Ensure HEAD is correct when skipping remote fetch

A recent optimization (2fb6466f79) skips
performing a remote fetch if we already know we have the sha1 we want.
However, that optimization skipped initialization steps that ensure HEAD
points to the correct sha1.  This change makes sure not to skip those
steps.

Here is an example of how to test this change:

"""""""""
url=<manifest url>
branch1=<branch name>
branch2=<branch name>
project=<project with revision set to different sha1 in each branch>

repo init -u $url -b $branch1 --mirror
repo sync $project
first=$(cd $project.git; git rev-parse HEAD)

repo init -b $branch2
repo sync $project
second=$(cd platform/build.git; git rev-parse HEAD)

if [[ $first == $second ]]
then
    echo 'problem!'
else
    echo 'no problem!'
fi
"""""""""

.gitattributes

@@ -0,0 +1,4 @@
+# Prevent /bin/sh scripts from being clobbered by autocrlf=true
+git_ssh text eol=lf
+main.py text eol=lf
+repo text eol=lf

.gitignore

@@ -1,2 +1,3 @@
 *.pyc
 .repopickle_*
+/repoc

.project

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <projectDescription>
-	<name>repo</name>
+	<name>git-repo</name>
 	<comment></comment>
 	<projects>
 	</projects>

.pydevproject

@@ -3,7 +3,7 @@
 
 <pydev_project>
 <pydev_pathproperty name="org.python.pydev.PROJECT_SOURCE_PATH">
-<path>/repo</path>
+<path>/git-repo</path>
 </pydev_pathproperty>
 <pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python 2.6</pydev_property>
 <pydev_property name="org.python.pydev.PYTHON_PROJECT_INTERPRETER">Default</pydev_property>

.pylintrc

@@ -53,7 +53,7 @@ load-plugins=
 enable=RP0004
 
 # Disable the message(s) with the given id(s).
-disable=R0903,R0912,R0913,R0914,R0915,W0141,C0111,C0103,C0323,C0322,C0324,W0603,W0703,R0911,C0301,C0302,R0902,R0904,W0142,W0212,E1101,E1103,R0201,W0201,W0122,W0232,W0311,RP0001,RP0003,RP0101,RP0002,RP0401,RP0701,RP0801
+disable=R0903,R0912,R0913,R0914,R0915,W0141,C0111,C0103,W0603,W0703,R0911,C0301,C0302,R0902,R0904,W0142,W0212,E1101,E1103,R0201,W0201,W0122,W0232,RP0001,RP0003,RP0101,RP0002,RP0401,RP0701,RP0801,F0401,E0611,R0801,I0011
 
 [REPORTS]
 

color.py

@@ -36,50 +36,51 @@ ATTRS = {None     :-1,
          'blink'  : 5,
          'reverse': 7}
 
-RESET = "\033[m"
+RESET = "\033[m"  # pylint: disable=W1401
+                  # backslash is not anomalous
 
 def is_color(s):
-    return s in COLORS
+  return s in COLORS
 
 def is_attr(s):
-    return s in ATTRS
+  return s in ATTRS
 
 def _Color(fg = None, bg = None, attr = None):
-    fg = COLORS[fg]
+  fg = COLORS[fg]
-    bg = COLORS[bg]
+  bg = COLORS[bg]
-    attr = ATTRS[attr]
+  attr = ATTRS[attr]
 
-    if attr >= 0 or fg >= 0 or bg >= 0:
+  if attr >= 0 or fg >= 0 or bg >= 0:
-      need_sep = False
+    need_sep = False
-      code = "\033["
+    code = "\033["  #pylint: disable=W1401
 
-      if attr >= 0:
+    if attr >= 0:
-        code += chr(ord('0') + attr)
+      code += chr(ord('0') + attr)
-        need_sep = True
+      need_sep = True
 
-      if fg >= 0:
+    if fg >= 0:
-        if need_sep:
+      if need_sep:
-          code += ';'
+        code += ';'
-        need_sep = True
+      need_sep = True
 
-        if fg < 8:
+      if fg < 8:
-          code += '3%c' % (ord('0') + fg)
+        code += '3%c' % (ord('0') + fg)
-        else:
+      else:
-          code += '38;5;%d' % fg
+        code += '38;5;%d' % fg
 
-      if bg >= 0:
+    if bg >= 0:
-        if need_sep:
+      if need_sep:
-          code += ';'
+        code += ';'
-        need_sep = True
+      need_sep = True
 
-        if bg < 8:
+      if bg < 8:
-          code += '4%c' % (ord('0') + bg)
+        code += '4%c' % (ord('0') + bg)
-        else:
+      else:
-          code += '48;5;%d' % bg
+        code += '48;5;%d' % bg
-      code += 'm'
+    code += 'm'
-    else:
+  else:
-      code = ''
+    code = ''
-    return code
+  return code
 
 
 class Coloring(object):
@@ -125,6 +126,13 @@ class Coloring(object):
       s._out.write(c(fmt, *args))
     return f
 
+  def nofmt_printer(self, opt=None, fg=None, bg=None, attr=None):
+    s = self
+    c = self.nofmt_colorer(opt, fg, bg, attr)
+    def f(fmt):
+      s._out.write(c(fmt))
+    return f
+
   def colorer(self, opt=None, fg=None, bg=None, attr=None):
     if self._on:
       c = self._parse(opt, fg, bg, attr)
@@ -137,6 +145,17 @@ class Coloring(object):
         return fmt % args
       return f
 
+  def nofmt_colorer(self, opt=None, fg=None, bg=None, attr=None):
+    if self._on:
+      c = self._parse(opt, fg, bg, attr)
+      def f(fmt):
+        return ''.join([c, fmt, RESET])
+      return f
+    else:
+      def f(fmt):
+        return fmt
+      return f
+
   def _parse(self, opt, fg, bg, attr):
     if not opt:
       return _Color(fg, bg, attr)

command.py

@@ -22,6 +22,7 @@ import sys
 from error import NoSuchProjectError
 from error import InvalidProjectGroupsError
 
+
 class Command(object):
   """Base class for any command line action in repo.
   """
@@ -33,6 +34,27 @@ class Command(object):
   def WantPager(self, opt):
     return False
 
+  def ReadEnvironmentOptions(self, opts):
+    """ Set options from environment variables. """
+
+    env_options = self._RegisteredEnvironmentOptions()
+
+    for env_key, opt_key in env_options.items():
+      # Get the user-set option value if any
+      opt_value = getattr(opts, opt_key)
+
+      # If the value is set, it means the user has passed it as a command
+      # line option, and we should use that.  Otherwise we can try to set it
+      # with the value from the corresponding environment variable.
+      if opt_value is not None:
+        continue
+
+      env_value = os.environ.get(env_key)
+      if env_value is not None:
+        setattr(opts, opt_key, env_value)
+
+    return opts
+
   @property
   def OptionParser(self):
     if self._optparse is None:
@@ -49,6 +71,24 @@ class Command(object):
     """Initialize the option parser.
     """
 
+  def _RegisteredEnvironmentOptions(self):
+    """Get options that can be set from environment variables.
+
+    Return a dictionary mapping environment variable name
+    to option key name that it can override.
+
+    Example: {'REPO_MY_OPTION': 'my_option'}
+
+    Will allow the option with key value 'my_option' to be set
+    from the value in the environment variable named 'REPO_MY_OPTION'.
+
+    Note: This does not work properly for options that are explicitly
+    set to None by the user, or options that are defined with a
+    default value other than None.
+
+    """
+    return {}
+
   def Usage(self):
     """Display usage and terminate.
     """
@@ -86,46 +126,44 @@ class Command(object):
         pass
     return project
 
-  def GetProjects(self, args, missing_ok=False):
+  def GetProjects(self, args, missing_ok=False, submodules_ok=False):
     """A list of projects that match the arguments.
     """
-    all_projects = self.manifest.projects
+    all_projects_list = self.manifest.projects
     result = []
 
     mp = self.manifest.manifestProject
 
     groups = mp.config.GetString('manifest.groups')
     if not groups:
-      groups = 'all,-notdefault,platform-' + platform.system().lower()
+      groups = 'default,platform-' + platform.system().lower()
-    groups = [x for x in re.split('[,\s]+', groups) if x]
+    groups = [x for x in re.split(r'[,\s]+', groups) if x]
 
     if not args:
-      all_projects_list = all_projects.values()
+      derived_projects = {}
-      derived_projects = []
       for project in all_projects_list:
-        if project.Registered:
+        if submodules_ok or project.sync_s:
-          # Do not search registered subproject for derived projects
+          derived_projects.update((p.name, p)
-          # since its parent has been searched already
+                                  for p in project.GetDerivedSubprojects())
-          continue
+      all_projects_list.extend(derived_projects.values())
-        derived_projects.extend(project.GetDerivedSubprojects())
-      all_projects_list.extend(derived_projects)
       for project in all_projects_list:
         if ((missing_ok or project.Exists) and
             project.MatchesGroups(groups)):
           result.append(project)
     else:
-      self._ResetPathToProjectMap(all_projects.values())
+      self._ResetPathToProjectMap(all_projects_list)
 
       for arg in args:
-        project = all_projects.get(arg)
+        projects = self.manifest.GetProjectsWithName(arg)
 
-        if not project:
+        if not projects:
           path = os.path.abspath(arg).replace('\\', '/')
           project = self._GetProjectByPath(path)
 
           # If it's not a derived project, update path->project mapping and
           # search again, as arg might actually point to a derived subproject.
-          if project and not project.Derived:
+          if (project and not project.Derived and
+              (submodules_ok or project.sync_s)):
             search_again = False
             for subproject in project.GetDerivedSubprojects():
               self._UpdatePathToProjectMap(subproject)
@@ -133,20 +171,36 @@ class Command(object):
             if search_again:
               project = self._GetProjectByPath(path) or project
 
-        if not project:
+          if project:
-          raise NoSuchProjectError(arg)
+            projects = [project]
-        if not missing_ok and not project.Exists:
-          raise NoSuchProjectError(arg)
-        if not project.MatchesGroups(groups):
-          raise InvalidProjectGroupsError(arg)
 
-        result.append(project)
+        if not projects:
+          raise NoSuchProjectError(arg)
+
+        for project in projects:
+          if not missing_ok and not project.Exists:
+            raise NoSuchProjectError(arg)
+          if not project.MatchesGroups(groups):
+            raise InvalidProjectGroupsError(arg)
+
+        result.extend(projects)
 
     def _getpath(x):
       return x.relpath
     result.sort(key=_getpath)
     return result
 
+  def FindProjects(self, args):
+    result = []
+    patterns = [re.compile(r'%s' % a, re.IGNORECASE) for a in args]
+    for project in self.GetProjects(''):
+      for pattern in patterns:
+        if pattern.search(project.name) or pattern.search(project.relpath):
+          result.append(project)
+          break
+    result.sort(key=lambda project: project.relpath)
+    return result
+
 # pylint: disable=W0223
 # Pylint warns that the `InteractiveCommand` and `PagedCommand` classes do not
 # override method `Execute` which is abstract in `Command`.  Since that method

docs/manifest-format.txt

@@ -27,38 +27,45 @@ following DTD:
                         remove-project*,
                         project*,
                         repo-hooks?)>
-  
+
     <!ELEMENT notice (#PCDATA)>
-  
+
     <!ELEMENT remote (EMPTY)>
     <!ATTLIST remote name         ID    #REQUIRED>
     <!ATTLIST remote alias        CDATA #IMPLIED>
     <!ATTLIST remote fetch        CDATA #REQUIRED>
     <!ATTLIST remote review       CDATA #IMPLIED>
-  
+
     <!ELEMENT default (EMPTY)>
-    <!ATTLIST default remote   IDREF #IMPLIED>
+    <!ATTLIST default remote      IDREF #IMPLIED>
-    <!ATTLIST default revision CDATA #IMPLIED>
+    <!ATTLIST default revision    CDATA #IMPLIED>
-    <!ATTLIST default sync-j   CDATA #IMPLIED>
+    <!ATTLIST default dest-branch CDATA #IMPLIED>
-    <!ATTLIST default sync-c   CDATA #IMPLIED>
+    <!ATTLIST default sync-j      CDATA #IMPLIED>
+    <!ATTLIST default sync-c      CDATA #IMPLIED>
+    <!ATTLIST default sync-s      CDATA #IMPLIED>
 
     <!ELEMENT manifest-server (EMPTY)>
     <!ATTLIST url              CDATA #REQUIRED>
-  
+
-    <!ELEMENT project (annotation?,
+    <!ELEMENT project (annotation*,
                        project*)>
-    <!ATTLIST project name     CDATA #REQUIRED>
+    <!ATTLIST project name        CDATA #REQUIRED>
-    <!ATTLIST project path     CDATA #IMPLIED>
+    <!ATTLIST project path        CDATA #IMPLIED>
-    <!ATTLIST project remote   IDREF #IMPLIED>
+    <!ATTLIST project remote      IDREF #IMPLIED>
-    <!ATTLIST project revision CDATA #IMPLIED>
+    <!ATTLIST project revision    CDATA #IMPLIED>
-    <!ATTLIST project groups   CDATA #IMPLIED>
+    <!ATTLIST project dest-branch CDATA #IMPLIED>
-    <!ATTLIST project sync-c   CDATA #IMPLIED>
+    <!ATTLIST project groups      CDATA #IMPLIED>
+    <!ATTLIST project sync-c      CDATA #IMPLIED>
+    <!ATTLIST project sync-s      CDATA #IMPLIED>
+    <!ATTLIST project upstream CDATA #IMPLIED>
+    <!ATTLIST project clone-depth CDATA #IMPLIED>
+    <!ATTLIST project force-path CDATA #IMPLIED>
 
     <!ELEMENT annotation (EMPTY)>
     <!ATTLIST annotation name  CDATA #REQUIRED>
     <!ATTLIST annotation value CDATA #REQUIRED>
     <!ATTLIST annotation keep  CDATA "true">
-  
+
     <!ELEMENT remove-project (EMPTY)>
     <!ATTLIST remove-project name  CDATA #REQUIRED>
 
@@ -120,6 +127,20 @@ Attribute `revision`: Name of a Git branch (e.g. `master` or
 `refs/heads/master`).  Project elements lacking their own
 revision attribute will use this revision.
 
+Attribute `dest-branch`: Name of a Git branch (e.g. `master`).
+Project elements not setting their own `dest-branch` will inherit
+this value. If this value is not set, projects will use `revision`
+by default instead.
+
+Attribute `sync_j`: Number of parallel jobs to use when synching.
+
+Attribute `sync_c`: Set to true to only sync the given Git
+branch (specified in the `revision` attribute) rather than the
+whole ref space.  Project elements lacking a sync_c element of
+their own will use this value.
+
+Attribute `sync_s`: Set to true to also sync sub-projects.
+
 
 Element manifest-server
 -----------------------
@@ -189,6 +210,11 @@ Tags and/or explicit SHA-1s should work in theory, but have not
 been extensively tested.  If not supplied the revision given by
 the default element is used.
 
+Attribute `dest-branch`: Name of a Git branch (e.g. `master`).
+When using `repo upload`, changes will be submitted for code
+review on this branch. If unspecified both here and in the
+default element, `revision` is used instead.
+
 Attribute `groups`: List of groups to which this project belongs,
 whitespace or comma separated.  All projects belong to the group
 "all", and each project automatically belongs to a group of
@@ -200,6 +226,26 @@ group "notdefault", it will not be automatically downloaded by repo.
 If the project has a parent element, the `name` and `path` here
 are the prefixed ones.
 
+Attribute `sync_c`: Set to true to only sync the given Git
+branch (specified in the `revision` attribute) rather than the
+whole ref space.
+
+Attribute `sync_s`: Set to true to also sync sub-projects.
+
+Attribute `upstream`: Name of the Git branch in which a sha1
+can be found.  Used when syncing a revision locked manifest in
+-c mode to avoid having to sync the entire ref space.
+
+Attribute `clone-depth`: Set the depth to use when fetching this
+project.  If specified, this value will override any value given
+to repo init with the --depth option on the command line.
+
+Attribute `force-path`: Set to true to force this project to create the
+local mirror repository according to its `path` attribute (if supplied)
+rather than the `name` attribute.  This attribute only applies to the
+local mirrors syncing, it will be ignored when syncing the projects in a
+client working directory.
+
 Element annotation
 ------------------
 
@@ -218,7 +264,7 @@ Deletes the named project from the internal manifest table, possibly
 allowing a subsequent project element in the same manifest file to
 replace the project with a different source.
 
-This element is mostly useful in the local_manifest.xml, where
+This element is mostly useful in a local manifest file, where
 the user can remove a project, and possibly replace it with their
 own definition.
 
@@ -227,21 +273,25 @@ Element include
 
 This element provides the capability of including another manifest
 file into the originating manifest.  Normal rules apply for the
-target manifest to include- it must be a usable manifest on it's own.
+target manifest to include - it must be a usable manifest on its own.
 
-Attribute `name`; the manifest to include, specified relative to
+Attribute `name`: the manifest to include, specified relative to
-the manifest repositories root.
+the manifest repository's root.
 
 
-Local Manifest
+Local Manifests
-==============
+===============
 
-Additional remotes and projects may be added through a local
+Additional remotes and projects may be added through local manifest
-manifest, stored in `$TOP_DIR/.repo/local_manifest.xml`.
+files stored in `$TOP_DIR/.repo/local_manifests/*.xml`.
 
 For example:
 
-  $ cat .repo/local_manifest.xml
+  $ ls .repo/local_manifests
+  local_manifest.xml
+  another_local_manifest.xml
+
+  $ cat .repo/local_manifests/local_manifest.xml
   <?xml version="1.0" encoding="UTF-8"?>
   <manifest>
     <project path="manifest"
@@ -250,6 +300,17 @@ For example:
              name="platform/manifest" />
   </manifest>
 
-Users may add projects to the local manifest prior to a `repo sync`
+Users may add projects to the local manifest(s) prior to a `repo sync`
 invocation, instructing repo to automatically download and manage
 these extra projects.
+
+Manifest files stored in `$TOP_DIR/.repo/local_manifests/*.xml` will
+be loaded in alphabetical order.
+
+Additional remotes and projects may also be added through a local
+manifest, stored in `$TOP_DIR/.repo/local_manifest.xml`. This method
+is deprecated in favor of using multiple manifest files as mentioned
+above.
+
+If `$TOP_DIR/.repo/local_manifest.xml` exists, it will be loaded before
+any manifest files stored in `$TOP_DIR/.repo/local_manifests/*.xml`.

editor.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import os
 import re
 import sys
@@ -53,10 +54,10 @@ class Editor(object):
       return e
 
     if os.getenv('TERM') == 'dumb':
-      print >>sys.stderr,\
+      print(
 """No editor specified in GIT_EDITOR, core.editor, VISUAL or EDITOR.
 Tried to fall back to vi but terminal is dumb.  Please configure at
-least one of these before using this command."""
+least one of these before using this command.""", file=sys.stderr)
       sys.exit(1)
 
     return 'vi'
@@ -67,7 +68,7 @@ least one of these before using this command."""
 
        Args:
          data        : the text to edit
-  
+
       Returns:
         new value of edited text; None if editing did not succeed
     """

error.py

@@ -21,6 +21,17 @@ class ManifestInvalidRevisionError(Exception):
   """The revision value in a project is incorrect.
   """
 
+class NoManifestException(Exception):
+  """The required manifest does not exist.
+  """
+  def __init__(self, path, reason):
+    super(NoManifestException, self).__init__()
+    self.path = path
+    self.reason = reason
+
+  def __str__(self):
+    return self.reason
+
 class EditorError(Exception):
   """Unspecified error from the user's text editor.
   """

git_command.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import os
 import sys
 import subprocess
@@ -20,6 +21,7 @@ import tempfile
 from signal import SIGTERM
 from error import GitError
 from trace import REPO_TRACE, IsTrace, Trace
+from wrapper import Wrapper
 
 GIT = 'git'
 MIN_GIT_VERSION = (1, 5, 4)
@@ -83,16 +85,11 @@ class _GitCall(object):
 
   def version_tuple(self):
     global _git_version
-
     if _git_version is None:
-      ver_str = git.version()
+      ver_str = git.version().decode('utf-8')
-      if ver_str.startswith('git version '):
+      _git_version = Wrapper().ParseGitVersion(ver_str)
-        _git_version = tuple(
+      if _git_version is None:
-          map(lambda x: int(x),
+        print('fatal: "%s" unsupported' % ver_str, file=sys.stderr)
-            ver_str[len('git version '):].strip().split('-')[0].split('.')[0:3]
-          ))
-      else:
-        print >>sys.stderr, 'fatal: "%s" unsupported' % ver_str
         sys.exit(1)
     return _git_version
 
@@ -110,8 +107,8 @@ def git_require(min_version, fail=False):
   if min_version <= git_version:
     return True
   if fail:
-    need = '.'.join(map(lambda x: str(x), min_version))
+    need = '.'.join(map(str, min_version))
-    print >>sys.stderr, 'fatal: git %s or later required' % need
+    print('fatal: git %s or later required' % need, file=sys.stderr)
     sys.exit(1)
   return False
 
@@ -132,15 +129,15 @@ class GitCommand(object):
                gitdir = None):
     env = os.environ.copy()
 
-    for e in [REPO_TRACE,
+    for key in [REPO_TRACE,
               GIT_DIR,
               'GIT_ALTERNATE_OBJECT_DIRECTORIES',
               'GIT_OBJECT_DIRECTORY',
               'GIT_WORK_TREE',
               'GIT_GRAFT_FILE',
               'GIT_INDEX_FILE']:
-      if e in env:
+      if key in env:
-        del env[e]
+        del env[key]
 
     if disable_editor:
       _setenv(env, 'GIT_EDITOR', ':')

git_config.py

@@ -13,8 +13,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import cPickle
+from __future__ import print_function
+
 import os
+import pickle
 import re
 import subprocess
 import sys
@@ -23,11 +25,25 @@ try:
 except ImportError:
   import dummy_threading as _threading
 import time
-import urllib2
+
+from pyversion import is_python3
+if is_python3():
+  import urllib.request
+  import urllib.error
+else:
+  import urllib2
+  import imp
+  urllib = imp.new_module('urllib')
+  urllib.request = urllib2
+  urllib.error = urllib2
 
 from signal import SIGTERM
 from error import GitError, UploadError
 from trace import Trace
+if is_python3():
+  from http.client import HTTPException
+else:
+  from httplib import HTTPException
 
 from git_command import GitCommand
 from git_command import ssh_sock
@@ -35,7 +51,7 @@ from git_command import terminate_ssh_clients
 
 R_HEADS = 'refs/heads/'
 R_TAGS  = 'refs/tags/'
-ID_RE = re.compile('^[0-9a-f]{40}$')
+ID_RE = re.compile(r'^[0-9a-f]{40}$')
 
 REVIEW_CACHE = dict()
 
@@ -157,7 +173,7 @@ class GitConfig(object):
       elif old != value:
         self._cache[key] = list(value)
         self._do('--replace-all', name, value[0])
-        for i in xrange(1, len(value)):
+        for i in range(1, len(value)):
           self._do('--add', name, value[i])
 
     elif len(old) != 1 or old[0] != value:
@@ -250,7 +266,7 @@ class GitConfig(object):
       Trace(': unpickle %s', self.file)
       fd = open(self._pickle, 'rb')
       try:
-        return cPickle.load(fd)
+        return pickle.load(fd)
       finally:
         fd.close()
     except EOFError:
@@ -259,7 +275,7 @@ class GitConfig(object):
     except IOError:
       os.remove(self._pickle)
       return None
-    except cPickle.PickleError:
+    except pickle.PickleError:
       os.remove(self._pickle)
       return None
 
@@ -267,13 +283,13 @@ class GitConfig(object):
     try:
       fd = open(self._pickle, 'wb')
       try:
-        cPickle.dump(cache, fd, cPickle.HIGHEST_PROTOCOL)
+        pickle.dump(cache, fd, pickle.HIGHEST_PROTOCOL)
       finally:
         fd.close()
     except IOError:
       if os.path.exists(self._pickle):
         os.remove(self._pickle)
-    except cPickle.PickleError:
+    except pickle.PickleError:
       if os.path.exists(self._pickle):
         os.remove(self._pickle)
 
@@ -288,12 +304,13 @@ class GitConfig(object):
     d = self._do('--null', '--list')
     if d is None:
       return c
-    for line in d.rstrip('\0').split('\0'):
+    for line in d.decode('utf-8').rstrip('\0').split('\0'):  # pylint: disable=W1401
+                                                             # Backslash is not anomalous
       if '\n' in line:
-          key, val = line.split('\n', 1)
+        key, val = line.split('\n', 1)
       else:
-          key = line
+        key = line
-          val = None
+        val = None
 
       if key in c:
         c[key].append(val)
@@ -418,7 +435,7 @@ def _open_ssh(host, port=None):
                      '-o','ControlPath %s' % ssh_sock(),
                      host]
     if port is not None:
-      command_base[1:1] = ['-p',str(port)]
+      command_base[1:1] = ['-p', str(port)]
 
     # Since the key wasn't in _master_keys, we think that master isn't running.
     # ...but before actually starting a master, we'll double-check.  This can
@@ -451,9 +468,8 @@ def _open_ssh(host, port=None):
       p = subprocess.Popen(command)
     except Exception as e:
       _ssh_master = False
-      print >>sys.stderr, \
+      print('\nwarn: cannot enable ssh control master for %s:%s\n%s'
-        '\nwarn: cannot enable ssh control master for %s:%s\n%s' \
+             % (host,port, str(e)), file=sys.stderr)
-        % (host,port, str(e))
       return False
 
     _master_processes.append(p)
@@ -525,8 +541,8 @@ class Remote(object):
     self.url = self._Get('url')
     self.review = self._Get('review')
     self.projectname = self._Get('projectname')
-    self.fetch = map(lambda x: RefSpec.FromString(x),
+    self.fetch = list(map(RefSpec.FromString,
-                     self._Get('fetch', all_keys=True))
+                      self._Get('fetch', all_keys=True)))
     self._review_url = None
 
   def _InsteadOf(self):
@@ -560,7 +576,7 @@ class Remote(object):
         return None
 
       u = self.review
-      if not u.startswith('http:') and not u.startswith('https:'):
+      if u.split(':')[0] not in ('http', 'https', 'sso'):
         u = 'http://%s' % u
       if u.endswith('/Gerrit'):
         u = u[:len(u) - len('/Gerrit')]
@@ -576,26 +592,28 @@ class Remote(object):
         host, port = os.environ['REPO_HOST_PORT_INFO'].split()
         self._review_url = self._SshReviewUrl(userEmail, host, port)
         REVIEW_CACHE[u] = self._review_url
+      elif u.startswith('sso:'):
+        self._review_url = u  # Assume it's right
+        REVIEW_CACHE[u] = self._review_url
       else:
         try:
           info_url = u + 'ssh_info'
-          info = urllib2.urlopen(info_url).read()
+          info = urllib.request.urlopen(info_url).read()
-          if '<' in info:
+          if info == 'NOT_AVAILABLE' or '<' in info:
-            # Assume the server gave us some sort of HTML
+            # If `info` contains '<', we assume the server gave us some sort
-            # response back, like maybe a login page.
+            # of HTML response back, like maybe a login page.
             #
-            raise UploadError('%s: Cannot parse response' % info_url)
+            # Assume HTTP if SSH is not enabled or ssh_info doesn't look right.
-
+            self._review_url = http_url
-          if info == 'NOT_AVAILABLE':
-            # Assume HTTP if SSH is not enabled.
-            self._review_url = http_url + 'p/'
           else:
             host, port = info.split()
             self._review_url = self._SshReviewUrl(userEmail, host, port)
-        except urllib2.HTTPError as e:
+        except urllib.error.HTTPError as e:
           raise UploadError('%s: %s' % (self.review, str(e)))
-        except urllib2.URLError as e:
+        except urllib.error.URLError as e:
           raise UploadError('%s: %s' % (self.review, str(e)))
+        except HTTPException as e:
+          raise UploadError('%s: %s' % (self.review, e.__class__.__name__))
 
         REVIEW_CACHE[u] = self._review_url
     return self._review_url + self.projectname
@@ -645,7 +663,7 @@ class Remote(object):
     self._Set('url', self.url)
     self._Set('review', self.review)
     self._Set('projectname', self.projectname)
-    self._Set('fetch', map(lambda x: str(x), self.fetch))
+    self._Set('fetch', list(map(str, self.fetch)))
 
   def _Set(self, key, value):
     key = 'remote.%s.%s' % (self.name, key)

git_refs.py

@@ -66,7 +66,7 @@ class GitRefs(object):
   def _NeedUpdate(self):
     Trace(': scan refs %s', self._gitdir)
 
-    for name, mtime in self._mtime.iteritems():
+    for name, mtime in self._mtime.items():
       try:
         if mtime != os.path.getmtime(os.path.join(self._gitdir, name)):
           return True
@@ -89,7 +89,7 @@ class GitRefs(object):
     attempts = 0
     while scan and attempts < 5:
       scan_next = {}
-      for name, dest in scan.iteritems():
+      for name, dest in scan.items():
         if dest in self._phyref:
           self._phyref[name] = self._phyref[dest]
         else:
@@ -100,7 +100,7 @@ class GitRefs(object):
   def _ReadPackedRefs(self):
     path = os.path.join(self._gitdir, 'packed-refs')
     try:
-      fd = open(path, 'rb')
+      fd = open(path, 'r')
       mtime = os.path.getmtime(path)
     except IOError:
       return
@@ -108,6 +108,7 @@ class GitRefs(object):
       return
     try:
       for line in fd:
+        line = str(line)
         if line[0] == '#':
           continue
         if line[0] == '^':
@@ -138,18 +139,22 @@ class GitRefs(object):
   def _ReadLoose1(self, path, name):
     try:
       fd = open(path, 'rb')
-    except:
+    except IOError:
       return
 
     try:
       try:
         mtime = os.path.getmtime(path)
         ref_id = fd.readline()
-      except:
+      except (IOError, OSError):
         return
     finally:
       fd.close()
 
+    try:
+      ref_id = ref_id.decode()
+    except AttributeError:
+      pass
     if not ref_id:
       return
     ref_id = ref_id[:-1]

hooks/commit-msg

@@ -1,5 +1,5 @@
 #!/bin/sh
-# From Gerrit Code Review 2.5-rc0
+# From Gerrit Code Review 2.6
 #
 # Part of Gerrit Code Review (http://code.google.com/p/gerrit/)
 #
@@ -18,6 +18,8 @@
 # limitations under the License.
 #
 
+unset GREP_OPTIONS
+
 CHANGE_ID_AFTER="Bug|Issue"
 MSG="$1"
 
@@ -152,7 +154,7 @@ add_ChangeId() {
 		if (unprinted) {
 			print "Change-Id: I'"$id"'"
 		}
-	}' "$MSG" > $T && mv $T "$MSG" || rm -f $T
+	}' "$MSG" > "$T" && mv "$T" "$MSG" || rm -f "$T"
 }
 _gen_ChangeIdInput() {
 	echo "tree `git write-tree`"

hooks/pre-auto-gc

@@ -35,7 +35,7 @@ elif grep -q "AC Power \+: 1" /proc/pmu/info 2>/dev/null
 then
 	exit 0
 elif test -x /usr/bin/pmset && /usr/bin/pmset -g batt |
-	grep -q "Currently drawing from 'AC Power'"
+	grep -q "drawing from 'AC Power'"
 then
 	exit 0
 elif test -d /sys/bus/acpi/drivers/battery && test 0 = \

main.py

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env python
 #
 # Copyright (C) 2008 The Android Open Source Project
 #
@@ -14,23 +14,27 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-magic='--calling-python-from-/bin/sh--'
+from __future__ import print_function
-"""exec" python -E "$0" "$@" """#$magic"
-if __name__ == '__main__':
-  import sys
-  if sys.argv[-1] == '#%s' % magic:
-    del sys.argv[-1]
-del magic
-
 import getpass
 import imp
 import netrc
 import optparse
 import os
-import re
 import sys
 import time
-import urllib2
+
+from pyversion import is_python3
+if is_python3():
+  import urllib.request
+else:
+  import urllib2
+  urllib = imp.new_module('urllib')
+  urllib.request = urllib2
+
+try:
+  import kerberos
+except ImportError:
+  kerberos = None
 
 from trace import SetTrace
 from git_command import git, GitCommand
@@ -41,13 +45,21 @@ from subcmds.version import Version
 from editor import Editor
 from error import DownloadError
 from error import ManifestInvalidRevisionError
+from error import ManifestParseError
+from error import NoManifestException
 from error import NoSuchProjectError
 from error import RepoChangedException
 from manifest_xml import XmlManifest
 from pager import RunPager
+from wrapper import WrapperPath, Wrapper
 
 from subcmds import all_commands
 
+if not is_python3():
+  # pylint:disable=W0622
+  input = raw_input
+  # pylint:enable=W0622
+
 global_options = optparse.OptionParser(
                  usage="repo [-p|--paginate|--no-pager] COMMAND [ARGS]"
                  )
@@ -79,7 +91,7 @@ class _Repo(object):
     name = None
     glob = []
 
-    for i in xrange(0, len(argv)):
+    for i in range(len(argv)):
       if not argv[i].startswith('-'):
         name = argv[i]
         if i > 0:
@@ -98,15 +110,14 @@ class _Repo(object):
       if name == 'help':
         name = 'version'
       else:
-        print >>sys.stderr, 'fatal: invalid usage of --version'
+        print('fatal: invalid usage of --version', file=sys.stderr)
         return 1
 
     try:
       cmd = self.commands[name]
     except KeyError:
-      print >>sys.stderr,\
+      print("repo: '%s' is not a repo command.  See 'repo help'." % name,
-            "repo: '%s' is not a repo command.  See 'repo help'."\
+            file=sys.stderr)
-            % name
       return 1
 
     cmd.repodir = self.repodir
@@ -114,12 +125,19 @@ class _Repo(object):
     Editor.globalConfig = cmd.manifest.globalConfig
 
     if not isinstance(cmd, MirrorSafeCommand) and cmd.manifest.IsMirror:
-      print >>sys.stderr, \
+      print("fatal: '%s' requires a working directory" % name,
-            "fatal: '%s' requires a working directory"\
+            file=sys.stderr)
-            % name
       return 1
 
-    copts, cargs = cmd.OptionParser.parse_args(argv)
+    try:
+      copts, cargs = cmd.OptionParser.parse_args(argv)
+      copts = cmd.ReadEnvironmentOptions(copts)
+    except NoManifestException as e:
+      print('error: in `%s`: %s' % (' '.join([name] + argv), str(e)),
+        file=sys.stderr)
+      print('error: manifest missing or unreadable -- please run init',
+            file=sys.stderr)
+      return 1
 
     if not gopts.no_pager and not isinstance(cmd, InteractiveCommand):
       config = cmd.manifest.globalConfig
@@ -132,87 +150,75 @@ class _Repo(object):
       if use_pager:
         RunPager(config)
 
+    start = time.time()
     try:
-      start = time.time()
+      result = cmd.Execute(copts, cargs)
-      try:
+    except (DownloadError, ManifestInvalidRevisionError,
-        result = cmd.Execute(copts, cargs)
+        NoManifestException) as e:
-      finally:
+      print('error: in `%s`: %s' % (' '.join([name] + argv), str(e)),
-        elapsed = time.time() - start
+        file=sys.stderr)
-        hours, remainder = divmod(elapsed, 3600)
+      if isinstance(e, NoManifestException):
-        minutes, seconds = divmod(remainder, 60)
+        print('error: manifest missing or unreadable -- please run init',
-        if gopts.time:
+              file=sys.stderr)
-          if hours == 0:
+      result = 1
-            print >>sys.stderr, 'real\t%dm%.3fs' \
-              % (minutes, seconds)
-          else:
-            print >>sys.stderr, 'real\t%dh%dm%.3fs' \
-              % (hours, minutes, seconds)
-    except DownloadError as e:
-      print >>sys.stderr, 'error: %s' % str(e)
-      return 1
-    except ManifestInvalidRevisionError as e:
-      print >>sys.stderr, 'error: %s' % str(e)
-      return 1
     except NoSuchProjectError as e:
       if e.name:
-        print >>sys.stderr, 'error: project %s not found' % e.name
+        print('error: project %s not found' % e.name, file=sys.stderr)
       else:
-        print >>sys.stderr, 'error: no project in current directory'
+        print('error: no project in current directory', file=sys.stderr)
-      return 1
+      result = 1
+    finally:
+      elapsed = time.time() - start
+      hours, remainder = divmod(elapsed, 3600)
+      minutes, seconds = divmod(remainder, 60)
+      if gopts.time:
+        if hours == 0:
+          print('real\t%dm%.3fs' % (minutes, seconds), file=sys.stderr)
+        else:
+          print('real\t%dh%dm%.3fs' % (hours, minutes, seconds),
+                file=sys.stderr)
 
     return result
 
+
 def _MyRepoPath():
   return os.path.dirname(__file__)
 
-def _MyWrapperPath():
-  return os.path.join(os.path.dirname(__file__), 'repo')
-
-_wrapper_module = None
-def WrapperModule():
-  global _wrapper_module
-  if not _wrapper_module:
-    _wrapper_module = imp.load_source('wrapper', _MyWrapperPath())
-  return _wrapper_module
-
-def _CurrentWrapperVersion():
-  return WrapperModule().VERSION
 
 def _CheckWrapperVersion(ver, repo_path):
   if not repo_path:
     repo_path = '~/bin/repo'
 
   if not ver:
-    print >>sys.stderr, 'no --wrapper-version argument'
+    print('no --wrapper-version argument', file=sys.stderr)
     sys.exit(1)
 
-  exp = _CurrentWrapperVersion()
+  exp = Wrapper().VERSION
-  ver = tuple(map(lambda x: int(x), ver.split('.')))
+  ver = tuple(map(int, ver.split('.')))
   if len(ver) == 1:
     ver = (0, ver[0])
 
+  exp_str = '.'.join(map(str, exp))
   if exp[0] > ver[0] or ver < (0, 4):
-    exp_str = '.'.join(map(lambda x: str(x), exp))
+    print("""
-    print >>sys.stderr, """
 !!! A new repo command (%5s) is available.    !!!
 !!! You must upgrade before you can continue:   !!!
 
     cp %s %s
-""" % (exp_str, _MyWrapperPath(), repo_path)
+""" % (exp_str, WrapperPath(), repo_path), file=sys.stderr)
     sys.exit(1)
 
   if exp > ver:
-    exp_str = '.'.join(map(lambda x: str(x), exp))
+    print("""
-    print >>sys.stderr, """
 ... A new repo command (%5s) is available.
 ... You should upgrade soon:
 
     cp %s %s
-""" % (exp_str, _MyWrapperPath(), repo_path)
+""" % (exp_str, WrapperPath(), repo_path), file=sys.stderr)
 
 def _CheckRepoDir(repo_dir):
   if not repo_dir:
-    print >>sys.stderr, 'no --repo-dir argument'
+    print('no --repo-dir argument', file=sys.stderr)
     sys.exit(1)
 
 def _PruneOptions(argv, opt):
@@ -264,11 +270,11 @@ def _UserAgent():
     _user_agent = 'git-repo/%s (%s) git/%s Python/%d.%d.%d' % (
       repo_version,
       os_name,
-      '.'.join(map(lambda d: str(d), git.version_tuple())),
+      '.'.join(map(str, git.version_tuple())),
       py_version[0], py_version[1], py_version[2])
   return _user_agent
 
-class _UserAgentHandler(urllib2.BaseHandler):
+class _UserAgentHandler(urllib.request.BaseHandler):
   def http_request(self, req):
     req.add_header('User-Agent', _UserAgent())
     return req
@@ -278,22 +284,22 @@ class _UserAgentHandler(urllib2.BaseHandler):
     return req
 
 def _AddPasswordFromUserInput(handler, msg, req):
-    # If repo could not find auth info from netrc, try to get it from user input
+  # If repo could not find auth info from netrc, try to get it from user input
-    url = req.get_full_url()
+  url = req.get_full_url()
-    user, password = handler.passwd.find_user_password(None, url)
+  user, password = handler.passwd.find_user_password(None, url)
-    if user is None:
+  if user is None:
-      print msg
+    print(msg)
-      try:
+    try:
-        user = raw_input('User: ')
+      user = input('User: ')
-        password = getpass.getpass()
+      password = getpass.getpass()
-      except KeyboardInterrupt:
+    except KeyboardInterrupt:
-        return
+      return
-      handler.passwd.add_password(None, url, user, password)
+    handler.passwd.add_password(None, url, user, password)
 
-class _BasicAuthHandler(urllib2.HTTPBasicAuthHandler):
+class _BasicAuthHandler(urllib.request.HTTPBasicAuthHandler):
   def http_error_401(self, req, fp, code, msg, headers):
     _AddPasswordFromUserInput(self, msg, req)
-    return urllib2.HTTPBasicAuthHandler.http_error_401(
+    return urllib.request.HTTPBasicAuthHandler.http_error_401(
       self, req, fp, code, msg, headers)
 
   def http_error_auth_reqed(self, authreq, host, req, headers):
@@ -303,7 +309,7 @@ class _BasicAuthHandler(urllib2.HTTPBasicAuthHandler):
         val = val.replace('\n', '')
         old_add_header(name, val)
       req.add_header = _add_header
-      return urllib2.AbstractBasicAuthHandler.http_error_auth_reqed(
+      return urllib.request.AbstractBasicAuthHandler.http_error_auth_reqed(
         self, authreq, host, req, headers)
     except:
       reset = getattr(self, 'reset_retry_count', None)
@@ -313,10 +319,10 @@ class _BasicAuthHandler(urllib2.HTTPBasicAuthHandler):
         self.retried = 0
       raise
 
-class _DigestAuthHandler(urllib2.HTTPDigestAuthHandler):
+class _DigestAuthHandler(urllib.request.HTTPDigestAuthHandler):
   def http_error_401(self, req, fp, code, msg, headers):
     _AddPasswordFromUserInput(self, msg, req)
-    return urllib2.HTTPDigestAuthHandler.http_error_401(
+    return urllib.request.HTTPDigestAuthHandler.http_error_401(
       self, req, fp, code, msg, headers)
 
   def http_error_auth_reqed(self, auth_header, host, req, headers):
@@ -326,7 +332,7 @@ class _DigestAuthHandler(urllib2.HTTPDigestAuthHandler):
         val = val.replace('\n', '')
         old_add_header(name, val)
       req.add_header = _add_header
-      return urllib2.AbstractDigestAuthHandler.http_error_auth_reqed(
+      return urllib.request.AbstractDigestAuthHandler.http_error_auth_reqed(
         self, auth_header, host, req, headers)
     except:
       reset = getattr(self, 'reset_retry_count', None)
@@ -336,10 +342,90 @@ class _DigestAuthHandler(urllib2.HTTPDigestAuthHandler):
         self.retried = 0
       raise
 
+class _KerberosAuthHandler(urllib.request.BaseHandler):
+  def __init__(self):
+    self.retried = 0
+    self.context = None
+    self.handler_order = urllib.request.BaseHandler.handler_order - 50
+
+  def http_error_401(self, req, fp, code, msg, headers):
+    host = req.get_host()
+    retry = self.http_error_auth_reqed('www-authenticate', host, req, headers)
+    return retry
+
+  def http_error_auth_reqed(self, auth_header, host, req, headers):
+    try:
+      spn = "HTTP@%s" % host
+      authdata = self._negotiate_get_authdata(auth_header, headers)
+
+      if self.retried > 3:
+        raise urllib.request.HTTPError(req.get_full_url(), 401,
+          "Negotiate auth failed", headers, None)
+      else:
+        self.retried += 1
+
+      neghdr = self._negotiate_get_svctk(spn, authdata)
+      if neghdr is None:
+        return None
+
+      req.add_unredirected_header('Authorization', neghdr)
+      response = self.parent.open(req)
+
+      srvauth = self._negotiate_get_authdata(auth_header, response.info())
+      if self._validate_response(srvauth):
+        return response
+    except kerberos.GSSError:
+      return None
+    except:
+      self.reset_retry_count()
+      raise
+    finally:
+      self._clean_context()
+
+  def reset_retry_count(self):
+    self.retried = 0
+
+  def _negotiate_get_authdata(self, auth_header, headers):
+    authhdr = headers.get(auth_header, None)
+    if authhdr is not None:
+      for mech_tuple in authhdr.split(","):
+        mech, __, authdata = mech_tuple.strip().partition(" ")
+        if mech.lower() == "negotiate":
+          return authdata.strip()
+    return None
+
+  def _negotiate_get_svctk(self, spn, authdata):
+    if authdata is None:
+      return None
+
+    result, self.context = kerberos.authGSSClientInit(spn)
+    if result < kerberos.AUTH_GSS_COMPLETE:
+      return None
+
+    result = kerberos.authGSSClientStep(self.context, authdata)
+    if result < kerberos.AUTH_GSS_CONTINUE:
+      return None
+
+    response = kerberos.authGSSClientResponse(self.context)
+    return "Negotiate %s" % response
+
+  def _validate_response(self, authdata):
+    if authdata is None:
+      return None
+    result = kerberos.authGSSClientStep(self.context, authdata)
+    if result == kerberos.AUTH_GSS_COMPLETE:
+      return True
+    return None
+
+  def _clean_context(self):
+    if self.context is not None:
+      kerberos.authGSSClientClean(self.context)
+      self.context = None
+
 def init_http():
   handlers = [_UserAgentHandler()]
 
-  mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
+  mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
   try:
     n = netrc.netrc()
     for host in n.hosts:
@@ -352,14 +438,16 @@ def init_http():
     pass
   handlers.append(_BasicAuthHandler(mgr))
   handlers.append(_DigestAuthHandler(mgr))
+  if kerberos:
+    handlers.append(_KerberosAuthHandler())
 
   if 'http_proxy' in os.environ:
     url = os.environ['http_proxy']
-    handlers.append(urllib2.ProxyHandler({'http': url, 'https': url}))
+    handlers.append(urllib.request.ProxyHandler({'http': url, 'https': url}))
   if 'REPO_CURL_VERBOSE' in os.environ:
-    handlers.append(urllib2.HTTPHandler(debuglevel=1))
+    handlers.append(urllib.request.HTTPHandler(debuglevel=1))
-    handlers.append(urllib2.HTTPSHandler(debuglevel=1))
+    handlers.append(urllib.request.HTTPSHandler(debuglevel=1))
-  urllib2.install_opener(urllib2.build_opener(*handlers))
+  urllib.request.install_opener(urllib.request.build_opener(*handlers))
 
 def _Main(argv):
   result = 0
@@ -389,6 +477,10 @@ def _Main(argv):
     finally:
       close_ssh()
   except KeyboardInterrupt:
+    print('aborted by user', file=sys.stderr)
+    result = 1
+  except ManifestParseError as mpe:
+    print('fatal: %s' % mpe, file=sys.stderr)
     result = 1
   except RepoChangedException as rce:
     # If repo changed, re-exec ourselves.
@@ -398,8 +490,8 @@ def _Main(argv):
     try:
       os.execv(__file__, argv)
     except OSError as e:
-      print >>sys.stderr, 'fatal: cannot restart repo after upgrade'
+      print('fatal: cannot restart repo after upgrade', file=sys.stderr)
-      print >>sys.stderr, 'fatal: %s' % e
+      print('fatal: %s' % e, file=sys.stderr)
       result = 128
 
   sys.exit(result)

manifest_xml.py

@@ -13,31 +13,49 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import itertools
 import os
 import re
 import sys
-import urlparse
 import xml.dom.minidom
 
+from pyversion import is_python3
+if is_python3():
+  import urllib.parse
+else:
+  import imp
+  import urlparse
+  urllib = imp.new_module('urllib')
+  urllib.parse = urlparse
+
 from git_config import GitConfig
 from git_refs import R_HEADS, HEAD
 from project import RemoteSpec, Project, MetaProject
-from error import ManifestParseError
+from error import ManifestParseError, ManifestInvalidRevisionError
 
 MANIFEST_FILE_NAME = 'manifest.xml'
 LOCAL_MANIFEST_NAME = 'local_manifest.xml'
+LOCAL_MANIFESTS_DIR_NAME = 'local_manifests'
 
-urlparse.uses_relative.extend(['ssh', 'git'])
+urllib.parse.uses_relative.extend(['ssh', 'git'])
-urlparse.uses_netloc.extend(['ssh', 'git'])
+urllib.parse.uses_netloc.extend(['ssh', 'git'])
 
 class _Default(object):
   """Project defaults within the manifest."""
 
   revisionExpr = None
+  destBranchExpr = None
   remote = None
   sync_j = 1
   sync_c = False
+  sync_s = False
+
+  def __eq__(self, other):
+    return self.__dict__ == other.__dict__
+
+  def __ne__(self, other):
+    return self.__dict__ != other.__dict__
 
 class _XmlRemote(object):
   def __init__(self,
@@ -53,15 +71,30 @@ class _XmlRemote(object):
     self.reviewUrl = review
     self.resolvedFetchUrl = self._resolveFetchUrl()
 
+  def __eq__(self, other):
+    return self.__dict__ == other.__dict__
+
+  def __ne__(self, other):
+    return self.__dict__ != other.__dict__
+
   def _resolveFetchUrl(self):
     url = self.fetchUrl.rstrip('/')
     manifestUrl = self.manifestUrl.rstrip('/')
-    # urljoin will get confused if there is no scheme in the base url
+    # urljoin will gets confused over quite a few things.  The ones we care
-    # ie, if manifestUrl is of the form <hostname:port>
+    # about here are:
+    # * no scheme in the base url, like <hostname:port>
+    # * persistent-https://
+    # We handle this by replacing these with obscure protocols
+    # and then replacing them with the original when we are done.
+    # gopher -> <none>
+    # wais -> persistent-https
     if manifestUrl.find(':') != manifestUrl.find('/') - 1:
-        manifestUrl = 'gopher://' + manifestUrl
+      manifestUrl = 'gopher://' + manifestUrl
-    url = urlparse.urljoin(manifestUrl, url)
+    manifestUrl = re.sub(r'^persistent-https://', 'wais://', manifestUrl)
-    return re.sub(r'^gopher://', '', url)
+    url = urllib.parse.urljoin(manifestUrl, url)
+    url = re.sub(r'^gopher://', '', url)
+    url = re.sub(r'^wais://', 'persistent-https://', url)
+    return url
 
   def ToRemoteSpec(self, projectName):
     url = self.resolvedFetchUrl.rstrip('/') + '/' + projectName
@@ -78,6 +111,7 @@ class XmlManifest(object):
     self.topdir = os.path.dirname(self.repodir)
     self.manifestFile = os.path.join(self.repodir, MANIFEST_FILE_NAME)
     self.globalConfig = GitConfig.ForUser()
+    self.localManifestWarning = False
 
     self.repoProject = MetaProject(self, 'repo',
       gitdir   = os.path.join(repodir, 'repo/.git'),
@@ -110,17 +144,19 @@ class XmlManifest(object):
     self.Override(name)
 
     try:
-      if os.path.exists(self.manifestFile):
+      if os.path.lexists(self.manifestFile):
         os.remove(self.manifestFile)
       os.symlink('manifests/%s' % name, self.manifestFile)
-    except OSError:
+    except OSError as e:
-      raise ManifestParseError('cannot link manifest %s' % name)
+      raise ManifestParseError('cannot link manifest %s: %s' % (name, str(e)))
 
   def _RemoteToXml(self, r, doc, root):
     e = doc.createElement('remote')
     root.appendChild(e)
     e.setAttribute('name', r.name)
     e.setAttribute('fetch', r.fetchUrl)
+    if r.remoteAlias is not None:
+      e.setAttribute('alias', r.remoteAlias)
     if r.reviewUrl is not None:
       e.setAttribute('review', r.reviewUrl)
 
@@ -130,9 +166,8 @@ class XmlManifest(object):
     mp = self.manifestProject
 
     groups = mp.config.GetString('manifest.groups')
-    if not groups:
+    if groups:
-      groups = 'all'
+      groups = [x for x in re.split(r'[,\s]+', groups) if x]
-    groups = [x for x in re.split(r'[,\s]+', groups) if x]
 
     doc = xml.dom.minidom.Document()
     root = doc.createElement('manifest')
@@ -148,10 +183,8 @@ class XmlManifest(object):
       notice_element.appendChild(doc.createTextNode(indented_notice))
 
     d = self.default
-    sort_remotes = list(self.remotes.keys())
-    sort_remotes.sort()
 
-    for r in sort_remotes:
+    for r in sorted(self.remotes):
       self._RemoteToXml(self.remotes[r], doc, root)
     if self.remotes:
       root.appendChild(doc.createTextNode(''))
@@ -170,6 +203,9 @@ class XmlManifest(object):
     if d.sync_c:
       have_default = True
       e.setAttribute('sync-c', 'true')
+    if d.sync_s:
+      have_default = True
+      e.setAttribute('sync-s', 'true')
     if have_default:
       root.appendChild(e)
       root.appendChild(doc.createTextNode(''))
@@ -181,8 +217,9 @@ class XmlManifest(object):
       root.appendChild(doc.createTextNode(''))
 
     def output_projects(parent, parent_node, projects):
-      for p in projects:
+      for project_name in projects:
-        output_project(parent, parent_node, self.projects[p])
+        for project in self._projects[project_name]:
+          output_project(parent, parent_node, project)
 
     def output_project(parent, parent_node, p):
       if not p.MatchesGroups(groups):
@@ -199,7 +236,10 @@ class XmlManifest(object):
       e.setAttribute('name', name)
       if relpath != name:
         e.setAttribute('path', relpath)
-      if not d.remote or p.remote.name != d.remote.name:
+      remoteName = None
+      if d.remote:
+        remoteName = d.remote.remoteAlias or d.remote.name
+      if not d.remote or p.remote.name != remoteName:
         e.setAttribute('remote', p.remote.name)
       if peg_rev:
         if self.IsMirror:
@@ -221,6 +261,12 @@ class XmlManifest(object):
         ce.setAttribute('dest', c.dest)
         e.appendChild(ce)
 
+      for l in p.linkfiles:
+        le = doc.createElement('linkfile')
+        le.setAttribute('src', l.src)
+        le.setAttribute('dest', l.dest)
+        e.appendChild(le)
+
       default_groups = ['all', 'name:%s' % p.name, 'path:%s' % p.relpath]
       egroups = [g for g in p.groups if g not in default_groups]
       if egroups:
@@ -236,15 +282,15 @@ class XmlManifest(object):
       if p.sync_c:
         e.setAttribute('sync-c', 'true')
 
-      if p.subprojects:
+      if p.sync_s:
-        sort_projects = [subp.name for subp in p.subprojects]
+        e.setAttribute('sync-s', 'true')
-        sort_projects.sort()
-        output_projects(p, e, sort_projects)
 
-    sort_projects = [key for key in self.projects.keys()
+      if p.subprojects:
-                     if not self.projects[key].parent]
+        subprojects = set(subp.name for subp in p.subprojects)
-    sort_projects.sort()
+        output_projects(p, e, list(sorted(subprojects)))
-    output_projects(None, root, sort_projects)
+
+    projects = set(p.name for p in self._paths.values() if not p.parent)
+    output_projects(None, root, list(sorted(projects)))
 
     if self._repo_hooks_project:
       root.appendChild(doc.createTextNode(''))
@@ -256,10 +302,15 @@ class XmlManifest(object):
 
     doc.writexml(fd, '', '  ', '\n', 'UTF-8')
 
+  @property
+  def paths(self):
+    self._Load()
+    return self._paths
+
   @property
   def projects(self):
     self._Load()
-    return self._projects
+    return self._paths.values()
 
   @property
   def remotes(self):
@@ -290,9 +341,14 @@ class XmlManifest(object):
   def IsMirror(self):
     return self.manifestProject.config.GetBoolean('repo.mirror')
 
+  @property
+  def IsArchive(self):
+    return self.manifestProject.config.GetBoolean('repo.archive')
+
   def _Unload(self):
     self._loaded = False
     self._projects = {}
+    self._paths = {}
     self._remotes = {}
     self._default = None
     self._repo_hooks_project = None
@@ -314,9 +370,29 @@ class XmlManifest(object):
 
       local = os.path.join(self.repodir, LOCAL_MANIFEST_NAME)
       if os.path.exists(local):
+        if not self.localManifestWarning:
+          self.localManifestWarning = True
+          print('warning: %s is deprecated; put local manifests in `%s` instead'
+                % (LOCAL_MANIFEST_NAME, os.path.join(self.repodir, LOCAL_MANIFESTS_DIR_NAME)),
+                file=sys.stderr)
         nodes.append(self._ParseManifestXml(local, self.repodir))
 
-      self._ParseManifest(nodes)
+      local_dir = os.path.abspath(os.path.join(self.repodir, LOCAL_MANIFESTS_DIR_NAME))
+      try:
+        for local_file in sorted(os.listdir(local_dir)):
+          if local_file.endswith('.xml'):
+            local = os.path.join(local_dir, local_file)
+            nodes.append(self._ParseManifestXml(local, self.repodir))
+      except OSError:
+        pass
+
+      try:
+        self._ParseManifest(nodes)
+      except ManifestParseError as e:
+        # There was a problem parsing, unload ourselves in case they catch
+        # this error and try again later, we will show the correct error
+        self._Unload()
+        raise e
 
       if self.IsMirror:
         self._AddMetaProjectMirror(self.repoProject)
@@ -325,7 +401,11 @@ class XmlManifest(object):
       self._loaded = True
 
   def _ParseManifestXml(self, path, include_root):
-    root = xml.dom.minidom.parse(path)
+    try:
+      root = xml.dom.minidom.parse(path)
+    except (OSError, xml.parsers.expat.ExpatError) as e:
+      raise ManifestParseError("error parsing manifest %s: %s" % (path, e))
+
     if not root or not root.childNodes:
       raise ManifestParseError("no root node in %s" % (path,))
 
@@ -338,43 +418,47 @@ class XmlManifest(object):
     nodes = []
     for node in manifest.childNodes:  # pylint:disable=W0631
                                       # We only get here if manifest is initialised
-        if node.nodeName == 'include':
+      if node.nodeName == 'include':
-            name = self._reqatt(node, 'name')
+        name = self._reqatt(node, 'name')
-            fp = os.path.join(include_root, name)
+        fp = os.path.join(include_root, name)
-            if not os.path.isfile(fp):
+        if not os.path.isfile(fp):
-                raise ManifestParseError, \
+          raise ManifestParseError("include %s doesn't exist or isn't a file"
-                    "include %s doesn't exist or isn't a file" % \
+              % (name,))
-                    (name,)
+        try:
-            try:
+          nodes.extend(self._ParseManifestXml(fp, include_root))
-                nodes.extend(self._ParseManifestXml(fp, include_root))
+        # should isolate this to the exact exception, but that's
-            # should isolate this to the exact exception, but that's
+        # tricky.  actual parsing implementation may vary.
-            # tricky.  actual parsing implementation may vary.
+        except (KeyboardInterrupt, RuntimeError, SystemExit):
-            except (KeyboardInterrupt, RuntimeError, SystemExit):
+          raise
-                raise
+        except Exception as e:
-            except Exception as e:
+          raise ManifestParseError(
-                raise ManifestParseError(
+              "failed parsing included manifest %s: %s", (name, e))
-                    "failed parsing included manifest %s: %s", (name, e))
+      else:
-        else:
+        nodes.append(node)
-          nodes.append(node)
     return nodes
 
   def _ParseManifest(self, node_list):
     for node in itertools.chain(*node_list):
       if node.nodeName == 'remote':
         remote = self._ParseRemote(node)
-        if self._remotes.get(remote.name):
+        if remote:
-          raise ManifestParseError(
+          if remote.name in self._remotes:
-              'duplicate remote %s in %s' %
+            if remote != self._remotes[remote.name]:
-              (remote.name, self.manifestFile))
+              raise ManifestParseError(
-        self._remotes[remote.name] = remote
+                  'remote %s already exists with different attributes' %
+                  (remote.name))
+          else:
+            self._remotes[remote.name] = remote
 
     for node in itertools.chain(*node_list):
       if node.nodeName == 'default':
-        if self._default is not None:
+        new_default = self._ParseDefault(node)
-          raise ManifestParseError(
+        if self._default is None:
-              'duplicate default in %s' %
+          self._default = new_default
-              (self.manifestFile))
+        elif new_default != self._default:
-        self._default = self._ParseDefault(node)
+          raise ManifestParseError('duplicate default in %s' %
+                                   (self.manifestFile))
+
     if self._default is None:
       self._default = _Default()
 
@@ -390,22 +474,29 @@ class XmlManifest(object):
       if node.nodeName == 'manifest-server':
         url = self._reqatt(node, 'url')
         if self._manifest_server is not None:
-            raise ManifestParseError(
+          raise ManifestParseError(
-                'duplicate manifest-server in %s' %
+              'duplicate manifest-server in %s' %
-                (self.manifestFile))
+              (self.manifestFile))
         self._manifest_server = url
 
+    def recursively_add_projects(project):
+      projects = self._projects.setdefault(project.name, [])
+      if project.relpath is None:
+        raise ManifestParseError(
+            'missing path for %s in %s' %
+            (project.name, self.manifestFile))
+      if project.relpath in self._paths:
+        raise ManifestParseError(
+            'duplicate path %s in %s' %
+            (project.relpath, self.manifestFile))
+      self._paths[project.relpath] = project
+      projects.append(project)
+      for subproject in project.subprojects:
+        recursively_add_projects(subproject)
+
     for node in itertools.chain(*node_list):
       if node.nodeName == 'project':
         project = self._ParseProject(node)
-        def recursively_add_projects(project):
-          if self._projects.get(project.name):
-            raise ManifestParseError(
-                'duplicate project %s in %s' %
-                (project.name, self.manifestFile))
-          self._projects[project.name] = project
-          for subproject in project.subprojects:
-            recursively_add_projects(subproject)
         recursively_add_projects(project)
       if node.nodeName == 'repo-hooks':
         # Get the name of the project and the (space-separated) list of enabled.
@@ -420,22 +511,30 @@ class XmlManifest(object):
 
         # Store a reference to the Project.
         try:
-          self._repo_hooks_project = self._projects[repo_hooks_project]
+          repo_hooks_projects = self._projects[repo_hooks_project]
         except KeyError:
           raise ManifestParseError(
               'project %s not found for repo-hooks' %
               (repo_hooks_project))
 
+        if len(repo_hooks_projects) != 1:
+          raise ManifestParseError(
+              'internal error parsing repo-hooks in %s' %
+              (self.manifestFile))
+        self._repo_hooks_project = repo_hooks_projects[0]
+
         # Store the enabled hooks in the Project object.
         self._repo_hooks_project.enabled_repo_hooks = enabled_repo_hooks
       if node.nodeName == 'remove-project':
         name = self._reqatt(node, 'name')
-        try:
+
-          del self._projects[name]
+        if name not in self._projects:
-        except KeyError:
+          raise ManifestParseError('remove-project element specifies non-existent '
-          raise ManifestParseError(
+                                   'project: %s' % name)
-              'project %s not found' %
+
-              (name))
+        for p in self._projects[name]:
+          del self._paths[p.relpath]
+        del self._projects[name]
 
         # If the manifest removes the hooks project, treat it as if it deleted
         # the repo-hooks element too.
@@ -447,7 +546,7 @@ class XmlManifest(object):
     name = None
     m_url = m.GetRemote(m.remote.name).url
     if m_url.endswith('/.git'):
-      raise ManifestParseError, 'refusing to mirror %s' % m_url
+      raise ManifestParseError('refusing to mirror %s' % m_url)
 
     if self._default and self._default.remote:
       url = self._default.remote.resolvedFetchUrl
@@ -473,11 +572,13 @@ class XmlManifest(object):
                         name = name,
                         remote = remote.ToRemoteSpec(name),
                         gitdir = gitdir,
+                        objdir = gitdir,
                         worktree = None,
-                        relpath = None,
+                        relpath = name or None,
                         revisionExpr = m.revisionExpr,
                         revisionId = None)
-      self._projects[project.name] = project
+      self._projects[project.name] = [project]
+      self._paths[project.relpath] = project
 
   def _ParseRemote(self, node):
     """
@@ -504,6 +605,8 @@ class XmlManifest(object):
     if d.revisionExpr == '':
       d.revisionExpr = None
 
+    d.destBranchExpr = node.getAttribute('dest-branch') or None
+
     sync_j = node.getAttribute('sync-j')
     if sync_j == '' or sync_j is None:
       d.sync_j = 1
@@ -515,6 +618,12 @@ class XmlManifest(object):
       d.sync_c = False
     else:
       d.sync_c = sync_c.lower() in ("yes", "true", "1")
+
+    sync_s = node.getAttribute('sync-s')
+    if not sync_s:
+      d.sync_s = False
+    else:
+      d.sync_s = sync_s.lower() in ("yes", "true", "1")
     return d
 
   def _ParseNotice(self, node):
@@ -535,7 +644,7 @@ class XmlManifest(object):
 
     # Figure out minimum indentation, skipping the first line (the same line
     # as the <notice> tag)...
-    minIndent = sys.maxint
+    minIndent = sys.maxsize
     lines = notice.splitlines()
     for line in lines[1:]:
       lstrippedLine = line.lstrip()
@@ -574,25 +683,22 @@ class XmlManifest(object):
     if remote is None:
       remote = self._default.remote
     if remote is None:
-      raise ManifestParseError, \
+      raise ManifestParseError("no remote for project %s within %s" %
-            "no remote for project %s within %s" % \
+            (name, self.manifestFile))
-            (name, self.manifestFile)
 
     revisionExpr = node.getAttribute('revision')
     if not revisionExpr:
       revisionExpr = self._default.revisionExpr
     if not revisionExpr:
-      raise ManifestParseError, \
+      raise ManifestParseError("no revision for project %s within %s" %
-            "no revision for project %s within %s" % \
+            (name, self.manifestFile))
-            (name, self.manifestFile)
 
     path = node.getAttribute('path')
     if not path:
       path = name
     if path.startswith('/'):
-      raise ManifestParseError, \
+      raise ManifestParseError("project %s path cannot be absolute in %s" %
-            "project %s path cannot be absolute in %s" % \
+            (name, self.manifestFile))
-            (name, self.manifestFile)
 
     rebase = node.getAttribute('rebase')
     if not rebase:
@@ -606,25 +712,49 @@ class XmlManifest(object):
     else:
       sync_c = sync_c.lower() in ("yes", "true", "1")
 
+    sync_s = node.getAttribute('sync-s')
+    if not sync_s:
+      sync_s = self._default.sync_s
+    else:
+      sync_s = sync_s.lower() in ("yes", "true", "1")
+
+    clone_depth = node.getAttribute('clone-depth')
+    if clone_depth:
+      try:
+        clone_depth = int(clone_depth)
+        if  clone_depth <= 0:
+          raise ValueError()
+      except ValueError:
+        raise ManifestParseError('invalid clone-depth %s in %s' %
+                                 (clone_depth, self.manifestFile))
+
+    dest_branch = node.getAttribute('dest-branch') or self._default.destBranchExpr
+
     upstream = node.getAttribute('upstream')
 
     groups = ''
     if node.hasAttribute('groups'):
       groups = node.getAttribute('groups')
-    groups = [x for x in re.split('[,\s]+', groups) if x]
+    groups = [x for x in re.split(r'[,\s]+', groups) if x]
 
     if parent is None:
-      relpath, worktree, gitdir = self.GetProjectPaths(name, path)
+      relpath, worktree, gitdir, objdir = self.GetProjectPaths(name, path)
     else:
-      relpath, worktree, gitdir = self.GetSubprojectPaths(parent, path)
+      relpath, worktree, gitdir, objdir = \
+          self.GetSubprojectPaths(parent, name, path)
 
     default_groups = ['all', 'name:%s' % name, 'path:%s' % relpath]
     groups.extend(set(default_groups).difference(groups))
 
+    if self.IsMirror and node.hasAttribute('force-path'):
+      if node.getAttribute('force-path').lower() in ("yes", "true", "1"):
+        gitdir = os.path.join(self.topdir, '%s.git' % path)
+
     project = Project(manifest = self,
                       name = name,
                       remote = remote.ToRemoteSpec(name),
                       gitdir = gitdir,
+                      objdir = objdir,
                       worktree = worktree,
                       relpath = relpath,
                       revisionExpr = revisionExpr,
@@ -632,12 +762,17 @@ class XmlManifest(object):
                       rebase = rebase,
                       groups = groups,
                       sync_c = sync_c,
+                      sync_s = sync_s,
+                      clone_depth = clone_depth,
                       upstream = upstream,
-                      parent = parent)
+                      parent = parent,
+                      dest_branch = dest_branch)
 
     for n in node.childNodes:
       if n.nodeName == 'copyfile':
         self._ParseCopyFile(project, n)
+      if n.nodeName == 'linkfile':
+        self._ParseLinkFile(project, n)
       if n.nodeName == 'annotation':
         self._ParseAnnotation(project, n)
       if n.nodeName == 'project':
@@ -650,10 +785,15 @@ class XmlManifest(object):
     if self.IsMirror:
       worktree = None
       gitdir = os.path.join(self.topdir, '%s.git' % name)
+      objdir = gitdir
     else:
       worktree = os.path.join(self.topdir, path).replace('\\', '/')
       gitdir = os.path.join(self.repodir, 'projects', '%s.git' % path)
-    return relpath, worktree, gitdir
+      objdir = os.path.join(self.repodir, 'project-objects', '%s.git' % name)
+    return relpath, worktree, gitdir, objdir
+
+  def GetProjectsWithName(self, name):
+    return self._projects.get(name, [])
 
   def GetSubprojectName(self, parent, submodule_path):
     return os.path.join(parent.name, submodule_path)
@@ -664,14 +804,15 @@ class XmlManifest(object):
   def _UnjoinRelpath(self, parent_relpath, relpath):
     return os.path.relpath(relpath, parent_relpath)
 
-  def GetSubprojectPaths(self, parent, path):
+  def GetSubprojectPaths(self, parent, name, path):
     relpath = self._JoinRelpath(parent.relpath, path)
     gitdir = os.path.join(parent.gitdir, 'subprojects', '%s.git' % path)
+    objdir = os.path.join(parent.gitdir, 'subproject-objects', '%s.git' % name)
     if self.IsMirror:
       worktree = None
     else:
       worktree = os.path.join(parent.worktree, path).replace('\\', '/')
-    return relpath, worktree, gitdir
+    return relpath, worktree, gitdir, objdir
 
   def _ParseCopyFile(self, project, node):
     src = self._reqatt(node, 'src')
@@ -681,6 +822,14 @@ class XmlManifest(object):
       # dest is relative to the top of the tree
       project.AddCopyFile(src, dest, os.path.join(self.topdir, dest))
 
+  def _ParseLinkFile(self, project, node):
+    src = self._reqatt(node, 'src')
+    dest = self._reqatt(node, 'dest')
+    if not self.IsMirror:
+      # src is project relative;
+      # dest is relative to the top of the tree
+      project.AddLinkFile(src, dest, os.path.join(self.topdir, dest))
+
   def _ParseAnnotation(self, project, node):
     name = self._reqatt(node, 'name')
     value = self._reqatt(node, 'value')
@@ -689,7 +838,8 @@ class XmlManifest(object):
     except ManifestParseError:
       keep = "true"
     if keep != "true" and keep != "false":
-      raise ManifestParseError, "optional \"keep\" attribute must be \"true\" or \"false\""
+      raise ManifestParseError('optional "keep" attribute must be '
+            '"true" or "false"')
     project.AddAnnotation(name, value, keep)
 
   def _get_remote(self, node):
@@ -699,9 +849,8 @@ class XmlManifest(object):
 
     v = self._remotes.get(name)
     if not v:
-      raise ManifestParseError, \
+      raise ManifestParseError("remote %s not defined in %s" %
-            "remote %s not defined in %s" % \
+            (name, self.manifestFile))
-            (name, self.manifestFile)
     return v
 
   def _reqatt(self, node, attname):
@@ -710,7 +859,43 @@ class XmlManifest(object):
     """
     v = node.getAttribute(attname)
     if not v:
-      raise ManifestParseError, \
+      raise ManifestParseError("no %s in <%s> within %s" %
-            "no %s in <%s> within %s" % \
+            (attname, node.nodeName, self.manifestFile))
-            (attname, node.nodeName, self.manifestFile)
     return v
+
+  def projectsDiff(self, manifest):
+    """return the projects differences between two manifests.
+
+    The diff will be from self to given manifest.
+
+    """
+    fromProjects = self.paths
+    toProjects = manifest.paths
+
+    fromKeys = fromProjects.keys()
+    fromKeys.sort()
+    toKeys = toProjects.keys()
+    toKeys.sort()
+
+    diff = {'added': [], 'removed': [], 'changed': [], 'unreachable': []}
+
+    for proj in fromKeys:
+      if not proj in toKeys:
+        diff['removed'].append(fromProjects[proj])
+      else:
+        fromProj = fromProjects[proj]
+        toProj = toProjects[proj]
+        try:
+          fromRevId = fromProj.GetCommitRevisionId()
+          toRevId = toProj.GetCommitRevisionId()
+        except ManifestInvalidRevisionError:
+          diff['unreachable'].append((fromProj, toProj))
+        else:
+          if fromRevId != toRevId:
+            diff['changed'].append((fromProj, toProj))
+        toKeys.remove(proj)
+
+    for proj in toKeys:
+      diff['added'].append(toProjects[proj])
+
+    return diff

pager.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import os
 import select
 import sys
@@ -49,7 +50,7 @@ def RunPager(globalConfig):
 
     _BecomePager(pager)
   except Exception:
-    print >>sys.stderr, "fatal: cannot start pager '%s'" % pager
+    print("fatal: cannot start pager '%s'" % pager, file=sys.stderr)
     sys.exit(255)
 
 def _SelectPager(globalConfig):

pyversion.py

@@ -0,0 +1,19 @@
+#
+# Copyright (C) 2013 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+def is_python3():
+  return sys.version_info[0] == 3

repo

@@ -1,9 +1,9 @@
-#!/bin/sh
+#!/usr/bin/env python
 
 ## repo default configuration
 ##
-REPO_URL='https://gerrit.googlesource.com/git-repo'
+REPO_URL = 'https://gerrit.googlesource.com/git-repo'
-REPO_REV='stable'
+REPO_REV = 'stable'
 
 # Copyright (C) 2008 Google Inc.
 #
@@ -19,19 +19,11 @@ REPO_REV='stable'
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-magic='--calling-python-from-/bin/sh--'
-"""exec" python -E "$0" "$@" """#$magic"
-if __name__ == '__main__':
-  import sys
-  if sys.argv[-1] == '#%s' % magic:
-    del sys.argv[-1]
-del magic
-
 # increment this whenever we make important changes to this script
-VERSION = (1, 18)
+VERSION = (1, 21)
 
 # increment this if the MAINTAINER_KEYS block is modified
-KEYRING_VERSION = (1,1)
+KEYRING_VERSION = (1, 2)
 MAINTAINER_KEYS = """
 
      Repo Maintainer <repo@android.kernel.org>
@@ -80,49 +72,82 @@ TACbBS+Up3RpfYVfd63c1cDdlru13pQAn3NQy/SN858MkxN+zym86UBgOad2
 -----BEGIN PGP PUBLIC KEY BLOCK-----
 Version: GnuPG v1.4.11 (GNU/Linux)
 
-mQENBFBiLPwBCACvISTASOgFXwADw2GYRH2I2z9RvYkYoZ6ThTTNlMXbbYYKO2Wo
+mQENBFHRvc8BCADFg45Xx/y6QDC+T7Y/gGc7vx0ww7qfOwIKlAZ9xG3qKunMxo+S
-a9LQDNW0TbCEekg5UKk0FD13XOdWaqUt4Gtuvq9c43GRSjMO6NXH+0BjcQ8vUtY2
+hPCnzEl3cq+6I1Ww/ndop/HB3N3toPXRCoN8Vs4/Hc7by+SnaLFnacrm+tV5/OgT
-/W4CYUevwdo4nQ1+1zsOCu1XYe/CReXq0fdugv3hgmRmh3sz1soo37Q44W2frxxg
+V37Lzt8lhay1Kl+YfpFwHYYpIEBLFV9knyfRXS/428W2qhdzYfvB15/AasRmwmor
-U7Rz3Da4FjgAL0RQ8qndD+LwRHXTY7H7wYM8V/3cYFZV7pSodd75q3MAXYQLf0ZV
+py4NIzSs8UD/SPr1ihqNCdZM76+MQyN5HMYXW/ALZXUFG0pwluHFA7hrfPG74i8C
-QR1XATu5l1QnXrxgHvz7MmDwb1D+jX3YPKnZveaukigQ6hDHdiVcePBiGXmk8LZC
+zMiP7qvMWIl/r/jtzHioH1dRKgbod+LZsrDJ8mBaqsZaDmNJMhss9g76XvfMyLra
-2jQkdXeF7Su1ZYpr2nnEHLJ6vOLcCpPGb8gDABEBAAG0H0NvbmxleSBPd2VucyA8
+9DI9/iFuBpGzeqBv0hwOGQspLRrEoyTeR6n1ABEBAAG0H0NvbmxleSBPd2VucyA8
-Y2NvM0BhbmRyb2lkLmNvbT6JATgEEwECACIFAlBiLPwCGwMGCwkIBwMCBhUIAgkK
+Y2NvM0BhbmRyb2lkLmNvbT6JATgEEwECACIFAlHRvc8CGwMGCwkIBwMCBhUIAgkK
-CwQWAgMBAh4BAheAAAoJEBkmlFUziHGkHVkH/2Hks2Cif5i2xPtv2IFZcjL42joU
+CwQWAgMBAh4BAheAAAoJEGe35EhpKzgsP6AIAJKJmNtn4l7hkYHKHFSo3egb6RjQ
-T7lO5XFqUYS9ZNHpGa/V0eiPt7rHoO16glR83NZtwlrq2cSN89i9HfOhMYV/qLu8
+zEIP3MFTcu8HFX1kF1ZFbrp7xqurLaE53kEkKuAAvjJDAgI8mcZHP1JyplubqjQA
-fLCHcV2muw+yCB5s5bxnI5UkToiNZyBNqFkcOt/Kbj9Hpy68A1kmc6myVEaUYebq
+xvv84gK+OGP3Xk+QK1ZjUQSbjOpjEiSZpRhWcHci3dgOUH4blJfByHw25hlgHowd
-2Chx/f3xuEthan099t746v1K+/6SvQGDNctHuaMr9cWdxZtHjdRf31SQRc99Phe5
+a/2PrNKZVcJ92YienaxxGjcXEUcd0uYEG2+rwllQigFcnMFDhr9B71MfalRHjFKE
-w+ZGR/ebxNDKRK9mKgZT8wVFHlXerJsRqWIqtx1fsW1UgLgbpcpe2MChm6B5wTu0
+fmdoypqLrri61YBc59P88Rw2/WUpTQjgNubSqa3A2+CKdaRyaRw+2fdF4TdR0h8W
-s1ltzox3l4q71FyRRPUJxXyvGkDLZWpK7EpiHSCOYq/KP3HkKeXU3xqHpcG5AQ0E
+zbg+lbaPtJHsV+3mJC7fq26MiJDRJa5ZztpMn8su20gbLgi2ShBOaHAYDDi5AQ0E
-UGIs/AEIAKzO/7lO9cB6dshmZYo8Vy/b7aGicThE+ChcDSfhvyOXVdEM2GKAjsR+
+UdG9zwEIAMoOBq+QLNozAhxOOl5GL3StTStGRgPRXINfmViTsihrqGCWBBUfXlUE
-rlBWbTFX3It301p2HwZPFEi9nEvJxVlqqBiW0bPmNMkDRR55l2vbWg35wwkg6RyE
+OytC0mYcrDUQev/8ToVoyqw+iGSwDkcSXkrEUCKFtHV/GECWtk1keyHgR10YKI1R
-Bc5/TQjhXI2w8IvlimoGoUff4t3JmMOnWrnKSvL+5iuRj12p9WmanCHzw3Ee7ztf
+mquSXoubWGqPeG1PAI74XWaRx8UrL8uCXUtmD8Q5J7mDjKR5NpxaXrwlA0bKsf2E
-/aU/q+FTpr3DLerb6S8xbv86ySgnJT6o5CyL2DCWRtnYQyGVi0ZmLzEouAYiO0hs
+Gp9tu1kKauuToZhWHMRMqYSOGikQJwWSFYKT1KdNcOXLQF6+bfoJ6sjVYdwfmNQL
-z0AAu28Mj+12g2WwePRz6gfM9rHtI37ylYW3oT/9M9mO9ei/Bc/1D7Dz6qNV+0vg
+Ixn8QVhoTDedcqClSWB17VDEFDFa7MmqXZz2qtM3X1R/MUMHqPtegQzBGNhRdnI2
-uSVJxM2Bl6GalHPZLhHntFEdIA6EdoUAEQEAAYkBHwQYAQIACQUCUGIs/AIbDAAK
+V45+1Nnx/uuCxDbeI4RbHzujnxDiq70AEQEAAYkBHwQYAQIACQUCUdG9zwIbDAAK
-CRAZJpRVM4hxpNfkB/0W/hP5WK/NETXBlWXXW7JPaWO2c5kGwD0lnj5RRmridyo1
+CRBnt+RIaSs4LNVeB/0Y2pZ8I7gAAcEM0Xw8drr4omg2fUoK1J33ozlA/RxeA/lJ
-vbm5PdM91jOsDQYqRu6YOoYBnDnEhB2wL2bPh34HWwwrA+LwB8hlcAV2z1bdwyfl
+I3KnyCDTpXuIeBKPGkdL8uMATC9Z8DnBBajRlftNDVZS3Hz4G09G9QpMojvJkFJV
-3R823fReKN3QcvLHzmvZPrF4Rk97M9UIyKS0RtnfTWykRgDWHIsrtQPoNwsXrWoT
+By+01Flw/X+eeN8NpqSuLV4W+AjEO8at/VvgKr1AFvBRdZ7GkpI1o6DgPe7ZqX+1
-9LrM2v+1+9mp3vuXnE473/NHxmiWEQH9Ez+O/mOxQ7rSOlqGRiKq/IBZCfioJOtV
+dzQZt3e13W0rVBb/bUgx9iSLoeWP3aq/k+/GRGOR+S6F6BBSl0SQ2EF2+dIywb1x
-fTQeIu/yASZnsLBqr6SJEGwYBoWcyjG++k4fyw8ocOAo4uGDYbxgN7yYfNQ0OH7o
+JuinEP+AwLAUZ1Bsx9ISC0Agpk2VeHXPL3FGhroEmoMvBzO0kTFGyoeT7PR/BfKv
-V6pfUgqKLWa/aK7/N1ZHnPdFLD8Xt0Dmy4BPwrKC
++H/g3HsL2LOB9uoIm8/5p2TTU5ttYCXMHhQZ81AY
-=O7am
+=AUp4
 -----END PGP PUBLIC KEY BLOCK-----
 """
 
 GIT = 'git'                     # our git command
-MIN_GIT_VERSION = (1, 5, 4)     # minimum supported git version
+MIN_GIT_VERSION = (1, 7, 2)     # minimum supported git version
 repodir = '.repo'               # name of repo's private directory
 S_repo = 'repo'                 # special repo repository
 S_manifests = 'manifests'       # special manifest repository
 REPO_MAIN = S_repo + '/main.py' # main script
+MIN_PYTHON_VERSION = (2, 6)     # minimum supported python version
 
 
+import errno
 import optparse
 import os
 import re
+import shutil
+import stat
 import subprocess
 import sys
-import urllib2
+
+if sys.version_info[0] == 3:
+  import urllib.request
+  import urllib.error
+else:
+  import imp
+  import urllib2
+  urllib = imp.new_module('urllib')
+  urllib.request = urllib2
+  urllib.error = urllib2
+
+
+def _print(*objects, **kwargs):
+  sep = kwargs.get('sep', ' ')
+  end = kwargs.get('end', '\n')
+  out = kwargs.get('file', sys.stdout)
+  out.write(sep.join(objects) + end)
+
+
+# Python version check
+ver = sys.version_info
+if ver[0] == 3:
+  _print('warning: Python 3 support is currently experimental. YMMV.\n'
+         'Please use Python 2.6 - 2.7 instead.',
+         file=sys.stderr)
+if (ver[0], ver[1]) < MIN_PYTHON_VERSION:
+  _print('error: Python version %s unsupported.\n'
+         'Please use Python 2.6 - 2.7 instead.'
+         % sys.version.split(' ')[0], file=sys.stderr)
+  sys.exit(1)
 
 home_dot_repo = os.path.expanduser('~/.repoconfig')
 gpg_dir = os.path.join(home_dot_repo, 'gnupg')
@@ -149,16 +174,22 @@ group.add_option('-m', '--manifest-name',
                  help='initial manifest file', metavar='NAME.xml')
 group.add_option('--mirror',
                  dest='mirror', action='store_true',
-                 help='mirror the forrest')
+                 help='create a replica of the remote repositories '
+                      'rather than a client working directory')
 group.add_option('--reference',
                  dest='reference',
                  help='location of mirror directory', metavar='DIR')
 group.add_option('--depth', type='int', default=None,
                  dest='depth',
                  help='create a shallow clone with given depth; see git clone')
+group.add_option('--archive',
+                 dest='archive', action='store_true',
+                 help='checkout an archive instead of a git repository for '
+                      'each project. See git archive.')
 group.add_option('-g', '--groups',
                  dest='groups', default='default',
-                 help='restrict manifest projects to ones with a specified group',
+                 help='restrict manifest projects to ones with specified '
+                      'group(s) [default|all|G1,G2,G3|G4,-G5,-G6]',
                  metavar='GROUP')
 group.add_option('-p', '--platform',
                  dest='platform', default="auto",
@@ -211,17 +242,16 @@ def _Init(args):
   if branch.startswith('refs/heads/'):
     branch = branch[len('refs/heads/'):]
   if branch.startswith('refs/'):
-    print >>sys.stderr, "fatal: invalid branch name '%s'" % branch
+    _print("fatal: invalid branch name '%s'" % branch, file=sys.stderr)
     raise CloneFailure()
 
-  if not os.path.isdir(repodir):
+  try:
-    try:
+    os.mkdir(repodir)
-      os.mkdir(repodir)
+  except OSError as e:
-    except OSError as e:
+    if e.errno != errno.EEXIST:
-      print >>sys.stderr, \
+      _print('fatal: cannot make %s directory: %s'
-            'fatal: cannot make %s directory: %s' % (
+             % (repodir, e.strerror), file=sys.stderr)
-            repodir, e.strerror)
+      # Don't raise CloneFailure; that would delete the
-      # Don't faise CloneFailure; that would delete the
       # name. Instead exit immediately.
       #
       sys.exit(1)
@@ -244,37 +274,50 @@ def _Init(args):
     _Checkout(dst, branch, rev, opt.quiet)
   except CloneFailure:
     if opt.quiet:
-      print >>sys.stderr, \
+      _print('fatal: repo init failed; run without --quiet to see why',
-        'fatal: repo init failed; run without --quiet to see why'
+             file=sys.stderr)
     raise
 
 
+def ParseGitVersion(ver_str):
+  if not ver_str.startswith('git version '):
+    return None
+
+  num_ver_str = ver_str[len('git version '):].strip().split('-')[0]
+  to_tuple = []
+  for num_str in num_ver_str.split('.')[:3]:
+    if num_str.isdigit():
+      to_tuple.append(int(num_str))
+    else:
+      to_tuple.append(0)
+  return tuple(to_tuple)
+
+
 def _CheckGitVersion():
   cmd = [GIT, '--version']
   try:
     proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
   except OSError as e:
-    print >>sys.stderr
+    _print(file=sys.stderr)
-    print >>sys.stderr, "fatal: '%s' is not available" % GIT
+    _print("fatal: '%s' is not available" % GIT, file=sys.stderr)
-    print >>sys.stderr, 'fatal: %s' % e
+    _print('fatal: %s' % e, file=sys.stderr)
-    print >>sys.stderr
+    _print(file=sys.stderr)
-    print >>sys.stderr, 'Please make sure %s is installed'\
+    _print('Please make sure %s is installed and in your path.' % GIT,
-                        ' and in your path.' % GIT
+           file=sys.stderr)
     raise CloneFailure()
 
   ver_str = proc.stdout.read().strip()
   proc.stdout.close()
   proc.wait()
 
-  if not ver_str.startswith('git version '):
+  ver_act = ParseGitVersion(ver_str)
-    print >>sys.stderr, 'error: "%s" unsupported' % ver_str
+  if ver_act is None:
+    _print('error: "%s" unsupported' % ver_str, file=sys.stderr)
     raise CloneFailure()
 
-  ver_str = ver_str[len('git version '):].strip()
-  ver_act = tuple(map(lambda x: int(x), ver_str.split('.')[0:3]))
   if ver_act < MIN_GIT_VERSION:
-    need = '.'.join(map(lambda x: str(x), MIN_GIT_VERSION))
+    need = '.'.join(map(str, MIN_GIT_VERSION))
-    print >>sys.stderr, 'fatal: git %s or later required' % need
+    _print('fatal: git %s or later required' % need, file=sys.stderr)
     raise CloneFailure()
 
 
@@ -290,29 +333,27 @@ def NeedSetupGnuPG():
   if not kv:
     return True
 
-  kv = tuple(map(lambda x: int(x), kv.split('.')))
+  kv = tuple(map(int, kv.split('.')))
   if kv < KEYRING_VERSION:
     return True
   return False
 
 
 def SetupGnuPG(quiet):
-  if not os.path.isdir(home_dot_repo):
+  try:
-    try:
+    os.mkdir(home_dot_repo)
-      os.mkdir(home_dot_repo)
+  except OSError as e:
-    except OSError as e:
+    if e.errno != errno.EEXIST:
-      print >>sys.stderr, \
+      _print('fatal: cannot make %s directory: %s'
-            'fatal: cannot make %s directory: %s' % (
+             % (home_dot_repo, e.strerror), file=sys.stderr)
-            home_dot_repo, e.strerror)
       sys.exit(1)
 
-  if not os.path.isdir(gpg_dir):
+  try:
-    try:
+    os.mkdir(gpg_dir, stat.S_IRWXU)
-      os.mkdir(gpg_dir, 0700)
+  except OSError as e:
-    except OSError as e:
+    if e.errno != errno.EEXIST:
-      print >>sys.stderr, \
+      _print('fatal: cannot make %s directory: %s' % (gpg_dir, e.strerror),
-            'fatal: cannot make %s directory: %s' % (
+             file=sys.stderr)
-            gpg_dir, e.strerror)
       sys.exit(1)
 
   env = os.environ.copy()
@@ -325,21 +366,21 @@ def SetupGnuPG(quiet):
                             stdin = subprocess.PIPE)
   except OSError as e:
     if not quiet:
-      print >>sys.stderr, 'warning: gpg (GnuPG) is not available.'
+      _print('warning: gpg (GnuPG) is not available.', file=sys.stderr)
-      print >>sys.stderr, 'warning: Installing it is strongly encouraged.'
+      _print('warning: Installing it is strongly encouraged.', file=sys.stderr)
-      print >>sys.stderr
+      _print(file=sys.stderr)
     return False
 
   proc.stdin.write(MAINTAINER_KEYS)
   proc.stdin.close()
 
   if proc.wait() != 0:
-    print >>sys.stderr, 'fatal: registering repo maintainer keys failed'
+    _print('fatal: registering repo maintainer keys failed', file=sys.stderr)
     sys.exit(1)
-  print
+  _print()
 
   fd = open(os.path.join(home_dot_repo, 'keyring-version'), 'w')
-  fd.write('.'.join(map(lambda x: str(x), KEYRING_VERSION)) + '\n')
+  fd.write('.'.join(map(str, KEYRING_VERSION)) + '\n')
   fd.close()
   return True
 
@@ -355,7 +396,7 @@ def _SetConfig(local, name, value):
 def _InitHttp():
   handlers = []
 
-  mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
+  mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
   try:
     import netrc
     n = netrc.netrc()
@@ -365,20 +406,20 @@ def _InitHttp():
       mgr.add_password(p[1], 'https://%s/' % host, p[0], p[2])
   except:
     pass
-  handlers.append(urllib2.HTTPBasicAuthHandler(mgr))
+  handlers.append(urllib.request.HTTPBasicAuthHandler(mgr))
-  handlers.append(urllib2.HTTPDigestAuthHandler(mgr))
+  handlers.append(urllib.request.HTTPDigestAuthHandler(mgr))
 
   if 'http_proxy' in os.environ:
     url = os.environ['http_proxy']
-    handlers.append(urllib2.ProxyHandler({'http': url, 'https': url}))
+    handlers.append(urllib.request.ProxyHandler({'http': url, 'https': url}))
   if 'REPO_CURL_VERBOSE' in os.environ:
-    handlers.append(urllib2.HTTPHandler(debuglevel=1))
+    handlers.append(urllib.request.HTTPHandler(debuglevel=1))
-    handlers.append(urllib2.HTTPSHandler(debuglevel=1))
+    handlers.append(urllib.request.HTTPSHandler(debuglevel=1))
-  urllib2.install_opener(urllib2.build_opener(*handlers))
+  urllib.request.install_opener(urllib.request.build_opener(*handlers))
 
 def _Fetch(url, local, src, quiet):
   if not quiet:
-    print >>sys.stderr, 'Get %s' % url
+    _print('Get %s' % url, file=sys.stderr)
 
   cmd = [GIT, 'fetch']
   if quiet:
@@ -423,20 +464,20 @@ def _DownloadBundle(url, local, quiet):
   dest = open(os.path.join(local, '.git', 'clone.bundle'), 'w+b')
   try:
     try:
-      r = urllib2.urlopen(url)
+      r = urllib.request.urlopen(url)
-    except urllib2.HTTPError as e:
+    except urllib.error.HTTPError as e:
-      if e.code == 404:
+      if e.code in [403, 404]:
         return False
-      print >>sys.stderr, 'fatal: Cannot get %s' % url
+      _print('fatal: Cannot get %s' % url, file=sys.stderr)
-      print >>sys.stderr, 'fatal: HTTP error %s' % e.code
+      _print('fatal: HTTP error %s' % e.code, file=sys.stderr)
       raise CloneFailure()
-    except urllib2.URLError as e:
+    except urllib.error.URLError as e:
-      print >>sys.stderr, 'fatal: Cannot get %s' % url
+      _print('fatal: Cannot get %s' % url, file=sys.stderr)
-      print >>sys.stderr, 'fatal: error %s' % e.reason
+      _print('fatal: error %s' % e.reason, file=sys.stderr)
       raise CloneFailure()
     try:
       if not quiet:
-        print >>sys.stderr, 'Get %s' % url
+        _print('Get %s' % url, file=sys.stderr)
       while True:
         buf = r.read(8192)
         if buf == '':
@@ -460,24 +501,23 @@ def _Clone(url, local, quiet):
   try:
     os.mkdir(local)
   except OSError as e:
-    print >>sys.stderr, \
+    _print('fatal: cannot make %s directory: %s' % (local, e.strerror),
-          'fatal: cannot make %s directory: %s' \
+           file=sys.stderr)
-          % (local, e.strerror)
     raise CloneFailure()
 
   cmd = [GIT, 'init', '--quiet']
   try:
     proc = subprocess.Popen(cmd, cwd = local)
   except OSError as e:
-    print >>sys.stderr
+    _print(file=sys.stderr)
-    print >>sys.stderr, "fatal: '%s' is not available" % GIT
+    _print("fatal: '%s' is not available" % GIT, file=sys.stderr)
-    print >>sys.stderr, 'fatal: %s' % e
+    _print('fatal: %s' % e, file=sys.stderr)
-    print >>sys.stderr
+    _print(file=sys.stderr)
-    print >>sys.stderr, 'Please make sure %s is installed'\
+    _print('Please make sure %s is installed and in your path.' % GIT,
-                        ' and in your path.' % GIT
+          file=sys.stderr)
     raise CloneFailure()
   if proc.wait() != 0:
-    print >>sys.stderr, 'fatal: could not create %s' % local
+    _print('fatal: could not create %s' % local, file=sys.stderr)
     raise CloneFailure()
 
   _InitHttp()
@@ -505,21 +545,18 @@ def _Verify(cwd, branch, quiet):
   proc.stderr.close()
 
   if proc.wait() != 0 or not cur:
-    print >>sys.stderr
+    _print(file=sys.stderr)
-    print >>sys.stderr,\
+    _print("fatal: branch '%s' has not been signed" % branch, file=sys.stderr)
-      "fatal: branch '%s' has not been signed" \
-      % branch
     raise CloneFailure()
 
   m = re.compile(r'^(.*)-[0-9]{1,}-g[0-9a-f]{1,}$').match(cur)
   if m:
     cur = m.group(1)
     if not quiet:
-      print >>sys.stderr
+      _print(file=sys.stderr)
-      print >>sys.stderr, \
+      _print("info: Ignoring branch '%s'; using tagged release '%s'"
-        "info: Ignoring branch '%s'; using tagged release '%s'" \
+            % (branch, cur), file=sys.stderr)
-        % (branch, cur)
+      _print(file=sys.stderr)
-      print >>sys.stderr
 
   env = os.environ.copy()
   env['GNUPGHOME'] = gpg_dir.encode()
@@ -537,10 +574,10 @@ def _Verify(cwd, branch, quiet):
   proc.stderr.close()
 
   if proc.wait() != 0:
-    print >>sys.stderr
+    _print(file=sys.stderr)
-    print >>sys.stderr, out
+    _print(out, file=sys.stderr)
-    print >>sys.stderr, err
+    _print(err, file=sys.stderr)
-    print >>sys.stderr
+    _print(file=sys.stderr)
     raise CloneFailure()
   return '%s^0' % cur
 
@@ -594,7 +631,7 @@ def _ParseArguments(args):
   opt = _Options()
   arg = []
 
-  for i in xrange(0, len(args)):
+  for i in range(len(args)):
     a = args[i]
     if a == '-h' or a == '--help':
       opt.help = True
@@ -607,7 +644,7 @@ def _ParseArguments(args):
 
 
 def _Usage():
-  print >>sys.stderr,\
+  _print(
 """usage: repo COMMAND [ARGS]
 
 repo is not yet installed.  Use "repo init" to install it here.
@@ -618,7 +655,7 @@ The most commonly used repo commands are:
   help      Display detailed help on a command
 
 For access to the full online help, install repo ("repo init").
-"""
+""", file=sys.stderr)
   sys.exit(1)
 
 
@@ -628,25 +665,23 @@ def _Help(args):
       init_optparse.print_help()
       sys.exit(0)
     else:
-      print >>sys.stderr,\
+      _print("error: '%s' is not a bootstrap command.\n"
-      "error: '%s' is not a bootstrap command.\n"\
+             '        For access to online help, install repo ("repo init").'
-      '        For access to online help, install repo ("repo init").'\
+             % args[0], file=sys.stderr)
-      % args[0]
   else:
     _Usage()
   sys.exit(1)
 
 
 def _NotInstalled():
-  print >>sys.stderr,\
+  _print('error: repo is not installed.  Use "repo init" to install it here.',
-'error: repo is not installed.  Use "repo init" to install it here.'
+         file=sys.stderr)
   sys.exit(1)
 
 
 def _NoCommands(cmd):
-  print >>sys.stderr,\
+  _print("""error: command '%s' requires repo to be installed first.
-"""error: command '%s' requires repo to be installed first.
+         Use "repo init" to install it here.""" % cmd, file=sys.stderr)
-       Use "repo init" to install it here.""" % cmd
   sys.exit(1)
 
 
@@ -683,7 +718,7 @@ def _SetDefaultsTo(gitdir):
   proc.stderr.close()
 
   if proc.wait() != 0:
-    print >>sys.stderr, 'fatal: %s has no current branch' % gitdir
+    _print('fatal: %s has no current branch' % gitdir, file=sys.stderr)
     sys.exit(1)
 
 
@@ -707,12 +742,7 @@ def main(orig_args):
       try:
         _Init(args)
       except CloneFailure:
-        for root, dirs, files in os.walk(repodir, topdown=False):
+        shutil.rmtree(repodir, ignore_errors=True)
-          for name in files:
-            os.remove(os.path.join(root, name))
-          for name in dirs:
-            os.rmdir(os.path.join(root, name))
-        os.rmdir(repodir)
         sys.exit(1)
       repo_main, rel_repo_dir = _FindRepo()
     else:
@@ -721,8 +751,8 @@ def main(orig_args):
   if my_main:
     repo_main = my_main
 
-  ver_str = '.'.join(map(lambda x: str(x), VERSION))
+  ver_str = '.'.join(map(str, VERSION))
-  me = [repo_main,
+  me = [sys.executable, repo_main,
         '--repo-dir=%s' % rel_repo_dir,
         '--wrapper-version=%s' % ver_str,
         '--wrapper-path=%s' % wrapper_path,
@@ -730,10 +760,10 @@ def main(orig_args):
   me.extend(orig_args)
   me.extend(extra_args)
   try:
-    os.execv(repo_main, me)
+    os.execv(sys.executable, me)
   except OSError as e:
-    print >>sys.stderr, "fatal: unable to start %s" % repo_main
+    _print("fatal: unable to start %s" % repo_main, file=sys.stderr)
-    print >>sys.stderr, "fatal: %s" % e
+    _print("fatal: %s" % e, file=sys.stderr)
     sys.exit(148)
 
 

subcmds/__init__.py

@@ -38,8 +38,8 @@ for py in os.listdir(my_dir):
     try:
       cmd = getattr(mod, clsn)()
     except AttributeError:
-      raise SyntaxError, '%s/%s does not define class %s' % (
+      raise SyntaxError('%s/%s does not define class %s' % (
-                         __name__, py, clsn)
+                         __name__, py, clsn))
 
     name = name.replace('_', '-')
     cmd.NAME = name

subcmds/abandon.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import sys
 from command import Command
 from git_command import git
@@ -36,7 +37,7 @@ It is equivalent to "git branch -D <branchname>".
 
     nb = args[0]
     if not git.check_ref_format('heads/%s' % nb):
-      print >>sys.stderr, "error: '%s' is not a valid name" % nb
+      print("error: '%s' is not a valid name" % nb, file=sys.stderr)
       sys.exit(1)
 
     nb = args[0]
@@ -58,13 +59,13 @@ It is equivalent to "git branch -D <branchname>".
 
     if err:
       for p in err:
-        print >>sys.stderr,\
+        print("error: %s/: cannot abandon %s" % (p.relpath, nb),
-          "error: %s/: cannot abandon %s" \
+              file=sys.stderr)
-          % (p.relpath, nb)
       sys.exit(1)
     elif not success:
-      print >>sys.stderr, 'error: no project has branch %s' % nb
+      print('error: no project has branch %s' % nb, file=sys.stderr)
       sys.exit(1)
     else:
-      print >>sys.stderr, 'Abandoned in %d project(s):\n  %s' % (
+      print('Abandoned in %d project(s):\n  %s'
-            len(success), '\n  '.join(p.relpath for p in success))
+            % (len(success), '\n  '.join(p.relpath for p in success)),
+            file=sys.stderr)

subcmds/branches.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import sys
 from color import Coloring
 from command import Command
@@ -97,17 +98,16 @@ is shown, then the branch appears in all projects.
     project_cnt = len(projects)
 
     for project in projects:
-      for name, b in project.GetBranches().iteritems():
+      for name, b in project.GetBranches().items():
         b.project = project
         if name not in all_branches:
           all_branches[name] = BranchInfo(name)
         all_branches[name].add(b)
 
-    names = all_branches.keys()
+    names = list(sorted(all_branches))
-    names.sort()
 
     if not names:
-      print >>sys.stderr, '   (no branches)'
+      print('   (no branches)', file=sys.stderr)
       return
 
     width = 25
@@ -139,7 +139,7 @@ is shown, then the branch appears in all projects.
       if in_cnt < project_cnt:
         fmt = out.write
         paths = []
-        if in_cnt < project_cnt - in_cnt: 
+        if in_cnt < project_cnt - in_cnt:
           in_type = 'in'
           for b in i.projects:
             paths.append(b.project.relpath)

subcmds/checkout.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import sys
 from command import Command
 from progress import Progress
@@ -55,10 +56,9 @@ The command is equivalent to:
 
     if err:
       for p in err:
-        print >>sys.stderr,\
+        print("error: %s/: cannot checkout %s" % (p.relpath, nb),
-          "error: %s/: cannot checkout %s" \
+              file=sys.stderr)
-          % (p.relpath, nb)
       sys.exit(1)
     elif not success:
-      print >>sys.stderr, 'error: no project has branch %s' % nb
+      print('error: no project has branch %s' % nb, file=sys.stderr)
       sys.exit(1)

subcmds/cherry_pick.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import re
 import sys
 from command import Command
@@ -46,13 +47,13 @@ change id will be added.
                    capture_stdout = True,
                    capture_stderr = True)
     if p.Wait() != 0:
-      print >>sys.stderr, p.stderr
+      print(p.stderr, file=sys.stderr)
       sys.exit(1)
     sha1 = p.stdout.strip()
 
     p = GitCommand(None, ['cat-file', 'commit', sha1], capture_stdout=True)
     if p.Wait() != 0:
-      print >>sys.stderr, "error: Failed to retrieve old commit message"
+      print("error: Failed to retrieve old commit message", file=sys.stderr)
       sys.exit(1)
     old_msg = self._StripHeader(p.stdout)
 
@@ -62,8 +63,8 @@ change id will be added.
                    capture_stderr = True)
     status = p.Wait()
 
-    print >>sys.stdout, p.stdout
+    print(p.stdout, file=sys.stdout)
-    print >>sys.stderr, p.stderr
+    print(p.stderr, file=sys.stderr)
 
     if status == 0:
       # The cherry-pick was applied correctly. We just need to edit the
@@ -76,16 +77,14 @@ change id will be added.
                      capture_stderr = True)
       p.stdin.write(new_msg)
       if p.Wait() != 0:
-        print >>sys.stderr, "error: Failed to update commit message"
+        print("error: Failed to update commit message", file=sys.stderr)
         sys.exit(1)
 
     else:
-      print >>sys.stderr, """\
+      print('NOTE: When committing (please see above) and editing the commit '
-NOTE: When committing (please see above) and editing the commit message,
+            'message, please remove the old Change-Id-line and add:')
-please remove the old Change-Id-line and add:
+      print(self._GetReference(sha1), file=sys.stderr)
-"""
+      print(file=sys.stderr)
-      print >>sys.stderr, self._GetReference(sha1)
-      print >>sys.stderr
 
   def _IsChangeId(self, line):
     return CHANGE_ID_RE.match(line)

subcmds/diffmanifests.py

@@ -0,0 +1,195 @@
+#
+# Copyright (C) 2014 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from color import Coloring
+from command import PagedCommand
+from manifest_xml import XmlManifest
+
+class _Coloring(Coloring):
+  def __init__(self, config):
+    Coloring.__init__(self, config, "status")
+
+class Diffmanifests(PagedCommand):
+  """ A command to see logs in projects represented by manifests
+
+  This is used to see deeper differences between manifests. Where a simple
+  diff would only show a diff of sha1s for example, this command will display
+  the logs of the project between both sha1s, allowing user to see diff at a
+  deeper level.
+  """
+
+  common = True
+  helpSummary = "Manifest diff utility"
+  helpUsage = """%prog manifest1.xml [manifest2.xml] [options]"""
+
+  helpDescription = """
+The %prog command shows differences between project revisions of manifest1 and
+manifest2. if manifest2 is not specified, current manifest.xml will be used
+instead. Both absolute and relative paths may be used for manifests. Relative
+paths start from project's ".repo/manifests" folder.
+
+The --raw option Displays the diff in a way that facilitates parsing, the
+project pattern will be <status> <path> <revision from> [<revision to>] and the
+commit pattern will be <status> <onelined log> with status values respectively :
+
+  A = Added project
+  R = Removed project
+  C = Changed project
+  U = Project with unreachable revision(s) (revision(s) not found)
+
+for project, and
+
+   A = Added commit
+   R = Removed commit
+
+for a commit.
+
+Only changed projects may contain commits, and commit status always starts with
+a space, and are part of last printed project.
+Unreachable revisions may occur if project is not up to date or if repo has not
+been initialized with all the groups, in which case some projects won't be
+synced and their revisions won't be found.
+
+"""
+
+  def _Options(self, p):
+    p.add_option('--raw',
+                 dest='raw', action='store_true',
+                 help='Display raw diff.')
+    p.add_option('--no-color',
+                 dest='color', action='store_false', default=True,
+                 help='does not display the diff in color.')
+
+  def _printRawDiff(self, diff):
+    for project in diff['added']:
+      self.printText("A %s %s" % (project.relpath, project.revisionExpr))
+      self.out.nl()
+
+    for project in diff['removed']:
+      self.printText("R %s %s" % (project.relpath, project.revisionExpr))
+      self.out.nl()
+
+    for project, otherProject in diff['changed']:
+      self.printText("C %s %s %s" % (project.relpath, project.revisionExpr,
+                                     otherProject.revisionExpr))
+      self.out.nl()
+      self._printLogs(project, otherProject, raw=True, color=False)
+
+    for project, otherProject in diff['unreachable']:
+      self.printText("U %s %s %s" % (project.relpath, project.revisionExpr,
+                                     otherProject.revisionExpr))
+      self.out.nl()
+
+  def _printDiff(self, diff, color=True):
+    if diff['added']:
+      self.out.nl()
+      self.printText('added projects : \n')
+      self.out.nl()
+      for project in diff['added']:
+        self.printProject('\t%s' % (project.relpath))
+        self.printText(' at revision ')
+        self.printRevision(project.revisionExpr)
+        self.out.nl()
+
+    if diff['removed']:
+      self.out.nl()
+      self.printText('removed projects : \n')
+      self.out.nl()
+      for project in diff['removed']:
+        self.printProject('\t%s' % (project.relpath))
+        self.printText(' at revision ')
+        self.printRevision(project.revisionExpr)
+        self.out.nl()
+
+    if diff['changed']:
+      self.out.nl()
+      self.printText('changed projects : \n')
+      self.out.nl()
+      for project, otherProject in diff['changed']:
+        self.printProject('\t%s' % (project.relpath))
+        self.printText(' changed from ')
+        self.printRevision(project.revisionExpr)
+        self.printText(' to ')
+        self.printRevision(otherProject.revisionExpr)
+        self.out.nl()
+        self._printLogs(project, otherProject, raw=False, color=color)
+        self.out.nl()
+
+    if diff['unreachable']:
+      self.out.nl()
+      self.printText('projects with unreachable revisions : \n')
+      self.out.nl()
+      for project, otherProject in diff['unreachable']:
+        self.printProject('\t%s ' % (project.relpath))
+        self.printRevision(project.revisionExpr)
+        self.printText(' or ')
+        self.printRevision(otherProject.revisionExpr)
+        self.printText(' not found')
+        self.out.nl()
+
+  def _printLogs(self, project, otherProject, raw=False, color=True):
+    logs = project.getAddedAndRemovedLogs(otherProject, oneline=True,
+                                          color=color)
+    if logs['removed']:
+      removedLogs = logs['removed'].split('\n')
+      for log in removedLogs:
+        if log.strip():
+          if raw:
+            self.printText(' R ' + log)
+            self.out.nl()
+          else:
+            self.printRemoved('\t\t[-] ')
+            self.printText(log)
+            self.out.nl()
+
+    if logs['added']:
+      addedLogs = logs['added'].split('\n')
+      for log in addedLogs:
+        if log.strip():
+          if raw:
+            self.printText(' A ' + log)
+            self.out.nl()
+          else:
+            self.printAdded('\t\t[+] ')
+            self.printText(log)
+            self.out.nl()
+
+  def Execute(self, opt, args):
+    if not args or len(args) > 2:
+      self.Usage()
+
+    self.out = _Coloring(self.manifest.globalConfig)
+    self.printText = self.out.nofmt_printer('text')
+    if opt.color:
+      self.printProject = self.out.nofmt_printer('project', attr = 'bold')
+      self.printAdded = self.out.nofmt_printer('green', fg = 'green', attr = 'bold')
+      self.printRemoved = self.out.nofmt_printer('red', fg = 'red', attr = 'bold')
+      self.printRevision = self.out.nofmt_printer('revision', fg = 'yellow')
+    else:
+      self.printProject = self.printAdded = self.printRemoved = self.printRevision = self.printText
+
+    manifest1 = XmlManifest(self.manifest.repodir)
+    manifest1.Override(args[0])
+    if len(args) == 1:
+      manifest2 = self.manifest
+    else:
+      manifest2 = XmlManifest(self.manifest.repodir)
+      manifest2.Override(args[1])
+
+    diff = manifest1.projectsDiff(manifest2)
+    if opt.raw:
+      self._printRawDiff(diff)
+    else:
+      self._printDiff(diff, color=opt.color)

subcmds/download.py

@@ -13,10 +13,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import re
 import sys
 
 from command import Command
+from error import GitError
 
 CHANGE_RE = re.compile(r'^([1-9][0-9]*)(?:[/\.-]([1-9][0-9]*))?$')
 
@@ -32,13 +34,13 @@ makes it available in your project's local working directory.
 """
 
   def _Options(self, p):
-    p.add_option('-c','--cherry-pick',
+    p.add_option('-c', '--cherry-pick',
                  dest='cherrypick', action='store_true',
                  help="cherry-pick instead of checkout")
-    p.add_option('-r','--revert',
+    p.add_option('-r', '--revert',
                  dest='revert', action='store_true',
                  help="revert instead of checkout")
-    p.add_option('-f','--ff-only',
+    p.add_option('-f', '--ff-only',
                  dest='ffonly', action='store_true',
                  help="force fast-forward merge")
 
@@ -68,25 +70,30 @@ makes it available in your project's local working directory.
     for project, change_id, ps_id in self._ParseChangeIds(args):
       dl = project.DownloadPatchSet(change_id, ps_id)
       if not dl:
-        print >>sys.stderr, \
+        print('[%s] change %d/%d not found'
-          '[%s] change %d/%d not found' \
+              % (project.name, change_id, ps_id),
-          % (project.name, change_id, ps_id)
+              file=sys.stderr)
         sys.exit(1)
 
       if not opt.revert and not dl.commits:
-        print >>sys.stderr, \
+        print('[%s] change %d/%d has already been merged'
-          '[%s] change %d/%d has already been merged' \
+              % (project.name, change_id, ps_id),
-          % (project.name, change_id, ps_id)
+              file=sys.stderr)
         continue
 
       if len(dl.commits) > 1:
-        print >>sys.stderr, \
+        print('[%s] %d/%d depends on %d unmerged changes:' \
-          '[%s] %d/%d depends on %d unmerged changes:' \
+              % (project.name, change_id, ps_id, len(dl.commits)),
-          % (project.name, change_id, ps_id, len(dl.commits))
+              file=sys.stderr)
         for c in dl.commits:
-          print >>sys.stderr, '  %s' % (c)
+          print('  %s' % (c), file=sys.stderr)
       if opt.cherrypick:
-        project._CherryPick(dl.commit)
+        try:
+          project._CherryPick(dl.commit)
+        except GitError:
+          print('[%s] Could not complete the cherry-pick of %s' \
+                % (project.name, dl.commit), file=sys.stderr)
+
       elif opt.revert:
         project._Revert(dl.commit)
       elif opt.ffonly:

subcmds/forall.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import fcntl
 import re
 import os
@@ -41,10 +42,14 @@ class Forall(Command, MirrorSafeCommand):
   helpSummary = "Run a shell command in each project"
   helpUsage = """
 %prog [<project>...] -c <command> [<arg>...]
+%prog -r str1 [str2] ... -c <command> [<arg>...]"
 """
   helpDescription = """
 Executes the same shell command in each project.
 
+The -r option allows running the command only on projects matching
+regex or wildcard expression.
+
 Output Formatting
 -----------------
 
@@ -82,6 +87,12 @@ revision to a locally executed git command, use REPO_LREV.
 REPO_RREV is the name of the revision from the manifest, exactly
 as written in the manifest.
 
+REPO_COUNT is the total number of projects being iterated.
+
+REPO_I is the current (1-based) iteration count. Can be used in
+conjunction with REPO_COUNT to add a simple progress indicator to your
+command.
+
 REPO__* are any extra environment variables, specified by the
 "annotation" element under any project element.  This can be useful
 for differentiating trees based on user-specific criteria, or simply
@@ -92,6 +103,9 @@ following <command>.
 
 Unless -p is used, stdin, stdout, stderr are inherited from the
 terminal and are not redirected.
+
+If -e is used, when a command exits unsuccessfully, '%prog' will abort
+without iterating through the remaining projects.
 """
 
   def _Options(self, p):
@@ -99,11 +113,17 @@ terminal and are not redirected.
       setattr(parser.values, option.dest, list(parser.rargs))
       while parser.rargs:
         del parser.rargs[0]
+    p.add_option('-r', '--regex',
+                 dest='regex', action='store_true',
+                 help="Execute the command only on projects matching regex or wildcard expression")
     p.add_option('-c', '--command',
                  help='Command (and arguments) to execute',
                  dest='command',
                  action='callback',
                  callback=cmd)
+    p.add_option('-e', '--abort-on-errors',
+                 dest='abort_on_errors', action='store_true',
+                 help='Abort if a command exits unsuccessfully')
 
     g = p.add_option_group('Output')
     g.add_option('-p',
@@ -159,7 +179,14 @@ terminal and are not redirected.
     rc = 0
     first = True
 
-    for project in self.GetProjects(args):
+    if not opt.regex:
+      projects = self.GetProjects(args)
+    else:
+      projects = self.FindProjects(args)
+
+    os.environ['REPO_COUNT'] = str(len(projects))
+
+    for (cnt, project) in enumerate(projects):
       env = os.environ.copy()
       def setenv(name, val):
         if val is None:
@@ -171,6 +198,7 @@ terminal and are not redirected.
       setenv('REPO_REMOTE', project.remote.name)
       setenv('REPO_LREV', project.GetRevisionId())
       setenv('REPO_RREV', project.revisionExpr)
+      setenv('REPO_I', str(cnt + 1))
       for a in project.annotations:
         setenv("REPO__%s" % (a.name), a.value)
 
@@ -183,7 +211,7 @@ terminal and are not redirected.
       if not os.path.exists(cwd):
         if (opt.project_header and opt.verbose) \
         or not opt.project_header:
-          print >>sys.stderr, 'skipping %s/' % project.relpath
+          print('skipping %s/' % project.relpath, file=sys.stderr)
         continue
 
       if opt.project_header:
@@ -241,7 +269,12 @@ terminal and are not redirected.
                 first = False
               else:
                 out.nl()
-              out.project('project %s/', project.relpath)
+
+              if mirror:
+                project_header_path = project.name
+              else:
+                project_header_path = project.relpath
+              out.project('project %s/', project_header_path)
               out.nl()
               out.flush()
               if errbuf:
@@ -254,7 +287,12 @@ terminal and are not redirected.
             s.dest.flush()
 
       r = p.wait()
-      if r != 0 and r != rc:
+      if r != 0:
-        rc = r
+        if r != rc:
+          rc = r
+        if opt.abort_on_errors:
+          print("error: %s: Aborting due to previous error" % project.relpath,
+                file=sys.stderr)
+          sys.exit(r)
     if rc != 0:
       sys.exit(rc)

subcmds/grep.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import sys
 from color import Coloring
 from command import PagedCommand
@@ -51,7 +52,7 @@ Examples
 
 Look for a line that has '#define' and either 'MAX_PATH or 'PATH_MAX':
 
-  repo grep -e '#define' --and -\( -e MAX_PATH -e PATH_MAX \)
+  repo grep -e '#define' --and -\\( -e MAX_PATH -e PATH_MAX \\)
 
 Look for a line that has 'NODE' or 'Unexpected' in files that
 contain a line that matches both expressions:
@@ -84,7 +85,7 @@ contain a line that matches both expressions:
     g.add_option('--cached',
                  action='callback', callback=carry,
                  help='Search the index, instead of the work tree')
-    g.add_option('-r','--revision',
+    g.add_option('-r', '--revision',
                  dest='revision', action='append', metavar='TREEish',
                  help='Search TREEish, instead of the work tree')
 
@@ -96,7 +97,7 @@ contain a line that matches both expressions:
     g.add_option('-i', '--ignore-case',
                  action='callback', callback=carry,
                  help='Ignore case differences')
-    g.add_option('-a','--text',
+    g.add_option('-a', '--text',
                  action='callback', callback=carry,
                  help="Process binary files as if they were text")
     g.add_option('-I',
@@ -125,7 +126,7 @@ contain a line that matches both expressions:
     g.add_option('--and', '--or', '--not',
                  action='callback', callback=carry,
                  help='Boolean operators to combine patterns')
-    g.add_option('-(','-)',
+    g.add_option('-(', '-)',
                  action='callback', callback=carry,
                  help='Boolean operator grouping')
 
@@ -145,10 +146,10 @@ contain a line that matches both expressions:
                  action='callback', callback=carry,
                  metavar='CONTEXT', type='str',
                  help='Show CONTEXT lines after match')
-    g.add_option('-l','--name-only','--files-with-matches',
+    g.add_option('-l', '--name-only', '--files-with-matches',
                  action='callback', callback=carry,
                  help='Show only file names containing matching lines')
-    g.add_option('-L','--files-without-match',
+    g.add_option('-L', '--files-without-match',
                  action='callback', callback=carry,
                  help='Show only file names not containing matching lines')
 
@@ -157,9 +158,9 @@ contain a line that matches both expressions:
     out = GrepColoring(self.manifest.manifestProject.config)
 
     cmd_argv = ['grep']
-    if out.is_on and git_require((1,6,3)):
+    if out.is_on and git_require((1, 6, 3)):
       cmd_argv.append('--color')
-    cmd_argv.extend(getattr(opt,'cmd_argv',[]))
+    cmd_argv.extend(getattr(opt, 'cmd_argv', []))
 
     if '-e' not in cmd_argv:
       if not args:
@@ -178,8 +179,7 @@ contain a line that matches both expressions:
     have_rev = False
     if opt.revision:
       if '--cached' in cmd_argv:
-        print >>sys.stderr,\
+        print('fatal: cannot combine --cached and --revision', file=sys.stderr)
-          'fatal: cannot combine --cached and --revision'
         sys.exit(1)
       have_rev = True
       cmd_argv.extend(opt.revision)
@@ -230,13 +230,13 @@ contain a line that matches both expressions:
           out.nl()
       else:
         for line in r:
-          print line
+          print(line)
 
     if have_match:
       sys.exit(0)
     elif have_rev and bad_rev:
       for r in opt.revision:
-        print >>sys.stderr, "error: can't search revision %s" % r
+        print("error: can't search revision %s" % r, file=sys.stderr)
       sys.exit(1)
     else:
       sys.exit(1)

subcmds/help.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import re
 import sys
 from formatter import AbstractFormatter, DumbWriter
@@ -31,12 +32,9 @@ Displays detailed usage information about a command.
 """
 
   def _PrintAllCommands(self):
-    print 'usage: repo COMMAND [ARGS]'
+    print('usage: repo COMMAND [ARGS]')
-    print """
+    print('The complete list of recognized repo commands are:')
-The complete list of recognized repo commands are:
+    commandNames = list(sorted(self.commands))
-"""
-    commandNames = self.commands.keys()
-    commandNames.sort()
 
     maxlen = 0
     for name in commandNames:
@@ -49,20 +47,16 @@ The complete list of recognized repo commands are:
         summary = command.helpSummary.strip()
       except AttributeError:
         summary = ''
-      print fmt % (name, summary)
+      print(fmt % (name, summary))
-    print """
+    print("See 'repo help <command>' for more information on a "
-See 'repo help <command>' for more information on a specific command.
+          'specific command.')
-"""
 
   def _PrintCommonCommands(self):
-    print 'usage: repo COMMAND [ARGS]'
+    print('usage: repo COMMAND [ARGS]')
-    print """
+    print('The most commonly used repo commands are:')
-The most commonly used repo commands are:
+    commandNames = list(sorted([name
-"""
+                    for name, command in self.commands.items()
-    commandNames = [name 
+                    if command.common]))
-                    for name in self.commands.keys()
-                    if self.commands[name].common]
-    commandNames.sort()
 
     maxlen = 0
     for name in commandNames:
@@ -75,11 +69,10 @@ The most commonly used repo commands are:
         summary = command.helpSummary.strip()
       except AttributeError:
         summary = ''
-      print fmt % (name, summary)
+      print(fmt % (name, summary))
-    print """
+    print(
-See 'repo help <command>' for more information on a specific command.
+"See 'repo help <command>' for more information on a specific command.\n"
-See 'repo help --all' for a complete list of recognized commands.
+"See 'repo help --all' for a complete list of recognized commands.")
-"""
 
   def _PrintCommandHelp(self, cmd):
     class _Out(Coloring):
@@ -131,7 +124,7 @@ See 'repo help --all' for a complete list of recognized commands.
 
             p('%s', title)
             self.nl()
-            p('%s', ''.ljust(len(title),section_type[0]))
+            p('%s', ''.ljust(len(title), section_type[0]))
             self.nl()
             continue
 
@@ -162,7 +155,7 @@ See 'repo help --all' for a complete list of recognized commands.
       try:
         cmd = self.commands[name]
       except KeyError:
-        print >>sys.stderr, "repo: '%s' is not a repo command." % name
+        print("repo: '%s' is not a repo command." % name, file=sys.stderr)
         sys.exit(1)
 
       cmd.manifest = self.manifest

subcmds/info.py

@@ -0,0 +1,201 @@
+#
+# Copyright (C) 2012 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from command import PagedCommand
+from color import Coloring
+from error import NoSuchProjectError
+from git_refs import R_M
+
+class _Coloring(Coloring):
+  def __init__(self, config):
+    Coloring.__init__(self, config, "status")
+
+class Info(PagedCommand):
+  common = True
+  helpSummary = "Get info on the manifest branch, current branch or unmerged branches"
+  helpUsage = "%prog [-dl] [-o [-b]] [<project>...]"
+
+  def _Options(self, p):
+    p.add_option('-d', '--diff',
+                 dest='all', action='store_true',
+                 help="show full info and commit diff including remote branches")
+    p.add_option('-o', '--overview',
+                 dest='overview', action='store_true',
+                 help='show overview of all local commits')
+    p.add_option('-b', '--current-branch',
+                 dest="current_branch", action="store_true",
+                 help="consider only checked out branches")
+    p.add_option('-l', '--local-only',
+                 dest="local", action="store_true",
+                 help="Disable all remote operations")
+
+
+  def Execute(self, opt, args):
+    self.out = _Coloring(self.manifest.globalConfig)
+    self.heading = self.out.printer('heading', attr = 'bold')
+    self.headtext = self.out.printer('headtext', fg = 'yellow')
+    self.redtext = self.out.printer('redtext', fg = 'red')
+    self.sha = self.out.printer("sha", fg = 'yellow')
+    self.text = self.out.nofmt_printer('text')
+    self.dimtext = self.out.printer('dimtext', attr = 'dim')
+
+    self.opt = opt
+
+    manifestConfig = self.manifest.manifestProject.config
+    mergeBranch = manifestConfig.GetBranch("default").merge
+    manifestGroups = (manifestConfig.GetString('manifest.groups')
+                      or 'all,-notdefault')
+
+    self.heading("Manifest branch: ")
+    self.headtext(self.manifest.default.revisionExpr)
+    self.out.nl()
+    self.heading("Manifest merge branch: ")
+    self.headtext(mergeBranch)
+    self.out.nl()
+    self.heading("Manifest groups: ")
+    self.headtext(manifestGroups)
+    self.out.nl()
+
+    self.printSeparator()
+
+    if not opt.overview:
+      self.printDiffInfo(args)
+    else:
+      self.printCommitOverview(args)
+
+  def printSeparator(self):
+    self.text("----------------------------")
+    self.out.nl()
+
+  def printDiffInfo(self, args):
+    try:
+      projs = self.GetProjects(args)
+    except NoSuchProjectError:
+      return
+
+    for p in projs:
+      self.heading("Project: ")
+      self.headtext(p.name)
+      self.out.nl()
+
+      self.heading("Mount path: ")
+      self.headtext(p.worktree)
+      self.out.nl()
+
+      self.heading("Current revision: ")
+      self.headtext(p.revisionExpr)
+      self.out.nl()
+
+      localBranches = p.GetBranches().keys()
+      self.heading("Local Branches: ")
+      self.redtext(str(len(localBranches)))
+      if len(localBranches) > 0:
+        self.text(" [")
+        self.text(", ".join(localBranches))
+        self.text("]")
+      self.out.nl()
+
+      if self.opt.all:
+        self.findRemoteLocalDiff(p)
+
+      self.printSeparator()
+
+  def findRemoteLocalDiff(self, project):
+    #Fetch all the latest commits
+    if not self.opt.local:
+      project.Sync_NetworkHalf(quiet=True, current_branch_only=True)
+
+    logTarget = R_M + self.manifest.manifestProject.config.GetBranch("default").merge
+
+    bareTmp = project.bare_git._bare
+    project.bare_git._bare = False
+    localCommits = project.bare_git.rev_list(
+        '--abbrev=8',
+        '--abbrev-commit',
+        '--pretty=oneline',
+        logTarget + "..",
+        '--')
+
+    originCommits = project.bare_git.rev_list(
+        '--abbrev=8',
+        '--abbrev-commit',
+        '--pretty=oneline',
+        ".." + logTarget,
+        '--')
+    project.bare_git._bare = bareTmp
+
+    self.heading("Local Commits: ")
+    self.redtext(str(len(localCommits)))
+    self.dimtext(" (on current branch)")
+    self.out.nl()
+
+    for c in localCommits:
+      split = c.split()
+      self.sha(split[0] + " ")
+      self.text(" ".join(split[1:]))
+      self.out.nl()
+
+    self.printSeparator()
+
+    self.heading("Remote Commits: ")
+    self.redtext(str(len(originCommits)))
+    self.out.nl()
+
+    for c in originCommits:
+      split = c.split()
+      self.sha(split[0] + " ")
+      self.text(" ".join(split[1:]))
+      self.out.nl()
+
+  def printCommitOverview(self, args):
+    all_branches = []
+    for project in self.GetProjects(args):
+      br = [project.GetUploadableBranch(x)
+            for x in project.GetBranches()]
+      br = [x for x in br if x]
+      if self.opt.current_branch:
+        br = [x for x in br if x.name == project.CurrentBranch]
+      all_branches.extend(br)
+
+    if not all_branches:
+      return
+
+    self.out.nl()
+    self.heading('Projects Overview')
+    project = None
+
+    for branch in all_branches:
+      if project != branch.project:
+        project = branch.project
+        self.out.nl()
+        self.headtext(project.relpath)
+        self.out.nl()
+
+      commits = branch.commits
+      date = branch.date
+      self.text('%s %-33s (%2d commit%s, %s)' % (
+        branch.name == project.CurrentBranch and '*' or ' ',
+        branch.name,
+        len(commits),
+        len(commits) != 1 and 's' or '',
+        date))
+      self.out.nl()
+
+      for commit in commits:
+        split = commit.split()
+        self.text('{0:38}{1} '.format('','-'))
+        self.sha(split[0] + " ")
+        self.text(" ".join(split[1:]))
+        self.out.nl()

subcmds/init.py

@@ -13,12 +13,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import os
 import platform
 import re
 import shutil
 import sys
 
+from pyversion import is_python3
+if is_python3():
+  import urllib.parse
+else:
+  import imp
+  import urlparse
+  urllib = imp.new_module('urllib')
+  urllib.parse = urlparse.urlparse
+
 from color import Coloring
 from command import InteractiveCommand, MirrorSafeCommand
 from error import ManifestParseError
@@ -89,9 +99,14 @@ to update the working directory files.
     g.add_option('--depth', type='int', default=None,
                  dest='depth',
                  help='create a shallow clone with given depth; see git clone')
+    g.add_option('--archive',
+                 dest='archive', action='store_true',
+                 help='checkout an archive instead of a git repository for '
+                      'each project. See git archive.')
     g.add_option('-g', '--groups',
-                 dest='groups', default='all,-notdefault',
+                 dest='groups', default='default',
-                 help='restrict manifest projects to ones with a specified group',
+                 help='restrict manifest projects to ones with specified '
+                      'group(s) [default|all|G1,G2,G3|G4,-G5,-G6]',
                  metavar='GROUP')
     g.add_option('-p', '--platform',
                  dest='platform', default='auto',
@@ -117,19 +132,35 @@ to update the working directory files.
                  dest='config_name', action="store_true", default=False,
                  help='Always prompt for name/e-mail')
 
+  def _RegisteredEnvironmentOptions(self):
+    return {'REPO_MANIFEST_URL': 'manifest_url',
+            'REPO_MIRROR_LOCATION': 'reference'}
+
   def _SyncManifest(self, opt):
     m = self.manifest.manifestProject
     is_new = not m.Exists
 
     if is_new:
       if not opt.manifest_url:
-        print >>sys.stderr, 'fatal: manifest url (-u) is required.'
+        print('fatal: manifest url (-u) is required.', file=sys.stderr)
         sys.exit(1)
 
       if not opt.quiet:
-        print >>sys.stderr, 'Get %s' \
+        print('Get %s' % GitConfig.ForUser().UrlInsteadOf(opt.manifest_url),
-          % GitConfig.ForUser().UrlInsteadOf(opt.manifest_url)
+              file=sys.stderr)
-      m._InitGitDir()
+
+      # The manifest project object doesn't keep track of the path on the
+      # server where this git is located, so let's save that here.
+      mirrored_manifest_git = None
+      if opt.reference:
+        manifest_git_path = urllib.parse(opt.manifest_url).path[1:]
+        mirrored_manifest_git = os.path.join(opt.reference, manifest_git_path)
+        if not mirrored_manifest_git.endswith(".git"):
+          mirrored_manifest_git += ".git"
+        if not os.path.exists(mirrored_manifest_git):
+          mirrored_manifest_git = os.path.join(opt.reference + '/.repo/manifests.git')
+
+      m._InitGitDir(mirror_git=mirrored_manifest_git)
 
       if opt.manifest_branch:
         m.revisionExpr = opt.manifest_branch
@@ -147,7 +178,7 @@ to update the working directory files.
       r.ResetFetch()
       r.Save()
 
-    groups = re.split('[,\s]+', opt.groups)
+    groups = re.split(r'[,\s]+', opt.groups)
     all_platforms = ['linux', 'darwin']
     platformize = lambda x: 'platform-' + x
     if opt.platform == 'auto':
@@ -159,28 +190,41 @@ to update the working directory files.
     elif opt.platform in all_platforms:
       groups.extend(platformize(opt.platform))
     elif opt.platform != 'none':
-      print >>sys.stderr, 'fatal: invalid platform flag'
+      print('fatal: invalid platform flag', file=sys.stderr)
       sys.exit(1)
 
     groups = [x for x in groups if x]
     groupstr = ','.join(groups)
-    if opt.platform == 'auto' and groupstr == 'all,-notdefault,platform-' + platform.system().lower():
+    if opt.platform == 'auto' and groupstr == 'default,platform-' + platform.system().lower():
       groupstr = None
     m.config.SetString('manifest.groups', groupstr)
 
     if opt.reference:
       m.config.SetString('repo.reference', opt.reference)
 
+    if opt.archive:
+      if is_new:
+        m.config.SetString('repo.archive', 'true')
+      else:
+        print('fatal: --archive is only supported when initializing a new '
+              'workspace.', file=sys.stderr)
+        print('Either delete the .repo folder in this workspace, or initialize '
+              'in another location.', file=sys.stderr)
+        sys.exit(1)
+
     if opt.mirror:
       if is_new:
         m.config.SetString('repo.mirror', 'true')
       else:
-        print >>sys.stderr, 'fatal: --mirror not supported on existing client'
+        print('fatal: --mirror is only supported when initializing a new '
+              'workspace.', file=sys.stderr)
+        print('Either delete the .repo folder in this workspace, or initialize '
+              'in another location.', file=sys.stderr)
         sys.exit(1)
 
     if not m.Sync_NetworkHalf(is_new=is_new):
       r = m.GetRemote(m.remote.name)
-      print >>sys.stderr, 'fatal: cannot obtain manifest %s' % r.url
+      print('fatal: cannot obtain manifest %s' % r.url, file=sys.stderr)
 
       # Better delete the manifest git dir if we created it; otherwise next
       # time (when user fixes problems) we won't go through the "is_new" logic.
@@ -197,19 +241,19 @@ to update the working directory files.
 
     if is_new or m.CurrentBranch is None:
       if not m.StartBranch('default'):
-        print >>sys.stderr, 'fatal: cannot create default in manifest'
+        print('fatal: cannot create default in manifest', file=sys.stderr)
         sys.exit(1)
 
   def _LinkManifest(self, name):
     if not name:
-      print >>sys.stderr, 'fatal: manifest name (-m) is required.'
+      print('fatal: manifest name (-m) is required.', file=sys.stderr)
       sys.exit(1)
 
     try:
       self.manifest.Link(name)
     except ManifestParseError as e:
-      print >>sys.stderr, "fatal: manifest '%s' not available" % name
+      print("fatal: manifest '%s' not available" % name, file=sys.stderr)
-      print >>sys.stderr, 'fatal: %s' % str(e)
+      print('fatal: %s' % str(e), file=sys.stderr)
       sys.exit(1)
 
   def _Prompt(self, prompt, value):
@@ -231,24 +275,24 @@ to update the working directory files.
       mp.config.SetString('user.name', gc.GetString('user.name'))
       mp.config.SetString('user.email', gc.GetString('user.email'))
 
-    print ''
+    print()
-    print 'Your identity is: %s <%s>' % (mp.config.GetString('user.name'),
+    print('Your identity is: %s <%s>' % (mp.config.GetString('user.name'),
-                                         mp.config.GetString('user.email'))
+                                         mp.config.GetString('user.email')))
-    print 'If you want to change this, please re-run \'repo init\' with --config-name'
+    print('If you want to change this, please re-run \'repo init\' with --config-name')
     return False
 
   def _ConfigureUser(self):
     mp = self.manifest.manifestProject
 
     while True:
-      print ''
+      print()
       name  = self._Prompt('Your Name', mp.UserName)
       email = self._Prompt('Your Email', mp.UserEmail)
 
-      print ''
+      print()
-      print 'Your identity is: %s <%s>' % (name, email)
+      print('Your identity is: %s <%s>' % (name, email))
       sys.stdout.write('is this correct [y/N]? ')
-      a = sys.stdin.readline().strip()
+      a = sys.stdin.readline().strip().lower()
       if a in ('yes', 'y', 't', 'true'):
         break
 
@@ -274,17 +318,17 @@ to update the working directory files.
         self._on = True
     out = _Test()
 
-    print ''
+    print()
-    print "Testing colorized output (for 'repo diff', 'repo status'):"
+    print("Testing colorized output (for 'repo diff', 'repo status'):")
 
-    for c in ['black','red','green','yellow','blue','magenta','cyan']:
+    for c in ['black', 'red', 'green', 'yellow', 'blue', 'magenta', 'cyan']:
       out.write(' ')
       out.printer(fg=c)(' %-6s ', c)
     out.write(' ')
     out.printer(fg='white', bg='black')(' %s ' % 'white')
     out.nl()
 
-    for c in ['bold','dim','ul','reverse']:
+    for c in ['bold', 'dim', 'ul', 'reverse']:
       out.write(' ')
       out.printer(fg='black', attr=c)(' %-6s ', c)
     out.nl()
@@ -313,12 +357,36 @@ to update the working directory files.
       # We store the depth in the main manifest project.
       self.manifest.manifestProject.config.SetString('repo.depth', depth)
 
+  def _DisplayResult(self):
+    if self.manifest.IsMirror:
+      init_type = 'mirror '
+    else:
+      init_type = ''
+
+    print()
+    print('repo %shas been initialized in %s'
+          % (init_type, self.manifest.topdir))
+
+    current_dir = os.getcwd()
+    if current_dir != self.manifest.topdir:
+      print('If this is not the directory in which you want to initialize '
+            'repo, please run:')
+      print('   rm -r %s/.repo' % self.manifest.topdir)
+      print('and try again.')
+
   def Execute(self, opt, args):
     git_require(MIN_GIT_VERSION, fail=True)
 
     if opt.reference:
       opt.reference = os.path.expanduser(opt.reference)
 
+    # Check this here, else manifest will be tagged "not new" and init won't be
+    # possible anymore without removing the .repo/manifests directory.
+    if opt.archive and opt.mirror:
+      print('fatal: --mirror and --archive cannot be used together.',
+            file=sys.stderr)
+      sys.exit(1)
+
     self._SyncManifest(opt)
     self._LinkManifest(opt.manifest_name)
 
@@ -329,10 +397,4 @@ to update the working directory files.
 
     self._ConfigureDepth(opt)
 
-    if self.manifest.IsMirror:
+    self._DisplayResult()
-      init_type = 'mirror '
-    else:
-      init_type = ''
-
-    print ''
-    print 'repo %sinitialized in %s' % (init_type, self.manifest.topdir)

subcmds/list.py

@@ -13,7 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import re
+from __future__ import print_function
+import sys
 
 from command import Command, MirrorSafeCommand
 
@@ -30,13 +31,19 @@ List all projects; pass '.' to list the project for the cwd.
 This is similar to running: repo forall -c 'echo "$REPO_PATH : $REPO_PROJECT"'.
 """
 
-  def _Options(self, p, show_smart=True):
+  def _Options(self, p):
     p.add_option('-r', '--regex',
                  dest='regex', action='store_true',
                  help="Filter the project list based on regex or wildcard matching of strings")
     p.add_option('-f', '--fullpath',
                  dest='fullpath', action='store_true',
                  help="Display the full work tree path instead of the relative path")
+    p.add_option('-n', '--name-only',
+                 dest='name_only', action='store_true',
+                 help="Display only the name of the repository")
+    p.add_option('-p', '--path-only',
+                 dest='path_only', action='store_true',
+                 help="Display only the path of the repository")
 
   def Execute(self, opt, args):
     """List all projects and the associated directories.
@@ -49,6 +56,11 @@ This is similar to running: repo forall -c 'echo "$REPO_PATH : $REPO_PROJECT"'.
       opt: The options.
       args: Positional args.  Can be a list of projects to list, or empty.
     """
+
+    if opt.fullpath and opt.name_only:
+      print('error: cannot combine -f and -n', file=sys.stderr)
+      sys.exit(1)
+
     if not opt.regex:
       projects = self.GetProjects(args)
     else:
@@ -61,18 +73,12 @@ This is similar to running: repo forall -c 'echo "$REPO_PATH : $REPO_PROJECT"'.
 
     lines = []
     for project in projects:
-      lines.append("%s : %s" % (_getpath(project), project.name))
+      if opt.name_only and not opt.path_only:
+        lines.append("%s" % ( project.name))
+      elif opt.path_only and not opt.name_only:
+        lines.append("%s" % (_getpath(project)))
+      else:
+        lines.append("%s : %s" % (_getpath(project), project.name))
 
     lines.sort()
-    print '\n'.join(lines)
+    print('\n'.join(lines))
-
-  def FindProjects(self, args):
-    result = []
-    for project in self.GetProjects(''):
-      for arg in args:
-        pattern = re.compile(r'%s' % arg, re.IGNORECASE)
-        if pattern.search(project.name) or pattern.search(project.relpath):
-          result.append(project)
-          break
-    result.sort(key=lambda project: project.relpath)
-    return result

subcmds/manifest.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import os
 import sys
 
@@ -69,7 +70,7 @@ in a Git repository for use during future 'repo init' invocations.
                        peg_rev_upstream = opt.peg_rev_upstream)
     fd.close()
     if opt.output_file != '-':
-      print >>sys.stderr, 'Saved manifest to %s' % opt.output_file
+      print('Saved manifest to %s' % opt.output_file, file=sys.stderr)
 
   def Execute(self, opt, args):
     if args:
@@ -79,6 +80,6 @@ in a Git repository for use during future 'repo init' invocations.
       self._Output(opt)
       return
 
-    print >>sys.stderr, 'error: no operation to perform'
+    print('error: no operation to perform', file=sys.stderr)
-    print >>sys.stderr, 'error: see repo help manifest'
+    print('error: see repo help manifest', file=sys.stderr)
     sys.exit(1)

subcmds/overview.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 from color import Coloring
 from command import PagedCommand
 
@@ -41,7 +42,7 @@ are displayed.
     all_branches = []
     for project in self.GetProjects(args):
       br = [project.GetUploadableBranch(x)
-            for x in project.GetBranches().keys()]
+            for x in project.GetBranches()]
       br = [x for x in br if x]
       if opt.current_branch:
         br = [x for x in br if x.name == project.CurrentBranch]
@@ -54,8 +55,11 @@ are displayed.
       def __init__(self, config):
         Coloring.__init__(self, config, 'status')
         self.project = self.printer('header', attr='bold')
+        self.text = self.printer('text')
 
     out = Report(all_branches[0].project.config)
+    out.text("Deprecated. See repo info -o.")
+    out.nl()
     out.project('Projects Overview')
     out.nl()
 
@@ -70,11 +74,11 @@ are displayed.
 
       commits = branch.commits
       date = branch.date
-      print '%s %-33s (%2d commit%s, %s)' % (
+      print('%s %-33s (%2d commit%s, %s)' % (
             branch.name == project.CurrentBranch and '*' or ' ',
             branch.name,
             len(commits),
             len(commits) != 1 and 's' or ' ',
-            date)
+            date))
       for commit in commits:
-        print '%-35s   - %s' % ('', commit)
+        print('%-35s   - %s' % ('', commit))

subcmds/prune.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 from color import Coloring
 from command import PagedCommand
 
@@ -51,9 +52,9 @@ class Prune(PagedCommand):
 
       commits = branch.commits
       date = branch.date
-      print '%s %-33s (%2d commit%s, %s)' % (
+      print('%s %-33s (%2d commit%s, %s)' % (
             branch.name == project.CurrentBranch and '*' or ' ',
             branch.name,
             len(commits),
             len(commits) != 1 and 's' or ' ',
-            date)
+            date))

subcmds/rebase.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import sys
 
 from command import Command
@@ -59,14 +60,19 @@ branch but need to incorporate new upstream changes "underneath" them.
     one_project = len(all_projects) == 1
 
     if opt.interactive and not one_project:
-      print >>sys.stderr, 'error: interactive rebase not supported with multiple projects'
+      print('error: interactive rebase not supported with multiple projects',
+            file=sys.stderr)
+      if len(args) == 1:
+        print('note: project %s is mapped to more than one path' % (args[0],),
+            file=sys.stderr)
       return -1
 
     for project in all_projects:
       cb = project.CurrentBranch
       if not cb:
         if one_project:
-          print >>sys.stderr, "error: project %s has a detatched HEAD" % project.relpath
+          print("error: project %s has a detached HEAD" % project.relpath,
+                file=sys.stderr)
           return -1
         # ignore branches with detatched HEADs
         continue
@@ -74,7 +80,8 @@ branch but need to incorporate new upstream changes "underneath" them.
       upbranch = project.GetBranch(cb)
       if not upbranch.LocalMerge:
         if one_project:
-          print >>sys.stderr, "error: project %s does not track any remote branches" % project.relpath
+          print("error: project %s does not track any remote branches"
+                % project.relpath, file=sys.stderr)
           return -1
         # ignore branches without remotes
         continue
@@ -101,8 +108,8 @@ branch but need to incorporate new upstream changes "underneath" them.
 
       args.append(upbranch.LocalMerge)
 
-      print >>sys.stderr, '# %s: rebasing %s -> %s' % \
+      print('# %s: rebasing %s -> %s'
-        (project.relpath, cb, upbranch.LocalMerge)
+            % (project.relpath, cb, upbranch.LocalMerge), file=sys.stderr)
 
       needs_stash = False
       if opt.auto_stash:

subcmds/selfupdate.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 from optparse import SUPPRESS_HELP
 import sys
 
@@ -52,7 +53,7 @@ need to be performed by an end-user.
 
     else:
       if not rp.Sync_NetworkHalf():
-        print >>sys.stderr, "error: can't update repo"
+        print("error: can't update repo", file=sys.stderr)
         sys.exit(1)
 
       rp.bare_git.gc('--auto')

subcmds/stage.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import sys
 
 from color import Coloring
@@ -48,9 +49,9 @@ The '%prog' command stages files to prepare the next commit.
       self.Usage()
 
   def _Interactive(self, opt, args):
-    all_projects = filter(lambda x: x.IsDirty(), self.GetProjects(args))
+    all_projects = [p for p in self.GetProjects(args) if p.IsDirty()]
     if not all_projects:
-      print >>sys.stderr,'no projects have uncommitted modifications'
+      print('no projects have uncommitted modifications', file=sys.stderr)
       return
 
     out = _ProjectList(self.manifest.manifestProject.config)
@@ -58,7 +59,7 @@ The '%prog' command stages files to prepare the next commit.
       out.header('        %s', 'project')
       out.nl()
 
-      for i in xrange(0, len(all_projects)):
+      for i in range(len(all_projects)):
         p = all_projects[i]
         out.write('%3d:    %s', i + 1, p.relpath + '/')
         out.nl()
@@ -97,11 +98,11 @@ The '%prog' command stages files to prepare the next commit.
           _AddI(all_projects[a_index - 1])
           continue
 
-      p = filter(lambda x: x.name == a or x.relpath == a, all_projects)
+      projects = [p for p in all_projects if a in [p.name, p.relpath]]
-      if len(p) == 1:
+      if len(projects) == 1:
-        _AddI(p[0])
+        _AddI(projects[0])
         continue
-    print 'Bye.'
+    print('Bye.')
 
 def _AddI(project):
   p = GitCommand(project, ['add', '--interactive'], bare=False)

subcmds/start.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import sys
 from command import Command
 from git_config import IsId
@@ -41,7 +42,7 @@ revision specified in the manifest.
 
     nb = args[0]
     if not git.check_ref_format('heads/%s' % nb):
-      print >>sys.stderr, "error: '%s' is not a valid name" % nb
+      print("error: '%s' is not a valid name" % nb, file=sys.stderr)
       sys.exit(1)
 
     err = []
@@ -49,7 +50,7 @@ revision specified in the manifest.
     if not opt.all:
       projects = args[1:]
       if len(projects) < 1:
-        print >>sys.stderr, "error: at least one project must be specified"
+        print("error: at least one project must be specified", file=sys.stderr)
         sys.exit(1)
 
     all_projects = self.GetProjects(projects)
@@ -67,7 +68,6 @@ revision specified in the manifest.
 
     if err:
       for p in err:
-        print >>sys.stderr,\
+        print("error: %s/: cannot start %s" % (p.relpath, nb),
-          "error: %s/: cannot start %s" \
+              file=sys.stderr)
-          % (p.relpath, nb)
       sys.exit(1)

subcmds/status.py

@@ -20,9 +20,19 @@ try:
 except ImportError:
   import dummy_threading as _threading
 
+import glob
+
+from pyversion import is_python3
+if is_python3():
+  import io
+else:
+  import StringIO as io
+
 import itertools
+import os
 import sys
-import StringIO
+
+from color import Coloring
 
 class Status(PagedCommand):
   common = True
@@ -39,6 +49,13 @@ is a difference between these three states.
 The -j/--jobs option can be used to run multiple status queries
 in parallel.
 
+The -o/--orphans option can be used to show objects that are in
+the working directory, but not associated with a repo project.
+This includes unmanaged top-level files and directories, but also
+includes deeper items.  For example, if dir/subdir/proj1 and
+dir/subdir/proj2 are repo projects, dir/subdir/proj3 will be shown
+if it is not known to repo.
+
 Status Display
 --------------
 
@@ -76,6 +93,9 @@ the following meanings:
     p.add_option('-j', '--jobs',
                  dest='jobs', action='store', type='int', default=2,
                  help="number of projects to check simultaneously")
+    p.add_option('-o', '--orphans',
+                 dest='orphans', action='store_true',
+                 help="include objects in working directory outside of repo projects")
 
   def _StatusHelper(self, project, clean_counter, sem, output):
     """Obtains the status for a specific project.
@@ -97,6 +117,22 @@ the following meanings:
     finally:
       sem.release()
 
+  def _FindOrphans(self, dirs, proj_dirs, proj_dirs_parents, outstring):
+    """find 'dirs' that are present in 'proj_dirs_parents' but not in 'proj_dirs'"""
+    status_header = ' --\t'
+    for item in dirs:
+      if not os.path.isdir(item):
+        outstring.write(''.join([status_header, item]))
+        continue
+      if item in proj_dirs:
+        continue
+      if item in proj_dirs_parents:
+        self._FindOrphans(glob.glob('%s/.*' % item) + \
+            glob.glob('%s/*' % item), \
+            proj_dirs, proj_dirs_parents, outstring)
+        continue
+      outstring.write(''.join([status_header, item, '/']))
+
   def Execute(self, opt, args):
     all_projects = self.GetProjects(args)
     counter = itertools.count()
@@ -112,7 +148,7 @@ the following meanings:
       for project in all_projects:
         sem.acquire()
 
-        class BufList(StringIO.StringIO):
+        class BufList(io.StringIO):
           def dump(self, ostream):
             for entry in self.buflist:
               ostream.write(entry)
@@ -129,4 +165,46 @@ the following meanings:
         output.dump(sys.stdout)
         output.close()
     if len(all_projects) == counter.next():
-      print 'nothing to commit (working directory clean)'
+      print('nothing to commit (working directory clean)')
+
+    if opt.orphans:
+      proj_dirs = set()
+      proj_dirs_parents = set()
+      for project in self.GetProjects(None, missing_ok=True):
+        proj_dirs.add(project.relpath)
+        (head, _tail) = os.path.split(project.relpath)
+        while head != "":
+          proj_dirs_parents.add(head)
+          (head, _tail) = os.path.split(head)
+      proj_dirs.add('.repo')
+
+      class StatusColoring(Coloring):
+        def __init__(self, config):
+          Coloring.__init__(self, config, 'status')
+          self.project = self.printer('header', attr = 'bold')
+          self.untracked = self.printer('untracked', fg = 'red')
+
+      orig_path = os.getcwd()
+      try:
+        os.chdir(self.manifest.topdir)
+
+        outstring = io.StringIO()
+        self._FindOrphans(glob.glob('.*') + \
+            glob.glob('*'), \
+            proj_dirs, proj_dirs_parents, outstring)
+
+        if outstring.buflist:
+          output = StatusColoring(self.manifest.globalConfig)
+          output.project('Objects not within a project (orphans)')
+          output.nl()
+          for entry in outstring.buflist:
+            output.untracked(entry)
+            output.nl()
+        else:
+          print('No orphan files or directories')
+
+        outstring.close()
+
+      finally:
+        # Restore CWD.
+        os.chdir(orig_path)

subcmds/sync.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import netrc
 from optparse import SUPPRESS_HELP
 import os
@@ -23,8 +24,19 @@ import socket
 import subprocess
 import sys
 import time
-import urlparse
+
-import xmlrpclib
+from pyversion import is_python3
+if is_python3():
+  import urllib.parse
+  import xmlrpc.client
+else:
+  import imp
+  import urlparse
+  import xmlrpclib
+  urllib = imp.new_module('urllib')
+  urllib.parse = urlparse
+  xmlrpc = imp.new_module('xmlrpc')
+  xmlrpc.client = xmlrpclib
 
 try:
   import threading as _threading
@@ -44,15 +56,15 @@ try:
 except ImportError:
   multiprocessing = None
 
-from git_command import GIT
+from git_command import GIT, git_require
 from git_refs import R_HEADS, HEAD
-from main import WrapperModule
 from project import Project
 from project import RemoteSpec
 from command import Command, MirrorSafeCommand
-from error import RepoChangedException, GitError
+from error import RepoChangedException, GitError, ManifestParseError
 from project import SyncBuffer
 from progress import Progress
+from wrapper import Wrapper
 
 _ONE_DAY_S = 24 * 60 * 60
 
@@ -113,6 +125,9 @@ resumeable bundle file on a content delivery network. This
 may be necessary if there are problems with the local Python
 HTTP client or proxy configuration, but the Git binary works.
 
+The --fetch-submodules option enables fetching Git submodules
+of a project from server.
+
 SSH Connections
 ---------------
 
@@ -144,27 +159,30 @@ later is required to fix a server side protocol bug.
 """
 
   def _Options(self, p, show_smart=True):
-    self.jobs = self.manifest.default.sync_j
+    try:
+      self.jobs = self.manifest.default.sync_j
+    except ManifestParseError:
+      self.jobs = 1
 
     p.add_option('-f', '--force-broken',
                  dest='force_broken', action='store_true',
                  help="continue sync even if a project fails to sync")
-    p.add_option('-l','--local-only',
+    p.add_option('-l', '--local-only',
                  dest='local_only', action='store_true',
                  help="only update working tree, don't fetch")
-    p.add_option('-n','--network-only',
+    p.add_option('-n', '--network-only',
                  dest='network_only', action='store_true',
                  help="fetch only, don't update working tree")
-    p.add_option('-d','--detach',
+    p.add_option('-d', '--detach',
                  dest='detach_head', action='store_true',
                  help='detach projects back to manifest revision')
-    p.add_option('-c','--current-branch',
+    p.add_option('-c', '--current-branch',
                  dest='current_branch_only', action='store_true',
                  help='fetch only current branch from server')
-    p.add_option('-q','--quiet',
+    p.add_option('-q', '--quiet',
                  dest='quiet', action='store_true',
                  help='be more quiet')
-    p.add_option('-j','--jobs',
+    p.add_option('-j', '--jobs',
                  dest='jobs', action='store', type='int',
                  help="projects to fetch simultaneously (default %d)" % self.jobs)
     p.add_option('-m', '--manifest-name',
@@ -173,6 +191,18 @@ later is required to fix a server side protocol bug.
     p.add_option('--no-clone-bundle',
                  dest='no_clone_bundle', action='store_true',
                  help='disable use of /clone.bundle on HTTP/HTTPS')
+    p.add_option('-u', '--manifest-server-username', action='store',
+                 dest='manifest_server_username',
+                 help='username to authenticate with the manifest server')
+    p.add_option('-p', '--manifest-server-password', action='store',
+                 dest='manifest_server_password',
+                 help='password to authenticate with the manifest server')
+    p.add_option('--fetch-submodules',
+                 dest='fetch_submodules', action='store_true',
+                 help='fetch submodules from server')
+    p.add_option('--no-tags',
+                 dest='no_tags', action='store_true',
+                 help="don't fetch tags")
     if show_smart:
       p.add_option('-s', '--smart-sync',
                    dest='smart_sync', action='store_true',
@@ -180,12 +210,6 @@ later is required to fix a server side protocol bug.
       p.add_option('-t', '--smart-tag',
                    dest='smart_tag', action='store',
                    help='smart sync using manifest from a known tag')
-      p.add_option('-u', '--manifest-server-username', action='store',
-                   dest='manifest_server_username',
-                   help='username to authenticate with the manifest server')
-      p.add_option('-p', '--manifest-server-password', action='store',
-                   dest='manifest_server_password',
-                   help='password to authenticate with the manifest server')
 
     g = p.add_option_group('repo Version options')
     g.add_option('--no-repo-verify',
@@ -195,130 +219,156 @@ later is required to fix a server side protocol bug.
                  dest='repo_upgraded', action='store_true',
                  help=SUPPRESS_HELP)
 
+  def _FetchProjectList(self, opt, projects, *args, **kwargs):
+    """Main function of the fetch threads when jobs are > 1.
+
+    Delegates most of the work to _FetchHelper.
+
+    Args:
+      opt: Program options returned from optparse.  See _Options().
+      projects: Projects to fetch.
+      *args, **kwargs: Remaining arguments to pass to _FetchHelper. See the
+          _FetchHelper docstring for details.
+    """
+    for project in projects:
+      success = self._FetchHelper(opt, project, *args, **kwargs)
+      if not success and not opt.force_broken:
+        break
+
   def _FetchHelper(self, opt, project, lock, fetched, pm, sem, err_event):
-      """Main function of the fetch threads when jobs are > 1.
+    """Fetch git objects for a single project.
 
-      Args:
+    Args:
-        opt: Program options returned from optparse.  See _Options().
+      opt: Program options returned from optparse.  See _Options().
-        project: Project object for the project to fetch.
+      project: Project object for the project to fetch.
-        lock: Lock for accessing objects that are shared amongst multiple
+      lock: Lock for accessing objects that are shared amongst multiple
-            _FetchHelper() threads.
+          _FetchHelper() threads.
-        fetched: set object that we will add project.gitdir to when we're done
+      fetched: set object that we will add project.gitdir to when we're done
-            (with our lock held).
+          (with our lock held).
-        pm: Instance of a Project object.  We will call pm.update() (with our
+      pm: Instance of a Project object.  We will call pm.update() (with our
-            lock held).
+          lock held).
-        sem: We'll release() this semaphore when we exit so that another thread
+      sem: We'll release() this semaphore when we exit so that another thread
-            can be started up.
+          can be started up.
-        err_event: We'll set this event in the case of an error (after printing
+      err_event: We'll set this event in the case of an error (after printing
-            out info about the error).
+          out info about the error).
-      """
-      # We'll set to true once we've locked the lock.
-      did_lock = False
 
-      # Encapsulate everything in a try/except/finally so that:
+    Returns:
-      # - We always set err_event in the case of an exception.
+      Whether the fetch was successful.
-      # - We always make sure we call sem.release().
+    """
-      # - We always make sure we unlock the lock if we locked it.
+    # We'll set to true once we've locked the lock.
+    did_lock = False
+
+    if not opt.quiet:
+      print('Fetching project %s' % project.name)
+
+    # Encapsulate everything in a try/except/finally so that:
+    # - We always set err_event in the case of an exception.
+    # - We always make sure we call sem.release().
+    # - We always make sure we unlock the lock if we locked it.
+    try:
       try:
-        try:
+        start = time.time()
-          start = time.time()
+        success = project.Sync_NetworkHalf(
-          success = project.Sync_NetworkHalf(
+          quiet=opt.quiet,
-            quiet=opt.quiet,
+          current_branch_only=opt.current_branch_only,
-            current_branch_only=opt.current_branch_only,
+          clone_bundle=not opt.no_clone_bundle,
-            clone_bundle=not opt.no_clone_bundle)
+          no_tags=opt.no_tags, archive=self.manifest.IsArchive)
-          self._fetch_times.Set(project, time.time() - start)
+        self._fetch_times.Set(project, time.time() - start)
 
-          # Lock around all the rest of the code, since printing, updating a set
+        # Lock around all the rest of the code, since printing, updating a set
-          # and Progress.update() are not thread safe.
+        # and Progress.update() are not thread safe.
-          lock.acquire()
+        lock.acquire()
-          did_lock = True
+        did_lock = True
 
-          if not success:
+        if not success:
-            print >>sys.stderr, 'error: Cannot fetch %s' % project.name
+          print('error: Cannot fetch %s' % project.name, file=sys.stderr)
-            if opt.force_broken:
+          if opt.force_broken:
-              print >>sys.stderr, 'warn: --force-broken, continuing to sync'
+            print('warn: --force-broken, continuing to sync',
-            else:
+                  file=sys.stderr)
-              raise _FetchError()
+          else:
+            raise _FetchError()
 
-          fetched.add(project.gitdir)
+        fetched.add(project.gitdir)
-          pm.update()
+        pm.update()
-        except _FetchError:
+      except _FetchError:
-          err_event.set()
+        err_event.set()
-        except:
+      except:
-          err_event.set()
+        err_event.set()
-          raise
+        raise
-      finally:
+    finally:
-        if did_lock:
+      if did_lock:
-          lock.release()
+        lock.release()
-        sem.release()
+      sem.release()
+
+    return success
 
   def _Fetch(self, projects, opt):
     fetched = set()
+    lock = _threading.Lock()
     pm = Progress('Fetching projects', len(projects))
 
-    if self.jobs == 1:
+    objdir_project_map = dict()
-      for project in projects:
+    for project in projects:
-        pm.update()
+      objdir_project_map.setdefault(project.objdir, []).append(project)
-        if project.Sync_NetworkHalf(
-            quiet=opt.quiet,
-            current_branch_only=opt.current_branch_only,
-            clone_bundle=not opt.no_clone_bundle):
-          fetched.add(project.gitdir)
-        else:
-          print >>sys.stderr, 'error: Cannot fetch %s' % project.name
-          if opt.force_broken:
-            print >>sys.stderr, 'warn: --force-broken, continuing to sync'
-          else:
-            sys.exit(1)
-    else:
-      threads = set()
-      lock = _threading.Lock()
-      sem = _threading.Semaphore(self.jobs)
-      err_event = _threading.Event()
-      for project in projects:
-        # Check for any errors before starting any new threads.
-        # ...we'll let existing threads finish, though.
-        if err_event.isSet():
-          break
 
-        sem.acquire()
+    threads = set()
-        t = _threading.Thread(target = self._FetchHelper,
+    sem = _threading.Semaphore(self.jobs)
-                              args = (opt,
+    err_event = _threading.Event()
-                                      project,
+    for project_list in objdir_project_map.values():
-                                      lock,
+      # Check for any errors before running any more tasks.
-                                      fetched,
+      # ...we'll let existing threads finish, though.
-                                      pm,
+      if err_event.isSet() and not opt.force_broken:
-                                      sem,
+        break
-                                      err_event))
+
+      sem.acquire()
+      kwargs = dict(opt=opt,
+                    projects=project_list,
+                    lock=lock,
+                    fetched=fetched,
+                    pm=pm,
+                    sem=sem,
+                    err_event=err_event)
+      if self.jobs > 1:
+        t = _threading.Thread(target = self._FetchProjectList,
+                              kwargs = kwargs)
         # Ensure that Ctrl-C will not freeze the repo process.
         t.daemon = True
         threads.add(t)
         t.start()
+      else:
+        self._FetchProjectList(**kwargs)
 
-      for t in threads:
+    for t in threads:
-        t.join()
+      t.join()
 
-      # If we saw an error, exit with code 1 so that other scripts can check.
+    # If we saw an error, exit with code 1 so that other scripts can check.
-      if err_event.isSet():
+    if err_event.isSet():
-        print >>sys.stderr, '\nerror: Exited sync due to fetch errors'
+      print('\nerror: Exited sync due to fetch errors', file=sys.stderr)
-        sys.exit(1)
+      sys.exit(1)
 
     pm.end()
     self._fetch_times.Save()
 
-    self._GCProjects(projects)
+    if not self.manifest.IsArchive:
+      self._GCProjects(projects)
+
     return fetched
 
   def _GCProjects(self, projects):
-    if multiprocessing:
+    gitdirs = {}
+    for project in projects:
+      gitdirs[project.gitdir] = project.bare_git
+
+    has_dash_c = git_require((1, 7, 2))
+    if multiprocessing and has_dash_c:
       cpu_count = multiprocessing.cpu_count()
     else:
       cpu_count = 1
     jobs = min(self.jobs, cpu_count)
 
     if jobs < 2:
-      for project in projects:
+      for bare_git in gitdirs.values():
-        project.bare_git.gc('--auto')
+        bare_git.gc('--auto')
       return
 
     config = {'pack.threads': cpu_count / jobs if cpu_count > jobs else 1}
@@ -327,10 +377,10 @@ later is required to fix a server side protocol bug.
     sem = _threading.Semaphore(jobs)
     err_event = _threading.Event()
 
-    def GC(project):
+    def GC(bare_git):
       try:
         try:
-          project.bare_git.gc('--auto', config=config)
+          bare_git.gc('--auto', config=config)
         except GitError:
           err_event.set()
         except:
@@ -339,11 +389,11 @@ later is required to fix a server side protocol bug.
       finally:
         sem.release()
 
-    for project in projects:
+    for bare_git in gitdirs.values():
       if err_event.isSet():
         break
       sem.acquire()
-      t = _threading.Thread(target=GC, args=(project,))
+      t = _threading.Thread(target=GC, args=(bare_git,))
       t.daemon = True
       threads.add(t)
       t.start()
@@ -352,9 +402,16 @@ later is required to fix a server side protocol bug.
       t.join()
 
     if err_event.isSet():
-      print >>sys.stderr, '\nerror: Exited sync due to gc errors'
+      print('\nerror: Exited sync due to gc errors', file=sys.stderr)
       sys.exit(1)
 
+  def _ReloadManifest(self, manifest_name=None):
+    if manifest_name:
+      # Override calls _Unload already
+      self.manifest.Override(manifest_name)
+    else:
+      self.manifest._Unload()
+
   def UpdateProjectList(self):
     new_project_paths = []
     for project in self.GetProjects(None, missing_ok=True):
@@ -376,34 +433,37 @@ later is required to fix a server side protocol bug.
         if path not in new_project_paths:
           # If the path has already been deleted, we don't need to do it
           if os.path.exists(self.manifest.topdir + '/' + path):
-              project = Project(
+            gitdir = os.path.join(self.manifest.topdir, path, '.git')
-                             manifest = self.manifest,
+            project = Project(
-                             name = path,
+                           manifest = self.manifest,
-                             remote = RemoteSpec('origin'),
+                           name = path,
-                             gitdir = os.path.join(self.manifest.topdir,
+                           remote = RemoteSpec('origin'),
-                                                   path, '.git'),
+                           gitdir = gitdir,
-                             worktree = os.path.join(self.manifest.topdir, path),
+                           objdir = gitdir,
-                             relpath = path,
+                           worktree = os.path.join(self.manifest.topdir, path),
-                             revisionExpr = 'HEAD',
+                           relpath = path,
-                             revisionId = None,
+                           revisionExpr = 'HEAD',
-                             groups = None)
+                           revisionId = None,
+                           groups = None)
 
-              if project.IsDirty():
+            if project.IsDirty():
-                print >>sys.stderr, 'error: Cannot remove project "%s": \
+              print('error: Cannot remove project "%s": uncommitted changes '
-uncommitted changes are present' % project.relpath
+                    'are present' % project.relpath, file=sys.stderr)
-                print >>sys.stderr, '       commit changes, then run sync again'
+              print('       commit changes, then run sync again',
-                return -1
+                    file=sys.stderr)
-              else:
+              return -1
-                print >>sys.stderr, 'Deleting obsolete path %s' % project.worktree
+            else:
-                shutil.rmtree(project.worktree)
+              print('Deleting obsolete path %s' % project.worktree,
-                # Try deleting parent subdirs if they are empty
+                    file=sys.stderr)
-                project_dir = os.path.dirname(project.worktree)
+              shutil.rmtree(project.worktree)
-                while project_dir != self.manifest.topdir:
+              # Try deleting parent subdirs if they are empty
-                  try:
+              project_dir = os.path.dirname(project.worktree)
-                    os.rmdir(project_dir)
+              while project_dir != self.manifest.topdir:
-                  except OSError:
+                try:
-                    break
+                  os.rmdir(project_dir)
-                  project_dir = os.path.dirname(project_dir)
+                except OSError:
+                  break
+                project_dir = os.path.dirname(project_dir)
 
     new_project_paths.sort()
     fd = open(file_path, 'w')
@@ -422,36 +482,40 @@ uncommitted changes are present' % project.relpath
       self.jobs = min(self.jobs, (soft_limit - 5) / 3)
 
     if opt.network_only and opt.detach_head:
-      print >>sys.stderr, 'error: cannot combine -n and -d'
+      print('error: cannot combine -n and -d', file=sys.stderr)
       sys.exit(1)
     if opt.network_only and opt.local_only:
-      print >>sys.stderr, 'error: cannot combine -n and -l'
+      print('error: cannot combine -n and -l', file=sys.stderr)
       sys.exit(1)
     if opt.manifest_name and opt.smart_sync:
-      print >>sys.stderr, 'error: cannot combine -m and -s'
+      print('error: cannot combine -m and -s', file=sys.stderr)
       sys.exit(1)
     if opt.manifest_name and opt.smart_tag:
-      print >>sys.stderr, 'error: cannot combine -m and -t'
+      print('error: cannot combine -m and -t', file=sys.stderr)
       sys.exit(1)
     if opt.manifest_server_username or opt.manifest_server_password:
       if not (opt.smart_sync or opt.smart_tag):
-        print >>sys.stderr, 'error: -u and -p may only be combined with ' \
+        print('error: -u and -p may only be combined with -s or -t',
-                            '-s or -t'
+              file=sys.stderr)
         sys.exit(1)
       if None in [opt.manifest_server_username, opt.manifest_server_password]:
-        print >>sys.stderr, 'error: both -u and -p must be given'
+        print('error: both -u and -p must be given', file=sys.stderr)
         sys.exit(1)
 
     if opt.manifest_name:
       self.manifest.Override(opt.manifest_name)
 
+    manifest_name = opt.manifest_name
+
     if opt.smart_sync or opt.smart_tag:
       if not self.manifest.manifest_server:
-        print >>sys.stderr, \
+        print('error: cannot smart sync: no manifest server defined in '
-            'error: cannot smart sync: no manifest server defined in manifest'
+              'manifest', file=sys.stderr)
         sys.exit(1)
 
       manifest_server = self.manifest.manifest_server
+      if not opt.quiet:
+        print('Using manifest server %s' % manifest_server)
 
       if not '@' in manifest_server:
         username = None
@@ -463,20 +527,21 @@ uncommitted changes are present' % project.relpath
           try:
             info = netrc.netrc()
           except IOError:
-            print >>sys.stderr, '.netrc file does not exist or could not be opened'
+            print('.netrc file does not exist or could not be opened',
+                  file=sys.stderr)
           else:
             try:
-              parse_result = urlparse.urlparse(manifest_server)
+              parse_result = urllib.parse.urlparse(manifest_server)
               if parse_result.hostname:
                 username, _account, password = \
                   info.authenticators(parse_result.hostname)
             except TypeError:
               # TypeError is raised when the given hostname is not present
               # in the .netrc file.
-              print >>sys.stderr, 'No credentials found for %s in .netrc' % \
+              print('No credentials found for %s in .netrc'
-                                  parse_result.hostname
+                    % parse_result.hostname, file=sys.stderr)
             except netrc.NetrcParseError as e:
-              print >>sys.stderr, 'Error parsing .netrc file: %s' % e
+              print('Error parsing .netrc file: %s' % e, file=sys.stderr)
 
         if (username and password):
           manifest_server = manifest_server.replace('://', '://%s:%s@' %
@@ -484,7 +549,7 @@ uncommitted changes are present' % project.relpath
                                                     1)
 
       try:
-        server = xmlrpclib.Server(manifest_server)
+        server = xmlrpc.client.Server(manifest_server)
         if opt.smart_sync:
           p = self.manifest.manifestProject
           b = p.GetBranch(p.CurrentBranch)
@@ -493,8 +558,7 @@ uncommitted changes are present' % project.relpath
             branch = branch[len(R_HEADS):]
 
           env = os.environ.copy()
-          if (env.has_key('TARGET_PRODUCT') and
+          if 'TARGET_PRODUCT' in env and 'TARGET_BUILD_VARIANT' in env:
-              env.has_key('TARGET_BUILD_VARIANT')):
             target = '%s-%s' % (env['TARGET_PRODUCT'],
                                 env['TARGET_BUILD_VARIANT'])
             [success, manifest_str] = server.GetApprovedManifest(branch, target)
@@ -515,20 +579,22 @@ uncommitted changes are present' % project.relpath
             finally:
               f.close()
           except IOError:
-            print >>sys.stderr, 'error: cannot write manifest to %s' % \
+            print('error: cannot write manifest to %s' % manifest_path,
-                manifest_path
+                  file=sys.stderr)
             sys.exit(1)
-          self.manifest.Override(manifest_name)
+          self._ReloadManifest(manifest_name)
         else:
-          print >>sys.stderr, 'error: %s' % manifest_str
+          print('error: manifest server RPC call failed: %s' %
+                manifest_str, file=sys.stderr)
           sys.exit(1)
-      except (socket.error, IOError, xmlrpclib.Fault) as e:
+      except (socket.error, IOError, xmlrpc.client.Fault) as e:
-        print >>sys.stderr, 'error: cannot connect to manifest server %s:\n%s' % (
+        print('error: cannot connect to manifest server %s:\n%s'
-            self.manifest.manifest_server, e)
+              % (self.manifest.manifest_server, e), file=sys.stderr)
         sys.exit(1)
-      except xmlrpclib.ProtocolError as e:
+      except xmlrpc.client.ProtocolError as e:
-        print >>sys.stderr, 'error: cannot connect to manifest server %s:\n%d %s' % (
+        print('error: cannot connect to manifest server %s:\n%d %s'
-            self.manifest.manifest_server, e.errcode, e.errmsg)
+              % (self.manifest.manifest_server, e.errcode, e.errmsg),
+              file=sys.stderr)
         sys.exit(1)
 
     rp = self.manifest.repoProject
@@ -538,21 +604,24 @@ uncommitted changes are present' % project.relpath
     mp.PreSync()
 
     if opt.repo_upgraded:
-      _PostRepoUpgrade(self.manifest, opt)
+      _PostRepoUpgrade(self.manifest, quiet=opt.quiet)
 
     if not opt.local_only:
       mp.Sync_NetworkHalf(quiet=opt.quiet,
-                          current_branch_only=opt.current_branch_only)
+                          current_branch_only=opt.current_branch_only,
+                          no_tags=opt.no_tags)
 
     if mp.HasChanges:
       syncbuf = SyncBuffer(mp.config)
       mp.Sync_LocalHalf(syncbuf)
       if not syncbuf.Finish():
         sys.exit(1)
-      self.manifest._Unload()
+      self._ReloadManifest(manifest_name)
       if opt.jobs is None:
         self.jobs = self.manifest.default.sync_j
-    all_projects = self.GetProjects(args, missing_ok=True)
+    all_projects = self.GetProjects(args,
+                                    missing_ok=True,
+                                    submodules_ok=opt.fetch_submodules)
 
     self._fetch_times = _FetchTimes(self.manifest)
     if not opt.local_only:
@@ -572,8 +641,10 @@ uncommitted changes are present' % project.relpath
       # Iteratively fetch missing and/or nested unregistered submodules
       previously_missing_set = set()
       while True:
-        self.manifest._Unload()
+        self._ReloadManifest(manifest_name)
-        all_projects = self.GetProjects(args, missing_ok=True)
+        all_projects = self.GetProjects(args,
+                                        missing_ok=True,
+                                        submodules_ok=opt.fetch_submodules)
         missing = []
         for project in all_projects:
           if project.gitdir not in fetched:
@@ -588,7 +659,7 @@ uncommitted changes are present' % project.relpath
         previously_missing_set = missing_set
         fetched.update(self._Fetch(missing, opt))
 
-    if self.manifest.IsMirror:
+    if self.manifest.IsMirror or self.manifest.IsArchive:
       # bail out now, we have no working tree
       return
 
@@ -603,46 +674,47 @@ uncommitted changes are present' % project.relpath
       if project.worktree:
         project.Sync_LocalHalf(syncbuf)
     pm.end()
-    print >>sys.stderr
+    print(file=sys.stderr)
     if not syncbuf.Finish():
       sys.exit(1)
 
     # If there's a notice that's supposed to print at the end of the sync, print
     # it now...
     if self.manifest.notice:
-      print self.manifest.notice
+      print(self.manifest.notice)
 
-def _PostRepoUpgrade(manifest, opt):
+def _PostRepoUpgrade(manifest, quiet=False):
-  wrapper = WrapperModule()
+  wrapper = Wrapper()
   if wrapper.NeedSetupGnuPG():
-    wrapper.SetupGnuPG(opt.quiet)
+    wrapper.SetupGnuPG(quiet)
-  for project in manifest.projects.values():
+  for project in manifest.projects:
     if project.Exists:
       project.PostRepoUpgrade()
 
 def _PostRepoFetch(rp, no_repo_verify=False, verbose=False):
   if rp.HasChanges:
-    print >>sys.stderr, 'info: A new version of repo is available'
+    print('info: A new version of repo is available', file=sys.stderr)
-    print >>sys.stderr, ''
+    print(file=sys.stderr)
     if no_repo_verify or _VerifyTag(rp):
       syncbuf = SyncBuffer(rp.config)
       rp.Sync_LocalHalf(syncbuf)
       if not syncbuf.Finish():
         sys.exit(1)
-      print >>sys.stderr, 'info: Restarting repo with latest version'
+      print('info: Restarting repo with latest version', file=sys.stderr)
       raise RepoChangedException(['--repo-upgraded'])
     else:
-      print >>sys.stderr, 'warning: Skipped upgrade to unverified version'
+      print('warning: Skipped upgrade to unverified version', file=sys.stderr)
   else:
     if verbose:
-      print >>sys.stderr, 'repo version %s is current' % rp.work_git.describe(HEAD)
+      print('repo version %s is current' % rp.work_git.describe(HEAD),
+            file=sys.stderr)
 
 def _VerifyTag(project):
   gpg_dir = os.path.expanduser('~/.repoconfig/gnupg')
   if not os.path.exists(gpg_dir):
-    print >>sys.stderr,\
+    print('warning: GnuPG was not available during last "repo init"\n'
-"""warning: GnuPG was not available during last "repo init"
+          'warning: Cannot automatically authenticate repo."""',
-warning: Cannot automatically authenticate repo."""
+          file=sys.stderr)
     return True
 
   try:
@@ -656,10 +728,9 @@ warning: Cannot automatically authenticate repo."""
     if rev.startswith(R_HEADS):
       rev = rev[len(R_HEADS):]
 
-    print >>sys.stderr
+    print(file=sys.stderr)
-    print >>sys.stderr,\
+    print("warning: project '%s' branch '%s' is not signed"
-      "warning: project '%s' branch '%s' is not signed" \
+          % (project.name, rev), file=sys.stderr)
-      % (project.name, rev)
     return False
 
   env = os.environ.copy()
@@ -678,10 +749,10 @@ warning: Cannot automatically authenticate repo."""
   proc.stderr.close()
 
   if proc.wait() != 0:
-    print >>sys.stderr
+    print(file=sys.stderr)
-    print >>sys.stderr, out
+    print(out, file=sys.stderr)
-    print >>sys.stderr, err
+    print(err, file=sys.stderr)
-    print >>sys.stderr
+    print(file=sys.stderr)
     return False
   return True
 
@@ -708,14 +779,14 @@ class _FetchTimes(object):
   def _Load(self):
     if self._times is None:
       try:
-        f = open(self._path)
+        f = open(self._path, 'rb')
       except IOError:
         self._times = {}
         return self._times
       try:
         try:
           self._times = pickle.load(f)
-        except:
+        except IOError:
           try:
             os.remove(self._path)
           except OSError:

subcmds/upload.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import copy
 import re
 import sys
@@ -20,22 +21,31 @@ import sys
 from command import InteractiveCommand
 from editor import Editor
 from error import HookError, UploadError
+from git_command import GitCommand
 from project import RepoHook
 
+from pyversion import is_python3
+if not is_python3():
+  # pylint:disable=W0622
+  input = raw_input
+  # pylint:enable=W0622
+
 UNUSUAL_COMMIT_THRESHOLD = 5
 
 def _ConfirmManyUploads(multiple_branches=False):
   if multiple_branches:
-    print "ATTENTION: One or more branches has an unusually high number of commits."
+    print('ATTENTION: One or more branches has an unusually high number '
+          'of commits.')
   else:
-    print "ATTENTION: You are uploading an unusually high number of commits."
+    print('ATTENTION: You are uploading an unusually high number of commits.')
-  print "YOU PROBABLY DO NOT MEAN TO DO THIS. (Did you rebase across branches?)"
+  print('YOU PROBABLY DO NOT MEAN TO DO THIS. (Did you rebase across '
-  answer = raw_input("If you are sure you intend to do this, type 'yes': ").strip()
+        'branches?)')
+  answer = input("If you are sure you intend to do this, type 'yes': ").strip()
   return answer == "yes"
 
 def _die(fmt, *args):
   msg = fmt % args
-  print >>sys.stderr, 'error: %s' % msg
+  print('error: %s' % msg, file=sys.stderr)
   sys.exit(1)
 
 def _SplitEmails(values):
@@ -47,7 +57,7 @@ def _SplitEmails(values):
 class Upload(InteractiveCommand):
   common = True
   helpSummary = "Upload changes for code review"
-  helpUsage="""
+  helpUsage = """
 %prog [--re --cc] [<project>]...
 """
   helpDescription = """
@@ -79,6 +89,11 @@ to "true" then repo will assume you always answer "y" at the prompt,
 and will not prompt you further.  If it is set to "false" then repo
 will assume you always answer "n", and will abort.
 
+review.URL.autoreviewer:
+
+To automatically append a user or mailing list to reviews, you can set
+a per-project or global Git option to do so.
+
 review.URL.autocopy:
 
 To automatically copy a user or mailing list to all uploaded reviews,
@@ -137,6 +152,10 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
     p.add_option('-d', '--draft',
                  action='store_true', dest='draft', default=False,
                  help='If specified, upload as a draft.')
+    p.add_option('-D', '--destination', '--dest',
+                 type='string', action='store', dest='dest_branch',
+                 metavar='BRANCH',
+                 help='Submit for review on this target branch.')
 
     # Options relating to upload hook.  Note that verify and no-verify are NOT
     # opposites of each other, which is why they store to different locations.
@@ -176,18 +195,19 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
       date = branch.date
       commit_list = branch.commits
 
-      print 'Upload project %s/ to remote branch %s:' % (project.relpath, project.revisionExpr)
+      destination = opt.dest_branch or project.dest_branch or project.revisionExpr
-      print '  branch %s (%2d commit%s, %s):' % (
+      print('Upload project %s/ to remote branch %s:' % (project.relpath, destination))
+      print('  branch %s (%2d commit%s, %s):' % (
                     name,
                     len(commit_list),
                     len(commit_list) != 1 and 's' or '',
-                    date)
+                    date))
       for commit in commit_list:
-        print '         %s' % commit
+        print('         %s' % commit)
 
       sys.stdout.write('to %s (y/N)? ' % remote.review)
-      answer = sys.stdin.readline().strip()
+      answer = sys.stdin.readline().strip().lower()
-      answer = answer in ('y', 'Y', 'yes', '1', 'true', 't')
+      answer = answer in ('y', 'yes', '1', 'true', 't')
 
     if answer:
       if len(branch.commits) > UNUSUAL_COMMIT_THRESHOLD:
@@ -210,18 +230,21 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
 
       b = {}
       for branch in avail:
+        if branch is None:
+          continue
         name = branch.name
         date = branch.date
         commit_list = branch.commits
 
         if b:
           script.append('#')
+        destination = opt.dest_branch or project.dest_branch or project.revisionExpr
         script.append('#  branch %s (%2d commit%s, %s) to remote branch %s:' % (
                       name,
                       len(commit_list),
                       len(commit_list) != 1 and 's' or '',
                       date,
-                      project.revisionExpr))
+                      destination))
         for commit in commit_list:
           script.append('#         %s' % commit)
         b[name] = branch
@@ -275,14 +298,20 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
 
     self._UploadAndReport(opt, todo, people)
 
-  def _AppendAutoCcList(self, branch, people):
+  def _AppendAutoList(self, branch, people):
     """
+    Appends the list of reviewers in the git project's config.
     Appends the list of users in the CC list in the git project's config if a
     non-empty reviewer list was found.
     """
-
     name = branch.name
     project = branch.project
+
+    key = 'review.%s.autoreviewer' % project.GetBranch(name).remote.review
+    raw_list = project.config.GetString(key)
+    if not raw_list is None:
+      people[0].extend([entry.strip() for entry in raw_list.split(',')])
+
     key = 'review.%s.autocopy' % project.GetBranch(name).remote.review
     raw_list = project.config.GetString(key)
     if not raw_list is None and len(people[0]) > 0:
@@ -297,7 +326,7 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
     try:
       # refs/changes/XYZ/N --> XYZ
       return refs.get(last_pub).split('/')[-2]
-    except:
+    except (AttributeError, IndexError):
       return ""
 
   def _UploadAndReport(self, opt, todo, original_people):
@@ -305,37 +334,52 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
     for branch in todo:
       try:
         people = copy.deepcopy(original_people)
-        self._AppendAutoCcList(branch, people)
+        self._AppendAutoList(branch, people)
 
         # Check if there are local changes that may have been forgotten
         if branch.project.HasChanges():
-            key = 'review.%s.autoupload' % branch.project.remote.review
+          key = 'review.%s.autoupload' % branch.project.remote.review
-            answer = branch.project.config.GetBoolean(key)
+          answer = branch.project.config.GetBoolean(key)
 
-            # if they want to auto upload, let's not ask because it could be automated
+          # if they want to auto upload, let's not ask because it could be automated
-            if answer is None:
+          if answer is None:
-                sys.stdout.write('Uncommitted changes in ' + branch.project.name + ' (did you forget to amend?). Continue uploading? (y/N) ')
+            sys.stdout.write('Uncommitted changes in ' + branch.project.name + ' (did you forget to amend?). Continue uploading? (y/N) ')
-                a = sys.stdin.readline().strip().lower()
+            a = sys.stdin.readline().strip().lower()
-                if a not in ('y', 'yes', 't', 'true', 'on'):
+            if a not in ('y', 'yes', 't', 'true', 'on'):
-                    print >>sys.stderr, "skipping upload"
+              print("skipping upload", file=sys.stderr)
-                    branch.uploaded = False
+              branch.uploaded = False
-                    branch.error = 'User aborted'
+              branch.error = 'User aborted'
-                    continue
+              continue
 
         # Check if topic branches should be sent to the server during upload
         if opt.auto_topic is not True:
-           key = 'review.%s.uploadtopic' % branch.project.remote.review
+          key = 'review.%s.uploadtopic' % branch.project.remote.review
-           opt.auto_topic = branch.project.config.GetBoolean(key)
+          opt.auto_topic = branch.project.config.GetBoolean(key)
 
-        branch.UploadForReview(people, auto_topic=opt.auto_topic, draft=opt.draft)
+        destination = opt.dest_branch or branch.project.dest_branch
+
+        # Make sure our local branch is not setup to track a different remote branch
+        merge_branch = self._GetMergeBranch(branch.project)
+        if destination:
+          full_dest = 'refs/heads/%s' % destination
+          if not opt.dest_branch and merge_branch and merge_branch != full_dest:
+            print('merge branch %s does not match destination branch %s'
+                  % (merge_branch, full_dest))
+            print('skipping upload.')
+            print('Please use `--destination %s` if this is intentional'
+                  % destination)
+            branch.uploaded = False
+            continue
+
+        branch.UploadForReview(people, auto_topic=opt.auto_topic, draft=opt.draft, dest_branch=destination)
         branch.uploaded = True
       except UploadError as e:
         branch.error = e
         branch.uploaded = False
         have_errors = True
 
-    print >>sys.stderr, ''
+    print(file=sys.stderr)
-    print >>sys.stderr, '----------------------------------------------------------------------'
+    print('----------------------------------------------------------------------', file=sys.stderr)
 
     if have_errors:
       for branch in todo:
@@ -344,21 +388,38 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
             fmt = ' (%s)'
           else:
             fmt = '\n       (%s)'
-          print >>sys.stderr, ('[FAILED] %-15s %-15s' + fmt) % (
+          print(('[FAILED] %-15s %-15s' + fmt) % (
                  branch.project.relpath + '/', \
                  branch.name, \
-                 str(branch.error))
+                 str(branch.error)),
-      print >>sys.stderr, ''
+                 file=sys.stderr)
+      print()
 
     for branch in todo:
-        if branch.uploaded:
+      if branch.uploaded:
-          print >>sys.stderr, '[OK    ] %-15s %s' % (
+        print('[OK    ] %-15s %s' % (
-                 branch.project.relpath + '/',
+               branch.project.relpath + '/',
-                 branch.name)
+               branch.name),
+               file=sys.stderr)
 
     if have_errors:
       sys.exit(1)
 
+  def _GetMergeBranch(self, project):
+    p = GitCommand(project,
+                   ['rev-parse', '--abbrev-ref', 'HEAD'],
+                   capture_stdout = True,
+                   capture_stderr = True)
+    p.Wait()
+    local_branch = p.stdout.strip()
+    p = GitCommand(project,
+                   ['config', '--get', 'branch.%s.merge' % local_branch],
+                   capture_stdout = True,
+                   capture_stderr = True)
+    p.Wait()
+    merge_branch = p.stdout.strip()
+    return merge_branch
+
   def Execute(self, opt, args):
     project_list = self.GetProjects(args)
     pending = []
@@ -372,7 +433,16 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
     for project in project_list:
       if opt.current_branch:
         cbr = project.CurrentBranch
-        avail = [project.GetUploadableBranch(cbr)] if cbr else None
+        up_branch = project.GetUploadableBranch(cbr)
+        if up_branch:
+          avail = [up_branch]
+        else:
+          avail = None
+          print('ERROR: Current branch (%s) not uploadable. '
+                'You may be able to type '
+                '"git branch --set-upstream-to m/master" to fix '
+                'your branch.' % str(cbr),
+                file=sys.stderr)
       else:
         avail = project.GetUploadableBranches(branch)
       if avail:
@@ -382,20 +452,22 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
       hook = RepoHook('pre-upload', self.manifest.repo_hooks_project,
                       self.manifest.topdir, abort_if_user_denies=True)
       pending_proj_names = [project.name for (project, avail) in pending]
+      pending_worktrees = [project.worktree for (project, avail) in pending]
       try:
-        hook.Run(opt.allow_all_hooks, project_list=pending_proj_names)
+        hook.Run(opt.allow_all_hooks, project_list=pending_proj_names,
+                 worktree_list=pending_worktrees)
       except HookError as e:
-        print >>sys.stderr, "ERROR: %s" % str(e)
+        print("ERROR: %s" % str(e), file=sys.stderr)
         return
 
     if opt.reviewers:
       reviewers = _SplitEmails(opt.reviewers)
     if opt.cc:
       cc = _SplitEmails(opt.cc)
-    people = (reviewers,cc)
+    people = (reviewers, cc)
 
     if not pending:
-      print >>sys.stdout, "no branches ready for upload"
+      print("no branches ready for upload", file=sys.stderr)
     elif len(pending) == 1 and len(pending[0][1]) == 1:
       self._SingleBranch(opt, pending[0][1][0], people)
     else:

subcmds/version.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import sys
 from command import Command, MirrorSafeCommand
 from git_command import git
@@ -32,12 +33,12 @@ class Version(Command, MirrorSafeCommand):
     rp = self.manifest.repoProject
     rem = rp.GetRemote(rp.remote.name)
 
-    print 'repo version %s' % rp.work_git.describe(HEAD)
+    print('repo version %s' % rp.work_git.describe(HEAD))
-    print '       (from %s)' % rem.url
+    print('       (from %s)' % rem.url)
 
     if Version.wrapper_path is not None:
-      print 'repo launcher version %s' % Version.wrapper_version
+      print('repo launcher version %s' % Version.wrapper_version)
-      print '       (from %s)' % Version.wrapper_path
+      print('       (from %s)' % Version.wrapper_path)
 
-    print git.version().strip()
+    print(git.version().strip())
-    print 'Python %s' % sys.version
+    print('Python %s' % sys.version)

tests/test_git_config.py

@@ -4,49 +4,49 @@ import unittest
 import git_config
 
 def fixture(*paths):
-    """Return a path relative to test/fixtures.
+  """Return a path relative to test/fixtures.
-    """
+  """
-    return os.path.join(os.path.dirname(__file__), 'fixtures', *paths)
+  return os.path.join(os.path.dirname(__file__), 'fixtures', *paths)
 
 class GitConfigUnitTest(unittest.TestCase):
-    """Tests the GitConfig class.
+  """Tests the GitConfig class.
+  """
+  def setUp(self):
+    """Create a GitConfig object using the test.gitconfig fixture.
     """
-    def setUp(self):
+    config_fixture = fixture('test.gitconfig')
-        """Create a GitConfig object using the test.gitconfig fixture.
+    self.config = git_config.GitConfig(config_fixture)
-        """
-        config_fixture = fixture('test.gitconfig')
-        self.config = git_config.GitConfig(config_fixture)
 
-    def test_GetString_with_empty_config_values(self):
+  def test_GetString_with_empty_config_values(self):
-        """
+    """
-        Test config entries with no value.
+    Test config entries with no value.
 
-        [section]
+    [section]
-            empty
+        empty
 
-        """
+    """
-        val = self.config.GetString('section.empty')
+    val = self.config.GetString('section.empty')
-        self.assertEqual(val, None)
+    self.assertEqual(val, None)
 
-    def test_GetString_with_true_value(self):
+  def test_GetString_with_true_value(self):
-        """
+    """
-        Test config entries with a string value.
+    Test config entries with a string value.
 
-        [section]
+    [section]
-            nonempty = true
+        nonempty = true
 
-        """
+    """
-        val = self.config.GetString('section.nonempty')
+    val = self.config.GetString('section.nonempty')
-        self.assertEqual(val, 'true')
+    self.assertEqual(val, 'true')
 
-    def test_GetString_from_missing_file(self):
+  def test_GetString_from_missing_file(self):
-        """
+    """
-        Test missing config file
+    Test missing config file
-        """
+    """
-        config_fixture = fixture('not.present.gitconfig')
+    config_fixture = fixture('not.present.gitconfig')
-        config = git_config.GitConfig(config_fixture)
+    config = git_config.GitConfig(config_fixture)
-        val = config.GetString('empty')
+    val = config.GetString('empty')
-        self.assertEqual(val, None)
+    self.assertEqual(val, None)
 
 if __name__ == '__main__':
-    unittest.main()
+  unittest.main()

trace.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import sys
 import os
 REPO_TRACE = 'REPO_TRACE'
@@ -31,4 +32,4 @@ def SetTrace():
 
 def Trace(fmt, *args):
   if IsTrace():
-    print >>sys.stderr, fmt % args
+    print(fmt % args, file=sys.stderr)

wrapper.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2014 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+import imp
+import os
+
+
+def WrapperPath():
+  return os.path.join(os.path.dirname(__file__), 'repo')
+
+_wrapper_module = None
+def Wrapper():
+  global _wrapper_module
+  if not _wrapper_module:
+    _wrapper_module = imp.load_source('wrapper', WrapperPath())
+  return _wrapper_module