Merge "Only fetch current branch on shallow clients"

Fetching a new branch on a shallow client may download the entire
project history, as the depth parameter is not passed to git
fetch. Force the fetch to only download the current branch.

Change-Id: Ie17ce8eb5e3487c24d90b2cae8227319dea482c8

.gitattributes

@@ -0,0 +1,4 @@
+# Prevent /bin/sh scripts from being clobbered by autocrlf=true
+git_ssh text eol=lf
+main.py text eol=lf
+repo text eol=lf

.gitignore

@@ -1,2 +1,3 @@
 *.pyc
 .repopickle_*
+/repoc

.project

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <projectDescription>
-	<name>repo</name>
+	<name>git-repo</name>
 	<comment></comment>
 	<projects>
 	</projects>

.pydevproject

@@ -3,7 +3,7 @@
 
 <pydev_project>
 <pydev_pathproperty name="org.python.pydev.PROJECT_SOURCE_PATH">
-<path>/repo</path>
+<path>/git-repo</path>
 </pydev_pathproperty>
 <pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python 2.6</pydev_property>
 <pydev_property name="org.python.pydev.PYTHON_PROJECT_INTERPRETER">Default</pydev_property>

.pylintrc

@@ -53,7 +53,7 @@ load-plugins=
 enable=RP0004
 
 # Disable the message(s) with the given id(s).
-disable=R0903,R0912,R0913,R0914,R0915,W0141,C0111,C0103,C0323,C0322,C0324,W0603,W0703,R0911,C0301,C0302,R0902,R0904,W0142,W0212,E1101,E1103,R0201,W0201,W0122,W0232,W0311,RP0001,RP0003,RP0101,RP0002,RP0401,RP0701,RP0801
+disable=R0903,R0912,R0913,R0914,R0915,W0141,C0111,C0103,W0603,W0703,R0911,C0301,C0302,R0902,R0904,W0142,W0212,E1101,E1103,R0201,W0201,W0122,W0232,RP0001,RP0003,RP0101,RP0002,RP0401,RP0701,RP0801,F0401,E0611,R0801,I0011
 
 [REPORTS]
 

color.py

@@ -36,50 +36,51 @@ ATTRS = {None     :-1,
          'blink'  : 5,
          'reverse': 7}
 
-RESET = "\033[m"
+RESET = "\033[m"  # pylint: disable=W1401
+                  # backslash is not anomalous
 
 def is_color(s):
-    return s in COLORS
+  return s in COLORS
 
 def is_attr(s):
-    return s in ATTRS
+  return s in ATTRS
 
 def _Color(fg = None, bg = None, attr = None):
-    fg = COLORS[fg]
-    bg = COLORS[bg]
-    attr = ATTRS[attr]
+  fg = COLORS[fg]
+  bg = COLORS[bg]
+  attr = ATTRS[attr]
 
-    if attr >= 0 or fg >= 0 or bg >= 0:
-      need_sep = False
-      code = "\033["
+  if attr >= 0 or fg >= 0 or bg >= 0:
+    need_sep = False
+    code = "\033["  #pylint: disable=W1401
 
-      if attr >= 0:
-        code += chr(ord('0') + attr)
-        need_sep = True
+    if attr >= 0:
+      code += chr(ord('0') + attr)
+      need_sep = True
 
-      if fg >= 0:
-        if need_sep:
-          code += ';'
-        need_sep = True
+    if fg >= 0:
+      if need_sep:
+        code += ';'
+      need_sep = True
 
-        if fg < 8:
-          code += '3%c' % (ord('0') + fg)
-        else:
-          code += '38;5;%d' % fg
+      if fg < 8:
+        code += '3%c' % (ord('0') + fg)
+      else:
+        code += '38;5;%d' % fg
 
-      if bg >= 0:
-        if need_sep:
-          code += ';'
-        need_sep = True
+    if bg >= 0:
+      if need_sep:
+        code += ';'
+      need_sep = True
 
-        if bg < 8:
-          code += '4%c' % (ord('0') + bg)
-        else:
-          code += '48;5;%d' % bg
-      code += 'm'
-    else:
-      code = ''
-    return code
+      if bg < 8:
+        code += '4%c' % (ord('0') + bg)
+      else:
+        code += '48;5;%d' % bg
+    code += 'm'
+  else:
+    code = ''
+  return code
 
 
 class Coloring(object):
@@ -125,6 +126,13 @@ class Coloring(object):
       s._out.write(c(fmt, *args))
     return f
 
+  def nofmt_printer(self, opt=None, fg=None, bg=None, attr=None):
+    s = self
+    c = self.nofmt_colorer(opt, fg, bg, attr)
+    def f(fmt):
+      s._out.write(c(fmt))
+    return f
+
   def colorer(self, opt=None, fg=None, bg=None, attr=None):
     if self._on:
       c = self._parse(opt, fg, bg, attr)
@@ -137,6 +145,17 @@ class Coloring(object):
         return fmt % args
       return f
 
+  def nofmt_colorer(self, opt=None, fg=None, bg=None, attr=None):
+    if self._on:
+      c = self._parse(opt, fg, bg, attr)
+      def f(fmt):
+        return ''.join([c, fmt, RESET])
+      return f
+    else:
+      def f(fmt):
+        return fmt
+      return f
+
   def _parse(self, opt, fg, bg, attr):
     if not opt:
       return _Color(fg, bg, attr)

command.py

@@ -22,6 +22,7 @@ import sys
 from error import NoSuchProjectError
 from error import InvalidProjectGroupsError
 
+
 class Command(object):
   """Base class for any command line action in repo.
   """
@@ -33,6 +34,27 @@ class Command(object):
   def WantPager(self, opt):
     return False
 
+  def ReadEnvironmentOptions(self, opts):
+    """ Set options from environment variables. """
+
+    env_options = self._RegisteredEnvironmentOptions()
+
+    for env_key, opt_key in env_options.items():
+      # Get the user-set option value if any
+      opt_value = getattr(opts, opt_key)
+
+      # If the value is set, it means the user has passed it as a command
+      # line option, and we should use that.  Otherwise we can try to set it
+      # with the value from the corresponding environment variable.
+      if opt_value is not None:
+        continue
+
+      env_value = os.environ.get(env_key)
+      if env_value is not None:
+        setattr(opts, opt_key, env_value)
+
+    return opts
+
   @property
   def OptionParser(self):
     if self._optparse is None:
@@ -49,6 +71,24 @@ class Command(object):
     """Initialize the option parser.
     """
 
+  def _RegisteredEnvironmentOptions(self):
+    """Get options that can be set from environment variables.
+
+    Return a dictionary mapping environment variable name
+    to option key name that it can override.
+
+    Example: {'REPO_MY_OPTION': 'my_option'}
+
+    Will allow the option with key value 'my_option' to be set
+    from the value in the environment variable named 'REPO_MY_OPTION'.
+
+    Note: This does not work properly for options that are explicitly
+    set to None by the user, or options that are defined with a
+    default value other than None.
+
+    """
+    return {}
+
   def Usage(self):
     """Display usage and terminate.
     """
@@ -86,46 +126,44 @@ class Command(object):
         pass
     return project
 
-  def GetProjects(self, args, missing_ok=False):
+  def GetProjects(self, args, missing_ok=False, submodules_ok=False):
     """A list of projects that match the arguments.
     """
-    all_projects = self.manifest.projects
+    all_projects_list = self.manifest.projects
     result = []
 
     mp = self.manifest.manifestProject
 
     groups = mp.config.GetString('manifest.groups')
     if not groups:
-      groups = 'all,-notdefault,platform-' + platform.system().lower()
-    groups = [x for x in re.split('[,\s]+', groups) if x]
+      groups = 'default,platform-' + platform.system().lower()
+    groups = [x for x in re.split(r'[,\s]+', groups) if x]
 
     if not args:
-      all_projects_list = all_projects.values()
-      derived_projects = []
+      derived_projects = {}
       for project in all_projects_list:
-        if project.Registered:
-          # Do not search registered subproject for derived projects
-          # since its parent has been searched already
-          continue
-        derived_projects.extend(project.GetDerivedSubprojects())
-      all_projects_list.extend(derived_projects)
+        if submodules_ok or project.sync_s:
+          derived_projects.update((p.name, p)
+                                  for p in project.GetDerivedSubprojects())
+      all_projects_list.extend(derived_projects.values())
       for project in all_projects_list:
         if ((missing_ok or project.Exists) and
             project.MatchesGroups(groups)):
           result.append(project)
     else:
-      self._ResetPathToProjectMap(all_projects.values())
+      self._ResetPathToProjectMap(all_projects_list)
 
       for arg in args:
-        project = all_projects.get(arg)
+        projects = self.manifest.GetProjectsWithName(arg)
 
-        if not project:
+        if not projects:
           path = os.path.abspath(arg).replace('\\', '/')
           project = self._GetProjectByPath(path)
 
           # If it's not a derived project, update path->project mapping and
           # search again, as arg might actually point to a derived subproject.
-          if project and not project.Derived:
+          if (project and not project.Derived and
+              (submodules_ok or project.sync_s)):
             search_again = False
             for subproject in project.GetDerivedSubprojects():
               self._UpdatePathToProjectMap(subproject)
@@ -133,20 +171,36 @@ class Command(object):
             if search_again:
               project = self._GetProjectByPath(path) or project
 
-        if not project:
-          raise NoSuchProjectError(arg)
-        if not missing_ok and not project.Exists:
-          raise NoSuchProjectError(arg)
-        if not project.MatchesGroups(groups):
-          raise InvalidProjectGroupsError(arg)
+          if project:
+            projects = [project]
 
-        result.append(project)
+        if not projects:
+          raise NoSuchProjectError(arg)
+
+        for project in projects:
+          if not missing_ok and not project.Exists:
+            raise NoSuchProjectError(arg)
+          if not project.MatchesGroups(groups):
+            raise InvalidProjectGroupsError(arg)
+
+        result.extend(projects)
 
     def _getpath(x):
       return x.relpath
     result.sort(key=_getpath)
     return result
 
+  def FindProjects(self, args):
+    result = []
+    patterns = [re.compile(r'%s' % a, re.IGNORECASE) for a in args]
+    for project in self.GetProjects(''):
+      for pattern in patterns:
+        if pattern.search(project.name) or pattern.search(project.relpath):
+          result.append(project)
+          break
+    result.sort(key=lambda project: project.relpath)
+    return result
+
 # pylint: disable=W0223
 # Pylint warns that the `InteractiveCommand` and `PagedCommand` classes do not
 # override method `Execute` which is abstract in `Command`.  Since that method

docs/manifest-format.txt

@@ -27,38 +27,45 @@ following DTD:
                         remove-project*,
                         project*,
                         repo-hooks?)>
-  
+
     <!ELEMENT notice (#PCDATA)>
-  
+
     <!ELEMENT remote (EMPTY)>
     <!ATTLIST remote name         ID    #REQUIRED>
     <!ATTLIST remote alias        CDATA #IMPLIED>
     <!ATTLIST remote fetch        CDATA #REQUIRED>
     <!ATTLIST remote review       CDATA #IMPLIED>
-  
+
     <!ELEMENT default (EMPTY)>
-    <!ATTLIST default remote   IDREF #IMPLIED>
-    <!ATTLIST default revision CDATA #IMPLIED>
-    <!ATTLIST default sync-j   CDATA #IMPLIED>
-    <!ATTLIST default sync-c   CDATA #IMPLIED>
+    <!ATTLIST default remote      IDREF #IMPLIED>
+    <!ATTLIST default revision    CDATA #IMPLIED>
+    <!ATTLIST default dest-branch CDATA #IMPLIED>
+    <!ATTLIST default sync-j      CDATA #IMPLIED>
+    <!ATTLIST default sync-c      CDATA #IMPLIED>
+    <!ATTLIST default sync-s      CDATA #IMPLIED>
 
     <!ELEMENT manifest-server (EMPTY)>
     <!ATTLIST url              CDATA #REQUIRED>
-  
-    <!ELEMENT project (annotation?,
+
+    <!ELEMENT project (annotation*,
                        project*)>
-    <!ATTLIST project name     CDATA #REQUIRED>
-    <!ATTLIST project path     CDATA #IMPLIED>
-    <!ATTLIST project remote   IDREF #IMPLIED>
-    <!ATTLIST project revision CDATA #IMPLIED>
-    <!ATTLIST project groups   CDATA #IMPLIED>
-    <!ATTLIST project sync-c   CDATA #IMPLIED>
+    <!ATTLIST project name        CDATA #REQUIRED>
+    <!ATTLIST project path        CDATA #IMPLIED>
+    <!ATTLIST project remote      IDREF #IMPLIED>
+    <!ATTLIST project revision    CDATA #IMPLIED>
+    <!ATTLIST project dest-branch CDATA #IMPLIED>
+    <!ATTLIST project groups      CDATA #IMPLIED>
+    <!ATTLIST project sync-c      CDATA #IMPLIED>
+    <!ATTLIST project sync-s      CDATA #IMPLIED>
+    <!ATTLIST project upstream CDATA #IMPLIED>
+    <!ATTLIST project clone-depth CDATA #IMPLIED>
+    <!ATTLIST project force-path CDATA #IMPLIED>
 
     <!ELEMENT annotation (EMPTY)>
     <!ATTLIST annotation name  CDATA #REQUIRED>
     <!ATTLIST annotation value CDATA #REQUIRED>
     <!ATTLIST annotation keep  CDATA "true">
-  
+
     <!ELEMENT remove-project (EMPTY)>
     <!ATTLIST remove-project name  CDATA #REQUIRED>
 
@@ -120,6 +127,20 @@ Attribute `revision`: Name of a Git branch (e.g. `master` or
 `refs/heads/master`).  Project elements lacking their own
 revision attribute will use this revision.
 
+Attribute `dest-branch`: Name of a Git branch (e.g. `master`).
+Project elements not setting their own `dest-branch` will inherit
+this value. If this value is not set, projects will use `revision`
+by default instead.
+
+Attribute `sync_j`: Number of parallel jobs to use when synching.
+
+Attribute `sync_c`: Set to true to only sync the given Git
+branch (specified in the `revision` attribute) rather than the
+whole ref space.  Project elements lacking a sync_c element of
+their own will use this value.
+
+Attribute `sync_s`: Set to true to also sync sub-projects.
+
 
 Element manifest-server
 -----------------------
@@ -189,6 +210,11 @@ Tags and/or explicit SHA-1s should work in theory, but have not
 been extensively tested.  If not supplied the revision given by
 the default element is used.
 
+Attribute `dest-branch`: Name of a Git branch (e.g. `master`).
+When using `repo upload`, changes will be submitted for code
+review on this branch. If unspecified both here and in the
+default element, `revision` is used instead.
+
 Attribute `groups`: List of groups to which this project belongs,
 whitespace or comma separated.  All projects belong to the group
 "all", and each project automatically belongs to a group of
@@ -200,6 +226,26 @@ group "notdefault", it will not be automatically downloaded by repo.
 If the project has a parent element, the `name` and `path` here
 are the prefixed ones.
 
+Attribute `sync_c`: Set to true to only sync the given Git
+branch (specified in the `revision` attribute) rather than the
+whole ref space.
+
+Attribute `sync_s`: Set to true to also sync sub-projects.
+
+Attribute `upstream`: Name of the Git branch in which a sha1
+can be found.  Used when syncing a revision locked manifest in
+-c mode to avoid having to sync the entire ref space.
+
+Attribute `clone-depth`: Set the depth to use when fetching this
+project.  If specified, this value will override any value given
+to repo init with the --depth option on the command line.
+
+Attribute `force-path`: Set to true to force this project to create the
+local mirror repository according to its `path` attribute (if supplied)
+rather than the `name` attribute.  This attribute only applies to the
+local mirrors syncing, it will be ignored when syncing the projects in a
+client working directory.
+
 Element annotation
 ------------------
 
@@ -218,7 +264,7 @@ Deletes the named project from the internal manifest table, possibly
 allowing a subsequent project element in the same manifest file to
 replace the project with a different source.
 
-This element is mostly useful in the local_manifest.xml, where
+This element is mostly useful in a local manifest file, where
 the user can remove a project, and possibly replace it with their
 own definition.
 
@@ -227,21 +273,25 @@ Element include
 
 This element provides the capability of including another manifest
 file into the originating manifest.  Normal rules apply for the
-target manifest to include- it must be a usable manifest on it's own.
+target manifest to include - it must be a usable manifest on its own.
 
-Attribute `name`; the manifest to include, specified relative to
-the manifest repositories root.
+Attribute `name`: the manifest to include, specified relative to
+the manifest repository's root.
 
 
-Local Manifest
-==============
+Local Manifests
+===============
 
-Additional remotes and projects may be added through a local
-manifest, stored in `$TOP_DIR/.repo/local_manifest.xml`.
+Additional remotes and projects may be added through local manifest
+files stored in `$TOP_DIR/.repo/local_manifests/*.xml`.
 
 For example:
 
-  $ cat .repo/local_manifest.xml
+  $ ls .repo/local_manifests
+  local_manifest.xml
+  another_local_manifest.xml
+
+  $ cat .repo/local_manifests/local_manifest.xml
   <?xml version="1.0" encoding="UTF-8"?>
   <manifest>
     <project path="manifest"
@@ -250,6 +300,17 @@ For example:
              name="platform/manifest" />
   </manifest>
 
-Users may add projects to the local manifest prior to a `repo sync`
+Users may add projects to the local manifest(s) prior to a `repo sync`
 invocation, instructing repo to automatically download and manage
 these extra projects.
+
+Manifest files stored in `$TOP_DIR/.repo/local_manifests/*.xml` will
+be loaded in alphabetical order.
+
+Additional remotes and projects may also be added through a local
+manifest, stored in `$TOP_DIR/.repo/local_manifest.xml`. This method
+is deprecated in favor of using multiple manifest files as mentioned
+above.
+
+If `$TOP_DIR/.repo/local_manifest.xml` exists, it will be loaded before
+any manifest files stored in `$TOP_DIR/.repo/local_manifests/*.xml`.

editor.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import os
 import re
 import sys
@@ -53,10 +54,10 @@ class Editor(object):
       return e
 
     if os.getenv('TERM') == 'dumb':
-      print >>sys.stderr,\
+      print(
 """No editor specified in GIT_EDITOR, core.editor, VISUAL or EDITOR.
 Tried to fall back to vi but terminal is dumb.  Please configure at
-least one of these before using this command."""
+least one of these before using this command.""", file=sys.stderr)
       sys.exit(1)
 
     return 'vi'
@@ -67,7 +68,7 @@ least one of these before using this command."""
 
        Args:
          data        : the text to edit
-  
+
       Returns:
         new value of edited text; None if editing did not succeed
     """

error.py

@@ -21,6 +21,10 @@ class ManifestInvalidRevisionError(Exception):
   """The revision value in a project is incorrect.
   """
 
+class NoManifestException(Exception):
+  """The required manifest does not exist.
+  """
+
 class EditorError(Exception):
   """Unspecified error from the user's text editor.
   """

git_command.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import os
 import sys
 import subprocess
@@ -85,14 +86,14 @@ class _GitCall(object):
     global _git_version
 
     if _git_version is None:
-      ver_str = git.version()
+      ver_str = git.version().decode('utf-8')
       if ver_str.startswith('git version '):
         _git_version = tuple(
-          map(lambda x: int(x),
+          map(int,
             ver_str[len('git version '):].strip().split('-')[0].split('.')[0:3]
           ))
       else:
-        print >>sys.stderr, 'fatal: "%s" unsupported' % ver_str
+        print('fatal: "%s" unsupported' % ver_str, file=sys.stderr)
         sys.exit(1)
     return _git_version
 
@@ -110,8 +111,8 @@ def git_require(min_version, fail=False):
   if min_version <= git_version:
     return True
   if fail:
-    need = '.'.join(map(lambda x: str(x), min_version))
-    print >>sys.stderr, 'fatal: git %s or later required' % need
+    need = '.'.join(map(str, min_version))
+    print('fatal: git %s or later required' % need, file=sys.stderr)
     sys.exit(1)
   return False
 
@@ -132,15 +133,15 @@ class GitCommand(object):
                gitdir = None):
     env = os.environ.copy()
 
-    for e in [REPO_TRACE,
+    for key in [REPO_TRACE,
               GIT_DIR,
               'GIT_ALTERNATE_OBJECT_DIRECTORIES',
               'GIT_OBJECT_DIRECTORY',
               'GIT_WORK_TREE',
               'GIT_GRAFT_FILE',
               'GIT_INDEX_FILE']:
-      if e in env:
-        del env[e]
+      if key in env:
+        del env[key]
 
     if disable_editor:
       _setenv(env, 'GIT_EDITOR', ':')

git_config.py

@@ -13,8 +13,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import cPickle
+from __future__ import print_function
+
 import os
+import pickle
 import re
 import subprocess
 import sys
@@ -23,11 +25,25 @@ try:
 except ImportError:
   import dummy_threading as _threading
 import time
-import urllib2
+
+from pyversion import is_python3
+if is_python3():
+  import urllib.request
+  import urllib.error
+else:
+  import urllib2
+  import imp
+  urllib = imp.new_module('urllib')
+  urllib.request = urllib2
+  urllib.error = urllib2
 
 from signal import SIGTERM
 from error import GitError, UploadError
 from trace import Trace
+if is_python3():
+  from http.client import HTTPException
+else:
+  from httplib import HTTPException
 
 from git_command import GitCommand
 from git_command import ssh_sock
@@ -35,7 +51,7 @@ from git_command import terminate_ssh_clients
 
 R_HEADS = 'refs/heads/'
 R_TAGS  = 'refs/tags/'
-ID_RE = re.compile('^[0-9a-f]{40}$')
+ID_RE = re.compile(r'^[0-9a-f]{40}$')
 
 REVIEW_CACHE = dict()
 
@@ -157,7 +173,7 @@ class GitConfig(object):
       elif old != value:
         self._cache[key] = list(value)
         self._do('--replace-all', name, value[0])
-        for i in xrange(1, len(value)):
+        for i in range(1, len(value)):
           self._do('--add', name, value[i])
 
     elif len(old) != 1 or old[0] != value:
@@ -250,7 +266,7 @@ class GitConfig(object):
       Trace(': unpickle %s', self.file)
       fd = open(self._pickle, 'rb')
       try:
-        return cPickle.load(fd)
+        return pickle.load(fd)
       finally:
         fd.close()
     except EOFError:
@@ -259,7 +275,7 @@ class GitConfig(object):
     except IOError:
       os.remove(self._pickle)
       return None
-    except cPickle.PickleError:
+    except pickle.PickleError:
       os.remove(self._pickle)
       return None
 
@@ -267,13 +283,13 @@ class GitConfig(object):
     try:
       fd = open(self._pickle, 'wb')
       try:
-        cPickle.dump(cache, fd, cPickle.HIGHEST_PROTOCOL)
+        pickle.dump(cache, fd, pickle.HIGHEST_PROTOCOL)
       finally:
         fd.close()
     except IOError:
       if os.path.exists(self._pickle):
         os.remove(self._pickle)
-    except cPickle.PickleError:
+    except pickle.PickleError:
       if os.path.exists(self._pickle):
         os.remove(self._pickle)
 
@@ -288,12 +304,13 @@ class GitConfig(object):
     d = self._do('--null', '--list')
     if d is None:
       return c
-    for line in d.rstrip('\0').split('\0'):
+    for line in d.decode('utf-8').rstrip('\0').split('\0'):  # pylint: disable=W1401
+                                                             # Backslash is not anomalous
       if '\n' in line:
-          key, val = line.split('\n', 1)
+        key, val = line.split('\n', 1)
       else:
-          key = line
-          val = None
+        key = line
+        val = None
 
       if key in c:
         c[key].append(val)
@@ -418,7 +435,7 @@ def _open_ssh(host, port=None):
                      '-o','ControlPath %s' % ssh_sock(),
                      host]
     if port is not None:
-      command_base[1:1] = ['-p',str(port)]
+      command_base[1:1] = ['-p', str(port)]
 
     # Since the key wasn't in _master_keys, we think that master isn't running.
     # ...but before actually starting a master, we'll double-check.  This can
@@ -451,9 +468,8 @@ def _open_ssh(host, port=None):
       p = subprocess.Popen(command)
     except Exception as e:
       _ssh_master = False
-      print >>sys.stderr, \
-        '\nwarn: cannot enable ssh control master for %s:%s\n%s' \
-        % (host,port, str(e))
+      print('\nwarn: cannot enable ssh control master for %s:%s\n%s'
+             % (host,port, str(e)), file=sys.stderr)
       return False
 
     _master_processes.append(p)
@@ -525,8 +541,8 @@ class Remote(object):
     self.url = self._Get('url')
     self.review = self._Get('review')
     self.projectname = self._Get('projectname')
-    self.fetch = map(lambda x: RefSpec.FromString(x),
-                     self._Get('fetch', all_keys=True))
+    self.fetch = list(map(RefSpec.FromString,
+                      self._Get('fetch', all_keys=True)))
     self._review_url = None
 
   def _InsteadOf(self):
@@ -579,23 +595,22 @@ class Remote(object):
       else:
         try:
           info_url = u + 'ssh_info'
-          info = urllib2.urlopen(info_url).read()
-          if '<' in info:
-            # Assume the server gave us some sort of HTML
-            # response back, like maybe a login page.
+          info = urllib.request.urlopen(info_url).read()
+          if info == 'NOT_AVAILABLE' or '<' in info:
+            # If `info` contains '<', we assume the server gave us some sort
+            # of HTML response back, like maybe a login page.
             #
-            raise UploadError('%s: Cannot parse response' % info_url)
-
-          if info == 'NOT_AVAILABLE':
-            # Assume HTTP if SSH is not enabled.
+            # Assume HTTP if SSH is not enabled or ssh_info doesn't look right.
             self._review_url = http_url + 'p/'
           else:
             host, port = info.split()
             self._review_url = self._SshReviewUrl(userEmail, host, port)
-        except urllib2.HTTPError as e:
+        except urllib.error.HTTPError as e:
           raise UploadError('%s: %s' % (self.review, str(e)))
-        except urllib2.URLError as e:
+        except urllib.error.URLError as e:
           raise UploadError('%s: %s' % (self.review, str(e)))
+        except HTTPException as e:
+          raise UploadError('%s: %s' % (self.review, e.__class__.__name__))
 
         REVIEW_CACHE[u] = self._review_url
     return self._review_url + self.projectname
@@ -645,7 +660,7 @@ class Remote(object):
     self._Set('url', self.url)
     self._Set('review', self.review)
     self._Set('projectname', self.projectname)
-    self._Set('fetch', map(lambda x: str(x), self.fetch))
+    self._Set('fetch', list(map(str, self.fetch)))
 
   def _Set(self, key, value):
     key = 'remote.%s.%s' % (self.name, key)

git_refs.py

@@ -66,7 +66,7 @@ class GitRefs(object):
   def _NeedUpdate(self):
     Trace(': scan refs %s', self._gitdir)
 
-    for name, mtime in self._mtime.iteritems():
+    for name, mtime in self._mtime.items():
       try:
         if mtime != os.path.getmtime(os.path.join(self._gitdir, name)):
           return True
@@ -89,7 +89,7 @@ class GitRefs(object):
     attempts = 0
     while scan and attempts < 5:
       scan_next = {}
-      for name, dest in scan.iteritems():
+      for name, dest in scan.items():
         if dest in self._phyref:
           self._phyref[name] = self._phyref[dest]
         else:
@@ -100,7 +100,7 @@ class GitRefs(object):
   def _ReadPackedRefs(self):
     path = os.path.join(self._gitdir, 'packed-refs')
     try:
-      fd = open(path, 'rb')
+      fd = open(path, 'r')
       mtime = os.path.getmtime(path)
     except IOError:
       return
@@ -108,6 +108,7 @@ class GitRefs(object):
       return
     try:
       for line in fd:
+        line = str(line)
         if line[0] == '#':
           continue
         if line[0] == '^':
@@ -138,18 +139,22 @@ class GitRefs(object):
   def _ReadLoose1(self, path, name):
     try:
       fd = open(path, 'rb')
-    except:
+    except IOError:
       return
 
     try:
       try:
         mtime = os.path.getmtime(path)
         ref_id = fd.readline()
-      except:
+      except (IOError, OSError):
         return
     finally:
       fd.close()
 
+    try:
+      ref_id = ref_id.decode()
+    except AttributeError:
+      pass
     if not ref_id:
       return
     ref_id = ref_id[:-1]

hooks/commit-msg

@@ -1,5 +1,5 @@
 #!/bin/sh
-# From Gerrit Code Review 2.5-rc0
+# From Gerrit Code Review 2.6
 #
 # Part of Gerrit Code Review (http://code.google.com/p/gerrit/)
 #
@@ -18,6 +18,8 @@
 # limitations under the License.
 #
 
+unset GREP_OPTIONS
+
 CHANGE_ID_AFTER="Bug|Issue"
 MSG="$1"
 
@@ -152,7 +154,7 @@ add_ChangeId() {
 		if (unprinted) {
 			print "Change-Id: I'"$id"'"
 		}
-	}' "$MSG" > $T && mv $T "$MSG" || rm -f $T
+	}' "$MSG" > "$T" && mv "$T" "$MSG" || rm -f "$T"
 }
 _gen_ChangeIdInput() {
 	echo "tree `git write-tree`"

hooks/pre-auto-gc

@@ -35,7 +35,7 @@ elif grep -q "AC Power \+: 1" /proc/pmu/info 2>/dev/null
 then
 	exit 0
 elif test -x /usr/bin/pmset && /usr/bin/pmset -g batt |
-	grep -q "Currently drawing from 'AC Power'"
+	grep -q "drawing from 'AC Power'"
 then
 	exit 0
 elif test -d /sys/bus/acpi/drivers/battery && test 0 = \

main.py

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env python
 #
 # Copyright (C) 2008 The Android Open Source Project
 #
@@ -14,23 +14,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-magic='--calling-python-from-/bin/sh--'
-"""exec" python -E "$0" "$@" """#$magic"
-if __name__ == '__main__':
-  import sys
-  if sys.argv[-1] == '#%s' % magic:
-    del sys.argv[-1]
-del magic
-
+from __future__ import print_function
 import getpass
 import imp
 import netrc
 import optparse
 import os
-import re
 import sys
 import time
-import urllib2
+
+from pyversion import is_python3
+if is_python3():
+  import urllib.request
+else:
+  import urllib2
+  urllib = imp.new_module('urllib')
+  urllib.request = urllib2
 
 from trace import SetTrace
 from git_command import git, GitCommand
@@ -41,6 +40,8 @@ from subcmds.version import Version
 from editor import Editor
 from error import DownloadError
 from error import ManifestInvalidRevisionError
+from error import ManifestParseError
+from error import NoManifestException
 from error import NoSuchProjectError
 from error import RepoChangedException
 from manifest_xml import XmlManifest
@@ -48,6 +49,11 @@ from pager import RunPager
 
 from subcmds import all_commands
 
+if not is_python3():
+  # pylint:disable=W0622
+  input = raw_input
+  # pylint:enable=W0622
+
 global_options = optparse.OptionParser(
                  usage="repo [-p|--paginate|--no-pager] COMMAND [ARGS]"
                  )
@@ -79,7 +85,7 @@ class _Repo(object):
     name = None
     glob = []
 
-    for i in xrange(0, len(argv)):
+    for i in range(len(argv)):
       if not argv[i].startswith('-'):
         name = argv[i]
         if i > 0:
@@ -98,15 +104,14 @@ class _Repo(object):
       if name == 'help':
         name = 'version'
       else:
-        print >>sys.stderr, 'fatal: invalid usage of --version'
+        print('fatal: invalid usage of --version', file=sys.stderr)
         return 1
 
     try:
       cmd = self.commands[name]
     except KeyError:
-      print >>sys.stderr,\
-            "repo: '%s' is not a repo command.  See 'repo help'."\
-            % name
+      print("repo: '%s' is not a repo command.  See 'repo help'." % name,
+            file=sys.stderr)
       return 1
 
     cmd.repodir = self.repodir
@@ -114,12 +119,12 @@ class _Repo(object):
     Editor.globalConfig = cmd.manifest.globalConfig
 
     if not isinstance(cmd, MirrorSafeCommand) and cmd.manifest.IsMirror:
-      print >>sys.stderr, \
-            "fatal: '%s' requires a working directory"\
-            % name
+      print("fatal: '%s' requires a working directory" % name,
+            file=sys.stderr)
       return 1
 
     copts, cargs = cmd.OptionParser.parse_args(argv)
+    copts = cmd.ReadEnvironmentOptions(copts)
 
     if not gopts.no_pager and not isinstance(cmd, InteractiveCommand):
       config = cmd.manifest.globalConfig
@@ -132,33 +137,35 @@ class _Repo(object):
       if use_pager:
         RunPager(config)
 
+    start = time.time()
     try:
-      start = time.time()
-      try:
-        result = cmd.Execute(copts, cargs)
-      finally:
-        elapsed = time.time() - start
-        hours, remainder = divmod(elapsed, 3600)
-        minutes, seconds = divmod(remainder, 60)
-        if gopts.time:
-          if hours == 0:
-            print >>sys.stderr, 'real\t%dm%.3fs' \
-              % (minutes, seconds)
-          else:
-            print >>sys.stderr, 'real\t%dh%dm%.3fs' \
-              % (hours, minutes, seconds)
+      result = cmd.Execute(copts, cargs)
     except DownloadError as e:
-      print >>sys.stderr, 'error: %s' % str(e)
-      return 1
+      print('error: %s' % str(e), file=sys.stderr)
+      result = 1
     except ManifestInvalidRevisionError as e:
-      print >>sys.stderr, 'error: %s' % str(e)
-      return 1
+      print('error: %s' % str(e), file=sys.stderr)
+      result = 1
+    except NoManifestException as e:
+      print('error: manifest required for this command -- please run init',
+            file=sys.stderr)
+      result = 1
     except NoSuchProjectError as e:
       if e.name:
-        print >>sys.stderr, 'error: project %s not found' % e.name
+        print('error: project %s not found' % e.name, file=sys.stderr)
       else:
-        print >>sys.stderr, 'error: no project in current directory'
-      return 1
+        print('error: no project in current directory', file=sys.stderr)
+      result = 1
+    finally:
+      elapsed = time.time() - start
+      hours, remainder = divmod(elapsed, 3600)
+      minutes, seconds = divmod(remainder, 60)
+      if gopts.time:
+        if hours == 0:
+          print('real\t%dm%.3fs' % (minutes, seconds), file=sys.stderr)
+        else:
+          print('real\t%dh%dm%.3fs' % (hours, minutes, seconds),
+                file=sys.stderr)
 
     return result
 
@@ -183,36 +190,35 @@ def _CheckWrapperVersion(ver, repo_path):
     repo_path = '~/bin/repo'
 
   if not ver:
-    print >>sys.stderr, 'no --wrapper-version argument'
+    print('no --wrapper-version argument', file=sys.stderr)
     sys.exit(1)
 
   exp = _CurrentWrapperVersion()
-  ver = tuple(map(lambda x: int(x), ver.split('.')))
+  ver = tuple(map(int, ver.split('.')))
   if len(ver) == 1:
     ver = (0, ver[0])
 
+  exp_str = '.'.join(map(str, exp))
   if exp[0] > ver[0] or ver < (0, 4):
-    exp_str = '.'.join(map(lambda x: str(x), exp))
-    print >>sys.stderr, """
+    print("""
 !!! A new repo command (%5s) is available.    !!!
 !!! You must upgrade before you can continue:   !!!
 
     cp %s %s
-""" % (exp_str, _MyWrapperPath(), repo_path)
+""" % (exp_str, _MyWrapperPath(), repo_path), file=sys.stderr)
     sys.exit(1)
 
   if exp > ver:
-    exp_str = '.'.join(map(lambda x: str(x), exp))
-    print >>sys.stderr, """
+    print("""
 ... A new repo command (%5s) is available.
 ... You should upgrade soon:
 
     cp %s %s
-""" % (exp_str, _MyWrapperPath(), repo_path)
+""" % (exp_str, _MyWrapperPath(), repo_path), file=sys.stderr)
 
 def _CheckRepoDir(repo_dir):
   if not repo_dir:
-    print >>sys.stderr, 'no --repo-dir argument'
+    print('no --repo-dir argument', file=sys.stderr)
     sys.exit(1)
 
 def _PruneOptions(argv, opt):
@@ -264,11 +270,11 @@ def _UserAgent():
     _user_agent = 'git-repo/%s (%s) git/%s Python/%d.%d.%d' % (
       repo_version,
       os_name,
-      '.'.join(map(lambda d: str(d), git.version_tuple())),
+      '.'.join(map(str, git.version_tuple())),
       py_version[0], py_version[1], py_version[2])
   return _user_agent
 
-class _UserAgentHandler(urllib2.BaseHandler):
+class _UserAgentHandler(urllib.request.BaseHandler):
   def http_request(self, req):
     req.add_header('User-Agent', _UserAgent())
     return req
@@ -278,22 +284,22 @@ class _UserAgentHandler(urllib2.BaseHandler):
     return req
 
 def _AddPasswordFromUserInput(handler, msg, req):
-    # If repo could not find auth info from netrc, try to get it from user input
-    url = req.get_full_url()
-    user, password = handler.passwd.find_user_password(None, url)
-    if user is None:
-      print msg
-      try:
-        user = raw_input('User: ')
-        password = getpass.getpass()
-      except KeyboardInterrupt:
-        return
-      handler.passwd.add_password(None, url, user, password)
+  # If repo could not find auth info from netrc, try to get it from user input
+  url = req.get_full_url()
+  user, password = handler.passwd.find_user_password(None, url)
+  if user is None:
+    print(msg)
+    try:
+      user = input('User: ')
+      password = getpass.getpass()
+    except KeyboardInterrupt:
+      return
+    handler.passwd.add_password(None, url, user, password)
 
-class _BasicAuthHandler(urllib2.HTTPBasicAuthHandler):
+class _BasicAuthHandler(urllib.request.HTTPBasicAuthHandler):
   def http_error_401(self, req, fp, code, msg, headers):
     _AddPasswordFromUserInput(self, msg, req)
-    return urllib2.HTTPBasicAuthHandler.http_error_401(
+    return urllib.request.HTTPBasicAuthHandler.http_error_401(
       self, req, fp, code, msg, headers)
 
   def http_error_auth_reqed(self, authreq, host, req, headers):
@@ -303,7 +309,7 @@ class _BasicAuthHandler(urllib2.HTTPBasicAuthHandler):
         val = val.replace('\n', '')
         old_add_header(name, val)
       req.add_header = _add_header
-      return urllib2.AbstractBasicAuthHandler.http_error_auth_reqed(
+      return urllib.request.AbstractBasicAuthHandler.http_error_auth_reqed(
         self, authreq, host, req, headers)
     except:
       reset = getattr(self, 'reset_retry_count', None)
@@ -313,10 +319,10 @@ class _BasicAuthHandler(urllib2.HTTPBasicAuthHandler):
         self.retried = 0
       raise
 
-class _DigestAuthHandler(urllib2.HTTPDigestAuthHandler):
+class _DigestAuthHandler(urllib.request.HTTPDigestAuthHandler):
   def http_error_401(self, req, fp, code, msg, headers):
     _AddPasswordFromUserInput(self, msg, req)
-    return urllib2.HTTPDigestAuthHandler.http_error_401(
+    return urllib.request.HTTPDigestAuthHandler.http_error_401(
       self, req, fp, code, msg, headers)
 
   def http_error_auth_reqed(self, auth_header, host, req, headers):
@@ -326,7 +332,7 @@ class _DigestAuthHandler(urllib2.HTTPDigestAuthHandler):
         val = val.replace('\n', '')
         old_add_header(name, val)
       req.add_header = _add_header
-      return urllib2.AbstractDigestAuthHandler.http_error_auth_reqed(
+      return urllib.request.AbstractDigestAuthHandler.http_error_auth_reqed(
         self, auth_header, host, req, headers)
     except:
       reset = getattr(self, 'reset_retry_count', None)
@@ -339,7 +345,7 @@ class _DigestAuthHandler(urllib2.HTTPDigestAuthHandler):
 def init_http():
   handlers = [_UserAgentHandler()]
 
-  mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
+  mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
   try:
     n = netrc.netrc()
     for host in n.hosts:
@@ -355,11 +361,11 @@ def init_http():
 
   if 'http_proxy' in os.environ:
     url = os.environ['http_proxy']
-    handlers.append(urllib2.ProxyHandler({'http': url, 'https': url}))
+    handlers.append(urllib.request.ProxyHandler({'http': url, 'https': url}))
   if 'REPO_CURL_VERBOSE' in os.environ:
-    handlers.append(urllib2.HTTPHandler(debuglevel=1))
-    handlers.append(urllib2.HTTPSHandler(debuglevel=1))
-  urllib2.install_opener(urllib2.build_opener(*handlers))
+    handlers.append(urllib.request.HTTPHandler(debuglevel=1))
+    handlers.append(urllib.request.HTTPSHandler(debuglevel=1))
+  urllib.request.install_opener(urllib.request.build_opener(*handlers))
 
 def _Main(argv):
   result = 0
@@ -389,6 +395,10 @@ def _Main(argv):
     finally:
       close_ssh()
   except KeyboardInterrupt:
+    print('aborted by user', file=sys.stderr)
+    result = 1
+  except ManifestParseError as mpe:
+    print('fatal: %s' % mpe, file=sys.stderr)
     result = 1
   except RepoChangedException as rce:
     # If repo changed, re-exec ourselves.
@@ -398,8 +408,8 @@ def _Main(argv):
     try:
       os.execv(__file__, argv)
     except OSError as e:
-      print >>sys.stderr, 'fatal: cannot restart repo after upgrade'
-      print >>sys.stderr, 'fatal: %s' % e
+      print('fatal: cannot restart repo after upgrade', file=sys.stderr)
+      print('fatal: %s' % e, file=sys.stderr)
       result = 128
 
   sys.exit(result)

manifest_xml.py

@@ -13,13 +13,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import itertools
 import os
 import re
 import sys
-import urlparse
 import xml.dom.minidom
 
+from pyversion import is_python3
+if is_python3():
+  import urllib.parse
+else:
+  import imp
+  import urlparse
+  urllib = imp.new_module('urllib')
+  urllib.parse = urlparse
+
 from git_config import GitConfig
 from git_refs import R_HEADS, HEAD
 from project import RemoteSpec, Project, MetaProject
@@ -27,17 +36,26 @@ from error import ManifestParseError
 
 MANIFEST_FILE_NAME = 'manifest.xml'
 LOCAL_MANIFEST_NAME = 'local_manifest.xml'
+LOCAL_MANIFESTS_DIR_NAME = 'local_manifests'
 
-urlparse.uses_relative.extend(['ssh', 'git'])
-urlparse.uses_netloc.extend(['ssh', 'git'])
+urllib.parse.uses_relative.extend(['ssh', 'git'])
+urllib.parse.uses_netloc.extend(['ssh', 'git'])
 
 class _Default(object):
   """Project defaults within the manifest."""
 
   revisionExpr = None
+  destBranchExpr = None
   remote = None
   sync_j = 1
   sync_c = False
+  sync_s = False
+
+  def __eq__(self, other):
+    return self.__dict__ == other.__dict__
+
+  def __ne__(self, other):
+    return self.__dict__ != other.__dict__
 
 class _XmlRemote(object):
   def __init__(self,
@@ -53,15 +71,28 @@ class _XmlRemote(object):
     self.reviewUrl = review
     self.resolvedFetchUrl = self._resolveFetchUrl()
 
+  def __eq__(self, other):
+    return self.__dict__ == other.__dict__
+
+  def __ne__(self, other):
+    return self.__dict__ != other.__dict__
+
   def _resolveFetchUrl(self):
     url = self.fetchUrl.rstrip('/')
     manifestUrl = self.manifestUrl.rstrip('/')
+    p = manifestUrl.startswith('persistent-http')
+    if p:
+      manifestUrl = manifestUrl[len('persistent-'):]
+
     # urljoin will get confused if there is no scheme in the base url
     # ie, if manifestUrl is of the form <hostname:port>
     if manifestUrl.find(':') != manifestUrl.find('/') - 1:
-        manifestUrl = 'gopher://' + manifestUrl
-    url = urlparse.urljoin(manifestUrl, url)
-    return re.sub(r'^gopher://', '', url)
+      manifestUrl = 'gopher://' + manifestUrl
+    url = urllib.parse.urljoin(manifestUrl, url)
+    url = re.sub(r'^gopher://', '', url)
+    if p:
+      url = 'persistent-' + url
+    return url
 
   def ToRemoteSpec(self, projectName):
     url = self.resolvedFetchUrl.rstrip('/') + '/' + projectName
@@ -78,6 +109,7 @@ class XmlManifest(object):
     self.topdir = os.path.dirname(self.repodir)
     self.manifestFile = os.path.join(self.repodir, MANIFEST_FILE_NAME)
     self.globalConfig = GitConfig.ForUser()
+    self.localManifestWarning = False
 
     self.repoProject = MetaProject(self, 'repo',
       gitdir   = os.path.join(repodir, 'repo/.git'),
@@ -110,17 +142,19 @@ class XmlManifest(object):
     self.Override(name)
 
     try:
-      if os.path.exists(self.manifestFile):
+      if os.path.lexists(self.manifestFile):
         os.remove(self.manifestFile)
       os.symlink('manifests/%s' % name, self.manifestFile)
-    except OSError:
-      raise ManifestParseError('cannot link manifest %s' % name)
+    except OSError as e:
+      raise ManifestParseError('cannot link manifest %s: %s' % (name, str(e)))
 
   def _RemoteToXml(self, r, doc, root):
     e = doc.createElement('remote')
     root.appendChild(e)
     e.setAttribute('name', r.name)
     e.setAttribute('fetch', r.fetchUrl)
+    if r.remoteAlias is not None:
+      e.setAttribute('alias', r.remoteAlias)
     if r.reviewUrl is not None:
       e.setAttribute('review', r.reviewUrl)
 
@@ -130,9 +164,8 @@ class XmlManifest(object):
     mp = self.manifestProject
 
     groups = mp.config.GetString('manifest.groups')
-    if not groups:
-      groups = 'all'
-    groups = [x for x in re.split(r'[,\s]+', groups) if x]
+    if groups:
+      groups = [x for x in re.split(r'[,\s]+', groups) if x]
 
     doc = xml.dom.minidom.Document()
     root = doc.createElement('manifest')
@@ -148,10 +181,8 @@ class XmlManifest(object):
       notice_element.appendChild(doc.createTextNode(indented_notice))
 
     d = self.default
-    sort_remotes = list(self.remotes.keys())
-    sort_remotes.sort()
 
-    for r in sort_remotes:
+    for r in sorted(self.remotes):
       self._RemoteToXml(self.remotes[r], doc, root)
     if self.remotes:
       root.appendChild(doc.createTextNode(''))
@@ -170,6 +201,9 @@ class XmlManifest(object):
     if d.sync_c:
       have_default = True
       e.setAttribute('sync-c', 'true')
+    if d.sync_s:
+      have_default = True
+      e.setAttribute('sync-s', 'true')
     if have_default:
       root.appendChild(e)
       root.appendChild(doc.createTextNode(''))
@@ -181,8 +215,9 @@ class XmlManifest(object):
       root.appendChild(doc.createTextNode(''))
 
     def output_projects(parent, parent_node, projects):
-      for p in projects:
-        output_project(parent, parent_node, self.projects[p])
+      for project_name in projects:
+        for project in self._projects[project_name]:
+          output_project(parent, parent_node, project)
 
     def output_project(parent, parent_node, p):
       if not p.MatchesGroups(groups):
@@ -199,7 +234,10 @@ class XmlManifest(object):
       e.setAttribute('name', name)
       if relpath != name:
         e.setAttribute('path', relpath)
-      if not d.remote or p.remote.name != d.remote.name:
+      remoteName = None
+      if d.remote:
+        remoteName = d.remote.remoteAlias or d.remote.name
+      if not d.remote or p.remote.name != remoteName:
         e.setAttribute('remote', p.remote.name)
       if peg_rev:
         if self.IsMirror:
@@ -236,15 +274,15 @@ class XmlManifest(object):
       if p.sync_c:
         e.setAttribute('sync-c', 'true')
 
-      if p.subprojects:
-        sort_projects = [subp.name for subp in p.subprojects]
-        sort_projects.sort()
-        output_projects(p, e, sort_projects)
+      if p.sync_s:
+        e.setAttribute('sync-s', 'true')
 
-    sort_projects = [key for key in self.projects.keys()
-                     if not self.projects[key].parent]
-    sort_projects.sort()
-    output_projects(None, root, sort_projects)
+      if p.subprojects:
+        subprojects = set(subp.name for subp in p.subprojects)
+        output_projects(p, e, list(sorted(subprojects)))
+
+    projects = set(p.name for p in self._paths.values() if not p.parent)
+    output_projects(None, root, list(sorted(projects)))
 
     if self._repo_hooks_project:
       root.appendChild(doc.createTextNode(''))
@@ -256,10 +294,15 @@ class XmlManifest(object):
 
     doc.writexml(fd, '', '  ', '\n', 'UTF-8')
 
+  @property
+  def paths(self):
+    self._Load()
+    return self._paths
+
   @property
   def projects(self):
     self._Load()
-    return self._projects
+    return self._paths.values()
 
   @property
   def remotes(self):
@@ -290,9 +333,14 @@ class XmlManifest(object):
   def IsMirror(self):
     return self.manifestProject.config.GetBoolean('repo.mirror')
 
+  @property
+  def IsArchive(self):
+    return self.manifestProject.config.GetBoolean('repo.archive')
+
   def _Unload(self):
     self._loaded = False
     self._projects = {}
+    self._paths = {}
     self._remotes = {}
     self._default = None
     self._repo_hooks_project = None
@@ -314,9 +362,29 @@ class XmlManifest(object):
 
       local = os.path.join(self.repodir, LOCAL_MANIFEST_NAME)
       if os.path.exists(local):
+        if not self.localManifestWarning:
+          self.localManifestWarning = True
+          print('warning: %s is deprecated; put local manifests in `%s` instead'
+                % (LOCAL_MANIFEST_NAME, os.path.join(self.repodir, LOCAL_MANIFESTS_DIR_NAME)),
+                file=sys.stderr)
         nodes.append(self._ParseManifestXml(local, self.repodir))
 
-      self._ParseManifest(nodes)
+      local_dir = os.path.abspath(os.path.join(self.repodir, LOCAL_MANIFESTS_DIR_NAME))
+      try:
+        for local_file in sorted(os.listdir(local_dir)):
+          if local_file.endswith('.xml'):
+            local = os.path.join(local_dir, local_file)
+            nodes.append(self._ParseManifestXml(local, self.repodir))
+      except OSError:
+        pass
+
+      try:
+        self._ParseManifest(nodes)
+      except ManifestParseError as e:
+        # There was a problem parsing, unload ourselves in case they catch
+        # this error and try again later, we will show the correct error
+        self._Unload()
+        raise e
 
       if self.IsMirror:
         self._AddMetaProjectMirror(self.repoProject)
@@ -325,7 +393,11 @@ class XmlManifest(object):
       self._loaded = True
 
   def _ParseManifestXml(self, path, include_root):
-    root = xml.dom.minidom.parse(path)
+    try:
+      root = xml.dom.minidom.parse(path)
+    except (OSError, xml.parsers.expat.ExpatError) as e:
+      raise ManifestParseError("error parsing manifest %s: %s" % (path, e))
+
     if not root or not root.childNodes:
       raise ManifestParseError("no root node in %s" % (path,))
 
@@ -338,43 +410,47 @@ class XmlManifest(object):
     nodes = []
     for node in manifest.childNodes:  # pylint:disable=W0631
                                       # We only get here if manifest is initialised
-        if node.nodeName == 'include':
-            name = self._reqatt(node, 'name')
-            fp = os.path.join(include_root, name)
-            if not os.path.isfile(fp):
-                raise ManifestParseError, \
-                    "include %s doesn't exist or isn't a file" % \
-                    (name,)
-            try:
-                nodes.extend(self._ParseManifestXml(fp, include_root))
-            # should isolate this to the exact exception, but that's
-            # tricky.  actual parsing implementation may vary.
-            except (KeyboardInterrupt, RuntimeError, SystemExit):
-                raise
-            except Exception as e:
-                raise ManifestParseError(
-                    "failed parsing included manifest %s: %s", (name, e))
-        else:
-          nodes.append(node)
+      if node.nodeName == 'include':
+        name = self._reqatt(node, 'name')
+        fp = os.path.join(include_root, name)
+        if not os.path.isfile(fp):
+          raise ManifestParseError("include %s doesn't exist or isn't a file"
+              % (name,))
+        try:
+          nodes.extend(self._ParseManifestXml(fp, include_root))
+        # should isolate this to the exact exception, but that's
+        # tricky.  actual parsing implementation may vary.
+        except (KeyboardInterrupt, RuntimeError, SystemExit):
+          raise
+        except Exception as e:
+          raise ManifestParseError(
+              "failed parsing included manifest %s: %s", (name, e))
+      else:
+        nodes.append(node)
     return nodes
 
   def _ParseManifest(self, node_list):
     for node in itertools.chain(*node_list):
       if node.nodeName == 'remote':
         remote = self._ParseRemote(node)
-        if self._remotes.get(remote.name):
-          raise ManifestParseError(
-              'duplicate remote %s in %s' %
-              (remote.name, self.manifestFile))
-        self._remotes[remote.name] = remote
+        if remote:
+          if remote.name in self._remotes:
+            if remote != self._remotes[remote.name]:
+              raise ManifestParseError(
+                  'remote %s already exists with different attributes' %
+                  (remote.name))
+          else:
+            self._remotes[remote.name] = remote
 
     for node in itertools.chain(*node_list):
       if node.nodeName == 'default':
-        if self._default is not None:
-          raise ManifestParseError(
-              'duplicate default in %s' %
-              (self.manifestFile))
-        self._default = self._ParseDefault(node)
+        new_default = self._ParseDefault(node)
+        if self._default is None:
+          self._default = new_default
+        elif new_default != self._default:
+          raise ManifestParseError('duplicate default in %s' %
+                                   (self.manifestFile))
+
     if self._default is None:
       self._default = _Default()
 
@@ -390,22 +466,29 @@ class XmlManifest(object):
       if node.nodeName == 'manifest-server':
         url = self._reqatt(node, 'url')
         if self._manifest_server is not None:
-            raise ManifestParseError(
-                'duplicate manifest-server in %s' %
-                (self.manifestFile))
+          raise ManifestParseError(
+              'duplicate manifest-server in %s' %
+              (self.manifestFile))
         self._manifest_server = url
 
+    def recursively_add_projects(project):
+      projects = self._projects.setdefault(project.name, [])
+      if project.relpath is None:
+        raise ManifestParseError(
+            'missing path for %s in %s' %
+            (project.name, self.manifestFile))
+      if project.relpath in self._paths:
+        raise ManifestParseError(
+            'duplicate path %s in %s' %
+            (project.relpath, self.manifestFile))
+      self._paths[project.relpath] = project
+      projects.append(project)
+      for subproject in project.subprojects:
+        recursively_add_projects(subproject)
+
     for node in itertools.chain(*node_list):
       if node.nodeName == 'project':
         project = self._ParseProject(node)
-        def recursively_add_projects(project):
-          if self._projects.get(project.name):
-            raise ManifestParseError(
-                'duplicate project %s in %s' %
-                (project.name, self.manifestFile))
-          self._projects[project.name] = project
-          for subproject in project.subprojects:
-            recursively_add_projects(subproject)
         recursively_add_projects(project)
       if node.nodeName == 'repo-hooks':
         # Get the name of the project and the (space-separated) list of enabled.
@@ -420,12 +503,18 @@ class XmlManifest(object):
 
         # Store a reference to the Project.
         try:
-          self._repo_hooks_project = self._projects[repo_hooks_project]
+          repo_hooks_projects = self._projects[repo_hooks_project]
         except KeyError:
           raise ManifestParseError(
               'project %s not found for repo-hooks' %
               (repo_hooks_project))
 
+        if len(repo_hooks_projects) != 1:
+          raise ManifestParseError(
+              'internal error parsing repo-hooks in %s' %
+              (self.manifestFile))
+        self._repo_hooks_project = repo_hooks_projects[0]
+
         # Store the enabled hooks in the Project object.
         self._repo_hooks_project.enabled_repo_hooks = enabled_repo_hooks
       if node.nodeName == 'remove-project':
@@ -433,9 +522,8 @@ class XmlManifest(object):
         try:
           del self._projects[name]
         except KeyError:
-          raise ManifestParseError(
-              'project %s not found' %
-              (name))
+          raise ManifestParseError('remove-project element specifies non-existent '
+                                   'project: %s' % name)
 
         # If the manifest removes the hooks project, treat it as if it deleted
         # the repo-hooks element too.
@@ -447,7 +535,7 @@ class XmlManifest(object):
     name = None
     m_url = m.GetRemote(m.remote.name).url
     if m_url.endswith('/.git'):
-      raise ManifestParseError, 'refusing to mirror %s' % m_url
+      raise ManifestParseError('refusing to mirror %s' % m_url)
 
     if self._default and self._default.remote:
       url = self._default.remote.resolvedFetchUrl
@@ -473,11 +561,12 @@ class XmlManifest(object):
                         name = name,
                         remote = remote.ToRemoteSpec(name),
                         gitdir = gitdir,
+                        objdir = gitdir,
                         worktree = None,
                         relpath = None,
                         revisionExpr = m.revisionExpr,
                         revisionId = None)
-      self._projects[project.name] = project
+      self._projects[project.name] = [project]
 
   def _ParseRemote(self, node):
     """
@@ -504,6 +593,8 @@ class XmlManifest(object):
     if d.revisionExpr == '':
       d.revisionExpr = None
 
+    d.destBranchExpr = node.getAttribute('dest-branch') or None
+
     sync_j = node.getAttribute('sync-j')
     if sync_j == '' or sync_j is None:
       d.sync_j = 1
@@ -515,6 +606,12 @@ class XmlManifest(object):
       d.sync_c = False
     else:
       d.sync_c = sync_c.lower() in ("yes", "true", "1")
+
+    sync_s = node.getAttribute('sync-s')
+    if not sync_s:
+      d.sync_s = False
+    else:
+      d.sync_s = sync_s.lower() in ("yes", "true", "1")
     return d
 
   def _ParseNotice(self, node):
@@ -535,7 +632,7 @@ class XmlManifest(object):
 
     # Figure out minimum indentation, skipping the first line (the same line
     # as the <notice> tag)...
-    minIndent = sys.maxint
+    minIndent = sys.maxsize
     lines = notice.splitlines()
     for line in lines[1:]:
       lstrippedLine = line.lstrip()
@@ -574,25 +671,22 @@ class XmlManifest(object):
     if remote is None:
       remote = self._default.remote
     if remote is None:
-      raise ManifestParseError, \
-            "no remote for project %s within %s" % \
-            (name, self.manifestFile)
+      raise ManifestParseError("no remote for project %s within %s" %
+            (name, self.manifestFile))
 
     revisionExpr = node.getAttribute('revision')
     if not revisionExpr:
       revisionExpr = self._default.revisionExpr
     if not revisionExpr:
-      raise ManifestParseError, \
-            "no revision for project %s within %s" % \
-            (name, self.manifestFile)
+      raise ManifestParseError("no revision for project %s within %s" %
+            (name, self.manifestFile))
 
     path = node.getAttribute('path')
     if not path:
       path = name
     if path.startswith('/'):
-      raise ManifestParseError, \
-            "project %s path cannot be absolute in %s" % \
-            (name, self.manifestFile)
+      raise ManifestParseError("project %s path cannot be absolute in %s" %
+            (name, self.manifestFile))
 
     rebase = node.getAttribute('rebase')
     if not rebase:
@@ -606,25 +700,49 @@ class XmlManifest(object):
     else:
       sync_c = sync_c.lower() in ("yes", "true", "1")
 
+    sync_s = node.getAttribute('sync-s')
+    if not sync_s:
+      sync_s = self._default.sync_s
+    else:
+      sync_s = sync_s.lower() in ("yes", "true", "1")
+
+    clone_depth = node.getAttribute('clone-depth')
+    if clone_depth:
+      try:
+        clone_depth = int(clone_depth)
+        if  clone_depth <= 0:
+          raise ValueError()
+      except ValueError:
+        raise ManifestParseError('invalid clone-depth %s in %s' %
+                                 (clone_depth, self.manifestFile))
+
+    dest_branch = node.getAttribute('dest-branch') or self._default.destBranchExpr
+
     upstream = node.getAttribute('upstream')
 
     groups = ''
     if node.hasAttribute('groups'):
       groups = node.getAttribute('groups')
-    groups = [x for x in re.split('[,\s]+', groups) if x]
+    groups = [x for x in re.split(r'[,\s]+', groups) if x]
 
     if parent is None:
-      relpath, worktree, gitdir = self.GetProjectPaths(name, path)
+      relpath, worktree, gitdir, objdir = self.GetProjectPaths(name, path)
     else:
-      relpath, worktree, gitdir = self.GetSubprojectPaths(parent, path)
+      relpath, worktree, gitdir, objdir = \
+          self.GetSubprojectPaths(parent, name, path)
 
     default_groups = ['all', 'name:%s' % name, 'path:%s' % relpath]
     groups.extend(set(default_groups).difference(groups))
 
+    if self.IsMirror and node.hasAttribute('force-path'):
+      if node.getAttribute('force-path').lower() in ("yes", "true", "1"):
+        gitdir = os.path.join(self.topdir, '%s.git' % path)
+
     project = Project(manifest = self,
                       name = name,
                       remote = remote.ToRemoteSpec(name),
                       gitdir = gitdir,
+                      objdir = objdir,
                       worktree = worktree,
                       relpath = relpath,
                       revisionExpr = revisionExpr,
@@ -632,8 +750,11 @@ class XmlManifest(object):
                       rebase = rebase,
                       groups = groups,
                       sync_c = sync_c,
+                      sync_s = sync_s,
+                      clone_depth = clone_depth,
                       upstream = upstream,
-                      parent = parent)
+                      parent = parent,
+                      dest_branch = dest_branch)
 
     for n in node.childNodes:
       if n.nodeName == 'copyfile':
@@ -650,10 +771,15 @@ class XmlManifest(object):
     if self.IsMirror:
       worktree = None
       gitdir = os.path.join(self.topdir, '%s.git' % name)
+      objdir = gitdir
     else:
       worktree = os.path.join(self.topdir, path).replace('\\', '/')
       gitdir = os.path.join(self.repodir, 'projects', '%s.git' % path)
-    return relpath, worktree, gitdir
+      objdir = os.path.join(self.repodir, 'project-objects', '%s.git' % name)
+    return relpath, worktree, gitdir, objdir
+
+  def GetProjectsWithName(self, name):
+    return self._projects.get(name, [])
 
   def GetSubprojectName(self, parent, submodule_path):
     return os.path.join(parent.name, submodule_path)
@@ -664,14 +790,15 @@ class XmlManifest(object):
   def _UnjoinRelpath(self, parent_relpath, relpath):
     return os.path.relpath(relpath, parent_relpath)
 
-  def GetSubprojectPaths(self, parent, path):
+  def GetSubprojectPaths(self, parent, name, path):
     relpath = self._JoinRelpath(parent.relpath, path)
     gitdir = os.path.join(parent.gitdir, 'subprojects', '%s.git' % path)
+    objdir = os.path.join(parent.gitdir, 'subproject-objects', '%s.git' % name)
     if self.IsMirror:
       worktree = None
     else:
       worktree = os.path.join(parent.worktree, path).replace('\\', '/')
-    return relpath, worktree, gitdir
+    return relpath, worktree, gitdir, objdir
 
   def _ParseCopyFile(self, project, node):
     src = self._reqatt(node, 'src')
@@ -689,7 +816,8 @@ class XmlManifest(object):
     except ManifestParseError:
       keep = "true"
     if keep != "true" and keep != "false":
-      raise ManifestParseError, "optional \"keep\" attribute must be \"true\" or \"false\""
+      raise ManifestParseError('optional "keep" attribute must be '
+            '"true" or "false"')
     project.AddAnnotation(name, value, keep)
 
   def _get_remote(self, node):
@@ -699,9 +827,8 @@ class XmlManifest(object):
 
     v = self._remotes.get(name)
     if not v:
-      raise ManifestParseError, \
-            "remote %s not defined in %s" % \
-            (name, self.manifestFile)
+      raise ManifestParseError("remote %s not defined in %s" %
+            (name, self.manifestFile))
     return v
 
   def _reqatt(self, node, attname):
@@ -710,7 +837,6 @@ class XmlManifest(object):
     """
     v = node.getAttribute(attname)
     if not v:
-      raise ManifestParseError, \
-            "no %s in <%s> within %s" % \
-            (attname, node.nodeName, self.manifestFile)
+      raise ManifestParseError("no %s in <%s> within %s" %
+            (attname, node.nodeName, self.manifestFile))
     return v

pager.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import os
 import select
 import sys
@@ -49,7 +50,7 @@ def RunPager(globalConfig):
 
     _BecomePager(pager)
   except Exception:
-    print >>sys.stderr, "fatal: cannot start pager '%s'" % pager
+    print("fatal: cannot start pager '%s'" % pager, file=sys.stderr)
     sys.exit(255)
 
 def _SelectPager(globalConfig):

project.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import traceback
 import errno
 import filecmp
@@ -22,18 +23,26 @@ import shutil
 import stat
 import subprocess
 import sys
+import tarfile
 import tempfile
 import time
 
 from color import Coloring
-from git_command import GitCommand
+from git_command import GitCommand, git_require
 from git_config import GitConfig, IsId, GetSchemeFromUrl, ID_RE
 from error import GitError, HookError, UploadError
 from error import ManifestInvalidRevisionError
+from error import NoManifestException
 from trace import IsTrace, Trace
 
 from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M
 
+from pyversion import is_python3
+if not is_python3():
+  # pylint:disable=W0622
+  input = raw_input
+  # pylint:enable=W0622
+
 def _lwrite(path, content):
   lock = '%s.lock' % path
 
@@ -51,7 +60,7 @@ def _lwrite(path, content):
 
 def _error(fmt, *args):
   msg = fmt % args
-  print >>sys.stderr, 'error: %s' % msg
+  print('error: %s' % msg, file=sys.stderr)
 
 def not_rev(r):
   return '^' + r
@@ -74,9 +83,9 @@ def _ProjectHooks():
   """
   global _project_hook_list
   if _project_hook_list is None:
-    d = os.path.abspath(os.path.dirname(__file__))
+    d = os.path.realpath(os.path.abspath(os.path.dirname(__file__)))
     d = os.path.join(d , 'hooks')
-    _project_hook_list = map(lambda x: os.path.join(d, x), os.listdir(d))
+    _project_hook_list = [os.path.join(d, x) for x in os.listdir(d)]
   return _project_hook_list
 
 
@@ -149,11 +158,12 @@ class ReviewableBranch(object):
       R_HEADS + self.name,
       '--')
 
-  def UploadForReview(self, people, auto_topic=False, draft=False):
+  def UploadForReview(self, people, auto_topic=False, draft=False, dest_branch=None):
     self.project.UploadForReview(self.name,
                                  people,
                                  auto_topic=auto_topic,
-                                 draft=draft)
+                                 draft=draft,
+                                 dest_branch=dest_branch)
 
   def GetPublishedRefs(self):
     refs = {}
@@ -359,8 +369,8 @@ class RepoHook(object):
                  'Do you want to allow this script to run '
                  '(yes/yes-never-ask-again/NO)? ') % (
                  self._GetMustVerb(), self._script_fullpath)
-      response = raw_input(prompt).lower()
-      print
+      response = input(prompt).lower()
+      print()
 
       # User is doing a one-time approval.
       if response in ('y', 'yes'):
@@ -478,6 +488,7 @@ class Project(object):
                name,
                remote,
                gitdir,
+               objdir,
                worktree,
                relpath,
                revisionExpr,
@@ -485,9 +496,12 @@ class Project(object):
                rebase = True,
                groups = None,
                sync_c = False,
+               sync_s = False,
+               clone_depth = None,
                upstream = None,
                parent = None,
-               is_derived = False):
+               is_derived = False,
+               dest_branch = None):
     """Init a Project object.
 
     Args:
@@ -495,6 +509,7 @@ class Project(object):
       name: The `name` attribute of manifest.xml's project element.
       remote: RemoteSpec object specifying its remote's properties.
       gitdir: Absolute path of git directory.
+      objdir: Absolute path of directory to store git objects.
       worktree: Absolute path of git working tree.
       relpath: Relative path of git working tree to repo's top directory.
       revisionExpr: The `revision` attribute of manifest.xml's project element.
@@ -502,15 +517,18 @@ class Project(object):
       rebase: The `rebase` attribute of manifest.xml's project element.
       groups: The `groups` attribute of manifest.xml's project element.
       sync_c: The `sync-c` attribute of manifest.xml's project element.
+      sync_s: The `sync-s` attribute of manifest.xml's project element.
       upstream: The `upstream` attribute of manifest.xml's project element.
       parent: The parent Project object.
       is_derived: False if the project was explicitly defined in the manifest;
                   True if the project is a discovered submodule.
+      dest_branch: The branch to which to push changes for review by default.
     """
     self.manifest = manifest
     self.name = name
     self.remote = remote
     self.gitdir = gitdir.replace('\\', '/')
+    self.objdir = objdir.replace('\\', '/')
     if worktree:
       self.worktree = worktree.replace('\\', '/')
     else:
@@ -528,6 +546,8 @@ class Project(object):
     self.rebase = rebase
     self.groups = groups
     self.sync_c = sync_c
+    self.sync_s = sync_s
+    self.clone_depth = clone_depth
     self.upstream = upstream
     self.parent = parent
     self.is_derived = is_derived
@@ -541,20 +561,18 @@ class Project(object):
                     defaults =  self.manifest.globalConfig)
 
     if self.worktree:
-      self.work_git = self._GitGetByExec(self, bare=False)
+      self.work_git = self._GitGetByExec(self, bare=False, gitdir=gitdir)
     else:
       self.work_git = None
-    self.bare_git = self._GitGetByExec(self, bare=True)
+    self.bare_git = self._GitGetByExec(self, bare=True, gitdir=gitdir)
     self.bare_ref = GitRefs(gitdir)
+    self.bare_objdir = self._GitGetByExec(self, bare=True, gitdir=objdir)
+    self.dest_branch = dest_branch
 
     # This will be filled in if a project is later identified to be the
     # project containing repo hooks.
     self.enabled_repo_hooks = []
 
-  @property
-  def Registered(self):
-    return self.parent and not self.is_derived
-
   @property
   def Derived(self):
     return self.is_derived
@@ -588,7 +606,7 @@ class Project(object):
                                '--unmerged',
                                '--ignore-missing',
                                '--refresh')
-    if self.work_git.DiffZ('diff-index','-M','--cached',HEAD):
+    if self.work_git.DiffZ('diff-index', '-M', '--cached', HEAD):
       return True
     if self.work_git.DiffZ('diff-files'):
       return True
@@ -617,14 +635,14 @@ class Project(object):
     return self._userident_email
 
   def _LoadUserIdentity(self):
-      u = self.bare_git.var('GIT_COMMITTER_IDENT')
-      m = re.compile("^(.*) <([^>]*)> ").match(u)
-      if m:
-        self._userident_name = m.group(1)
-        self._userident_email = m.group(2)
-      else:
-        self._userident_name = ''
-        self._userident_email = ''
+    u = self.bare_git.var('GIT_COMMITTER_IDENT')
+    m = re.compile("^(.*) <([^>]*)> ").match(u)
+    if m:
+      self._userident_name = m.group(1)
+      self._userident_email = m.group(2)
+    else:
+      self._userident_name = ''
+      self._userident_email = ''
 
   def GetRemote(self, name):
     """Get the configuration for a single remote.
@@ -643,7 +661,7 @@ class Project(object):
     all_refs = self._allrefs
     heads = {}
 
-    for name, ref_id in all_refs.iteritems():
+    for name, ref_id in all_refs.items():
       if name.startswith(R_HEADS):
         name = name[len(R_HEADS):]
         b = self.GetBranch(name)
@@ -652,7 +670,7 @@ class Project(object):
         b.revision = ref_id
         heads[name] = b
 
-    for name, ref_id in all_refs.iteritems():
+    for name, ref_id in all_refs.items():
       if name.startswith(R_PUB):
         name = name[len(R_PUB):]
         b = heads.get(name)
@@ -671,9 +689,14 @@ class Project(object):
        project_groups: "all,group1,group2"
        manifest_groups: "-group1,group2"
        the project will be matched.
+
+       The special manifest group "default" will match any project that
+       does not have the special project group "notdefault"
     """
-    expanded_manifest_groups = manifest_groups or ['all', '-notdefault']
+    expanded_manifest_groups = manifest_groups or ['default']
     expanded_project_groups = ['all'] + (self.groups or [])
+    if not 'notdefault' in expanded_project_groups:
+      expanded_project_groups += ['default']
 
     matched = False
     for group in expanded_manifest_groups:
@@ -716,9 +739,9 @@ class Project(object):
     if not os.path.isdir(self.worktree):
       if output_redir == None:
         output_redir = sys.stdout
-      print >>output_redir, ''
-      print >>output_redir, 'project %s/' % self.relpath
-      print >>output_redir, '  missing (run "repo sync")'
+      print(file=output_redir)
+      print('project %s/' % self.relpath, file=output_redir)
+      print('  missing (run "repo sync")', file=output_redir)
       return
 
     self.work_git.update_index('-q',
@@ -753,10 +776,7 @@ class Project(object):
     paths.extend(df.keys())
     paths.extend(do)
 
-    paths = list(set(paths))
-    paths.sort()
-
-    for p in paths:
+    for p in sorted(set(paths)):
       try:
         i = di[p]
       except KeyError:
@@ -818,7 +838,7 @@ class Project(object):
         out.project('project %s/' % self.relpath)
         out.nl()
         has_diff = True
-      print line[:-1]
+      print(line[:-1])
     p.Wait()
 
 
@@ -848,13 +868,13 @@ class Project(object):
       all_refs = self._allrefs
     heads = set()
     canrm = {}
-    for name, ref_id in all_refs.iteritems():
+    for name, ref_id in all_refs.items():
       if name.startswith(R_HEADS):
         heads.add(name)
       elif name.startswith(R_PUB):
         canrm[name] = ref_id
 
-    for name, ref_id in canrm.iteritems():
+    for name, ref_id in canrm.items():
       n = name[len(R_PUB):]
       if R_HEADS + n not in heads:
         self.bare_git.DeleteRef(name, ref_id)
@@ -865,14 +885,14 @@ class Project(object):
     heads = {}
     pubed = {}
 
-    for name, ref_id in self._allrefs.iteritems():
+    for name, ref_id in self._allrefs.items():
       if name.startswith(R_HEADS):
         heads[name[len(R_HEADS):]] = ref_id
       elif name.startswith(R_PUB):
         pubed[name[len(R_PUB):]] = ref_id
 
     ready = []
-    for branch, ref_id in heads.iteritems():
+    for branch, ref_id in heads.items():
       if branch in pubed and pubed[branch] == ref_id:
         continue
       if selected_branch and branch != selected_branch:
@@ -897,7 +917,8 @@ class Project(object):
   def UploadForReview(self, branch=None,
                       people=([],[]),
                       auto_topic=False,
-                      draft=False):
+                      draft=False,
+                      dest_branch=None):
     """Uploads the named branch for code review.
     """
     if branch is None:
@@ -911,7 +932,10 @@ class Project(object):
     if not branch.remote.review:
       raise GitError('remote %s has no review url' % branch.remote.name)
 
-    dest_branch = branch.merge
+    if dest_branch is None:
+      dest_branch = self.dest_branch
+    if dest_branch is None:
+      dest_branch = branch.merge
     if not dest_branch.startswith(R_HEADS):
       dest_branch = R_HEADS + dest_branch
 
@@ -945,6 +969,11 @@ class Project(object):
                                   dest_branch)
     if auto_topic:
       ref_spec = ref_spec + '/' + branch.name
+    if not url.startswith('ssh://'):
+      rp = ['r=%s' % p for p in people[0]] + \
+           ['cc=%s' % p for p in people[1]]
+      if rp:
+        ref_spec = ref_spec + '%' + ','.join(rp)
     cmd.append(ref_spec)
 
     if GitCommand(self, cmd, bare = True).Wait() != 0:
@@ -958,18 +987,68 @@ class Project(object):
 
 ## Sync ##
 
+  def _ExtractArchive(self, tarpath, path=None):
+    """Extract the given tar on its current location
+
+    Args:
+        - tarpath: The path to the actual tar file
+
+    """
+    try:
+      with tarfile.open(tarpath, 'r') as tar:
+        tar.extractall(path=path)
+        return True
+    except (IOError, tarfile.TarError) as e:
+      print("error: Cannot extract archive %s: "
+            "%s" % (tarpath, str(e)), file=sys.stderr)
+    return False
+
   def Sync_NetworkHalf(self,
       quiet=False,
       is_new=None,
       current_branch_only=False,
-      clone_bundle=True):
+      clone_bundle=True,
+      no_tags=False,
+      archive=False):
     """Perform only the network IO portion of the sync process.
        Local working directory/branch state is not affected.
     """
+    if archive and not isinstance(self, MetaProject):
+      if self.remote.url.startswith(('http://', 'https://')):
+        print("error: %s: Cannot fetch archives from http/https "
+              "remotes." % self.name, file=sys.stderr)
+        return False
+
+      name = self.relpath.replace('\\', '/')
+      name = name.replace('/', '_')
+      tarpath = '%s.tar' % name
+      topdir = self.manifest.topdir
+
+      try:
+        self._FetchArchive(tarpath, cwd=topdir)
+      except GitError as e:
+        print('error: %s' % str(e), file=sys.stderr)
+        return False
+
+      # From now on, we only need absolute tarpath
+      tarpath = os.path.join(topdir, tarpath)
+
+      if not self._ExtractArchive(tarpath, path=topdir):
+        return False
+      try:
+        os.remove(tarpath)
+      except OSError as e:
+        print("warn: Cannot remove archive %s: "
+              "%s" % (tarpath, str(e)), file=sys.stderr)
+      self._CopyFiles()
+      return True
+
     if is_new is None:
       is_new = not self.Exists
     if is_new:
       self._InitGitDir()
+    else:
+      self._UpdateHooks()
     self._InitRemote()
 
     if is_new:
@@ -1000,7 +1079,8 @@ class Project(object):
         current_branch_only = True
 
     if not self._RemoteFetch(initial=is_new, quiet=quiet, alt_dir=alt_dir,
-                             current_branch_only=current_branch_only):
+                             current_branch_only=current_branch_only,
+                             no_tags=no_tags):
       return False
 
     if self.worktree:
@@ -1041,11 +1121,15 @@ class Project(object):
     """Perform only the local IO portion of the sync process.
        Network access is not required.
     """
+    self._InitWorkTree()
     all_refs = self.bare_ref.all
     self.CleanPublishedCache(all_refs)
     revid = self.GetRevisionId(all_refs)
 
-    self._InitWorkTree()
+    def _doff():
+      self._FastForward(revid)
+      self._CopyFiles()
+
     head = self.work_git.GetHead()
     if head.startswith(R_HEADS):
       branch = head[len(R_HEADS):]
@@ -1123,9 +1207,6 @@ class Project(object):
         # All published commits are merged, and thus we are a
         # strict subset.  We can fast-forward safely.
         #
-        def _doff():
-          self._FastForward(revid)
-          self._CopyFiles()
         syncbuf.later1(self, _doff)
         return
 
@@ -1136,7 +1217,7 @@ class Project(object):
     last_mine = None
     cnt_mine = 0
     for commit in local_changes:
-      commit_id, committer_email = commit.split(' ', 1)
+      commit_id, committer_email = commit.decode('utf-8').split(' ', 1)
       if committer_email == self.UserEmail:
         last_mine = commit_id
         cnt_mine += 1
@@ -1188,9 +1269,6 @@ class Project(object):
         syncbuf.fail(self, e)
         return
     else:
-      def _doff():
-        self._FastForward(revid)
-        self._CopyFiles()
       syncbuf.later1(self, _doff)
 
   def AddCopyFile(self, src, dest, absdest):
@@ -1210,7 +1288,6 @@ class Project(object):
     cmd = ['fetch', remote.name]
     cmd.append('refs/changes/%2.2d/%d/%d' \
                % (change_id % 100, change_id, patch_id))
-    cmd.extend(map(lambda x: str(x), remote.fetch))
     if GitCommand(self, cmd, bare=True).Wait() != 0:
       return None
     return DownloadedChange(self,
@@ -1439,8 +1516,8 @@ class Project(object):
         submodules.append((sub_rev, sub_path, sub_url))
       return submodules
 
-    re_path = re.compile(r'submodule.(\w+).path')
-    re_url = re.compile(r'submodule.(\w+).url')
+    re_path = re.compile(r'^submodule\.([^.]+)\.path=(.*)$')
+    re_url = re.compile(r'^submodule\.([^.]+)\.url=(.*)$')
     def parse_gitmodules(gitdir, rev):
       cmd = ['cat-file', 'blob', '%s:.gitmodules' % rev]
       try:
@@ -1473,19 +1550,19 @@ class Project(object):
       for line in gitmodules_lines:
         if not line:
           continue
-        key, value = line.split('=')
-        m = re_path.match(key)
+        m = re_path.match(line)
         if m:
           names.add(m.group(1))
-          paths[m.group(1)] = value
+          paths[m.group(1)] = m.group(2)
           continue
-        m = re_url.match(key)
+        m = re_url.match(line)
         if m:
           names.add(m.group(1))
-          urls[m.group(1)] = value
+          urls[m.group(1)] = m.group(2)
           continue
       names = sorted(names)
-      return [paths[name] for name in names], [urls[name] for name in names]
+      return ([paths.get(name, '') for name in names],
+              [urls.get(name, '') for name in names])
 
     def git_ls_tree(gitdir, rev, paths):
       cmd = ['ls-tree', rev, '--']
@@ -1519,13 +1596,13 @@ class Project(object):
       return result
     for rev, path, url in self._GetSubmodules():
       name = self.manifest.GetSubprojectName(self, path)
-      project = self.manifest.projects.get(name)
-      if project and project.Registered:
-        # If it has been registered, skip it because we are searching
-        # derived subprojects, but search for its derived subprojects.
+      relpath, worktree, gitdir, objdir = \
+          self.manifest.GetSubprojectPaths(self, name, path)
+      project = self.manifest.paths.get(relpath)
+      if project:
         result.extend(project.GetDerivedSubprojects())
         continue
-      relpath, worktree, gitdir = self.manifest.GetSubprojectPaths(self, path)
+
       remote = RemoteSpec(self.remote.name,
                           url = url,
                           review = self.remote.review)
@@ -1533,6 +1610,7 @@ class Project(object):
                            name = name,
                            remote = remote,
                            gitdir = gitdir,
+                           objdir = objdir,
                            worktree = worktree,
                            relpath = relpath,
                            revisionExpr = self.revisionExpr,
@@ -1540,6 +1618,7 @@ class Project(object):
                            rebase = self.rebase,
                            groups = self.groups,
                            sync_c = self.sync_c,
+                           sync_s = self.sync_s,
                            parent = self,
                            is_derived = True)
       result.append(subproject)
@@ -1549,24 +1628,45 @@ class Project(object):
 
 ## Direct Git Commands ##
 
+  def _FetchArchive(self, tarpath, cwd=None):
+    cmd = ['archive', '-v', '-o', tarpath]
+    cmd.append('--remote=%s' % self.remote.url)
+    cmd.append('--prefix=%s/' % self.relpath)
+    cmd.append(self.revisionExpr)
+
+    command = GitCommand(self, cmd, cwd=cwd,
+                         capture_stdout=True,
+                         capture_stderr=True)
+
+    if command.Wait() != 0:
+      raise GitError('git archive %s: %s' % (self.name, command.stderr))
+
   def _RemoteFetch(self, name=None,
                    current_branch_only=False,
                    initial=False,
                    quiet=False,
-                   alt_dir=None):
+                   alt_dir=None,
+                   no_tags=False):
 
     is_sha1 = False
     tag_name = None
 
     def CheckForSha1():
-        try:
-          # if revision (sha or tag) is not present then following function
-          # throws an error.
-          self.bare_git.rev_parse('--verify', '%s^0' % self.revisionExpr)
-          return True
-        except GitError:
-          # There is no such persistent revision. We have to fetch it.
-          return False
+      try:
+        # if revision (sha or tag) is not present then following function
+        # throws an error.
+        self.bare_git.rev_parse('--verify', '%s^0' % self.revisionExpr)
+        return True
+      except GitError:
+        # There is no such persistent revision. We have to fetch it.
+        return False
+
+    if self.clone_depth:
+      depth = self.clone_depth
+    else:
+      depth = self.manifest.manifestProject.config.GetString('repo.depth')
+    if depth:
+      current_branch_only = True
 
     if current_branch_only:
       if ID_RE.match(self.revisionExpr) is not None:
@@ -1599,7 +1699,7 @@ class Project(object):
         ids = set(all_refs.values())
         tmp = set()
 
-        for r, ref_id in GitRefs(ref_dir).all.iteritems():
+        for r, ref_id in GitRefs(ref_dir).all.items():
           if r not in all_refs:
             if r.startswith(R_TAGS) or remote.WritesTo(r):
               all_refs[r] = ref_id
@@ -1614,13 +1714,10 @@ class Project(object):
           ids.add(ref_id)
           tmp.add(r)
 
-        ref_names = list(all_refs.keys())
-        ref_names.sort()
-
         tmp_packed = ''
         old_packed = ''
 
-        for r in ref_names:
+        for r in sorted(all_refs):
           line = '%s %s\n' % (all_refs[r], r)
           tmp_packed += line
           if r not in tmp:
@@ -1634,7 +1731,6 @@ class Project(object):
 
     # The --depth option only affects the initial fetch; after that we'll do
     # full fetches of changes.
-    depth = self.manifest.manifestProject.config.GetString('repo.depth')
     if depth and initial:
       cmd.append('--depth=%s' % depth)
 
@@ -1646,8 +1742,13 @@ class Project(object):
 
     if not current_branch_only:
       # Fetch whole repo
-      cmd.append('--tags')
-      cmd.append((u'+refs/heads/*:') + remote.ToLocal('refs/heads/*'))
+      # If using depth then we should not get all the tags since they may
+      # be outside of the depth.
+      if no_tags or depth:
+        cmd.append('--no-tags')
+      else:
+        cmd.append('--tags')
+      cmd.append(str((u'+refs/heads/*:') + remote.ToLocal('refs/heads/*')))
     elif tag_name is not None:
       cmd.append('tag')
       cmd.append(tag_name)
@@ -1657,7 +1758,7 @@ class Project(object):
         branch = self.upstream
       if branch.startswith(R_HEADS):
         branch = branch[len(R_HEADS):]
-      cmd.append((u'+refs/heads/%s:' % branch) + remote.ToLocal('refs/heads/%s' % branch))
+      cmd.append(str((u'+refs/heads/%s:' % branch) + remote.ToLocal('refs/heads/%s' % branch)))
 
     ok = False
     for _i in range(2):
@@ -1691,15 +1792,14 @@ class Project(object):
     return ok
 
   def _ApplyCloneBundle(self, initial=False, quiet=False):
-    if initial and self.manifest.manifestProject.config.GetString('repo.depth'):
+    if initial and (self.manifest.manifestProject.config.GetString('repo.depth') or self.clone_depth):
       return False
 
     remote = self.GetRemote(self.remote.name)
     bundle_url = remote.url + '/clone.bundle'
     bundle_url = GitConfig.ForUser().UrlInsteadOf(bundle_url)
-    if GetSchemeFromUrl(bundle_url) in ('persistent-http', 'persistent-https'):
-      bundle_url = bundle_url[len('persistent-'):]
-    if GetSchemeFromUrl(bundle_url) not in ('http', 'https'):
+    if GetSchemeFromUrl(bundle_url) not in (
+        'http', 'https', 'persistent-http', 'persistent-https'):
       return False
 
     bundle_dst = os.path.join(self.gitdir, 'clone.bundle')
@@ -1748,6 +1848,11 @@ class Project(object):
         os.remove(tmpPath)
     if 'http_proxy' in os.environ and 'darwin' == sys.platform:
       cmd += ['--proxy', os.environ['http_proxy']]
+    cookiefile = self._GetBundleCookieFile(srcUrl)
+    if cookiefile:
+      cmd += ['--cookie', cookiefile]
+    if srcUrl.startswith('persistent-'):
+      srcUrl = srcUrl[len('persistent-'):]
     cmd += [srcUrl]
 
     if IsTrace():
@@ -1765,11 +1870,12 @@ class Project(object):
       # returned another error with the HTTP error code being 400 or above.
       # This return code only appears if -f, --fail is used.
       if not quiet:
-        print >> sys.stderr, "Server does not provide clone.bundle; ignoring."
+        print("Server does not provide clone.bundle; ignoring.",
+              file=sys.stderr)
       return False
 
     if os.path.exists(tmpPath):
-      if curlret == 0 and os.stat(tmpPath).st_size > 16:
+      if curlret == 0 and self._IsValidBundle(tmpPath):
         os.rename(tmpPath, dstPath)
         return True
       else:
@@ -1778,6 +1884,46 @@ class Project(object):
     else:
       return False
 
+  def _IsValidBundle(self, path):
+    try:
+      with open(path) as f:
+        if f.read(16) == '# v2 git bundle\n':
+          return True
+        else:
+          print("Invalid clone.bundle file; ignoring.", file=sys.stderr)
+          return False
+    except OSError:
+      return False
+
+  def _GetBundleCookieFile(self, url):
+    if url.startswith('persistent-'):
+      try:
+        p = subprocess.Popen(
+            ['git-remote-persistent-https', '-print_config', url],
+            stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE)
+        p.stdin.close()  # Tell subprocess it's ok to close.
+        prefix = 'http.cookiefile='
+        cookiefile = None
+        for line in p.stdout:
+          line = line.strip()
+          if line.startswith(prefix):
+            cookiefile = line[len(prefix):]
+            break
+        if p.wait():
+          err_msg = p.stderr.read()
+          if ' -print_config' in err_msg:
+            pass  # Persistent proxy doesn't support -print_config.
+          else:
+            print(err_msg, file=sys.stderr)
+        if cookiefile:
+          return cookiefile
+      except OSError as e:
+        if e.errno == errno.ENOENT:
+          pass  # No persistent proxy.
+        raise
+    return GitConfig.ForUser().GetString('http.cookiefile')
+
   def _Checkout(self, rev, quiet=False):
     cmd = ['checkout']
     if quiet:
@@ -1828,16 +1974,26 @@ class Project(object):
     if GitCommand(self, cmd).Wait() != 0:
       raise GitError('%s merge %s ' % (self.name, head))
 
-  def _InitGitDir(self):
+  def _InitGitDir(self, mirror_git=None):
     if not os.path.exists(self.gitdir):
-      os.makedirs(self.gitdir)
-      self.bare_git.init()
+
+      # Initialize the bare repository, which contains all of the objects.
+      if not os.path.exists(self.objdir):
+        os.makedirs(self.objdir)
+        self.bare_objdir.init()
+
+      # If we have a separate directory to hold refs, initialize it as well.
+      if self.objdir != self.gitdir:
+        os.makedirs(self.gitdir)
+        self._ReferenceGitDir(self.objdir, self.gitdir, share_refs=False,
+                              copy_all=True)
 
       mp = self.manifest.manifestProject
-      ref_dir = mp.config.GetString('repo.reference')
+      ref_dir = mp.config.GetString('repo.reference') or ''
 
-      if ref_dir:
-        mirror_git = os.path.join(ref_dir, self.name + '.git')
+      if ref_dir or mirror_git:
+        if not mirror_git:
+          mirror_git = os.path.join(ref_dir, self.name + '.git')
         repo_git = os.path.join(ref_dir, '.repo', 'projects',
                                 self.relpath + '.git')
 
@@ -1854,11 +2010,21 @@ class Project(object):
           _lwrite(os.path.join(self.gitdir, 'objects/info/alternates'),
                   os.path.join(ref_dir, 'objects') + '\n')
 
+      self._UpdateHooks()
+
+      m = self.manifest.manifestProject.config
+      for key in ['user.name', 'user.email']:
+        if m.Has(key, include_defaults = False):
+          self.config.SetString(key, m.GetString(key))
       if self.manifest.IsMirror:
         self.config.SetString('core.bare', 'true')
       else:
         self.config.SetString('core.bare', None)
 
+  def _UpdateHooks(self):
+    if os.path.exists(self.gitdir):
+      # Always recreate hooks since they can have been changed
+      # since the latest update.
       hooks = self._gitdir_path('hooks')
       try:
         to_rm = os.listdir(hooks)
@@ -1868,13 +2034,8 @@ class Project(object):
         os.remove(os.path.join(hooks, old_hook))
       self._InitHooks()
 
-      m = self.manifest.manifestProject.config
-      for key in ['user.name', 'user.email']:
-        if m.Has(key, include_defaults = False):
-          self.config.SetString(key, m.GetString(key))
-
   def _InitHooks(self):
-    hooks = self._gitdir_path('hooks')
+    hooks = os.path.realpath(self._gitdir_path('hooks'))
     if not os.path.exists(hooks):
       os.makedirs(hooks)
     for stock_hook in _ProjectHooks():
@@ -1941,33 +2102,61 @@ class Project(object):
         msg = 'manifest set to %s' % self.revisionExpr
         self.bare_git.symbolic_ref('-m', msg, ref, dst)
 
+  def _ReferenceGitDir(self, gitdir, dotgit, share_refs, copy_all):
+    """Update |dotgit| to reference |gitdir|, using symlinks where possible.
+
+    Args:
+      gitdir: The bare git repository. Must already be initialized.
+      dotgit: The repository you would like to initialize.
+      share_refs: If true, |dotgit| will store its refs under |gitdir|.
+          Only one work tree can store refs under a given |gitdir|.
+      copy_all: If true, copy all remaining files from |gitdir| -> |dotgit|.
+          This saves you the effort of initializing |dotgit| yourself.
+    """
+    # These objects can be shared between several working trees.
+    symlink_files = ['description', 'info']
+    symlink_dirs = ['hooks', 'objects', 'rr-cache', 'svn']
+    if share_refs:
+      # These objects can only be used by a single working tree.
+      symlink_files += ['config', 'packed-refs']
+      symlink_dirs += ['logs', 'refs']
+    to_symlink = symlink_files + symlink_dirs
+
+    to_copy = []
+    if copy_all:
+      to_copy = os.listdir(gitdir)
+
+    for name in set(to_copy).union(to_symlink):
+      try:
+        src = os.path.realpath(os.path.join(gitdir, name))
+        dst = os.path.realpath(os.path.join(dotgit, name))
+
+        if os.path.lexists(dst) and not os.path.islink(dst):
+          raise GitError('cannot overwrite a local work tree')
+
+        # If the source dir doesn't exist, create an empty dir.
+        if name in symlink_dirs and not os.path.lexists(src):
+          os.makedirs(src)
+
+        if name in to_symlink:
+          os.symlink(os.path.relpath(src, os.path.dirname(dst)), dst)
+        elif copy_all and not os.path.islink(dst):
+          if os.path.isdir(src):
+            shutil.copytree(src, dst)
+          elif os.path.isfile(src):
+            shutil.copy(src, dst)
+      except OSError as e:
+        if e.errno == errno.EPERM:
+          raise GitError('filesystem must support symlinks')
+        else:
+          raise
+
   def _InitWorkTree(self):
     dotgit = os.path.join(self.worktree, '.git')
     if not os.path.exists(dotgit):
       os.makedirs(dotgit)
-
-      for name in ['config',
-                   'description',
-                   'hooks',
-                   'info',
-                   'logs',
-                   'objects',
-                   'packed-refs',
-                   'refs',
-                   'rr-cache',
-                   'svn']:
-        try:
-          src = os.path.join(self.gitdir, name)
-          dst = os.path.join(dotgit, name)
-          if os.path.islink(dst) or not os.path.exists(dst):
-            os.symlink(os.path.relpath(src, os.path.dirname(dst)), dst)
-          else:
-            raise GitError('cannot overwrite a local work tree')
-        except OSError as e:
-          if e.errno == errno.EPERM:
-            raise GitError('filesystem must support symlinks')
-          else:
-            raise
+      self._ReferenceGitDir(self.gitdir, dotgit, share_refs=True,
+                            copy_all=False)
 
       _lwrite(os.path.join(dotgit, HEAD), '%s\n' % self.GetRevisionId())
 
@@ -1977,14 +2166,10 @@ class Project(object):
       if GitCommand(self, cmd).Wait() != 0:
         raise GitError("cannot initialize work tree")
 
-      rr_cache = os.path.join(self.gitdir, 'rr-cache')
-      if not os.path.exists(rr_cache):
-        os.makedirs(rr_cache)
-
       self._CopyFiles()
 
   def _gitdir_path(self, path):
-    return os.path.join(self.gitdir, path)
+    return os.path.realpath(os.path.join(self.gitdir, path))
 
   def _revlist(self, *args, **kw):
     a = []
@@ -1997,9 +2182,10 @@ class Project(object):
     return self.bare_ref.all
 
   class _GitGetByExec(object):
-    def __init__(self, project, bare):
+    def __init__(self, project, bare, gitdir):
       self._project = project
       self._bare = bare
+      self._gitdir = gitdir
 
     def LsOthers(self):
       p = GitCommand(self._project,
@@ -2008,12 +2194,14 @@ class Project(object):
                       '--others',
                       '--exclude-standard'],
                      bare = False,
+                     gitdir=self._gitdir,
                      capture_stdout = True,
                      capture_stderr = True)
       if p.Wait() == 0:
         out = p.stdout
         if out:
-          return out[:-1].split("\0")
+          return out[:-1].split('\0')  # pylint: disable=W1401
+                                       # Backslash is not anomalous
       return []
 
     def DiffZ(self, name, *args):
@@ -2022,6 +2210,7 @@ class Project(object):
       cmd.extend(args)
       p = GitCommand(self._project,
                      cmd,
+                     gitdir=self._gitdir,
                      bare = False,
                      capture_stdout = True,
                      capture_stderr = True)
@@ -2029,7 +2218,7 @@ class Project(object):
         out = p.process.stdout.read()
         r = {}
         if out:
-          out = iter(out[:-1].split('\0'))
+          out = iter(out[:-1].split('\0'))  # pylint: disable=W1401
           while out:
             try:
               info = out.next()
@@ -2056,7 +2245,7 @@ class Project(object):
                     self.level = self.level[1:]
 
             info = info[1:].split(' ')
-            info =_Info(path, *info)
+            info = _Info(path, *info)
             if info.status in ('R', 'C'):
               info.src_path = info.path
               info.path = out.next()
@@ -2070,11 +2259,18 @@ class Project(object):
         path = os.path.join(self._project.gitdir, HEAD)
       else:
         path = os.path.join(self._project.worktree, '.git', HEAD)
-      fd = open(path, 'rb')
+      try:
+        fd = open(path, 'rb')
+      except IOError:
+        raise NoManifestException(path)
       try:
         line = fd.read()
       finally:
         fd.close()
+      try:
+        line = line.decode()
+      except AttributeError:
+        pass
       if line.startswith('ref: '):
         return line[5:-1]
       return line[:-1]
@@ -2124,6 +2320,7 @@ class Project(object):
       p = GitCommand(self._project,
                      cmdv,
                      bare = self._bare,
+                     gitdir=self._gitdir,
                      capture_stdout = True,
                      capture_stderr = True)
       r = []
@@ -2165,7 +2362,10 @@ class Project(object):
           raise TypeError('%s() got an unexpected keyword argument %r'
                           % (name, k))
         if config is not None:
-          for k, v in config.iteritems():
+          if not git_require((1, 7, 2)):
+            raise ValueError('cannot set config on command line for %s()'
+                             % name)
+          for k, v in config.items():
             cmdv.append('-c')
             cmdv.append('%s=%s' % (k, v))
         cmdv.append(name)
@@ -2173,6 +2373,7 @@ class Project(object):
         p = GitCommand(self._project,
                        cmdv,
                        bare = self._bare,
+                       gitdir=self._gitdir,
                        capture_stdout = True,
                        capture_stderr = True)
         if p.Wait() != 0:
@@ -2181,6 +2382,10 @@ class Project(object):
                          name,
                          p.stderr))
         r = p.stdout
+        try:
+          r = r.decode('utf-8')
+        except AttributeError:
+          pass
         if r.endswith('\n') and r.index('\n') == len(r) - 1:
           return r[:-1]
         return r
@@ -2302,6 +2507,7 @@ class MetaProject(Project):
                      manifest = manifest,
                      name = name,
                      gitdir = gitdir,
+                     objdir = gitdir,
                      worktree = worktree,
                      remote = RemoteSpec('origin'),
                      relpath = '.repo/%s' % name,

pyversion.py

@@ -0,0 +1,19 @@
+#
+# Copyright (C) 2013 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+def is_python3():
+  return sys.version_info[0] == 3

repo

@@ -1,9 +1,9 @@
-#!/bin/sh
+#!/usr/bin/env python
 
 ## repo default configuration
 ##
-REPO_URL='https://gerrit.googlesource.com/git-repo'
-REPO_REV='stable'
+REPO_URL = 'https://gerrit.googlesource.com/git-repo'
+REPO_REV = 'stable'
 
 # Copyright (C) 2008 Google Inc.
 #
@@ -19,19 +19,11 @@ REPO_REV='stable'
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-magic='--calling-python-from-/bin/sh--'
-"""exec" python -E "$0" "$@" """#$magic"
-if __name__ == '__main__':
-  import sys
-  if sys.argv[-1] == '#%s' % magic:
-    del sys.argv[-1]
-del magic
-
 # increment this whenever we make important changes to this script
-VERSION = (1, 18)
+VERSION = (1, 20)
 
 # increment this if the MAINTAINER_KEYS block is modified
-KEYRING_VERSION = (1,1)
+KEYRING_VERSION = (1, 2)
 MAINTAINER_KEYS = """
 
      Repo Maintainer <repo@android.kernel.org>
@@ -80,49 +72,81 @@ TACbBS+Up3RpfYVfd63c1cDdlru13pQAn3NQy/SN858MkxN+zym86UBgOad2
 -----BEGIN PGP PUBLIC KEY BLOCK-----
 Version: GnuPG v1.4.11 (GNU/Linux)
 
-mQENBFBiLPwBCACvISTASOgFXwADw2GYRH2I2z9RvYkYoZ6ThTTNlMXbbYYKO2Wo
-a9LQDNW0TbCEekg5UKk0FD13XOdWaqUt4Gtuvq9c43GRSjMO6NXH+0BjcQ8vUtY2
-/W4CYUevwdo4nQ1+1zsOCu1XYe/CReXq0fdugv3hgmRmh3sz1soo37Q44W2frxxg
-U7Rz3Da4FjgAL0RQ8qndD+LwRHXTY7H7wYM8V/3cYFZV7pSodd75q3MAXYQLf0ZV
-QR1XATu5l1QnXrxgHvz7MmDwb1D+jX3YPKnZveaukigQ6hDHdiVcePBiGXmk8LZC
-2jQkdXeF7Su1ZYpr2nnEHLJ6vOLcCpPGb8gDABEBAAG0H0NvbmxleSBPd2VucyA8
-Y2NvM0BhbmRyb2lkLmNvbT6JATgEEwECACIFAlBiLPwCGwMGCwkIBwMCBhUIAgkK
-CwQWAgMBAh4BAheAAAoJEBkmlFUziHGkHVkH/2Hks2Cif5i2xPtv2IFZcjL42joU
-T7lO5XFqUYS9ZNHpGa/V0eiPt7rHoO16glR83NZtwlrq2cSN89i9HfOhMYV/qLu8
-fLCHcV2muw+yCB5s5bxnI5UkToiNZyBNqFkcOt/Kbj9Hpy68A1kmc6myVEaUYebq
-2Chx/f3xuEthan099t746v1K+/6SvQGDNctHuaMr9cWdxZtHjdRf31SQRc99Phe5
-w+ZGR/ebxNDKRK9mKgZT8wVFHlXerJsRqWIqtx1fsW1UgLgbpcpe2MChm6B5wTu0
-s1ltzox3l4q71FyRRPUJxXyvGkDLZWpK7EpiHSCOYq/KP3HkKeXU3xqHpcG5AQ0E
-UGIs/AEIAKzO/7lO9cB6dshmZYo8Vy/b7aGicThE+ChcDSfhvyOXVdEM2GKAjsR+
-rlBWbTFX3It301p2HwZPFEi9nEvJxVlqqBiW0bPmNMkDRR55l2vbWg35wwkg6RyE
-Bc5/TQjhXI2w8IvlimoGoUff4t3JmMOnWrnKSvL+5iuRj12p9WmanCHzw3Ee7ztf
-/aU/q+FTpr3DLerb6S8xbv86ySgnJT6o5CyL2DCWRtnYQyGVi0ZmLzEouAYiO0hs
-z0AAu28Mj+12g2WwePRz6gfM9rHtI37ylYW3oT/9M9mO9ei/Bc/1D7Dz6qNV+0vg
-uSVJxM2Bl6GalHPZLhHntFEdIA6EdoUAEQEAAYkBHwQYAQIACQUCUGIs/AIbDAAK
-CRAZJpRVM4hxpNfkB/0W/hP5WK/NETXBlWXXW7JPaWO2c5kGwD0lnj5RRmridyo1
-vbm5PdM91jOsDQYqRu6YOoYBnDnEhB2wL2bPh34HWwwrA+LwB8hlcAV2z1bdwyfl
-3R823fReKN3QcvLHzmvZPrF4Rk97M9UIyKS0RtnfTWykRgDWHIsrtQPoNwsXrWoT
-9LrM2v+1+9mp3vuXnE473/NHxmiWEQH9Ez+O/mOxQ7rSOlqGRiKq/IBZCfioJOtV
-fTQeIu/yASZnsLBqr6SJEGwYBoWcyjG++k4fyw8ocOAo4uGDYbxgN7yYfNQ0OH7o
-V6pfUgqKLWa/aK7/N1ZHnPdFLD8Xt0Dmy4BPwrKC
-=O7am
+mQENBFHRvc8BCADFg45Xx/y6QDC+T7Y/gGc7vx0ww7qfOwIKlAZ9xG3qKunMxo+S
+hPCnzEl3cq+6I1Ww/ndop/HB3N3toPXRCoN8Vs4/Hc7by+SnaLFnacrm+tV5/OgT
+V37Lzt8lhay1Kl+YfpFwHYYpIEBLFV9knyfRXS/428W2qhdzYfvB15/AasRmwmor
+py4NIzSs8UD/SPr1ihqNCdZM76+MQyN5HMYXW/ALZXUFG0pwluHFA7hrfPG74i8C
+zMiP7qvMWIl/r/jtzHioH1dRKgbod+LZsrDJ8mBaqsZaDmNJMhss9g76XvfMyLra
+9DI9/iFuBpGzeqBv0hwOGQspLRrEoyTeR6n1ABEBAAG0H0NvbmxleSBPd2VucyA8
+Y2NvM0BhbmRyb2lkLmNvbT6JATgEEwECACIFAlHRvc8CGwMGCwkIBwMCBhUIAgkK
+CwQWAgMBAh4BAheAAAoJEGe35EhpKzgsP6AIAJKJmNtn4l7hkYHKHFSo3egb6RjQ
+zEIP3MFTcu8HFX1kF1ZFbrp7xqurLaE53kEkKuAAvjJDAgI8mcZHP1JyplubqjQA
+xvv84gK+OGP3Xk+QK1ZjUQSbjOpjEiSZpRhWcHci3dgOUH4blJfByHw25hlgHowd
+a/2PrNKZVcJ92YienaxxGjcXEUcd0uYEG2+rwllQigFcnMFDhr9B71MfalRHjFKE
+fmdoypqLrri61YBc59P88Rw2/WUpTQjgNubSqa3A2+CKdaRyaRw+2fdF4TdR0h8W
+zbg+lbaPtJHsV+3mJC7fq26MiJDRJa5ZztpMn8su20gbLgi2ShBOaHAYDDi5AQ0E
+UdG9zwEIAMoOBq+QLNozAhxOOl5GL3StTStGRgPRXINfmViTsihrqGCWBBUfXlUE
+OytC0mYcrDUQev/8ToVoyqw+iGSwDkcSXkrEUCKFtHV/GECWtk1keyHgR10YKI1R
+mquSXoubWGqPeG1PAI74XWaRx8UrL8uCXUtmD8Q5J7mDjKR5NpxaXrwlA0bKsf2E
+Gp9tu1kKauuToZhWHMRMqYSOGikQJwWSFYKT1KdNcOXLQF6+bfoJ6sjVYdwfmNQL
+Ixn8QVhoTDedcqClSWB17VDEFDFa7MmqXZz2qtM3X1R/MUMHqPtegQzBGNhRdnI2
+V45+1Nnx/uuCxDbeI4RbHzujnxDiq70AEQEAAYkBHwQYAQIACQUCUdG9zwIbDAAK
+CRBnt+RIaSs4LNVeB/0Y2pZ8I7gAAcEM0Xw8drr4omg2fUoK1J33ozlA/RxeA/lJ
+I3KnyCDTpXuIeBKPGkdL8uMATC9Z8DnBBajRlftNDVZS3Hz4G09G9QpMojvJkFJV
+By+01Flw/X+eeN8NpqSuLV4W+AjEO8at/VvgKr1AFvBRdZ7GkpI1o6DgPe7ZqX+1
+dzQZt3e13W0rVBb/bUgx9iSLoeWP3aq/k+/GRGOR+S6F6BBSl0SQ2EF2+dIywb1x
+JuinEP+AwLAUZ1Bsx9ISC0Agpk2VeHXPL3FGhroEmoMvBzO0kTFGyoeT7PR/BfKv
++H/g3HsL2LOB9uoIm8/5p2TTU5ttYCXMHhQZ81AY
+=AUp4
 -----END PGP PUBLIC KEY BLOCK-----
 """
 
 GIT = 'git'                     # our git command
-MIN_GIT_VERSION = (1, 5, 4)     # minimum supported git version
+MIN_GIT_VERSION = (1, 7, 2)     # minimum supported git version
 repodir = '.repo'               # name of repo's private directory
 S_repo = 'repo'                 # special repo repository
 S_manifests = 'manifests'       # special manifest repository
 REPO_MAIN = S_repo + '/main.py' # main script
+MIN_PYTHON_VERSION = (2, 6)     # minimum supported python version
 
 
+import errno
 import optparse
 import os
 import re
+import stat
 import subprocess
 import sys
-import urllib2
+
+if sys.version_info[0] == 3:
+  import urllib.request
+  import urllib.error
+else:
+  import imp
+  import urllib2
+  urllib = imp.new_module('urllib')
+  urllib.request = urllib2
+  urllib.error = urllib2
+
+
+def _print(*objects, **kwargs):
+  sep = kwargs.get('sep', ' ')
+  end = kwargs.get('end', '\n')
+  out = kwargs.get('file', sys.stdout)
+  out.write(sep.join(objects) + end)
+
+
+# Python version check
+ver = sys.version_info
+if ver[0] == 3:
+  _print('warning: Python 3 support is currently experimental. YMMV.\n'
+         'Please use Python 2.6 - 2.7 instead.',
+         file=sys.stderr)
+if (ver[0], ver[1]) < MIN_PYTHON_VERSION:
+  _print('error: Python version %s unsupported.\n'
+         'Please use Python 2.6 - 2.7 instead.'
+         % sys.version.split(' ')[0], file=sys.stderr)
+  sys.exit(1)
 
 home_dot_repo = os.path.expanduser('~/.repoconfig')
 gpg_dir = os.path.join(home_dot_repo, 'gnupg')
@@ -149,16 +173,22 @@ group.add_option('-m', '--manifest-name',
                  help='initial manifest file', metavar='NAME.xml')
 group.add_option('--mirror',
                  dest='mirror', action='store_true',
-                 help='mirror the forrest')
+                 help='create a replica of the remote repositories '
+                      'rather than a client working directory')
 group.add_option('--reference',
                  dest='reference',
                  help='location of mirror directory', metavar='DIR')
 group.add_option('--depth', type='int', default=None,
                  dest='depth',
                  help='create a shallow clone with given depth; see git clone')
+group.add_option('--archive',
+                 dest='archive', action='store_true',
+                 help='checkout an archive instead of a git repository for '
+                      'each project. See git archive.')
 group.add_option('-g', '--groups',
                  dest='groups', default='default',
-                 help='restrict manifest projects to ones with a specified group',
+                 help='restrict manifest projects to ones with specified '
+                      'group(s) [default|all|G1,G2,G3|G4,-G5,-G6]',
                  metavar='GROUP')
 group.add_option('-p', '--platform',
                  dest='platform', default="auto",
@@ -211,17 +241,16 @@ def _Init(args):
   if branch.startswith('refs/heads/'):
     branch = branch[len('refs/heads/'):]
   if branch.startswith('refs/'):
-    print >>sys.stderr, "fatal: invalid branch name '%s'" % branch
+    _print("fatal: invalid branch name '%s'" % branch, file=sys.stderr)
     raise CloneFailure()
 
-  if not os.path.isdir(repodir):
-    try:
-      os.mkdir(repodir)
-    except OSError as e:
-      print >>sys.stderr, \
-            'fatal: cannot make %s directory: %s' % (
-            repodir, e.strerror)
-      # Don't faise CloneFailure; that would delete the
+  try:
+    os.mkdir(repodir)
+  except OSError as e:
+    if e.errno != errno.EEXIST:
+      _print('fatal: cannot make %s directory: %s'
+             % (repodir, e.strerror), file=sys.stderr)
+      # Don't raise CloneFailure; that would delete the
       # name. Instead exit immediately.
       #
       sys.exit(1)
@@ -244,8 +273,8 @@ def _Init(args):
     _Checkout(dst, branch, rev, opt.quiet)
   except CloneFailure:
     if opt.quiet:
-      print >>sys.stderr, \
-        'fatal: repo init failed; run without --quiet to see why'
+      _print('fatal: repo init failed; run without --quiet to see why',
+             file=sys.stderr)
     raise
 
 
@@ -254,12 +283,12 @@ def _CheckGitVersion():
   try:
     proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
   except OSError as e:
-    print >>sys.stderr
-    print >>sys.stderr, "fatal: '%s' is not available" % GIT
-    print >>sys.stderr, 'fatal: %s' % e
-    print >>sys.stderr
-    print >>sys.stderr, 'Please make sure %s is installed'\
-                        ' and in your path.' % GIT
+    _print(file=sys.stderr)
+    _print("fatal: '%s' is not available" % GIT, file=sys.stderr)
+    _print('fatal: %s' % e, file=sys.stderr)
+    _print(file=sys.stderr)
+    _print('Please make sure %s is installed and in your path.' % GIT,
+           file=sys.stderr)
     raise CloneFailure()
 
   ver_str = proc.stdout.read().strip()
@@ -267,14 +296,14 @@ def _CheckGitVersion():
   proc.wait()
 
   if not ver_str.startswith('git version '):
-    print >>sys.stderr, 'error: "%s" unsupported' % ver_str
+    _print('error: "%s" unsupported' % ver_str, file=sys.stderr)
     raise CloneFailure()
 
   ver_str = ver_str[len('git version '):].strip()
-  ver_act = tuple(map(lambda x: int(x), ver_str.split('.')[0:3]))
+  ver_act = tuple(map(int, ver_str.split('.')[0:3]))
   if ver_act < MIN_GIT_VERSION:
-    need = '.'.join(map(lambda x: str(x), MIN_GIT_VERSION))
-    print >>sys.stderr, 'fatal: git %s or later required' % need
+    need = '.'.join(map(str, MIN_GIT_VERSION))
+    _print('fatal: git %s or later required' % need, file=sys.stderr)
     raise CloneFailure()
 
 
@@ -290,29 +319,27 @@ def NeedSetupGnuPG():
   if not kv:
     return True
 
-  kv = tuple(map(lambda x: int(x), kv.split('.')))
+  kv = tuple(map(int, kv.split('.')))
   if kv < KEYRING_VERSION:
     return True
   return False
 
 
 def SetupGnuPG(quiet):
-  if not os.path.isdir(home_dot_repo):
-    try:
-      os.mkdir(home_dot_repo)
-    except OSError as e:
-      print >>sys.stderr, \
-            'fatal: cannot make %s directory: %s' % (
-            home_dot_repo, e.strerror)
+  try:
+    os.mkdir(home_dot_repo)
+  except OSError as e:
+    if e.errno != errno.EEXIST:
+      _print('fatal: cannot make %s directory: %s'
+             % (home_dot_repo, e.strerror), file=sys.stderr)
       sys.exit(1)
 
-  if not os.path.isdir(gpg_dir):
-    try:
-      os.mkdir(gpg_dir, 0700)
-    except OSError as e:
-      print >>sys.stderr, \
-            'fatal: cannot make %s directory: %s' % (
-            gpg_dir, e.strerror)
+  try:
+    os.mkdir(gpg_dir, stat.S_IRWXU)
+  except OSError as e:
+    if e.errno != errno.EEXIST:
+      _print('fatal: cannot make %s directory: %s' % (gpg_dir, e.strerror),
+             file=sys.stderr)
       sys.exit(1)
 
   env = os.environ.copy()
@@ -325,21 +352,21 @@ def SetupGnuPG(quiet):
                             stdin = subprocess.PIPE)
   except OSError as e:
     if not quiet:
-      print >>sys.stderr, 'warning: gpg (GnuPG) is not available.'
-      print >>sys.stderr, 'warning: Installing it is strongly encouraged.'
-      print >>sys.stderr
+      _print('warning: gpg (GnuPG) is not available.', file=sys.stderr)
+      _print('warning: Installing it is strongly encouraged.', file=sys.stderr)
+      _print(file=sys.stderr)
     return False
 
   proc.stdin.write(MAINTAINER_KEYS)
   proc.stdin.close()
 
   if proc.wait() != 0:
-    print >>sys.stderr, 'fatal: registering repo maintainer keys failed'
+    _print('fatal: registering repo maintainer keys failed', file=sys.stderr)
     sys.exit(1)
-  print
+  _print()
 
   fd = open(os.path.join(home_dot_repo, 'keyring-version'), 'w')
-  fd.write('.'.join(map(lambda x: str(x), KEYRING_VERSION)) + '\n')
+  fd.write('.'.join(map(str, KEYRING_VERSION)) + '\n')
   fd.close()
   return True
 
@@ -355,7 +382,7 @@ def _SetConfig(local, name, value):
 def _InitHttp():
   handlers = []
 
-  mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
+  mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
   try:
     import netrc
     n = netrc.netrc()
@@ -365,20 +392,20 @@ def _InitHttp():
       mgr.add_password(p[1], 'https://%s/' % host, p[0], p[2])
   except:
     pass
-  handlers.append(urllib2.HTTPBasicAuthHandler(mgr))
-  handlers.append(urllib2.HTTPDigestAuthHandler(mgr))
+  handlers.append(urllib.request.HTTPBasicAuthHandler(mgr))
+  handlers.append(urllib.request.HTTPDigestAuthHandler(mgr))
 
   if 'http_proxy' in os.environ:
     url = os.environ['http_proxy']
-    handlers.append(urllib2.ProxyHandler({'http': url, 'https': url}))
+    handlers.append(urllib.request.ProxyHandler({'http': url, 'https': url}))
   if 'REPO_CURL_VERBOSE' in os.environ:
-    handlers.append(urllib2.HTTPHandler(debuglevel=1))
-    handlers.append(urllib2.HTTPSHandler(debuglevel=1))
-  urllib2.install_opener(urllib2.build_opener(*handlers))
+    handlers.append(urllib.request.HTTPHandler(debuglevel=1))
+    handlers.append(urllib.request.HTTPSHandler(debuglevel=1))
+  urllib.request.install_opener(urllib.request.build_opener(*handlers))
 
 def _Fetch(url, local, src, quiet):
   if not quiet:
-    print >>sys.stderr, 'Get %s' % url
+    _print('Get %s' % url, file=sys.stderr)
 
   cmd = [GIT, 'fetch']
   if quiet:
@@ -423,20 +450,20 @@ def _DownloadBundle(url, local, quiet):
   dest = open(os.path.join(local, '.git', 'clone.bundle'), 'w+b')
   try:
     try:
-      r = urllib2.urlopen(url)
-    except urllib2.HTTPError as e:
-      if e.code == 404:
+      r = urllib.request.urlopen(url)
+    except urllib.error.HTTPError as e:
+      if e.code in [403, 404]:
         return False
-      print >>sys.stderr, 'fatal: Cannot get %s' % url
-      print >>sys.stderr, 'fatal: HTTP error %s' % e.code
+      _print('fatal: Cannot get %s' % url, file=sys.stderr)
+      _print('fatal: HTTP error %s' % e.code, file=sys.stderr)
       raise CloneFailure()
-    except urllib2.URLError as e:
-      print >>sys.stderr, 'fatal: Cannot get %s' % url
-      print >>sys.stderr, 'fatal: error %s' % e.reason
+    except urllib.error.URLError as e:
+      _print('fatal: Cannot get %s' % url, file=sys.stderr)
+      _print('fatal: error %s' % e.reason, file=sys.stderr)
       raise CloneFailure()
     try:
       if not quiet:
-        print >>sys.stderr, 'Get %s' % url
+        _print('Get %s' % url, file=sys.stderr)
       while True:
         buf = r.read(8192)
         if buf == '':
@@ -460,24 +487,23 @@ def _Clone(url, local, quiet):
   try:
     os.mkdir(local)
   except OSError as e:
-    print >>sys.stderr, \
-          'fatal: cannot make %s directory: %s' \
-          % (local, e.strerror)
+    _print('fatal: cannot make %s directory: %s' % (local, e.strerror),
+           file=sys.stderr)
     raise CloneFailure()
 
   cmd = [GIT, 'init', '--quiet']
   try:
     proc = subprocess.Popen(cmd, cwd = local)
   except OSError as e:
-    print >>sys.stderr
-    print >>sys.stderr, "fatal: '%s' is not available" % GIT
-    print >>sys.stderr, 'fatal: %s' % e
-    print >>sys.stderr
-    print >>sys.stderr, 'Please make sure %s is installed'\
-                        ' and in your path.' % GIT
+    _print(file=sys.stderr)
+    _print("fatal: '%s' is not available" % GIT, file=sys.stderr)
+    _print('fatal: %s' % e, file=sys.stderr)
+    _print(file=sys.stderr)
+    _print('Please make sure %s is installed and in your path.' % GIT,
+          file=sys.stderr)
     raise CloneFailure()
   if proc.wait() != 0:
-    print >>sys.stderr, 'fatal: could not create %s' % local
+    _print('fatal: could not create %s' % local, file=sys.stderr)
     raise CloneFailure()
 
   _InitHttp()
@@ -505,21 +531,18 @@ def _Verify(cwd, branch, quiet):
   proc.stderr.close()
 
   if proc.wait() != 0 or not cur:
-    print >>sys.stderr
-    print >>sys.stderr,\
-      "fatal: branch '%s' has not been signed" \
-      % branch
+    _print(file=sys.stderr)
+    _print("fatal: branch '%s' has not been signed" % branch, file=sys.stderr)
     raise CloneFailure()
 
   m = re.compile(r'^(.*)-[0-9]{1,}-g[0-9a-f]{1,}$').match(cur)
   if m:
     cur = m.group(1)
     if not quiet:
-      print >>sys.stderr
-      print >>sys.stderr, \
-        "info: Ignoring branch '%s'; using tagged release '%s'" \
-        % (branch, cur)
-      print >>sys.stderr
+      _print(file=sys.stderr)
+      _print("info: Ignoring branch '%s'; using tagged release '%s'"
+            % (branch, cur), file=sys.stderr)
+      _print(file=sys.stderr)
 
   env = os.environ.copy()
   env['GNUPGHOME'] = gpg_dir.encode()
@@ -537,10 +560,10 @@ def _Verify(cwd, branch, quiet):
   proc.stderr.close()
 
   if proc.wait() != 0:
-    print >>sys.stderr
-    print >>sys.stderr, out
-    print >>sys.stderr, err
-    print >>sys.stderr
+    _print(file=sys.stderr)
+    _print(out, file=sys.stderr)
+    _print(err, file=sys.stderr)
+    _print(file=sys.stderr)
     raise CloneFailure()
   return '%s^0' % cur
 
@@ -594,7 +617,7 @@ def _ParseArguments(args):
   opt = _Options()
   arg = []
 
-  for i in xrange(0, len(args)):
+  for i in range(len(args)):
     a = args[i]
     if a == '-h' or a == '--help':
       opt.help = True
@@ -607,7 +630,7 @@ def _ParseArguments(args):
 
 
 def _Usage():
-  print >>sys.stderr,\
+  _print(
 """usage: repo COMMAND [ARGS]
 
 repo is not yet installed.  Use "repo init" to install it here.
@@ -618,7 +641,7 @@ The most commonly used repo commands are:
   help      Display detailed help on a command
 
 For access to the full online help, install repo ("repo init").
-"""
+""", file=sys.stderr)
   sys.exit(1)
 
 
@@ -628,25 +651,23 @@ def _Help(args):
       init_optparse.print_help()
       sys.exit(0)
     else:
-      print >>sys.stderr,\
-      "error: '%s' is not a bootstrap command.\n"\
-      '        For access to online help, install repo ("repo init").'\
-      % args[0]
+      _print("error: '%s' is not a bootstrap command.\n"
+             '        For access to online help, install repo ("repo init").'
+             % args[0], file=sys.stderr)
   else:
     _Usage()
   sys.exit(1)
 
 
 def _NotInstalled():
-  print >>sys.stderr,\
-'error: repo is not installed.  Use "repo init" to install it here.'
+  _print('error: repo is not installed.  Use "repo init" to install it here.',
+         file=sys.stderr)
   sys.exit(1)
 
 
 def _NoCommands(cmd):
-  print >>sys.stderr,\
-"""error: command '%s' requires repo to be installed first.
-       Use "repo init" to install it here.""" % cmd
+  _print("""error: command '%s' requires repo to be installed first.
+         Use "repo init" to install it here.""" % cmd, file=sys.stderr)
   sys.exit(1)
 
 
@@ -683,7 +704,7 @@ def _SetDefaultsTo(gitdir):
   proc.stderr.close()
 
   if proc.wait() != 0:
-    print >>sys.stderr, 'fatal: %s has no current branch' % gitdir
+    _print('fatal: %s has no current branch' % gitdir, file=sys.stderr)
     sys.exit(1)
 
 
@@ -721,8 +742,8 @@ def main(orig_args):
   if my_main:
     repo_main = my_main
 
-  ver_str = '.'.join(map(lambda x: str(x), VERSION))
-  me = [repo_main,
+  ver_str = '.'.join(map(str, VERSION))
+  me = [sys.executable, repo_main,
         '--repo-dir=%s' % rel_repo_dir,
         '--wrapper-version=%s' % ver_str,
         '--wrapper-path=%s' % wrapper_path,
@@ -730,10 +751,10 @@ def main(orig_args):
   me.extend(orig_args)
   me.extend(extra_args)
   try:
-    os.execv(repo_main, me)
+    os.execv(sys.executable, me)
   except OSError as e:
-    print >>sys.stderr, "fatal: unable to start %s" % repo_main
-    print >>sys.stderr, "fatal: %s" % e
+    _print("fatal: unable to start %s" % repo_main, file=sys.stderr)
+    _print("fatal: %s" % e, file=sys.stderr)
     sys.exit(148)
 
 

subcmds/__init__.py

@@ -38,8 +38,8 @@ for py in os.listdir(my_dir):
     try:
       cmd = getattr(mod, clsn)()
     except AttributeError:
-      raise SyntaxError, '%s/%s does not define class %s' % (
-                         __name__, py, clsn)
+      raise SyntaxError('%s/%s does not define class %s' % (
+                         __name__, py, clsn))
 
     name = name.replace('_', '-')
     cmd.NAME = name

subcmds/abandon.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import sys
 from command import Command
 from git_command import git
@@ -36,7 +37,7 @@ It is equivalent to "git branch -D <branchname>".
 
     nb = args[0]
     if not git.check_ref_format('heads/%s' % nb):
-      print >>sys.stderr, "error: '%s' is not a valid name" % nb
+      print("error: '%s' is not a valid name" % nb, file=sys.stderr)
       sys.exit(1)
 
     nb = args[0]
@@ -58,13 +59,13 @@ It is equivalent to "git branch -D <branchname>".
 
     if err:
       for p in err:
-        print >>sys.stderr,\
-          "error: %s/: cannot abandon %s" \
-          % (p.relpath, nb)
+        print("error: %s/: cannot abandon %s" % (p.relpath, nb),
+              file=sys.stderr)
       sys.exit(1)
     elif not success:
-      print >>sys.stderr, 'error: no project has branch %s' % nb
+      print('error: no project has branch %s' % nb, file=sys.stderr)
       sys.exit(1)
     else:
-      print >>sys.stderr, 'Abandoned in %d project(s):\n  %s' % (
-            len(success), '\n  '.join(p.relpath for p in success))
+      print('Abandoned in %d project(s):\n  %s'
+            % (len(success), '\n  '.join(p.relpath for p in success)),
+            file=sys.stderr)

subcmds/branches.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import sys
 from color import Coloring
 from command import Command
@@ -97,17 +98,16 @@ is shown, then the branch appears in all projects.
     project_cnt = len(projects)
 
     for project in projects:
-      for name, b in project.GetBranches().iteritems():
+      for name, b in project.GetBranches().items():
         b.project = project
         if name not in all_branches:
           all_branches[name] = BranchInfo(name)
         all_branches[name].add(b)
 
-    names = all_branches.keys()
-    names.sort()
+    names = list(sorted(all_branches))
 
     if not names:
-      print >>sys.stderr, '   (no branches)'
+      print('   (no branches)', file=sys.stderr)
       return
 
     width = 25
@@ -139,7 +139,7 @@ is shown, then the branch appears in all projects.
       if in_cnt < project_cnt:
         fmt = out.write
         paths = []
-        if in_cnt < project_cnt - in_cnt: 
+        if in_cnt < project_cnt - in_cnt:
           in_type = 'in'
           for b in i.projects:
             paths.append(b.project.relpath)

subcmds/checkout.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import sys
 from command import Command
 from progress import Progress
@@ -55,10 +56,9 @@ The command is equivalent to:
 
     if err:
       for p in err:
-        print >>sys.stderr,\
-          "error: %s/: cannot checkout %s" \
-          % (p.relpath, nb)
+        print("error: %s/: cannot checkout %s" % (p.relpath, nb),
+              file=sys.stderr)
       sys.exit(1)
     elif not success:
-      print >>sys.stderr, 'error: no project has branch %s' % nb
+      print('error: no project has branch %s' % nb, file=sys.stderr)
       sys.exit(1)

subcmds/cherry_pick.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import re
 import sys
 from command import Command
@@ -46,13 +47,13 @@ change id will be added.
                    capture_stdout = True,
                    capture_stderr = True)
     if p.Wait() != 0:
-      print >>sys.stderr, p.stderr
+      print(p.stderr, file=sys.stderr)
       sys.exit(1)
     sha1 = p.stdout.strip()
 
     p = GitCommand(None, ['cat-file', 'commit', sha1], capture_stdout=True)
     if p.Wait() != 0:
-      print >>sys.stderr, "error: Failed to retrieve old commit message"
+      print("error: Failed to retrieve old commit message", file=sys.stderr)
       sys.exit(1)
     old_msg = self._StripHeader(p.stdout)
 
@@ -62,8 +63,8 @@ change id will be added.
                    capture_stderr = True)
     status = p.Wait()
 
-    print >>sys.stdout, p.stdout
-    print >>sys.stderr, p.stderr
+    print(p.stdout, file=sys.stdout)
+    print(p.stderr, file=sys.stderr)
 
     if status == 0:
       # The cherry-pick was applied correctly. We just need to edit the
@@ -76,16 +77,14 @@ change id will be added.
                      capture_stderr = True)
       p.stdin.write(new_msg)
       if p.Wait() != 0:
-        print >>sys.stderr, "error: Failed to update commit message"
+        print("error: Failed to update commit message", file=sys.stderr)
         sys.exit(1)
 
     else:
-      print >>sys.stderr, """\
-NOTE: When committing (please see above) and editing the commit message,
-please remove the old Change-Id-line and add:
-"""
-      print >>sys.stderr, self._GetReference(sha1)
-      print >>sys.stderr
+      print('NOTE: When committing (please see above) and editing the commit '
+            'message, please remove the old Change-Id-line and add:')
+      print(self._GetReference(sha1), file=sys.stderr)
+      print(file=sys.stderr)
 
   def _IsChangeId(self, line):
     return CHANGE_ID_RE.match(line)

subcmds/download.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import re
 import sys
 
@@ -32,13 +33,13 @@ makes it available in your project's local working directory.
 """
 
   def _Options(self, p):
-    p.add_option('-c','--cherry-pick',
+    p.add_option('-c', '--cherry-pick',
                  dest='cherrypick', action='store_true',
                  help="cherry-pick instead of checkout")
-    p.add_option('-r','--revert',
+    p.add_option('-r', '--revert',
                  dest='revert', action='store_true',
                  help="revert instead of checkout")
-    p.add_option('-f','--ff-only',
+    p.add_option('-f', '--ff-only',
                  dest='ffonly', action='store_true',
                  help="force fast-forward merge")
 
@@ -68,23 +69,23 @@ makes it available in your project's local working directory.
     for project, change_id, ps_id in self._ParseChangeIds(args):
       dl = project.DownloadPatchSet(change_id, ps_id)
       if not dl:
-        print >>sys.stderr, \
-          '[%s] change %d/%d not found' \
-          % (project.name, change_id, ps_id)
+        print('[%s] change %d/%d not found'
+              % (project.name, change_id, ps_id),
+              file=sys.stderr)
         sys.exit(1)
 
       if not opt.revert and not dl.commits:
-        print >>sys.stderr, \
-          '[%s] change %d/%d has already been merged' \
-          % (project.name, change_id, ps_id)
+        print('[%s] change %d/%d has already been merged'
+              % (project.name, change_id, ps_id),
+              file=sys.stderr)
         continue
 
       if len(dl.commits) > 1:
-        print >>sys.stderr, \
-          '[%s] %d/%d depends on %d unmerged changes:' \
-          % (project.name, change_id, ps_id, len(dl.commits))
+        print('[%s] %d/%d depends on %d unmerged changes:' \
+              % (project.name, change_id, ps_id, len(dl.commits)),
+              file=sys.stderr)
         for c in dl.commits:
-          print >>sys.stderr, '  %s' % (c)
+          print('  %s' % (c), file=sys.stderr)
       if opt.cherrypick:
         project._CherryPick(dl.commit)
       elif opt.revert:

subcmds/forall.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import fcntl
 import re
 import os
@@ -41,10 +42,14 @@ class Forall(Command, MirrorSafeCommand):
   helpSummary = "Run a shell command in each project"
   helpUsage = """
 %prog [<project>...] -c <command> [<arg>...]
+%prog -r str1 [str2] ... -c <command> [<arg>...]"
 """
   helpDescription = """
 Executes the same shell command in each project.
 
+The -r option allows running the command only on projects matching
+regex or wildcard expression.
+
 Output Formatting
 -----------------
 
@@ -92,6 +97,9 @@ following <command>.
 
 Unless -p is used, stdin, stdout, stderr are inherited from the
 terminal and are not redirected.
+
+If -e is used, when a command exits unsuccessfully, '%prog' will abort
+without iterating through the remaining projects.
 """
 
   def _Options(self, p):
@@ -99,11 +107,17 @@ terminal and are not redirected.
       setattr(parser.values, option.dest, list(parser.rargs))
       while parser.rargs:
         del parser.rargs[0]
+    p.add_option('-r', '--regex',
+                 dest='regex', action='store_true',
+                 help="Execute the command only on projects matching regex or wildcard expression")
     p.add_option('-c', '--command',
                  help='Command (and arguments) to execute',
                  dest='command',
                  action='callback',
                  callback=cmd)
+    p.add_option('-e', '--abort-on-errors',
+                 dest='abort_on_errors', action='store_true',
+                 help='Abort if a command exits unsuccessfully')
 
     g = p.add_option_group('Output')
     g.add_option('-p',
@@ -159,7 +173,12 @@ terminal and are not redirected.
     rc = 0
     first = True
 
-    for project in self.GetProjects(args):
+    if not opt.regex:
+      projects = self.GetProjects(args)
+    else:
+      projects = self.FindProjects(args)
+
+    for project in projects:
       env = os.environ.copy()
       def setenv(name, val):
         if val is None:
@@ -183,7 +202,7 @@ terminal and are not redirected.
       if not os.path.exists(cwd):
         if (opt.project_header and opt.verbose) \
         or not opt.project_header:
-          print >>sys.stderr, 'skipping %s/' % project.relpath
+          print('skipping %s/' % project.relpath, file=sys.stderr)
         continue
 
       if opt.project_header:
@@ -241,7 +260,12 @@ terminal and are not redirected.
                 first = False
               else:
                 out.nl()
-              out.project('project %s/', project.relpath)
+
+              if mirror:
+                project_header_path = project.name
+              else:
+                project_header_path = project.relpath
+              out.project('project %s/', project_header_path)
               out.nl()
               out.flush()
               if errbuf:
@@ -254,7 +278,12 @@ terminal and are not redirected.
             s.dest.flush()
 
       r = p.wait()
-      if r != 0 and r != rc:
-        rc = r
+      if r != 0:
+        if r != rc:
+          rc = r
+        if opt.abort_on_errors:
+          print("error: %s: Aborting due to previous error" % project.relpath,
+                file=sys.stderr)
+          sys.exit(r)
     if rc != 0:
       sys.exit(rc)

subcmds/grep.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import sys
 from color import Coloring
 from command import PagedCommand
@@ -51,7 +52,7 @@ Examples
 
 Look for a line that has '#define' and either 'MAX_PATH or 'PATH_MAX':
 
-  repo grep -e '#define' --and -\( -e MAX_PATH -e PATH_MAX \)
+  repo grep -e '#define' --and -\\( -e MAX_PATH -e PATH_MAX \\)
 
 Look for a line that has 'NODE' or 'Unexpected' in files that
 contain a line that matches both expressions:
@@ -84,7 +85,7 @@ contain a line that matches both expressions:
     g.add_option('--cached',
                  action='callback', callback=carry,
                  help='Search the index, instead of the work tree')
-    g.add_option('-r','--revision',
+    g.add_option('-r', '--revision',
                  dest='revision', action='append', metavar='TREEish',
                  help='Search TREEish, instead of the work tree')
 
@@ -96,7 +97,7 @@ contain a line that matches both expressions:
     g.add_option('-i', '--ignore-case',
                  action='callback', callback=carry,
                  help='Ignore case differences')
-    g.add_option('-a','--text',
+    g.add_option('-a', '--text',
                  action='callback', callback=carry,
                  help="Process binary files as if they were text")
     g.add_option('-I',
@@ -125,7 +126,7 @@ contain a line that matches both expressions:
     g.add_option('--and', '--or', '--not',
                  action='callback', callback=carry,
                  help='Boolean operators to combine patterns')
-    g.add_option('-(','-)',
+    g.add_option('-(', '-)',
                  action='callback', callback=carry,
                  help='Boolean operator grouping')
 
@@ -145,10 +146,10 @@ contain a line that matches both expressions:
                  action='callback', callback=carry,
                  metavar='CONTEXT', type='str',
                  help='Show CONTEXT lines after match')
-    g.add_option('-l','--name-only','--files-with-matches',
+    g.add_option('-l', '--name-only', '--files-with-matches',
                  action='callback', callback=carry,
                  help='Show only file names containing matching lines')
-    g.add_option('-L','--files-without-match',
+    g.add_option('-L', '--files-without-match',
                  action='callback', callback=carry,
                  help='Show only file names not containing matching lines')
 
@@ -157,9 +158,9 @@ contain a line that matches both expressions:
     out = GrepColoring(self.manifest.manifestProject.config)
 
     cmd_argv = ['grep']
-    if out.is_on and git_require((1,6,3)):
+    if out.is_on and git_require((1, 6, 3)):
       cmd_argv.append('--color')
-    cmd_argv.extend(getattr(opt,'cmd_argv',[]))
+    cmd_argv.extend(getattr(opt, 'cmd_argv', []))
 
     if '-e' not in cmd_argv:
       if not args:
@@ -178,8 +179,7 @@ contain a line that matches both expressions:
     have_rev = False
     if opt.revision:
       if '--cached' in cmd_argv:
-        print >>sys.stderr,\
-          'fatal: cannot combine --cached and --revision'
+        print('fatal: cannot combine --cached and --revision', file=sys.stderr)
         sys.exit(1)
       have_rev = True
       cmd_argv.extend(opt.revision)
@@ -230,13 +230,13 @@ contain a line that matches both expressions:
           out.nl()
       else:
         for line in r:
-          print line
+          print(line)
 
     if have_match:
       sys.exit(0)
     elif have_rev and bad_rev:
       for r in opt.revision:
-        print >>sys.stderr, "error: can't search revision %s" % r
+        print("error: can't search revision %s" % r, file=sys.stderr)
       sys.exit(1)
     else:
       sys.exit(1)

subcmds/help.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import re
 import sys
 from formatter import AbstractFormatter, DumbWriter
@@ -31,12 +32,9 @@ Displays detailed usage information about a command.
 """
 
   def _PrintAllCommands(self):
-    print 'usage: repo COMMAND [ARGS]'
-    print """
-The complete list of recognized repo commands are:
-"""
-    commandNames = self.commands.keys()
-    commandNames.sort()
+    print('usage: repo COMMAND [ARGS]')
+    print('The complete list of recognized repo commands are:')
+    commandNames = list(sorted(self.commands))
 
     maxlen = 0
     for name in commandNames:
@@ -49,20 +47,16 @@ The complete list of recognized repo commands are:
         summary = command.helpSummary.strip()
       except AttributeError:
         summary = ''
-      print fmt % (name, summary)
-    print """
-See 'repo help <command>' for more information on a specific command.
-"""
+      print(fmt % (name, summary))
+    print("See 'repo help <command>' for more information on a "
+          'specific command.')
 
   def _PrintCommonCommands(self):
-    print 'usage: repo COMMAND [ARGS]'
-    print """
-The most commonly used repo commands are:
-"""
-    commandNames = [name 
-                    for name in self.commands.keys()
-                    if self.commands[name].common]
-    commandNames.sort()
+    print('usage: repo COMMAND [ARGS]')
+    print('The most commonly used repo commands are:')
+    commandNames = list(sorted([name
+                    for name, command in self.commands.items()
+                    if command.common]))
 
     maxlen = 0
     for name in commandNames:
@@ -75,11 +69,10 @@ The most commonly used repo commands are:
         summary = command.helpSummary.strip()
       except AttributeError:
         summary = ''
-      print fmt % (name, summary)
-    print """
-See 'repo help <command>' for more information on a specific command.
-See 'repo help --all' for a complete list of recognized commands.
-"""
+      print(fmt % (name, summary))
+    print(
+"See 'repo help <command>' for more information on a specific command.\n"
+"See 'repo help --all' for a complete list of recognized commands.")
 
   def _PrintCommandHelp(self, cmd):
     class _Out(Coloring):
@@ -131,7 +124,7 @@ See 'repo help --all' for a complete list of recognized commands.
 
             p('%s', title)
             self.nl()
-            p('%s', ''.ljust(len(title),section_type[0]))
+            p('%s', ''.ljust(len(title), section_type[0]))
             self.nl()
             continue
 
@@ -162,7 +155,7 @@ See 'repo help --all' for a complete list of recognized commands.
       try:
         cmd = self.commands[name]
       except KeyError:
-        print >>sys.stderr, "repo: '%s' is not a repo command." % name
+        print("repo: '%s' is not a repo command." % name, file=sys.stderr)
         sys.exit(1)
 
       cmd.manifest = self.manifest

subcmds/info.py

@@ -0,0 +1,201 @@
+#
+# Copyright (C) 2012 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from command import PagedCommand
+from color import Coloring
+from error import NoSuchProjectError
+from git_refs import R_M
+
+class _Coloring(Coloring):
+  def __init__(self, config):
+    Coloring.__init__(self, config, "status")
+
+class Info(PagedCommand):
+  common = True
+  helpSummary = "Get info on the manifest branch, current branch or unmerged branches"
+  helpUsage = "%prog [-dl] [-o [-b]] [<project>...]"
+
+  def _Options(self, p):
+    p.add_option('-d', '--diff',
+                 dest='all', action='store_true',
+                 help="show full info and commit diff including remote branches")
+    p.add_option('-o', '--overview',
+                 dest='overview', action='store_true',
+                 help='show overview of all local commits')
+    p.add_option('-b', '--current-branch',
+                 dest="current_branch", action="store_true",
+                 help="consider only checked out branches")
+    p.add_option('-l', '--local-only',
+                 dest="local", action="store_true",
+                 help="Disable all remote operations")
+
+
+  def Execute(self, opt, args):
+    self.out = _Coloring(self.manifest.globalConfig)
+    self.heading = self.out.printer('heading', attr = 'bold')
+    self.headtext = self.out.printer('headtext', fg = 'yellow')
+    self.redtext = self.out.printer('redtext', fg = 'red')
+    self.sha = self.out.printer("sha", fg = 'yellow')
+    self.text = self.out.nofmt_printer('text')
+    self.dimtext = self.out.printer('dimtext', attr = 'dim')
+
+    self.opt = opt
+
+    manifestConfig = self.manifest.manifestProject.config
+    mergeBranch = manifestConfig.GetBranch("default").merge
+    manifestGroups = (manifestConfig.GetString('manifest.groups')
+                      or 'all,-notdefault')
+
+    self.heading("Manifest branch: ")
+    self.headtext(self.manifest.default.revisionExpr)
+    self.out.nl()
+    self.heading("Manifest merge branch: ")
+    self.headtext(mergeBranch)
+    self.out.nl()
+    self.heading("Manifest groups: ")
+    self.headtext(manifestGroups)
+    self.out.nl()
+
+    self.printSeparator()
+
+    if not opt.overview:
+      self.printDiffInfo(args)
+    else:
+      self.printCommitOverview(args)
+
+  def printSeparator(self):
+    self.text("----------------------------")
+    self.out.nl()
+
+  def printDiffInfo(self, args):
+    try:
+      projs = self.GetProjects(args)
+    except NoSuchProjectError:
+      return
+
+    for p in projs:
+      self.heading("Project: ")
+      self.headtext(p.name)
+      self.out.nl()
+
+      self.heading("Mount path: ")
+      self.headtext(p.worktree)
+      self.out.nl()
+
+      self.heading("Current revision: ")
+      self.headtext(p.revisionExpr)
+      self.out.nl()
+
+      localBranches = p.GetBranches().keys()
+      self.heading("Local Branches: ")
+      self.redtext(str(len(localBranches)))
+      if len(localBranches) > 0:
+        self.text(" [")
+        self.text(", ".join(localBranches))
+        self.text("]")
+      self.out.nl()
+
+      if self.opt.all:
+        self.findRemoteLocalDiff(p)
+
+      self.printSeparator()
+
+  def findRemoteLocalDiff(self, project):
+    #Fetch all the latest commits
+    if not self.opt.local:
+      project.Sync_NetworkHalf(quiet=True, current_branch_only=True)
+
+    logTarget = R_M + self.manifest.manifestProject.config.GetBranch("default").merge
+
+    bareTmp = project.bare_git._bare
+    project.bare_git._bare = False
+    localCommits = project.bare_git.rev_list(
+        '--abbrev=8',
+        '--abbrev-commit',
+        '--pretty=oneline',
+        logTarget + "..",
+        '--')
+
+    originCommits = project.bare_git.rev_list(
+        '--abbrev=8',
+        '--abbrev-commit',
+        '--pretty=oneline',
+        ".." + logTarget,
+        '--')
+    project.bare_git._bare = bareTmp
+
+    self.heading("Local Commits: ")
+    self.redtext(str(len(localCommits)))
+    self.dimtext(" (on current branch)")
+    self.out.nl()
+
+    for c in localCommits:
+      split = c.split()
+      self.sha(split[0] + " ")
+      self.text(" ".join(split[1:]))
+      self.out.nl()
+
+    self.printSeparator()
+
+    self.heading("Remote Commits: ")
+    self.redtext(str(len(originCommits)))
+    self.out.nl()
+
+    for c in originCommits:
+      split = c.split()
+      self.sha(split[0] + " ")
+      self.text(" ".join(split[1:]))
+      self.out.nl()
+
+  def printCommitOverview(self, args):
+    all_branches = []
+    for project in self.GetProjects(args):
+      br = [project.GetUploadableBranch(x)
+            for x in project.GetBranches()]
+      br = [x for x in br if x]
+      if self.opt.current_branch:
+        br = [x for x in br if x.name == project.CurrentBranch]
+      all_branches.extend(br)
+
+    if not all_branches:
+      return
+
+    self.out.nl()
+    self.heading('Projects Overview')
+    project = None
+
+    for branch in all_branches:
+      if project != branch.project:
+        project = branch.project
+        self.out.nl()
+        self.headtext(project.relpath)
+        self.out.nl()
+
+      commits = branch.commits
+      date = branch.date
+      self.text('%s %-33s (%2d commit%s, %s)' % (
+        branch.name == project.CurrentBranch and '*' or ' ',
+        branch.name,
+        len(commits),
+        len(commits) != 1 and 's' or '',
+        date))
+      self.out.nl()
+
+      for commit in commits:
+        split = commit.split()
+        self.text('{0:38}{1} '.format('','-'))
+        self.sha(split[0] + " ")
+        self.text(" ".join(split[1:]))
+        self.out.nl()

subcmds/init.py

@@ -13,12 +13,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import os
 import platform
 import re
 import shutil
 import sys
 
+from pyversion import is_python3
+if is_python3():
+  import urllib.parse
+else:
+  import imp
+  import urlparse
+  urllib = imp.new_module('urllib')
+  urllib.parse = urlparse.urlparse
+
 from color import Coloring
 from command import InteractiveCommand, MirrorSafeCommand
 from error import ManifestParseError
@@ -89,9 +99,14 @@ to update the working directory files.
     g.add_option('--depth', type='int', default=None,
                  dest='depth',
                  help='create a shallow clone with given depth; see git clone')
+    g.add_option('--archive',
+                 dest='archive', action='store_true',
+                 help='checkout an archive instead of a git repository for '
+                      'each project. See git archive.')
     g.add_option('-g', '--groups',
-                 dest='groups', default='all,-notdefault',
-                 help='restrict manifest projects to ones with a specified group',
+                 dest='groups', default='default',
+                 help='restrict manifest projects to ones with specified '
+                      'group(s) [default|all|G1,G2,G3|G4,-G5,-G6]',
                  metavar='GROUP')
     g.add_option('-p', '--platform',
                  dest='platform', default='auto',
@@ -117,19 +132,35 @@ to update the working directory files.
                  dest='config_name', action="store_true", default=False,
                  help='Always prompt for name/e-mail')
 
+  def _RegisteredEnvironmentOptions(self):
+    return {'REPO_MANIFEST_URL': 'manifest_url',
+            'REPO_MIRROR_LOCATION': 'reference'}
+
   def _SyncManifest(self, opt):
     m = self.manifest.manifestProject
     is_new = not m.Exists
 
     if is_new:
       if not opt.manifest_url:
-        print >>sys.stderr, 'fatal: manifest url (-u) is required.'
+        print('fatal: manifest url (-u) is required.', file=sys.stderr)
         sys.exit(1)
 
       if not opt.quiet:
-        print >>sys.stderr, 'Get %s' \
-          % GitConfig.ForUser().UrlInsteadOf(opt.manifest_url)
-      m._InitGitDir()
+        print('Get %s' % GitConfig.ForUser().UrlInsteadOf(opt.manifest_url),
+              file=sys.stderr)
+
+      # The manifest project object doesn't keep track of the path on the
+      # server where this git is located, so let's save that here.
+      mirrored_manifest_git = None
+      if opt.reference:
+        manifest_git_path = urllib.parse(opt.manifest_url).path[1:]
+        mirrored_manifest_git = os.path.join(opt.reference, manifest_git_path)
+        if not mirrored_manifest_git.endswith(".git"):
+          mirrored_manifest_git += ".git"
+        if not os.path.exists(mirrored_manifest_git):
+          mirrored_manifest_git = os.path.join(opt.reference + '/.repo/manifests.git')
+
+      m._InitGitDir(mirror_git=mirrored_manifest_git)
 
       if opt.manifest_branch:
         m.revisionExpr = opt.manifest_branch
@@ -147,7 +178,7 @@ to update the working directory files.
       r.ResetFetch()
       r.Save()
 
-    groups = re.split('[,\s]+', opt.groups)
+    groups = re.split(r'[,\s]+', opt.groups)
     all_platforms = ['linux', 'darwin']
     platformize = lambda x: 'platform-' + x
     if opt.platform == 'auto':
@@ -159,28 +190,41 @@ to update the working directory files.
     elif opt.platform in all_platforms:
       groups.extend(platformize(opt.platform))
     elif opt.platform != 'none':
-      print >>sys.stderr, 'fatal: invalid platform flag'
+      print('fatal: invalid platform flag', file=sys.stderr)
       sys.exit(1)
 
     groups = [x for x in groups if x]
     groupstr = ','.join(groups)
-    if opt.platform == 'auto' and groupstr == 'all,-notdefault,platform-' + platform.system().lower():
+    if opt.platform == 'auto' and groupstr == 'default,platform-' + platform.system().lower():
       groupstr = None
     m.config.SetString('manifest.groups', groupstr)
 
     if opt.reference:
       m.config.SetString('repo.reference', opt.reference)
 
+    if opt.archive:
+      if is_new:
+        m.config.SetString('repo.archive', 'true')
+      else:
+        print('fatal: --archive is only supported when initializing a new '
+              'workspace.', file=sys.stderr)
+        print('Either delete the .repo folder in this workspace, or initialize '
+              'in another location.', file=sys.stderr)
+        sys.exit(1)
+
     if opt.mirror:
       if is_new:
         m.config.SetString('repo.mirror', 'true')
       else:
-        print >>sys.stderr, 'fatal: --mirror not supported on existing client'
+        print('fatal: --mirror is only supported when initializing a new '
+              'workspace.', file=sys.stderr)
+        print('Either delete the .repo folder in this workspace, or initialize '
+              'in another location.', file=sys.stderr)
         sys.exit(1)
 
     if not m.Sync_NetworkHalf(is_new=is_new):
       r = m.GetRemote(m.remote.name)
-      print >>sys.stderr, 'fatal: cannot obtain manifest %s' % r.url
+      print('fatal: cannot obtain manifest %s' % r.url, file=sys.stderr)
 
       # Better delete the manifest git dir if we created it; otherwise next
       # time (when user fixes problems) we won't go through the "is_new" logic.
@@ -197,19 +241,19 @@ to update the working directory files.
 
     if is_new or m.CurrentBranch is None:
       if not m.StartBranch('default'):
-        print >>sys.stderr, 'fatal: cannot create default in manifest'
+        print('fatal: cannot create default in manifest', file=sys.stderr)
         sys.exit(1)
 
   def _LinkManifest(self, name):
     if not name:
-      print >>sys.stderr, 'fatal: manifest name (-m) is required.'
+      print('fatal: manifest name (-m) is required.', file=sys.stderr)
       sys.exit(1)
 
     try:
       self.manifest.Link(name)
     except ManifestParseError as e:
-      print >>sys.stderr, "fatal: manifest '%s' not available" % name
-      print >>sys.stderr, 'fatal: %s' % str(e)
+      print("fatal: manifest '%s' not available" % name, file=sys.stderr)
+      print('fatal: %s' % str(e), file=sys.stderr)
       sys.exit(1)
 
   def _Prompt(self, prompt, value):
@@ -231,24 +275,24 @@ to update the working directory files.
       mp.config.SetString('user.name', gc.GetString('user.name'))
       mp.config.SetString('user.email', gc.GetString('user.email'))
 
-    print ''
-    print 'Your identity is: %s <%s>' % (mp.config.GetString('user.name'),
-                                         mp.config.GetString('user.email'))
-    print 'If you want to change this, please re-run \'repo init\' with --config-name'
+    print()
+    print('Your identity is: %s <%s>' % (mp.config.GetString('user.name'),
+                                         mp.config.GetString('user.email')))
+    print('If you want to change this, please re-run \'repo init\' with --config-name')
     return False
 
   def _ConfigureUser(self):
     mp = self.manifest.manifestProject
 
     while True:
-      print ''
+      print()
       name  = self._Prompt('Your Name', mp.UserName)
       email = self._Prompt('Your Email', mp.UserEmail)
 
-      print ''
-      print 'Your identity is: %s <%s>' % (name, email)
+      print()
+      print('Your identity is: %s <%s>' % (name, email))
       sys.stdout.write('is this correct [y/N]? ')
-      a = sys.stdin.readline().strip()
+      a = sys.stdin.readline().strip().lower()
       if a in ('yes', 'y', 't', 'true'):
         break
 
@@ -274,17 +318,17 @@ to update the working directory files.
         self._on = True
     out = _Test()
 
-    print ''
-    print "Testing colorized output (for 'repo diff', 'repo status'):"
+    print()
+    print("Testing colorized output (for 'repo diff', 'repo status'):")
 
-    for c in ['black','red','green','yellow','blue','magenta','cyan']:
+    for c in ['black', 'red', 'green', 'yellow', 'blue', 'magenta', 'cyan']:
       out.write(' ')
       out.printer(fg=c)(' %-6s ', c)
     out.write(' ')
     out.printer(fg='white', bg='black')(' %s ' % 'white')
     out.nl()
 
-    for c in ['bold','dim','ul','reverse']:
+    for c in ['bold', 'dim', 'ul', 'reverse']:
       out.write(' ')
       out.printer(fg='black', attr=c)(' %-6s ', c)
     out.nl()
@@ -313,12 +357,36 @@ to update the working directory files.
       # We store the depth in the main manifest project.
       self.manifest.manifestProject.config.SetString('repo.depth', depth)
 
+  def _DisplayResult(self):
+    if self.manifest.IsMirror:
+      init_type = 'mirror '
+    else:
+      init_type = ''
+
+    print()
+    print('repo %shas been initialized in %s'
+          % (init_type, self.manifest.topdir))
+
+    current_dir = os.getcwd()
+    if current_dir != self.manifest.topdir:
+      print('If this is not the directory in which you want to initialize '
+            'repo, please run:')
+      print('   rm -r %s/.repo' % self.manifest.topdir)
+      print('and try again.')
+
   def Execute(self, opt, args):
     git_require(MIN_GIT_VERSION, fail=True)
 
     if opt.reference:
       opt.reference = os.path.expanduser(opt.reference)
 
+    # Check this here, else manifest will be tagged "not new" and init won't be
+    # possible anymore without removing the .repo/manifests directory.
+    if opt.archive and opt.mirror:
+      print('fatal: --mirror and --archive cannot be used together.',
+            file=sys.stderr)
+      sys.exit(1)
+
     self._SyncManifest(opt)
     self._LinkManifest(opt.manifest_name)
 
@@ -329,10 +397,4 @@ to update the working directory files.
 
     self._ConfigureDepth(opt)
 
-    if self.manifest.IsMirror:
-      init_type = 'mirror '
-    else:
-      init_type = ''
-
-    print ''
-    print 'repo %sinitialized in %s' % (init_type, self.manifest.topdir)
+    self._DisplayResult()

subcmds/list.py

@@ -13,7 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import re
+from __future__ import print_function
+import sys
 
 from command import Command, MirrorSafeCommand
 
@@ -30,13 +31,19 @@ List all projects; pass '.' to list the project for the cwd.
 This is similar to running: repo forall -c 'echo "$REPO_PATH : $REPO_PROJECT"'.
 """
 
-  def _Options(self, p, show_smart=True):
+  def _Options(self, p):
     p.add_option('-r', '--regex',
                  dest='regex', action='store_true',
                  help="Filter the project list based on regex or wildcard matching of strings")
     p.add_option('-f', '--fullpath',
                  dest='fullpath', action='store_true',
                  help="Display the full work tree path instead of the relative path")
+    p.add_option('-n', '--name-only',
+                 dest='name_only', action='store_true',
+                 help="Display only the name of the repository")
+    p.add_option('-p', '--path-only',
+                 dest='path_only', action='store_true',
+                 help="Display only the path of the repository")
 
   def Execute(self, opt, args):
     """List all projects and the associated directories.
@@ -49,6 +56,11 @@ This is similar to running: repo forall -c 'echo "$REPO_PATH : $REPO_PROJECT"'.
       opt: The options.
       args: Positional args.  Can be a list of projects to list, or empty.
     """
+
+    if opt.fullpath and opt.name_only:
+      print('error: cannot combine -f and -n', file=sys.stderr)
+      sys.exit(1)
+
     if not opt.regex:
       projects = self.GetProjects(args)
     else:
@@ -61,18 +73,12 @@ This is similar to running: repo forall -c 'echo "$REPO_PATH : $REPO_PROJECT"'.
 
     lines = []
     for project in projects:
-      lines.append("%s : %s" % (_getpath(project), project.name))
+      if opt.name_only and not opt.path_only:
+        lines.append("%s" % ( project.name))
+      elif opt.path_only and not opt.name_only:
+        lines.append("%s" % (_getpath(project)))
+      else:
+        lines.append("%s : %s" % (_getpath(project), project.name))
 
     lines.sort()
-    print '\n'.join(lines)
-
-  def FindProjects(self, args):
-    result = []
-    for project in self.GetProjects(''):
-      for arg in args:
-        pattern = re.compile(r'%s' % arg, re.IGNORECASE)
-        if pattern.search(project.name) or pattern.search(project.relpath):
-          result.append(project)
-          break
-    result.sort(key=lambda project: project.relpath)
-    return result
+    print('\n'.join(lines))

subcmds/manifest.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import os
 import sys
 
@@ -69,7 +70,7 @@ in a Git repository for use during future 'repo init' invocations.
                        peg_rev_upstream = opt.peg_rev_upstream)
     fd.close()
     if opt.output_file != '-':
-      print >>sys.stderr, 'Saved manifest to %s' % opt.output_file
+      print('Saved manifest to %s' % opt.output_file, file=sys.stderr)
 
   def Execute(self, opt, args):
     if args:
@@ -79,6 +80,6 @@ in a Git repository for use during future 'repo init' invocations.
       self._Output(opt)
       return
 
-    print >>sys.stderr, 'error: no operation to perform'
-    print >>sys.stderr, 'error: see repo help manifest'
+    print('error: no operation to perform', file=sys.stderr)
+    print('error: see repo help manifest', file=sys.stderr)
     sys.exit(1)

subcmds/overview.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 from color import Coloring
 from command import PagedCommand
 
@@ -41,7 +42,7 @@ are displayed.
     all_branches = []
     for project in self.GetProjects(args):
       br = [project.GetUploadableBranch(x)
-            for x in project.GetBranches().keys()]
+            for x in project.GetBranches()]
       br = [x for x in br if x]
       if opt.current_branch:
         br = [x for x in br if x.name == project.CurrentBranch]
@@ -54,8 +55,11 @@ are displayed.
       def __init__(self, config):
         Coloring.__init__(self, config, 'status')
         self.project = self.printer('header', attr='bold')
+        self.text = self.printer('text')
 
     out = Report(all_branches[0].project.config)
+    out.text("Deprecated. See repo info -o.")
+    out.nl()
     out.project('Projects Overview')
     out.nl()
 
@@ -70,11 +74,11 @@ are displayed.
 
       commits = branch.commits
       date = branch.date
-      print '%s %-33s (%2d commit%s, %s)' % (
+      print('%s %-33s (%2d commit%s, %s)' % (
             branch.name == project.CurrentBranch and '*' or ' ',
             branch.name,
             len(commits),
             len(commits) != 1 and 's' or ' ',
-            date)
+            date))
       for commit in commits:
-        print '%-35s   - %s' % ('', commit)
+        print('%-35s   - %s' % ('', commit))

subcmds/prune.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 from color import Coloring
 from command import PagedCommand
 
@@ -51,9 +52,9 @@ class Prune(PagedCommand):
 
       commits = branch.commits
       date = branch.date
-      print '%s %-33s (%2d commit%s, %s)' % (
+      print('%s %-33s (%2d commit%s, %s)' % (
             branch.name == project.CurrentBranch and '*' or ' ',
             branch.name,
             len(commits),
             len(commits) != 1 and 's' or ' ',
-            date)
+            date))

subcmds/rebase.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import sys
 
 from command import Command
@@ -59,14 +60,19 @@ branch but need to incorporate new upstream changes "underneath" them.
     one_project = len(all_projects) == 1
 
     if opt.interactive and not one_project:
-      print >>sys.stderr, 'error: interactive rebase not supported with multiple projects'
+      print('error: interactive rebase not supported with multiple projects',
+            file=sys.stderr)
+      if len(args) == 1:
+        print('note: project %s is mapped to more than one path' % (args[0],),
+            file=sys.stderr)
       return -1
 
     for project in all_projects:
       cb = project.CurrentBranch
       if not cb:
         if one_project:
-          print >>sys.stderr, "error: project %s has a detatched HEAD" % project.relpath
+          print("error: project %s has a detached HEAD" % project.relpath,
+                file=sys.stderr)
           return -1
         # ignore branches with detatched HEADs
         continue
@@ -74,7 +80,8 @@ branch but need to incorporate new upstream changes "underneath" them.
       upbranch = project.GetBranch(cb)
       if not upbranch.LocalMerge:
         if one_project:
-          print >>sys.stderr, "error: project %s does not track any remote branches" % project.relpath
+          print("error: project %s does not track any remote branches"
+                % project.relpath, file=sys.stderr)
           return -1
         # ignore branches without remotes
         continue
@@ -101,8 +108,8 @@ branch but need to incorporate new upstream changes "underneath" them.
 
       args.append(upbranch.LocalMerge)
 
-      print >>sys.stderr, '# %s: rebasing %s -> %s' % \
-        (project.relpath, cb, upbranch.LocalMerge)
+      print('# %s: rebasing %s -> %s'
+            % (project.relpath, cb, upbranch.LocalMerge), file=sys.stderr)
 
       needs_stash = False
       if opt.auto_stash:

subcmds/selfupdate.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 from optparse import SUPPRESS_HELP
 import sys
 
@@ -52,7 +53,7 @@ need to be performed by an end-user.
 
     else:
       if not rp.Sync_NetworkHalf():
-        print >>sys.stderr, "error: can't update repo"
+        print("error: can't update repo", file=sys.stderr)
         sys.exit(1)
 
       rp.bare_git.gc('--auto')

subcmds/stage.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import sys
 
 from color import Coloring
@@ -48,9 +49,9 @@ The '%prog' command stages files to prepare the next commit.
       self.Usage()
 
   def _Interactive(self, opt, args):
-    all_projects = filter(lambda x: x.IsDirty(), self.GetProjects(args))
+    all_projects = [p for p in self.GetProjects(args) if p.IsDirty()]
     if not all_projects:
-      print >>sys.stderr,'no projects have uncommitted modifications'
+      print('no projects have uncommitted modifications', file=sys.stderr)
       return
 
     out = _ProjectList(self.manifest.manifestProject.config)
@@ -58,7 +59,7 @@ The '%prog' command stages files to prepare the next commit.
       out.header('        %s', 'project')
       out.nl()
 
-      for i in xrange(0, len(all_projects)):
+      for i in range(len(all_projects)):
         p = all_projects[i]
         out.write('%3d:    %s', i + 1, p.relpath + '/')
         out.nl()
@@ -97,11 +98,11 @@ The '%prog' command stages files to prepare the next commit.
           _AddI(all_projects[a_index - 1])
           continue
 
-      p = filter(lambda x: x.name == a or x.relpath == a, all_projects)
-      if len(p) == 1:
-        _AddI(p[0])
+      projects = [p for p in all_projects if a in [p.name, p.relpath]]
+      if len(projects) == 1:
+        _AddI(projects[0])
         continue
-    print 'Bye.'
+    print('Bye.')
 
 def _AddI(project):
   p = GitCommand(project, ['add', '--interactive'], bare=False)

subcmds/start.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import sys
 from command import Command
 from git_config import IsId
@@ -41,7 +42,7 @@ revision specified in the manifest.
 
     nb = args[0]
     if not git.check_ref_format('heads/%s' % nb):
-      print >>sys.stderr, "error: '%s' is not a valid name" % nb
+      print("error: '%s' is not a valid name" % nb, file=sys.stderr)
       sys.exit(1)
 
     err = []
@@ -49,7 +50,7 @@ revision specified in the manifest.
     if not opt.all:
       projects = args[1:]
       if len(projects) < 1:
-        print >>sys.stderr, "error: at least one project must be specified"
+        print("error: at least one project must be specified", file=sys.stderr)
         sys.exit(1)
 
     all_projects = self.GetProjects(projects)
@@ -67,7 +68,6 @@ revision specified in the manifest.
 
     if err:
       for p in err:
-        print >>sys.stderr,\
-          "error: %s/: cannot start %s" \
-          % (p.relpath, nb)
+        print("error: %s/: cannot start %s" % (p.relpath, nb),
+              file=sys.stderr)
       sys.exit(1)

subcmds/status.py

@@ -20,9 +20,19 @@ try:
 except ImportError:
   import dummy_threading as _threading
 
+import glob
+
+from pyversion import is_python3
+if is_python3():
+  import io
+else:
+  import StringIO as io
+
 import itertools
+import os
 import sys
-import StringIO
+
+from color import Coloring
 
 class Status(PagedCommand):
   common = True
@@ -39,6 +49,13 @@ is a difference between these three states.
 The -j/--jobs option can be used to run multiple status queries
 in parallel.
 
+The -o/--orphans option can be used to show objects that are in
+the working directory, but not associated with a repo project.
+This includes unmanaged top-level files and directories, but also
+includes deeper items.  For example, if dir/subdir/proj1 and
+dir/subdir/proj2 are repo projects, dir/subdir/proj3 will be shown
+if it is not known to repo.
+
 Status Display
 --------------
 
@@ -76,6 +93,9 @@ the following meanings:
     p.add_option('-j', '--jobs',
                  dest='jobs', action='store', type='int', default=2,
                  help="number of projects to check simultaneously")
+    p.add_option('-o', '--orphans',
+                 dest='orphans', action='store_true',
+                 help="include objects in working directory outside of repo projects")
 
   def _StatusHelper(self, project, clean_counter, sem, output):
     """Obtains the status for a specific project.
@@ -97,6 +117,22 @@ the following meanings:
     finally:
       sem.release()
 
+  def _FindOrphans(self, dirs, proj_dirs, proj_dirs_parents, outstring):
+    """find 'dirs' that are present in 'proj_dirs_parents' but not in 'proj_dirs'"""
+    status_header = ' --\t'
+    for item in dirs:
+      if not os.path.isdir(item):
+        outstring.write(''.join([status_header, item]))
+        continue
+      if item in proj_dirs:
+        continue
+      if item in proj_dirs_parents:
+        self._FindOrphans(glob.glob('%s/.*' % item) + \
+            glob.glob('%s/*' % item), \
+            proj_dirs, proj_dirs_parents, outstring)
+        continue
+      outstring.write(''.join([status_header, item, '/']))
+
   def Execute(self, opt, args):
     all_projects = self.GetProjects(args)
     counter = itertools.count()
@@ -112,7 +148,7 @@ the following meanings:
       for project in all_projects:
         sem.acquire()
 
-        class BufList(StringIO.StringIO):
+        class BufList(io.StringIO):
           def dump(self, ostream):
             for entry in self.buflist:
               ostream.write(entry)
@@ -129,4 +165,46 @@ the following meanings:
         output.dump(sys.stdout)
         output.close()
     if len(all_projects) == counter.next():
-      print 'nothing to commit (working directory clean)'
+      print('nothing to commit (working directory clean)')
+
+    if opt.orphans:
+      proj_dirs = set()
+      proj_dirs_parents = set()
+      for project in self.GetProjects(None, missing_ok=True):
+        proj_dirs.add(project.relpath)
+        (head, _tail) = os.path.split(project.relpath)
+        while head != "":
+          proj_dirs_parents.add(head)
+          (head, _tail) = os.path.split(head)
+      proj_dirs.add('.repo')
+
+      class StatusColoring(Coloring):
+        def __init__(self, config):
+          Coloring.__init__(self, config, 'status')
+          self.project = self.printer('header', attr = 'bold')
+          self.untracked = self.printer('untracked', fg = 'red')
+
+      orig_path = os.getcwd()
+      try:
+        os.chdir(self.manifest.topdir)
+
+        outstring = io.StringIO()
+        self._FindOrphans(glob.glob('.*') + \
+            glob.glob('*'), \
+            proj_dirs, proj_dirs_parents, outstring)
+
+        if outstring.buflist:
+          output = StatusColoring(self.manifest.globalConfig)
+          output.project('Objects not within a project (orphans)')
+          output.nl()
+          for entry in outstring.buflist:
+            output.untracked(entry)
+            output.nl()
+        else:
+          print('No orphan files or directories')
+
+        outstring.close()
+
+      finally:
+        # Restore CWD.
+        os.chdir(orig_path)

subcmds/sync.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import netrc
 from optparse import SUPPRESS_HELP
 import os
@@ -23,8 +24,19 @@ import socket
 import subprocess
 import sys
 import time
-import urlparse
-import xmlrpclib
+
+from pyversion import is_python3
+if is_python3():
+  import urllib.parse
+  import xmlrpc.client
+else:
+  import imp
+  import urlparse
+  import xmlrpclib
+  urllib = imp.new_module('urllib')
+  urllib.parse = urlparse
+  xmlrpc = imp.new_module('xmlrpc')
+  xmlrpc.client = xmlrpclib
 
 try:
   import threading as _threading
@@ -44,13 +56,13 @@ try:
 except ImportError:
   multiprocessing = None
 
-from git_command import GIT
+from git_command import GIT, git_require
 from git_refs import R_HEADS, HEAD
 from main import WrapperModule
 from project import Project
 from project import RemoteSpec
 from command import Command, MirrorSafeCommand
-from error import RepoChangedException, GitError
+from error import RepoChangedException, GitError, ManifestParseError
 from project import SyncBuffer
 from progress import Progress
 
@@ -113,6 +125,9 @@ resumeable bundle file on a content delivery network. This
 may be necessary if there are problems with the local Python
 HTTP client or proxy configuration, but the Git binary works.
 
+The --fetch-submodules option enables fetching Git submodules
+of a project from server.
+
 SSH Connections
 ---------------
 
@@ -144,27 +159,30 @@ later is required to fix a server side protocol bug.
 """
 
   def _Options(self, p, show_smart=True):
-    self.jobs = self.manifest.default.sync_j
+    try:
+      self.jobs = self.manifest.default.sync_j
+    except ManifestParseError:
+      self.jobs = 1
 
     p.add_option('-f', '--force-broken',
                  dest='force_broken', action='store_true',
                  help="continue sync even if a project fails to sync")
-    p.add_option('-l','--local-only',
+    p.add_option('-l', '--local-only',
                  dest='local_only', action='store_true',
                  help="only update working tree, don't fetch")
-    p.add_option('-n','--network-only',
+    p.add_option('-n', '--network-only',
                  dest='network_only', action='store_true',
                  help="fetch only, don't update working tree")
-    p.add_option('-d','--detach',
+    p.add_option('-d', '--detach',
                  dest='detach_head', action='store_true',
                  help='detach projects back to manifest revision')
-    p.add_option('-c','--current-branch',
+    p.add_option('-c', '--current-branch',
                  dest='current_branch_only', action='store_true',
                  help='fetch only current branch from server')
-    p.add_option('-q','--quiet',
+    p.add_option('-q', '--quiet',
                  dest='quiet', action='store_true',
                  help='be more quiet')
-    p.add_option('-j','--jobs',
+    p.add_option('-j', '--jobs',
                  dest='jobs', action='store', type='int',
                  help="projects to fetch simultaneously (default %d)" % self.jobs)
     p.add_option('-m', '--manifest-name',
@@ -173,6 +191,18 @@ later is required to fix a server side protocol bug.
     p.add_option('--no-clone-bundle',
                  dest='no_clone_bundle', action='store_true',
                  help='disable use of /clone.bundle on HTTP/HTTPS')
+    p.add_option('-u', '--manifest-server-username', action='store',
+                 dest='manifest_server_username',
+                 help='username to authenticate with the manifest server')
+    p.add_option('-p', '--manifest-server-password', action='store',
+                 dest='manifest_server_password',
+                 help='password to authenticate with the manifest server')
+    p.add_option('--fetch-submodules',
+                 dest='fetch_submodules', action='store_true',
+                 help='fetch submodules from server')
+    p.add_option('--no-tags',
+                 dest='no_tags', action='store_true',
+                 help="don't fetch tags")
     if show_smart:
       p.add_option('-s', '--smart-sync',
                    dest='smart_sync', action='store_true',
@@ -180,12 +210,6 @@ later is required to fix a server side protocol bug.
       p.add_option('-t', '--smart-tag',
                    dest='smart_tag', action='store',
                    help='smart sync using manifest from a known tag')
-      p.add_option('-u', '--manifest-server-username', action='store',
-                   dest='manifest_server_username',
-                   help='username to authenticate with the manifest server')
-      p.add_option('-p', '--manifest-server-password', action='store',
-                   dest='manifest_server_password',
-                   help='password to authenticate with the manifest server')
 
     g = p.add_option_group('repo Version options')
     g.add_option('--no-repo-verify',
@@ -195,62 +219,88 @@ later is required to fix a server side protocol bug.
                  dest='repo_upgraded', action='store_true',
                  help=SUPPRESS_HELP)
 
+  def _FetchProjectList(self, opt, projects, *args):
+    """Main function of the fetch threads when jobs are > 1.
+
+    Delegates most of the work to _FetchHelper.
+
+    Args:
+      opt: Program options returned from optparse.  See _Options().
+      projects: Projects to fetch.
+      *args: Remaining arguments to pass to _FetchHelper. See the
+          _FetchHelper docstring for details.
+    """
+    for project in projects:
+      success = self._FetchHelper(opt, project, *args)
+      if not success and not opt.force_broken:
+        break
+
   def _FetchHelper(self, opt, project, lock, fetched, pm, sem, err_event):
-      """Main function of the fetch threads when jobs are > 1.
+    """Fetch git objects for a single project.
 
-      Args:
-        opt: Program options returned from optparse.  See _Options().
-        project: Project object for the project to fetch.
-        lock: Lock for accessing objects that are shared amongst multiple
-            _FetchHelper() threads.
-        fetched: set object that we will add project.gitdir to when we're done
-            (with our lock held).
-        pm: Instance of a Project object.  We will call pm.update() (with our
-            lock held).
-        sem: We'll release() this semaphore when we exit so that another thread
-            can be started up.
-        err_event: We'll set this event in the case of an error (after printing
-            out info about the error).
-      """
-      # We'll set to true once we've locked the lock.
-      did_lock = False
+    Args:
+      opt: Program options returned from optparse.  See _Options().
+      project: Project object for the project to fetch.
+      lock: Lock for accessing objects that are shared amongst multiple
+          _FetchHelper() threads.
+      fetched: set object that we will add project.gitdir to when we're done
+          (with our lock held).
+      pm: Instance of a Project object.  We will call pm.update() (with our
+          lock held).
+      sem: We'll release() this semaphore when we exit so that another thread
+          can be started up.
+      err_event: We'll set this event in the case of an error (after printing
+          out info about the error).
 
-      # Encapsulate everything in a try/except/finally so that:
-      # - We always set err_event in the case of an exception.
-      # - We always make sure we call sem.release().
-      # - We always make sure we unlock the lock if we locked it.
+    Returns:
+      Whether the fetch was successful.
+    """
+    # We'll set to true once we've locked the lock.
+    did_lock = False
+
+    if not opt.quiet:
+      print('Fetching project %s' % project.name)
+
+    # Encapsulate everything in a try/except/finally so that:
+    # - We always set err_event in the case of an exception.
+    # - We always make sure we call sem.release().
+    # - We always make sure we unlock the lock if we locked it.
+    try:
       try:
-        try:
-          start = time.time()
-          success = project.Sync_NetworkHalf(
-            quiet=opt.quiet,
-            current_branch_only=opt.current_branch_only,
-            clone_bundle=not opt.no_clone_bundle)
-          self._fetch_times.Set(project, time.time() - start)
+        start = time.time()
+        success = project.Sync_NetworkHalf(
+          quiet=opt.quiet,
+          current_branch_only=opt.current_branch_only,
+          clone_bundle=not opt.no_clone_bundle,
+          no_tags=opt.no_tags, archive=self.manifest.IsArchive)
+        self._fetch_times.Set(project, time.time() - start)
 
-          # Lock around all the rest of the code, since printing, updating a set
-          # and Progress.update() are not thread safe.
-          lock.acquire()
-          did_lock = True
+        # Lock around all the rest of the code, since printing, updating a set
+        # and Progress.update() are not thread safe.
+        lock.acquire()
+        did_lock = True
 
-          if not success:
-            print >>sys.stderr, 'error: Cannot fetch %s' % project.name
-            if opt.force_broken:
-              print >>sys.stderr, 'warn: --force-broken, continuing to sync'
-            else:
-              raise _FetchError()
+        if not success:
+          print('error: Cannot fetch %s' % project.name, file=sys.stderr)
+          if opt.force_broken:
+            print('warn: --force-broken, continuing to sync',
+                  file=sys.stderr)
+          else:
+            raise _FetchError()
 
-          fetched.add(project.gitdir)
-          pm.update()
-        except _FetchError:
-          err_event.set()
-        except:
-          err_event.set()
-          raise
-      finally:
-        if did_lock:
-          lock.release()
-        sem.release()
+        fetched.add(project.gitdir)
+        pm.update()
+      except _FetchError:
+        err_event.set()
+      except:
+        err_event.set()
+        raise
+    finally:
+      if did_lock:
+        lock.release()
+      sem.release()
+
+    return success
 
   def _Fetch(self, projects, opt):
     fetched = set()
@@ -259,32 +309,40 @@ later is required to fix a server side protocol bug.
     if self.jobs == 1:
       for project in projects:
         pm.update()
+        if not opt.quiet:
+          print('Fetching project %s' % project.name)
         if project.Sync_NetworkHalf(
             quiet=opt.quiet,
             current_branch_only=opt.current_branch_only,
-            clone_bundle=not opt.no_clone_bundle):
+            clone_bundle=not opt.no_clone_bundle,
+            no_tags=opt.no_tags,
+            archive=self.manifest.IsArchive):
           fetched.add(project.gitdir)
         else:
-          print >>sys.stderr, 'error: Cannot fetch %s' % project.name
+          print('error: Cannot fetch %s' % project.name, file=sys.stderr)
           if opt.force_broken:
-            print >>sys.stderr, 'warn: --force-broken, continuing to sync'
+            print('warn: --force-broken, continuing to sync', file=sys.stderr)
           else:
             sys.exit(1)
     else:
+      objdir_project_map = dict()
+      for project in projects:
+        objdir_project_map.setdefault(project.objdir, []).append(project)
+
       threads = set()
       lock = _threading.Lock()
       sem = _threading.Semaphore(self.jobs)
       err_event = _threading.Event()
-      for project in projects:
+      for project_list in objdir_project_map.values():
         # Check for any errors before starting any new threads.
         # ...we'll let existing threads finish, though.
         if err_event.isSet():
           break
 
         sem.acquire()
-        t = _threading.Thread(target = self._FetchHelper,
+        t = _threading.Thread(target = self._FetchProjectList,
                               args = (opt,
-                                      project,
+                                      project_list,
                                       lock,
                                       fetched,
                                       pm,
@@ -300,25 +358,32 @@ later is required to fix a server side protocol bug.
 
       # If we saw an error, exit with code 1 so that other scripts can check.
       if err_event.isSet():
-        print >>sys.stderr, '\nerror: Exited sync due to fetch errors'
+        print('\nerror: Exited sync due to fetch errors', file=sys.stderr)
         sys.exit(1)
 
     pm.end()
     self._fetch_times.Save()
 
-    self._GCProjects(projects)
+    if not self.manifest.IsArchive:
+      self._GCProjects(projects)
+
     return fetched
 
   def _GCProjects(self, projects):
-    if multiprocessing:
+    gitdirs = {}
+    for project in projects:
+      gitdirs[project.gitdir] = project.bare_git
+
+    has_dash_c = git_require((1, 7, 2))
+    if multiprocessing and has_dash_c:
       cpu_count = multiprocessing.cpu_count()
     else:
       cpu_count = 1
     jobs = min(self.jobs, cpu_count)
 
     if jobs < 2:
-      for project in projects:
-        project.bare_git.gc('--auto')
+      for bare_git in gitdirs.values():
+        bare_git.gc('--auto')
       return
 
     config = {'pack.threads': cpu_count / jobs if cpu_count > jobs else 1}
@@ -327,10 +392,10 @@ later is required to fix a server side protocol bug.
     sem = _threading.Semaphore(jobs)
     err_event = _threading.Event()
 
-    def GC(project):
+    def GC(bare_git):
       try:
         try:
-          project.bare_git.gc('--auto', config=config)
+          bare_git.gc('--auto', config=config)
         except GitError:
           err_event.set()
         except:
@@ -339,11 +404,11 @@ later is required to fix a server side protocol bug.
       finally:
         sem.release()
 
-    for project in projects:
+    for bare_git in gitdirs.values():
       if err_event.isSet():
         break
       sem.acquire()
-      t = _threading.Thread(target=GC, args=(project,))
+      t = _threading.Thread(target=GC, args=(bare_git,))
       t.daemon = True
       threads.add(t)
       t.start()
@@ -352,9 +417,16 @@ later is required to fix a server side protocol bug.
       t.join()
 
     if err_event.isSet():
-      print >>sys.stderr, '\nerror: Exited sync due to gc errors'
+      print('\nerror: Exited sync due to gc errors', file=sys.stderr)
       sys.exit(1)
 
+  def _ReloadManifest(self, manifest_name=None):
+    if manifest_name:
+      # Override calls _Unload already
+      self.manifest.Override(manifest_name)
+    else:
+      self.manifest._Unload()
+
   def UpdateProjectList(self):
     new_project_paths = []
     for project in self.GetProjects(None, missing_ok=True):
@@ -376,34 +448,37 @@ later is required to fix a server side protocol bug.
         if path not in new_project_paths:
           # If the path has already been deleted, we don't need to do it
           if os.path.exists(self.manifest.topdir + '/' + path):
-              project = Project(
-                             manifest = self.manifest,
-                             name = path,
-                             remote = RemoteSpec('origin'),
-                             gitdir = os.path.join(self.manifest.topdir,
-                                                   path, '.git'),
-                             worktree = os.path.join(self.manifest.topdir, path),
-                             relpath = path,
-                             revisionExpr = 'HEAD',
-                             revisionId = None,
-                             groups = None)
+            gitdir = os.path.join(self.manifest.topdir, path, '.git')
+            project = Project(
+                           manifest = self.manifest,
+                           name = path,
+                           remote = RemoteSpec('origin'),
+                           gitdir = gitdir,
+                           objdir = gitdir,
+                           worktree = os.path.join(self.manifest.topdir, path),
+                           relpath = path,
+                           revisionExpr = 'HEAD',
+                           revisionId = None,
+                           groups = None)
 
-              if project.IsDirty():
-                print >>sys.stderr, 'error: Cannot remove project "%s": \
-uncommitted changes are present' % project.relpath
-                print >>sys.stderr, '       commit changes, then run sync again'
-                return -1
-              else:
-                print >>sys.stderr, 'Deleting obsolete path %s' % project.worktree
-                shutil.rmtree(project.worktree)
-                # Try deleting parent subdirs if they are empty
-                project_dir = os.path.dirname(project.worktree)
-                while project_dir != self.manifest.topdir:
-                  try:
-                    os.rmdir(project_dir)
-                  except OSError:
-                    break
-                  project_dir = os.path.dirname(project_dir)
+            if project.IsDirty():
+              print('error: Cannot remove project "%s": uncommitted changes '
+                    'are present' % project.relpath, file=sys.stderr)
+              print('       commit changes, then run sync again',
+                    file=sys.stderr)
+              return -1
+            else:
+              print('Deleting obsolete path %s' % project.worktree,
+                    file=sys.stderr)
+              shutil.rmtree(project.worktree)
+              # Try deleting parent subdirs if they are empty
+              project_dir = os.path.dirname(project.worktree)
+              while project_dir != self.manifest.topdir:
+                try:
+                  os.rmdir(project_dir)
+                except OSError:
+                  break
+                project_dir = os.path.dirname(project_dir)
 
     new_project_paths.sort()
     fd = open(file_path, 'w')
@@ -422,36 +497,40 @@ uncommitted changes are present' % project.relpath
       self.jobs = min(self.jobs, (soft_limit - 5) / 3)
 
     if opt.network_only and opt.detach_head:
-      print >>sys.stderr, 'error: cannot combine -n and -d'
+      print('error: cannot combine -n and -d', file=sys.stderr)
       sys.exit(1)
     if opt.network_only and opt.local_only:
-      print >>sys.stderr, 'error: cannot combine -n and -l'
+      print('error: cannot combine -n and -l', file=sys.stderr)
       sys.exit(1)
     if opt.manifest_name and opt.smart_sync:
-      print >>sys.stderr, 'error: cannot combine -m and -s'
+      print('error: cannot combine -m and -s', file=sys.stderr)
       sys.exit(1)
     if opt.manifest_name and opt.smart_tag:
-      print >>sys.stderr, 'error: cannot combine -m and -t'
+      print('error: cannot combine -m and -t', file=sys.stderr)
       sys.exit(1)
     if opt.manifest_server_username or opt.manifest_server_password:
       if not (opt.smart_sync or opt.smart_tag):
-        print >>sys.stderr, 'error: -u and -p may only be combined with ' \
-                            '-s or -t'
+        print('error: -u and -p may only be combined with -s or -t',
+              file=sys.stderr)
         sys.exit(1)
       if None in [opt.manifest_server_username, opt.manifest_server_password]:
-        print >>sys.stderr, 'error: both -u and -p must be given'
+        print('error: both -u and -p must be given', file=sys.stderr)
         sys.exit(1)
 
     if opt.manifest_name:
       self.manifest.Override(opt.manifest_name)
 
+    manifest_name = opt.manifest_name
+
     if opt.smart_sync or opt.smart_tag:
       if not self.manifest.manifest_server:
-        print >>sys.stderr, \
-            'error: cannot smart sync: no manifest server defined in manifest'
+        print('error: cannot smart sync: no manifest server defined in '
+              'manifest', file=sys.stderr)
         sys.exit(1)
 
       manifest_server = self.manifest.manifest_server
+      if not opt.quiet:
+        print('Using manifest server %s' % manifest_server)
 
       if not '@' in manifest_server:
         username = None
@@ -463,20 +542,21 @@ uncommitted changes are present' % project.relpath
           try:
             info = netrc.netrc()
           except IOError:
-            print >>sys.stderr, '.netrc file does not exist or could not be opened'
+            print('.netrc file does not exist or could not be opened',
+                  file=sys.stderr)
           else:
             try:
-              parse_result = urlparse.urlparse(manifest_server)
+              parse_result = urllib.parse.urlparse(manifest_server)
               if parse_result.hostname:
                 username, _account, password = \
                   info.authenticators(parse_result.hostname)
             except TypeError:
               # TypeError is raised when the given hostname is not present
               # in the .netrc file.
-              print >>sys.stderr, 'No credentials found for %s in .netrc' % \
-                                  parse_result.hostname
+              print('No credentials found for %s in .netrc'
+                    % parse_result.hostname, file=sys.stderr)
             except netrc.NetrcParseError as e:
-              print >>sys.stderr, 'Error parsing .netrc file: %s' % e
+              print('Error parsing .netrc file: %s' % e, file=sys.stderr)
 
         if (username and password):
           manifest_server = manifest_server.replace('://', '://%s:%s@' %
@@ -484,7 +564,7 @@ uncommitted changes are present' % project.relpath
                                                     1)
 
       try:
-        server = xmlrpclib.Server(manifest_server)
+        server = xmlrpc.client.Server(manifest_server)
         if opt.smart_sync:
           p = self.manifest.manifestProject
           b = p.GetBranch(p.CurrentBranch)
@@ -493,8 +573,7 @@ uncommitted changes are present' % project.relpath
             branch = branch[len(R_HEADS):]
 
           env = os.environ.copy()
-          if (env.has_key('TARGET_PRODUCT') and
-              env.has_key('TARGET_BUILD_VARIANT')):
+          if 'TARGET_PRODUCT' in env and 'TARGET_BUILD_VARIANT' in env:
             target = '%s-%s' % (env['TARGET_PRODUCT'],
                                 env['TARGET_BUILD_VARIANT'])
             [success, manifest_str] = server.GetApprovedManifest(branch, target)
@@ -515,20 +594,22 @@ uncommitted changes are present' % project.relpath
             finally:
               f.close()
           except IOError:
-            print >>sys.stderr, 'error: cannot write manifest to %s' % \
-                manifest_path
+            print('error: cannot write manifest to %s' % manifest_path,
+                  file=sys.stderr)
             sys.exit(1)
-          self.manifest.Override(manifest_name)
+          self._ReloadManifest(manifest_name)
         else:
-          print >>sys.stderr, 'error: %s' % manifest_str
+          print('error: manifest server RPC call failed: %s' %
+                manifest_str, file=sys.stderr)
           sys.exit(1)
-      except (socket.error, IOError, xmlrpclib.Fault) as e:
-        print >>sys.stderr, 'error: cannot connect to manifest server %s:\n%s' % (
-            self.manifest.manifest_server, e)
+      except (socket.error, IOError, xmlrpc.client.Fault) as e:
+        print('error: cannot connect to manifest server %s:\n%s'
+              % (self.manifest.manifest_server, e), file=sys.stderr)
         sys.exit(1)
-      except xmlrpclib.ProtocolError as e:
-        print >>sys.stderr, 'error: cannot connect to manifest server %s:\n%d %s' % (
-            self.manifest.manifest_server, e.errcode, e.errmsg)
+      except xmlrpc.client.ProtocolError as e:
+        print('error: cannot connect to manifest server %s:\n%d %s'
+              % (self.manifest.manifest_server, e.errcode, e.errmsg),
+              file=sys.stderr)
         sys.exit(1)
 
     rp = self.manifest.repoProject
@@ -538,21 +619,24 @@ uncommitted changes are present' % project.relpath
     mp.PreSync()
 
     if opt.repo_upgraded:
-      _PostRepoUpgrade(self.manifest, opt)
+      _PostRepoUpgrade(self.manifest, quiet=opt.quiet)
 
     if not opt.local_only:
       mp.Sync_NetworkHalf(quiet=opt.quiet,
-                          current_branch_only=opt.current_branch_only)
+                          current_branch_only=opt.current_branch_only,
+                          no_tags=opt.no_tags)
 
     if mp.HasChanges:
       syncbuf = SyncBuffer(mp.config)
       mp.Sync_LocalHalf(syncbuf)
       if not syncbuf.Finish():
         sys.exit(1)
-      self.manifest._Unload()
+      self._ReloadManifest(manifest_name)
       if opt.jobs is None:
         self.jobs = self.manifest.default.sync_j
-    all_projects = self.GetProjects(args, missing_ok=True)
+    all_projects = self.GetProjects(args,
+                                    missing_ok=True,
+                                    submodules_ok=opt.fetch_submodules)
 
     self._fetch_times = _FetchTimes(self.manifest)
     if not opt.local_only:
@@ -572,8 +656,10 @@ uncommitted changes are present' % project.relpath
       # Iteratively fetch missing and/or nested unregistered submodules
       previously_missing_set = set()
       while True:
-        self.manifest._Unload()
-        all_projects = self.GetProjects(args, missing_ok=True)
+        self._ReloadManifest(manifest_name)
+        all_projects = self.GetProjects(args,
+                                        missing_ok=True,
+                                        submodules_ok=opt.fetch_submodules)
         missing = []
         for project in all_projects:
           if project.gitdir not in fetched:
@@ -588,7 +674,7 @@ uncommitted changes are present' % project.relpath
         previously_missing_set = missing_set
         fetched.update(self._Fetch(missing, opt))
 
-    if self.manifest.IsMirror:
+    if self.manifest.IsMirror or self.manifest.IsArchive:
       # bail out now, we have no working tree
       return
 
@@ -603,46 +689,47 @@ uncommitted changes are present' % project.relpath
       if project.worktree:
         project.Sync_LocalHalf(syncbuf)
     pm.end()
-    print >>sys.stderr
+    print(file=sys.stderr)
     if not syncbuf.Finish():
       sys.exit(1)
 
     # If there's a notice that's supposed to print at the end of the sync, print
     # it now...
     if self.manifest.notice:
-      print self.manifest.notice
+      print(self.manifest.notice)
 
-def _PostRepoUpgrade(manifest, opt):
+def _PostRepoUpgrade(manifest, quiet=False):
   wrapper = WrapperModule()
   if wrapper.NeedSetupGnuPG():
-    wrapper.SetupGnuPG(opt.quiet)
+    wrapper.SetupGnuPG(quiet)
   for project in manifest.projects.values():
     if project.Exists:
       project.PostRepoUpgrade()
 
 def _PostRepoFetch(rp, no_repo_verify=False, verbose=False):
   if rp.HasChanges:
-    print >>sys.stderr, 'info: A new version of repo is available'
-    print >>sys.stderr, ''
+    print('info: A new version of repo is available', file=sys.stderr)
+    print(file=sys.stderr)
     if no_repo_verify or _VerifyTag(rp):
       syncbuf = SyncBuffer(rp.config)
       rp.Sync_LocalHalf(syncbuf)
       if not syncbuf.Finish():
         sys.exit(1)
-      print >>sys.stderr, 'info: Restarting repo with latest version'
+      print('info: Restarting repo with latest version', file=sys.stderr)
       raise RepoChangedException(['--repo-upgraded'])
     else:
-      print >>sys.stderr, 'warning: Skipped upgrade to unverified version'
+      print('warning: Skipped upgrade to unverified version', file=sys.stderr)
   else:
     if verbose:
-      print >>sys.stderr, 'repo version %s is current' % rp.work_git.describe(HEAD)
+      print('repo version %s is current' % rp.work_git.describe(HEAD),
+            file=sys.stderr)
 
 def _VerifyTag(project):
   gpg_dir = os.path.expanduser('~/.repoconfig/gnupg')
   if not os.path.exists(gpg_dir):
-    print >>sys.stderr,\
-"""warning: GnuPG was not available during last "repo init"
-warning: Cannot automatically authenticate repo."""
+    print('warning: GnuPG was not available during last "repo init"\n'
+          'warning: Cannot automatically authenticate repo."""',
+          file=sys.stderr)
     return True
 
   try:
@@ -656,10 +743,9 @@ warning: Cannot automatically authenticate repo."""
     if rev.startswith(R_HEADS):
       rev = rev[len(R_HEADS):]
 
-    print >>sys.stderr
-    print >>sys.stderr,\
-      "warning: project '%s' branch '%s' is not signed" \
-      % (project.name, rev)
+    print(file=sys.stderr)
+    print("warning: project '%s' branch '%s' is not signed"
+          % (project.name, rev), file=sys.stderr)
     return False
 
   env = os.environ.copy()
@@ -678,10 +764,10 @@ warning: Cannot automatically authenticate repo."""
   proc.stderr.close()
 
   if proc.wait() != 0:
-    print >>sys.stderr
-    print >>sys.stderr, out
-    print >>sys.stderr, err
-    print >>sys.stderr
+    print(file=sys.stderr)
+    print(out, file=sys.stderr)
+    print(err, file=sys.stderr)
+    print(file=sys.stderr)
     return False
   return True
 
@@ -708,14 +794,14 @@ class _FetchTimes(object):
   def _Load(self):
     if self._times is None:
       try:
-        f = open(self._path)
+        f = open(self._path, 'rb')
       except IOError:
         self._times = {}
         return self._times
       try:
         try:
           self._times = pickle.load(f)
-        except:
+        except IOError:
           try:
             os.remove(self._path)
           except OSError:

subcmds/upload.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import copy
 import re
 import sys
@@ -20,22 +21,31 @@ import sys
 from command import InteractiveCommand
 from editor import Editor
 from error import HookError, UploadError
+from git_command import GitCommand
 from project import RepoHook
 
+from pyversion import is_python3
+if not is_python3():
+  # pylint:disable=W0622
+  input = raw_input
+  # pylint:enable=W0622
+
 UNUSUAL_COMMIT_THRESHOLD = 5
 
 def _ConfirmManyUploads(multiple_branches=False):
   if multiple_branches:
-    print "ATTENTION: One or more branches has an unusually high number of commits."
+    print('ATTENTION: One or more branches has an unusually high number '
+          'of commits.')
   else:
-    print "ATTENTION: You are uploading an unusually high number of commits."
-  print "YOU PROBABLY DO NOT MEAN TO DO THIS. (Did you rebase across branches?)"
-  answer = raw_input("If you are sure you intend to do this, type 'yes': ").strip()
+    print('ATTENTION: You are uploading an unusually high number of commits.')
+  print('YOU PROBABLY DO NOT MEAN TO DO THIS. (Did you rebase across '
+        'branches?)')
+  answer = input("If you are sure you intend to do this, type 'yes': ").strip()
   return answer == "yes"
 
 def _die(fmt, *args):
   msg = fmt % args
-  print >>sys.stderr, 'error: %s' % msg
+  print('error: %s' % msg, file=sys.stderr)
   sys.exit(1)
 
 def _SplitEmails(values):
@@ -47,7 +57,7 @@ def _SplitEmails(values):
 class Upload(InteractiveCommand):
   common = True
   helpSummary = "Upload changes for code review"
-  helpUsage="""
+  helpUsage = """
 %prog [--re --cc] [<project>]...
 """
   helpDescription = """
@@ -137,6 +147,10 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
     p.add_option('-d', '--draft',
                  action='store_true', dest='draft', default=False,
                  help='If specified, upload as a draft.')
+    p.add_option('-D', '--destination', '--dest',
+                 type='string', action='store', dest='dest_branch',
+                 metavar='BRANCH',
+                 help='Submit for review on this target branch.')
 
     # Options relating to upload hook.  Note that verify and no-verify are NOT
     # opposites of each other, which is why they store to different locations.
@@ -176,18 +190,19 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
       date = branch.date
       commit_list = branch.commits
 
-      print 'Upload project %s/ to remote branch %s:' % (project.relpath, project.revisionExpr)
-      print '  branch %s (%2d commit%s, %s):' % (
+      destination = opt.dest_branch or project.dest_branch or project.revisionExpr
+      print('Upload project %s/ to remote branch %s:' % (project.relpath, destination))
+      print('  branch %s (%2d commit%s, %s):' % (
                     name,
                     len(commit_list),
                     len(commit_list) != 1 and 's' or '',
-                    date)
+                    date))
       for commit in commit_list:
-        print '         %s' % commit
+        print('         %s' % commit)
 
       sys.stdout.write('to %s (y/N)? ' % remote.review)
-      answer = sys.stdin.readline().strip()
-      answer = answer in ('y', 'Y', 'yes', '1', 'true', 't')
+      answer = sys.stdin.readline().strip().lower()
+      answer = answer in ('y', 'yes', '1', 'true', 't')
 
     if answer:
       if len(branch.commits) > UNUSUAL_COMMIT_THRESHOLD:
@@ -210,18 +225,21 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
 
       b = {}
       for branch in avail:
+        if branch is None:
+          continue
         name = branch.name
         date = branch.date
         commit_list = branch.commits
 
         if b:
           script.append('#')
+        destination = opt.dest_branch or project.dest_branch or project.revisionExpr
         script.append('#  branch %s (%2d commit%s, %s) to remote branch %s:' % (
                       name,
                       len(commit_list),
                       len(commit_list) != 1 and 's' or '',
                       date,
-                      project.revisionExpr))
+                      destination))
         for commit in commit_list:
           script.append('#         %s' % commit)
         b[name] = branch
@@ -297,7 +315,7 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
     try:
       # refs/changes/XYZ/N --> XYZ
       return refs.get(last_pub).split('/')[-2]
-    except:
+    except (AttributeError, IndexError):
       return ""
 
   def _UploadAndReport(self, opt, todo, original_people):
@@ -309,33 +327,48 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
 
         # Check if there are local changes that may have been forgotten
         if branch.project.HasChanges():
-            key = 'review.%s.autoupload' % branch.project.remote.review
-            answer = branch.project.config.GetBoolean(key)
+          key = 'review.%s.autoupload' % branch.project.remote.review
+          answer = branch.project.config.GetBoolean(key)
 
-            # if they want to auto upload, let's not ask because it could be automated
-            if answer is None:
-                sys.stdout.write('Uncommitted changes in ' + branch.project.name + ' (did you forget to amend?). Continue uploading? (y/N) ')
-                a = sys.stdin.readline().strip().lower()
-                if a not in ('y', 'yes', 't', 'true', 'on'):
-                    print >>sys.stderr, "skipping upload"
-                    branch.uploaded = False
-                    branch.error = 'User aborted'
-                    continue
+          # if they want to auto upload, let's not ask because it could be automated
+          if answer is None:
+            sys.stdout.write('Uncommitted changes in ' + branch.project.name + ' (did you forget to amend?). Continue uploading? (y/N) ')
+            a = sys.stdin.readline().strip().lower()
+            if a not in ('y', 'yes', 't', 'true', 'on'):
+              print("skipping upload", file=sys.stderr)
+              branch.uploaded = False
+              branch.error = 'User aborted'
+              continue
 
         # Check if topic branches should be sent to the server during upload
         if opt.auto_topic is not True:
-           key = 'review.%s.uploadtopic' % branch.project.remote.review
-           opt.auto_topic = branch.project.config.GetBoolean(key)
+          key = 'review.%s.uploadtopic' % branch.project.remote.review
+          opt.auto_topic = branch.project.config.GetBoolean(key)
 
-        branch.UploadForReview(people, auto_topic=opt.auto_topic, draft=opt.draft)
+        destination = opt.dest_branch or branch.project.dest_branch
+
+        # Make sure our local branch is not setup to track a different remote branch
+        merge_branch = self._GetMergeBranch(branch.project)
+        if destination:
+          full_dest = 'refs/heads/%s' % destination
+          if not opt.dest_branch and merge_branch and merge_branch != full_dest:
+            print('merge branch %s does not match destination branch %s'
+                  % (merge_branch, full_dest))
+            print('skipping upload.')
+            print('Please use `--destination %s` if this is intentional'
+                  % destination)
+            branch.uploaded = False
+            continue
+
+        branch.UploadForReview(people, auto_topic=opt.auto_topic, draft=opt.draft, dest_branch=destination)
         branch.uploaded = True
       except UploadError as e:
         branch.error = e
         branch.uploaded = False
         have_errors = True
 
-    print >>sys.stderr, ''
-    print >>sys.stderr, '----------------------------------------------------------------------'
+    print(file=sys.stderr)
+    print('----------------------------------------------------------------------', file=sys.stderr)
 
     if have_errors:
       for branch in todo:
@@ -344,21 +377,38 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
             fmt = ' (%s)'
           else:
             fmt = '\n       (%s)'
-          print >>sys.stderr, ('[FAILED] %-15s %-15s' + fmt) % (
+          print(('[FAILED] %-15s %-15s' + fmt) % (
                  branch.project.relpath + '/', \
                  branch.name, \
-                 str(branch.error))
-      print >>sys.stderr, ''
+                 str(branch.error)),
+                 file=sys.stderr)
+      print()
 
     for branch in todo:
-        if branch.uploaded:
-          print >>sys.stderr, '[OK    ] %-15s %s' % (
-                 branch.project.relpath + '/',
-                 branch.name)
+      if branch.uploaded:
+        print('[OK    ] %-15s %s' % (
+               branch.project.relpath + '/',
+               branch.name),
+               file=sys.stderr)
 
     if have_errors:
       sys.exit(1)
 
+  def _GetMergeBranch(self, project):
+    p = GitCommand(project,
+                   ['rev-parse', '--abbrev-ref', 'HEAD'],
+                   capture_stdout = True,
+                   capture_stderr = True)
+    p.Wait()
+    local_branch = p.stdout.strip()
+    p = GitCommand(project,
+                   ['config', '--get', 'branch.%s.merge' % local_branch],
+                   capture_stdout = True,
+                   capture_stderr = True)
+    p.Wait()
+    merge_branch = p.stdout.strip()
+    return merge_branch
+
   def Execute(self, opt, args):
     project_list = self.GetProjects(args)
     pending = []
@@ -372,7 +422,16 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
     for project in project_list:
       if opt.current_branch:
         cbr = project.CurrentBranch
-        avail = [project.GetUploadableBranch(cbr)] if cbr else None
+        up_branch = project.GetUploadableBranch(cbr)
+        if up_branch:
+          avail = [up_branch]
+        else:
+          avail = None
+          print('ERROR: Current branch (%s) not uploadable. '
+                'You may be able to type '
+                '"git branch --set-upstream-to m/master" to fix '
+                'your branch.' % str(cbr),
+                file=sys.stderr)
       else:
         avail = project.GetUploadableBranches(branch)
       if avail:
@@ -382,20 +441,22 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
       hook = RepoHook('pre-upload', self.manifest.repo_hooks_project,
                       self.manifest.topdir, abort_if_user_denies=True)
       pending_proj_names = [project.name for (project, avail) in pending]
+      pending_worktrees = [project.worktree for (project, avail) in pending]
       try:
-        hook.Run(opt.allow_all_hooks, project_list=pending_proj_names)
+        hook.Run(opt.allow_all_hooks, project_list=pending_proj_names,
+                 worktree_list=pending_worktrees)
       except HookError as e:
-        print >>sys.stderr, "ERROR: %s" % str(e)
+        print("ERROR: %s" % str(e), file=sys.stderr)
         return
 
     if opt.reviewers:
       reviewers = _SplitEmails(opt.reviewers)
     if opt.cc:
       cc = _SplitEmails(opt.cc)
-    people = (reviewers,cc)
+    people = (reviewers, cc)
 
     if not pending:
-      print >>sys.stdout, "no branches ready for upload"
+      print("no branches ready for upload", file=sys.stderr)
     elif len(pending) == 1 and len(pending[0][1]) == 1:
       self._SingleBranch(opt, pending[0][1][0], people)
     else:

subcmds/version.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import sys
 from command import Command, MirrorSafeCommand
 from git_command import git
@@ -32,12 +33,12 @@ class Version(Command, MirrorSafeCommand):
     rp = self.manifest.repoProject
     rem = rp.GetRemote(rp.remote.name)
 
-    print 'repo version %s' % rp.work_git.describe(HEAD)
-    print '       (from %s)' % rem.url
+    print('repo version %s' % rp.work_git.describe(HEAD))
+    print('       (from %s)' % rem.url)
 
     if Version.wrapper_path is not None:
-      print 'repo launcher version %s' % Version.wrapper_version
-      print '       (from %s)' % Version.wrapper_path
+      print('repo launcher version %s' % Version.wrapper_version)
+      print('       (from %s)' % Version.wrapper_path)
 
-    print git.version().strip()
-    print 'Python %s' % sys.version
+    print(git.version().strip())
+    print('Python %s' % sys.version)

tests/test_git_config.py

@@ -4,49 +4,49 @@ import unittest
 import git_config
 
 def fixture(*paths):
-    """Return a path relative to test/fixtures.
-    """
-    return os.path.join(os.path.dirname(__file__), 'fixtures', *paths)
+  """Return a path relative to test/fixtures.
+  """
+  return os.path.join(os.path.dirname(__file__), 'fixtures', *paths)
 
 class GitConfigUnitTest(unittest.TestCase):
-    """Tests the GitConfig class.
+  """Tests the GitConfig class.
+  """
+  def setUp(self):
+    """Create a GitConfig object using the test.gitconfig fixture.
     """
-    def setUp(self):
-        """Create a GitConfig object using the test.gitconfig fixture.
-        """
-        config_fixture = fixture('test.gitconfig')
-        self.config = git_config.GitConfig(config_fixture)
+    config_fixture = fixture('test.gitconfig')
+    self.config = git_config.GitConfig(config_fixture)
 
-    def test_GetString_with_empty_config_values(self):
-        """
-        Test config entries with no value.
+  def test_GetString_with_empty_config_values(self):
+    """
+    Test config entries with no value.
 
-        [section]
-            empty
+    [section]
+        empty
 
-        """
-        val = self.config.GetString('section.empty')
-        self.assertEqual(val, None)
+    """
+    val = self.config.GetString('section.empty')
+    self.assertEqual(val, None)
 
-    def test_GetString_with_true_value(self):
-        """
-        Test config entries with a string value.
+  def test_GetString_with_true_value(self):
+    """
+    Test config entries with a string value.
 
-        [section]
-            nonempty = true
+    [section]
+        nonempty = true
 
-        """
-        val = self.config.GetString('section.nonempty')
-        self.assertEqual(val, 'true')
+    """
+    val = self.config.GetString('section.nonempty')
+    self.assertEqual(val, 'true')
 
-    def test_GetString_from_missing_file(self):
-        """
-        Test missing config file
-        """
-        config_fixture = fixture('not.present.gitconfig')
-        config = git_config.GitConfig(config_fixture)
-        val = config.GetString('empty')
-        self.assertEqual(val, None)
+  def test_GetString_from_missing_file(self):
+    """
+    Test missing config file
+    """
+    config_fixture = fixture('not.present.gitconfig')
+    config = git_config.GitConfig(config_fixture)
+    val = config.GetString('empty')
+    self.assertEqual(val, None)
 
 if __name__ == '__main__':
-    unittest.main()
+  unittest.main()

trace.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import sys
 import os
 REPO_TRACE = 'REPO_TRACE'
@@ -31,4 +32,4 @@ def SetTrace():
 
 def Trace(fmt, *args):
   if IsTrace():
-    print >>sys.stderr, fmt % args
+    print(fmt % args, file=sys.stderr)