Revert "Implementation of manifest defined githooks"

This reverts commit 38e4387f8e.

A "repo init" followed by "repo sync" is meant to be as safe as
"git clone".  In particular it should not run arbitrary code provided
by the manifest owner.

It would still be nice to have support for manifest-defined git hooks
--- they'd just need a prompt like the upload RepoHook has.  Hopefully
a later change can bring them back.

Change-Id: I5ecd90fb5c2ed64f103d856d1ffcba38a47b062d
Signed-off-by: Jonathan Nieder <jrn@google.com>

color.py

@@ -83,12 +83,35 @@ def _Color(fg = None, bg = None, attr = None):
   return code
 
 
+DEFAULT = None
+
+def SetDefaultColoring(state):
+  """Set coloring behavior to |state|.
+
+  This is useful for overriding config options via the command line.
+  """
+  if state is None:
+    # Leave it alone -- return quick!
+    return
+
+  global DEFAULT
+  state = state.lower()
+  if state in ('auto',):
+    DEFAULT = state
+  elif state in ('always', 'yes', 'true', True):
+    DEFAULT = 'always'
+  elif state in ('never', 'no', 'false', False):
+    DEFAULT = 'never'
+
+
 class Coloring(object):
   def __init__(self, config, section_type):
     self._section = 'color.%s' % section_type
     self._config = config
     self._out = sys.stdout
 
+    on = DEFAULT
+    if on is None:
       on = self._config.GetString(self._section)
       if on is None:
         on = self._config.GetString('color.ui')

command.py

@@ -129,7 +129,7 @@ class Command(object):
   def GetProjects(self, args, missing_ok=False, submodules_ok=False):
     """A list of projects that match the arguments.
     """
-    all_projects = self.manifest.projects
+    all_projects_list = self.manifest.projects
     result = []
 
     mp = self.manifest.manifestProject
@@ -140,7 +140,6 @@ class Command(object):
     groups = [x for x in re.split(r'[,\s]+', groups) if x]
 
     if not args:
-      all_projects_list = list(all_projects.values())
       derived_projects = {}
       for project in all_projects_list:
         if submodules_ok or project.sync_s:
@@ -152,12 +151,12 @@ class Command(object):
             project.MatchesGroups(groups)):
           result.append(project)
     else:
-      self._ResetPathToProjectMap(all_projects.values())
+      self._ResetPathToProjectMap(all_projects_list)
 
       for arg in args:
-        project = all_projects.get(arg)
+        projects = self.manifest.GetProjectsWithName(arg)
 
-        if not project:
+        if not projects:
           path = os.path.abspath(arg).replace('\\', '/')
           project = self._GetProjectByPath(path)
 
@@ -172,14 +171,19 @@ class Command(object):
             if search_again:
               project = self._GetProjectByPath(path) or project
 
-        if not project:
+          if project:
+            projects = [project]
+
+        if not projects:
           raise NoSuchProjectError(arg)
+
+        for project in projects:
           if not missing_ok and not project.Exists:
             raise NoSuchProjectError(arg)
           if not project.MatchesGroups(groups):
             raise InvalidProjectGroupsError(arg)
 
-        result.append(project)
+        result.extend(projects)
 
     def _getpath(x):
       return x.relpath

docs/manifest-format.txt

@@ -26,6 +26,7 @@ following DTD:
                         manifest-server?,
                         remove-project*,
                         project*,
+                        extend-project*,
                         repo-hooks?)>
 
     <!ELEMENT notice (#PCDATA)>
@@ -35,6 +36,7 @@ following DTD:
     <!ATTLIST remote alias        CDATA #IMPLIED>
     <!ATTLIST remote fetch        CDATA #REQUIRED>
     <!ATTLIST remote review       CDATA #IMPLIED>
+    <!ATTLIST remote revision     CDATA #IMPLIED>
 
     <!ELEMENT default (EMPTY)>
     <!ATTLIST default remote      IDREF #IMPLIED>
@@ -47,7 +49,7 @@ following DTD:
     <!ELEMENT manifest-server (EMPTY)>
     <!ATTLIST url              CDATA #REQUIRED>
 
-    <!ELEMENT project (annotation?,
+    <!ELEMENT project (annotation*,
                        project*)>
     <!ATTLIST project name        CDATA #REQUIRED>
     <!ATTLIST project path        CDATA #IMPLIED>
@@ -66,6 +68,11 @@ following DTD:
     <!ATTLIST annotation value CDATA #REQUIRED>
     <!ATTLIST annotation keep  CDATA "true">
 
+    <!ELEMENT extend-project>
+    <!ATTLIST extend-project name CDATA #REQUIRED>
+    <!ATTLIST extend-project path CDATA #IMPLIED>
+    <!ATTLIST extend-project groups CDATA #IMPLIED>
+
     <!ELEMENT remove-project (EMPTY)>
     <!ATTLIST remove-project name  CDATA #REQUIRED>
 
@@ -112,6 +119,10 @@ Attribute `review`: Hostname of the Gerrit server where reviews
 are uploaded to by `repo upload`.  This attribute is optional;
 if not specified then `repo upload` will not function.
 
+Attribute `revision`: Name of a Git branch (e.g. `master` or
+`refs/heads/master`). Remotes with their own revision will override
+the default revision.
+
 Element default
 ---------------
 
@@ -132,14 +143,14 @@ Project elements not setting their own `dest-branch` will inherit
 this value. If this value is not set, projects will use `revision`
 by default instead.
 
-Attribute `sync_j`: Number of parallel jobs to use when synching.
+Attribute `sync-j`: Number of parallel jobs to use when synching.
 
-Attribute `sync_c`: Set to true to only sync the given Git
+Attribute `sync-c`: Set to true to only sync the given Git
 branch (specified in the `revision` attribute) rather than the
-whole ref space.  Project elements lacking a sync_c element of
+whole ref space.  Project elements lacking a sync-c element of
 their own will use this value.
 
-Attribute `sync_s`: Set to true to also sync sub-projects.
+Attribute `sync-s`: Set to true to also sync sub-projects.
 
 
 Element manifest-server
@@ -208,7 +219,8 @@ to track for this project.  Names can be relative to refs/heads
 (e.g. just "master") or absolute (e.g. "refs/heads/master").
 Tags and/or explicit SHA-1s should work in theory, but have not
 been extensively tested.  If not supplied the revision given by
-the default element is used.
+the remote element is used if applicable, else the default
+element is used.
 
 Attribute `dest-branch`: Name of a Git branch (e.g. `master`).
 When using `repo upload`, changes will be submitted for code
@@ -226,13 +238,13 @@ group "notdefault", it will not be automatically downloaded by repo.
 If the project has a parent element, the `name` and `path` here
 are the prefixed ones.
 
-Attribute `sync_c`: Set to true to only sync the given Git
+Attribute `sync-c`: Set to true to only sync the given Git
 branch (specified in the `revision` attribute) rather than the
 whole ref space.
 
-Attribute `sync_s`: Set to true to also sync sub-projects.
+Attribute `sync-s`: Set to true to also sync sub-projects.
 
-Attribute `upstream`: Name of the Git branch in which a sha1
+Attribute `upstream`: Name of the Git ref in which a sha1
 can be found.  Used when syncing a revision locked manifest in
 -c mode to avoid having to sync the entire ref space.
 
@@ -246,6 +258,22 @@ rather than the `name` attribute.  This attribute only applies to the
 local mirrors syncing, it will be ignored when syncing the projects in a
 client working directory.
 
+Element extend-project
+----------------------
+
+Modify the attributes of the named project.
+
+This element is mostly useful in a local manifest file, to modify the
+attributes of an existing project without completely replacing the
+existing project definition.  This makes the local manifest more robust
+against changes to the original manifest.
+
+Attribute `path`: If specified, limit the change to projects checked out
+at the specified path, rather than all projects with the given name.
+
+Attribute `groups`: List of additional groups to which this project
+belongs.  Same syntax as the corresponding element of `project`.
+
 Element annotation
 ------------------
 

error.py

@@ -24,6 +24,13 @@ class ManifestInvalidRevisionError(Exception):
 class NoManifestException(Exception):
   """The required manifest does not exist.
   """
+  def __init__(self, path, reason):
+    super(NoManifestException, self).__init__()
+    self.path = path
+    self.reason = reason
+
+  def __str__(self):
+    return self.reason
 
 class EditorError(Exception):
   """Unspecified error from the user's text editor.

git_command.py

@@ -21,6 +21,7 @@ import tempfile
 from signal import SIGTERM
 from error import GitError
 from trace import REPO_TRACE, IsTrace, Trace
+from wrapper import Wrapper
 
 GIT = 'git'
 MIN_GIT_VERSION = (1, 5, 4)
@@ -79,20 +80,15 @@ class _GitCall(object):
   def version(self):
     p = GitCommand(None, ['--version'], capture_stdout=True)
     if p.Wait() == 0:
-      return p.stdout
+      return p.stdout.decode('utf-8')
     return None
 
   def version_tuple(self):
     global _git_version
-
     if _git_version is None:
       ver_str = git.version()
-      if ver_str.startswith('git version '):
-        _git_version = tuple(
-          map(int,
-            ver_str[len('git version '):].strip().split('-')[0].split('.')[0:3]
-          ))
-      else:
+      _git_version = Wrapper().ParseGitVersion(ver_str)
+      if _git_version is None:
         print('fatal: "%s" unsupported' % ver_str, file=sys.stderr)
         sys.exit(1)
     return _git_version

git_config.py

@@ -15,8 +15,8 @@
 
 from __future__ import print_function
 
+import json
 import os
-import pickle
 import re
 import subprocess
 import sys
@@ -80,7 +80,7 @@ class GitConfig(object):
     return cls(configfile = os.path.join(gitdir, 'config'),
                defaults = defaults)
 
-  def __init__(self, configfile, defaults=None, pickleFile=None):
+  def __init__(self, configfile, defaults=None, jsonFile=None):
     self.file = configfile
     self.defaults = defaults
     self._cache_dict = None
@@ -88,12 +88,11 @@ class GitConfig(object):
     self._remotes = {}
     self._branches = {}
 
-    if pickleFile is None:
-      self._pickle = os.path.join(
+    self._json = jsonFile
+    if self._json is None:
+      self._json = os.path.join(
         os.path.dirname(self.file),
-        '.repopickle_' + os.path.basename(self.file))
-    else:
-      self._pickle = pickleFile
+        '.repo_' + os.path.basename(self.file) + '.json')
 
   def Has(self, name, include_defaults = True):
     """Return true if this configuration file has the key.
@@ -217,7 +216,7 @@ class GitConfig(object):
     """Resolve any url.*.insteadof references.
     """
     for new_url in self.GetSubSections('url'):
-      old_url = self.GetString('url.%s.insteadof' % new_url)
+      for old_url in self.GetString('url.%s.insteadof' % new_url, True):
         if old_url is not None and url.startswith(old_url):
           return new_url + url[len(old_url):]
     return url
@@ -248,50 +247,41 @@ class GitConfig(object):
     return self._cache_dict
 
   def _Read(self):
-    d = self._ReadPickle()
+    d = self._ReadJson()
     if d is None:
       d = self._ReadGit()
-      self._SavePickle(d)
+      self._SaveJson(d)
     return d
 
-  def _ReadPickle(self):
+  def _ReadJson(self):
     try:
-      if os.path.getmtime(self._pickle) \
+      if os.path.getmtime(self._json) \
       <= os.path.getmtime(self.file):
-        os.remove(self._pickle)
+        os.remove(self._json)
         return None
     except OSError:
       return None
     try:
-      Trace(': unpickle %s', self.file)
-      fd = open(self._pickle, 'rb')
+      Trace(': parsing %s', self.file)
+      fd = open(self._json)
       try:
-        return pickle.load(fd)
+        return json.load(fd)
       finally:
         fd.close()
-    except EOFError:
-      os.remove(self._pickle)
-      return None
-    except IOError:
-      os.remove(self._pickle)
-      return None
-    except pickle.PickleError:
-      os.remove(self._pickle)
+    except (IOError, ValueError):
+      os.remove(self._json)
       return None
 
-  def _SavePickle(self, cache):
+  def _SaveJson(self, cache):
     try:
-      fd = open(self._pickle, 'wb')
+      fd = open(self._json, 'w')
       try:
-        pickle.dump(cache, fd, pickle.HIGHEST_PROTOCOL)
+        json.dump(cache, fd, indent=2)
       finally:
         fd.close()
-    except IOError:
-      if os.path.exists(self._pickle):
-        os.remove(self._pickle)
-    except pickle.PickleError:
-      if os.path.exists(self._pickle):
-        os.remove(self._pickle)
+    except (IOError, TypeError):
+      if os.path.exists(self.json):
+        os.remove(self._json)
 
   def _ReadGit(self):
     """
@@ -304,7 +294,7 @@ class GitConfig(object):
     d = self._do('--null', '--list')
     if d is None:
       return c
-    for line in d.rstrip('\0').split('\0'):  # pylint: disable=W1401
+    for line in d.decode('utf-8').rstrip('\0').split('\0'):  # pylint: disable=W1401
                                                              # Backslash is not anomalous
       if '\n' in line:
         key, val = line.split('\n', 1)
@@ -576,7 +566,9 @@ class Remote(object):
         return None
 
       u = self.review
-      if not u.startswith('http:') and not u.startswith('https:'):
+      if u.startswith('persistent-'):
+        u = u[len('persistent-'):]
+      if u.split(':')[0] not in ('http', 'https', 'sso'):
         u = 'http://%s' % u
       if u.endswith('/Gerrit'):
         u = u[:len(u) - len('/Gerrit')]
@@ -592,6 +584,9 @@ class Remote(object):
         host, port = os.environ['REPO_HOST_PORT_INFO'].split()
         self._review_url = self._SshReviewUrl(userEmail, host, port)
         REVIEW_CACHE[u] = self._review_url
+      elif u.startswith('sso:'):
+        self._review_url = u  # Assume it's right
+        REVIEW_CACHE[u] = self._review_url
       else:
         try:
           info_url = u + 'ssh_info'
@@ -601,7 +596,7 @@ class Remote(object):
             # of HTML response back, like maybe a login page.
             #
             # Assume HTTP if SSH is not enabled or ssh_info doesn't look right.
-            self._review_url = http_url + 'p/'
+            self._review_url = http_url
           else:
             host, port = info.split()
             self._review_url = self._SshReviewUrl(userEmail, host, port)
@@ -624,9 +619,7 @@ class Remote(object):
   def ToLocal(self, rev):
     """Convert a remote revision string to something we have locally.
     """
-    if IsId(rev):
-      return rev
-    if rev.startswith(R_TAGS):
+    if self.name == '.' or IsId(rev):
       return rev
 
     if not rev.startswith('refs/'):
@@ -635,6 +628,10 @@ class Remote(object):
     for spec in self.fetch:
       if spec.SourceMatches(rev):
         return spec.MapSource(rev)
+
+    if not rev.startswith(R_HEADS):
+      return rev
+
     raise GitError('remote %s does not have %s' % (self.name, rev))
 
   def WritesTo(self, ref):
@@ -704,7 +701,7 @@ class Branch(object):
       self._Set('merge', self.merge)
 
     else:
-      fd = open(self._config.file, 'ab')
+      fd = open(self._config.file, 'a')
       try:
         fd.write('[branch "%s"]\n' % self.name)
         if self.remote:

git_refs.py

@@ -100,7 +100,7 @@ class GitRefs(object):
   def _ReadPackedRefs(self):
     path = os.path.join(self._gitdir, 'packed-refs')
     try:
-      fd = open(path, 'rb')
+      fd = open(path, 'r')
       mtime = os.path.getmtime(path)
     except IOError:
       return

hooks/commit-msg

@@ -1,5 +1,4 @@
 #!/bin/sh
-# From Gerrit Code Review 2.5.2
 #
 # Part of Gerrit Code Review (http://code.google.com/p/gerrit/)
 #
@@ -27,7 +26,7 @@ MSG="$1"
 #
 add_ChangeId() {
 	clean_message=`sed -e '
-		/^diff --git a\/.*/{
+		/^diff --git .*/{
 			s///
 			q
 		}
@@ -39,6 +38,11 @@ add_ChangeId() {
 		return
 	fi
 
+	if test "false" = "`git config --bool --get gerrit.createChangeId`"
+	then
+		return
+	fi
+
 	# Does Change-Id: already exist? if so, exit (no change).
 	if grep -i '^Change-Id:' "$MSG" >/dev/null
 	then
@@ -77,7 +81,7 @@ add_ChangeId() {
 	# Skip the line starting with the diff command and everything after it,
 	# up to the end of the file, assuming it is only patch data.
 	# If more than one line before the diff was empty, strip all but one.
-	/^diff --git a/ {
+	/^diff --git / {
 		blankLines = 0
 		while (getline) { }
 		next
@@ -154,7 +158,7 @@ add_ChangeId() {
 		if (unprinted) {
 			print "Change-Id: I'"$id"'"
 		}
-	}' "$MSG" > $T && mv $T "$MSG" || rm -f $T
+	}' "$MSG" > "$T" && mv "$T" "$MSG" || rm -f "$T"
 }
 _gen_ChangeIdInput() {
 	echo "tree `git write-tree`"

hooks/pre-auto-gc

@@ -35,7 +35,7 @@ elif grep -q "AC Power \+: 1" /proc/pmu/info 2>/dev/null
 then
 	exit 0
 elif test -x /usr/bin/pmset && /usr/bin/pmset -g batt |
-	grep -q "Currently drawing from 'AC Power'"
+	grep -q "drawing from 'AC Power'"
 then
 	exit 0
 elif test -d /sys/bus/acpi/drivers/battery && test 0 = \

main.py

@@ -31,6 +31,12 @@ else:
   urllib = imp.new_module('urllib')
   urllib.request = urllib2
 
+try:
+  import kerberos
+except ImportError:
+  kerberos = None
+
+from color import SetDefaultColoring
 from trace import SetTrace
 from git_command import git, GitCommand
 from git_config import init_ssh, close_ssh
@@ -46,6 +52,7 @@ from error import NoSuchProjectError
 from error import RepoChangedException
 from manifest_xml import XmlManifest
 from pager import RunPager
+from wrapper import WrapperPath, Wrapper
 
 from subcmds import all_commands
 
@@ -63,6 +70,9 @@ global_options.add_option('-p', '--paginate',
 global_options.add_option('--no-pager',
                           dest='no_pager', action='store_true',
                           help='disable the pager')
+global_options.add_option('--color',
+                          choices=('auto', 'always', 'never'), default=None,
+                          help='control color usage: auto, always, never')
 global_options.add_option('--trace',
                           dest='trace', action='store_true',
                           help='trace git command execution')
@@ -107,6 +117,8 @@ class _Repo(object):
         print('fatal: invalid usage of --version', file=sys.stderr)
         return 1
 
+    SetDefaultColoring(gopts.color)
+
     try:
       cmd = self.commands[name]
     except KeyError:
@@ -123,8 +135,15 @@ class _Repo(object):
             file=sys.stderr)
       return 1
 
+    try:
       copts, cargs = cmd.OptionParser.parse_args(argv)
       copts = cmd.ReadEnvironmentOptions(copts)
+    except NoManifestException as e:
+      print('error: in `%s`: %s' % (' '.join([name] + argv), str(e)),
+        file=sys.stderr)
+      print('error: manifest missing or unreadable -- please run init',
+            file=sys.stderr)
+      return 1
 
     if not gopts.no_pager and not isinstance(cmd, InteractiveCommand):
       config = cmd.manifest.globalConfig
@@ -140,14 +159,12 @@ class _Repo(object):
     start = time.time()
     try:
       result = cmd.Execute(copts, cargs)
-    except DownloadError as e:
-      print('error: %s' % str(e), file=sys.stderr)
-      result = 1
-    except ManifestInvalidRevisionError as e:
-      print('error: %s' % str(e), file=sys.stderr)
-      result = 1
-    except NoManifestException as e:
-      print('error: manifest required for this command -- please run init',
+    except (DownloadError, ManifestInvalidRevisionError,
+        NoManifestException) as e:
+      print('error: in `%s`: %s' % (' '.join([name] + argv), str(e)),
+        file=sys.stderr)
+      if isinstance(e, NoManifestException):
+        print('error: manifest missing or unreadable -- please run init',
               file=sys.stderr)
       result = 1
     except NoSuchProjectError as e:
@@ -169,21 +186,10 @@ class _Repo(object):
 
     return result
 
+
 def _MyRepoPath():
   return os.path.dirname(__file__)
 
-def _MyWrapperPath():
-  return os.path.join(os.path.dirname(__file__), 'repo')
-
-_wrapper_module = None
-def WrapperModule():
-  global _wrapper_module
-  if not _wrapper_module:
-    _wrapper_module = imp.load_source('wrapper', _MyWrapperPath())
-  return _wrapper_module
-
-def _CurrentWrapperVersion():
-  return WrapperModule().VERSION
 
 def _CheckWrapperVersion(ver, repo_path):
   if not repo_path:
@@ -193,7 +199,7 @@ def _CheckWrapperVersion(ver, repo_path):
     print('no --wrapper-version argument', file=sys.stderr)
     sys.exit(1)
 
-  exp = _CurrentWrapperVersion()
+  exp = Wrapper().VERSION
   ver = tuple(map(int, ver.split('.')))
   if len(ver) == 1:
     ver = (0, ver[0])
@@ -205,7 +211,7 @@ def _CheckWrapperVersion(ver, repo_path):
 !!! You must upgrade before you can continue:   !!!
 
     cp %s %s
-""" % (exp_str, _MyWrapperPath(), repo_path), file=sys.stderr)
+""" % (exp_str, WrapperPath(), repo_path), file=sys.stderr)
     sys.exit(1)
 
   if exp > ver:
@@ -214,7 +220,7 @@ def _CheckWrapperVersion(ver, repo_path):
 ... You should upgrade soon:
 
     cp %s %s
-""" % (exp_str, _MyWrapperPath(), repo_path), file=sys.stderr)
+""" % (exp_str, WrapperPath(), repo_path), file=sys.stderr)
 
 def _CheckRepoDir(repo_dir):
   if not repo_dir:
@@ -342,6 +348,86 @@ class _DigestAuthHandler(urllib.request.HTTPDigestAuthHandler):
         self.retried = 0
       raise
 
+class _KerberosAuthHandler(urllib.request.BaseHandler):
+  def __init__(self):
+    self.retried = 0
+    self.context = None
+    self.handler_order = urllib.request.BaseHandler.handler_order - 50
+
+  def http_error_401(self, req, fp, code, msg, headers):
+    host = req.get_host()
+    retry = self.http_error_auth_reqed('www-authenticate', host, req, headers)
+    return retry
+
+  def http_error_auth_reqed(self, auth_header, host, req, headers):
+    try:
+      spn = "HTTP@%s" % host
+      authdata = self._negotiate_get_authdata(auth_header, headers)
+
+      if self.retried > 3:
+        raise urllib.request.HTTPError(req.get_full_url(), 401,
+          "Negotiate auth failed", headers, None)
+      else:
+        self.retried += 1
+
+      neghdr = self._negotiate_get_svctk(spn, authdata)
+      if neghdr is None:
+        return None
+
+      req.add_unredirected_header('Authorization', neghdr)
+      response = self.parent.open(req)
+
+      srvauth = self._negotiate_get_authdata(auth_header, response.info())
+      if self._validate_response(srvauth):
+        return response
+    except kerberos.GSSError:
+      return None
+    except:
+      self.reset_retry_count()
+      raise
+    finally:
+      self._clean_context()
+
+  def reset_retry_count(self):
+    self.retried = 0
+
+  def _negotiate_get_authdata(self, auth_header, headers):
+    authhdr = headers.get(auth_header, None)
+    if authhdr is not None:
+      for mech_tuple in authhdr.split(","):
+        mech, __, authdata = mech_tuple.strip().partition(" ")
+        if mech.lower() == "negotiate":
+          return authdata.strip()
+    return None
+
+  def _negotiate_get_svctk(self, spn, authdata):
+    if authdata is None:
+      return None
+
+    result, self.context = kerberos.authGSSClientInit(spn)
+    if result < kerberos.AUTH_GSS_COMPLETE:
+      return None
+
+    result = kerberos.authGSSClientStep(self.context, authdata)
+    if result < kerberos.AUTH_GSS_CONTINUE:
+      return None
+
+    response = kerberos.authGSSClientResponse(self.context)
+    return "Negotiate %s" % response
+
+  def _validate_response(self, authdata):
+    if authdata is None:
+      return None
+    result = kerberos.authGSSClientStep(self.context, authdata)
+    if result == kerberos.AUTH_GSS_COMPLETE:
+      return True
+    return None
+
+  def _clean_context(self):
+    if self.context is not None:
+      kerberos.authGSSClientClean(self.context)
+      self.context = None
+
 def init_http():
   handlers = [_UserAgentHandler()]
 
@@ -358,6 +444,8 @@ def init_http():
     pass
   handlers.append(_BasicAuthHandler(mgr))
   handlers.append(_DigestAuthHandler(mgr))
+  if kerberos:
+    handlers.append(_KerberosAuthHandler())
 
   if 'http_proxy' in os.environ:
     url = os.environ['http_proxy']

manifest_xml.py

@@ -32,7 +32,7 @@ else:
 from git_config import GitConfig
 from git_refs import R_HEADS, HEAD
 from project import RemoteSpec, Project, MetaProject
-from error import ManifestParseError
+from error import ManifestParseError, ManifestInvalidRevisionError
 
 MANIFEST_FILE_NAME = 'manifest.xml'
 LOCAL_MANIFEST_NAME = 'local_manifest.xml'
@@ -51,18 +51,26 @@ class _Default(object):
   sync_c = False
   sync_s = False
 
+  def __eq__(self, other):
+    return self.__dict__ == other.__dict__
+
+  def __ne__(self, other):
+    return self.__dict__ != other.__dict__
+
 class _XmlRemote(object):
   def __init__(self,
                name,
                alias=None,
                fetch=None,
                manifestUrl=None,
-               review=None):
+               review=None,
+               revision=None):
     self.name = name
     self.fetchUrl = fetch
     self.manifestUrl = manifestUrl
     self.remoteAlias = alias
     self.reviewUrl = review
+    self.revision = revision
     self.resolvedFetchUrl = self._resolveFetchUrl()
 
   def __eq__(self, other):
@@ -74,23 +82,31 @@ class _XmlRemote(object):
   def _resolveFetchUrl(self):
     url = self.fetchUrl.rstrip('/')
     manifestUrl = self.manifestUrl.rstrip('/')
-    p = manifestUrl.startswith('persistent-http')
-    if p:
-      manifestUrl = manifestUrl[len('persistent-'):]
-
-    # urljoin will get confused if there is no scheme in the base url
-    # ie, if manifestUrl is of the form <hostname:port>
+    # urljoin will gets confused over quite a few things.  The ones we care
+    # about here are:
+    # * no scheme in the base url, like <hostname:port>
+    # * persistent-https://
+    # * rpc://
+    # We handle this by replacing these with obscure protocols
+    # and then replacing them with the original when we are done.
+    # gopher -> <none>
+    # wais -> persistent-https
+    # nntp -> rpc
     if manifestUrl.find(':') != manifestUrl.find('/') - 1:
       manifestUrl = 'gopher://' + manifestUrl
+    manifestUrl = re.sub(r'^persistent-https://', 'wais://', manifestUrl)
+    manifestUrl = re.sub(r'^rpc://', 'nntp://', manifestUrl)
     url = urllib.parse.urljoin(manifestUrl, url)
     url = re.sub(r'^gopher://', '', url)
-    if p:
-      url = 'persistent-' + url
+    url = re.sub(r'^wais://', 'persistent-https://', url)
+    url = re.sub(r'^nntp://', 'rpc://', url)
     return url
 
   def ToRemoteSpec(self, projectName):
     url = self.resolvedFetchUrl.rstrip('/') + '/' + projectName
     remoteName = self.name
+    if self.remoteAlias:
+      remoteName = self.remoteAlias
     return RemoteSpec(remoteName, url, self.reviewUrl)
 
 class XmlManifest(object):
@@ -145,8 +161,15 @@ class XmlManifest(object):
     root.appendChild(e)
     e.setAttribute('name', r.name)
     e.setAttribute('fetch', r.fetchUrl)
+    if r.remoteAlias is not None:
+      e.setAttribute('alias', r.remoteAlias)
     if r.reviewUrl is not None:
       e.setAttribute('review', r.reviewUrl)
+    if r.revision is not None:
+      e.setAttribute('revision', r.revision)
+
+  def _ParseGroups(self, groups):
+    return [x for x in re.split(r'[,\s]+', groups) if x]
 
   def Save(self, fd, peg_rev=False, peg_rev_upstream=True):
     """Write the current manifest out to the given file descriptor.
@@ -155,7 +178,7 @@ class XmlManifest(object):
 
     groups = mp.config.GetString('manifest.groups')
     if groups:
-      groups = [x for x in re.split(r'[,\s]+', groups) if x]
+      groups = self._ParseGroups(groups)
 
     doc = xml.dom.minidom.Document()
     root = doc.createElement('manifest')
@@ -205,8 +228,9 @@ class XmlManifest(object):
       root.appendChild(doc.createTextNode(''))
 
     def output_projects(parent, parent_node, projects):
-      for p in projects:
-        output_project(parent, parent_node, self.projects[p])
+      for project_name in projects:
+        for project in self._projects[project_name]:
+          output_project(parent, parent_node, project)
 
     def output_project(parent, parent_node, p):
       if not p.MatchesGroups(groups):
@@ -223,8 +247,12 @@ class XmlManifest(object):
       e.setAttribute('name', name)
       if relpath != name:
         e.setAttribute('path', relpath)
-      if not d.remote or p.remote.name != d.remote.name:
-        e.setAttribute('remote', p.remote.name)
+      remoteName = None
+      if d.remote:
+        remoteName = d.remote.remoteAlias or d.remote.name
+      if not d.remote or p.remote.name != remoteName:
+        remoteName = p.remote.name
+        e.setAttribute('remote', remoteName)
       if peg_rev:
         if self.IsMirror:
           value = p.bare_git.rev_parse(p.revisionExpr + '^0')
@@ -236,8 +264,12 @@ class XmlManifest(object):
           # isn't our value, and the if the default doesn't already have that
           # covered.
           e.setAttribute('upstream', p.revisionExpr)
-      elif not d.revisionExpr or p.revisionExpr != d.revisionExpr:
+      else:
+        revision = self.remotes[remoteName].revision or d.revisionExpr
+        if not revision or revision != p.revisionExpr:
           e.setAttribute('revision', p.revisionExpr)
+        if p.upstream and p.upstream != p.revisionExpr:
+          e.setAttribute('upstream', p.upstream)
 
       for c in p.copyfiles:
         ce = doc.createElement('copyfile')
@@ -245,6 +277,12 @@ class XmlManifest(object):
         ce.setAttribute('dest', c.dest)
         e.appendChild(ce)
 
+      for l in p.linkfiles:
+        le = doc.createElement('linkfile')
+        le.setAttribute('src', l.src)
+        le.setAttribute('dest', l.dest)
+        e.appendChild(le)
+
       default_groups = ['all', 'name:%s' % p.name, 'path:%s' % p.relpath]
       egroups = [g for g in p.groups if g not in default_groups]
       if egroups:
@@ -264,13 +302,11 @@ class XmlManifest(object):
         e.setAttribute('sync-s', 'true')
 
       if p.subprojects:
-        sort_projects = list(sorted([subp.name for subp in p.subprojects]))
-        output_projects(p, e, sort_projects)
+        subprojects = set(subp.name for subp in p.subprojects)
+        output_projects(p, e, list(sorted(subprojects)))
 
-    sort_projects = list(sorted([key for key, value in self.projects.items()
-                     if not value.parent]))
-    sort_projects.sort()
-    output_projects(None, root, sort_projects)
+    projects = set(p.name for p in self._paths.values() if not p.parent)
+    output_projects(None, root, list(sorted(projects)))
 
     if self._repo_hooks_project:
       root.appendChild(doc.createTextNode(''))
@@ -282,10 +318,15 @@ class XmlManifest(object):
 
     doc.writexml(fd, '', '  ', '\n', 'UTF-8')
 
+  @property
+  def paths(self):
+    self._Load()
+    return self._paths
+
   @property
   def projects(self):
     self._Load()
-    return self._projects
+    return list(self._paths.values())
 
   @property
   def remotes(self):
@@ -316,9 +357,14 @@ class XmlManifest(object):
   def IsMirror(self):
     return self.manifestProject.config.GetBoolean('repo.mirror')
 
+  @property
+  def IsArchive(self):
+    return self.manifestProject.config.GetBoolean('repo.archive')
+
   def _Unload(self):
     self._loaded = False
     self._projects = {}
+    self._paths = {}
     self._remotes = {}
     self._default = None
     self._repo_hooks_project = None
@@ -422,11 +468,13 @@ class XmlManifest(object):
 
     for node in itertools.chain(*node_list):
       if node.nodeName == 'default':
-        if self._default is not None:
-          raise ManifestParseError(
-              'duplicate default in %s' %
+        new_default = self._ParseDefault(node)
+        if self._default is None:
+          self._default = new_default
+        elif new_default != self._default:
+          raise ManifestParseError('duplicate default in %s' %
                                    (self.manifestFile))
-        self._default = self._ParseDefault(node)
+
     if self._default is None:
       self._default = _Default()
 
@@ -448,11 +496,17 @@ class XmlManifest(object):
         self._manifest_server = url
 
     def recursively_add_projects(project):
-      if self._projects.get(project.name):
+      projects = self._projects.setdefault(project.name, [])
+      if project.relpath is None:
         raise ManifestParseError(
-            'duplicate project %s in %s' %
+            'missing path for %s in %s' %
             (project.name, self.manifestFile))
-      self._projects[project.name] = project
+      if project.relpath in self._paths:
+        raise ManifestParseError(
+            'duplicate path %s in %s' %
+            (project.relpath, self.manifestFile))
+      self._paths[project.relpath] = project
+      projects.append(project)
       for subproject in project.subprojects:
         recursively_add_projects(subproject)
 
@@ -460,6 +514,23 @@ class XmlManifest(object):
       if node.nodeName == 'project':
         project = self._ParseProject(node)
         recursively_add_projects(project)
+      if node.nodeName == 'extend-project':
+        name = self._reqatt(node, 'name')
+
+        if name not in self._projects:
+          raise ManifestParseError('extend-project element specifies non-existent '
+                                   'project: %s' % name)
+
+        path = node.getAttribute('path')
+        groups = node.getAttribute('groups')
+        if groups:
+          groups = self._ParseGroups(groups)
+
+        for p in self._projects[name]:
+          if path and p.relpath != path:
+            continue
+          if groups:
+            p.groups.extend(groups)
       if node.nodeName == 'repo-hooks':
         # Get the name of the project and the (space-separated) list of enabled.
         repo_hooks_project = self._reqatt(node, 'in-project')
@@ -473,22 +544,31 @@ class XmlManifest(object):
 
         # Store a reference to the Project.
         try:
-          self._repo_hooks_project = self._projects[repo_hooks_project]
+          repo_hooks_projects = self._projects[repo_hooks_project]
         except KeyError:
           raise ManifestParseError(
               'project %s not found for repo-hooks' %
               (repo_hooks_project))
 
+        if len(repo_hooks_projects) != 1:
+          raise ManifestParseError(
+              'internal error parsing repo-hooks in %s' %
+              (self.manifestFile))
+        self._repo_hooks_project = repo_hooks_projects[0]
+
         # Store the enabled hooks in the Project object.
         self._repo_hooks_project.enabled_repo_hooks = enabled_repo_hooks
       if node.nodeName == 'remove-project':
         name = self._reqatt(node, 'name')
-        try:
-          del self._projects[name]
-        except KeyError:
+
+        if name not in self._projects:
           raise ManifestParseError('remove-project element specifies non-existent '
                                    'project: %s' % name)
 
+        for p in self._projects[name]:
+          del self._paths[p.relpath]
+        del self._projects[name]
+
         # If the manifest removes the hooks project, treat it as if it deleted
         # the repo-hooks element too.
         if self._repo_hooks_project and (self._repo_hooks_project.name == name):
@@ -525,11 +605,13 @@ class XmlManifest(object):
                         name = name,
                         remote = remote.ToRemoteSpec(name),
                         gitdir = gitdir,
+                        objdir = gitdir,
                         worktree = None,
-                        relpath = None,
+                        relpath = name or None,
                         revisionExpr = m.revisionExpr,
                         revisionId = None)
-      self._projects[project.name] = project
+      self._projects[project.name] = [project]
+      self._paths[project.relpath] = project
 
   def _ParseRemote(self, node):
     """
@@ -543,8 +625,11 @@ class XmlManifest(object):
     review = node.getAttribute('review')
     if review == '':
       review = None
+    revision = node.getAttribute('revision')
+    if revision == '':
+      revision = None
     manifestUrl = self.manifestProject.config.GetString('remote.origin.url')
-    return _XmlRemote(name, alias, fetch, manifestUrl, review)
+    return _XmlRemote(name, alias, fetch, manifestUrl, review, revision)
 
   def _ParseDefault(self, node):
     """
@@ -637,7 +722,7 @@ class XmlManifest(object):
       raise ManifestParseError("no remote for project %s within %s" %
             (name, self.manifestFile))
 
-    revisionExpr = node.getAttribute('revision')
+    revisionExpr = node.getAttribute('revision') or remote.revision
     if not revisionExpr:
       revisionExpr = self._default.revisionExpr
     if not revisionExpr:
@@ -686,12 +771,13 @@ class XmlManifest(object):
     groups = ''
     if node.hasAttribute('groups'):
       groups = node.getAttribute('groups')
-    groups = [x for x in re.split(r'[,\s]+', groups) if x]
+    groups = self._ParseGroups(groups)
 
     if parent is None:
-      relpath, worktree, gitdir = self.GetProjectPaths(name, path)
+      relpath, worktree, gitdir, objdir = self.GetProjectPaths(name, path)
     else:
-      relpath, worktree, gitdir = self.GetSubprojectPaths(parent, path)
+      relpath, worktree, gitdir, objdir = \
+          self.GetSubprojectPaths(parent, name, path)
 
     default_groups = ['all', 'name:%s' % name, 'path:%s' % relpath]
     groups.extend(set(default_groups).difference(groups))
@@ -704,6 +790,7 @@ class XmlManifest(object):
                       name = name,
                       remote = remote.ToRemoteSpec(name),
                       gitdir = gitdir,
+                      objdir = objdir,
                       worktree = worktree,
                       relpath = relpath,
                       revisionExpr = revisionExpr,
@@ -720,6 +807,8 @@ class XmlManifest(object):
     for n in node.childNodes:
       if n.nodeName == 'copyfile':
         self._ParseCopyFile(project, n)
+      if n.nodeName == 'linkfile':
+        self._ParseLinkFile(project, n)
       if n.nodeName == 'annotation':
         self._ParseAnnotation(project, n)
       if n.nodeName == 'project':
@@ -732,10 +821,15 @@ class XmlManifest(object):
     if self.IsMirror:
       worktree = None
       gitdir = os.path.join(self.topdir, '%s.git' % name)
+      objdir = gitdir
     else:
       worktree = os.path.join(self.topdir, path).replace('\\', '/')
       gitdir = os.path.join(self.repodir, 'projects', '%s.git' % path)
-    return relpath, worktree, gitdir
+      objdir = os.path.join(self.repodir, 'project-objects', '%s.git' % name)
+    return relpath, worktree, gitdir, objdir
+
+  def GetProjectsWithName(self, name):
+    return self._projects.get(name, [])
 
   def GetSubprojectName(self, parent, submodule_path):
     return os.path.join(parent.name, submodule_path)
@@ -746,14 +840,15 @@ class XmlManifest(object):
   def _UnjoinRelpath(self, parent_relpath, relpath):
     return os.path.relpath(relpath, parent_relpath)
 
-  def GetSubprojectPaths(self, parent, path):
+  def GetSubprojectPaths(self, parent, name, path):
     relpath = self._JoinRelpath(parent.relpath, path)
     gitdir = os.path.join(parent.gitdir, 'subprojects', '%s.git' % path)
+    objdir = os.path.join(parent.gitdir, 'subproject-objects', '%s.git' % name)
     if self.IsMirror:
       worktree = None
     else:
       worktree = os.path.join(parent.worktree, path).replace('\\', '/')
-    return relpath, worktree, gitdir
+    return relpath, worktree, gitdir, objdir
 
   def _ParseCopyFile(self, project, node):
     src = self._reqatt(node, 'src')
@@ -763,6 +858,14 @@ class XmlManifest(object):
       # dest is relative to the top of the tree
       project.AddCopyFile(src, dest, os.path.join(self.topdir, dest))
 
+  def _ParseLinkFile(self, project, node):
+    src = self._reqatt(node, 'src')
+    dest = self._reqatt(node, 'dest')
+    if not self.IsMirror:
+      # src is project relative;
+      # dest is relative to the top of the tree
+      project.AddLinkFile(src, dest, os.path.join(self.topdir, dest))
+
   def _ParseAnnotation(self, project, node):
     name = self._reqatt(node, 'name')
     value = self._reqatt(node, 'value')
@@ -795,3 +898,38 @@ class XmlManifest(object):
       raise ManifestParseError("no %s in <%s> within %s" %
             (attname, node.nodeName, self.manifestFile))
     return v
+
+  def projectsDiff(self, manifest):
+    """return the projects differences between two manifests.
+
+    The diff will be from self to given manifest.
+
+    """
+    fromProjects = self.paths
+    toProjects = manifest.paths
+
+    fromKeys = sorted(fromProjects.keys())
+    toKeys = sorted(toProjects.keys())
+
+    diff = {'added': [], 'removed': [], 'changed': [], 'unreachable': []}
+
+    for proj in fromKeys:
+      if not proj in toKeys:
+        diff['removed'].append(fromProjects[proj])
+      else:
+        fromProj = fromProjects[proj]
+        toProj = toProjects[proj]
+        try:
+          fromRevId = fromProj.GetCommitRevisionId()
+          toRevId = toProj.GetCommitRevisionId()
+        except ManifestInvalidRevisionError:
+          diff['unreachable'].append((fromProj, toProj))
+        else:
+          if fromRevId != toRevId:
+            diff['changed'].append((fromProj, toProj))
+        toKeys.remove(proj)
+
+    for proj in toKeys:
+      diff['added'].append(toProjects[proj])
+
+    return diff

project.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 from __future__ import print_function
-import traceback
+import contextlib
 import errno
 import filecmp
 import os
@@ -23,8 +23,10 @@ import shutil
 import stat
 import subprocess
 import sys
+import tarfile
 import tempfile
 import time
+import traceback
 
 from color import Coloring
 from git_command import GitCommand, git_require
@@ -45,7 +47,7 @@ if not is_python3():
 def _lwrite(path, content):
   lock = '%s.lock' % path
 
-  fd = open(lock, 'wb')
+  fd = open(lock, 'w')
   try:
     fd.write(content)
   finally:
@@ -82,7 +84,7 @@ def _ProjectHooks():
   """
   global _project_hook_list
   if _project_hook_list is None:
-    d = os.path.abspath(os.path.dirname(__file__))
+    d = os.path.realpath(os.path.abspath(os.path.dirname(__file__)))
     d = os.path.join(d, 'hooks')
     _project_hook_list = [os.path.join(d, x) for x in os.listdir(d)]
   return _project_hook_list
@@ -196,13 +198,13 @@ class DiffColoring(Coloring):
     Coloring.__init__(self, config, 'diff')
     self.project = self.printer('header', attr='bold')
 
-class _Annotation:
+class _Annotation(object):
   def __init__(self, name, value, keep):
     self.name = name
     self.value = value
     self.keep = keep
 
-class _CopyFile:
+class _CopyFile(object):
   def __init__(self, src, dest, abssrc, absdest):
     self.src = src
     self.dest = dest
@@ -230,14 +232,40 @@ class _CopyFile:
       except IOError:
         _error('Cannot copy file %s to %s', src, dest)
 
+class _LinkFile(object):
+  def __init__(self, src, dest, abssrc, absdest):
+    self.src = src
+    self.dest = dest
+    self.abs_src = abssrc
+    self.abs_dest = absdest
+
+  def _Link(self):
+    src = self.abs_src
+    dest = self.abs_dest
+    # link file if it does not exist or is out of date
+    if not os.path.islink(dest) or os.readlink(dest) != src:
+      try:
+        # remove existing file first, since it might be read-only
+        if os.path.exists(dest):
+          os.remove(dest)
+        else:
+          dest_dir = os.path.dirname(dest)
+          if not os.path.isdir(dest_dir):
+            os.makedirs(dest_dir)
+        os.symlink(src, dest)
+      except IOError:
+        _error('Cannot link file %s to %s', src, dest)
+
 class RemoteSpec(object):
   def __init__(self,
                name,
                url=None,
-               review = None):
+               review=None,
+               revision=None):
     self.name = name
     self.url = url
     self.review = review
+    self.revision = revision
 
 class RepoHook(object):
   """A RepoHook contains information about a script to run as a hook.
@@ -413,7 +441,8 @@ class RepoHook(object):
       # and  convert to a HookError w/ just the failing traceback.
       context = {}
       try:
-        execfile(self._script_fullpath, context)
+        exec(compile(open(self._script_fullpath).read(),
+                     self._script_fullpath, 'exec'), context)
       except Exception:
         raise HookError('%s\nFailed to import %s hook; see traceback above.' % (
                         traceback.format_exc(), self._hook_type))
@@ -487,6 +516,7 @@ class Project(object):
                name,
                remote,
                gitdir,
+               objdir,
                worktree,
                relpath,
                revisionExpr,
@@ -507,6 +537,7 @@ class Project(object):
       name: The `name` attribute of manifest.xml's project element.
       remote: RemoteSpec object specifying its remote's properties.
       gitdir: Absolute path of git directory.
+      objdir: Absolute path of directory to store git objects.
       worktree: Absolute path of git working tree.
       relpath: Relative path of git working tree to repo's top directory.
       revisionExpr: The `revision` attribute of manifest.xml's project element.
@@ -525,6 +556,7 @@ class Project(object):
     self.name = name
     self.remote = remote
     self.gitdir = gitdir.replace('\\', '/')
+    self.objdir = objdir.replace('\\', '/')
     if worktree:
       self.worktree = worktree.replace('\\', '/')
     else:
@@ -551,17 +583,19 @@ class Project(object):
 
     self.snapshots = {}
     self.copyfiles = []
+    self.linkfiles = []
     self.annotations = []
     self.config = GitConfig.ForRepository(
                     gitdir=self.gitdir,
                     defaults=self.manifest.globalConfig)
 
     if self.worktree:
-      self.work_git = self._GitGetByExec(self, bare=False)
+      self.work_git = self._GitGetByExec(self, bare=False, gitdir=gitdir)
     else:
       self.work_git = None
-    self.bare_git = self._GitGetByExec(self, bare=True)
+    self.bare_git = self._GitGetByExec(self, bare=True, gitdir=gitdir)
     self.bare_ref = GitRefs(gitdir)
+    self.bare_objdir = self._GitGetByExec(self, bare=True, gitdir=objdir)
     self.dest_branch = dest_branch
 
     # This will be filled in if a project is later identified to be the
@@ -703,26 +737,48 @@ class Project(object):
     return matched
 
 ## Status Display ##
+  def UncommitedFiles(self, get_all=True):
+    """Returns a list of strings, uncommitted files in the git tree.
 
-  def HasChanges(self):
-    """Returns true if there are uncommitted changes.
+    Args:
+      get_all: a boolean, if True - get information about all different
+               uncommitted files. If False - return as soon as any kind of
+               uncommitted files is detected.
     """
+    details = []
     self.work_git.update_index('-q',
                                '--unmerged',
                                '--ignore-missing',
                                '--refresh')
     if self.IsRebaseInProgress():
-      return True
+      details.append("rebase in progress")
+      if not get_all:
+        return details
 
-    if self.work_git.DiffZ('diff-index', '--cached', HEAD):
-      return True
+    changes = self.work_git.DiffZ('diff-index', '--cached', HEAD).keys()
+    if changes:
+      details.extend(changes)
+      if not get_all:
+        return details
 
-    if self.work_git.DiffZ('diff-files'):
-      return True
+    changes = self.work_git.DiffZ('diff-files').keys()
+    if changes:
+      details.extend(changes)
+      if not get_all:
+        return details
 
-    if self.work_git.LsOthers():
-      return True
+    changes = self.work_git.LsOthers()
+    if changes:
+      details.extend(changes)
 
+    return details
+
+  def HasChanges(self):
+    """Returns true if there are uncommitted changes.
+    """
+    if self.UncommitedFiles(get_all=False):
+      return True
+    else:
       return False
 
   def PrintWorkTreeStatus(self, output_redir=None):
@@ -982,15 +1038,62 @@ class Project(object):
 
 ## Sync ##
 
+  def _ExtractArchive(self, tarpath, path=None):
+    """Extract the given tar on its current location
+
+    Args:
+        - tarpath: The path to the actual tar file
+
+    """
+    try:
+      with tarfile.open(tarpath, 'r') as tar:
+        tar.extractall(path=path)
+        return True
+    except (IOError, tarfile.TarError) as e:
+      print("error: Cannot extract archive %s: "
+            "%s" % (tarpath, str(e)), file=sys.stderr)
+    return False
+
   def Sync_NetworkHalf(self,
       quiet=False,
       is_new=None,
       current_branch_only=False,
       clone_bundle=True,
-      no_tags=False):
+      no_tags=False,
+      archive=False):
     """Perform only the network IO portion of the sync process.
        Local working directory/branch state is not affected.
     """
+    if archive and not isinstance(self, MetaProject):
+      if self.remote.url.startswith(('http://', 'https://')):
+        print("error: %s: Cannot fetch archives from http/https "
+              "remotes." % self.name, file=sys.stderr)
+        return False
+
+      name = self.relpath.replace('\\', '/')
+      name = name.replace('/', '_')
+      tarpath = '%s.tar' % name
+      topdir = self.manifest.topdir
+
+      try:
+        self._FetchArchive(tarpath, cwd=topdir)
+      except GitError as e:
+        print('error: %s' % str(e), file=sys.stderr)
+        return False
+
+      # From now on, we only need absolute tarpath
+      tarpath = os.path.join(topdir, tarpath)
+
+      if not self._ExtractArchive(tarpath, path=topdir):
+        return False
+      try:
+        os.remove(tarpath)
+      except OSError as e:
+        print("warn: Cannot remove archive %s: "
+              "%s" % (tarpath, str(e)), file=sys.stderr)
+      self._CopyAndLinkFiles()
+      return True
+
     if is_new is None:
       is_new = not self.Exists
     if is_new:
@@ -1026,9 +1129,11 @@ class Project(object):
       elif self.manifest.default.sync_c:
         current_branch_only = True
 
-    if not self._RemoteFetch(initial=is_new, quiet=quiet, alt_dir=alt_dir,
+    has_sha1 = ID_RE.match(self.revisionExpr) and self._CheckForSha1()
+    if (not has_sha1  #Need to fetch since we don't already have this revision
+        and not self._RemoteFetch(initial=is_new, quiet=quiet, alt_dir=alt_dir,
                                   current_branch_only=current_branch_only,
-                             no_tags=no_tags):
+                                  no_tags=no_tags)):
       return False
 
     if self.worktree:
@@ -1044,9 +1149,28 @@ class Project(object):
   def PostRepoUpgrade(self):
     self._InitHooks()
 
-  def _CopyFiles(self):
+  def _CopyAndLinkFiles(self):
     for copyfile in self.copyfiles:
       copyfile._Copy()
+    for linkfile in self.linkfiles:
+      linkfile._Link()
+
+  def GetCommitRevisionId(self):
+    """Get revisionId of a commit.
+
+    Use this method instead of GetRevisionId to get the id of the commit rather
+    than the id of the current git object (for example, a tag)
+
+    """
+    if not self.revisionExpr.startswith(R_TAGS):
+      return self.GetRevisionId(self._allrefs)
+
+    try:
+      return self.bare_git.rev_list(self.revisionExpr, '-1')[0]
+    except GitError:
+      raise ManifestInvalidRevisionError(
+        'revision %s in %s not found' % (self.revisionExpr,
+                                         self.name))
 
   def GetRevisionId(self, all_refs=None):
     if self.revisionId:
@@ -1069,15 +1193,15 @@ class Project(object):
     """Perform only the local IO portion of the sync process.
        Network access is not required.
     """
+    self._InitWorkTree()
     all_refs = self.bare_ref.all
     self.CleanPublishedCache(all_refs)
     revid = self.GetRevisionId(all_refs)
 
     def _doff():
       self._FastForward(revid)
-      self._CopyFiles()
+      self._CopyAndLinkFiles()
 
-    self._InitWorkTree()
     head = self.work_git.GetHead()
     if head.startswith(R_HEADS):
       branch = head[len(R_HEADS):]
@@ -1112,7 +1236,7 @@ class Project(object):
       except GitError as e:
         syncbuf.fail(self, e)
         return
-      self._CopyFiles()
+      self._CopyAndLinkFiles()
       return
 
     if head == revid:
@@ -1134,7 +1258,7 @@ class Project(object):
       except GitError as e:
         syncbuf.fail(self, e)
         return
-      self._CopyFiles()
+      self._CopyAndLinkFiles()
       return
 
     upstream_gain = self._revlist(not_rev(HEAD), revid)
@@ -1165,7 +1289,7 @@ class Project(object):
     last_mine = None
     cnt_mine = 0
     for commit in local_changes:
-      commit_id, committer_email = commit.split(' ', 1)
+      commit_id, committer_email = commit.decode('utf-8').split(' ', 1)
       if committer_email == self.UserEmail:
         last_mine = commit_id
         cnt_mine += 1
@@ -1207,12 +1331,12 @@ class Project(object):
     if cnt_mine > 0 and self.rebase:
       def _dorebase():
         self._Rebase(upstream='%s^1' % last_mine, onto=revid)
-        self._CopyFiles()
+        self._CopyAndLinkFiles()
       syncbuf.later2(self, _dorebase)
     elif local_changes:
       try:
         self._ResetHard(revid)
-        self._CopyFiles()
+        self._CopyAndLinkFiles()
       except GitError as e:
         syncbuf.fail(self, e)
         return
@@ -1225,6 +1349,12 @@ class Project(object):
     abssrc = os.path.join(self.worktree, src)
     self.copyfiles.append(_CopyFile(src, dest, abssrc, absdest))
 
+  def AddLinkFile(self, src, dest, absdest):
+    # dest should already be an absolute path, but src is project relative
+    # make src an absolute path
+    abssrc = os.path.join(self.worktree, src)
+    self.linkfiles.append(_LinkFile(src, dest, abssrc, absdest))
+
   def AddAnnotation(self, name, value, keep):
     self.annotations.append(_Annotation(name, value, keep))
 
@@ -1255,7 +1385,7 @@ class Project(object):
       return True
 
     all_refs = self.bare_ref.all
-    if (R_HEADS + name) in all_refs:
+    if R_HEADS + name in all_refs:
       return GitCommand(self,
                         ['checkout', name, '--'],
                         capture_stdout=True,
@@ -1419,7 +1549,7 @@ class Project(object):
 
     kept = []
     for branch in kill:
-      if (R_HEADS + branch) in left:
+      if R_HEADS + branch in left:
         branch = self.GetBranch(branch)
         base = branch.LocalMerge
         if not base:
@@ -1544,18 +1674,22 @@ class Project(object):
       return result
     for rev, path, url in self._GetSubmodules():
       name = self.manifest.GetSubprojectName(self, path)
-      project = self.manifest.projects.get(name)
+      relpath, worktree, gitdir, objdir = \
+          self.manifest.GetSubprojectPaths(self, name, path)
+      project = self.manifest.paths.get(relpath)
       if project:
         result.extend(project.GetDerivedSubprojects())
         continue
-      relpath, worktree, gitdir = self.manifest.GetSubprojectPaths(self, path)
+
       remote = RemoteSpec(self.remote.name,
                           url=url,
-                          review = self.remote.review)
+                          review=self.remote.review,
+                          revision=self.remote.revision)
       subproject = Project(manifest=self.manifest,
                            name=name,
                            remote=remote,
                            gitdir=gitdir,
+                           objdir=objdir,
                            worktree=worktree,
                            relpath=relpath,
                            revisionExpr=self.revisionExpr,
@@ -1572,6 +1706,29 @@ class Project(object):
 
 
 ## Direct Git Commands ##
+  def _CheckForSha1(self):
+    try:
+      # if revision (sha or tag) is not present then following function
+      # throws an error.
+      self.bare_git.rev_parse('--verify', '%s^0' % self.revisionExpr)
+      return True
+    except GitError:
+      # There is no such persistent revision. We have to fetch it.
+      return False
+
+  def _FetchArchive(self, tarpath, cwd=None):
+    cmd = ['archive', '-v', '-o', tarpath]
+    cmd.append('--remote=%s' % self.remote.url)
+    cmd.append('--prefix=%s/' % self.relpath)
+    cmd.append(self.revisionExpr)
+
+    command = GitCommand(self, cmd, cwd=cwd,
+                         capture_stdout=True,
+                         capture_stderr=True)
+
+    if command.Wait() != 0:
+      raise GitError('git archive %s: %s' % (self.name, command.stderr))
+
 
   def _RemoteFetch(self, name=None,
                    current_branch_only=False,
@@ -1582,29 +1739,43 @@ class Project(object):
 
     is_sha1 = False
     tag_name = None
+    depth = None
 
-    def CheckForSha1():
-      try:
-        # if revision (sha or tag) is not present then following function
-        # throws an error.
-        self.bare_git.rev_parse('--verify', '%s^0' % self.revisionExpr)
-        return True
-      except GitError:
-        # There is no such persistent revision. We have to fetch it.
-        return False
+    # The depth should not be used when fetching to a mirror because
+    # it will result in a shallow repository that cannot be cloned or
+    # fetched from.
+    if not self.manifest.IsMirror:
+      if self.clone_depth:
+        depth = self.clone_depth
+      else:
+        depth = self.manifest.manifestProject.config.GetString('repo.depth')
+      # The repo project should never be synced with partial depth
+      if self.relpath == '.repo/repo':
+        depth = None
+
+    if depth:
+      current_branch_only = True
 
-    if current_branch_only:
     if ID_RE.match(self.revisionExpr) is not None:
       is_sha1 = True
-      elif self.revisionExpr.startswith(R_TAGS):
+
+    if current_branch_only:
+      if self.revisionExpr.startswith(R_TAGS):
         # this is a tag and its sha1 value should never change
         tag_name = self.revisionExpr[len(R_TAGS):]
 
       if is_sha1 or tag_name is not None:
-        if CheckForSha1():
+        if self._CheckForSha1():
           return True
-      if is_sha1 and (not self.upstream or ID_RE.match(self.upstream)):
-        current_branch_only = False
+      if is_sha1 and not depth:
+        # When syncing a specific commit and --depth is not set:
+        # * if upstream is explicitly specified and is not a sha1, fetch only
+        #   upstream as users expect only upstream to be fetch.
+        #   Note: The commit might not be in upstream in which case the sync
+        #   will fail.
+        # * otherwise, fetch all branches to make sure we end up with the
+        #   specific commit.
+        current_branch_only = self.upstream and not ID_RE.match(self.upstream)
 
     if not name:
       name = self.remote.name
@@ -1654,13 +1825,7 @@ class Project(object):
 
     cmd = ['fetch']
 
-    # The --depth option only affects the initial fetch; after that we'll do
-    # full fetches of changes.
-    if self.clone_depth:
-      depth = self.clone_depth
-    else:
-      depth = self.manifest.manifestProject.config.GetString('repo.depth')
-    if depth and initial:
+    if depth:
       cmd.append('--depth=%s' % depth)
 
     if quiet:
@@ -1669,32 +1834,64 @@ class Project(object):
       cmd.append('--update-head-ok')
     cmd.append(name)
 
-    if not current_branch_only:
-      # Fetch whole repo
     # If using depth then we should not get all the tags since they may
     # be outside of the depth.
     if no_tags or depth:
       cmd.append('--no-tags')
     else:
       cmd.append('--tags')
-      cmd.append(str((u'+refs/heads/*:') + remote.ToLocal('refs/heads/*')))
+
+    spec = []
+    if not current_branch_only:
+      # Fetch whole repo
+      spec.append(str((u'+refs/heads/*:') + remote.ToLocal('refs/heads/*')))
     elif tag_name is not None:
-      cmd.append('tag')
-      cmd.append(tag_name)
-    else:
+      spec.append('tag')
+      spec.append(tag_name)
+
     branch = self.revisionExpr
+    if is_sha1 and depth:
+      # Shallow checkout of a specific commit, fetch from that commit and not
+      # the heads only as the commit might be deeper in the history.
+      spec.append(branch)
+    else:
       if is_sha1:
         branch = self.upstream
-      if branch.startswith(R_HEADS):
-        branch = branch[len(R_HEADS):]
-      cmd.append(str((u'+refs/heads/%s:' % branch) + remote.ToLocal('refs/heads/%s' % branch)))
+      if branch is not None and branch.strip():
+        if not branch.startswith('refs/'):
+          branch = R_HEADS + branch
+        spec.append(str((u'+%s:' % branch) + remote.ToLocal(branch)))
+    cmd.extend(spec)
+
+    shallowfetch = self.config.GetString('repo.shallowfetch')
+    if shallowfetch and shallowfetch != ' '.join(spec):
+      GitCommand(self, ['fetch', '--unshallow', name] + shallowfetch.split(),
+                 bare=True, ssh_proxy=ssh_proxy).Wait()
+    if depth:
+      self.config.SetString('repo.shallowfetch', ' '.join(spec))
+    else:
+      self.config.SetString('repo.shallowfetch', None)
 
     ok = False
     for _i in range(2):
-      ret = GitCommand(self, cmd, bare=True, ssh_proxy=ssh_proxy).Wait()
+      gitcmd = GitCommand(self, cmd, bare=True, capture_stderr=True,
+                          ssh_proxy=ssh_proxy)
+      ret = gitcmd.Wait()
+      print(gitcmd.stderr, file=sys.stderr, end='')
       if ret == 0:
         ok = True
         break
+      # If needed, run the 'git remote prune' the first time through the loop
+      elif (not _i and
+            "error:" in gitcmd.stderr and
+            "git remote prune" in gitcmd.stderr):
+        prunecmd = GitCommand(self, ['remote', 'prune', name], bare=True,
+                              capture_stderr=True, ssh_proxy=ssh_proxy)
+        ret = prunecmd.Wait()
+        print(prunecmd.stderr, file=sys.stderr, end='')
+        if ret:
+          break
+        continue
       elif current_branch_only and is_sha1 and ret == 128:
         # Exit code 128 means "couldn't find the ref you asked for"; if we're in sha1
         # mode, we just tried sync'ing from the upstream field; it doesn't exist, thus
@@ -1714,7 +1911,7 @@ class Project(object):
       # We just synced the upstream given branch; verify we
       # got what we wanted, else trigger a second run of all
       # refs.
-      if not CheckForSha1():
+      if not self._CheckForSha1():
         return self._RemoteFetch(name=name, current_branch_only=False,
                                  initial=False, quiet=quiet, alt_dir=alt_dir)
 
@@ -1777,9 +1974,9 @@ class Project(object):
         os.remove(tmpPath)
     if 'http_proxy' in os.environ and 'darwin' == sys.platform:
       cmd += ['--proxy', os.environ['http_proxy']]
-    cookiefile = self._GetBundleCookieFile(srcUrl)
+    with self._GetBundleCookieFile(srcUrl, quiet) as cookiefile:
       if cookiefile:
-      cmd += ['--cookie', cookiefile]
+        cmd += ['--cookie', cookiefile, '--cookie-jar', cookiefile]
       if srcUrl.startswith('persistent-'):
         srcUrl = srcUrl[len('persistent-'):]
       cmd += [srcUrl]
@@ -1804,7 +2001,7 @@ class Project(object):
         return False
 
     if os.path.exists(tmpPath):
-      if curlret == 0 and self._IsValidBundle(tmpPath):
+      if curlret == 0 and self._IsValidBundle(tmpPath, quiet):
         os.rename(tmpPath, dstPath)
         return True
       else:
@@ -1813,25 +2010,27 @@ class Project(object):
     else:
       return False
 
-  def _IsValidBundle(self, path):
+  def _IsValidBundle(self, path, quiet):
     try:
       with open(path) as f:
         if f.read(16) == '# v2 git bundle\n':
           return True
         else:
+          if not quiet:
             print("Invalid clone.bundle file; ignoring.", file=sys.stderr)
           return False
     except OSError:
       return False
 
-  def _GetBundleCookieFile(self, url):
+  @contextlib.contextmanager
+  def _GetBundleCookieFile(self, url, quiet):
     if url.startswith('persistent-'):
       try:
         p = subprocess.Popen(
             ['git-remote-persistent-https', '-print_config', url],
             stdin=subprocess.PIPE, stdout=subprocess.PIPE,
             stderr=subprocess.PIPE)
-        p.stdin.close()  # Tell subprocess it's ok to close.
+        try:
           prefix = 'http.cookiefile='
           cookiefile = None
           for line in p.stdout:
@@ -1839,19 +2038,23 @@ class Project(object):
             if line.startswith(prefix):
               cookiefile = line[len(prefix):]
               break
-        if p.wait():
-          line = iter(p.stderr).next()
-          if ' -print_config' in line:
-            pass  # Persistent proxy doesn't support -print_config.
-          else:
-            print(line + p.stderr.read(), file=sys.stderr)
+          # Leave subprocess open, as cookie file may be transient.
           if cookiefile:
-          return cookiefile
+            yield cookiefile
+            return
+        finally:
+          p.stdin.close()
+          if p.wait():
+            err_msg = p.stderr.read()
+            if ' -print_config' in err_msg:
+              pass  # Persistent proxy doesn't support -print_config.
+            elif not quiet:
+              print(err_msg, file=sys.stderr)
       except OSError as e:
         if e.errno == errno.ENOENT:
           pass  # No persistent proxy.
         raise
-    return GitConfig.ForUser().GetString('http.cookiefile')
+    yield GitConfig.ForUser().GetString('http.cookiefile')
 
   def _Checkout(self, rev, quiet=False):
     cmd = ['checkout']
@@ -1863,7 +2066,7 @@ class Project(object):
       if self._allrefs:
         raise GitError('%s checkout %s ' % (self.name, rev))
 
-  def _CherryPick(self, rev, quiet=False):
+  def _CherryPick(self, rev):
     cmd = ['cherry-pick']
     cmd.append(rev)
     cmd.append('--')
@@ -1871,7 +2074,7 @@ class Project(object):
       if self._allrefs:
         raise GitError('%s cherry-pick %s ' % (self.name, rev))
 
-  def _Revert(self, rev, quiet=False):
+  def _Revert(self, rev):
     cmd = ['revert']
     cmd.append('--no-edit')
     cmd.append(rev)
@@ -1905,8 +2108,17 @@ class Project(object):
 
   def _InitGitDir(self, mirror_git=None):
     if not os.path.exists(self.gitdir):
+
+      # Initialize the bare repository, which contains all of the objects.
+      if not os.path.exists(self.objdir):
+        os.makedirs(self.objdir)
+        self.bare_objdir.init()
+
+      # If we have a separate directory to hold refs, initialize it as well.
+      if self.objdir != self.gitdir:
         os.makedirs(self.gitdir)
-      self.bare_git.init()
+        self._ReferenceGitDir(self.objdir, self.gitdir, share_refs=False,
+                              copy_all=True)
 
       mp = self.manifest.manifestProject
       ref_dir = mp.config.GetString('repo.reference') or ''
@@ -1943,19 +2155,10 @@ class Project(object):
 
   def _UpdateHooks(self):
     if os.path.exists(self.gitdir):
-      # Always recreate hooks since they can have been changed
-      # since the latest update.
-      hooks = self._gitdir_path('hooks')
-      try:
-        to_rm = os.listdir(hooks)
-      except OSError:
-        to_rm = []
-      for old_hook in to_rm:
-        os.remove(os.path.join(hooks, old_hook))
       self._InitHooks()
 
   def _InitHooks(self):
-    hooks = self._gitdir_path('hooks')
+    hooks = os.path.realpath(self._gitdir_path('hooks'))
     if not os.path.exists(hooks):
       os.makedirs(hooks)
     for stock_hook in _ProjectHooks():
@@ -2022,34 +2225,70 @@ class Project(object):
         msg = 'manifest set to %s' % self.revisionExpr
         self.bare_git.symbolic_ref('-m', msg, ref, dst)
 
-  def _InitWorkTree(self):
-    dotgit = os.path.join(self.worktree, '.git')
-    if not os.path.exists(dotgit):
-      os.makedirs(dotgit)
+  def _ReferenceGitDir(self, gitdir, dotgit, share_refs, copy_all):
+    """Update |dotgit| to reference |gitdir|, using symlinks where possible.
 
-      for name in ['config',
-                   'description',
-                   'hooks',
-                   'info',
-                   'logs',
-                   'objects',
-                   'packed-refs',
-                   'refs',
-                   'rr-cache',
-                   'svn']:
+    Args:
+      gitdir: The bare git repository. Must already be initialized.
+      dotgit: The repository you would like to initialize.
+      share_refs: If true, |dotgit| will store its refs under |gitdir|.
+          Only one work tree can store refs under a given |gitdir|.
+      copy_all: If true, copy all remaining files from |gitdir| -> |dotgit|.
+          This saves you the effort of initializing |dotgit| yourself.
+    """
+    # These objects can be shared between several working trees.
+    symlink_files = ['description', 'info']
+    symlink_dirs = ['hooks', 'objects', 'rr-cache', 'svn']
+    if share_refs:
+      # These objects can only be used by a single working tree.
+      symlink_files += ['config', 'packed-refs', 'shallow']
+      symlink_dirs += ['logs', 'refs']
+    to_symlink = symlink_files + symlink_dirs
+
+    to_copy = []
+    if copy_all:
+      to_copy = os.listdir(gitdir)
+
+    for name in set(to_copy).union(to_symlink):
       try:
-          src = os.path.join(self.gitdir, name)
-          dst = os.path.join(dotgit, name)
-          if os.path.islink(dst) or not os.path.exists(dst):
-            os.symlink(os.path.relpath(src, os.path.dirname(dst)), dst)
-          else:
+        src = os.path.realpath(os.path.join(gitdir, name))
+        dst = os.path.realpath(os.path.join(dotgit, name))
+
+        if os.path.lexists(dst) and not os.path.islink(dst):
           raise GitError('cannot overwrite a local work tree')
+
+        # If the source dir doesn't exist, create an empty dir.
+        if name in symlink_dirs and not os.path.lexists(src):
+          os.makedirs(src)
+
+        # If the source file doesn't exist, ensure the destination
+        # file doesn't either.
+        if name in symlink_files and not os.path.lexists(src):
+          try:
+            os.remove(dst)
+          except OSError:
+            pass
+
+        if name in to_symlink:
+          os.symlink(os.path.relpath(src, os.path.dirname(dst)), dst)
+        elif copy_all and not os.path.islink(dst):
+          if os.path.isdir(src):
+            shutil.copytree(src, dst)
+          elif os.path.isfile(src):
+            shutil.copy(src, dst)
       except OSError as e:
         if e.errno == errno.EPERM:
           raise GitError('filesystem must support symlinks')
         else:
           raise
 
+  def _InitWorkTree(self):
+    dotgit = os.path.join(self.worktree, '.git')
+    if not os.path.exists(dotgit):
+      os.makedirs(dotgit)
+      self._ReferenceGitDir(self.gitdir, dotgit, share_refs=True,
+                            copy_all=False)
+
       _lwrite(os.path.join(dotgit, HEAD), '%s\n' % self.GetRevisionId())
 
       cmd = ['read-tree', '--reset', '-u']
@@ -2058,14 +2297,10 @@ class Project(object):
       if GitCommand(self, cmd).Wait() != 0:
         raise GitError("cannot initialize work tree")
 
-      rr_cache = os.path.join(self.gitdir, 'rr-cache')
-      if not os.path.exists(rr_cache):
-        os.makedirs(rr_cache)
-
-      self._CopyFiles()
+      self._CopyAndLinkFiles()
 
   def _gitdir_path(self, path):
-    return os.path.join(self.gitdir, path)
+    return os.path.realpath(os.path.join(self.gitdir, path))
 
   def _revlist(self, *args, **kw):
     a = []
@@ -2077,10 +2312,48 @@ class Project(object):
   def _allrefs(self):
     return self.bare_ref.all
 
+  def _getLogs(self, rev1, rev2, oneline=False, color=True):
+    """Get logs between two revisions of this project."""
+    comp = '..'
+    if rev1:
+      revs = [rev1]
+      if rev2:
+        revs.extend([comp, rev2])
+      cmd = ['log', ''.join(revs)]
+      out = DiffColoring(self.config)
+      if out.is_on and color:
+        cmd.append('--color')
+      if oneline:
+        cmd.append('--oneline')
+
+      try:
+        log = GitCommand(self, cmd, capture_stdout=True, capture_stderr=True)
+        if log.Wait() == 0:
+          return log.stdout
+      except GitError:
+        # worktree may not exist if groups changed for example. In that case,
+        # try in gitdir instead.
+        if not os.path.exists(self.worktree):
+          return self.bare_git.log(*cmd[1:])
+        else:
+          raise
+    return None
+
+  def getAddedAndRemovedLogs(self, toProject, oneline=False, color=True):
+    """Get the list of logs from this revision to given revisionId"""
+    logs = {}
+    selfId = self.GetRevisionId(self._allrefs)
+    toId = toProject.GetRevisionId(toProject._allrefs)
+
+    logs['added'] = self._getLogs(selfId, toId, oneline=oneline, color=color)
+    logs['removed'] = self._getLogs(toId, selfId, oneline=oneline, color=color)
+    return logs
+
   class _GitGetByExec(object):
-    def __init__(self, project, bare):
+    def __init__(self, project, bare, gitdir):
       self._project = project
       self._bare = bare
+      self._gitdir = gitdir
 
     def LsOthers(self):
       p = GitCommand(self._project,
@@ -2089,6 +2362,7 @@ class Project(object):
                       '--others',
                       '--exclude-standard'],
                      bare=False,
+                     gitdir=self._gitdir,
                      capture_stdout=True,
                      capture_stderr=True)
       if p.Wait() == 0:
@@ -2104,6 +2378,7 @@ class Project(object):
       cmd.extend(args)
       p = GitCommand(self._project,
                      cmd,
+                     gitdir=self._gitdir,
                      bare=False,
                      capture_stdout=True,
                      capture_stderr=True)
@@ -2114,8 +2389,8 @@ class Project(object):
           out = iter(out[:-1].split('\0'))  # pylint: disable=W1401
           while out:
             try:
-              info = out.next()
-              path = out.next()
+              info = next(out)
+              path = next(out)
             except StopIteration:
               break
 
@@ -2141,7 +2416,7 @@ class Project(object):
             info = _Info(path, *info)
             if info.status in ('R', 'C'):
               info.src_path = info.path
-              info.path = out.next()
+              info.path = next(out)
             r[info.path] = info
         return r
       finally:
@@ -2154,8 +2429,8 @@ class Project(object):
         path = os.path.join(self._project.worktree, '.git', HEAD)
       try:
         fd = open(path, 'rb')
-      except IOError:
-        raise NoManifestException(path)
+      except IOError as e:
+        raise NoManifestException(path, str(e))
       try:
         line = fd.read()
       finally:
@@ -2213,6 +2488,7 @@ class Project(object):
       p = GitCommand(self._project,
                      cmdv,
                      bare=self._bare,
+                     gitdir=self._gitdir,
                      capture_stdout=True,
                      capture_stderr=True)
       r = []
@@ -2265,6 +2541,7 @@ class Project(object):
         p = GitCommand(self._project,
                        cmdv,
                        bare=self._bare,
+                       gitdir=self._gitdir,
                        capture_stdout=True,
                        capture_stderr=True)
         if p.Wait() != 0:
@@ -2398,6 +2675,7 @@ class MetaProject(Project):
                      manifest=manifest,
                      name=name,
                      gitdir=gitdir,
+                     objdir=gitdir,
                      worktree=worktree,
                      remote=RemoteSpec('origin'),
                      relpath='.repo/%s' % name,
@@ -2414,7 +2692,7 @@ class MetaProject(Project):
           self.revisionExpr = base
           self.revisionId = None
 
-  def MetaBranchSwitch(self, target):
+  def MetaBranchSwitch(self):
     """ Prepare MetaProject for manifest branch switch
     """
 

repo

@@ -20,7 +20,7 @@ REPO_REV = 'stable'
 # limitations under the License.
 
 # increment this whenever we make important changes to this script
-VERSION = (1, 20)
+VERSION = (1, 21)
 
 # increment this if the MAINTAINER_KEYS block is modified
 KEYRING_VERSION = (1, 2)
@@ -110,9 +110,11 @@ REPO_MAIN = S_repo + '/main.py' # main script
 MIN_PYTHON_VERSION = (2, 6)     # minimum supported python version
 
 
+import errno
 import optparse
 import os
 import re
+import shutil
 import stat
 import subprocess
 import sys
@@ -137,11 +139,6 @@ def _print(*objects, **kwargs):
 
 # Python version check
 ver = sys.version_info
-if ver[0] == 3:
-  _print('error: Python 3 support is not fully implemented in repo yet.\n'
-         'Please use Python 2.6 - 2.7 instead.',
-         file=sys.stderr)
-  sys.exit(1)
 if (ver[0], ver[1]) < MIN_PYTHON_VERSION:
   _print('error: Python version %s unsupported.\n'
          'Please use Python 2.6 - 2.7 instead.'
@@ -181,6 +178,10 @@ group.add_option('--reference',
 group.add_option('--depth', type='int', default=None,
                  dest='depth',
                  help='create a shallow clone with given depth; see git clone')
+group.add_option('--archive',
+                 dest='archive', action='store_true',
+                 help='checkout an archive instead of a git repository for '
+                      'each project. See git archive.')
 group.add_option('-g', '--groups',
                  dest='groups', default='default',
                  help='restrict manifest projects to ones with specified '
@@ -240,10 +241,10 @@ def _Init(args):
     _print("fatal: invalid branch name '%s'" % branch, file=sys.stderr)
     raise CloneFailure()
 
-  if not os.path.isdir(repodir):
   try:
     os.mkdir(repodir)
   except OSError as e:
+    if e.errno != errno.EEXIST:
       _print('fatal: cannot make %s directory: %s'
              % (repodir, e.strerror), file=sys.stderr)
       # Don't raise CloneFailure; that would delete the
@@ -274,6 +275,20 @@ def _Init(args):
     raise
 
 
+def ParseGitVersion(ver_str):
+  if not ver_str.startswith('git version '):
+    return None
+
+  num_ver_str = ver_str[len('git version '):].strip().split('-')[0]
+  to_tuple = []
+  for num_str in num_ver_str.split('.')[:3]:
+    if num_str.isdigit():
+      to_tuple.append(int(num_str))
+    else:
+      to_tuple.append(0)
+  return tuple(to_tuple)
+
+
 def _CheckGitVersion():
   cmd = [GIT, '--version']
   try:
@@ -291,12 +306,11 @@ def _CheckGitVersion():
   proc.stdout.close()
   proc.wait()
 
-  if not ver_str.startswith('git version '):
+  ver_act = ParseGitVersion(ver_str)
+  if ver_act is None:
     _print('error: "%s" unsupported' % ver_str, file=sys.stderr)
     raise CloneFailure()
 
-  ver_str = ver_str[len('git version '):].strip()
-  ver_act = tuple(map(int, ver_str.split('.')[0:3]))
   if ver_act < MIN_GIT_VERSION:
     need = '.'.join(map(str, MIN_GIT_VERSION))
     _print('fatal: git %s or later required' % need, file=sys.stderr)
@@ -322,18 +336,18 @@ def NeedSetupGnuPG():
 
 
 def SetupGnuPG(quiet):
-  if not os.path.isdir(home_dot_repo):
   try:
     os.mkdir(home_dot_repo)
   except OSError as e:
+    if e.errno != errno.EEXIST:
       _print('fatal: cannot make %s directory: %s'
              % (home_dot_repo, e.strerror), file=sys.stderr)
       sys.exit(1)
 
-  if not os.path.isdir(gpg_dir):
   try:
     os.mkdir(gpg_dir, stat.S_IRWXU)
   except OSError as e:
+    if e.errno != errno.EEXIST:
       _print('fatal: cannot make %s directory: %s' % (gpg_dir, e.strerror),
              file=sys.stderr)
       sys.exit(1)
@@ -448,7 +462,7 @@ def _DownloadBundle(url, local, quiet):
     try:
       r = urllib.request.urlopen(url)
     except urllib.error.HTTPError as e:
-      if e.code in [403, 404]:
+      if e.code in [401, 403, 404]:
         return False
       _print('fatal: Cannot get %s' % url, file=sys.stderr)
       _print('fatal: HTTP error %s' % e.code, file=sys.stderr)
@@ -724,12 +738,7 @@ def main(orig_args):
       try:
         _Init(args)
       except CloneFailure:
-        for root, dirs, files in os.walk(repodir, topdown=False):
-          for name in files:
-            os.remove(os.path.join(root, name))
-          for name in dirs:
-            os.rmdir(os.path.join(root, name))
-        os.rmdir(repodir)
+        shutil.rmtree(os.path.join(repodir, S_repo), ignore_errors=True)
         sys.exit(1)
       repo_main, rel_repo_dir = _FindRepo()
     else:
@@ -739,7 +748,7 @@ def main(orig_args):
     repo_main = my_main
 
   ver_str = '.'.join(map(str, VERSION))
-  me = [repo_main,
+  me = [sys.executable, repo_main,
         '--repo-dir=%s' % rel_repo_dir,
         '--wrapper-version=%s' % ver_str,
         '--wrapper-path=%s' % wrapper_path,
@@ -747,7 +756,7 @@ def main(orig_args):
   me.extend(orig_args)
   me.extend(extra_args)
   try:
-    os.execv(repo_main, me)
+    os.execv(sys.executable, me)
   except OSError as e:
     _print("fatal: unable to start %s" % repo_main, file=sys.stderr)
     _print("fatal: %s" % e, file=sys.stderr)
@@ -755,4 +764,8 @@ def main(orig_args):
 
 
 if __name__ == '__main__':
+  if ver[0] == 3:
+    _print('warning: Python 3 support is currently experimental. YMMV.\n'
+           'Please use Python 2.6 - 2.7 instead.',
+           file=sys.stderr)
   main(sys.argv[1:])

subcmds/branches.py

@@ -46,6 +46,10 @@ class BranchInfo(object):
   def IsCurrent(self):
     return self.current > 0
 
+  @property
+  def IsSplitCurrent(self):
+    return self.current != 0 and self.current != len(self.projects)
+
   @property
   def IsPublished(self):
     return self.published > 0
@@ -139,10 +143,14 @@ is shown, then the branch appears in all projects.
       if in_cnt < project_cnt:
         fmt = out.write
         paths = []
-        if in_cnt < project_cnt - in_cnt: 
+        non_cur_paths = []
+        if i.IsSplitCurrent or (in_cnt < project_cnt - in_cnt):
           in_type = 'in'
           for b in i.projects:
+            if not i.IsSplitCurrent or b.current:
               paths.append(b.project.relpath)
+            else:
+              non_cur_paths.append(b.project.relpath)
         else:
           fmt = out.notinproject
           in_type = 'not in'
@@ -154,13 +162,19 @@ is shown, then the branch appears in all projects.
               paths.append(p.relpath)
 
         s = ' %s %s' % (in_type, ', '.join(paths))
-        if width + 7 + len(s) < 80:
+        if not i.IsSplitCurrent and (width + 7 + len(s) < 80):
+          fmt = out.current if i.IsCurrent else fmt
           fmt(s)
         else:
           fmt(' %s:' % in_type)
+          fmt = out.current if i.IsCurrent else out.write
           for p in paths:
             out.nl()
             fmt(width*' ' + '          %s' % p)
+          fmt = out.write
+          for p in non_cur_paths:
+            out.nl()
+            fmt(width*' ' + '          %s' % p)
       else:
         out.write(' in all projects')
       out.nl()

subcmds/diffmanifests.py

@@ -0,0 +1,195 @@
+#
+# Copyright (C) 2014 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from color import Coloring
+from command import PagedCommand
+from manifest_xml import XmlManifest
+
+class _Coloring(Coloring):
+  def __init__(self, config):
+    Coloring.__init__(self, config, "status")
+
+class Diffmanifests(PagedCommand):
+  """ A command to see logs in projects represented by manifests
+
+  This is used to see deeper differences between manifests. Where a simple
+  diff would only show a diff of sha1s for example, this command will display
+  the logs of the project between both sha1s, allowing user to see diff at a
+  deeper level.
+  """
+
+  common = True
+  helpSummary = "Manifest diff utility"
+  helpUsage = """%prog manifest1.xml [manifest2.xml] [options]"""
+
+  helpDescription = """
+The %prog command shows differences between project revisions of manifest1 and
+manifest2. if manifest2 is not specified, current manifest.xml will be used
+instead. Both absolute and relative paths may be used for manifests. Relative
+paths start from project's ".repo/manifests" folder.
+
+The --raw option Displays the diff in a way that facilitates parsing, the
+project pattern will be <status> <path> <revision from> [<revision to>] and the
+commit pattern will be <status> <onelined log> with status values respectively :
+
+  A = Added project
+  R = Removed project
+  C = Changed project
+  U = Project with unreachable revision(s) (revision(s) not found)
+
+for project, and
+
+   A = Added commit
+   R = Removed commit
+
+for a commit.
+
+Only changed projects may contain commits, and commit status always starts with
+a space, and are part of last printed project.
+Unreachable revisions may occur if project is not up to date or if repo has not
+been initialized with all the groups, in which case some projects won't be
+synced and their revisions won't be found.
+
+"""
+
+  def _Options(self, p):
+    p.add_option('--raw',
+                 dest='raw', action='store_true',
+                 help='Display raw diff.')
+    p.add_option('--no-color',
+                 dest='color', action='store_false', default=True,
+                 help='does not display the diff in color.')
+
+  def _printRawDiff(self, diff):
+    for project in diff['added']:
+      self.printText("A %s %s" % (project.relpath, project.revisionExpr))
+      self.out.nl()
+
+    for project in diff['removed']:
+      self.printText("R %s %s" % (project.relpath, project.revisionExpr))
+      self.out.nl()
+
+    for project, otherProject in diff['changed']:
+      self.printText("C %s %s %s" % (project.relpath, project.revisionExpr,
+                                     otherProject.revisionExpr))
+      self.out.nl()
+      self._printLogs(project, otherProject, raw=True, color=False)
+
+    for project, otherProject in diff['unreachable']:
+      self.printText("U %s %s %s" % (project.relpath, project.revisionExpr,
+                                     otherProject.revisionExpr))
+      self.out.nl()
+
+  def _printDiff(self, diff, color=True):
+    if diff['added']:
+      self.out.nl()
+      self.printText('added projects : \n')
+      self.out.nl()
+      for project in diff['added']:
+        self.printProject('\t%s' % (project.relpath))
+        self.printText(' at revision ')
+        self.printRevision(project.revisionExpr)
+        self.out.nl()
+
+    if diff['removed']:
+      self.out.nl()
+      self.printText('removed projects : \n')
+      self.out.nl()
+      for project in diff['removed']:
+        self.printProject('\t%s' % (project.relpath))
+        self.printText(' at revision ')
+        self.printRevision(project.revisionExpr)
+        self.out.nl()
+
+    if diff['changed']:
+      self.out.nl()
+      self.printText('changed projects : \n')
+      self.out.nl()
+      for project, otherProject in diff['changed']:
+        self.printProject('\t%s' % (project.relpath))
+        self.printText(' changed from ')
+        self.printRevision(project.revisionExpr)
+        self.printText(' to ')
+        self.printRevision(otherProject.revisionExpr)
+        self.out.nl()
+        self._printLogs(project, otherProject, raw=False, color=color)
+        self.out.nl()
+
+    if diff['unreachable']:
+      self.out.nl()
+      self.printText('projects with unreachable revisions : \n')
+      self.out.nl()
+      for project, otherProject in diff['unreachable']:
+        self.printProject('\t%s ' % (project.relpath))
+        self.printRevision(project.revisionExpr)
+        self.printText(' or ')
+        self.printRevision(otherProject.revisionExpr)
+        self.printText(' not found')
+        self.out.nl()
+
+  def _printLogs(self, project, otherProject, raw=False, color=True):
+    logs = project.getAddedAndRemovedLogs(otherProject, oneline=True,
+                                          color=color)
+    if logs['removed']:
+      removedLogs = logs['removed'].split('\n')
+      for log in removedLogs:
+        if log.strip():
+          if raw:
+            self.printText(' R ' + log)
+            self.out.nl()
+          else:
+            self.printRemoved('\t\t[-] ')
+            self.printText(log)
+            self.out.nl()
+
+    if logs['added']:
+      addedLogs = logs['added'].split('\n')
+      for log in addedLogs:
+        if log.strip():
+          if raw:
+            self.printText(' A ' + log)
+            self.out.nl()
+          else:
+            self.printAdded('\t\t[+] ')
+            self.printText(log)
+            self.out.nl()
+
+  def Execute(self, opt, args):
+    if not args or len(args) > 2:
+      self.Usage()
+
+    self.out = _Coloring(self.manifest.globalConfig)
+    self.printText = self.out.nofmt_printer('text')
+    if opt.color:
+      self.printProject = self.out.nofmt_printer('project', attr = 'bold')
+      self.printAdded = self.out.nofmt_printer('green', fg = 'green', attr = 'bold')
+      self.printRemoved = self.out.nofmt_printer('red', fg = 'red', attr = 'bold')
+      self.printRevision = self.out.nofmt_printer('revision', fg = 'yellow')
+    else:
+      self.printProject = self.printAdded = self.printRemoved = self.printRevision = self.printText
+
+    manifest1 = XmlManifest(self.manifest.repodir)
+    manifest1.Override(args[0])
+    if len(args) == 1:
+      manifest2 = self.manifest
+    else:
+      manifest2 = XmlManifest(self.manifest.repodir)
+      manifest2.Override(args[1])
+
+    diff = manifest1.projectsDiff(manifest2)
+    if opt.raw:
+      self._printRawDiff(diff)
+    else:
+      self._printDiff(diff, color=opt.color)

subcmds/download.py

@@ -18,6 +18,7 @@ import re
 import sys
 
 from command import Command
+from error import GitError
 
 CHANGE_RE = re.compile(r'^([1-9][0-9]*)(?:[/\.-]([1-9][0-9]*))?$')
 
@@ -87,7 +88,13 @@ makes it available in your project's local working directory.
         for c in dl.commits:
           print('  %s' % (c), file=sys.stderr)
       if opt.cherrypick:
+        try:
           project._CherryPick(dl.commit)
+        except GitError:
+          print('[%s] Could not complete the cherry-pick of %s' \
+                % (project.name, dl.commit), file=sys.stderr)
+          sys.exit(1)
+
       elif opt.revert:
         project._Revert(dl.commit)
       elif opt.ffonly:

subcmds/forall.py

@@ -14,7 +14,9 @@
 # limitations under the License.
 
 from __future__ import print_function
+import errno
 import fcntl
+import multiprocessing
 import re
 import os
 import select
@@ -31,6 +33,7 @@ _CAN_COLOR = [
   'log',
 ]
 
+
 class ForallColoring(Coloring):
   def __init__(self, config):
     Coloring.__init__(self, config, 'forall')
@@ -87,6 +90,12 @@ revision to a locally executed git command, use REPO_LREV.
 REPO_RREV is the name of the revision from the manifest, exactly
 as written in the manifest.
 
+REPO_COUNT is the total number of projects being iterated.
+
+REPO_I is the current (1-based) iteration count. Can be used in
+conjunction with REPO_COUNT to add a simple progress indicator to your
+command.
+
 REPO__* are any extra environment variables, specified by the
 "annotation" element under any project element.  This can be useful
 for differentiating trees based on user-specific criteria, or simply
@@ -126,9 +135,31 @@ without iterating through the remaining projects.
     g.add_option('-v', '--verbose',
                  dest='verbose', action='store_true',
                  help='Show command error messages')
+    g.add_option('-j', '--jobs',
+                 dest='jobs', action='store', type='int', default=1,
+                 help='number of commands to execute simultaneously')
 
   def WantPager(self, opt):
-    return opt.project_header
+    return opt.project_header and opt.jobs == 1
+
+  def _SerializeProject(self, project):
+    """ Serialize a project._GitGetByExec instance.
+
+    project._GitGetByExec is not pickle-able. Instead of trying to pass it
+    around between processes, make a dict ourselves containing only the
+    attributes that we need.
+
+    """
+    return {
+      'name': project.name,
+      'relpath': project.relpath,
+      'remote_name': project.remote.name,
+      'lrev': project.GetRevisionId(),
+      'rrev': project.revisionExpr,
+      'annotations': dict((a.name, a.value) for a in project.annotations),
+      'gitdir': project.gitdir,
+      'worktree': project.worktree,
+    }
 
   def Execute(self, opt, args):
     if not opt.command:
@@ -167,43 +198,93 @@ without iterating through the remaining projects.
       # pylint: enable=W0631
 
     mirror = self.manifest.IsMirror
-    out = ForallColoring(self.manifest.manifestProject.config)
-    out.redirect(sys.stdout)
-
     rc = 0
-    first = True
 
     if not opt.regex:
       projects = self.GetProjects(args)
     else:
       projects = self.FindProjects(args)
 
-    for project in projects:
+    os.environ['REPO_COUNT'] = str(len(projects))
+
+    pool = multiprocessing.Pool(opt.jobs)
+    try:
+      config = self.manifest.manifestProject.config
+      results_it = pool.imap(
+         DoWorkWrapper,
+         [[mirror, opt, cmd, shell, cnt, config, self._SerializeProject(p)]
+          for cnt, p in enumerate(projects)]
+      )
+      pool.close()
+      for r in results_it:
+        rc = rc or r
+        if r != 0 and opt.abort_on_errors:
+          raise Exception('Aborting due to previous error')
+    except (KeyboardInterrupt, WorkerKeyboardInterrupt):
+      # Catch KeyboardInterrupt raised inside and outside of workers
+      print('Interrupted - terminating the pool')
+      pool.terminate()
+      rc = rc or errno.EINTR
+    except Exception as e:
+      # Catch any other exceptions raised
+      print('Got an error, terminating the pool: %r' % e,
+            file=sys.stderr)
+      pool.terminate()
+      rc = rc or getattr(e, 'errno', 1)
+    finally:
+      pool.join()
+    if rc != 0:
+      sys.exit(rc)
+
+
+class WorkerKeyboardInterrupt(Exception):
+  """ Keyboard interrupt exception for worker processes. """
+  pass
+
+
+def DoWorkWrapper(args):
+  """ A wrapper around the DoWork() method.
+
+  Catch the KeyboardInterrupt exceptions here and re-raise them as a different,
+  ``Exception``-based exception to stop it flooding the console with stacktraces
+  and making the parent hang indefinitely.
+
+  """
+  project = args.pop()
+  try:
+    return DoWork(project, *args)
+  except KeyboardInterrupt:
+    print('%s: Worker interrupted' % project['name'])
+    raise WorkerKeyboardInterrupt()
+
+
+def DoWork(project, mirror, opt, cmd, shell, cnt, config):
   env = os.environ.copy()
   def setenv(name, val):
     if val is None:
       val = ''
     env[name] = val.encode()
 
-      setenv('REPO_PROJECT', project.name)
-      setenv('REPO_PATH', project.relpath)
-      setenv('REPO_REMOTE', project.remote.name)
-      setenv('REPO_LREV', project.GetRevisionId())
-      setenv('REPO_RREV', project.revisionExpr)
-      for a in project.annotations:
-        setenv("REPO__%s" % (a.name), a.value)
+  setenv('REPO_PROJECT', project['name'])
+  setenv('REPO_PATH', project['relpath'])
+  setenv('REPO_REMOTE', project['remote_name'])
+  setenv('REPO_LREV', project['lrev'])
+  setenv('REPO_RREV', project['rrev'])
+  setenv('REPO_I', str(cnt + 1))
+  for name in project['annotations']:
+    setenv("REPO__%s" % (name), project['annotations'][name])
 
   if mirror:
-        setenv('GIT_DIR', project.gitdir)
-        cwd = project.gitdir
+    setenv('GIT_DIR', project['gitdir'])
+    cwd = project['gitdir']
   else:
-        cwd = project.worktree
+    cwd = project['worktree']
 
   if not os.path.exists(cwd):
     if (opt.project_header and opt.verbose) \
     or not opt.project_header:
-          print('skipping %s/' % project.relpath, file=sys.stderr)
-        continue
+      print('skipping %s/' % project['relpath'], file=sys.stderr)
+    return
 
   if opt.project_header:
     stdin = subprocess.PIPE
@@ -223,6 +304,8 @@ without iterating through the remaining projects.
                        stderr=stderr)
 
   if opt.project_header:
+    out = ForallColoring(config)
+    out.redirect(sys.stdout)
     class sfd(object):
       def __init__(self, fd, dest):
         self.fd = fd
@@ -255,16 +338,14 @@ without iterating through the remaining projects.
             errbuf += buf
             continue
 
-            if empty:
-              if first:
-                first = False
-              else:
+        if empty and out:
+          if not cnt == 0:
             out.nl()
 
           if mirror:
-                project_header_path = project.name
+            project_header_path = project['name']
           else:
-                project_header_path = project.relpath
+            project_header_path = project['relpath']
           out.project('project %s/', project_header_path)
           out.nl()
           out.flush()
@@ -278,12 +359,4 @@ without iterating through the remaining projects.
         s.dest.flush()
 
   r = p.wait()
-      if r != 0:
-        if r != rc:
-          rc = r
-        if opt.abort_on_errors:
-          print("error: %s: Aborting due to previous error" % project.relpath,
-                file=sys.stderr)
-          sys.exit(r)
-    if rc != 0:
-      sys.exit(rc)
+  return r

subcmds/init.py

@@ -99,6 +99,10 @@ to update the working directory files.
     g.add_option('--depth', type='int', default=None,
                  dest='depth',
                  help='create a shallow clone with given depth; see git clone')
+    g.add_option('--archive',
+                 dest='archive', action='store_true',
+                 help='checkout an archive instead of a git repository for '
+                      'each project. See git archive.')
     g.add_option('-g', '--groups',
                  dest='groups', default='default',
                  help='restrict manifest projects to ones with specified '
@@ -198,6 +202,16 @@ to update the working directory files.
     if opt.reference:
       m.config.SetString('repo.reference', opt.reference)
 
+    if opt.archive:
+      if is_new:
+        m.config.SetString('repo.archive', 'true')
+      else:
+        print('fatal: --archive is only supported when initializing a new '
+              'workspace.', file=sys.stderr)
+        print('Either delete the .repo folder in this workspace, or initialize '
+              'in another location.', file=sys.stderr)
+        sys.exit(1)
+
     if opt.mirror:
       if is_new:
         m.config.SetString('repo.mirror', 'true')
@@ -219,7 +233,7 @@ to update the working directory files.
       sys.exit(1)
 
     if opt.manifest_branch:
-      m.MetaBranchSwitch(opt.manifest_branch)
+      m.MetaBranchSwitch()
 
     syncbuf = SyncBuffer(m.config)
     m.Sync_LocalHalf(syncbuf)
@@ -366,6 +380,13 @@ to update the working directory files.
     if opt.reference:
       opt.reference = os.path.expanduser(opt.reference)
 
+    # Check this here, else manifest will be tagged "not new" and init won't be
+    # possible anymore without removing the .repo/manifests directory.
+    if opt.archive and opt.mirror:
+      print('fatal: --mirror and --archive cannot be used together.',
+            file=sys.stderr)
+      sys.exit(1)
+
     self._SyncManifest(opt)
     self._LinkManifest(opt.manifest_name)
 

subcmds/rebase.py

@@ -62,6 +62,9 @@ branch but need to incorporate new upstream changes "underneath" them.
     if opt.interactive and not one_project:
       print('error: interactive rebase not supported with multiple projects',
             file=sys.stderr)
+      if len(args) == 1:
+        print('note: project %s is mapped to more than one path' % (args[0],),
+            file=sys.stderr)
       return -1
 
     for project in all_projects:

subcmds/start.py

@@ -59,8 +59,12 @@ revision specified in the manifest.
     for project in all_projects:
       pm.update()
       # If the current revision is a specific SHA1 then we can't push back
-      # to it so substitute the manifest default revision instead.
+      # to it; so substitute with dest_branch if defined, or with manifest
+      # default revision instead.
       if IsId(project.revisionExpr):
+        if project.dest_branch:
+          project.revisionExpr = project.dest_branch
+        else:
           project.revisionExpr = self.manifest.default.revisionExpr
       if not project.StartBranch(nb):
         err.append(project)

subcmds/status.py

@@ -113,7 +113,7 @@ the following meanings:
     try:
       state = project.PrintWorkTreeStatus(output)
       if state == 'CLEAN':
-        clean_counter.next()
+        next(clean_counter)
     finally:
       sem.release()
 
@@ -141,7 +141,7 @@ the following meanings:
       for project in all_projects:
         state = project.PrintWorkTreeStatus()
         if state == 'CLEAN':
-          counter.next()
+          next(counter)
     else:
       sem = _threading.Semaphore(opt.jobs)
       threads_and_output = []
@@ -164,7 +164,7 @@ the following meanings:
         t.join()
         output.dump(sys.stdout)
         output.close()
-    if len(all_projects) == counter.next():
+    if len(all_projects) == next(counter):
       print('nothing to commit (working directory clean)')
 
     if opt.orphans:

subcmds/sync.py

@@ -14,10 +14,10 @@
 # limitations under the License.
 
 from __future__ import print_function
+import json
 import netrc
 from optparse import SUPPRESS_HELP
 import os
-import pickle
 import re
 import shutil
 import socket
@@ -58,13 +58,13 @@ except ImportError:
 
 from git_command import GIT, git_require
 from git_refs import R_HEADS, HEAD
-from main import WrapperModule
 from project import Project
 from project import RemoteSpec
 from command import Command, MirrorSafeCommand
 from error import RepoChangedException, GitError, ManifestParseError
 from project import SyncBuffer
 from progress import Progress
+from wrapper import Wrapper
 
 _ONE_DAY_S = 24 * 60 * 60
 
@@ -128,6 +128,9 @@ HTTP client or proxy configuration, but the Git binary works.
 The --fetch-submodules option enables fetching Git submodules
 of a project from server.
 
+The -c/--current-branch option can be used to only fetch objects that
+are on the branch specified by a project's revision.
+
 SSH Connections
 ---------------
 
@@ -219,9 +222,25 @@ later is required to fix a server side protocol bug.
                  dest='repo_upgraded', action='store_true',
                  help=SUPPRESS_HELP)
 
-  def _FetchHelper(self, opt, project, lock, fetched, pm, sem, err_event):
+  def _FetchProjectList(self, opt, projects, *args, **kwargs):
     """Main function of the fetch threads when jobs are > 1.
 
+    Delegates most of the work to _FetchHelper.
+
+    Args:
+      opt: Program options returned from optparse.  See _Options().
+      projects: Projects to fetch.
+      *args, **kwargs: Remaining arguments to pass to _FetchHelper. See the
+          _FetchHelper docstring for details.
+    """
+    for project in projects:
+      success = self._FetchHelper(opt, project, *args, **kwargs)
+      if not success and not opt.force_broken:
+        break
+
+  def _FetchHelper(self, opt, project, lock, fetched, pm, sem, err_event):
+    """Fetch git objects for a single project.
+
     Args:
       opt: Program options returned from optparse.  See _Options().
       project: Project object for the project to fetch.
@@ -235,6 +254,9 @@ later is required to fix a server side protocol bug.
           can be started up.
       err_event: We'll set this event in the case of an error (after printing
           out info about the error).
+
+    Returns:
+      Whether the fetch was successful.
     """
     # We'll set to true once we've locked the lock.
     did_lock = False
@@ -253,7 +275,7 @@ later is required to fix a server side protocol bug.
           quiet=opt.quiet,
           current_branch_only=opt.current_branch_only,
           clone_bundle=not opt.no_clone_bundle,
-          no_tags=opt.no_tags)
+          no_tags=opt.no_tags, archive=self.manifest.IsArchive)
         self._fetch_times.Set(project, time.time() - start)
 
         # Lock around all the rest of the code, since printing, updating a set
@@ -281,51 +303,43 @@ later is required to fix a server side protocol bug.
         lock.release()
       sem.release()
 
+    return success
+
   def _Fetch(self, projects, opt):
     fetched = set()
+    lock = _threading.Lock()
     pm = Progress('Fetching projects', len(projects))
 
-    if self.jobs == 1:
+    objdir_project_map = dict()
     for project in projects:
-        pm.update()
-        if not opt.quiet:
-          print('Fetching project %s' % project.name)
-        if project.Sync_NetworkHalf(
-            quiet=opt.quiet,
-            current_branch_only=opt.current_branch_only,
-            clone_bundle=not opt.no_clone_bundle,
-            no_tags=opt.no_tags):
-          fetched.add(project.gitdir)
-        else:
-          print('error: Cannot fetch %s' % project.name, file=sys.stderr)
-          if opt.force_broken:
-            print('warn: --force-broken, continuing to sync', file=sys.stderr)
-          else:
-            sys.exit(1)
-    else:
+      objdir_project_map.setdefault(project.objdir, []).append(project)
+
     threads = set()
-      lock = _threading.Lock()
     sem = _threading.Semaphore(self.jobs)
     err_event = _threading.Event()
-      for project in projects:
-        # Check for any errors before starting any new threads.
+    for project_list in objdir_project_map.values():
+      # Check for any errors before running any more tasks.
       # ...we'll let existing threads finish, though.
-        if err_event.isSet():
+      if err_event.isSet() and not opt.force_broken:
         break
 
       sem.acquire()
-        t = _threading.Thread(target = self._FetchHelper,
-                              args = (opt,
-                                      project,
-                                      lock,
-                                      fetched,
-                                      pm,
-                                      sem,
-                                      err_event))
+      kwargs = dict(opt=opt,
+                    projects=project_list,
+                    lock=lock,
+                    fetched=fetched,
+                    pm=pm,
+                    sem=sem,
+                    err_event=err_event)
+      if self.jobs > 1:
+        t = _threading.Thread(target = self._FetchProjectList,
+                              kwargs = kwargs)
         # Ensure that Ctrl-C will not freeze the repo process.
         t.daemon = True
         threads.add(t)
         t.start()
+      else:
+        self._FetchProjectList(**kwargs)
 
     for t in threads:
       t.join()
@@ -338,10 +352,16 @@ later is required to fix a server side protocol bug.
     pm.end()
     self._fetch_times.Save()
 
+    if not self.manifest.IsArchive:
       self._GCProjects(projects)
+
     return fetched
 
   def _GCProjects(self, projects):
+    gitdirs = {}
+    for project in projects:
+      gitdirs[project.gitdir] = project.bare_git
+
     has_dash_c = git_require((1, 7, 2))
     if multiprocessing and has_dash_c:
       cpu_count = multiprocessing.cpu_count()
@@ -350,8 +370,8 @@ later is required to fix a server side protocol bug.
     jobs = min(self.jobs, cpu_count)
 
     if jobs < 2:
-      for project in projects:
-        project.bare_git.gc('--auto')
+      for bare_git in gitdirs.values():
+        bare_git.gc('--auto')
       return
 
     config = {'pack.threads': cpu_count / jobs if cpu_count > jobs else 1}
@@ -360,10 +380,10 @@ later is required to fix a server side protocol bug.
     sem = _threading.Semaphore(jobs)
     err_event = _threading.Event()
 
-    def GC(project):
+    def GC(bare_git):
       try:
         try:
-          project.bare_git.gc('--auto', config=config)
+          bare_git.gc('--auto', config=config)
         except GitError:
           err_event.set()
         except:
@@ -372,11 +392,11 @@ later is required to fix a server side protocol bug.
       finally:
         sem.release()
 
-    for project in projects:
+    for bare_git in gitdirs.values():
       if err_event.isSet():
         break
       sem.acquire()
-      t = _threading.Thread(target=GC, args=(project,))
+      t = _threading.Thread(target=GC, args=(bare_git,))
       t.daemon = True
       threads.add(t)
       t.start()
@@ -416,12 +436,13 @@ later is required to fix a server side protocol bug.
         if path not in new_project_paths:
           # If the path has already been deleted, we don't need to do it
           if os.path.exists(self.manifest.topdir + '/' + path):
+            gitdir = os.path.join(self.manifest.topdir, path, '.git')
             project = Project(
                            manifest = self.manifest,
                            name = path,
                            remote = RemoteSpec('origin'),
-                           gitdir = os.path.join(self.manifest.topdir,
-                                                 path, '.git'),
+                           gitdir = gitdir,
+                           objdir = gitdir,
                            worktree = os.path.join(self.manifest.topdir, path),
                            relpath = path,
                            revisionExpr = 'HEAD',
@@ -540,7 +561,10 @@ later is required to fix a server side protocol bug.
             branch = branch[len(R_HEADS):]
 
           env = os.environ.copy()
-          if 'TARGET_PRODUCT' in env and 'TARGET_BUILD_VARIANT' in env:
+          if 'SYNC_TARGET' in env:
+            target = env['SYNC_TARGET']
+            [success, manifest_str] = server.GetApprovedManifest(branch, target)
+          elif 'TARGET_PRODUCT' in env and 'TARGET_BUILD_VARIANT' in env:
             target = '%s-%s' % (env['TARGET_PRODUCT'],
                                 env['TARGET_BUILD_VARIANT'])
             [success, manifest_str] = server.GetApprovedManifest(branch, target)
@@ -641,7 +665,7 @@ later is required to fix a server side protocol bug.
         previously_missing_set = missing_set
         fetched.update(self._Fetch(missing, opt))
 
-    if self.manifest.IsMirror:
+    if self.manifest.IsMirror or self.manifest.IsArchive:
       # bail out now, we have no working tree
       return
 
@@ -666,10 +690,10 @@ later is required to fix a server side protocol bug.
       print(self.manifest.notice)
 
 def _PostRepoUpgrade(manifest, quiet=False):
-  wrapper = WrapperModule()
+  wrapper = Wrapper()
   if wrapper.NeedSetupGnuPG():
     wrapper.SetupGnuPG(quiet)
-  for project in manifest.projects.values():
+  for project in manifest.projects:
     if project.Exists:
       project.PostRepoUpgrade()
 
@@ -742,7 +766,7 @@ class _FetchTimes(object):
   _ALPHA = 0.5
 
   def __init__(self, manifest):
-    self._path = os.path.join(manifest.repodir, '.repopickle_fetchtimes')
+    self._path = os.path.join(manifest.repodir, '.repo_fetchtimes.json')
     self._times = None
     self._seen = set()
 
@@ -762,21 +786,16 @@ class _FetchTimes(object):
     if self._times is None:
       try:
         f = open(self._path)
-      except IOError:
-        self._times = {}
-        return self._times
         try:
-        try:
-          self._times = pickle.load(f)
-        except IOError:
+          self._times = json.load(f)
+        finally:
+          f.close()
+      except (IOError, ValueError):
         try:
           os.remove(self._path)
         except OSError:
           pass
         self._times = {}
-      finally:
-        f.close()
-    return self._times
 
   def Save(self):
     if self._times is None:
@@ -790,13 +809,13 @@ class _FetchTimes(object):
       del self._times[name]
 
     try:
-      f = open(self._path, 'wb')
+      f = open(self._path, 'w')
       try:
-        pickle.dump(self._times, f)
-      except (IOError, OSError, pickle.PickleError):
+        json.dump(self._times, f, indent=2)
+      finally:
+        f.close()
+    except (IOError, TypeError):
       try:
         os.remove(self._path)
       except OSError:
         pass
-    finally:
-      f.close()

subcmds/upload.py

@@ -21,12 +21,15 @@ import sys
 from command import InteractiveCommand
 from editor import Editor
 from error import HookError, UploadError
+from git_command import GitCommand
 from project import RepoHook
 
 from pyversion import is_python3
-if not is_python3():
 # pylint:disable=W0622
+if not is_python3():
   input = raw_input
+else:
+  unicode = str
 # pylint:enable=W0622
 
 UNUSUAL_COMMIT_THRESHOLD = 5
@@ -88,6 +91,11 @@ to "true" then repo will assume you always answer "y" at the prompt,
 and will not prompt you further.  If it is set to "false" then repo
 will assume you always answer "n", and will abort.
 
+review.URL.autoreviewer:
+
+To automatically append a user or mailing list to reviews, you can set
+a per-project or global Git option to do so.
+
 review.URL.autocopy:
 
 To automatically copy a user or mailing list to all uploaded reviews,
@@ -292,14 +300,20 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
 
     self._UploadAndReport(opt, todo, people)
 
-  def _AppendAutoCcList(self, branch, people):
+  def _AppendAutoList(self, branch, people):
     """
+    Appends the list of reviewers in the git project's config.
     Appends the list of users in the CC list in the git project's config if a
     non-empty reviewer list was found.
     """
-
     name = branch.name
     project = branch.project
+
+    key = 'review.%s.autoreviewer' % project.GetBranch(name).remote.review
+    raw_list = project.config.GetString(key)
+    if not raw_list is None:
+      people[0].extend([entry.strip() for entry in raw_list.split(',')])
+
     key = 'review.%s.autocopy' % project.GetBranch(name).remote.review
     raw_list = project.config.GetString(key)
     if not raw_list is None and len(people[0]) > 0:
@@ -322,16 +336,20 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
     for branch in todo:
       try:
         people = copy.deepcopy(original_people)
-        self._AppendAutoCcList(branch, people)
+        self._AppendAutoList(branch, people)
 
         # Check if there are local changes that may have been forgotten
-        if branch.project.HasChanges():
+        changes = branch.project.UncommitedFiles()
+        if changes:
           key = 'review.%s.autoupload' % branch.project.remote.review
           answer = branch.project.config.GetBoolean(key)
 
           # if they want to auto upload, let's not ask because it could be automated
           if answer is None:
-            sys.stdout.write('Uncommitted changes in ' + branch.project.name + ' (did you forget to amend?). Continue uploading? (y/N) ')
+            sys.stdout.write('Uncommitted changes in ' + branch.project.name)
+            sys.stdout.write(' (did you forget to amend?):\n')
+            sys.stdout.write('\n'.join(changes) + '\n')
+            sys.stdout.write('Continue uploading? (y/N) ')
             a = sys.stdin.readline().strip().lower()
             if a not in ('y', 'yes', 't', 'true', 'on'):
               print("skipping upload", file=sys.stderr)
@@ -344,7 +362,21 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
           key = 'review.%s.uploadtopic' % branch.project.remote.review
           opt.auto_topic = branch.project.config.GetBoolean(key)
 
-        destination = opt.dest_branch or branch.project.dest_branch or branch.project.revisionExpr
+        destination = opt.dest_branch or branch.project.dest_branch
+
+        # Make sure our local branch is not setup to track a different remote branch
+        merge_branch = self._GetMergeBranch(branch.project)
+        if destination:
+          full_dest = 'refs/heads/%s' % destination
+          if not opt.dest_branch and merge_branch and merge_branch != full_dest:
+            print('merge branch %s does not match destination branch %s'
+                  % (merge_branch, full_dest))
+            print('skipping upload.')
+            print('Please use `--destination %s` if this is intentional'
+                  % destination)
+            branch.uploaded = False
+            continue
+
         branch.UploadForReview(people, auto_topic=opt.auto_topic, draft=opt.draft, dest_branch=destination)
         branch.uploaded = True
       except UploadError as e:
@@ -379,6 +411,21 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
     if have_errors:
       sys.exit(1)
 
+  def _GetMergeBranch(self, project):
+    p = GitCommand(project,
+                   ['rev-parse', '--abbrev-ref', 'HEAD'],
+                   capture_stdout = True,
+                   capture_stderr = True)
+    p.Wait()
+    local_branch = p.stdout.strip()
+    p = GitCommand(project,
+                   ['config', '--get', 'branch.%s.merge' % local_branch],
+                   capture_stdout = True,
+                   capture_stderr = True)
+    p.Wait()
+    merge_branch = p.stdout.strip()
+    return merge_branch
+
   def Execute(self, opt, args):
     project_list = self.GetProjects(args)
     pending = []
@@ -392,7 +439,16 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
     for project in project_list:
       if opt.current_branch:
         cbr = project.CurrentBranch
-        avail = [project.GetUploadableBranch(cbr)] if cbr else None
+        up_branch = project.GetUploadableBranch(cbr)
+        if up_branch:
+          avail = [up_branch]
+        else:
+          avail = None
+          print('ERROR: Current branch (%s) not uploadable. '
+                'You may be able to type '
+                '"git branch --set-upstream-to m/master" to fix '
+                'your branch.' % str(cbr),
+                file=sys.stderr)
       else:
         avail = project.GetUploadableBranches(branch)
       if avail:
@@ -402,8 +458,10 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
       hook = RepoHook('pre-upload', self.manifest.repo_hooks_project,
                       self.manifest.topdir, abort_if_user_denies=True)
       pending_proj_names = [project.name for (project, avail) in pending]
+      pending_worktrees = [project.worktree for (project, avail) in pending]
       try:
-        hook.Run(opt.allow_all_hooks, project_list=pending_proj_names)
+        hook.Run(opt.allow_all_hooks, project_list=pending_proj_names,
+                 worktree_list=pending_worktrees)
       except HookError as e:
         print("ERROR: %s" % str(e), file=sys.stderr)
         return

wrapper.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2014 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+import imp
+import os
+
+
+def WrapperPath():
+  return os.path.join(os.path.dirname(__file__), 'repo')
+
+_wrapper_module = None
+def Wrapper():
+  global _wrapper_module
+  if not _wrapper_module:
+    _wrapper_module = imp.load_source('wrapper', WrapperPath())
+  return _wrapper_module