Ensure HEAD is correct when skipping remote fetch

A recent optimization (2fb6466f79) skips
performing a remote fetch if we already know we have the sha1 we want.
However, that optimization skipped initialization steps that ensure HEAD
points to the correct sha1.  This change makes sure not to skip those
steps.

Here is an example of how to test this change:

"""""""""
url=<manifest url>
branch1=<branch name>
branch2=<branch name>
project=<project with revision set to different sha1 in each branch>

repo init -u $url -b $branch1 --mirror
repo sync $project
first=$(cd $project.git; git rev-parse HEAD)

repo init -b $branch2
repo sync $project
second=$(cd platform/build.git; git rev-parse HEAD)

if [[ $first == $second ]]
then
    echo 'problem!'
else
    echo 'no problem!'
fi
"""""""""

error.py

@@ -24,6 +24,13 @@ class ManifestInvalidRevisionError(Exception):
 class NoManifestException(Exception):
   """The required manifest does not exist.
   """
+  def __init__(self, path, reason):
+    super(NoManifestException, self).__init__()
+    self.path = path
+    self.reason = reason
+
+  def __str__(self):
+    return self.reason
 
 class EditorError(Exception):
   """Unspecified error from the user's text editor.

git_command.py

@@ -21,6 +21,7 @@ import tempfile
 from signal import SIGTERM
 from error import GitError
 from trace import REPO_TRACE, IsTrace, Trace
+from wrapper import Wrapper
 
 GIT = 'git'
 MIN_GIT_VERSION = (1, 5, 4)
@@ -84,15 +85,10 @@ class _GitCall(object):
 
   def version_tuple(self):
     global _git_version
-
     if _git_version is None:
       ver_str = git.version().decode('utf-8')
-      if ver_str.startswith('git version '):
-        _git_version = tuple(
-          map(int,
-            ver_str[len('git version '):].strip().split('-')[0].split('.')[0:3]
-          ))
-      else:
+      _git_version = Wrapper().ParseGitVersion(ver_str)
+      if _git_version is None:
         print('fatal: "%s" unsupported' % ver_str, file=sys.stderr)
         sys.exit(1)
     return _git_version

git_config.py

@@ -576,7 +576,7 @@ class Remote(object):
         return None
 
       u = self.review
-      if not u.startswith('http:') and not u.startswith('https:'):
+      if u.split(':')[0] not in ('http', 'https', 'sso'):
         u = 'http://%s' % u
       if u.endswith('/Gerrit'):
         u = u[:len(u) - len('/Gerrit')]
@@ -592,6 +592,9 @@ class Remote(object):
         host, port = os.environ['REPO_HOST_PORT_INFO'].split()
         self._review_url = self._SshReviewUrl(userEmail, host, port)
         REVIEW_CACHE[u] = self._review_url
+      elif u.startswith('sso:'):
+        self._review_url = u  # Assume it's right
+        REVIEW_CACHE[u] = self._review_url
       else:
         try:
           info_url = u + 'ssh_info'
@@ -601,7 +604,7 @@ class Remote(object):
             # of HTML response back, like maybe a login page.
             #
             # Assume HTTP if SSH is not enabled or ssh_info doesn't look right.
-            self._review_url = http_url + 'p/'
+            self._review_url = http_url
           else:
             host, port = info.split()
             self._review_url = self._SshReviewUrl(userEmail, host, port)

main.py

@@ -31,6 +31,11 @@ else:
   urllib = imp.new_module('urllib')
   urllib.request = urllib2
 
+try:
+  import kerberos
+except ImportError:
+  kerberos = None
+
 from trace import SetTrace
 from git_command import git, GitCommand
 from git_config import init_ssh, close_ssh
@@ -46,6 +51,7 @@ from error import NoSuchProjectError
 from error import RepoChangedException
 from manifest_xml import XmlManifest
 from pager import RunPager
+from wrapper import WrapperPath, Wrapper
 
 from subcmds import all_commands
 
@@ -123,8 +129,15 @@ class _Repo(object):
             file=sys.stderr)
       return 1
 
-    copts, cargs = cmd.OptionParser.parse_args(argv)
-    copts = cmd.ReadEnvironmentOptions(copts)
+    try:
+      copts, cargs = cmd.OptionParser.parse_args(argv)
+      copts = cmd.ReadEnvironmentOptions(copts)
+    except NoManifestException as e:
+      print('error: in `%s`: %s' % (' '.join([name] + argv), str(e)),
+        file=sys.stderr)
+      print('error: manifest missing or unreadable -- please run init',
+            file=sys.stderr)
+      return 1
 
     if not gopts.no_pager and not isinstance(cmd, InteractiveCommand):
       config = cmd.manifest.globalConfig
@@ -140,15 +153,13 @@ class _Repo(object):
     start = time.time()
     try:
       result = cmd.Execute(copts, cargs)
-    except DownloadError as e:
-      print('error: %s' % str(e), file=sys.stderr)
-      result = 1
-    except ManifestInvalidRevisionError as e:
-      print('error: %s' % str(e), file=sys.stderr)
-      result = 1
-    except NoManifestException as e:
-      print('error: manifest required for this command -- please run init',
-            file=sys.stderr)
+    except (DownloadError, ManifestInvalidRevisionError,
+        NoManifestException) as e:
+      print('error: in `%s`: %s' % (' '.join([name] + argv), str(e)),
+        file=sys.stderr)
+      if isinstance(e, NoManifestException):
+        print('error: manifest missing or unreadable -- please run init',
+              file=sys.stderr)
       result = 1
     except NoSuchProjectError as e:
       if e.name:
@@ -169,21 +180,10 @@ class _Repo(object):
 
     return result
 
+
 def _MyRepoPath():
   return os.path.dirname(__file__)
 
-def _MyWrapperPath():
-  return os.path.join(os.path.dirname(__file__), 'repo')
-
-_wrapper_module = None
-def WrapperModule():
-  global _wrapper_module
-  if not _wrapper_module:
-    _wrapper_module = imp.load_source('wrapper', _MyWrapperPath())
-  return _wrapper_module
-
-def _CurrentWrapperVersion():
-  return WrapperModule().VERSION
 
 def _CheckWrapperVersion(ver, repo_path):
   if not repo_path:
@@ -193,7 +193,7 @@ def _CheckWrapperVersion(ver, repo_path):
     print('no --wrapper-version argument', file=sys.stderr)
     sys.exit(1)
 
-  exp = _CurrentWrapperVersion()
+  exp = Wrapper().VERSION
   ver = tuple(map(int, ver.split('.')))
   if len(ver) == 1:
     ver = (0, ver[0])
@@ -205,7 +205,7 @@ def _CheckWrapperVersion(ver, repo_path):
 !!! You must upgrade before you can continue:   !!!
 
     cp %s %s
-""" % (exp_str, _MyWrapperPath(), repo_path), file=sys.stderr)
+""" % (exp_str, WrapperPath(), repo_path), file=sys.stderr)
     sys.exit(1)
 
   if exp > ver:
@@ -214,7 +214,7 @@ def _CheckWrapperVersion(ver, repo_path):
 ... You should upgrade soon:
 
     cp %s %s
-""" % (exp_str, _MyWrapperPath(), repo_path), file=sys.stderr)
+""" % (exp_str, WrapperPath(), repo_path), file=sys.stderr)
 
 def _CheckRepoDir(repo_dir):
   if not repo_dir:
@@ -342,6 +342,86 @@ class _DigestAuthHandler(urllib.request.HTTPDigestAuthHandler):
         self.retried = 0
       raise
 
+class _KerberosAuthHandler(urllib.request.BaseHandler):
+  def __init__(self):
+    self.retried = 0
+    self.context = None
+    self.handler_order = urllib.request.BaseHandler.handler_order - 50
+
+  def http_error_401(self, req, fp, code, msg, headers):
+    host = req.get_host()
+    retry = self.http_error_auth_reqed('www-authenticate', host, req, headers)
+    return retry
+
+  def http_error_auth_reqed(self, auth_header, host, req, headers):
+    try:
+      spn = "HTTP@%s" % host
+      authdata = self._negotiate_get_authdata(auth_header, headers)
+
+      if self.retried > 3:
+        raise urllib.request.HTTPError(req.get_full_url(), 401,
+          "Negotiate auth failed", headers, None)
+      else:
+        self.retried += 1
+
+      neghdr = self._negotiate_get_svctk(spn, authdata)
+      if neghdr is None:
+        return None
+
+      req.add_unredirected_header('Authorization', neghdr)
+      response = self.parent.open(req)
+
+      srvauth = self._negotiate_get_authdata(auth_header, response.info())
+      if self._validate_response(srvauth):
+        return response
+    except kerberos.GSSError:
+      return None
+    except:
+      self.reset_retry_count()
+      raise
+    finally:
+      self._clean_context()
+
+  def reset_retry_count(self):
+    self.retried = 0
+
+  def _negotiate_get_authdata(self, auth_header, headers):
+    authhdr = headers.get(auth_header, None)
+    if authhdr is not None:
+      for mech_tuple in authhdr.split(","):
+        mech, __, authdata = mech_tuple.strip().partition(" ")
+        if mech.lower() == "negotiate":
+          return authdata.strip()
+    return None
+
+  def _negotiate_get_svctk(self, spn, authdata):
+    if authdata is None:
+      return None
+
+    result, self.context = kerberos.authGSSClientInit(spn)
+    if result < kerberos.AUTH_GSS_COMPLETE:
+      return None
+
+    result = kerberos.authGSSClientStep(self.context, authdata)
+    if result < kerberos.AUTH_GSS_CONTINUE:
+      return None
+
+    response = kerberos.authGSSClientResponse(self.context)
+    return "Negotiate %s" % response
+
+  def _validate_response(self, authdata):
+    if authdata is None:
+      return None
+    result = kerberos.authGSSClientStep(self.context, authdata)
+    if result == kerberos.AUTH_GSS_COMPLETE:
+      return True
+    return None
+
+  def _clean_context(self):
+    if self.context is not None:
+      kerberos.authGSSClientClean(self.context)
+      self.context = None
+
 def init_http():
   handlers = [_UserAgentHandler()]
 
@@ -358,6 +438,8 @@ def init_http():
     pass
   handlers.append(_BasicAuthHandler(mgr))
   handlers.append(_DigestAuthHandler(mgr))
+  if kerberos:
+    handlers.append(_KerberosAuthHandler())
 
   if 'http_proxy' in os.environ:
     url = os.environ['http_proxy']

manifest_xml.py

@@ -32,7 +32,7 @@ else:
 from git_config import GitConfig
 from git_refs import R_HEADS, HEAD
 from project import RemoteSpec, Project, MetaProject
-from error import ManifestParseError
+from error import ManifestParseError, ManifestInvalidRevisionError
 
 MANIFEST_FILE_NAME = 'manifest.xml'
 LOCAL_MANIFEST_NAME = 'local_manifest.xml'
@@ -80,18 +80,20 @@ class _XmlRemote(object):
   def _resolveFetchUrl(self):
     url = self.fetchUrl.rstrip('/')
     manifestUrl = self.manifestUrl.rstrip('/')
-    p = manifestUrl.startswith('persistent-http')
-    if p:
-      manifestUrl = manifestUrl[len('persistent-'):]
-
-    # urljoin will get confused if there is no scheme in the base url
-    # ie, if manifestUrl is of the form <hostname:port>
+    # urljoin will gets confused over quite a few things.  The ones we care
+    # about here are:
+    # * no scheme in the base url, like <hostname:port>
+    # * persistent-https://
+    # We handle this by replacing these with obscure protocols
+    # and then replacing them with the original when we are done.
+    # gopher -> <none>
+    # wais -> persistent-https
     if manifestUrl.find(':') != manifestUrl.find('/') - 1:
       manifestUrl = 'gopher://' + manifestUrl
+    manifestUrl = re.sub(r'^persistent-https://', 'wais://', manifestUrl)
     url = urllib.parse.urljoin(manifestUrl, url)
     url = re.sub(r'^gopher://', '', url)
-    if p:
-      url = 'persistent-' + url
+    url = re.sub(r'^wais://', 'persistent-https://', url)
     return url
 
   def ToRemoteSpec(self, projectName):
@@ -259,6 +261,12 @@ class XmlManifest(object):
         ce.setAttribute('dest', c.dest)
         e.appendChild(ce)
 
+      for l in p.linkfiles:
+        le = doc.createElement('linkfile')
+        le.setAttribute('src', l.src)
+        le.setAttribute('dest', l.dest)
+        e.appendChild(le)
+
       default_groups = ['all', 'name:%s' % p.name, 'path:%s' % p.relpath]
       egroups = [g for g in p.groups if g not in default_groups]
       if egroups:
@@ -519,12 +527,15 @@ class XmlManifest(object):
         self._repo_hooks_project.enabled_repo_hooks = enabled_repo_hooks
       if node.nodeName == 'remove-project':
         name = self._reqatt(node, 'name')
-        try:
-          del self._projects[name]
-        except KeyError:
+
+        if name not in self._projects:
           raise ManifestParseError('remove-project element specifies non-existent '
                                    'project: %s' % name)
 
+        for p in self._projects[name]:
+          del self._paths[p.relpath]
+        del self._projects[name]
+
         # If the manifest removes the hooks project, treat it as if it deleted
         # the repo-hooks element too.
         if self._repo_hooks_project and (self._repo_hooks_project.name == name):
@@ -563,10 +574,11 @@ class XmlManifest(object):
                         gitdir = gitdir,
                         objdir = gitdir,
                         worktree = None,
-                        relpath = None,
+                        relpath = name or None,
                         revisionExpr = m.revisionExpr,
                         revisionId = None)
       self._projects[project.name] = [project]
+      self._paths[project.relpath] = project
 
   def _ParseRemote(self, node):
     """
@@ -759,6 +771,8 @@ class XmlManifest(object):
     for n in node.childNodes:
       if n.nodeName == 'copyfile':
         self._ParseCopyFile(project, n)
+      if n.nodeName == 'linkfile':
+        self._ParseLinkFile(project, n)
       if n.nodeName == 'annotation':
         self._ParseAnnotation(project, n)
       if n.nodeName == 'project':
@@ -808,6 +822,14 @@ class XmlManifest(object):
       # dest is relative to the top of the tree
       project.AddCopyFile(src, dest, os.path.join(self.topdir, dest))
 
+  def _ParseLinkFile(self, project, node):
+    src = self._reqatt(node, 'src')
+    dest = self._reqatt(node, 'dest')
+    if not self.IsMirror:
+      # src is project relative;
+      # dest is relative to the top of the tree
+      project.AddLinkFile(src, dest, os.path.join(self.topdir, dest))
+
   def _ParseAnnotation(self, project, node):
     name = self._reqatt(node, 'name')
     value = self._reqatt(node, 'value')
@@ -840,3 +862,40 @@ class XmlManifest(object):
       raise ManifestParseError("no %s in <%s> within %s" %
             (attname, node.nodeName, self.manifestFile))
     return v
+
+  def projectsDiff(self, manifest):
+    """return the projects differences between two manifests.
+
+    The diff will be from self to given manifest.
+
+    """
+    fromProjects = self.paths
+    toProjects = manifest.paths
+
+    fromKeys = fromProjects.keys()
+    fromKeys.sort()
+    toKeys = toProjects.keys()
+    toKeys.sort()
+
+    diff = {'added': [], 'removed': [], 'changed': [], 'unreachable': []}
+
+    for proj in fromKeys:
+      if not proj in toKeys:
+        diff['removed'].append(fromProjects[proj])
+      else:
+        fromProj = fromProjects[proj]
+        toProj = toProjects[proj]
+        try:
+          fromRevId = fromProj.GetCommitRevisionId()
+          toRevId = toProj.GetCommitRevisionId()
+        except ManifestInvalidRevisionError:
+          diff['unreachable'].append((fromProj, toProj))
+        else:
+          if fromRevId != toRevId:
+            diff['changed'].append((fromProj, toProj))
+        toKeys.remove(proj)
+
+    for proj in toKeys:
+      diff['added'].append(toProjects[proj])
+
+    return diff

project.py

@@ -231,6 +231,30 @@ class _CopyFile:
       except IOError:
         _error('Cannot copy file %s to %s', src, dest)
 
+class _LinkFile:
+  def __init__(self, src, dest, abssrc, absdest):
+    self.src = src
+    self.dest = dest
+    self.abs_src = abssrc
+    self.abs_dest = absdest
+
+  def _Link(self):
+    src = self.abs_src
+    dest = self.abs_dest
+    # link file if it does not exist or is out of date
+    if not os.path.islink(dest) or os.readlink(dest) != src:
+      try:
+        # remove existing file first, since it might be read-only
+        if os.path.exists(dest):
+          os.remove(dest)
+        else:
+          dest_dir = os.path.dirname(dest)
+          if not os.path.isdir(dest_dir):
+            os.makedirs(dest_dir)
+        os.symlink(src, dest)
+      except IOError:
+        _error('Cannot link file %s to %s', src, dest)
+
 class RemoteSpec(object):
   def __init__(self,
                name,
@@ -555,6 +579,7 @@ class Project(object):
 
     self.snapshots = {}
     self.copyfiles = []
+    self.linkfiles = []
     self.annotations = []
     self.config = GitConfig.ForRepository(
                     gitdir = self.gitdir,
@@ -1040,7 +1065,7 @@ class Project(object):
       except OSError as e:
         print("warn: Cannot remove archive %s: "
               "%s" % (tarpath, str(e)), file=sys.stderr)
-      self._CopyFiles()
+      self._CopyAndLinkFiles()
       return True
 
     if is_new is None:
@@ -1078,10 +1103,12 @@ class Project(object):
       elif self.manifest.default.sync_c:
         current_branch_only = True
 
-    if not self._RemoteFetch(initial=is_new, quiet=quiet, alt_dir=alt_dir,
-                             current_branch_only=current_branch_only,
-                             no_tags=no_tags):
-      return False
+    has_sha1 = ID_RE.match(self.revisionExpr) and self._CheckForSha1()
+    if (not has_sha1  #Need to fetch since we don't already have this revision
+        and not self._RemoteFetch(initial=is_new, quiet=quiet, alt_dir=alt_dir,
+                                  current_branch_only=current_branch_only,
+                                  no_tags=no_tags)):
+          return False
 
     if self.worktree:
       self._InitMRef()
@@ -1096,9 +1123,28 @@ class Project(object):
   def PostRepoUpgrade(self):
     self._InitHooks()
 
-  def _CopyFiles(self):
+  def _CopyAndLinkFiles(self):
     for copyfile in self.copyfiles:
       copyfile._Copy()
+    for linkfile in self.linkfiles:
+      linkfile._Link()
+
+  def GetCommitRevisionId(self):
+    """Get revisionId of a commit.
+
+    Use this method instead of GetRevisionId to get the id of the commit rather
+    than the id of the current git object (for example, a tag)
+
+    """
+    if not self.revisionExpr.startswith(R_TAGS):
+      return self.GetRevisionId(self._allrefs)
+
+    try:
+      return self.bare_git.rev_list(self.revisionExpr, '-1')[0]
+    except GitError:
+      raise ManifestInvalidRevisionError(
+        'revision %s in %s not found' % (self.revisionExpr,
+                                         self.name))
 
   def GetRevisionId(self, all_refs=None):
     if self.revisionId:
@@ -1128,7 +1174,7 @@ class Project(object):
 
     def _doff():
       self._FastForward(revid)
-      self._CopyFiles()
+      self._CopyAndLinkFiles()
 
     head = self.work_git.GetHead()
     if head.startswith(R_HEADS):
@@ -1164,7 +1210,7 @@ class Project(object):
       except GitError as e:
         syncbuf.fail(self, e)
         return
-      self._CopyFiles()
+      self._CopyAndLinkFiles()
       return
 
     if head == revid:
@@ -1186,7 +1232,7 @@ class Project(object):
       except GitError as e:
         syncbuf.fail(self, e)
         return
-      self._CopyFiles()
+      self._CopyAndLinkFiles()
       return
 
     upstream_gain = self._revlist(not_rev(HEAD), revid)
@@ -1259,12 +1305,12 @@ class Project(object):
     if cnt_mine > 0 and self.rebase:
       def _dorebase():
         self._Rebase(upstream = '%s^1' % last_mine, onto = revid)
-        self._CopyFiles()
+        self._CopyAndLinkFiles()
       syncbuf.later2(self, _dorebase)
     elif local_changes:
       try:
         self._ResetHard(revid)
-        self._CopyFiles()
+        self._CopyAndLinkFiles()
       except GitError as e:
         syncbuf.fail(self, e)
         return
@@ -1277,6 +1323,12 @@ class Project(object):
     abssrc = os.path.join(self.worktree, src)
     self.copyfiles.append(_CopyFile(src, dest, abssrc, absdest))
 
+  def AddLinkFile(self, src, dest, absdest):
+    # dest should already be an absolute path, but src is project relative
+    # make src an absolute path
+    abssrc = os.path.join(self.worktree, src)
+    self.linkfiles.append(_LinkFile(src, dest, abssrc, absdest))
+
   def AddAnnotation(self, name, value, keep):
     self.annotations.append(_Annotation(name, value, keep))
 
@@ -1627,6 +1679,15 @@ class Project(object):
 
 
 ## Direct Git Commands ##
+  def _CheckForSha1(self):
+    try:
+      # if revision (sha or tag) is not present then following function
+      # throws an error.
+      self.bare_git.rev_parse('--verify', '%s^0' % self.revisionExpr)
+      return True
+    except GitError:
+      # There is no such persistent revision. We have to fetch it.
+      return False
 
   def _FetchArchive(self, tarpath, cwd=None):
     cmd = ['archive', '-v', '-o', tarpath]
@@ -1650,21 +1711,17 @@ class Project(object):
 
     is_sha1 = False
     tag_name = None
+    depth = None
 
-    def CheckForSha1():
-      try:
-        # if revision (sha or tag) is not present then following function
-        # throws an error.
-        self.bare_git.rev_parse('--verify', '%s^0' % self.revisionExpr)
-        return True
-      except GitError:
-        # There is no such persistent revision. We have to fetch it.
-        return False
+    # The depth should not be used when fetching to a mirror because
+    # it will result in a shallow repository that cannot be cloned or
+    # fetched from.
+    if not self.manifest.IsMirror:
+      if self.clone_depth:
+        depth = self.clone_depth
+      else:
+        depth = self.manifest.manifestProject.config.GetString('repo.depth')
 
-    if self.clone_depth:
-      depth = self.clone_depth
-    else:
-      depth = self.manifest.manifestProject.config.GetString('repo.depth')
     if depth:
       current_branch_only = True
 
@@ -1676,7 +1733,7 @@ class Project(object):
         tag_name = self.revisionExpr[len(R_TAGS):]
 
       if is_sha1 or tag_name is not None:
-        if CheckForSha1():
+        if self._CheckForSha1():
           return True
       if is_sha1 and (not self.upstream or ID_RE.match(self.upstream)):
         current_branch_only = False
@@ -1740,14 +1797,15 @@ class Project(object):
       cmd.append('--update-head-ok')
     cmd.append(name)
 
+    # If using depth then we should not get all the tags since they may
+    # be outside of the depth.
+    if no_tags or depth:
+      cmd.append('--no-tags')
+    else:
+      cmd.append('--tags')
+
     if not current_branch_only:
       # Fetch whole repo
-      # If using depth then we should not get all the tags since they may
-      # be outside of the depth.
-      if no_tags or depth:
-        cmd.append('--no-tags')
-      else:
-        cmd.append('--tags')
       cmd.append(str((u'+refs/heads/*:') + remote.ToLocal('refs/heads/*')))
     elif tag_name is not None:
       cmd.append('tag')
@@ -1774,6 +1832,11 @@ class Project(object):
       time.sleep(random.randint(30, 45))
 
     if initial:
+      # Ensure that some refs exist.  Otherwise, we probably aren't looking
+      # at a real git repository and may have a bad url.
+      if not self.bare_ref.all:
+        ok = False
+
       if alt_dir:
         if old_packed != '':
           _lwrite(packed_refs, old_packed)
@@ -1785,7 +1848,7 @@ class Project(object):
       # We just synced the upstream given branch; verify we
       # got what we wanted, else trigger a second run of all
       # refs.
-      if not CheckForSha1():
+      if not self._CheckForSha1():
         return self._RemoteFetch(name=name, current_branch_only=False,
                                  initial=False, quiet=quiet, alt_dir=alt_dir)
 
@@ -2118,7 +2181,7 @@ class Project(object):
     symlink_dirs = ['hooks', 'objects', 'rr-cache', 'svn']
     if share_refs:
       # These objects can only be used by a single working tree.
-      symlink_files += ['config', 'packed-refs']
+      symlink_files += ['config', 'packed-refs', 'shallow']
       symlink_dirs += ['logs', 'refs']
     to_symlink = symlink_files + symlink_dirs
 
@@ -2166,7 +2229,7 @@ class Project(object):
       if GitCommand(self, cmd).Wait() != 0:
         raise GitError("cannot initialize work tree")
 
-      self._CopyFiles()
+      self._CopyAndLinkFiles()
 
   def _gitdir_path(self, path):
     return os.path.realpath(os.path.join(self.gitdir, path))
@@ -2181,6 +2244,43 @@ class Project(object):
   def _allrefs(self):
     return self.bare_ref.all
 
+  def _getLogs(self, rev1, rev2, oneline=False, color=True):
+    """Get logs between two revisions of this project."""
+    comp = '..'
+    if rev1:
+      revs = [rev1]
+      if rev2:
+        revs.extend([comp, rev2])
+      cmd = ['log', ''.join(revs)]
+      out = DiffColoring(self.config)
+      if out.is_on and color:
+        cmd.append('--color')
+      if oneline:
+        cmd.append('--oneline')
+
+      try:
+        log = GitCommand(self, cmd, capture_stdout=True, capture_stderr=True)
+        if log.Wait() == 0:
+          return log.stdout
+      except GitError:
+        # worktree may not exist if groups changed for example. In that case,
+        # try in gitdir instead.
+        if not os.path.exists(self.worktree):
+          return self.bare_git.log(*cmd[1:])
+        else:
+          raise
+    return None
+
+  def getAddedAndRemovedLogs(self, toProject, oneline=False, color=True):
+    """Get the list of logs from this revision to given revisionId"""
+    logs = {}
+    selfId = self.GetRevisionId(self._allrefs)
+    toId = toProject.GetRevisionId(toProject._allrefs)
+
+    logs['added'] = self._getLogs(selfId, toId, oneline=oneline, color=color)
+    logs['removed'] = self._getLogs(toId, selfId, oneline=oneline, color=color)
+    return logs
+
   class _GitGetByExec(object):
     def __init__(self, project, bare, gitdir):
       self._project = project
@@ -2261,8 +2361,8 @@ class Project(object):
         path = os.path.join(self._project.worktree, '.git', HEAD)
       try:
         fd = open(path, 'rb')
-      except IOError:
-        raise NoManifestException(path)
+      except IOError as e:
+        raise NoManifestException(path, str(e))
       try:
         line = fd.read()
       finally:

repo

@@ -20,7 +20,7 @@ REPO_REV = 'stable'
 # limitations under the License.
 
 # increment this whenever we make important changes to this script
-VERSION = (1, 20)
+VERSION = (1, 21)
 
 # increment this if the MAINTAINER_KEYS block is modified
 KEYRING_VERSION = (1, 2)
@@ -114,6 +114,7 @@ import errno
 import optparse
 import os
 import re
+import shutil
 import stat
 import subprocess
 import sys
@@ -278,6 +279,20 @@ def _Init(args):
     raise
 
 
+def ParseGitVersion(ver_str):
+  if not ver_str.startswith('git version '):
+    return None
+
+  num_ver_str = ver_str[len('git version '):].strip().split('-')[0]
+  to_tuple = []
+  for num_str in num_ver_str.split('.')[:3]:
+    if num_str.isdigit():
+      to_tuple.append(int(num_str))
+    else:
+      to_tuple.append(0)
+  return tuple(to_tuple)
+
+
 def _CheckGitVersion():
   cmd = [GIT, '--version']
   try:
@@ -295,12 +310,11 @@ def _CheckGitVersion():
   proc.stdout.close()
   proc.wait()
 
-  if not ver_str.startswith('git version '):
+  ver_act = ParseGitVersion(ver_str)
+  if ver_act is None:
     _print('error: "%s" unsupported' % ver_str, file=sys.stderr)
     raise CloneFailure()
 
-  ver_str = ver_str[len('git version '):].strip()
-  ver_act = tuple(map(int, ver_str.split('.')[0:3]))
   if ver_act < MIN_GIT_VERSION:
     need = '.'.join(map(str, MIN_GIT_VERSION))
     _print('fatal: git %s or later required' % need, file=sys.stderr)
@@ -728,12 +742,7 @@ def main(orig_args):
       try:
         _Init(args)
       except CloneFailure:
-        for root, dirs, files in os.walk(repodir, topdown=False):
-          for name in files:
-            os.remove(os.path.join(root, name))
-          for name in dirs:
-            os.rmdir(os.path.join(root, name))
-        os.rmdir(repodir)
+        shutil.rmtree(repodir, ignore_errors=True)
         sys.exit(1)
       repo_main, rel_repo_dir = _FindRepo()
     else:

subcmds/diffmanifests.py

@@ -0,0 +1,195 @@
+#
+# Copyright (C) 2014 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from color import Coloring
+from command import PagedCommand
+from manifest_xml import XmlManifest
+
+class _Coloring(Coloring):
+  def __init__(self, config):
+    Coloring.__init__(self, config, "status")
+
+class Diffmanifests(PagedCommand):
+  """ A command to see logs in projects represented by manifests
+
+  This is used to see deeper differences between manifests. Where a simple
+  diff would only show a diff of sha1s for example, this command will display
+  the logs of the project between both sha1s, allowing user to see diff at a
+  deeper level.
+  """
+
+  common = True
+  helpSummary = "Manifest diff utility"
+  helpUsage = """%prog manifest1.xml [manifest2.xml] [options]"""
+
+  helpDescription = """
+The %prog command shows differences between project revisions of manifest1 and
+manifest2. if manifest2 is not specified, current manifest.xml will be used
+instead. Both absolute and relative paths may be used for manifests. Relative
+paths start from project's ".repo/manifests" folder.
+
+The --raw option Displays the diff in a way that facilitates parsing, the
+project pattern will be <status> <path> <revision from> [<revision to>] and the
+commit pattern will be <status> <onelined log> with status values respectively :
+
+  A = Added project
+  R = Removed project
+  C = Changed project
+  U = Project with unreachable revision(s) (revision(s) not found)
+
+for project, and
+
+   A = Added commit
+   R = Removed commit
+
+for a commit.
+
+Only changed projects may contain commits, and commit status always starts with
+a space, and are part of last printed project.
+Unreachable revisions may occur if project is not up to date or if repo has not
+been initialized with all the groups, in which case some projects won't be
+synced and their revisions won't be found.
+
+"""
+
+  def _Options(self, p):
+    p.add_option('--raw',
+                 dest='raw', action='store_true',
+                 help='Display raw diff.')
+    p.add_option('--no-color',
+                 dest='color', action='store_false', default=True,
+                 help='does not display the diff in color.')
+
+  def _printRawDiff(self, diff):
+    for project in diff['added']:
+      self.printText("A %s %s" % (project.relpath, project.revisionExpr))
+      self.out.nl()
+
+    for project in diff['removed']:
+      self.printText("R %s %s" % (project.relpath, project.revisionExpr))
+      self.out.nl()
+
+    for project, otherProject in diff['changed']:
+      self.printText("C %s %s %s" % (project.relpath, project.revisionExpr,
+                                     otherProject.revisionExpr))
+      self.out.nl()
+      self._printLogs(project, otherProject, raw=True, color=False)
+
+    for project, otherProject in diff['unreachable']:
+      self.printText("U %s %s %s" % (project.relpath, project.revisionExpr,
+                                     otherProject.revisionExpr))
+      self.out.nl()
+
+  def _printDiff(self, diff, color=True):
+    if diff['added']:
+      self.out.nl()
+      self.printText('added projects : \n')
+      self.out.nl()
+      for project in diff['added']:
+        self.printProject('\t%s' % (project.relpath))
+        self.printText(' at revision ')
+        self.printRevision(project.revisionExpr)
+        self.out.nl()
+
+    if diff['removed']:
+      self.out.nl()
+      self.printText('removed projects : \n')
+      self.out.nl()
+      for project in diff['removed']:
+        self.printProject('\t%s' % (project.relpath))
+        self.printText(' at revision ')
+        self.printRevision(project.revisionExpr)
+        self.out.nl()
+
+    if diff['changed']:
+      self.out.nl()
+      self.printText('changed projects : \n')
+      self.out.nl()
+      for project, otherProject in diff['changed']:
+        self.printProject('\t%s' % (project.relpath))
+        self.printText(' changed from ')
+        self.printRevision(project.revisionExpr)
+        self.printText(' to ')
+        self.printRevision(otherProject.revisionExpr)
+        self.out.nl()
+        self._printLogs(project, otherProject, raw=False, color=color)
+        self.out.nl()
+
+    if diff['unreachable']:
+      self.out.nl()
+      self.printText('projects with unreachable revisions : \n')
+      self.out.nl()
+      for project, otherProject in diff['unreachable']:
+        self.printProject('\t%s ' % (project.relpath))
+        self.printRevision(project.revisionExpr)
+        self.printText(' or ')
+        self.printRevision(otherProject.revisionExpr)
+        self.printText(' not found')
+        self.out.nl()
+
+  def _printLogs(self, project, otherProject, raw=False, color=True):
+    logs = project.getAddedAndRemovedLogs(otherProject, oneline=True,
+                                          color=color)
+    if logs['removed']:
+      removedLogs = logs['removed'].split('\n')
+      for log in removedLogs:
+        if log.strip():
+          if raw:
+            self.printText(' R ' + log)
+            self.out.nl()
+          else:
+            self.printRemoved('\t\t[-] ')
+            self.printText(log)
+            self.out.nl()
+
+    if logs['added']:
+      addedLogs = logs['added'].split('\n')
+      for log in addedLogs:
+        if log.strip():
+          if raw:
+            self.printText(' A ' + log)
+            self.out.nl()
+          else:
+            self.printAdded('\t\t[+] ')
+            self.printText(log)
+            self.out.nl()
+
+  def Execute(self, opt, args):
+    if not args or len(args) > 2:
+      self.Usage()
+
+    self.out = _Coloring(self.manifest.globalConfig)
+    self.printText = self.out.nofmt_printer('text')
+    if opt.color:
+      self.printProject = self.out.nofmt_printer('project', attr = 'bold')
+      self.printAdded = self.out.nofmt_printer('green', fg = 'green', attr = 'bold')
+      self.printRemoved = self.out.nofmt_printer('red', fg = 'red', attr = 'bold')
+      self.printRevision = self.out.nofmt_printer('revision', fg = 'yellow')
+    else:
+      self.printProject = self.printAdded = self.printRemoved = self.printRevision = self.printText
+
+    manifest1 = XmlManifest(self.manifest.repodir)
+    manifest1.Override(args[0])
+    if len(args) == 1:
+      manifest2 = self.manifest
+    else:
+      manifest2 = XmlManifest(self.manifest.repodir)
+      manifest2.Override(args[1])
+
+    diff = manifest1.projectsDiff(manifest2)
+    if opt.raw:
+      self._printRawDiff(diff)
+    else:
+      self._printDiff(diff, color=opt.color)

subcmds/download.py

@@ -18,6 +18,7 @@ import re
 import sys
 
 from command import Command
+from error import GitError
 
 CHANGE_RE = re.compile(r'^([1-9][0-9]*)(?:[/\.-]([1-9][0-9]*))?$')
 
@@ -87,7 +88,12 @@ makes it available in your project's local working directory.
         for c in dl.commits:
           print('  %s' % (c), file=sys.stderr)
       if opt.cherrypick:
-        project._CherryPick(dl.commit)
+        try:
+          project._CherryPick(dl.commit)
+        except GitError:
+          print('[%s] Could not complete the cherry-pick of %s' \
+                % (project.name, dl.commit), file=sys.stderr)
+
       elif opt.revert:
         project._Revert(dl.commit)
       elif opt.ffonly:

subcmds/forall.py

@@ -87,6 +87,12 @@ revision to a locally executed git command, use REPO_LREV.
 REPO_RREV is the name of the revision from the manifest, exactly
 as written in the manifest.
 
+REPO_COUNT is the total number of projects being iterated.
+
+REPO_I is the current (1-based) iteration count. Can be used in
+conjunction with REPO_COUNT to add a simple progress indicator to your
+command.
+
 REPO__* are any extra environment variables, specified by the
 "annotation" element under any project element.  This can be useful
 for differentiating trees based on user-specific criteria, or simply
@@ -178,7 +184,9 @@ without iterating through the remaining projects.
     else:
       projects = self.FindProjects(args)
 
-    for project in projects:
+    os.environ['REPO_COUNT'] = str(len(projects))
+
+    for (cnt, project) in enumerate(projects):
       env = os.environ.copy()
       def setenv(name, val):
         if val is None:
@@ -190,6 +198,7 @@ without iterating through the remaining projects.
       setenv('REPO_REMOTE', project.remote.name)
       setenv('REPO_LREV', project.GetRevisionId())
       setenv('REPO_RREV', project.revisionExpr)
+      setenv('REPO_I', str(cnt + 1))
       for a in project.annotations:
         setenv("REPO__%s" % (a.name), a.value)
 

subcmds/sync.py

@@ -58,13 +58,13 @@ except ImportError:
 
 from git_command import GIT, git_require
 from git_refs import R_HEADS, HEAD
-from main import WrapperModule
 from project import Project
 from project import RemoteSpec
 from command import Command, MirrorSafeCommand
 from error import RepoChangedException, GitError, ManifestParseError
 from project import SyncBuffer
 from progress import Progress
+from wrapper import Wrapper
 
 _ONE_DAY_S = 24 * 60 * 60
 
@@ -219,7 +219,7 @@ later is required to fix a server side protocol bug.
                  dest='repo_upgraded', action='store_true',
                  help=SUPPRESS_HELP)
 
-  def _FetchProjectList(self, opt, projects, *args):
+  def _FetchProjectList(self, opt, projects, *args, **kwargs):
     """Main function of the fetch threads when jobs are > 1.
 
     Delegates most of the work to _FetchHelper.
@@ -227,11 +227,11 @@ later is required to fix a server side protocol bug.
     Args:
       opt: Program options returned from optparse.  See _Options().
       projects: Projects to fetch.
-      *args: Remaining arguments to pass to _FetchHelper. See the
+      *args, **kwargs: Remaining arguments to pass to _FetchHelper. See the
           _FetchHelper docstring for details.
     """
     for project in projects:
-      success = self._FetchHelper(opt, project, *args)
+      success = self._FetchHelper(opt, project, *args, **kwargs)
       if not success and not opt.force_broken:
         break
 
@@ -304,62 +304,47 @@ later is required to fix a server side protocol bug.
 
   def _Fetch(self, projects, opt):
     fetched = set()
+    lock = _threading.Lock()
     pm = Progress('Fetching projects', len(projects))
 
-    if self.jobs == 1:
-      for project in projects:
-        pm.update()
-        if not opt.quiet:
-          print('Fetching project %s' % project.name)
-        if project.Sync_NetworkHalf(
-            quiet=opt.quiet,
-            current_branch_only=opt.current_branch_only,
-            clone_bundle=not opt.no_clone_bundle,
-            no_tags=opt.no_tags,
-            archive=self.manifest.IsArchive):
-          fetched.add(project.gitdir)
-        else:
-          print('error: Cannot fetch %s' % project.name, file=sys.stderr)
-          if opt.force_broken:
-            print('warn: --force-broken, continuing to sync', file=sys.stderr)
-          else:
-            sys.exit(1)
-    else:
-      objdir_project_map = dict()
-      for project in projects:
-        objdir_project_map.setdefault(project.objdir, []).append(project)
+    objdir_project_map = dict()
+    for project in projects:
+      objdir_project_map.setdefault(project.objdir, []).append(project)
 
-      threads = set()
-      lock = _threading.Lock()
-      sem = _threading.Semaphore(self.jobs)
-      err_event = _threading.Event()
-      for project_list in objdir_project_map.values():
-        # Check for any errors before starting any new threads.
-        # ...we'll let existing threads finish, though.
-        if err_event.isSet():
-          break
+    threads = set()
+    sem = _threading.Semaphore(self.jobs)
+    err_event = _threading.Event()
+    for project_list in objdir_project_map.values():
+      # Check for any errors before running any more tasks.
+      # ...we'll let existing threads finish, though.
+      if err_event.isSet() and not opt.force_broken:
+        break
 
-        sem.acquire()
+      sem.acquire()
+      kwargs = dict(opt=opt,
+                    projects=project_list,
+                    lock=lock,
+                    fetched=fetched,
+                    pm=pm,
+                    sem=sem,
+                    err_event=err_event)
+      if self.jobs > 1:
         t = _threading.Thread(target = self._FetchProjectList,
-                              args = (opt,
-                                      project_list,
-                                      lock,
-                                      fetched,
-                                      pm,
-                                      sem,
-                                      err_event))
+                              kwargs = kwargs)
         # Ensure that Ctrl-C will not freeze the repo process.
         t.daemon = True
         threads.add(t)
         t.start()
+      else:
+        self._FetchProjectList(**kwargs)
 
-      for t in threads:
-        t.join()
+    for t in threads:
+      t.join()
 
-      # If we saw an error, exit with code 1 so that other scripts can check.
-      if err_event.isSet():
-        print('\nerror: Exited sync due to fetch errors', file=sys.stderr)
-        sys.exit(1)
+    # If we saw an error, exit with code 1 so that other scripts can check.
+    if err_event.isSet():
+      print('\nerror: Exited sync due to fetch errors', file=sys.stderr)
+      sys.exit(1)
 
     pm.end()
     self._fetch_times.Save()
@@ -699,7 +684,7 @@ later is required to fix a server side protocol bug.
       print(self.manifest.notice)
 
 def _PostRepoUpgrade(manifest, quiet=False):
-  wrapper = WrapperModule()
+  wrapper = Wrapper()
   if wrapper.NeedSetupGnuPG():
     wrapper.SetupGnuPG(quiet)
   for project in manifest.projects:

subcmds/upload.py

@@ -89,6 +89,11 @@ to "true" then repo will assume you always answer "y" at the prompt,
 and will not prompt you further.  If it is set to "false" then repo
 will assume you always answer "n", and will abort.
 
+review.URL.autoreviewer:
+
+To automatically append a user or mailing list to reviews, you can set
+a per-project or global Git option to do so.
+
 review.URL.autocopy:
 
 To automatically copy a user or mailing list to all uploaded reviews,
@@ -293,14 +298,20 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
 
     self._UploadAndReport(opt, todo, people)
 
-  def _AppendAutoCcList(self, branch, people):
+  def _AppendAutoList(self, branch, people):
     """
+    Appends the list of reviewers in the git project's config.
     Appends the list of users in the CC list in the git project's config if a
     non-empty reviewer list was found.
     """
-
     name = branch.name
     project = branch.project
+
+    key = 'review.%s.autoreviewer' % project.GetBranch(name).remote.review
+    raw_list = project.config.GetString(key)
+    if not raw_list is None:
+      people[0].extend([entry.strip() for entry in raw_list.split(',')])
+
     key = 'review.%s.autocopy' % project.GetBranch(name).remote.review
     raw_list = project.config.GetString(key)
     if not raw_list is None and len(people[0]) > 0:
@@ -323,7 +334,7 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
     for branch in todo:
       try:
         people = copy.deepcopy(original_people)
-        self._AppendAutoCcList(branch, people)
+        self._AppendAutoList(branch, people)
 
         # Check if there are local changes that may have been forgotten
         if branch.project.HasChanges():

wrapper.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2014 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+import imp
+import os
+
+
+def WrapperPath():
+  return os.path.join(os.path.dirname(__file__), 'repo')
+
+_wrapper_module = None
+def Wrapper():
+  global _wrapper_module
+  if not _wrapper_module:
+    _wrapper_module = imp.load_source('wrapper', WrapperPath())
+  return _wrapper_module