Fix initial sync broken by sync-c option

Change-Id: I308753da8944e6ce5c46e3bfee1bcd41d5b7e292

SUBMITTING_PATCHES

@@ -5,7 +5,7 @@ Short Version:
  - Make sure all code is under the Apache License, 2.0.
  - Publish your changes for review:
 
-   git push ssh://review.source.android.com:29418/tools/repo.git HEAD:refs/for/master
+   git push https://gerrit-review.googlesource.com/git-repo HEAD:refs/for/maint
 
 
 Long Version:
@@ -55,24 +55,23 @@ Do not email your patches to anyone.
 
 Instead, login to the Gerrit Code Review tool at:
 
-  https://review.source.android.com/
+  https://gerrit-review.googlesource.com/
 
 Ensure you have completed one of the necessary contributor
 agreements, providing documentation to the project maintainers that
 they have right to redistribute your work under the Apache License:
 
-  https://review.source.android.com/#settings,agreements
+  https://gerrit-review.googlesource.com/#/settings/agreements
 
-Ensure you have registered one or more SSH public keys, so you can
-push your commits directly over SSH:
+Ensure you have obtained an HTTP password to authenticate:
 
-  https://review.source.android.com/#settings,ssh-keys
+  https://gerrit-review.googlesource.com/new-password
 
-Push your patches over SSH to the review server, possibly through
+Push your patches over HTTPS to the review server, possibly through
 a remembered remote to make this easier in the future:
 
-   git config remote.review.url ssh://review.source.android.com:29418/tools/repo.git
-   git config remote.review.push HEAD:refs/for/master
+   git config remote.review.url https://gerrit-review.googlesource.com/git-repo
+   git config remote.review.push HEAD:refs/for/maint
 
    git push review
 

command.py

@@ -15,9 +15,12 @@
 
 import os
 import optparse
+import platform
+import re
 import sys
 
 from error import NoSuchProjectError
+from error import InvalidProjectGroupsError
 
 class Command(object):
   """Base class for any command line action in repo.
@@ -63,9 +66,17 @@ class Command(object):
     all = self.manifest.projects
     result = []
 
+    mp = self.manifest.manifestProject
+
+    groups = mp.config.GetString('manifest.groups')
+    if not groups:
+      groups = 'default,platform-' + platform.system().lower()
+    groups = [x for x in re.split('[,\s]+', groups) if x]
+
     if not args:
       for project in all.values():
-        if missing_ok or project.Exists:
+        if ((missing_ok or project.Exists) and
+            project.MatchesGroups(groups)):
           result.append(project)
     else:
       by_path = None
@@ -102,6 +113,8 @@ class Command(object):
           raise NoSuchProjectError(arg)
         if not missing_ok and not project.Exists:
           raise NoSuchProjectError(arg)
+        if not project.MatchesGroups(groups):
+          raise InvalidProjectGroupsError(arg)
 
         result.append(project)
 

docs/manifest-format.txt

@@ -39,15 +39,23 @@ following DTD:
     <!ATTLIST default remote   IDREF #IMPLIED>
     <!ATTLIST default revision CDATA #IMPLIED>
     <!ATTLIST default sync-j   CDATA #IMPLIED>
+    <!ATTLIST default sync-c   CDATA #IMPLIED>
 
     <!ELEMENT manifest-server (EMPTY)>
     <!ATTLIST url              CDATA #REQUIRED>
   
-    <!ELEMENT project (EMPTY)>
+    <!ELEMENT project (annotation?)>
     <!ATTLIST project name     CDATA #REQUIRED>
     <!ATTLIST project path     CDATA #IMPLIED>
     <!ATTLIST project remote   IDREF #IMPLIED>
     <!ATTLIST project revision CDATA #IMPLIED>
+    <!ATTLIST project groups   CDATA #IMPLIED>
+    <!ATTLIST project sync-c   CDATA #IMPLIED>
+
+    <!ELEMENT annotation (EMPTY)>
+    <!ATTLIST annotation name  CDATA #REQUIRED>
+    <!ATTLIST annotation value CDATA #REQUIRED>
+    <!ATTLIST annotation keep  CDATA "true">
   
     <!ELEMENT remove-project (EMPTY)>
     <!ATTLIST remove-project name  CDATA #REQUIRED>
@@ -55,6 +63,9 @@ following DTD:
     <!ELEMENT repo-hooks (EMPTY)>
     <!ATTLIST repo-hooks in-project CDATA #REQUIRED>
     <!ATTLIST repo-hooks enabled-list CDATA #REQUIRED>
+
+    <!ELEMENT include      (EMPTY)>
+    <!ATTLIST include name CDATA #REQUIRED>
   ]>
 
 A description of the elements and their attributes follows.
@@ -158,6 +169,21 @@ Tags and/or explicit SHA-1s should work in theory, but have not
 been extensively tested.  If not supplied the revision given by
 the default element is used.
 
+Attribute `groups`: List of groups to which this project belongs,
+whitespace or comma separated.  All projects belong to the group
+"default".
+
+Element annotation
+------------------
+
+Zero or more annotation elements may be specified as children of a
+project element. Each element describes a name-value pair that will be
+exported into each project's environment during a 'forall' command,
+prefixed with REPO__.  In addition, there is an optional attribute
+"keep" which accepts the case insensitive values "true" (default) or
+"false".  This attribute determines whether or not the annotation will
+be kept when exported with the manifest subcommand.
+
 Element remove-project
 ----------------------
 
@@ -169,6 +195,16 @@ This element is mostly useful in the local_manifest.xml, where
 the user can remove a project, and possibly replace it with their
 own definition.
 
+Element include
+---------------
+
+This element provides the capability of including another manifest
+file into the originating manifest.  Normal rules apply for the
+target manifest to include- it must be a usable manifest on it's own.
+
+Attribute `name`; the manifest to include, specified relative to
+the manifest repositories root.
+
 
 Local Manifest
 ==============

error.py

@@ -77,6 +77,18 @@ class NoSuchProjectError(Exception):
       return 'in current directory'
     return self.name
 
+
+class InvalidProjectGroupsError(Exception):
+  """A specified project is not suitable for the specified groups
+  """
+  def __init__(self, name=None):
+    self.name = name
+
+  def __str__(self):
+    if self.Name is None:
+      return 'in current directory'
+    return self.name
+
 class RepoChangedException(Exception):
   """Thrown if 'repo sync' results in repo updating its internal
      repo or manifest repositories.  In this special case we must

git_command.py

@@ -221,26 +221,10 @@ class GitCommand(object):
     self.stdin = p.stdin
 
   def Wait(self):
-    p = self.process
-
-    if p.stdin:
-      p.stdin.close()
-      self.stdin = None
-
-    if p.stdout:
-      self.stdout = p.stdout.read()
-      p.stdout.close()
-    else:
-      p.stdout = None
-
-    if p.stderr:
-      self.stderr = p.stderr.read()
-      p.stderr.close()
-    else:
-      p.stderr = None
-
     try:
-      rc = p.wait()
+      p = self.process
+      (self.stdout, self.stderr) = p.communicate()
+      rc = p.returncode
     finally:
       _remove_ssh_client(p)
     return rc

git_config.py

@@ -26,7 +26,6 @@ import time
 import urllib2
 
 from signal import SIGTERM
-from urllib2 import urlopen, HTTPError
 from error import GitError, UploadError
 from trace import Trace
 
@@ -489,7 +488,7 @@ def close_ssh():
   _master_keys_lock = None
 
 URI_SCP = re.compile(r'^([^@:]*@?[^:/]{1,}):')
-URI_ALL = re.compile(r'^([a-z][a-z+]*)://([^@/]*@?[^/]*)/')
+URI_ALL = re.compile(r'^([a-z][a-z+-]*)://([^@/]*@?[^/]*)/')
 
 def GetSchemeFromUrl(url):
   m = URI_ALL.match(url)
@@ -528,7 +527,7 @@ class Remote(object):
     self.projectname = self._Get('projectname')
     self.fetch = map(lambda x: RefSpec.FromString(x),
                      self._Get('fetch', all=True))
-    self._review_protocol = None
+    self._review_url = None
 
   def _InsteadOf(self):
     globCfg = GitConfig.ForUser()
@@ -555,9 +554,8 @@ class Remote(object):
     connectionUrl = self._InsteadOf()
     return _preconnect(connectionUrl)
 
-  @property
-  def ReviewProtocol(self):
-    if self._review_protocol is None:
+  def ReviewUrl(self, userEmail):
+    if self._review_url is None:
       if self.review is None:
         return None
 
@@ -566,57 +564,47 @@ class Remote(object):
         u = 'http://%s' % u
       if u.endswith('/Gerrit'):
         u = u[:len(u) - len('/Gerrit')]
-      if not u.endswith('/ssh_info'):
+      if u.endswith('/ssh_info'):
+        u = u[:len(u) - len('/ssh_info')]
       if not u.endswith('/'):
          u += '/'
-        u += 'ssh_info'
+      http_url = u
 
       if u in REVIEW_CACHE:
-        info = REVIEW_CACHE[u]
-        self._review_protocol = info[0]
-        self._review_host = info[1]
-        self._review_port = info[2]
+        self._review_url = REVIEW_CACHE[u]
+      elif 'REPO_HOST_PORT_INFO' in os.environ:
+        host, port = os.environ['REPO_HOST_PORT_INFO'].split()
+        self._review_url = self._SshReviewUrl(userEmail, host, port)
+        REVIEW_CACHE[u] = self._review_url
       else:
         try:
-          info = urlopen(u).read()
-          if info == 'NOT_AVAILABLE':
-            raise UploadError('%s: SSH disabled' % self.review)
+          info_url = u + 'ssh_info'
+          info = urllib2.urlopen(info_url).read()
           if '<' in info:
             # Assume the server gave us some sort of HTML
             # response back, like maybe a login page.
             #
-            raise UploadError('%s: Cannot parse response' % u)
+            raise UploadError('%s: Cannot parse response' % info_url)
 
-          self._review_protocol = 'ssh'
-          self._review_host = info.split(" ")[0]
-          self._review_port = info.split(" ")[1]
-        except urllib2.URLError, e:
-          raise UploadError('%s: %s' % (self.review, e.reason[1]))
-        except HTTPError, e:
-          if e.code == 404:
-            self._review_protocol = 'http-post'
-            self._review_host = None
-            self._review_port = None
+          if info == 'NOT_AVAILABLE':
+            # Assume HTTP if SSH is not enabled.
+            self._review_url = http_url + 'p/'
           else:
-            raise UploadError('Upload over ssh unavailable')
+            host, port = info.split()
+            self._review_url = self._SshReviewUrl(userEmail, host, port)
+        except urllib2.HTTPError, e:
+          raise UploadError('%s: %s' % (self.review, str(e)))
+        except urllib2.URLError, e:
+          raise UploadError('%s: %s' % (self.review, str(e)))
 
-        REVIEW_CACHE[u] = (
-          self._review_protocol,
-          self._review_host,
-          self._review_port)
-    return self._review_protocol
+        REVIEW_CACHE[u] = self._review_url
+    return self._review_url + self.projectname
 
-  def SshReviewUrl(self, userEmail):
-    if self.ReviewProtocol != 'ssh':
-      return None
+  def _SshReviewUrl(self, userEmail, host, port):
     username = self._config.GetString('review.%s.username' % self.review)
     if username is None:
-      username = userEmail.split("@")[0]
-    return 'ssh://%s@%s:%s/%s' % (
-      username,
-      self._review_host,
-      self._review_port,
-      self.projectname)
+      username = userEmail.split('@')[0]
+    return 'ssh://%s@%s:%s/' % (username, host, port)
 
   def ToLocal(self, rev):
     """Convert a remote revision string to something we have locally.

git_refs.py

@@ -139,13 +139,15 @@ class GitRefs(object):
   def _ReadLoose1(self, path, name):
     try:
       fd = open(path, 'rb')
-      mtime = os.path.getmtime(path)
-    except OSError:
-      return
-    except IOError:
+    except:
       return
+
     try:
+      try:
+        mtime = os.path.getmtime(path)
         id = fd.readline()
+      except:
+        return
     finally:
       fd.close()
 

main.py

@@ -36,6 +36,7 @@ from git_config import init_ssh, close_ssh
 from command import InteractiveCommand
 from command import MirrorSafeCommand
 from command import PagedCommand
+from subcmds.version import Version
 from editor import Editor
 from error import DownloadError
 from error import ManifestInvalidRevisionError
@@ -73,6 +74,7 @@ class _Repo(object):
     all_commands['branch'] = all_commands['branches']
 
   def _Run(self, argv):
+    result = 0
     name = None
     glob = []
 
@@ -96,7 +98,7 @@ class _Repo(object):
         name = 'version'
       else:
         print >>sys.stderr, 'fatal: invalid usage of --version'
-        sys.exit(1)
+        return 1
 
     try:
       cmd = self.commands[name]
@@ -104,7 +106,7 @@ class _Repo(object):
       print >>sys.stderr,\
             "repo: '%s' is not a repo command.  See 'repo help'."\
             % name
-      sys.exit(1)
+      return 1
 
     cmd.repodir = self.repodir
     cmd.manifest = XmlManifest(cmd.repodir)
@@ -114,7 +116,7 @@ class _Repo(object):
       print >>sys.stderr, \
             "fatal: '%s' requires a working directory"\
             % name
-      sys.exit(1)
+      return 1
 
     copts, cargs = cmd.OptionParser.parse_args(argv)
 
@@ -132,7 +134,7 @@ class _Repo(object):
     try:
       start = time.time()
       try:
-        cmd.Execute(copts, cargs)
+        result = cmd.Execute(copts, cargs)
       finally:
         elapsed = time.time() - start
         hours, remainder = divmod(elapsed, 3600)
@@ -146,16 +148,18 @@ class _Repo(object):
               % (hours, minutes, seconds)
     except DownloadError, e:
       print >>sys.stderr, 'error: %s' % str(e)
-      sys.exit(1)
+      return 1
     except ManifestInvalidRevisionError, e:
       print >>sys.stderr, 'error: %s' % str(e)
-      sys.exit(1)
+      return 1
     except NoSuchProjectError, e:
       if e.name:
         print >>sys.stderr, 'error: project %s not found' % e.name
       else:
         print >>sys.stderr, 'error: no project in current directory'
-      sys.exit(1)
+      return 1
+
+    return result
 
 def _MyRepoPath():
   return os.path.dirname(__file__)
@@ -273,6 +277,42 @@ class _UserAgentHandler(urllib2.BaseHandler):
     req.add_header('User-Agent', _UserAgent())
     return req
 
+class _BasicAuthHandler(urllib2.HTTPBasicAuthHandler):
+  def http_error_auth_reqed(self, authreq, host, req, headers):
+    try:
+      old_add_header = req.add_header
+      def _add_header(name, val):
+        val = val.replace('\n', '')
+        old_add_header(name, val)
+      req.add_header = _add_header
+      return urllib2.AbstractBasicAuthHandler.http_error_auth_reqed(
+        self, authreq, host, req, headers)
+    except:
+      reset = getattr(self, 'reset_retry_count', None)
+      if reset is not None:
+        reset()
+      elif getattr(self, 'retried', None):
+        self.retried = 0
+      raise
+
+class _DigestAuthHandler(urllib2.HTTPDigestAuthHandler):
+  def http_error_auth_reqed(self, auth_header, host, req, headers):
+    try:
+      old_add_header = req.add_header
+      def _add_header(name, val):
+        val = val.replace('\n', '')
+        old_add_header(name, val)
+      req.add_header = _add_header
+      return urllib2.AbstractDigestAuthHandler.http_error_auth_reqed(
+        self, auth_header, host, req, headers)
+    except:
+      reset = getattr(self, 'reset_retry_count', None)
+      if reset is not None:
+        reset()
+      elif getattr(self, 'retried', None):
+        self.retried = 0
+      raise
+
 def init_http():
   handlers = [_UserAgentHandler()]
 
@@ -281,13 +321,14 @@ def init_http():
     n = netrc.netrc()
     for host in n.hosts:
       p = n.hosts[host]
-      mgr.add_password(None, 'http://%s/'  % host, p[0], p[2])
-      mgr.add_password(None, 'https://%s/' % host, p[0], p[2])
+      mgr.add_password(p[1], 'http://%s/'  % host, p[0], p[2])
+      mgr.add_password(p[1], 'https://%s/' % host, p[0], p[2])
   except netrc.NetrcParseError:
     pass
   except IOError:
     pass
-  handlers.append(urllib2.HTTPBasicAuthHandler(mgr))
+  handlers.append(_BasicAuthHandler(mgr))
+  handlers.append(_DigestAuthHandler(mgr))
 
   if 'http_proxy' in os.environ:
     url = os.environ['http_proxy']
@@ -298,6 +339,8 @@ def init_http():
   urllib2.install_opener(urllib2.build_opener(*handlers))
 
 def _Main(argv):
+  result = 0
+
   opt = optparse.OptionParser(usage="repo wrapperinfo -- ...")
   opt.add_option("--repo-dir", dest="repodir",
                  help="path to .repo/")
@@ -311,16 +354,19 @@ def _Main(argv):
   _CheckWrapperVersion(opt.wrapper_version, opt.wrapper_path)
   _CheckRepoDir(opt.repodir)
 
+  Version.wrapper_version = opt.wrapper_version
+  Version.wrapper_path = opt.wrapper_path
+
   repo = _Repo(opt.repodir)
   try:
     try:
       init_ssh()
       init_http()
-      repo._Run(argv)
+      result = repo._Run(argv) or 0
     finally:
       close_ssh()
   except KeyboardInterrupt:
-    sys.exit(1)
+    result = 1
   except RepoChangedException, rce:
     # If repo changed, re-exec ourselves.
     #
@@ -331,7 +377,9 @@ def _Main(argv):
     except OSError, e:
       print >>sys.stderr, 'fatal: cannot restart repo after upgrade'
       print >>sys.stderr, 'fatal: %s' % e
-      sys.exit(128)
+      result = 128
+
+  sys.exit(result)
 
 if __name__ == '__main__':
   _Main(sys.argv[1:])

manifest_xml.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import itertools
 import os
 import re
 import sys
@@ -35,6 +36,7 @@ class _Default(object):
   revisionExpr = None
   remote = None
   sync_j = 1
+  sync_c = False
 
 class _XmlRemote(object):
   def __init__(self,
@@ -46,16 +48,20 @@ class _XmlRemote(object):
     self.fetchUrl = fetch
     self.manifestUrl = manifestUrl
     self.reviewUrl = review
+    self.resolvedFetchUrl = self._resolveFetchUrl()
 
-  def ToRemoteSpec(self, projectName):
-    url = self.fetchUrl.rstrip('/') + '/' + projectName + '.git'
+  def _resolveFetchUrl(self):
+    url = self.fetchUrl.rstrip('/')
     manifestUrl = self.manifestUrl.rstrip('/')
     # urljoin will get confused if there is no scheme in the base url
     # ie, if manifestUrl is of the form <hostname:port>
     if manifestUrl.find(':') != manifestUrl.find('/') - 1:
         manifestUrl = 'gopher://' + manifestUrl
     url = urlparse.urljoin(manifestUrl, url)
-    url = re.sub(r'^gopher://', '', url)
+    return re.sub(r'^gopher://', '', url)
+
+  def ToRemoteSpec(self, projectName):
+    url = self.resolvedFetchUrl.rstrip('/') + '/' + projectName
     return RemoteSpec(self.name, url, self.reviewUrl)
 
 class XmlManifest(object):
@@ -115,6 +121,13 @@ class XmlManifest(object):
   def Save(self, fd, peg_rev=False):
     """Write the current manifest out to the given file descriptor.
     """
+    mp = self.manifestProject
+
+    groups = mp.config.GetString('manifest.groups')
+    if not groups:
+      groups = 'default'
+    groups = [x for x in re.split(r'[,\s]+', groups) if x]
+
     doc = xml.dom.minidom.Document()
     root = doc.createElement('manifest')
     doc.appendChild(root)
@@ -148,6 +161,9 @@ class XmlManifest(object):
     if d.sync_j > 1:
       have_default = True
       e.setAttribute('sync-j', '%d' % d.sync_j)
+    if d.sync_c:
+      have_default = True
+      e.setAttribute('sync-c', 'true')
     if have_default:
       root.appendChild(e)
       root.appendChild(doc.createTextNode(''))
@@ -163,6 +179,10 @@ class XmlManifest(object):
 
     for p in sort_projects:
       p = self.projects[p]
+
+      if not p.MatchesGroups(groups):
+        continue
+
       e = doc.createElement('project')
       root.appendChild(e)
       e.setAttribute('name', p.name)
@@ -186,6 +206,20 @@ class XmlManifest(object):
         ce.setAttribute('dest', c.dest)
         e.appendChild(ce)
 
+      egroups = [g for g in p.groups if g != 'default']
+      if egroups:
+        e.setAttribute('groups', ','.join(egroups))
+
+      for a in p.annotations:
+        if a.keep == "true":
+          ae = doc.createElement('annotation')
+          ae.setAttribute('name', a.name)
+          ae.setAttribute('value', a.value)
+          e.appendChild(ae)
+
+      if p.sync_c:
+        e.setAttribute('sync-c', 'true')
+
     if self._repo_hooks_project:
       root.appendChild(doc.createTextNode(''))
       e = doc.createElement('repo-hooks')
@@ -248,16 +282,14 @@ class XmlManifest(object):
         b = b[len(R_HEADS):]
       self.branch = b
 
-      self._ParseManifest(True)
+      nodes = []
+      nodes.append(self._ParseManifestXml(self.manifestFile))
 
       local = os.path.join(self.repodir, LOCAL_MANIFEST_NAME)
       if os.path.exists(local):
-        try:
-          real = self.manifestFile
-          self.manifestFile = local
-          self._ParseManifest(False)
-        finally:
-          self.manifestFile = real
+        nodes.append(self._ParseManifestXml(local))
+
+      self._ParseManifest(nodes)
 
       if self.IsMirror:
         self._AddMetaProjectMirror(self.repoProject)
@@ -265,35 +297,39 @@ class XmlManifest(object):
 
       self._loaded = True
 
-  def _ParseManifest(self, is_root_file):
-    root = xml.dom.minidom.parse(self.manifestFile)
+  def _ParseManifestXml(self, path):
+    root = xml.dom.minidom.parse(path)
     if not root or not root.childNodes:
-      raise ManifestParseError(
-          "no root node in %s" %
-          self.manifestFile)
+      raise ManifestParseError("no root node in %s" % (path,))
 
     config = root.childNodes[0]
     if config.nodeName != 'manifest':
-      raise ManifestParseError(
-          "no <manifest> in %s" %
-          self.manifestFile)
+      raise ManifestParseError("no <manifest> in %s" % (path,))
 
+    nodes = []
     for node in config.childNodes:
-      if node.nodeName == 'remove-project':
+        if node.nodeName == 'include':
             name = self._reqatt(node, 'name')
+            fp = os.path.join(os.path.dirname(path), name)
+            if not os.path.isfile(fp):
+                raise ManifestParseError, \
+                    "include %s doesn't exist or isn't a file" % \
+                    (name,)
             try:
-          del self._projects[name]
-        except KeyError:
+                nodes.extend(self._ParseManifestXml(fp))
+            # should isolate this to the exact exception, but that's
+            # tricky.  actual parsing implementation may vary.
+            except (KeyboardInterrupt, RuntimeError, SystemExit):
+                raise
+            except Exception, e:
                 raise ManifestParseError(
-              'project %s not found' %
-              (name))
+                    "failed parsing included manifest %s: %s", (name, e))
+        else:
+          nodes.append(node)
+    return nodes
 
-        # If the manifest removes the hooks project, treat it as if it deleted
-        # the repo-hooks element too.
-        if self._repo_hooks_project and (self._repo_hooks_project.name == name):
-          self._repo_hooks_project = None
-
-    for node in config.childNodes:
+  def _ParseManifest(self, node_list):
+    for node in itertools.chain(*node_list):
       if node.nodeName == 'remote':
         remote = self._ParseRemote(node)
         if self._remotes.get(remote.name):
@@ -302,7 +338,7 @@ class XmlManifest(object):
               (remote.name, self.manifestFile))
         self._remotes[remote.name] = remote
 
-    for node in config.childNodes:
+    for node in itertools.chain(*node_list):
       if node.nodeName == 'default':
         if self._default is not None:
           raise ManifestParseError(
@@ -312,7 +348,7 @@ class XmlManifest(object):
     if self._default is None:
       self._default = _Default()
 
-    for node in config.childNodes:
+    for node in itertools.chain(*node_list):
       if node.nodeName == 'notice':
         if self._notice is not None:
           raise ManifestParseError(
@@ -320,7 +356,7 @@ class XmlManifest(object):
               (self.manifestFile))
         self._notice = self._ParseNotice(node)
 
-    for node in config.childNodes:
+    for node in itertools.chain(*node_list):
       if node.nodeName == 'manifest-server':
         url = self._reqatt(node, 'url')
         if self._manifest_server is not None:
@@ -329,7 +365,7 @@ class XmlManifest(object):
                 (self.manifestFile))
         self._manifest_server = url
 
-    for node in config.childNodes:
+    for node in itertools.chain(*node_list):
       if node.nodeName == 'project':
         project = self._ParseProject(node)
         if self._projects.get(project.name):
@@ -337,8 +373,6 @@ class XmlManifest(object):
               'duplicate project %s in %s' %
               (project.name, self.manifestFile))
         self._projects[project.name] = project
-
-    for node in config.childNodes:
       if node.nodeName == 'repo-hooks':
         # Get the name of the project and the (space-separated) list of enabled.
         repo_hooks_project = self._reqatt(node, 'in-project')
@@ -360,6 +394,20 @@ class XmlManifest(object):
 
         # Store the enabled hooks in the Project object.
         self._repo_hooks_project.enabled_repo_hooks = enabled_repo_hooks
+      if node.nodeName == 'remove-project':
+        name = self._reqatt(node, 'name')
+        try:
+          del self._projects[name]
+        except KeyError:
+          raise ManifestParseError(
+              'project %s not found' %
+              (name))
+
+        # If the manifest removes the hooks project, treat it as if it deleted
+        # the repo-hooks element too.
+        if self._repo_hooks_project and (self._repo_hooks_project.name == name):
+          self._repo_hooks_project = None
+
 
   def _AddMetaProjectMirror(self, m):
     name = None
@@ -368,7 +416,7 @@ class XmlManifest(object):
       raise ManifestParseError, 'refusing to mirror %s' % m_url
 
     if self._default and self._default.remote:
-      url = self._default.remote.fetchUrl
+      url = self._default.remote.resolvedFetchUrl
       if not url.endswith('/'):
         url += '/'
       if m_url.startswith(url):
@@ -418,11 +466,18 @@ class XmlManifest(object):
     d.revisionExpr = node.getAttribute('revision')
     if d.revisionExpr == '':
       d.revisionExpr = None
+
     sync_j = node.getAttribute('sync-j')
     if sync_j == '' or sync_j is None:
       d.sync_j = 1
     else:
       d.sync_j = int(sync_j)
+
+    sync_c = node.getAttribute('sync-c')
+    if not sync_c:
+      d.sync_c = False
+    else:
+      d.sync_c = sync_c.lower() in ("yes", "true", "1")
     return d
 
   def _ParseNotice(self, node):
@@ -494,6 +549,25 @@ class XmlManifest(object):
             "project %s path cannot be absolute in %s" % \
             (name, self.manifestFile)
 
+    rebase = node.getAttribute('rebase')
+    if not rebase:
+      rebase = True
+    else:
+      rebase = rebase.lower() in ("yes", "true", "1")
+
+    sync_c = node.getAttribute('sync-c')
+    if not sync_c:
+      sync_c = False
+    else:
+      sync_c = sync_c.lower() in ("yes", "true", "1")
+
+    groups = ''
+    if node.hasAttribute('groups'):
+      groups = node.getAttribute('groups')
+    groups = [x for x in re.split('[,\s]+', groups) if x]
+    if 'default' not in groups:
+      groups.append('default')
+
     if self.IsMirror:
       relpath = None
       worktree = None
@@ -509,11 +583,16 @@ class XmlManifest(object):
                       worktree = worktree,
                       relpath = path,
                       revisionExpr = revisionExpr,
-                      revisionId = None)
+                      revisionId = None,
+                      rebase = rebase,
+                      groups = groups,
+                      sync_c = sync_c)
 
     for n in node.childNodes:
       if n.nodeName == 'copyfile':
         self._ParseCopyFile(project, n)
+      if n.nodeName == 'annotation':
+        self._ParseAnnotation(project, n)
 
     return project
 
@@ -525,6 +604,17 @@ class XmlManifest(object):
       # dest is relative to the top of the tree
       project.AddCopyFile(src, dest, os.path.join(self.topdir, dest))
 
+  def _ParseAnnotation(self, project, node):
+    name = self._reqatt(node, 'name')
+    value = self._reqatt(node, 'value')
+    try:
+      keep = self._reqatt(node, 'keep').lower()
+    except ManifestParseError:
+      keep = "true"
+    if keep != "true" and keep != "false":
+      raise ManifestParseError, "optional \"keep\" attribute must be \"true\" or \"false\""
+    project.AddAnnotation(name, value, keep)
+
   def _get_remote(self, node):
     name = node.getAttribute('remote')
     if not name:

project.py

@@ -16,15 +16,27 @@ import traceback
 import errno
 import filecmp
 import os
+import random
 import re
 import shutil
 import stat
 import sys
+import time
 import urllib2
 
+try:
+  import threading as _threading
+except ImportError:
+  import dummy_threading as _threading
+
+try:
+  from os import SEEK_END
+except ImportError:
+  SEEK_END = 2
+
 from color import Coloring
 from git_command import GitCommand
-from git_config import GitConfig, IsId, GetSchemeFromUrl
+from git_config import GitConfig, IsId, GetSchemeFromUrl, ID_RE
 from error import DownloadError
 from error import GitError, HookError, ImportError, UploadError
 from error import ManifestInvalidRevisionError
@@ -32,6 +44,8 @@ from progress import Progress
 
 from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M
 
+_urllib_lock = _threading.Lock()
+
 def _lwrite(path, content):
   lock = '%s.lock' % path
 
@@ -199,6 +213,11 @@ class DiffColoring(Coloring):
     Coloring.__init__(self, config, 'diff')
     self.project   = self.printer('header',    attr = 'bold')
 
+class _Annotation:
+  def __init__(self, name, value, keep):
+    self.name = name
+    self.value = value
+    self.keep = keep
 
 class _CopyFile:
   def __init__(self, src, dest, abssrc, absdest):
@@ -489,7 +508,10 @@ class Project(object):
                worktree,
                relpath,
                revisionExpr,
-               revisionId):
+               revisionId,
+               rebase = True,
+               groups = None,
+               sync_c = False):
     self.manifest = manifest
     self.name = name
     self.remote = remote
@@ -508,8 +530,13 @@ class Project(object):
     else:
       self.revisionId = revisionId
 
+    self.rebase = rebase
+    self.groups = groups
+    self.sync_c = sync_c
+
     self.snapshots = {}
     self.copyfiles = []
+    self.annotations = []
     self.config = GitConfig.ForRepository(
                     gitdir = self.gitdir,
                     defaults =  self.manifest.globalConfig)
@@ -628,6 +655,25 @@ class Project(object):
 
     return heads
 
+  def MatchesGroups(self, manifest_groups):
+    """Returns true if the manifest groups specified at init should cause
+       this project to be synced.
+       Prefixing a manifest group with "-" inverts the meaning of a group.
+       All projects are implicitly labelled with "default".
+
+       labels are resolved in order.  In the example case of
+       project_groups: "default,group1,group2"
+       manifest_groups: "-group1,group2"
+       the project will be matched.
+    """
+    matched = False
+    for group in manifest_groups:
+      if group.startswith('-') and group[1:] in self.groups:
+        matched = False
+      elif group in self.groups:
+        matched = True
+
+    return matched
 
 ## Status Display ##
 
@@ -674,7 +720,7 @@ class Project(object):
     di = self.work_git.DiffZ('diff-index', '-M', '--cached', HEAD)
     df = self.work_git.DiffZ('diff-files')
     do = self.work_git.LsOthers()
-    if not rb and not di and not df and not do:
+    if not rb and not di and not df and not do and not self.CurrentBranch:
       return 'CLEAN'
 
     out = StatusColoring(self.config)
@@ -732,7 +778,7 @@ class Project(object):
 
     return 'DIRTY'
 
-  def PrintWorkTreeDiff(self):
+  def PrintWorkTreeDiff(self, absolute_paths=False):
     """Prints the status of the repository to stdout.
     """
     out = DiffColoring(self.config)
@@ -740,6 +786,9 @@ class Project(object):
     if out.is_on:
       cmd.append('--color')
     cmd.append(HEAD)
+    if absolute_paths:
+      cmd.append('--src-prefix=a/%s/' % self.relpath)
+      cmd.append('--dst-prefix=b/%s/' % self.relpath)
     cmd.append('--')
     p = GitCommand(self,
                    cmd,
@@ -852,32 +901,31 @@ class Project(object):
       branch.remote.projectname = self.name
       branch.remote.Save()
 
-    if branch.remote.ReviewProtocol == 'ssh':
-      if dest_branch.startswith(R_HEADS):
-        dest_branch = dest_branch[len(R_HEADS):]
+    url = branch.remote.ReviewUrl(self.UserEmail)
+    if url is None:
+      raise UploadError('review not configured')
+    cmd = ['push']
 
+    if url.startswith('ssh://'):
       rp = ['gerrit receive-pack']
       for e in people[0]:
         rp.append('--reviewer=%s' % sq(e))
       for e in people[1]:
         rp.append('--cc=%s' % sq(e))
+      cmd.append('--receive-pack=%s' % " ".join(rp))
 
+    cmd.append(url)
+
+    if dest_branch.startswith(R_HEADS):
+      dest_branch = dest_branch[len(R_HEADS):]
     ref_spec = '%s:refs/for/%s' % (R_HEADS + branch.name, dest_branch)
     if auto_topic:
       ref_spec = ref_spec + '/' + branch.name
-
-      cmd = ['push']
-      cmd.append('--receive-pack=%s' % " ".join(rp))
-      cmd.append(branch.remote.SshReviewUrl(self.UserEmail))
     cmd.append(ref_spec)
 
     if GitCommand(self, cmd, bare = True).Wait() != 0:
       raise UploadError('Upload failed')
 
-    else:
-        raise UploadError('Unsupported protocol %s' \
-          % branch.remote.review)
-
     msg = "posted to %s for %s" % (branch.remote.review, dest_branch)
     self.bare_git.UpdateRef(R_PUB + branch.name,
                             R_HEADS + branch.name,
@@ -886,7 +934,11 @@ class Project(object):
 
 ## Sync ##
 
-  def Sync_NetworkHalf(self, quiet=False, is_new=None):
+  def Sync_NetworkHalf(self,
+      quiet=False,
+      is_new=None,
+      current_branch_only=False,
+      clone_bundle=True):
     """Perform only the network IO portion of the sync process.
        Local working directory/branch state is not affected.
     """
@@ -894,22 +946,38 @@ class Project(object):
       is_new = not self.Exists
     if is_new:
       self._InitGitDir()
-
     self._InitRemote()
-    if not self._RemoteFetch(initial=is_new, quiet=quiet):
-      return False
 
-    #Check that the requested ref was found after fetch
-    #
+    if is_new:
+      alt = os.path.join(self.gitdir, 'objects/info/alternates')
       try:
-      self.GetRevisionId()
-    except ManifestInvalidRevisionError:
-      # if the ref is a tag. We can try fetching
-      # the tag manually as a last resort
-      #
-      rev = self.revisionExpr
-      if rev.startswith(R_TAGS):
-        self._RemoteFetch(None, rev[len(R_TAGS):], quiet=quiet)
+        fd = open(alt, 'rb')
+        try:
+          alt_dir = fd.readline().rstrip()
+        finally:
+          fd.close()
+      except IOError:
+        alt_dir = None
+    else:
+      alt_dir = None
+
+    if clone_bundle \
+    and alt_dir is None \
+    and self._ApplyCloneBundle(initial=is_new, quiet=quiet):
+      is_new = False
+
+    if not current_branch_only:
+      if self.sync_c:
+        current_branch_only = True
+      elif not self.manifest._loaded:
+        # Manifest cannot check defaults until it syncs.
+        current_branch_only = False
+      elif self.manifest.default.sync_c:
+        current_branch_only = True
+
+    if not self._RemoteFetch(initial=is_new, quiet=quiet, alt_dir=alt_dir,
+                             current_branch_only=current_branch_only):
+      return False
 
     if self.worktree:
       self._InitMRef()
@@ -997,7 +1065,7 @@ class Project(object):
 
     if not branch.LocalMerge:
       # The current branch has no tracking configuration.
-      # Jump off it to a deatched HEAD.
+      # Jump off it to a detached HEAD.
       #
       syncbuf.info(self,
                    "leaving %s; does not track upstream",
@@ -1075,10 +1143,12 @@ class Project(object):
                    len(local_changes) - cnt_mine)
 
     branch.remote = self.GetRemote(self.remote.name)
+    if not ID_RE.match(self.revisionExpr):
+      # in case of manifest sync the revisionExpr might be a SHA1
       branch.merge = self.revisionExpr
     branch.Save()
 
-    if cnt_mine > 0:
+    if cnt_mine > 0 and self.rebase:
       def _dorebase():
         self._Rebase(upstream = '%s^1' % last_mine, onto = revid)
         self._CopyFiles()
@@ -1102,6 +1172,9 @@ class Project(object):
     abssrc = os.path.join(self.worktree, src)
     self.copyfiles.append(_CopyFile(src, dest, abssrc, absdest))
 
+  def AddAnnotation(self, name, value, keep):
+    self.annotations.append(_Annotation(name, value, keep))
+
   def DownloadPatchSet(self, change_id, patch_id):
     """Download a single patch set of a single change to FETCH_HEAD.
     """
@@ -1305,9 +1378,32 @@ class Project(object):
 
 ## Direct Git Commands ##
 
-  def _RemoteFetch(self, name=None, tag=None,
+  def _RemoteFetch(self, name=None,
+                   current_branch_only=False,
                    initial=False,
-                   quiet=False):
+                   quiet=False,
+                   alt_dir=None):
+
+    is_sha1 = False
+    tag_name = None
+
+    if current_branch_only:
+      if ID_RE.match(self.revisionExpr) is not None:
+        is_sha1 = True
+      elif self.revisionExpr.startswith(R_TAGS):
+        # this is a tag and its sha1 value should never change
+        tag_name = self.revisionExpr[len(R_TAGS):]
+
+      if is_sha1 or tag_name is not None:
+        try:
+          # if revision (sha or tag) is not present then following function
+          # throws an error.
+          self.bare_git.rev_parse('--verify', '%s^0' % self.revisionExpr)
+          return True
+        except GitError:
+          # There is no such persistent revision. We have to fetch it.
+          pass
+
     if not name:
       name = self.remote.name
 
@@ -1316,29 +1412,9 @@ class Project(object):
     if remote.PreConnectFetch():
       ssh_proxy = True
 
-    bundle_dst = os.path.join(self.gitdir, 'clone.bundle')
-    bundle_tmp = os.path.join(self.gitdir, 'clone.bundle.tmp')
-    use_bundle = False
-    if os.path.exists(bundle_dst) or os.path.exists(bundle_tmp):
-      use_bundle = True
-
     if initial:
-      alt = os.path.join(self.gitdir, 'objects/info/alternates')
-      try:
-        fd = open(alt, 'rb')
-        try:
-          ref_dir = fd.readline()
-          if ref_dir and ref_dir.endswith('\n'):
-            ref_dir = ref_dir[:-1]
-        finally:
-          fd.close()
-      except IOError, e:
-        ref_dir = None
-
-      if ref_dir and 'objects' == os.path.basename(ref_dir):
-        if use_bundle:
-          use_bundle = False
-        ref_dir = os.path.dirname(ref_dir)
+      if alt_dir and 'objects' == os.path.basename(alt_dir):
+        ref_dir = os.path.dirname(alt_dir)
         packed_refs = os.path.join(self.gitdir, 'packed-refs')
         remote = self.GetRemote(name)
 
@@ -1374,10 +1450,8 @@ class Project(object):
             old_packed += line
 
         _lwrite(packed_refs, tmp_packed)
-
       else:
-        ref_dir = None
-        use_bundle = True
+        alt_dir = None
 
     cmd = ['fetch']
 
@@ -1386,66 +1460,95 @@ class Project(object):
     depth = self.manifest.manifestProject.config.GetString('repo.depth')
     if depth and initial:
       cmd.append('--depth=%s' % depth)
-      use_bundle = False
 
     if quiet:
       cmd.append('--quiet')
     if not self.worktree:
       cmd.append('--update-head-ok')
-
-    if use_bundle and not os.path.exists(bundle_dst):
-      bundle_url = remote.url + '/clone.bundle'
-      bundle_url = GitConfig.ForUser().UrlInsteadOf(bundle_url)
-      if GetSchemeFromUrl(bundle_url) in ('http', 'https'):
-        use_bundle = self._FetchBundle(
-          bundle_url,
-          bundle_tmp,
-          bundle_dst,
-          quiet=quiet)
-      else:
-        use_bundle = False
-
-    if use_bundle:
-      if not quiet:
-        cmd.append('--quiet')
-      cmd.append(bundle_dst)
-      for f in remote.fetch:
-        cmd.append(str(f))
-      cmd.append('refs/tags/*:refs/tags/*')
-    else:
     cmd.append(name)
-      if tag is not None:
-        cmd.append('tag')
-        cmd.append(tag)
 
-    ok = GitCommand(self,
-                    cmd,
-                    bare = True,
-                    ssh_proxy = ssh_proxy).Wait() == 0
+    if not current_branch_only or is_sha1:
+      # Fetch whole repo
+      cmd.append('--tags')
+      cmd.append((u'+refs/heads/*:') + remote.ToLocal('refs/heads/*'))
+    elif tag_name is not None:
+      cmd.append('tag')
+      cmd.append(tag_name)
+    else:
+      branch = self.revisionExpr
+      if branch.startswith(R_HEADS):
+        branch = branch[len(R_HEADS):]
+      cmd.append((u'+refs/heads/%s:' % branch) + remote.ToLocal('refs/heads/%s' % branch))
+
+    ok = False
+    for i in range(2):
+      if GitCommand(self, cmd, bare=True, ssh_proxy=ssh_proxy).Wait() == 0:
+        ok = True
+        break
+      time.sleep(random.randint(30, 45))
 
     if initial:
-      if ref_dir:
+      if alt_dir:
         if old_packed != '':
           _lwrite(packed_refs, old_packed)
         else:
           os.remove(packed_refs)
       self.bare_git.pack_refs('--all', '--prune')
+    return ok
 
+  def _ApplyCloneBundle(self, initial=False, quiet=False):
+    if initial and self.manifest.manifestProject.config.GetString('repo.depth'):
+      return False
+
+    remote = self.GetRemote(self.remote.name)
+    bundle_url = remote.url + '/clone.bundle'
+    bundle_url = GitConfig.ForUser().UrlInsteadOf(bundle_url)
+    if GetSchemeFromUrl(bundle_url) in ('persistent-http', 'persistent-https'):
+      bundle_url = bundle_url[len('persistent-'):]
+    if GetSchemeFromUrl(bundle_url) not in ('http', 'https'):
+      return False
+
+    bundle_dst = os.path.join(self.gitdir, 'clone.bundle')
+    bundle_tmp = os.path.join(self.gitdir, 'clone.bundle.tmp')
+
+    exist_dst = os.path.exists(bundle_dst)
+    exist_tmp = os.path.exists(bundle_tmp)
+
+    if not initial and not exist_dst and not exist_tmp:
+      return False
+
+    if not exist_dst:
+      exist_dst = self._FetchBundle(bundle_url, bundle_tmp, bundle_dst, quiet)
+    if not exist_dst:
+      return False
+
+    cmd = ['fetch']
+    if quiet:
+      cmd.append('--quiet')
+    if not self.worktree:
+      cmd.append('--update-head-ok')
+    cmd.append(bundle_dst)
+    for f in remote.fetch:
+      cmd.append(str(f))
+    cmd.append('refs/tags/*:refs/tags/*')
+
+    ok = GitCommand(self, cmd, bare=True).Wait() == 0
     if os.path.exists(bundle_dst):
       os.remove(bundle_dst)
     if os.path.exists(bundle_tmp):
       os.remove(bundle_tmp)
-
     return ok
 
-  def _FetchBundle(self, srcUrl, tmpPath, dstPath, quiet=False):
+  def _FetchBundle(self, srcUrl, tmpPath, dstPath, quiet):
     keep = True
     done = False
     dest = open(tmpPath, 'a+b')
     try:
-      dest.seek(0, os.SEEK_END)
+      dest.seek(0, SEEK_END)
       pos = dest.tell()
 
+      _urllib_lock.acquire()
+      try:
         req = urllib2.Request(srcUrl)
         if pos > 0:
           req.add_header('Range', 'bytes=%d-' % pos)
@@ -1453,10 +1556,16 @@ class Project(object):
         try:
           r = urllib2.urlopen(req)
         except urllib2.HTTPError, e:
-        if e.code == 404:
+          def _content_type():
+            try:
+              return e.info()['content-type']
+            except:
+              return None
+
+          if e.code in (401, 403, 404):
             keep = False
             return False
-        elif e.info()['content-type'] == 'text/plain':
+          elif _content_type() == 'text/plain':
             try:
               msg = e.read()
               if len(msg) > 0 and msg[-1] == '\n':
@@ -1473,11 +1582,13 @@ class Project(object):
               msg = ''
           raise DownloadError('HTTP %s%s' % (e.code, msg))
         except urllib2.URLError, e:
-        raise DownloadError('%s (%s)' % (e.reason, req.get_host()))
+          raise DownloadError('%s: %s ' % (req.get_host(), str(e)))
+      finally:
+        _urllib_lock.release()
 
       p = None
       try:
-        size = r.headers['content-length']
+        size = r.headers.get('content-length', 0)
         unit = 1 << 10
 
         if size and not quiet:
@@ -1534,6 +1645,23 @@ class Project(object):
       if self._allrefs:
         raise GitError('%s checkout %s ' % (self.name, rev))
 
+  def _CherryPick(self, rev, quiet=False):
+    cmd = ['cherry-pick']
+    cmd.append(rev)
+    cmd.append('--')
+    if GitCommand(self, cmd).Wait() != 0:
+      if self._allrefs:
+        raise GitError('%s cherry-pick %s ' % (self.name, rev))
+
+  def _Revert(self, rev, quiet=False):
+    cmd = ['revert']
+    cmd.append('--no-edit')
+    cmd.append(rev)
+    cmd.append('--')
+    if GitCommand(self, cmd).Wait() != 0:
+      if self._allrefs:
+        raise GitError('%s revert %s ' % (self.name, rev))
+
   def _ResetHard(self, rev, quiet=True):
     cmd = ['reset', '--hard']
     if quiet:
@@ -1550,8 +1678,10 @@ class Project(object):
     if GitCommand(self, cmd).Wait() != 0:
       raise GitError('%s rebase %s ' % (self.name, upstream))
 
-  def _FastForward(self, head):
+  def _FastForward(self, head, ffonly=False):
     cmd = ['merge', head]
+    if ffonly:
+      cmd.append("--ff-only")
     if GitCommand(self, cmd).Wait() != 0:
       raise GitError('%s merge %s ' % (self.name, head))
 
@@ -2023,7 +2153,8 @@ class MetaProject(Project):
                      remote = RemoteSpec('origin'),
                      relpath = '.repo/%s' % name,
                      revisionExpr = 'refs/heads/master',
-                     revisionId = None)
+                     revisionId = None,
+                     groups = None)
 
   def PreSync(self):
     if self.Exists:

repo

@@ -28,7 +28,7 @@ if __name__ == '__main__':
 del magic
 
 # increment this whenever we make important changes to this script
-VERSION = (1, 13)
+VERSION = (1, 16)
 
 # increment this if the MAINTAINER_KEYS block is modified
 KEYRING_VERSION = (1,0)
@@ -125,6 +125,15 @@ group.add_option('--reference',
 group.add_option('--depth', type='int', default=None,
                  dest='depth',
                  help='create a shallow clone with given depth; see git clone')
+group.add_option('-g', '--groups',
+                 dest='groups', default='default',
+                 help='restrict manifest projects to ones with a specified group',
+                 metavar='GROUP')
+group.add_option('-p', '--platform',
+                 dest='platform', default="auto",
+                 help='restrict manifest projects to ones with a specified'
+                      'platform group [auto|all|none|linux|darwin|...]',
+                 metavar='PLATFORM')
 
 
 # Tool
@@ -139,6 +148,11 @@ group.add_option('--no-repo-verify',
                  dest='no_repo_verify', action='store_true',
                  help='do not verify repo source code')
 
+# Other
+group = init_optparse.add_option_group('Other options')
+group.add_option('--config-name',
+                 dest='config_name', action="store_true", default=False,
+                 help='Always prompt for name/e-mail')
 
 class CloneFailure(Exception):
   """Indicate the remote clone of repo itself failed.
@@ -149,7 +163,7 @@ def _Init(args):
   """Installs repo by cloning it over the network.
   """
   opt, args = init_optparse.parse_args(args)
-  if args or not opt.manifest_url:
+  if args:
     init_optparse.print_usage()
     sys.exit(1)
 
@@ -306,11 +320,12 @@ def _InitHttp():
     n = netrc.netrc()
     for host in n.hosts:
       p = n.hosts[host]
-      mgr.add_password(None, 'http://%s/'  % host, p[0], p[2])
-      mgr.add_password(None, 'https://%s/' % host, p[0], p[2])
+      mgr.add_password(p[1], 'http://%s/'  % host, p[0], p[2])
+      mgr.add_password(p[1], 'https://%s/' % host, p[0], p[2])
   except:
     pass
   handlers.append(urllib2.HTTPBasicAuthHandler(mgr))
+  handlers.append(urllib2.HTTPDigestAuthHandler(mgr))
 
   if 'http_proxy' in os.environ:
     url = os.environ['http_proxy']

subcmds/diff.py

@@ -20,8 +20,21 @@ class Diff(PagedCommand):
   helpSummary = "Show changes between commit and working tree"
   helpUsage = """
 %prog [<project>...]
+
+The -u option causes '%prog' to generate diff output with file paths
+relative to the repository root, so the output can be applied
+to the Unix 'patch' command.
 """
 
+  def _Options(self, p):
+    def cmd(option, opt_str, value, parser):
+      setattr(parser.values, option.dest, list(parser.rargs))
+      while parser.rargs:
+        del parser.rargs[0]
+    p.add_option('-u', '--absolute',
+                 dest='absolute', action='store_true',
+                 help='Paths are relative to the repository root')
+
   def Execute(self, opt, args):
     for project in self.GetProjects(args):
-      project.PrintWorkTreeDiff()
+      project.PrintWorkTreeDiff(opt.absolute)

subcmds/download.py

@@ -33,7 +33,15 @@ makes it available in your project's local working directory.
 """
 
   def _Options(self, p):
-    pass
+    p.add_option('-c','--cherry-pick',
+                 dest='cherrypick', action='store_true',
+                 help="cherry-pick instead of checkout")
+    p.add_option('-r','--revert',
+                 dest='revert', action='store_true',
+                 help="revert instead of checkout")
+    p.add_option('-f','--ff-only',
+                 dest='ffonly', action='store_true',
+                 help="force fast-forward merge")
 
   def _ParseChangeIds(self, args):
     if not args:
@@ -66,7 +74,7 @@ makes it available in your project's local working directory.
           % (project.name, change_id, ps_id)
         sys.exit(1)
 
-      if not dl.commits:
+      if not opt.revert and not dl.commits:
         print >>sys.stderr, \
           '[%s] change %d/%d has already been merged' \
           % (project.name, change_id, ps_id)
@@ -78,4 +86,11 @@ makes it available in your project's local working directory.
           % (project.name, change_id, ps_id, len(dl.commits))
         for c in dl.commits:
           print >>sys.stderr, '  %s' % (c)
+      if opt.cherrypick:
+        project._CherryPick(dl.commit)
+      elif opt.revert:
+        project._Revert(dl.commit)
+      elif opt.ffonly:
+        project._FastForward(dl.commit, ffonly=True)
+      else:
         project._Checkout(dl.commit)

subcmds/forall.py

@@ -82,6 +82,11 @@ revision to a locally executed git command, use REPO_LREV.
 REPO_RREV is the name of the revision from the manifest, exactly
 as written in the manifest.
 
+REPO__* are any extra environment variables, specified by the
+"annotation" element under any project element.  This can be useful
+for differentiating trees based on user-specific criteria, or simply
+annotating tree details.
+
 shell positional arguments ($1, $2, .., $#) are set to any arguments
 following <command>.
 
@@ -162,6 +167,8 @@ terminal and are not redirected.
       setenv('REPO_REMOTE', project.remote.name)
       setenv('REPO_LREV', project.GetRevisionId())
       setenv('REPO_RREV', project.revisionExpr)
+      for a in project.annotations:
+        setenv("REPO__%s" % (a.name), a.value)
 
       if mirror:
         setenv('GIT_DIR', project.gitdir)

subcmds/help.py

@@ -165,6 +165,7 @@ See 'repo help --all' for a complete list of recognized commands.
         print >>sys.stderr, "repo: '%s' is not a repo command." % name
         sys.exit(1)
 
+      cmd.manifest = self.manifest
       self._PrintCommandHelp(cmd)
 
     else:

subcmds/init.py

@@ -14,6 +14,8 @@
 # limitations under the License.
 
 import os
+import platform
+import re
 import shutil
 import sys
 
@@ -86,6 +88,15 @@ to update the working directory files.
     g.add_option('--depth', type='int', default=None,
                  dest='depth',
                  help='create a shallow clone with given depth; see git clone')
+    g.add_option('-g', '--groups',
+                 dest='groups', default='default',
+                 help='restrict manifest projects to ones with a specified group',
+                 metavar='GROUP')
+    g.add_option('-p', '--platform',
+                 dest='platform', default='auto',
+                 help='restrict manifest projects to ones with a specified'
+                      'platform group [auto|all|none|linux|darwin|...]',
+                 metavar='PLATFORM')
 
     # Tool
     g = p.add_option_group('repo Version options')
@@ -99,6 +110,12 @@ to update the working directory files.
                  dest='no_repo_verify', action='store_true',
                  help='do not verify repo source code')
 
+    # Other
+    g = p.add_option_group('Other options')
+    g.add_option('--config-name',
+                 dest='config_name', action="store_true", default=False,
+                 help='Always prompt for name/e-mail')
+
   def _SyncManifest(self, opt):
     m = self.manifest.manifestProject
     is_new = not m.Exists
@@ -129,6 +146,27 @@ to update the working directory files.
       r.ResetFetch()
       r.Save()
 
+    groups = re.split('[,\s]+', opt.groups)
+    all_platforms = ['linux', 'darwin']
+    platformize = lambda x: 'platform-' + x
+    if opt.platform == 'auto':
+      if (not opt.mirror and
+          not m.config.GetString('repo.mirror') == 'true'):
+        groups.append(platformize(platform.system().lower()))
+    elif opt.platform == 'all':
+      groups.extend(map(platformize, all_platforms))
+    elif opt.platform in all_platforms:
+      groups.extend(platformize(opt.platform))
+    elif opt.platform != 'none':
+      print >>sys.stderr, 'fatal: invalid platform flag'
+      sys.exit(1)
+
+    groups = [x for x in groups if x]
+    groupstr = ','.join(groups)
+    if opt.platform == 'auto' and groupstr == 'default,platform-' + platform.system().lower():
+      groupstr = None
+    m.config.SetString('manifest.groups', groupstr)
+
     if opt.reference:
       m.config.SetString('repo.reference', opt.reference)
 
@@ -179,6 +217,24 @@ to update the working directory files.
       return value
     return a
 
+  def _ShouldConfigureUser(self):
+    gc = self.manifest.globalConfig
+    mp = self.manifest.manifestProject
+
+    # If we don't have local settings, get from global.
+    if not mp.config.Has('user.name') or not mp.config.Has('user.email'):
+      if not gc.Has('user.name') or not gc.Has('user.email'):
+        return True
+
+      mp.config.SetString('user.name', gc.GetString('user.name'))
+      mp.config.SetString('user.email', gc.GetString('user.email'))
+
+    print ''
+    print 'Your identity is: %s <%s>' % (mp.config.GetString('user.name'),
+                                         mp.config.GetString('user.email'))
+    print 'If you want to change this, please re-run \'repo init\' with --config-name'
+    return False
+
   def _ConfigureUser(self):
     mp = self.manifest.manifestProject
 
@@ -189,7 +245,7 @@ to update the working directory files.
 
       print ''
       print 'Your identity is: %s <%s>' % (name, email)
-      sys.stdout.write('is this correct [y/n]? ')
+      sys.stdout.write('is this correct [y/N]? ')
       a = sys.stdin.readline().strip()
       if a in ('yes', 'y', 't', 'true'):
         break
@@ -231,7 +287,7 @@ to update the working directory files.
       out.printer(fg='black', attr=c)(' %-6s ', c)
     out.nl()
 
-    sys.stdout.write('Enable color display in this user account (y/n)? ')
+    sys.stdout.write('Enable color display in this user account (y/N)? ')
     a = sys.stdin.readline().strip().lower()
     if a in ('y', 'yes', 't', 'true', 'on'):
       gc.SetString('color.ui', 'auto')
@@ -261,6 +317,7 @@ to update the working directory files.
     self._LinkManifest(opt.manifest_name)
 
     if os.isatty(0) and os.isatty(1) and not self.manifest.IsMirror:
+      if opt.config_name or self._ShouldConfigureUser():
         self._ConfigureUser()
       self._ConfigureColor()
 

subcmds/status.py

@@ -101,19 +101,6 @@ the following meanings:
     all = self.GetProjects(args)
     counter = itertools.count()
 
-    on = {}
-    for project in all:
-      cb = project.CurrentBranch
-      if cb:
-        if cb not in on:
-          on[cb] = []
-        on[cb].append(project)
-
-    branch_names = list(on.keys())
-    branch_names.sort()
-    for cb in branch_names:
-      print '# on branch %s' % cb
-
     if opt.jobs == 1:
       for project in all:
         state = project.PrintWorkTreeStatus()
@@ -124,14 +111,21 @@ the following meanings:
       threads_and_output = []
       for project in all:
         sem.acquire()
-        output = StringIO.StringIO()
+
+        class BufList(StringIO.StringIO):
+          def dump(self, ostream):
+            for entry in self.buflist:
+              ostream.write(entry)
+
+        output = BufList()
+
         t = _threading.Thread(target=self._StatusHelper,
                               args=(project, counter, sem, output))
         threads_and_output.append((t, output))
         t.start()
       for (t, output) in threads_and_output:
         t.join()
-        sys.stdout.write(output.getvalue())
+        output.dump(sys.stdout)
         output.close()
     if len(all) == counter.next():
       print 'nothing to commit (working directory clean)'

subcmds/sync.py

@@ -86,6 +86,12 @@ specify a custom tag/label.
 The -f/--force-broken option can be used to proceed with syncing
 other projects if a project sync fails.
 
+The --no-clone-bundle option disables any attempt to use
+$URL/clone.bundle to bootstrap a new Git repository from a
+resumeable bundle file on a content delivery network. This
+may be necessary if there are problems with the local Python
+HTTP client or proxy configuration, but the Git binary works.
+
 SSH Connections
 ---------------
 
@@ -131,12 +137,21 @@ later is required to fix a server side protocol bug.
     p.add_option('-d','--detach',
                  dest='detach_head', action='store_true',
                  help='detach projects back to manifest revision')
+    p.add_option('-c','--current-branch',
+                 dest='current_branch_only', action='store_true',
+                 help='fetch only current branch from server')
     p.add_option('-q','--quiet',
                  dest='quiet', action='store_true',
                  help='be more quiet')
     p.add_option('-j','--jobs',
                  dest='jobs', action='store', type='int',
                  help="projects to fetch simultaneously (default %d)" % self.jobs)
+    p.add_option('-m', '--manifest-name',
+                 dest='manifest_name',
+                 help='temporary manifest to use for this sync', metavar='NAME.xml')
+    p.add_option('--no-clone-bundle',
+                 dest='no_clone_bundle', action='store_true',
+                 help='disable use of /clone.bundle on HTTP/HTTPS')
     if show_smart:
       p.add_option('-s', '--smart-sync',
                    dest='smart_sync', action='store_true',
@@ -179,7 +194,10 @@ later is required to fix a server side protocol bug.
       # - We always make sure we unlock the lock if we locked it.
       try:
         try:
-          success = project.Sync_NetworkHalf(quiet=opt.quiet)
+          success = project.Sync_NetworkHalf(
+            quiet=opt.quiet,
+            current_branch_only=opt.current_branch_only,
+            clone_bundle=not opt.no_clone_bundle)
 
           # Lock around all the rest of the code, since printing, updating a set
           # and Progress.update() are not thread safe.
@@ -195,14 +213,10 @@ later is required to fix a server side protocol bug.
 
           fetched.add(project.gitdir)
           pm.update()
-        except BaseException, e:
-          # Notify the _Fetch() function about all errors.
+        except _FetchError:
+          err_event.set()
+        except:
           err_event.set()
-
-          # If we got our own _FetchError, we don't want a stack trace.
-          # However, if we got something else (something in Sync_NetworkHalf?),
-          # we'd like one (so re-raise after we've set err_event).
-          if not isinstance(e, _FetchError):
           raise
       finally:
         if did_lock:
@@ -216,7 +230,8 @@ later is required to fix a server side protocol bug.
     if self.jobs == 1:
       for project in projects:
         pm.update()
-        if project.Sync_NetworkHalf(quiet=opt.quiet):
+        if project.Sync_NetworkHalf(quiet=opt.quiet,
+                                    current_branch_only=opt.current_branch_only):
           fetched.add(project.gitdir)
         else:
           print >>sys.stderr, 'error: Cannot fetch %s' % project.name
@@ -262,7 +277,7 @@ later is required to fix a server side protocol bug.
 
   def UpdateProjectList(self):
     new_project_paths = []
-    for project in self.manifest.projects.values():
+    for project in self.GetProjects(None, missing_ok=True):
       if project.relpath:
         new_project_paths.append(project.relpath)
     file_name = 'project.list'
@@ -291,7 +306,8 @@ later is required to fix a server side protocol bug.
                              worktree = os.path.join(self.manifest.topdir, path),
                              relpath = path,
                              revisionExpr = 'HEAD',
-                             revisionId = None)
+                             revisionId = None,
+                             groups = None)
 
               if project.IsDirty():
                 print >>sys.stderr, 'error: Cannot remove project "%s": \
@@ -332,6 +348,15 @@ uncommitted changes are present' % project.relpath
     if opt.network_only and opt.local_only:
       print >>sys.stderr, 'error: cannot combine -n and -l'
       sys.exit(1)
+    if opt.manifest_name and opt.smart_sync:
+      print >>sys.stderr, 'error: cannot combine -m and -s'
+      sys.exit(1)
+    if opt.manifest_name and opt.smart_tag:
+      print >>sys.stderr, 'error: cannot combine -m and -t'
+      sys.exit(1)
+
+    if opt.manifest_name:
+      self.manifest.Override(opt.manifest_name)
 
     if opt.smart_sync or opt.smart_tag:
       if not self.manifest.manifest_server:
@@ -392,7 +417,8 @@ uncommitted changes are present' % project.relpath
       _PostRepoUpgrade(self.manifest)
 
     if not opt.local_only:
-      mp.Sync_NetworkHalf(quiet=opt.quiet)
+      mp.Sync_NetworkHalf(quiet=opt.quiet,
+                          current_branch_only=opt.current_branch_only)
 
     if mp.HasChanges:
       syncbuf = SyncBuffer(mp.config)

subcmds/upload.py

@@ -73,7 +73,7 @@ Configuration
 
 review.URL.autoupload:
 
-To disable the "Upload ... (y/n)?" prompt, you can set a per-project
+To disable the "Upload ... (y/N)?" prompt, you can set a per-project
 or global Git configuration option.  If review.URL.autoupload is set
 to "true" then repo will assume you always answer "y" at the prompt,
 and will not prompt you further.  If it is set to "false" then repo
@@ -123,6 +123,9 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
     p.add_option('--br',
                  type='string',  action='store', dest='branch',
                  help='Branch to upload.')
+    p.add_option('--cbr', '--current-branch',
+                 dest='current_branch', action='store_true',
+                 help='Upload current git branch.')
 
     # Options relating to upload hook.  Note that verify and no-verify are NOT
     # opposites of each other, which is why they store to different locations.
@@ -162,7 +165,7 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
       date = branch.date
       list = branch.commits
 
-      print 'Upload project %s/:' % project.relpath
+      print 'Upload project %s/ to remote branch %s:' % (project.relpath, project.revisionExpr)
       print '  branch %s (%2d commit%s, %s):' % (
                     name,
                     len(list),
@@ -171,7 +174,7 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
       for commit in list:
         print '         %s' % commit
 
-      sys.stdout.write('to %s (y/n)? ' % remote.review)
+      sys.stdout.write('to %s (y/N)? ' % remote.review)
       answer = sys.stdin.readline().strip()
       answer = answer in ('y', 'Y', 'yes', '1', 'true', 't')
 
@@ -202,11 +205,12 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
 
         if b:
           script.append('#')
-        script.append('#  branch %s (%2d commit%s, %s):' % (
+        script.append('#  branch %s (%2d commit%s, %s) to remote branch %s:' % (
                       name,
                       len(list),
                       len(list) != 1 and 's' or '',
-                      date))
+                      date,
+                      project.revisionExpr))
         for commit in list:
           script.append('#         %s' % commit)
         b[name] = branch
@@ -215,6 +219,11 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
       branches[project.name] = b
     script.append('')
 
+    script = [ x.encode('utf-8')
+             if issubclass(type(x), unicode)
+             else x
+             for x in script ]
+
     script = Editor.EditString("\n".join(script)).split("\n")
 
     project_re = re.compile(r'^#?\s*project\s*([^\s]+)/:$')
@@ -294,7 +303,7 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
 
             # if they want to auto upload, let's not ask because it could be automated
             if answer is None:
-                sys.stdout.write('Uncommitted changes in ' + branch.project.name + ' (did you forget to amend?). Continue uploading? (y/n) ')
+                sys.stdout.write('Uncommitted changes in ' + branch.project.name + ' (did you forget to amend?). Continue uploading? (y/N) ')
                 a = sys.stdin.readline().strip().lower()
                 if a not in ('y', 'yes', 't', 'true', 'on'):
                     print >>sys.stderr, "skipping upload"
@@ -345,6 +354,10 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
       branch = opt.branch
 
     for project in project_list:
+      if opt.current_branch:
+        cbr = project.CurrentBranch
+        avail = [project.GetUploadableBranch(cbr)] if cbr else None
+      else:
         avail = project.GetUploadableBranches(branch)
       if avail:
         pending.append((project, avail))

subcmds/version.py

@@ -19,6 +19,9 @@ from git_command import git
 from project import HEAD
 
 class Version(Command, MirrorSafeCommand):
+  wrapper_version = None
+  wrapper_path = None
+
   common = False
   helpSummary = "Display the version of repo"
   helpUsage = """
@@ -31,5 +34,10 @@ class Version(Command, MirrorSafeCommand):
 
     print 'repo version %s' % rp.work_git.describe(HEAD)
     print '       (from %s)' % rem.url
+
+    if Version.wrapper_path is not None:
+      print 'repo launcher version %s' % Version.wrapper_version
+      print '       (from %s)' % Version.wrapper_path
+
     print git.version().strip()
     print 'Python %s' % sys.version