Use curl command line tool for clone.bundle

urllib2 is not thread safe and may be causing sync to lock up or
not work correctly on various platforms. Instead use the command
line curl program.

Change-Id: I36eaf18bb4df089d26ea99d533cb015e7c616eb0

SUBMITTING_PATCHES

@@ -5,7 +5,7 @@ Short Version:
  - Make sure all code is under the Apache License, 2.0.
  - Publish your changes for review:
 
-   git push ssh://review.source.android.com:29418/tools/repo.git HEAD:refs/for/master
+   git push https://gerrit-review.googlesource.com/git-repo HEAD:refs/for/maint
 
 
 Long Version:
@@ -55,24 +55,23 @@ Do not email your patches to anyone.
 
 Instead, login to the Gerrit Code Review tool at:
 
-  https://review.source.android.com/
+  https://gerrit-review.googlesource.com/
 
 Ensure you have completed one of the necessary contributor
 agreements, providing documentation to the project maintainers that
 they have right to redistribute your work under the Apache License:
 
-  https://review.source.android.com/#settings,agreements
+  https://gerrit-review.googlesource.com/#/settings/agreements
 
-Ensure you have registered one or more SSH public keys, so you can
-push your commits directly over SSH:
+Ensure you have obtained an HTTP password to authenticate:
 
-  https://review.source.android.com/#settings,ssh-keys
+  https://gerrit-review.googlesource.com/new-password
 
-Push your patches over SSH to the review server, possibly through
+Push your patches over HTTPS to the review server, possibly through
 a remembered remote to make this easier in the future:
 
-   git config remote.review.url ssh://review.source.android.com:29418/tools/repo.git
-   git config remote.review.push HEAD:refs/for/master
+   git config remote.review.url https://gerrit-review.googlesource.com/git-repo
+   git config remote.review.push HEAD:refs/for/maint
 
    git push review
 

command.py

@@ -15,9 +15,12 @@
 
 import os
 import optparse
+import platform
+import re
 import sys
 
 from error import NoSuchProjectError
+from error import InvalidProjectGroupsError
 
 class Command(object):
   """Base class for any command line action in repo.
@@ -56,16 +59,24 @@ class Command(object):
     """Perform the action, after option parsing is complete.
     """
     raise NotImplementedError
- 
+
   def GetProjects(self, args, missing_ok=False):
     """A list of projects that match the arguments.
     """
     all = self.manifest.projects
     result = []
 
+    mp = self.manifest.manifestProject
+
+    groups = mp.config.GetString('manifest.groups')
+    if not groups:
+      groups = 'default,platform-' + platform.system().lower()
+    groups = [x for x in re.split('[,\s]+', groups) if x]
+
     if not args:
       for project in all.values():
-        if missing_ok or project.Exists:
+        if ((missing_ok or project.Exists) and
+            project.MatchesGroups(groups)):
           result.append(project)
     else:
       by_path = None
@@ -102,6 +113,8 @@ class Command(object):
           raise NoSuchProjectError(arg)
         if not missing_ok and not project.Exists:
           raise NoSuchProjectError(arg)
+        if not project.MatchesGroups(groups):
+          raise InvalidProjectGroupsError(arg)
 
         result.append(project)
 

docs/manifest-format.txt

@@ -32,21 +32,31 @@ following DTD:
   
     <!ELEMENT remote (EMPTY)>
     <!ATTLIST remote name         ID    #REQUIRED>
+    <!ATTLIST remote alias        CDATA #IMPLIED>
     <!ATTLIST remote fetch        CDATA #REQUIRED>
     <!ATTLIST remote review       CDATA #IMPLIED>
   
     <!ELEMENT default (EMPTY)>
     <!ATTLIST default remote   IDREF #IMPLIED>
     <!ATTLIST default revision CDATA #IMPLIED>
+    <!ATTLIST default sync-j   CDATA #IMPLIED>
+    <!ATTLIST default sync-c   CDATA #IMPLIED>
 
     <!ELEMENT manifest-server (EMPTY)>
     <!ATTLIST url              CDATA #REQUIRED>
   
-    <!ELEMENT project (EMPTY)>
+    <!ELEMENT project (annotation?)>
     <!ATTLIST project name     CDATA #REQUIRED>
     <!ATTLIST project path     CDATA #IMPLIED>
     <!ATTLIST project remote   IDREF #IMPLIED>
     <!ATTLIST project revision CDATA #IMPLIED>
+    <!ATTLIST project groups   CDATA #IMPLIED>
+    <!ATTLIST project sync-c   CDATA #IMPLIED>
+
+    <!ELEMENT annotation (EMPTY)>
+    <!ATTLIST annotation name  CDATA #REQUIRED>
+    <!ATTLIST annotation value CDATA #REQUIRED>
+    <!ATTLIST annotation keep  CDATA "true">
   
     <!ELEMENT remove-project (EMPTY)>
     <!ATTLIST remove-project name  CDATA #REQUIRED>
@@ -54,6 +64,9 @@ following DTD:
     <!ELEMENT repo-hooks (EMPTY)>
     <!ATTLIST repo-hooks in-project CDATA #REQUIRED>
     <!ATTLIST repo-hooks enabled-list CDATA #REQUIRED>
+
+    <!ELEMENT include      (EMPTY)>
+    <!ATTLIST include name CDATA #REQUIRED>
   ]>
 
 A description of the elements and their attributes follows.
@@ -77,6 +90,12 @@ name specified here is used as the remote name in each project's
 .git/config, and is therefore automatically available to commands
 like `git fetch`, `git remote`, `git pull` and `git push`.
 
+Attribute `alias`: The alias, if specified, is used to override
+`name` to be set as the remote name in each project's .git/config.
+Its value can be duplicated while attribute `name` has to be unique
+in the manifest file. This helps each project to be able to have
+same remote name which actually points to different remote url.
+
 Attribute `fetch`: The Git URL prefix for all projects which use
 this remote.  Each project's name is appended to this prefix to
 form the actual URL used to clone the project.
@@ -157,6 +176,25 @@ Tags and/or explicit SHA-1s should work in theory, but have not
 been extensively tested.  If not supplied the revision given by
 the default element is used.
 
+Attribute `groups`: List of groups to which this project belongs,
+whitespace or comma separated.  All projects belong to the group
+"default", and each project automatically belongs to a group of
+it's name:`name` and path:`path`.  E.g. for
+<project name="monkeys" path="barrel-of"/>, that project
+definition is implicitly in the following manifest groups:
+default, name:monkeys, and path:barrel-of.
+
+Element annotation
+------------------
+
+Zero or more annotation elements may be specified as children of a
+project element. Each element describes a name-value pair that will be
+exported into each project's environment during a 'forall' command,
+prefixed with REPO__.  In addition, there is an optional attribute
+"keep" which accepts the case insensitive values "true" (default) or
+"false".  This attribute determines whether or not the annotation will
+be kept when exported with the manifest subcommand.
+
 Element remove-project
 ----------------------
 
@@ -168,6 +206,16 @@ This element is mostly useful in the local_manifest.xml, where
 the user can remove a project, and possibly replace it with their
 own definition.
 
+Element include
+---------------
+
+This element provides the capability of including another manifest
+file into the originating manifest.  Normal rules apply for the
+target manifest to include- it must be a usable manifest on it's own.
+
+Attribute `name`; the manifest to include, specified relative to
+the manifest repositories root.
+
 
 Local Manifest
 ==============

error.py

@@ -57,6 +57,15 @@ class UploadError(Exception):
   def __str__(self):
     return self.reason
 
+class DownloadError(Exception):
+  """Cannot download a repository.
+  """
+  def __init__(self, reason):
+    self.reason = reason
+
+  def __str__(self):
+    return self.reason
+
 class NoSuchProjectError(Exception):
   """A specified project does not exist in the work tree.
   """
@@ -68,6 +77,18 @@ class NoSuchProjectError(Exception):
       return 'in current directory'
     return self.name
 
+
+class InvalidProjectGroupsError(Exception):
+  """A specified project is not suitable for the specified groups
+  """
+  def __init__(self, name=None):
+    self.name = name
+
+  def __str__(self):
+    if self.Name is None:
+      return 'in current directory'
+    return self.name
+
 class RepoChangedException(Exception):
   """Thrown if 'repo sync' results in repo updating its internal
      repo or manifest repositories.  In this special case we must

git_command.py

@@ -72,6 +72,8 @@ def terminate_ssh_clients():
       pass
   _ssh_clients = []
 
+_git_version = None
+
 class _GitCall(object):
   def version(self):
     p = GitCommand(None, ['--version'], capture_stdout=True)
@@ -79,6 +81,21 @@ class _GitCall(object):
       return p.stdout
     return None
 
+  def version_tuple(self):
+    global _git_version
+
+    if _git_version is None:
+      ver_str = git.version()
+      if ver_str.startswith('git version '):
+        _git_version = tuple(
+          map(lambda x: int(x),
+            ver_str[len('git version '):].strip().split('-')[0].split('.')[0:3]
+          ))
+      else:
+        print >>sys.stderr, 'fatal: "%s" unsupported' % ver_str
+        sys.exit(1)
+    return _git_version
+
   def __getattr__(self, name):
     name = name.replace('_','-')
     def fun(*cmdv):
@@ -88,23 +105,9 @@ class _GitCall(object):
     return fun
 git = _GitCall()
 
-_git_version = None
-
 def git_require(min_version, fail=False):
-  global _git_version
-
-  if _git_version is None:
-    ver_str = git.version()
-    if ver_str.startswith('git version '):
-      _git_version = tuple(
-        map(lambda x: int(x),
-          ver_str[len('git version '):].strip().split('.')[0:3]
-        ))
-    else:
-      print >>sys.stderr, 'fatal: "%s" unsupported' % ver_str
-      sys.exit(1)
-
-  if min_version <= _git_version:
+  git_version = git.version_tuple()
+  if min_version <= git_version:
     return True
   if fail:
     need = '.'.join(map(lambda x: str(x), min_version))
@@ -144,6 +147,12 @@ class GitCommand(object):
     if ssh_proxy:
       _setenv(env, 'REPO_SSH_SOCK', ssh_sock())
       _setenv(env, 'GIT_SSH', _ssh_proxy())
+    if 'http_proxy' in env and 'darwin' == sys.platform:
+      s = "'http.proxy=%s'" % (env['http_proxy'],)
+      p = env.get('GIT_CONFIG_PARAMETERS')
+      if p is not None:
+        s = p + ' ' + s
+      _setenv(env, 'GIT_CONFIG_PARAMETERS', s)
 
     if project:
       if not cwd:
@@ -218,26 +227,10 @@ class GitCommand(object):
     self.stdin = p.stdin
 
   def Wait(self):
-    p = self.process
-
-    if p.stdin:
-      p.stdin.close()
-      self.stdin = None
-
-    if p.stdout:
-      self.stdout = p.stdout.read()
-      p.stdout.close()
-    else:
-      p.stdout = None
-
-    if p.stderr:
-      self.stderr = p.stderr.read()
-      p.stderr.close()
-    else:
-      p.stderr = None
-
     try:
-      rc = p.wait()
+      p = self.process
+      (self.stdout, self.stderr) = p.communicate()
+      rc = p.returncode
     finally:
       _remove_ssh_client(p)
     return rc

git_config.py

@@ -26,7 +26,6 @@ import time
 import urllib2
 
 from signal import SIGTERM
-from urllib2 import urlopen, HTTPError
 from error import GitError, UploadError
 from trace import Trace
 
@@ -198,6 +197,15 @@ class GitConfig(object):
     except KeyError:
       return False
 
+  def UrlInsteadOf(self, url):
+    """Resolve any url.*.insteadof references.
+    """
+    for new_url in self.GetSubSections('url'):
+      old_url = self.GetString('url.%s.insteadof' % new_url)
+      if old_url is not None and url.startswith(old_url):
+        return new_url + url[len(old_url):]
+    return url
+
   @property
   def _sections(self):
     d = self._section_dict
@@ -480,7 +488,13 @@ def close_ssh():
   _master_keys_lock = None
 
 URI_SCP = re.compile(r'^([^@:]*@?[^:/]{1,}):')
-URI_ALL = re.compile(r'^([a-z][a-z+]*)://([^@/]*@?[^/]*)/')
+URI_ALL = re.compile(r'^([a-z][a-z+-]*)://([^@/]*@?[^/]*)/')
+
+def GetSchemeFromUrl(url):
+  m = URI_ALL.match(url)
+  if m:
+    return m.group(1)
+  return None
 
 def _preconnect(url):
   m = URI_ALL.match(url)
@@ -513,7 +527,7 @@ class Remote(object):
     self.projectname = self._Get('projectname')
     self.fetch = map(lambda x: RefSpec.FromString(x),
                      self._Get('fetch', all=True))
-    self._review_protocol = None
+    self._review_url = None
 
   def _InsteadOf(self):
     globCfg = GitConfig.ForUser()
@@ -540,9 +554,8 @@ class Remote(object):
     connectionUrl = self._InsteadOf()
     return _preconnect(connectionUrl)
 
-  @property
-  def ReviewProtocol(self):
-    if self._review_protocol is None:
+  def ReviewUrl(self, userEmail):
+    if self._review_url is None:
       if self.review is None:
         return None
 
@@ -551,57 +564,47 @@ class Remote(object):
         u = 'http://%s' % u
       if u.endswith('/Gerrit'):
         u = u[:len(u) - len('/Gerrit')]
-      if not u.endswith('/ssh_info'):
-        if not u.endswith('/'):
-          u += '/'
-        u += 'ssh_info'
+      if u.endswith('/ssh_info'):
+        u = u[:len(u) - len('/ssh_info')]
+      if not u.endswith('/'):
+         u += '/'
+      http_url = u
 
       if u in REVIEW_CACHE:
-        info = REVIEW_CACHE[u]
-        self._review_protocol = info[0]
-        self._review_host = info[1]
-        self._review_port = info[2]
+        self._review_url = REVIEW_CACHE[u]
+      elif 'REPO_HOST_PORT_INFO' in os.environ:
+        host, port = os.environ['REPO_HOST_PORT_INFO'].split()
+        self._review_url = self._SshReviewUrl(userEmail, host, port)
+        REVIEW_CACHE[u] = self._review_url
       else:
         try:
-          info = urlopen(u).read()
-          if info == 'NOT_AVAILABLE':
-            raise UploadError('%s: SSH disabled' % self.review)
+          info_url = u + 'ssh_info'
+          info = urllib2.urlopen(info_url).read()
           if '<' in info:
             # Assume the server gave us some sort of HTML
             # response back, like maybe a login page.
             #
-            raise UploadError('%s: Cannot parse response' % u)
+            raise UploadError('%s: Cannot parse response' % info_url)
 
-          self._review_protocol = 'ssh'
-          self._review_host = info.split(" ")[0]
-          self._review_port = info.split(" ")[1]
-        except urllib2.URLError, e:
-          raise UploadError('%s: %s' % (self.review, e.reason[1]))
-        except HTTPError, e:
-          if e.code == 404:
-            self._review_protocol = 'http-post'
-            self._review_host = None
-            self._review_port = None
+          if info == 'NOT_AVAILABLE':
+            # Assume HTTP if SSH is not enabled.
+            self._review_url = http_url + 'p/'
           else:
-            raise UploadError('Upload over ssh unavailable')
+            host, port = info.split()
+            self._review_url = self._SshReviewUrl(userEmail, host, port)
+        except urllib2.HTTPError, e:
+          raise UploadError('%s: %s' % (self.review, str(e)))
+        except urllib2.URLError, e:
+          raise UploadError('%s: %s' % (self.review, str(e)))
 
-        REVIEW_CACHE[u] = (
-          self._review_protocol,
-          self._review_host,
-          self._review_port)
-    return self._review_protocol
+        REVIEW_CACHE[u] = self._review_url
+    return self._review_url + self.projectname
 
-  def SshReviewUrl(self, userEmail):
-    if self.ReviewProtocol != 'ssh':
-      return None
+  def _SshReviewUrl(self, userEmail, host, port):
     username = self._config.GetString('review.%s.username' % self.review)
     if username is None:
-      username = userEmail.split("@")[0]
-    return 'ssh://%s@%s:%s/%s' % (
-      username,
-      self._review_host,
-      self._review_port,
-      self.projectname)
+      username = userEmail.split('@')[0]
+    return 'ssh://%s@%s:%s/' % (username, host, port)
 
   def ToLocal(self, rev):
     """Convert a remote revision string to something we have locally.

git_refs.py

@@ -139,13 +139,15 @@ class GitRefs(object):
   def _ReadLoose1(self, path, name):
     try:
       fd = open(path, 'rb')
-      mtime = os.path.getmtime(path)
-    except OSError:
-      return
-    except IOError:
+    except:
       return
+
     try:
-      id = fd.readline()
+      try:
+        mtime = os.path.getmtime(path)
+        id = fd.readline()
+      except:
+        return
     finally:
       fd.close()
 

hooks/pre-auto-gc

@@ -38,6 +38,11 @@ elif test -x /usr/bin/pmset && /usr/bin/pmset -g batt |
 	grep -q "Currently drawing from 'AC Power'"
 then
 	exit 0
+elif test -d /sys/bus/acpi/drivers/battery && test 0 = \
+  "$(find /sys/bus/acpi/drivers/battery/ -type l | wc -l)";
+then
+	# No battery exists.
+	exit 0
 fi
 
 echo "Auto packing deferred; not on AC"

main.py

@@ -22,17 +22,23 @@ if __name__ == '__main__':
     del sys.argv[-1]
 del magic
 
+import netrc
 import optparse
 import os
 import re
 import sys
+import time
+import urllib2
 
 from trace import SetTrace
+from git_command import git, GitCommand
 from git_config import init_ssh, close_ssh
 from command import InteractiveCommand
 from command import MirrorSafeCommand
 from command import PagedCommand
+from subcmds.version import Version
 from editor import Editor
+from error import DownloadError
 from error import ManifestInvalidRevisionError
 from error import NoSuchProjectError
 from error import RepoChangedException
@@ -53,6 +59,9 @@ global_options.add_option('--no-pager',
 global_options.add_option('--trace',
                           dest='trace', action='store_true',
                           help='trace git command execution')
+global_options.add_option('--time',
+                          dest='time', action='store_true',
+                          help='time repo command execution')
 global_options.add_option('--version',
                           dest='show_version', action='store_true',
                           help='display this version of repo')
@@ -65,6 +74,7 @@ class _Repo(object):
     all_commands['branch'] = all_commands['branches']
 
   def _Run(self, argv):
+    result = 0
     name = None
     glob = []
 
@@ -88,7 +98,7 @@ class _Repo(object):
         name = 'version'
       else:
         print >>sys.stderr, 'fatal: invalid usage of --version'
-        sys.exit(1)
+        return 1
 
     try:
       cmd = self.commands[name]
@@ -96,7 +106,7 @@ class _Repo(object):
       print >>sys.stderr,\
             "repo: '%s' is not a repo command.  See 'repo help'."\
             % name
-      sys.exit(1)
+      return 1
 
     cmd.repodir = self.repodir
     cmd.manifest = XmlManifest(cmd.repodir)
@@ -106,7 +116,7 @@ class _Repo(object):
       print >>sys.stderr, \
             "fatal: '%s' requires a working directory"\
             % name
-      sys.exit(1)
+      return 1
 
     copts, cargs = cmd.OptionParser.parse_args(argv)
 
@@ -122,16 +132,37 @@ class _Repo(object):
         RunPager(config)
 
     try:
-      cmd.Execute(copts, cargs)
+      start = time.time()
+      try:
+        result = cmd.Execute(copts, cargs)
+      finally:
+        elapsed = time.time() - start
+        hours, remainder = divmod(elapsed, 3600)
+        minutes, seconds = divmod(remainder, 60)
+        if gopts.time:
+          if hours == 0:
+            print >>sys.stderr, 'real\t%dm%.3fs' \
+              % (minutes, seconds)
+          else:
+            print >>sys.stderr, 'real\t%dh%dm%.3fs' \
+              % (hours, minutes, seconds)
+    except DownloadError, e:
+      print >>sys.stderr, 'error: %s' % str(e)
+      return 1
     except ManifestInvalidRevisionError, e:
       print >>sys.stderr, 'error: %s' % str(e)
-      sys.exit(1)
+      return 1
     except NoSuchProjectError, e:
       if e.name:
         print >>sys.stderr, 'error: project %s not found' % e.name
       else:
         print >>sys.stderr, 'error: no project in current directory'
-      sys.exit(1)
+      return 1
+
+    return result
+
+def _MyRepoPath():
+  return os.path.dirname(__file__)
 
 def _MyWrapperPath():
   return os.path.join(os.path.dirname(__file__), 'repo')
@@ -199,7 +230,117 @@ def _PruneOptions(argv, opt):
       continue
     i += 1
 
+_user_agent = None
+
+def _UserAgent():
+  global _user_agent
+
+  if _user_agent is None:
+    py_version = sys.version_info
+
+    os_name = sys.platform
+    if os_name == 'linux2':
+      os_name = 'Linux'
+    elif os_name == 'win32':
+      os_name = 'Win32'
+    elif os_name == 'cygwin':
+      os_name = 'Cygwin'
+    elif os_name == 'darwin':
+      os_name = 'Darwin'
+
+    p = GitCommand(
+      None, ['describe', 'HEAD'],
+      cwd = _MyRepoPath(),
+      capture_stdout = True)
+    if p.Wait() == 0:
+      repo_version = p.stdout
+      if len(repo_version) > 0 and repo_version[-1] == '\n':
+        repo_version = repo_version[0:-1]
+      if len(repo_version) > 0 and repo_version[0] == 'v':
+        repo_version = repo_version[1:]
+    else:
+      repo_version = 'unknown'
+
+    _user_agent = 'git-repo/%s (%s) git/%s Python/%d.%d.%d' % (
+      repo_version,
+      os_name,
+      '.'.join(map(lambda d: str(d), git.version_tuple())),
+      py_version[0], py_version[1], py_version[2])
+  return _user_agent
+
+class _UserAgentHandler(urllib2.BaseHandler):
+  def http_request(self, req):
+    req.add_header('User-Agent', _UserAgent())
+    return req
+
+  def https_request(self, req):
+    req.add_header('User-Agent', _UserAgent())
+    return req
+
+class _BasicAuthHandler(urllib2.HTTPBasicAuthHandler):
+  def http_error_auth_reqed(self, authreq, host, req, headers):
+    try:
+      old_add_header = req.add_header
+      def _add_header(name, val):
+        val = val.replace('\n', '')
+        old_add_header(name, val)
+      req.add_header = _add_header
+      return urllib2.AbstractBasicAuthHandler.http_error_auth_reqed(
+        self, authreq, host, req, headers)
+    except:
+      reset = getattr(self, 'reset_retry_count', None)
+      if reset is not None:
+        reset()
+      elif getattr(self, 'retried', None):
+        self.retried = 0
+      raise
+
+class _DigestAuthHandler(urllib2.HTTPDigestAuthHandler):
+  def http_error_auth_reqed(self, auth_header, host, req, headers):
+    try:
+      old_add_header = req.add_header
+      def _add_header(name, val):
+        val = val.replace('\n', '')
+        old_add_header(name, val)
+      req.add_header = _add_header
+      return urllib2.AbstractDigestAuthHandler.http_error_auth_reqed(
+        self, auth_header, host, req, headers)
+    except:
+      reset = getattr(self, 'reset_retry_count', None)
+      if reset is not None:
+        reset()
+      elif getattr(self, 'retried', None):
+        self.retried = 0
+      raise
+
+def init_http():
+  handlers = [_UserAgentHandler()]
+
+  mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
+  try:
+    n = netrc.netrc()
+    for host in n.hosts:
+      p = n.hosts[host]
+      mgr.add_password(p[1], 'http://%s/'  % host, p[0], p[2])
+      mgr.add_password(p[1], 'https://%s/' % host, p[0], p[2])
+  except netrc.NetrcParseError:
+    pass
+  except IOError:
+    pass
+  handlers.append(_BasicAuthHandler(mgr))
+  handlers.append(_DigestAuthHandler(mgr))
+
+  if 'http_proxy' in os.environ:
+    url = os.environ['http_proxy']
+    handlers.append(urllib2.ProxyHandler({'http': url, 'https': url}))
+  if 'REPO_CURL_VERBOSE' in os.environ:
+    handlers.append(urllib2.HTTPHandler(debuglevel=1))
+    handlers.append(urllib2.HTTPSHandler(debuglevel=1))
+  urllib2.install_opener(urllib2.build_opener(*handlers))
+
 def _Main(argv):
+  result = 0
+
   opt = optparse.OptionParser(usage="repo wrapperinfo -- ...")
   opt.add_option("--repo-dir", dest="repodir",
                  help="path to .repo/")
@@ -213,15 +354,19 @@ def _Main(argv):
   _CheckWrapperVersion(opt.wrapper_version, opt.wrapper_path)
   _CheckRepoDir(opt.repodir)
 
+  Version.wrapper_version = opt.wrapper_version
+  Version.wrapper_path = opt.wrapper_path
+
   repo = _Repo(opt.repodir)
   try:
     try:
       init_ssh()
-      repo._Run(argv)
+      init_http()
+      result = repo._Run(argv) or 0
     finally:
       close_ssh()
   except KeyboardInterrupt:
-    sys.exit(1)
+    result = 1
   except RepoChangedException, rce:
     # If repo changed, re-exec ourselves.
     #
@@ -232,7 +377,9 @@ def _Main(argv):
     except OSError, e:
       print >>sys.stderr, 'fatal: cannot restart repo after upgrade'
       print >>sys.stderr, 'fatal: %s' % e
-      sys.exit(128)
+      result = 128
+
+  sys.exit(result)
 
 if __name__ == '__main__':
   _Main(sys.argv[1:])

manifest_xml.py

@@ -13,8 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import itertools
 import os
+import re
 import sys
+import urlparse
 import xml.dom.minidom
 
 from git_config import GitConfig, IsId
@@ -24,27 +27,47 @@ from error import ManifestParseError
 MANIFEST_FILE_NAME = 'manifest.xml'
 LOCAL_MANIFEST_NAME = 'local_manifest.xml'
 
+urlparse.uses_relative.extend(['ssh', 'git'])
+urlparse.uses_netloc.extend(['ssh', 'git'])
+
 class _Default(object):
   """Project defaults within the manifest."""
 
   revisionExpr = None
   remote = None
+  sync_j = 1
+  sync_c = False
 
 class _XmlRemote(object):
   def __init__(self,
                name,
+               alias=None,
                fetch=None,
+               manifestUrl=None,
                review=None):
     self.name = name
     self.fetchUrl = fetch
+    self.manifestUrl = manifestUrl
+    self.remoteAlias = alias
     self.reviewUrl = review
+    self.resolvedFetchUrl = self._resolveFetchUrl()
+
+  def _resolveFetchUrl(self):
+    url = self.fetchUrl.rstrip('/')
+    manifestUrl = self.manifestUrl.rstrip('/')
+    # urljoin will get confused if there is no scheme in the base url
+    # ie, if manifestUrl is of the form <hostname:port>
+    if manifestUrl.find(':') != manifestUrl.find('/') - 1:
+        manifestUrl = 'gopher://' + manifestUrl
+    url = urlparse.urljoin(manifestUrl, url)
+    return re.sub(r'^gopher://', '', url)
 
   def ToRemoteSpec(self, projectName):
-    url = self.fetchUrl
-    while url.endswith('/'):
-      url = url[:-1]
-    url += '/%s.git' % projectName
-    return RemoteSpec(self.name, url, self.reviewUrl)
+    url = self.resolvedFetchUrl.rstrip('/') + '/' + projectName
+    remoteName = self.name
+    if self.remoteAlias:
+      remoteName = self.remoteAlias
+    return RemoteSpec(remoteName, url, self.reviewUrl)
 
 class XmlManifest(object):
   """manages the repo configuration file"""
@@ -103,6 +126,13 @@ class XmlManifest(object):
   def Save(self, fd, peg_rev=False):
     """Write the current manifest out to the given file descriptor.
     """
+    mp = self.manifestProject
+
+    groups = mp.config.GetString('manifest.groups')
+    if not groups:
+      groups = 'default'
+    groups = [x for x in re.split(r'[,\s]+', groups) if x]
+
     doc = xml.dom.minidom.Document()
     root = doc.createElement('manifest')
     doc.appendChild(root)
@@ -133,6 +163,12 @@ class XmlManifest(object):
     if d.revisionExpr:
       have_default = True
       e.setAttribute('revision', d.revisionExpr)
+    if d.sync_j > 1:
+      have_default = True
+      e.setAttribute('sync-j', '%d' % d.sync_j)
+    if d.sync_c:
+      have_default = True
+      e.setAttribute('sync-c', 'true')
     if have_default:
       root.appendChild(e)
       root.appendChild(doc.createTextNode(''))
@@ -148,6 +184,10 @@ class XmlManifest(object):
 
     for p in sort_projects:
       p = self.projects[p]
+
+      if not p.MatchesGroups(groups):
+        continue
+
       e = doc.createElement('project')
       root.appendChild(e)
       e.setAttribute('name', p.name)
@@ -171,6 +211,20 @@ class XmlManifest(object):
         ce.setAttribute('dest', c.dest)
         e.appendChild(ce)
 
+      egroups = [g for g in p.groups if g != 'default']
+      if egroups:
+        e.setAttribute('groups', ','.join(egroups))
+
+      for a in p.annotations:
+        if a.keep == "true":
+          ae = doc.createElement('annotation')
+          ae.setAttribute('name', a.name)
+          ae.setAttribute('value', a.value)
+          e.appendChild(ae)
+
+      if p.sync_c:
+        e.setAttribute('sync-c', 'true')
+
     if self._repo_hooks_project:
       root.appendChild(doc.createTextNode(''))
       e = doc.createElement('repo-hooks')
@@ -233,16 +287,15 @@ class XmlManifest(object):
         b = b[len(R_HEADS):]
       self.branch = b
 
-      self._ParseManifest(True)
+      nodes = []
+      nodes.append(self._ParseManifestXml(self.manifestFile,
+                                          self.manifestProject.worktree))
 
       local = os.path.join(self.repodir, LOCAL_MANIFEST_NAME)
       if os.path.exists(local):
-        try:
-          real = self.manifestFile
-          self.manifestFile = local
-          self._ParseManifest(False)
-        finally:
-          self.manifestFile = real
+        nodes.append(self._ParseManifestXml(local, self.repodir))
+
+      self._ParseManifest(nodes)
 
       if self.IsMirror:
         self._AddMetaProjectMirror(self.repoProject)
@@ -250,35 +303,39 @@ class XmlManifest(object):
 
       self._loaded = True
 
-  def _ParseManifest(self, is_root_file):
-    root = xml.dom.minidom.parse(self.manifestFile)
+  def _ParseManifestXml(self, path, include_root):
+    root = xml.dom.minidom.parse(path)
     if not root or not root.childNodes:
-      raise ManifestParseError(
-          "no root node in %s" %
-          self.manifestFile)
+      raise ManifestParseError("no root node in %s" % (path,))
 
     config = root.childNodes[0]
     if config.nodeName != 'manifest':
-      raise ManifestParseError(
-          "no <manifest> in %s" %
-          self.manifestFile)
+      raise ManifestParseError("no <manifest> in %s" % (path,))
 
+    nodes = []
     for node in config.childNodes:
-      if node.nodeName == 'remove-project':
-        name = self._reqatt(node, 'name')
-        try:
-          del self._projects[name]
-        except KeyError:
-          raise ManifestParseError(
-              'project %s not found' %
-              (name))
+        if node.nodeName == 'include':
+            name = self._reqatt(node, 'name')
+            fp = os.path.join(include_root, name)
+            if not os.path.isfile(fp):
+                raise ManifestParseError, \
+                    "include %s doesn't exist or isn't a file" % \
+                    (name,)
+            try:
+                nodes.extend(self._ParseManifestXml(fp, include_root))
+            # should isolate this to the exact exception, but that's
+            # tricky.  actual parsing implementation may vary.
+            except (KeyboardInterrupt, RuntimeError, SystemExit):
+                raise
+            except Exception, e:
+                raise ManifestParseError(
+                    "failed parsing included manifest %s: %s", (name, e))
+        else:
+          nodes.append(node)
+    return nodes
 
-        # If the manifest removes the hooks project, treat it as if it deleted
-        # the repo-hooks element too.
-        if self._repo_hooks_project and (self._repo_hooks_project.name == name):
-          self._repo_hooks_project = None
-
-    for node in config.childNodes:
+  def _ParseManifest(self, node_list):
+    for node in itertools.chain(*node_list):
       if node.nodeName == 'remote':
         remote = self._ParseRemote(node)
         if self._remotes.get(remote.name):
@@ -287,7 +344,7 @@ class XmlManifest(object):
               (remote.name, self.manifestFile))
         self._remotes[remote.name] = remote
 
-    for node in config.childNodes:
+    for node in itertools.chain(*node_list):
       if node.nodeName == 'default':
         if self._default is not None:
           raise ManifestParseError(
@@ -297,7 +354,7 @@ class XmlManifest(object):
     if self._default is None:
       self._default = _Default()
 
-    for node in config.childNodes:
+    for node in itertools.chain(*node_list):
       if node.nodeName == 'notice':
         if self._notice is not None:
           raise ManifestParseError(
@@ -305,7 +362,7 @@ class XmlManifest(object):
               (self.manifestFile))
         self._notice = self._ParseNotice(node)
 
-    for node in config.childNodes:
+    for node in itertools.chain(*node_list):
       if node.nodeName == 'manifest-server':
         url = self._reqatt(node, 'url')
         if self._manifest_server is not None:
@@ -314,7 +371,7 @@ class XmlManifest(object):
                 (self.manifestFile))
         self._manifest_server = url
 
-    for node in config.childNodes:
+    for node in itertools.chain(*node_list):
       if node.nodeName == 'project':
         project = self._ParseProject(node)
         if self._projects.get(project.name):
@@ -322,8 +379,6 @@ class XmlManifest(object):
               'duplicate project %s in %s' %
               (project.name, self.manifestFile))
         self._projects[project.name] = project
-
-    for node in config.childNodes:
       if node.nodeName == 'repo-hooks':
         # Get the name of the project and the (space-separated) list of enabled.
         repo_hooks_project = self._reqatt(node, 'in-project')
@@ -345,6 +400,20 @@ class XmlManifest(object):
 
         # Store the enabled hooks in the Project object.
         self._repo_hooks_project.enabled_repo_hooks = enabled_repo_hooks
+      if node.nodeName == 'remove-project':
+        name = self._reqatt(node, 'name')
+        try:
+          del self._projects[name]
+        except KeyError:
+          raise ManifestParseError(
+              'project %s not found' %
+              (name))
+
+        # If the manifest removes the hooks project, treat it as if it deleted
+        # the repo-hooks element too.
+        if self._repo_hooks_project and (self._repo_hooks_project.name == name):
+          self._repo_hooks_project = None
+
 
   def _AddMetaProjectMirror(self, m):
     name = None
@@ -353,7 +422,7 @@ class XmlManifest(object):
       raise ManifestParseError, 'refusing to mirror %s' % m_url
 
     if self._default and self._default.remote:
-      url = self._default.remote.fetchUrl
+      url = self._default.remote.resolvedFetchUrl
       if not url.endswith('/'):
         url += '/'
       if m_url.startswith(url):
@@ -362,7 +431,8 @@ class XmlManifest(object):
 
     if name is None:
       s = m_url.rindex('/') + 1
-      remote = _XmlRemote('origin', m_url[:s])
+      manifestUrl = self.manifestProject.config.GetString('remote.origin.url')
+      remote = _XmlRemote('origin', fetch=m_url[:s], manifestUrl=manifestUrl)
       name = m_url[s:]
 
     if name.endswith('.git'):
@@ -386,11 +456,15 @@ class XmlManifest(object):
     reads a <remote> element from the manifest file
     """
     name = self._reqatt(node, 'name')
+    alias = node.getAttribute('alias')
+    if alias == '':
+      alias = None
     fetch = self._reqatt(node, 'fetch')
     review = node.getAttribute('review')
     if review == '':
       review = None
-    return _XmlRemote(name, fetch, review)
+    manifestUrl = self.manifestProject.config.GetString('remote.origin.url')
+    return _XmlRemote(name, alias, fetch, manifestUrl, review)
 
   def _ParseDefault(self, node):
     """
@@ -401,6 +475,18 @@ class XmlManifest(object):
     d.revisionExpr = node.getAttribute('revision')
     if d.revisionExpr == '':
       d.revisionExpr = None
+
+    sync_j = node.getAttribute('sync-j')
+    if sync_j == '' or sync_j is None:
+      d.sync_j = 1
+    else:
+      d.sync_j = int(sync_j)
+
+    sync_c = node.getAttribute('sync-c')
+    if not sync_c:
+      d.sync_c = False
+    else:
+      d.sync_c = sync_c.lower() in ("yes", "true", "1")
     return d
 
   def _ParseNotice(self, node):
@@ -472,6 +558,26 @@ class XmlManifest(object):
             "project %s path cannot be absolute in %s" % \
             (name, self.manifestFile)
 
+    rebase = node.getAttribute('rebase')
+    if not rebase:
+      rebase = True
+    else:
+      rebase = rebase.lower() in ("yes", "true", "1")
+
+    sync_c = node.getAttribute('sync-c')
+    if not sync_c:
+      sync_c = False
+    else:
+      sync_c = sync_c.lower() in ("yes", "true", "1")
+
+    groups = ''
+    if node.hasAttribute('groups'):
+      groups = node.getAttribute('groups')
+    groups = [x for x in re.split('[,\s]+', groups) if x]
+
+    default_groups = ['default', 'name:%s' % name, 'path:%s' % path]
+    groups.extend(set(default_groups).difference(groups))
+
     if self.IsMirror:
       relpath = None
       worktree = None
@@ -487,11 +593,16 @@ class XmlManifest(object):
                       worktree = worktree,
                       relpath = path,
                       revisionExpr = revisionExpr,
-                      revisionId = None)
+                      revisionId = None,
+                      rebase = rebase,
+                      groups = groups,
+                      sync_c = sync_c)
 
     for n in node.childNodes:
       if n.nodeName == 'copyfile':
         self._ParseCopyFile(project, n)
+      if n.nodeName == 'annotation':
+        self._ParseAnnotation(project, n)
 
     return project
 
@@ -503,6 +614,17 @@ class XmlManifest(object):
       # dest is relative to the top of the tree
       project.AddCopyFile(src, dest, os.path.join(self.topdir, dest))
 
+  def _ParseAnnotation(self, project, node):
+    name = self._reqatt(node, 'name')
+    value = self._reqatt(node, 'value')
+    try:
+      keep = self._reqatt(node, 'keep').lower()
+    except ManifestParseError:
+      keep = "true"
+    if keep != "true" and keep != "false":
+      raise ManifestParseError, "optional \"keep\" attribute must be \"true\" or \"false\""
+    project.AddAnnotation(name, value, keep)
+
   def _get_remote(self, node):
     name = node.getAttribute('remote')
     if not name:

progress.py

@@ -21,13 +21,14 @@ from trace import IsTrace
 _NOT_TTY = not os.isatty(2)
 
 class Progress(object):
-  def __init__(self, title, total=0):
+  def __init__(self, title, total=0, units=''):
     self._title = title
     self._total = total
     self._done = 0
     self._lastp = -1
     self._start = time()
     self._show = False
+    self._units = units
 
   def update(self, inc=1):
     self._done += inc
@@ -51,11 +52,11 @@ class Progress(object):
 
       if self._lastp != p:
         self._lastp = p
-        sys.stderr.write('\r%s: %3d%% (%d/%d)  ' % (
+        sys.stderr.write('\r%s: %3d%% (%d%s/%d%s)  ' % (
           self._title,
           p,
-          self._done,
-          self._total))
+          self._done, self._units,
+          self._total, self._units))
         sys.stderr.flush()
 
   def end(self):
@@ -69,9 +70,9 @@ class Progress(object):
       sys.stderr.flush()
     else:
       p = (100 * self._done) / self._total
-      sys.stderr.write('\r%s: %3d%% (%d/%d), done.  \n' % (
+      sys.stderr.write('\r%s: %3d%% (%d%s/%d%s), done.  \n' % (
         self._title,
         p,
-        self._done,
-        self._total))
+        self._done, self._units,
+        self._total, self._units))
       sys.stderr.flush()

project.py

@@ -16,17 +16,22 @@ import traceback
 import errno
 import filecmp
 import os
+import random
 import re
 import shutil
 import stat
+import subprocess
 import sys
-import urllib2
+import time
 
 from color import Coloring
 from git_command import GitCommand
-from git_config import GitConfig, IsId
+from git_config import GitConfig, IsId, GetSchemeFromUrl, ID_RE
+from error import DownloadError
 from error import GitError, HookError, ImportError, UploadError
 from error import ManifestInvalidRevisionError
+from progress import Progress
+from trace import IsTrace, Trace
 
 from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M
 
@@ -160,10 +165,11 @@ class ReviewableBranch(object):
       R_HEADS + self.name,
       '--')
 
-  def UploadForReview(self, people, auto_topic=False):
+  def UploadForReview(self, people, auto_topic=False, draft=False):
     self.project.UploadForReview(self.name,
                                  people,
-                                 auto_topic=auto_topic)
+                                 auto_topic=auto_topic,
+                                 draft=draft)
 
   def GetPublishedRefs(self):
     refs = {}
@@ -197,6 +203,11 @@ class DiffColoring(Coloring):
     Coloring.__init__(self, config, 'diff')
     self.project   = self.printer('header',    attr = 'bold')
 
+class _Annotation:
+  def __init__(self, name, value, keep):
+    self.name = name
+    self.value = value
+    self.keep = keep
 
 class _CopyFile:
   def __init__(self, src, dest, abssrc, absdest):
@@ -487,7 +498,10 @@ class Project(object):
                worktree,
                relpath,
                revisionExpr,
-               revisionId):
+               revisionId,
+               rebase = True,
+               groups = None,
+               sync_c = False):
     self.manifest = manifest
     self.name = name
     self.remote = remote
@@ -506,8 +520,13 @@ class Project(object):
     else:
       self.revisionId = revisionId
 
+    self.rebase = rebase
+    self.groups = groups
+    self.sync_c = sync_c
+
     self.snapshots = {}
     self.copyfiles = []
+    self.annotations = []
     self.config = GitConfig.ForRepository(
                     gitdir = self.gitdir,
                     defaults =  self.manifest.globalConfig)
@@ -626,6 +645,27 @@ class Project(object):
 
     return heads
 
+  def MatchesGroups(self, manifest_groups):
+    """Returns true if the manifest groups specified at init should cause
+       this project to be synced.
+       Prefixing a manifest group with "-" inverts the meaning of a group.
+       All projects are implicitly labelled with "default".
+
+       labels are resolved in order.  In the example case of
+       project_groups: "default,group1,group2"
+       manifest_groups: "-group1,group2"
+       the project will be matched.
+    """
+    if self.groups is None:
+      return True
+    matched = False
+    for group in manifest_groups:
+      if group.startswith('-') and group[1:] in self.groups:
+        matched = False
+      elif group in self.groups:
+        matched = True
+
+    return matched
 
 ## Status Display ##
 
@@ -650,13 +690,18 @@ class Project(object):
 
     return False
 
-  def PrintWorkTreeStatus(self):
+  def PrintWorkTreeStatus(self, output_redir=None):
     """Prints the status of the repository to stdout.
+
+    Args:
+      output: If specified, redirect the output to this object.
     """
     if not os.path.isdir(self.worktree):
-      print ''
-      print 'project %s/' % self.relpath
-      print '  missing (run "repo sync")'
+      if output_redir == None:
+        output_redir = sys.stdout
+      print >>output_redir, ''
+      print >>output_redir, 'project %s/' % self.relpath
+      print >>output_redir, '  missing (run "repo sync")'
       return
 
     self.work_git.update_index('-q',
@@ -667,10 +712,12 @@ class Project(object):
     di = self.work_git.DiffZ('diff-index', '-M', '--cached', HEAD)
     df = self.work_git.DiffZ('diff-files')
     do = self.work_git.LsOthers()
-    if not rb and not di and not df and not do:
+    if not rb and not di and not df and not do and not self.CurrentBranch:
       return 'CLEAN'
 
     out = StatusColoring(self.config)
+    if not output_redir == None:
+      out.redirect(output_redir)
     out.project('project %-40s', self.relpath + '/')
 
     branch = self.CurrentBranch
@@ -720,9 +767,10 @@ class Project(object):
       else:
         out.write('%s', line)
       out.nl()
+
     return 'DIRTY'
 
-  def PrintWorkTreeDiff(self):
+  def PrintWorkTreeDiff(self, absolute_paths=False):
     """Prints the status of the repository to stdout.
     """
     out = DiffColoring(self.config)
@@ -730,6 +778,9 @@ class Project(object):
     if out.is_on:
       cmd.append('--color')
     cmd.append(HEAD)
+    if absolute_paths:
+      cmd.append('--src-prefix=a/%s/' % self.relpath)
+      cmd.append('--dst-prefix=b/%s/' % self.relpath)
     cmd.append('--')
     p = GitCommand(self,
                    cmd,
@@ -783,7 +834,7 @@ class Project(object):
       if R_HEADS + n not in heads:
         self.bare_git.DeleteRef(name, id)
 
-  def GetUploadableBranches(self):
+  def GetUploadableBranches(self, selected_branch=None):
     """List any branches which can be uploaded for review.
     """
     heads = {}
@@ -799,6 +850,8 @@ class Project(object):
     for branch, id in heads.iteritems():
       if branch in pubed and pubed[branch] == id:
         continue
+      if selected_branch and branch != selected_branch:
+        continue
 
       rb = self.GetUploadableBranch(branch)
       if rb:
@@ -818,7 +871,8 @@ class Project(object):
 
   def UploadForReview(self, branch=None,
                       people=([],[]),
-                      auto_topic=False):
+                      auto_topic=False,
+                      draft=False):
     """Uploads the named branch for code review.
     """
     if branch is None:
@@ -840,31 +894,36 @@ class Project(object):
       branch.remote.projectname = self.name
       branch.remote.Save()
 
-    if branch.remote.ReviewProtocol == 'ssh':
-      if dest_branch.startswith(R_HEADS):
-        dest_branch = dest_branch[len(R_HEADS):]
+    url = branch.remote.ReviewUrl(self.UserEmail)
+    if url is None:
+      raise UploadError('review not configured')
+    cmd = ['push']
 
+    if url.startswith('ssh://'):
       rp = ['gerrit receive-pack']
       for e in people[0]:
         rp.append('--reviewer=%s' % sq(e))
       for e in people[1]:
         rp.append('--cc=%s' % sq(e))
-
-      ref_spec = '%s:refs/for/%s' % (R_HEADS + branch.name, dest_branch)
-      if auto_topic:
-        ref_spec = ref_spec + '/' + branch.name
-
-      cmd = ['push']
       cmd.append('--receive-pack=%s' % " ".join(rp))
-      cmd.append(branch.remote.SshReviewUrl(self.UserEmail))
-      cmd.append(ref_spec)
 
-      if GitCommand(self, cmd, bare = True).Wait() != 0:
-        raise UploadError('Upload failed')
+    cmd.append(url)
 
-    else:
-        raise UploadError('Unsupported protocol %s' \
-          % branch.remote.review)
+    if dest_branch.startswith(R_HEADS):
+      dest_branch = dest_branch[len(R_HEADS):]
+
+    upload_type = 'for'
+    if draft:
+      upload_type = 'drafts'
+
+    ref_spec = '%s:refs/%s/%s' % (R_HEADS + branch.name, upload_type,
+                                  dest_branch)
+    if auto_topic:
+      ref_spec = ref_spec + '/' + branch.name
+    cmd.append(ref_spec)
+
+    if GitCommand(self, cmd, bare = True).Wait() != 0:
+      raise UploadError('Upload failed')
 
     msg = "posted to %s for %s" % (branch.remote.review, dest_branch)
     self.bare_git.UpdateRef(R_PUB + branch.name,
@@ -874,32 +933,50 @@ class Project(object):
 
 ## Sync ##
 
-  def Sync_NetworkHalf(self, quiet=False):
+  def Sync_NetworkHalf(self,
+      quiet=False,
+      is_new=None,
+      current_branch_only=False,
+      clone_bundle=True):
     """Perform only the network IO portion of the sync process.
        Local working directory/branch state is not affected.
     """
-    is_new = not self.Exists
+    if is_new is None:
+      is_new = not self.Exists
     if is_new:
-      if not quiet:
-        print >>sys.stderr
-        print >>sys.stderr, 'Initializing project %s ...' % self.name
       self._InitGitDir()
-
     self._InitRemote()
-    if not self._RemoteFetch(initial=is_new, quiet=quiet):
-      return False
 
-    #Check that the requested ref was found after fetch
-    #
-    try:
-      self.GetRevisionId()
-    except ManifestInvalidRevisionError:
-      # if the ref is a tag. We can try fetching
-      # the tag manually as a last resort
-      #
-      rev = self.revisionExpr
-      if rev.startswith(R_TAGS):
-        self._RemoteFetch(None, rev[len(R_TAGS):], quiet=quiet)
+    if is_new:
+      alt = os.path.join(self.gitdir, 'objects/info/alternates')
+      try:
+        fd = open(alt, 'rb')
+        try:
+          alt_dir = fd.readline().rstrip()
+        finally:
+          fd.close()
+      except IOError:
+        alt_dir = None
+    else:
+      alt_dir = None
+
+    if clone_bundle \
+    and alt_dir is None \
+    and self._ApplyCloneBundle(initial=is_new, quiet=quiet):
+      is_new = False
+
+    if not current_branch_only:
+      if self.sync_c:
+        current_branch_only = True
+      elif not self.manifest._loaded:
+        # Manifest cannot check defaults until it syncs.
+        current_branch_only = False
+      elif self.manifest.default.sync_c:
+        current_branch_only = True
+
+    if not self._RemoteFetch(initial=is_new, quiet=quiet, alt_dir=alt_dir,
+                             current_branch_only=current_branch_only):
+      return False
 
     if self.worktree:
       self._InitMRef()
@@ -964,12 +1041,15 @@ class Project(object):
 
       if head == revid:
         # No changes; don't do anything further.
+        # Except if the head needs to be detached
         #
-        return
+        if not syncbuf.detach_head:
+          return
+      else:
+        lost = self._revlist(not_rev(revid), HEAD)
+        if lost:
+          syncbuf.info(self, "discarding %d commits", len(lost))
 
-      lost = self._revlist(not_rev(revid), HEAD)
-      if lost:
-        syncbuf.info(self, "discarding %d commits", len(lost))
       try:
         self._Checkout(revid, quiet=True)
       except GitError, e:
@@ -987,7 +1067,7 @@ class Project(object):
 
     if not branch.LocalMerge:
       # The current branch has no tracking configuration.
-      # Jump off it to a deatched HEAD.
+      # Jump off it to a detached HEAD.
       #
       syncbuf.info(self,
                    "leaving %s; does not track upstream",
@@ -1065,10 +1145,12 @@ class Project(object):
                    len(local_changes) - cnt_mine)
 
     branch.remote = self.GetRemote(self.remote.name)
-    branch.merge = self.revisionExpr
+    if not ID_RE.match(self.revisionExpr):
+      # in case of manifest sync the revisionExpr might be a SHA1
+      branch.merge = self.revisionExpr
     branch.Save()
 
-    if cnt_mine > 0:
+    if cnt_mine > 0 and self.rebase:
       def _dorebase():
         self._Rebase(upstream = '%s^1' % last_mine, onto = revid)
         self._CopyFiles()
@@ -1092,6 +1174,9 @@ class Project(object):
     abssrc = os.path.join(self.worktree, src)
     self.copyfiles.append(_CopyFile(src, dest, abssrc, absdest))
 
+  def AddAnnotation(self, name, value, keep):
+    self.annotations.append(_Annotation(name, value, keep))
+
   def DownloadPatchSet(self, change_id, patch_id):
     """Download a single patch set of a single change to FETCH_HEAD.
     """
@@ -1159,6 +1244,13 @@ class Project(object):
 
   def CheckoutBranch(self, name):
     """Checkout a local topic branch.
+
+        Args:
+          name: The name of the branch to checkout.
+
+        Returns:
+          True if the checkout succeeded; False if it didn't; None if the branch
+          didn't exist.
     """
     rev = R_HEADS + name
     head = self.work_git.GetHead()
@@ -1173,7 +1265,7 @@ class Project(object):
     except KeyError:
       # Branch does not exist in this project
       #
-      return False
+      return None
 
     if head.startswith(R_HEADS):
       try:
@@ -1196,13 +1288,19 @@ class Project(object):
 
   def AbandonBranch(self, name):
     """Destroy a local topic branch.
+
+    Args:
+      name: The name of the branch to abandon.
+
+    Returns:
+      True if the abandon succeeded; False if it didn't; None if the branch
+      didn't exist.
     """
     rev = R_HEADS + name
     all = self.bare_ref.all
     if rev not in all:
-      # Doesn't exist; assume already abandoned.
-      #
-      return True
+      # Doesn't exist
+      return None
 
     head = self.work_git.GetHead()
     if head == rev:
@@ -1282,31 +1380,43 @@ class Project(object):
 
 ## Direct Git Commands ##
 
-  def _RemoteFetch(self, name=None, tag=None,
+  def _RemoteFetch(self, name=None,
+                   current_branch_only=False,
                    initial=False,
-                   quiet=False):
+                   quiet=False,
+                   alt_dir=None):
+
+    is_sha1 = False
+    tag_name = None
+
+    if current_branch_only:
+      if ID_RE.match(self.revisionExpr) is not None:
+        is_sha1 = True
+      elif self.revisionExpr.startswith(R_TAGS):
+        # this is a tag and its sha1 value should never change
+        tag_name = self.revisionExpr[len(R_TAGS):]
+
+      if is_sha1 or tag_name is not None:
+        try:
+          # if revision (sha or tag) is not present then following function
+          # throws an error.
+          self.bare_git.rev_parse('--verify', '%s^0' % self.revisionExpr)
+          return True
+        except GitError:
+          # There is no such persistent revision. We have to fetch it.
+          pass
+
     if not name:
       name = self.remote.name
 
     ssh_proxy = False
-    if self.GetRemote(name).PreConnectFetch():
+    remote = self.GetRemote(name)
+    if remote.PreConnectFetch():
       ssh_proxy = True
 
     if initial:
-      alt = os.path.join(self.gitdir, 'objects/info/alternates')
-      try:
-        fd = open(alt, 'rb')
-        try:
-          ref_dir = fd.readline()
-          if ref_dir and ref_dir.endswith('\n'):
-            ref_dir = ref_dir[:-1]
-        finally:
-          fd.close()
-      except IOError, e:
-        ref_dir = None
-
-      if ref_dir and 'objects' == os.path.basename(ref_dir):
-        ref_dir = os.path.dirname(ref_dir)
+      if alt_dir and 'objects' == os.path.basename(alt_dir):
+        ref_dir = os.path.dirname(alt_dir)
         packed_refs = os.path.join(self.gitdir, 'packed-refs')
         remote = self.GetRemote(name)
 
@@ -1342,35 +1452,130 @@ class Project(object):
             old_packed += line
 
         _lwrite(packed_refs, tmp_packed)
-
       else:
-        ref_dir = None
+        alt_dir = None
+
+    cmd = ['fetch']
+
+    # The --depth option only affects the initial fetch; after that we'll do
+    # full fetches of changes.
+    depth = self.manifest.manifestProject.config.GetString('repo.depth')
+    if depth and initial:
+      cmd.append('--depth=%s' % depth)
+
+    if quiet:
+      cmd.append('--quiet')
+    if not self.worktree:
+      cmd.append('--update-head-ok')
+    cmd.append(name)
+
+    if not current_branch_only or is_sha1:
+      # Fetch whole repo
+      cmd.append('--tags')
+      cmd.append((u'+refs/heads/*:') + remote.ToLocal('refs/heads/*'))
+    elif tag_name is not None:
+      cmd.append('tag')
+      cmd.append(tag_name)
+    else:
+      branch = self.revisionExpr
+      if branch.startswith(R_HEADS):
+        branch = branch[len(R_HEADS):]
+      cmd.append((u'+refs/heads/%s:' % branch) + remote.ToLocal('refs/heads/%s' % branch))
+
+    ok = False
+    for i in range(2):
+      if GitCommand(self, cmd, bare=True, ssh_proxy=ssh_proxy).Wait() == 0:
+        ok = True
+        break
+      time.sleep(random.randint(30, 45))
+
+    if initial:
+      if alt_dir:
+        if old_packed != '':
+          _lwrite(packed_refs, old_packed)
+        else:
+          os.remove(packed_refs)
+      self.bare_git.pack_refs('--all', '--prune')
+    return ok
+
+  def _ApplyCloneBundle(self, initial=False, quiet=False):
+    if initial and self.manifest.manifestProject.config.GetString('repo.depth'):
+      return False
+
+    remote = self.GetRemote(self.remote.name)
+    bundle_url = remote.url + '/clone.bundle'
+    bundle_url = GitConfig.ForUser().UrlInsteadOf(bundle_url)
+    if GetSchemeFromUrl(bundle_url) in ('persistent-http', 'persistent-https'):
+      bundle_url = bundle_url[len('persistent-'):]
+    if GetSchemeFromUrl(bundle_url) not in ('http', 'https'):
+      return False
+
+    bundle_dst = os.path.join(self.gitdir, 'clone.bundle')
+    bundle_tmp = os.path.join(self.gitdir, 'clone.bundle.tmp')
+
+    exist_dst = os.path.exists(bundle_dst)
+    exist_tmp = os.path.exists(bundle_tmp)
+
+    if not initial and not exist_dst and not exist_tmp:
+      return False
+
+    if not exist_dst:
+      exist_dst = self._FetchBundle(bundle_url, bundle_tmp, bundle_dst, quiet)
+    if not exist_dst:
+      return False
 
     cmd = ['fetch']
     if quiet:
       cmd.append('--quiet')
     if not self.worktree:
       cmd.append('--update-head-ok')
-    cmd.append(name)
-    if tag is not None:
-      cmd.append('tag')
-      cmd.append(tag)
-
-    ok = GitCommand(self,
-                    cmd,
-                    bare = True,
-                    ssh_proxy = ssh_proxy).Wait() == 0
-
-    if initial:
-      if ref_dir:
-        if old_packed != '':
-          _lwrite(packed_refs, old_packed)
-        else:
-          os.remove(packed_refs)
-      self.bare_git.pack_refs('--all', '--prune')
+    cmd.append(bundle_dst)
+    for f in remote.fetch:
+      cmd.append(str(f))
+    cmd.append('refs/tags/*:refs/tags/*')
 
+    ok = GitCommand(self, cmd, bare=True).Wait() == 0
+    if os.path.exists(bundle_dst):
+      os.remove(bundle_dst)
+    if os.path.exists(bundle_tmp):
+      os.remove(bundle_tmp)
     return ok
 
+  def _FetchBundle(self, srcUrl, tmpPath, dstPath, quiet):
+    if os.path.exists(dstPath):
+      os.remove(dstPath)
+
+    cmd = ['curl', '--output', tmpPath, '--netrc', '--location']
+    if quiet:
+      cmd += ['--silent']
+    if os.path.exists(tmpPath):
+      size = os.stat(tmpPath).st_size
+      if size >= 1024:
+        cmd += ['--continue-at', '%d' % (size,)]
+      else:
+        os.remove(tmpPath)
+    if 'http_proxy' in os.environ and 'darwin' == sys.platform:
+      cmd += ['--proxy', os.environ['http_proxy']]
+    cmd += [srcUrl]
+
+    if IsTrace():
+      Trace('%s', ' '.join(cmd))
+    try:
+      proc = subprocess.Popen(cmd)
+    except OSError:
+      return False
+
+    ok = proc.wait() == 0
+    if os.path.exists(tmpPath):
+      if ok and os.stat(tmpPath).st_size > 16:
+        os.rename(tmpPath, dstPath)
+        return True
+      else:
+        os.remove(tmpPath)
+        return False
+    else:
+      return False
+
   def _Checkout(self, rev, quiet=False):
     cmd = ['checkout']
     if quiet:
@@ -1381,6 +1586,23 @@ class Project(object):
       if self._allrefs:
         raise GitError('%s checkout %s ' % (self.name, rev))
 
+  def _CherryPick(self, rev, quiet=False):
+    cmd = ['cherry-pick']
+    cmd.append(rev)
+    cmd.append('--')
+    if GitCommand(self, cmd).Wait() != 0:
+      if self._allrefs:
+        raise GitError('%s cherry-pick %s ' % (self.name, rev))
+
+  def _Revert(self, rev, quiet=False):
+    cmd = ['revert']
+    cmd.append('--no-edit')
+    cmd.append(rev)
+    cmd.append('--')
+    if GitCommand(self, cmd).Wait() != 0:
+      if self._allrefs:
+        raise GitError('%s revert %s ' % (self.name, rev))
+
   def _ResetHard(self, rev, quiet=True):
     cmd = ['reset', '--hard']
     if quiet:
@@ -1397,8 +1619,10 @@ class Project(object):
     if GitCommand(self, cmd).Wait() != 0:
       raise GitError('%s rebase %s ' % (self.name, upstream))
 
-  def _FastForward(self, head):
+  def _FastForward(self, head, ffonly=False):
     cmd = ['merge', head]
+    if ffonly:
+      cmd.append("--ff-only")
     if GitCommand(self, cmd).Wait() != 0:
       raise GitError('%s merge %s ' % (self.name, head))
 
@@ -1454,10 +1678,13 @@ class Project(object):
     for stock_hook in _ProjectHooks():
       name = os.path.basename(stock_hook)
 
-      if name in ('commit-msg',) and not self.remote.review:
+      if name in ('commit-msg',) and not self.remote.review \
+            and not self is self.manifest.manifestProject:
         # Don't install a Gerrit Code Review hook if this
         # project does not appear to use it for reviews.
         #
+        # Since the manifest project is one of those, but also
+        # managed through gerrit, it's excluded
         continue
 
       dst = os.path.join(hooks, name)
@@ -1867,7 +2094,8 @@ class MetaProject(Project):
                      remote = RemoteSpec('origin'),
                      relpath = '.repo/%s' % name,
                      revisionExpr = 'refs/heads/master',
-                     revisionId = None)
+                     revisionId = None,
+                     groups = None)
 
   def PreSync(self):
     if self.Exists:
@@ -1878,6 +2106,22 @@ class MetaProject(Project):
           self.revisionExpr = base
           self.revisionId = None
 
+  def MetaBranchSwitch(self, target):
+    """ Prepare MetaProject for manifest branch switch
+    """
+
+    # detach and delete manifest branch, allowing a new
+    # branch to take over
+    syncbuf = SyncBuffer(self.config, detach_head = True)
+    self.Sync_LocalHalf(syncbuf)
+    syncbuf.Finish()
+
+    return GitCommand(self,
+                        ['update-ref', '-d', 'refs/heads/default'],
+                        capture_stdout = True,
+                        capture_stderr = True).Wait() == 0
+
+
   @property
   def LastFetch(self):
     try:

repo

@@ -2,7 +2,7 @@
 
 ## repo default configuration
 ##
-REPO_URL='git://android.git.kernel.org/tools/repo.git'
+REPO_URL='https://gerrit.googlesource.com/git-repo'
 REPO_REV='stable'
 
 # Copyright (C) 2008 Google Inc.
@@ -28,7 +28,7 @@ if __name__ == '__main__':
 del magic
 
 # increment this whenever we make important changes to this script
-VERSION = (1, 10)
+VERSION = (1, 17)
 
 # increment this if the MAINTAINER_KEYS block is modified
 KEYRING_VERSION = (1,0)
@@ -91,6 +91,7 @@ import re
 import readline
 import subprocess
 import sys
+import urllib2
 
 home_dot_repo = os.path.expanduser('~/.repoconfig')
 gpg_dir = os.path.join(home_dot_repo, 'gnupg')
@@ -121,6 +122,19 @@ group.add_option('--mirror',
 group.add_option('--reference',
                  dest='reference',
                  help='location of mirror directory', metavar='DIR')
+group.add_option('--depth', type='int', default=None,
+                 dest='depth',
+                 help='create a shallow clone with given depth; see git clone')
+group.add_option('-g', '--groups',
+                 dest='groups', default='default',
+                 help='restrict manifest projects to ones with a specified group',
+                 metavar='GROUP')
+group.add_option('-p', '--platform',
+                 dest='platform', default="auto",
+                 help='restrict manifest projects to ones with a specified'
+                      'platform group [auto|all|none|linux|darwin|...]',
+                 metavar='PLATFORM')
+
 
 # Tool
 group = init_optparse.add_option_group('repo Version options')
@@ -134,6 +148,11 @@ group.add_option('--no-repo-verify',
                  dest='no_repo_verify', action='store_true',
                  help='do not verify repo source code')
 
+# Other
+group = init_optparse.add_option_group('Other options')
+group.add_option('--config-name',
+                 dest='config_name', action="store_true", default=False,
+                 help='Always prompt for name/e-mail')
 
 class CloneFailure(Exception):
   """Indicate the remote clone of repo itself failed.
@@ -144,7 +163,7 @@ def _Init(args):
   """Installs repo by cloning it over the network.
   """
   opt, args = init_optparse.parse_args(args)
-  if args or not opt.manifest_url:
+  if args:
     init_optparse.print_usage()
     sys.exit(1)
 
@@ -183,10 +202,6 @@ def _Init(args):
     else:
       can_verify = True
 
-    if not opt.quiet:
-      print >>sys.stderr, 'Getting repo ...'
-      print >>sys.stderr, '   from %s' % url
-
     dst = os.path.abspath(os.path.join(repodir, S_repo))
     _Clone(url, dst, opt.quiet)
 
@@ -205,7 +220,17 @@ def _Init(args):
 
 def _CheckGitVersion():
   cmd = [GIT, '--version']
-  proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+  try:
+    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+  except OSError, e:
+    print >>sys.stderr
+    print >>sys.stderr, "fatal: '%s' is not available" % GIT
+    print >>sys.stderr, 'fatal: %s' % e
+    print >>sys.stderr
+    print >>sys.stderr, 'Please make sure %s is installed'\
+                        ' and in your path.' % GIT
+    raise CloneFailure()
+
   ver_str = proc.stdout.read().strip()
   proc.stdout.close()
   proc.wait()
@@ -296,15 +321,43 @@ def _SetConfig(local, name, value):
     raise CloneFailure()
 
 
-def _Fetch(local, quiet, *args):
+def _InitHttp():
+  handlers = []
+
+  mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
+  try:
+    import netrc
+    n = netrc.netrc()
+    for host in n.hosts:
+      p = n.hosts[host]
+      mgr.add_password(p[1], 'http://%s/'  % host, p[0], p[2])
+      mgr.add_password(p[1], 'https://%s/' % host, p[0], p[2])
+  except:
+    pass
+  handlers.append(urllib2.HTTPBasicAuthHandler(mgr))
+  handlers.append(urllib2.HTTPDigestAuthHandler(mgr))
+
+  if 'http_proxy' in os.environ:
+    url = os.environ['http_proxy']
+    handlers.append(urllib2.ProxyHandler({'http': url, 'https': url}))
+  if 'REPO_CURL_VERBOSE' in os.environ:
+    handlers.append(urllib2.HTTPHandler(debuglevel=1))
+    handlers.append(urllib2.HTTPSHandler(debuglevel=1))
+  urllib2.install_opener(urllib2.build_opener(*handlers))
+
+def _Fetch(url, local, src, quiet):
+  if not quiet:
+    print >>sys.stderr, 'Get %s' % url
+
   cmd = [GIT, 'fetch']
   if quiet:
     cmd.append('--quiet')
     err = subprocess.PIPE
   else:
     err = None
-  cmd.extend(args)
-  cmd.append('origin')
+  cmd.append(src)
+  cmd.append('+refs/heads/*:refs/remotes/origin/*')
+  cmd.append('refs/tags/*:refs/tags/*')
 
   proc = subprocess.Popen(cmd, cwd = local, stderr = err)
   if err:
@@ -313,6 +366,62 @@ def _Fetch(local, quiet, *args):
   if proc.wait() != 0:
     raise CloneFailure()
 
+def _DownloadBundle(url, local, quiet):
+  if not url.endswith('/'):
+    url += '/'
+  url += 'clone.bundle'
+
+  proc = subprocess.Popen(
+    [GIT, 'config', '--get-regexp', 'url.*.insteadof'],
+    cwd = local,
+    stdout = subprocess.PIPE)
+  for line in proc.stdout:
+    m = re.compile(r'^url\.(.*)\.insteadof (.*)$').match(line)
+    if m:
+      new_url = m.group(1)
+      old_url = m.group(2)
+      if url.startswith(old_url):
+        url = new_url + url[len(old_url):]
+        break
+  proc.stdout.close()
+  proc.wait()
+
+  if not url.startswith('http:') and not url.startswith('https:'):
+    return False
+
+  dest = open(os.path.join(local, '.git', 'clone.bundle'), 'w+b')
+  try:
+    try:
+      r = urllib2.urlopen(url)
+    except urllib2.HTTPError, e:
+      if e.code == 404:
+        return False
+      print >>sys.stderr, 'fatal: Cannot get %s' % url
+      print >>sys.stderr, 'fatal: HTTP error %s' % e.code
+      raise CloneFailure()
+    except urllib2.URLError, e:
+      print >>sys.stderr, 'fatal: Cannot get %s' % url
+      print >>sys.stderr, 'fatal: error %s' % e.reason
+      raise CloneFailure()
+    try:
+      if not quiet:
+        print >>sys.stderr, 'Get %s' % url
+      while True:
+        buf = r.read(8192)
+        if buf == '':
+          return True
+        dest.write(buf)
+    finally:
+      r.close()
+  finally:
+    dest.close()
+
+def _ImportBundle(local):
+  path = os.path.join(local, '.git', 'clone.bundle')
+  try:
+    _Fetch(local, local, path, True)
+  finally:
+    os.remove(path)
 
 def _Clone(url, local, quiet):
   """Clones a git repository to a new subdirectory of repodir
@@ -340,11 +449,14 @@ def _Clone(url, local, quiet):
     print >>sys.stderr, 'fatal: could not create %s' % local
     raise CloneFailure()
 
+  _InitHttp()
   _SetConfig(local, 'remote.origin.url', url)
   _SetConfig(local, 'remote.origin.fetch',
                     '+refs/heads/*:refs/remotes/origin/*')
-  _Fetch(local, quiet)
-  _Fetch(local, quiet, '--tags')
+  if _DownloadBundle(url, local, quiet):
+    _ImportBundle(local)
+  else:
+    _Fetch(url, local, 'origin', quiet)
 
 
 def _Verify(cwd, branch, quiet):
@@ -596,4 +708,3 @@ def main(orig_args):
 
 if __name__ == '__main__':
   main(sys.argv[1:])
-

subcmds/abandon.py

@@ -41,21 +41,30 @@ It is equivalent to "git branch -D <branchname>".
 
     nb = args[0]
     err = []
+    success = []
     all = self.GetProjects(args[1:])
 
     pm = Progress('Abandon %s' % nb, len(all))
     for project in all:
       pm.update()
-      if not project.AbandonBranch(nb):
-        err.append(project)
+
+      status = project.AbandonBranch(nb)
+      if status is not None:
+        if status:
+          success.append(project)
+        else:
+          err.append(project)
     pm.end()
 
     if err:
-      if len(err) == len(all):
-        print >>sys.stderr, 'error: no project has branch %s' % nb
-      else:
-        for p in err:
-          print >>sys.stderr,\
-            "error: %s/: cannot abandon %s" \
-            % (p.relpath, nb)
+      for p in err:
+        print >>sys.stderr,\
+          "error: %s/: cannot abandon %s" \
+          % (p.relpath, nb)
       sys.exit(1)
+    elif not success:
+      print >>sys.stderr, 'error: no project has branch %s' % nb
+      sys.exit(1)
+    else:
+      print >>sys.stderr, 'Abandoned in %d project(s):\n  %s' % (
+            len(success), '\n  '.join(p.relpath for p in success))

subcmds/checkout.py

@@ -38,21 +38,27 @@ The command is equivalent to:
 
     nb = args[0]
     err = []
+    success = []
     all = self.GetProjects(args[1:])
 
     pm = Progress('Checkout %s' % nb, len(all))
     for project in all:
       pm.update()
-      if not project.CheckoutBranch(nb):
-        err.append(project)
+
+      status = project.CheckoutBranch(nb)
+      if status is not None:
+        if status:
+          success.append(project)
+        else:
+          err.append(project)
     pm.end()
 
     if err:
-      if len(err) == len(all):
-        print >>sys.stderr, 'error: no project has branch %s' % nb
-      else:
-        for p in err:
-          print >>sys.stderr,\
-            "error: %s/: cannot checkout %s" \
-            % (p.relpath, nb)
+      for p in err:
+        print >>sys.stderr,\
+          "error: %s/: cannot checkout %s" \
+          % (p.relpath, nb)
+      sys.exit(1)
+    elif not success:
+      print >>sys.stderr, 'error: no project has branch %s' % nb
       sys.exit(1)

subcmds/cherry_pick.py

@@ -0,0 +1,114 @@
+#
+# Copyright (C) 2010 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys, re, string, random, os
+from command import Command
+from git_command import GitCommand
+
+CHANGE_ID_RE = re.compile(r'^\s*Change-Id: I([0-9a-f]{40})\s*$')
+
+class CherryPick(Command):
+  common = True
+  helpSummary = "Cherry-pick a change."
+  helpUsage = """
+%prog <sha1>
+"""
+  helpDescription = """
+'%prog' cherry-picks a change from one branch to another.
+The change id will be updated, and a reference to the old
+change id will be added.
+"""
+
+  def _Options(self, p):
+    pass
+
+  def Execute(self, opt, args):
+    if len(args) != 1:
+      self.Usage()
+
+    reference = args[0]
+
+    p = GitCommand(None,
+                   ['rev-parse', '--verify', reference],
+                   capture_stdout = True,
+                   capture_stderr = True)
+    if p.Wait() != 0:
+      print >>sys.stderr, p.stderr
+      sys.exit(1)
+    sha1 = p.stdout.strip()
+
+    p = GitCommand(None, ['cat-file', 'commit', sha1], capture_stdout=True)
+    if p.Wait() != 0:
+      print >>sys.stderr, "error: Failed to retrieve old commit message"
+      sys.exit(1)
+    old_msg = self._StripHeader(p.stdout)
+
+    p = GitCommand(None,
+                   ['cherry-pick', sha1],
+                   capture_stdout = True,
+                   capture_stderr = True)
+    status = p.Wait()
+
+    print >>sys.stdout, p.stdout
+    print >>sys.stderr, p.stderr
+
+    if status == 0:
+      # The cherry-pick was applied correctly. We just need to edit the
+      # commit message.
+      new_msg = self._Reformat(old_msg, sha1)
+
+      p = GitCommand(None, ['commit', '--amend', '-F', '-'],
+                     provide_stdin = True,
+                     capture_stdout = True,
+                     capture_stderr = True)
+      p.stdin.write(new_msg)
+      if p.Wait() != 0:
+        print >>sys.stderr, "error: Failed to update commit message"
+        sys.exit(1)
+
+    else:
+      print >>sys.stderr, """\
+NOTE: When committing (please see above) and editing the commit message,
+please remove the old Change-Id-line and add:
+"""
+      print >>sys.stderr, self._GetReference(sha1)
+      print >>sys.stderr
+
+  def _IsChangeId(self, line):
+    return CHANGE_ID_RE.match(line)
+
+  def _GetReference(self, sha1):
+    return "(cherry picked from commit %s)" % sha1
+
+  def _StripHeader(self, commit_msg):
+    lines = commit_msg.splitlines()
+    return "\n".join(lines[lines.index("")+1:])
+
+  def _Reformat(self, old_msg, sha1):
+    new_msg = []
+
+    for line in old_msg.splitlines():
+      if not self._IsChangeId(line):
+        new_msg.append(line)
+
+    # Add a blank line between the message and the change id/reference
+    try:
+      if new_msg[-1].strip() != "":
+        new_msg.append("")
+    except IndexError:
+      pass
+
+    new_msg.append(self._GetReference(sha1))
+    return "\n".join(new_msg)

subcmds/diff.py

@@ -20,8 +20,21 @@ class Diff(PagedCommand):
   helpSummary = "Show changes between commit and working tree"
   helpUsage = """
 %prog [<project>...]
+
+The -u option causes '%prog' to generate diff output with file paths
+relative to the repository root, so the output can be applied
+to the Unix 'patch' command.
 """
 
+  def _Options(self, p):
+    def cmd(option, opt_str, value, parser):
+      setattr(parser.values, option.dest, list(parser.rargs))
+      while parser.rargs:
+        del parser.rargs[0]
+    p.add_option('-u', '--absolute',
+                 dest='absolute', action='store_true',
+                 help='Paths are relative to the repository root')
+
   def Execute(self, opt, args):
     for project in self.GetProjects(args):
-      project.PrintWorkTreeDiff()
+      project.PrintWorkTreeDiff(opt.absolute)

subcmds/download.py

@@ -33,7 +33,15 @@ makes it available in your project's local working directory.
 """
 
   def _Options(self, p):
-    pass
+    p.add_option('-c','--cherry-pick',
+                 dest='cherrypick', action='store_true',
+                 help="cherry-pick instead of checkout")
+    p.add_option('-r','--revert',
+                 dest='revert', action='store_true',
+                 help="revert instead of checkout")
+    p.add_option('-f','--ff-only',
+                 dest='ffonly', action='store_true',
+                 help="force fast-forward merge")
 
   def _ParseChangeIds(self, args):
     if not args:
@@ -66,7 +74,7 @@ makes it available in your project's local working directory.
           % (project.name, change_id, ps_id)
         sys.exit(1)
 
-      if not dl.commits:
+      if not opt.revert and not dl.commits:
         print >>sys.stderr, \
           '[%s] change %d/%d has already been merged' \
           % (project.name, change_id, ps_id)
@@ -78,4 +86,11 @@ makes it available in your project's local working directory.
           % (project.name, change_id, ps_id, len(dl.commits))
         for c in dl.commits:
           print >>sys.stderr, '  %s' % (c)
-      project._Checkout(dl.commit)
+      if opt.cherrypick:
+        project._CherryPick(dl.commit)
+      elif opt.revert:
+        project._Revert(dl.commit)
+      elif opt.ffonly:
+        project._FastForward(dl.commit, ffonly=True)
+      else:
+        project._Checkout(dl.commit)

subcmds/forall.py

@@ -82,6 +82,11 @@ revision to a locally executed git command, use REPO_LREV.
 REPO_RREV is the name of the revision from the manifest, exactly
 as written in the manifest.
 
+REPO__* are any extra environment variables, specified by the
+"annotation" element under any project element.  This can be useful
+for differentiating trees based on user-specific criteria, or simply
+annotating tree details.
+
 shell positional arguments ($1, $2, .., $#) are set to any arguments
 following <command>.
 
@@ -162,6 +167,8 @@ terminal and are not redirected.
       setenv('REPO_REMOTE', project.remote.name)
       setenv('REPO_LREV', project.GetRevisionId())
       setenv('REPO_RREV', project.revisionExpr)
+      for a in project.annotations:
+        setenv("REPO__%s" % (a.name), a.value)
 
       if mirror:
         setenv('GIT_DIR', project.gitdir)

subcmds/help.py

@@ -165,6 +165,7 @@ See 'repo help --all' for a complete list of recognized commands.
         print >>sys.stderr, "repo: '%s' is not a repo command." % name
         sys.exit(1)
 
+      cmd.manifest = self.manifest
       self._PrintCommandHelp(cmd)
 
     else:

subcmds/init.py

@@ -14,12 +14,16 @@
 # limitations under the License.
 
 import os
+import platform
+import re
+import shutil
 import sys
 
 from color import Coloring
 from command import InteractiveCommand, MirrorSafeCommand
 from error import ManifestParseError
 from project import SyncBuffer
+from git_config import GitConfig
 from git_command import git_require, MIN_GIT_VERSION
 
 class Init(InteractiveCommand, MirrorSafeCommand):
@@ -81,6 +85,18 @@ to update the working directory files.
     g.add_option('--reference',
                  dest='reference',
                  help='location of mirror directory', metavar='DIR')
+    g.add_option('--depth', type='int', default=None,
+                 dest='depth',
+                 help='create a shallow clone with given depth; see git clone')
+    g.add_option('-g', '--groups',
+                 dest='groups', default='default',
+                 help='restrict manifest projects to ones with a specified group',
+                 metavar='GROUP')
+    g.add_option('-p', '--platform',
+                 dest='platform', default='auto',
+                 help='restrict manifest projects to ones with a specified'
+                      'platform group [auto|all|none|linux|darwin|...]',
+                 metavar='PLATFORM')
 
     # Tool
     g = p.add_option_group('repo Version options')
@@ -94,6 +110,12 @@ to update the working directory files.
                  dest='no_repo_verify', action='store_true',
                  help='do not verify repo source code')
 
+    # Other
+    g = p.add_option_group('Other options')
+    g.add_option('--config-name',
+                 dest='config_name', action="store_true", default=False,
+                 help='Always prompt for name/e-mail')
+
   def _SyncManifest(self, opt):
     m = self.manifest.manifestProject
     is_new = not m.Exists
@@ -104,8 +126,8 @@ to update the working directory files.
         sys.exit(1)
 
       if not opt.quiet:
-        print >>sys.stderr, 'Getting manifest ...'
-        print >>sys.stderr, '   from %s' % opt.manifest_url
+        print >>sys.stderr, 'Get %s' \
+          % GitConfig.ForUser().UrlInsteadOf(opt.manifest_url)
       m._InitGitDir()
 
       if opt.manifest_branch:
@@ -124,6 +146,27 @@ to update the working directory files.
       r.ResetFetch()
       r.Save()
 
+    groups = re.split('[,\s]+', opt.groups)
+    all_platforms = ['linux', 'darwin']
+    platformize = lambda x: 'platform-' + x
+    if opt.platform == 'auto':
+      if (not opt.mirror and
+          not m.config.GetString('repo.mirror') == 'true'):
+        groups.append(platformize(platform.system().lower()))
+    elif opt.platform == 'all':
+      groups.extend(map(platformize, all_platforms))
+    elif opt.platform in all_platforms:
+      groups.extend(platformize(opt.platform))
+    elif opt.platform != 'none':
+      print >>sys.stderr, 'fatal: invalid platform flag'
+      sys.exit(1)
+
+    groups = [x for x in groups if x]
+    groupstr = ','.join(groups)
+    if opt.platform == 'auto' and groupstr == 'default,platform-' + platform.system().lower():
+      groupstr = None
+    m.config.SetString('manifest.groups', groupstr)
+
     if opt.reference:
       m.config.SetString('repo.reference', opt.reference)
 
@@ -134,11 +177,19 @@ to update the working directory files.
         print >>sys.stderr, 'fatal: --mirror not supported on existing client'
         sys.exit(1)
 
-    if not m.Sync_NetworkHalf():
+    if not m.Sync_NetworkHalf(is_new=is_new):
       r = m.GetRemote(m.remote.name)
       print >>sys.stderr, 'fatal: cannot obtain manifest %s' % r.url
+
+      # Better delete the manifest git dir if we created it; otherwise next
+      # time (when user fixes problems) we won't go through the "is_new" logic.
+      if is_new:
+        shutil.rmtree(m.gitdir)
       sys.exit(1)
 
+    if opt.manifest_branch:
+      m.MetaBranchSwitch(opt.manifest_branch)
+
     syncbuf = SyncBuffer(m.config)
     m.Sync_LocalHalf(syncbuf)
     syncbuf.Finish()
@@ -169,6 +220,24 @@ to update the working directory files.
       return value
     return a
 
+  def _ShouldConfigureUser(self):
+    gc = self.manifest.globalConfig
+    mp = self.manifest.manifestProject
+
+    # If we don't have local settings, get from global.
+    if not mp.config.Has('user.name') or not mp.config.Has('user.email'):
+      if not gc.Has('user.name') or not gc.Has('user.email'):
+        return True
+
+      mp.config.SetString('user.name', gc.GetString('user.name'))
+      mp.config.SetString('user.email', gc.GetString('user.email'))
+
+    print ''
+    print 'Your identity is: %s <%s>' % (mp.config.GetString('user.name'),
+                                         mp.config.GetString('user.email'))
+    print 'If you want to change this, please re-run \'repo init\' with --config-name'
+    return False
+
   def _ConfigureUser(self):
     mp = self.manifest.manifestProject
 
@@ -179,7 +248,7 @@ to update the working directory files.
 
       print ''
       print 'Your identity is: %s <%s>' % (name, email)
-      sys.stdout.write('is this correct [y/n]? ')
+      sys.stdout.write('is this correct [y/N]? ')
       a = sys.stdin.readline().strip()
       if a in ('yes', 'y', 't', 'true'):
         break
@@ -221,20 +290,42 @@ to update the working directory files.
       out.printer(fg='black', attr=c)(' %-6s ', c)
     out.nl()
 
-    sys.stdout.write('Enable color display in this user account (y/n)? ')
+    sys.stdout.write('Enable color display in this user account (y/N)? ')
     a = sys.stdin.readline().strip().lower()
     if a in ('y', 'yes', 't', 'true', 'on'):
       gc.SetString('color.ui', 'auto')
 
+  def _ConfigureDepth(self, opt):
+    """Configure the depth we'll sync down.
+
+    Args:
+      opt: Options from optparse.  We care about opt.depth.
+    """
+    # Opt.depth will be non-None if user actually passed --depth to repo init.
+    if opt.depth is not None:
+      if opt.depth > 0:
+        # Positive values will set the depth.
+        depth = str(opt.depth)
+      else:
+        # Negative numbers will clear the depth; passing None to SetString
+        # will do that.
+        depth = None
+
+      # We store the depth in the main manifest project.
+      self.manifest.manifestProject.config.SetString('repo.depth', depth)
+
   def Execute(self, opt, args):
     git_require(MIN_GIT_VERSION, fail=True)
     self._SyncManifest(opt)
     self._LinkManifest(opt.manifest_name)
 
     if os.isatty(0) and os.isatty(1) and not self.manifest.IsMirror:
-      self._ConfigureUser()
+      if opt.config_name or self._ShouldConfigureUser():
+        self._ConfigureUser()
       self._ConfigureColor()
 
+    self._ConfigureDepth(opt)
+
     if self.manifest.IsMirror:
       type = 'mirror '
     else:

subcmds/overview.py

@@ -0,0 +1,80 @@
+#
+# Copyright (C) 2012 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from color import Coloring
+from command import PagedCommand
+
+
+class Overview(PagedCommand):
+  common = True
+  helpSummary = "Display overview of unmerged project branches"
+  helpUsage = """
+%prog [--current-branch] [<project>...]
+"""
+  helpDescription = """
+The '%prog' command is used to display an overview of the projects branches,
+and list any local commits that have not yet been merged into the project.
+
+The -b/--current-branch option can be used to restrict the output to only
+branches currently checked out in each project.  By default, all branches
+are displayed.
+"""
+
+  def _Options(self, p):
+    p.add_option('-b', '--current-branch',
+                 dest="current_branch", action="store_true",
+                 help="Consider only checked out branches")
+
+  def Execute(self, opt, args):
+    all = []
+    for project in self.GetProjects(args):
+      br = [project.GetUploadableBranch(x)
+            for x in project.GetBranches().keys()]
+      br = [x for x in br if x]
+      if opt.current_branch:
+        br = [x for x in br if x.name == project.CurrentBranch]
+      all.extend(br)
+
+    if not all:
+      return
+
+    class Report(Coloring):
+      def __init__(self, config):
+        Coloring.__init__(self, config, 'status')
+        self.project = self.printer('header', attr='bold')
+
+    out = Report(all[0].project.config)
+    out.project('Projects Overview')
+    out.nl()
+
+    project = None
+
+    for branch in all:
+      if project != branch.project:
+        project = branch.project
+        out.nl()
+        out.project('project %s/' % project.relpath)
+        out.nl()
+
+      commits = branch.commits
+      date = branch.date
+      print '%s %-33s (%2d commit%s, %s)' % (
+            branch.name == project.CurrentBranch and '*' or ' ',
+            branch.name,
+            len(commits),
+            len(commits) != 1 and 's' or ' ',
+            date)
+      for commit in commits:
+        print '%-35s   - %s' % ('', commit)

subcmds/rebase.py

@@ -52,6 +52,9 @@ branch but need to incorporate new upstream changes "underneath" them.
     p.add_option('--whitespace',
                  dest='whitespace', action='store', metavar='WS',
                  help='Pass --whitespace to git rebase')
+    p.add_option('--auto-stash',
+                 dest='auto_stash', action='store_true',
+                 help='Stash local modifications before starting')
 
   def Execute(self, opt, args):
     all = self.GetProjects(args)
@@ -103,5 +106,23 @@ branch but need to incorporate new upstream changes "underneath" them.
       print >>sys.stderr, '# %s: rebasing %s -> %s' % \
         (project.relpath, cb, upbranch.LocalMerge)
 
+      needs_stash = False
+      if opt.auto_stash:
+        stash_args = ["update-index", "--refresh", "-q"]
+
+        if GitCommand(project, stash_args).Wait() != 0:
+          needs_stash = True
+          # Dirty index, requires stash...
+          stash_args = ["stash"]
+
+          if GitCommand(project, stash_args).Wait() != 0:
+            return -1
+
       if GitCommand(project, args).Wait() != 0:
         return -1
+
+      if needs_stash:
+        stash_args.append('pop')
+        stash_args.append('--quiet')
+        if GitCommand(project, stash_args).Wait() != 0:
+          return -1

subcmds/start.py

@@ -15,6 +15,7 @@
 
 import sys
 from command import Command
+from git_config import IsId
 from git_command import git
 from progress import Progress
 
@@ -56,6 +57,10 @@ revision specified in the manifest.
     pm = Progress('Starting %s' % nb, len(all))
     for project in all:
       pm.update()
+      # If the current revision is a specific SHA1 then we can't push back
+      # to it so substitute the manifest default revision instead.
+      if IsId(project.revisionExpr):
+        project.revisionExpr = self.manifest.default.revisionExpr
       if not project.StartBranch(nb):
         err.append(project)
     pm.end()

subcmds/status.py

@@ -15,6 +15,15 @@
 
 from command import PagedCommand
 
+try:
+  import threading as _threading
+except ImportError:
+  import dummy_threading as _threading
+
+import itertools
+import sys
+import StringIO
+
 class Status(PagedCommand):
   common = True
   helpSummary = "Show the working tree status"
@@ -27,6 +36,9 @@ and the most recent commit on this branch (HEAD), in each project
 specified.  A summary is displayed, one line per file where there
 is a difference between these three states.
 
+The -j/--jobs option can be used to run multiple status queries
+in parallel.
+
 Status Display
 --------------
 
@@ -60,26 +72,60 @@ the following meanings:
 
 """
 
+  def _Options(self, p):
+    p.add_option('-j', '--jobs',
+                 dest='jobs', action='store', type='int', default=2,
+                 help="number of projects to check simultaneously")
+
+  def _StatusHelper(self, project, clean_counter, sem, output):
+    """Obtains the status for a specific project.
+
+    Obtains the status for a project, redirecting the output to
+    the specified object. It will release the semaphore
+    when done.
+
+    Args:
+      project: Project to get status of.
+      clean_counter: Counter for clean projects.
+      sem: Semaphore, will call release() when complete.
+      output: Where to output the status.
+    """
+    try:
+      state = project.PrintWorkTreeStatus(output)
+      if state == 'CLEAN':
+        clean_counter.next()
+    finally:
+      sem.release()
+
   def Execute(self, opt, args):
     all = self.GetProjects(args)
-    clean = 0
+    counter = itertools.count()
 
-    on = {}
-    for project in all:
-      cb = project.CurrentBranch
-      if cb:
-        if cb not in on:
-          on[cb] = []
-        on[cb].append(project)
+    if opt.jobs == 1:
+      for project in all:
+        state = project.PrintWorkTreeStatus()
+        if state == 'CLEAN':
+          counter.next()
+    else:
+      sem = _threading.Semaphore(opt.jobs)
+      threads_and_output = []
+      for project in all:
+        sem.acquire()
 
-    branch_names = list(on.keys())
-    branch_names.sort()
-    for cb in branch_names:
-      print '# on branch %s' % cb
+        class BufList(StringIO.StringIO):
+          def dump(self, ostream):
+            for entry in self.buflist:
+              ostream.write(entry)
 
-    for project in all:
-      state = project.PrintWorkTreeStatus()
-      if state == 'CLEAN':
-        clean += 1
-    if len(all) == clean:
+        output = BufList()
+
+        t = _threading.Thread(target=self._StatusHelper,
+                              args=(project, counter, sem, output))
+        threads_and_output.append((t, output))
+        t.start()
+      for (t, output) in threads_and_output:
+        t.join()
+        output.dump(sys.stdout)
+        output.close()
+    if len(all) == counter.next():
       print 'nothing to commit (working directory clean)'

subcmds/sync.py

@@ -28,6 +28,14 @@ try:
 except ImportError:
   import dummy_threading as _threading
 
+try:
+  import resource
+  def _rlimit_nofile():
+    return resource.getrlimit(resource.RLIMIT_NOFILE)
+except ImportError:
+  def _rlimit_nofile():
+    return (256, 256)
+
 from git_command import GIT
 from git_refs import R_HEADS
 from project import HEAD
@@ -72,11 +80,18 @@ revision is temporarily needed.
 
 The -s/--smart-sync option can be used to sync to a known good
 build as specified by the manifest-server element in the current
-manifest.
+manifest. The -t/--smart-tag option is similar and allows you to
+specify a custom tag/label.
 
 The -f/--force-broken option can be used to proceed with syncing
 other projects if a project sync fails.
 
+The --no-clone-bundle option disables any attempt to use
+$URL/clone.bundle to bootstrap a new Git repository from a
+resumeable bundle file on a content delivery network. This
+may be necessary if there are problems with the local Python
+HTTP client or proxy configuration, but the Git binary works.
+
 SSH Connections
 ---------------
 
@@ -108,6 +123,8 @@ later is required to fix a server side protocol bug.
 """
 
   def _Options(self, p, show_smart=True):
+    self.jobs = self.manifest.default.sync_j
+
     p.add_option('-f', '--force-broken',
                  dest='force_broken', action='store_true',
                  help="continue sync even if a project fails to sync")
@@ -120,16 +137,28 @@ later is required to fix a server side protocol bug.
     p.add_option('-d','--detach',
                  dest='detach_head', action='store_true',
                  help='detach projects back to manifest revision')
+    p.add_option('-c','--current-branch',
+                 dest='current_branch_only', action='store_true',
+                 help='fetch only current branch from server')
     p.add_option('-q','--quiet',
                  dest='quiet', action='store_true',
                  help='be more quiet')
     p.add_option('-j','--jobs',
                  dest='jobs', action='store', type='int',
-                 help="number of projects to fetch simultaneously")
+                 help="projects to fetch simultaneously (default %d)" % self.jobs)
+    p.add_option('-m', '--manifest-name',
+                 dest='manifest_name',
+                 help='temporary manifest to use for this sync', metavar='NAME.xml')
+    p.add_option('--no-clone-bundle',
+                 dest='no_clone_bundle', action='store_true',
+                 help='disable use of /clone.bundle on HTTP/HTTPS')
     if show_smart:
       p.add_option('-s', '--smart-sync',
                    dest='smart_sync', action='store_true',
                    help='smart sync using manifest from a known good build')
+      p.add_option('-t', '--smart-tag',
+                   dest='smart_tag', action='store',
+                   help='smart sync using manifest from a known tag')
 
     g = p.add_option_group('repo Version options')
     g.add_option('--no-repo-verify',
@@ -164,30 +193,30 @@ later is required to fix a server side protocol bug.
       # - We always make sure we call sem.release().
       # - We always make sure we unlock the lock if we locked it.
       try:
-        success = project.Sync_NetworkHalf(quiet=opt.quiet)
+        try:
+          success = project.Sync_NetworkHalf(
+            quiet=opt.quiet,
+            current_branch_only=opt.current_branch_only,
+            clone_bundle=not opt.no_clone_bundle)
 
-        # Lock around all the rest of the code, since printing, updating a set
-        # and Progress.update() are not thread safe.
-        lock.acquire()
-        did_lock = True
+          # Lock around all the rest of the code, since printing, updating a set
+          # and Progress.update() are not thread safe.
+          lock.acquire()
+          did_lock = True
 
-        if not success:
-          print >>sys.stderr, 'error: Cannot fetch %s' % project.name
-          if opt.force_broken:
-            print >>sys.stderr, 'warn: --force-broken, continuing to sync'
-          else:
-            raise _FetchError()
+          if not success:
+            print >>sys.stderr, 'error: Cannot fetch %s' % project.name
+            if opt.force_broken:
+              print >>sys.stderr, 'warn: --force-broken, continuing to sync'
+            else:
+              raise _FetchError()
 
-        fetched.add(project.gitdir)
-        pm.update()
-      except BaseException, e:
-        # Notify the _Fetch() function about all errors.
-        err_event.set()
-
-        # If we got our own _FetchError, we don't want a stack trace.
-        # However, if we got something else (something in Sync_NetworkHalf?),
-        # we'd like one (so re-raise after we've set err_event).
-        if not isinstance(e, _FetchError):
+          fetched.add(project.gitdir)
+          pm.update()
+        except _FetchError:
+          err_event.set()
+        except:
+          err_event.set()
           raise
       finally:
         if did_lock:
@@ -201,7 +230,10 @@ later is required to fix a server side protocol bug.
     if self.jobs == 1:
       for project in projects:
         pm.update()
-        if project.Sync_NetworkHalf(quiet=opt.quiet):
+        if project.Sync_NetworkHalf(
+            quiet=opt.quiet,
+            current_branch_only=opt.current_branch_only,
+            clone_bundle=not opt.no_clone_bundle):
           fetched.add(project.gitdir)
         else:
           print >>sys.stderr, 'error: Cannot fetch %s' % project.name
@@ -217,7 +249,7 @@ later is required to fix a server side protocol bug.
       for project in projects:
         # Check for any errors before starting any new threads.
         # ...we'll let existing threads finish, though.
-        if err_event.is_set():
+        if err_event.isSet():
           break
 
         sem.acquire()
@@ -236,7 +268,7 @@ later is required to fix a server side protocol bug.
         t.join()
 
       # If we saw an error, exit with code 1 so that other scripts can check.
-      if err_event.is_set():
+      if err_event.isSet():
         print >>sys.stderr, '\nerror: Exited sync due to fetch errors'
         sys.exit(1)
 
@@ -247,7 +279,7 @@ later is required to fix a server side protocol bug.
 
   def UpdateProjectList(self):
     new_project_paths = []
-    for project in self.manifest.projects.values():
+    for project in self.GetProjects(None, missing_ok=True):
       if project.relpath:
         new_project_paths.append(project.relpath)
     file_name = 'project.list'
@@ -276,7 +308,8 @@ later is required to fix a server side protocol bug.
                              worktree = os.path.join(self.manifest.topdir, path),
                              relpath = path,
                              revisionExpr = 'HEAD',
-                             revisionId = None)
+                             revisionId = None,
+                             groups = None)
 
               if project.IsDirty():
                 print >>sys.stderr, 'error: Cannot remove project "%s": \
@@ -307,34 +340,51 @@ uncommitted changes are present' % project.relpath
   def Execute(self, opt, args):
     if opt.jobs:
       self.jobs = opt.jobs
+    if self.jobs > 1:
+      soft_limit, _ = _rlimit_nofile()
+      self.jobs = min(self.jobs, (soft_limit - 5) / 3)
+
     if opt.network_only and opt.detach_head:
       print >>sys.stderr, 'error: cannot combine -n and -d'
       sys.exit(1)
     if opt.network_only and opt.local_only:
       print >>sys.stderr, 'error: cannot combine -n and -l'
       sys.exit(1)
+    if opt.manifest_name and opt.smart_sync:
+      print >>sys.stderr, 'error: cannot combine -m and -s'
+      sys.exit(1)
+    if opt.manifest_name and opt.smart_tag:
+      print >>sys.stderr, 'error: cannot combine -m and -t'
+      sys.exit(1)
 
-    if opt.smart_sync:
+    if opt.manifest_name:
+      self.manifest.Override(opt.manifest_name)
+
+    if opt.smart_sync or opt.smart_tag:
       if not self.manifest.manifest_server:
         print >>sys.stderr, \
             'error: cannot smart sync: no manifest server defined in manifest'
         sys.exit(1)
       try:
         server = xmlrpclib.Server(self.manifest.manifest_server)
-        p = self.manifest.manifestProject
-        b = p.GetBranch(p.CurrentBranch)
-        branch = b.merge
-        if branch.startswith(R_HEADS):
-          branch = branch[len(R_HEADS):]
+        if opt.smart_sync:
+          p = self.manifest.manifestProject
+          b = p.GetBranch(p.CurrentBranch)
+          branch = b.merge
+          if branch.startswith(R_HEADS):
+            branch = branch[len(R_HEADS):]
 
-        env = os.environ.copy()
-        if (env.has_key('TARGET_PRODUCT') and
-            env.has_key('TARGET_BUILD_VARIANT')):
-          target = '%s-%s' % (env['TARGET_PRODUCT'],
-                              env['TARGET_BUILD_VARIANT'])
-          [success, manifest_str] = server.GetApprovedManifest(branch, target)
+          env = os.environ.copy()
+          if (env.has_key('TARGET_PRODUCT') and
+              env.has_key('TARGET_BUILD_VARIANT')):
+            target = '%s-%s' % (env['TARGET_PRODUCT'],
+                                env['TARGET_BUILD_VARIANT'])
+            [success, manifest_str] = server.GetApprovedManifest(branch, target)
+          else:
+            [success, manifest_str] = server.GetApprovedManifest(branch)
         else:
-          [success, manifest_str] = server.GetApprovedManifest(branch)
+          assert(opt.smart_tag)
+          [success, manifest_str] = server.GetManifest(opt.smart_tag)
 
         if success:
           manifest_name = "smart_sync_override.xml"
@@ -369,7 +419,8 @@ uncommitted changes are present' % project.relpath
       _PostRepoUpgrade(self.manifest)
 
     if not opt.local_only:
-      mp.Sync_NetworkHalf(quiet=opt.quiet)
+      mp.Sync_NetworkHalf(quiet=opt.quiet,
+                          current_branch_only=opt.current_branch_only)
 
     if mp.HasChanges:
       syncbuf = SyncBuffer(mp.config)
@@ -377,6 +428,8 @@ uncommitted changes are present' % project.relpath
       if not syncbuf.Finish():
         sys.exit(1)
       self.manifest._Unload()
+      if opt.jobs is None:
+        self.jobs = self.manifest.default.sync_j
     all = self.GetProjects(args, missing_ok=True)
 
     if not opt.local_only:

subcmds/upload.py

@@ -73,7 +73,7 @@ Configuration
 
 review.URL.autoupload:
 
-To disable the "Upload ... (y/n)?" prompt, you can set a per-project
+To disable the "Upload ... (y/N)?" prompt, you can set a per-project
 or global Git configuration option.  If review.URL.autoupload is set
 to "true" then repo will assume you always answer "y" at the prompt,
 and will not prompt you further.  If it is set to "false" then repo
@@ -103,6 +103,14 @@ or in the .git/config within the project.  For example:
     autoupload = true
     autocopy = johndoe@company.com,my-team-alias@company.com
 
+review.URL.uploadtopic:
+
+To add a topic branch whenever uploading a commit, you can set a
+per-project or global Git option to do so. If review.URL.uploadtopic
+is set to "true" then repo will assume you always want the equivalent
+of the -t option to the repo command. If unset or set to "false" then
+repo will make use of only the command line option.
+
 References
 ----------
 
@@ -120,6 +128,15 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
     p.add_option('--cc',
                  type='string',  action='append', dest='cc',
                  help='Also send email to these email addresses.')
+    p.add_option('--br',
+                 type='string',  action='store', dest='branch',
+                 help='Branch to upload.')
+    p.add_option('--cbr', '--current-branch',
+                 dest='current_branch', action='store_true',
+                 help='Upload current git branch.')
+    p.add_option('-d', '--draft',
+                 action='store_true', dest='draft', default=False,
+                 help='If specified, upload as a draft.')
 
     # Options relating to upload hook.  Note that verify and no-verify are NOT
     # opposites of each other, which is why they store to different locations.
@@ -159,7 +176,7 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
       date = branch.date
       list = branch.commits
 
-      print 'Upload project %s/:' % project.relpath
+      print 'Upload project %s/ to remote branch %s:' % (project.relpath, project.revisionExpr)
       print '  branch %s (%2d commit%s, %s):' % (
                     name,
                     len(list),
@@ -168,7 +185,7 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
       for commit in list:
         print '         %s' % commit
 
-      sys.stdout.write('to %s (y/n)? ' % remote.review)
+      sys.stdout.write('to %s (y/N)? ' % remote.review)
       answer = sys.stdin.readline().strip()
       answer = answer in ('y', 'Y', 'yes', '1', 'true', 't')
 
@@ -199,11 +216,12 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
 
         if b:
           script.append('#')
-        script.append('#  branch %s (%2d commit%s, %s):' % (
+        script.append('#  branch %s (%2d commit%s, %s) to remote branch %s:' % (
                       name,
                       len(list),
                       len(list) != 1 and 's' or '',
-                      date))
+                      date,
+                      project.revisionExpr))
         for commit in list:
           script.append('#         %s' % commit)
         b[name] = branch
@@ -212,6 +230,11 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
       branches[project.name] = b
     script.append('')
 
+    script = [ x.encode('utf-8')
+             if issubclass(type(x), unicode)
+             else x
+             for x in script ]
+
     script = Editor.EditString("\n".join(script)).split("\n")
 
     project_re = re.compile(r'^#?\s*project\s*([^\s]+)/:$')
@@ -291,7 +314,7 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
 
             # if they want to auto upload, let's not ask because it could be automated
             if answer is None:
-                sys.stdout.write('Uncommitted changes in ' + branch.project.name + ' (did you forget to amend?). Continue uploading? (y/n) ')
+                sys.stdout.write('Uncommitted changes in ' + branch.project.name + ' (did you forget to amend?). Continue uploading? (y/N) ')
                 a = sys.stdin.readline().strip().lower()
                 if a not in ('y', 'yes', 't', 'true', 'on'):
                     print >>sys.stderr, "skipping upload"
@@ -299,7 +322,12 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
                     branch.error = 'User aborted'
                     continue
 
-        branch.UploadForReview(people, auto_topic=opt.auto_topic)
+        # Check if topic branches should be sent to the server during upload
+        if opt.auto_topic is not True:
+           key = 'review.%s.uploadtopic' % branch.project.remote.review
+           opt.auto_topic = branch.project.config.GetBoolean(key)
+
+        branch.UploadForReview(people, auto_topic=opt.auto_topic, draft=opt.draft)
         branch.uploaded = True
       except UploadError, e:
         branch.error = e
@@ -336,9 +364,17 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
     pending = []
     reviewers = []
     cc = []
+    branch = None
+
+    if opt.branch:
+      branch = opt.branch
 
     for project in project_list:
-      avail = project.GetUploadableBranches()
+      if opt.current_branch:
+        cbr = project.CurrentBranch
+        avail = [project.GetUploadableBranch(cbr)] if cbr else None
+      else:
+        avail = project.GetUploadableBranches(branch)
       if avail:
         pending.append((project, avail))
 

subcmds/version.py

@@ -19,6 +19,9 @@ from git_command import git
 from project import HEAD
 
 class Version(Command, MirrorSafeCommand):
+  wrapper_version = None
+  wrapper_path = None
+
   common = False
   helpSummary = "Display the version of repo"
   helpUsage = """
@@ -31,5 +34,10 @@ class Version(Command, MirrorSafeCommand):
 
     print 'repo version %s' % rp.work_git.describe(HEAD)
     print '       (from %s)' % rem.url
+
+    if Version.wrapper_path is not None:
+      print 'repo launcher version %s' % Version.wrapper_version
+      print '       (from %s)' % Version.wrapper_path
+
     print git.version().strip()
     print 'Python %s' % sys.version