sync: Allow -j to have a default in manifest

This permits manifest authors to suggest a number of parallel
fetch operations against a remote server. For example, Gerrit
Code Review servers support queuing of requests and processes
them in first-in, first-out order. Running concurrent fetches
can utilize multiple CPUs on the Gerrit server, but will also
decrease overall operation latency by having the request put
into the queue ready to execute as soon as a CPU is free.

Change-Id: I3d3904acb6f63516bae4b071c510ad57a2afab18
Signed-off-by: Shawn O. Pearce <sop@google.com>

docs/manifest-format.txt

@@ -38,6 +38,7 @@ following DTD:
     <!ELEMENT default (EMPTY)>
     <!ATTLIST default remote   IDREF #IMPLIED>
     <!ATTLIST default revision CDATA #IMPLIED>
+    <!ATTLIST default sync-j   CDATA #IMPLIED>
 
     <!ELEMENT manifest-server (EMPTY)>
     <!ATTLIST url              CDATA #REQUIRED>

git_command.py

@@ -72,6 +72,8 @@ def terminate_ssh_clients():
       pass
   _ssh_clients = []
 
+_git_version = None
+
 class _GitCall(object):
   def version(self):
     p = GitCommand(None, ['--version'], capture_stdout=True)
@@ -79,6 +81,21 @@ class _GitCall(object):
       return p.stdout
     return None
 
+  def version_tuple(self):
+    global _git_version
+
+    if _git_version is None:
+      ver_str = git.version()
+      if ver_str.startswith('git version '):
+        _git_version = tuple(
+          map(lambda x: int(x),
+            ver_str[len('git version '):].strip().split('.')[0:3]
+          ))
+      else:
+        print >>sys.stderr, 'fatal: "%s" unsupported' % ver_str
+        sys.exit(1)
+    return _git_version
+
   def __getattr__(self, name):
     name = name.replace('_','-')
     def fun(*cmdv):
@@ -88,23 +105,9 @@ class _GitCall(object):
     return fun
 git = _GitCall()
 
-_git_version = None
-
 def git_require(min_version, fail=False):
-  global _git_version
+  git_version = git.version_tuple()
-
+  if min_version <= git_version:
-  if _git_version is None:
-    ver_str = git.version()
-    if ver_str.startswith('git version '):
-      _git_version = tuple(
-        map(lambda x: int(x),
-          ver_str[len('git version '):].strip().split('.')[0:3]
-        ))
-    else:
-      print >>sys.stderr, 'fatal: "%s" unsupported' % ver_str
-      sys.exit(1)
-
-  if min_version <= _git_version:
     return True
   if fail:
     need = '.'.join(map(lambda x: str(x), min_version))

git_config.py

@@ -198,6 +198,15 @@ class GitConfig(object):
     except KeyError:
       return False
 
+  def UrlInsteadOf(self, url):
+    """Resolve any url.*.insteadof references.
+    """
+    for new_url in self.GetSubSections('url'):
+      old_url = self.GetString('url.%s.insteadof' % new_url)
+      if old_url is not None and url.startswith(old_url):
+        return new_url + url[len(old_url):]
+    return url
+
   @property
   def _sections(self):
     d = self._section_dict

main.py

@@ -22,12 +22,16 @@ if __name__ == '__main__':
     del sys.argv[-1]
 del magic
 
+import netrc
 import optparse
 import os
 import re
 import sys
+import time
+import urllib2
 
 from trace import SetTrace
+from git_command import git, GitCommand
 from git_config import init_ssh, close_ssh
 from command import InteractiveCommand
 from command import MirrorSafeCommand
@@ -53,6 +57,9 @@ global_options.add_option('--no-pager',
 global_options.add_option('--trace',
                           dest='trace', action='store_true',
                           help='trace git command execution')
+global_options.add_option('--time',
+                          dest='time', action='store_true',
+                          help='time repo command execution')
 global_options.add_option('--version',
                           dest='show_version', action='store_true',
                           help='display this version of repo')
@@ -122,7 +129,20 @@ class _Repo(object):
         RunPager(config)
 
     try:
-      cmd.Execute(copts, cargs)
+      start = time.time()
+      try:
+        cmd.Execute(copts, cargs)
+      finally:
+        elapsed = time.time() - start
+        hours, remainder = divmod(elapsed, 3600)
+        minutes, seconds = divmod(remainder, 60)
+        if gopts.time:
+          if hours == 0:
+            print >>sys.stderr, 'real\t%dm%.3fs' \
+              % (minutes, seconds)
+          else:
+            print >>sys.stderr, 'real\t%dh%dm%.3fs' \
+              % (hours, minutes, seconds)
     except ManifestInvalidRevisionError, e:
       print >>sys.stderr, 'error: %s' % str(e)
       sys.exit(1)
@@ -133,6 +153,9 @@ class _Repo(object):
         print >>sys.stderr, 'error: no project in current directory'
       sys.exit(1)
 
+def _MyRepoPath():
+  return os.path.dirname(__file__)
+
 def _MyWrapperPath():
   return os.path.join(os.path.dirname(__file__), 'repo')
 
@@ -199,6 +222,75 @@ def _PruneOptions(argv, opt):
       continue
     i += 1
 
+_user_agent = None
+
+def _UserAgent():
+  global _user_agent
+
+  if _user_agent is None:
+    py_version = sys.version_info
+
+    os_name = sys.platform
+    if os_name == 'linux2':
+      os_name = 'Linux'
+    elif os_name == 'win32':
+      os_name = 'Win32'
+    elif os_name == 'cygwin':
+      os_name = 'Cygwin'
+    elif os_name == 'darwin':
+      os_name = 'Darwin'
+
+    p = GitCommand(
+      None, ['describe', 'HEAD'],
+      cwd = _MyRepoPath(),
+      capture_stdout = True)
+    if p.Wait() == 0:
+      repo_version = p.stdout
+      if len(repo_version) > 0 and repo_version[-1] == '\n':
+        repo_version = repo_version[0:-1]
+      if len(repo_version) > 0 and repo_version[0] == 'v':
+        repo_version = repo_version[1:]
+    else:
+      repo_version = 'unknown'
+
+    _user_agent = 'git-repo/%s (%s) git/%s Python/%d.%d.%d' % (
+      repo_version,
+      os_name,
+      '.'.join(map(lambda d: str(d), git.version_tuple())),
+      py_version[0], py_version[1], py_version[2])
+  return _user_agent
+
+class _UserAgentHandler(urllib2.BaseHandler):
+  def http_request(self, req):
+    req.add_header('User-Agent', _UserAgent())
+    return req
+
+  def https_request(self, req):
+    req.add_header('User-Agent', _UserAgent())
+    return req
+
+def init_http():
+  handlers = [_UserAgentHandler()]
+
+  mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
+  try:
+    n = netrc.netrc()
+    for host in n.hosts:
+      p = n.hosts[host]
+      mgr.add_password(None, 'http://%s/'  % host, p[0], p[2])
+      mgr.add_password(None, 'https://%s/' % host, p[0], p[2])
+  except netrc.NetrcParseError:
+    pass
+  handlers.append(urllib2.HTTPBasicAuthHandler(mgr))
+
+  if 'http_proxy' in os.environ:
+    url = os.environ['http_proxy']
+    handlers.append(urllib2.ProxyHandler({'http': url, 'https': url}))
+  if 'REPO_CURL_VERBOSE' in os.environ:
+    handlers.append(urllib2.HTTPHandler(debuglevel=1))
+    handlers.append(urllib2.HTTPSHandler(debuglevel=1))
+  urllib2.install_opener(urllib2.build_opener(*handlers))
+
 def _Main(argv):
   opt = optparse.OptionParser(usage="repo wrapperinfo -- ...")
   opt.add_option("--repo-dir", dest="repodir",
@@ -217,6 +309,7 @@ def _Main(argv):
   try:
     try:
       init_ssh()
+      init_http()
       repo._Run(argv)
     finally:
       close_ssh()

manifest_xml.py

@@ -29,6 +29,7 @@ class _Default(object):
 
   revisionExpr = None
   remote = None
+  sync_j = 1
 
 class _XmlRemote(object):
   def __init__(self,
@@ -133,6 +134,9 @@ class XmlManifest(object):
     if d.revisionExpr:
       have_default = True
       e.setAttribute('revision', d.revisionExpr)
+    if d.sync_j > 1:
+      have_default = True
+      e.setAttribute('sync-j', '%d' % d.sync_j)
     if have_default:
       root.appendChild(e)
       root.appendChild(doc.createTextNode(''))
@@ -401,6 +405,11 @@ class XmlManifest(object):
     d.revisionExpr = node.getAttribute('revision')
     if d.revisionExpr == '':
       d.revisionExpr = None
+    sync_j = node.getAttribute('sync-j')
+    if sync_j == '' or sync_j is None:
+      d.sync_j = 1
+    else:
+      d.sync_j = int(sync_j)
     return d
 
   def _ParseNotice(self, node):

progress.py

@@ -21,13 +21,14 @@ from trace import IsTrace
 _NOT_TTY = not os.isatty(2)
 
 class Progress(object):
-  def __init__(self, title, total=0):
+  def __init__(self, title, total=0, units=''):
     self._title = title
     self._total = total
     self._done = 0
     self._lastp = -1
     self._start = time()
     self._show = False
+    self._units = units
 
   def update(self, inc=1):
     self._done += inc
@@ -51,11 +52,11 @@ class Progress(object):
 
       if self._lastp != p:
         self._lastp = p
-        sys.stderr.write('\r%s: %3d%% (%d/%d)  ' % (
+        sys.stderr.write('\r%s: %3d%% (%d%s/%d%s)  ' % (
           self._title,
           p,
-          self._done,
+          self._done, self._units,
-          self._total))
+          self._total, self._units))
         sys.stderr.flush()
 
   def end(self):
@@ -69,9 +70,9 @@ class Progress(object):
       sys.stderr.flush()
     else:
       p = (100 * self._done) / self._total
-      sys.stderr.write('\r%s: %3d%% (%d/%d), done.  \n' % (
+      sys.stderr.write('\r%s: %3d%% (%d%s/%d%s), done.  \n' % (
         self._title,
         p,
-        self._done,
+        self._done, self._units,
-        self._total))
+        self._total, self._units))
       sys.stderr.flush()

project.py

@@ -791,7 +791,7 @@ class Project(object):
       if R_HEADS + n not in heads:
         self.bare_git.DeleteRef(name, id)
 
-  def GetUploadableBranches(self):
+  def GetUploadableBranches(self, selected_branch=None):
     """List any branches which can be uploaded for review.
     """
     heads = {}
@@ -807,6 +807,8 @@ class Project(object):
     for branch, id in heads.iteritems():
       if branch in pubed and pubed[branch] == id:
         continue
+      if selected_branch and branch != selected_branch:
+        continue
 
       rb = self.GetUploadableBranch(branch)
       if rb:
@@ -1368,6 +1370,13 @@ class Project(object):
         ref_dir = None
 
     cmd = ['fetch']
+
+    # The --depth option only affects the initial fetch; after that we'll do
+    # full fetches of changes.
+    depth = self.manifest.manifestProject.config.GetString('repo.depth')
+    if depth and initial:
+      cmd.append('--depth=%s' % depth)
+
     if quiet:
       cmd.append('--quiet')
     if not self.worktree:
@@ -1475,10 +1484,13 @@ class Project(object):
     for stock_hook in _ProjectHooks():
       name = os.path.basename(stock_hook)
 
-      if name in ('commit-msg',) and not self.remote.review:
+      if name in ('commit-msg',) and not self.remote.review \
+            and not self is self.manifest.manifestProject:
         # Don't install a Gerrit Code Review hook if this
         # project does not appear to use it for reviews.
         #
+        # Since the manifest project is one of those, but also
+        # managed through gerrit, it's excluded
         continue
 
       dst = os.path.join(hooks, name)

repo

@@ -2,7 +2,7 @@
 
 ## repo default configuration
 ##
-REPO_URL='git://android.git.kernel.org/tools/repo.git'
+REPO_URL='https://code.google.com/p/git-repo/'
 REPO_REV='stable'
 
 # Copyright (C) 2008 Google Inc.
@@ -28,7 +28,7 @@ if __name__ == '__main__':
 del magic
 
 # increment this whenever we make important changes to this script
-VERSION = (1, 10)
+VERSION = (1, 12)
 
 # increment this if the MAINTAINER_KEYS block is modified
 KEYRING_VERSION = (1,0)
@@ -121,6 +121,10 @@ group.add_option('--mirror',
 group.add_option('--reference',
                  dest='reference',
                  help='location of mirror directory', metavar='DIR')
+group.add_option('--depth', type='int', default=None,
+                 dest='depth',
+                 help='create a shallow clone with given depth; see git clone')
+
 
 # Tool
 group = init_optparse.add_option_group('repo Version options')
@@ -596,4 +600,3 @@ def main(orig_args):
 
 if __name__ == '__main__':
   main(sys.argv[1:])
-

subcmds/init.py

@@ -82,6 +82,9 @@ to update the working directory files.
     g.add_option('--reference',
                  dest='reference',
                  help='location of mirror directory', metavar='DIR')
+    g.add_option('--depth', type='int', default=None,
+                 dest='depth',
+                 help='create a shallow clone with given depth; see git clone')
 
     # Tool
     g = p.add_option_group('repo Version options')
@@ -232,6 +235,25 @@ to update the working directory files.
     if a in ('y', 'yes', 't', 'true', 'on'):
       gc.SetString('color.ui', 'auto')
 
+  def _ConfigureDepth(self, opt):
+    """Configure the depth we'll sync down.
+
+    Args:
+      opt: Options from optparse.  We care about opt.depth.
+    """
+    # Opt.depth will be non-None if user actually passed --depth to repo init.
+    if opt.depth is not None:
+      if opt.depth > 0:
+        # Positive values will set the depth.
+        depth = str(opt.depth)
+      else:
+        # Negative numbers will clear the depth; passing None to SetString
+        # will do that.
+        depth = None
+
+      # We store the depth in the main manifest project.
+      self.manifest.manifestProject.config.SetString('repo.depth', depth)
+
   def Execute(self, opt, args):
     git_require(MIN_GIT_VERSION, fail=True)
     self._SyncManifest(opt)
@@ -241,6 +263,8 @@ to update the working directory files.
       self._ConfigureUser()
       self._ConfigureColor()
 
+    self._ConfigureDepth(opt)
+
     if self.manifest.IsMirror:
       type = 'mirror '
     else:

subcmds/sync.py

@@ -28,6 +28,14 @@ try:
 except ImportError:
   import dummy_threading as _threading
 
+try:
+  import resource
+  def _rlimit_nofile():
+    return resource.getrlimit(resource.RLIMIT_NOFILE)
+except ImportError:
+  def _rlimit_nofile():
+    return (256, 256)
+
 from git_command import GIT
 from git_refs import R_HEADS
 from project import HEAD
@@ -72,7 +80,8 @@ revision is temporarily needed.
 
 The -s/--smart-sync option can be used to sync to a known good
 build as specified by the manifest-server element in the current
-manifest.
+manifest. The -t/--smart-tag option is similar and allows you to
+specify a custom tag/label.
 
 The -f/--force-broken option can be used to proceed with syncing
 other projects if a project sync fails.
@@ -108,6 +117,8 @@ later is required to fix a server side protocol bug.
 """
 
   def _Options(self, p, show_smart=True):
+    self.jobs = self.manifest.default.sync_j
+
     p.add_option('-f', '--force-broken',
                  dest='force_broken', action='store_true',
                  help="continue sync even if a project fails to sync")
@@ -125,11 +136,15 @@ later is required to fix a server side protocol bug.
                  help='be more quiet')
     p.add_option('-j','--jobs',
                  dest='jobs', action='store', type='int',
-                 help="number of projects to fetch simultaneously")
+                 default=self.jobs,
+                 help="projects to fetch simultaneously (default %d)" % self.jobs)
     if show_smart:
       p.add_option('-s', '--smart-sync',
                    dest='smart_sync', action='store_true',
                    help='smart sync using manifest from a known good build')
+      p.add_option('-t', '--smart-tag',
+                   dest='smart_tag', action='store',
+                   help='smart sync using manifest from a known tag')
 
     g = p.add_option_group('repo Version options')
     g.add_option('--no-repo-verify',
@@ -308,6 +323,10 @@ uncommitted changes are present' % project.relpath
   def Execute(self, opt, args):
     if opt.jobs:
       self.jobs = opt.jobs
+    if self.jobs > 1:
+      soft_limit, _ = _rlimit_nofile()
+      self.jobs = min(self.jobs, (soft_limit - 5) / 3)
+
     if opt.network_only and opt.detach_head:
       print >>sys.stderr, 'error: cannot combine -n and -d'
       sys.exit(1)
@@ -315,27 +334,31 @@ uncommitted changes are present' % project.relpath
       print >>sys.stderr, 'error: cannot combine -n and -l'
       sys.exit(1)
 
-    if opt.smart_sync:
+    if opt.smart_sync or opt.smart_tag:
       if not self.manifest.manifest_server:
         print >>sys.stderr, \
             'error: cannot smart sync: no manifest server defined in manifest'
         sys.exit(1)
       try:
         server = xmlrpclib.Server(self.manifest.manifest_server)
-        p = self.manifest.manifestProject
+        if opt.smart_sync:
-        b = p.GetBranch(p.CurrentBranch)
+          p = self.manifest.manifestProject
-        branch = b.merge
+          b = p.GetBranch(p.CurrentBranch)
-        if branch.startswith(R_HEADS):
+          branch = b.merge
-          branch = branch[len(R_HEADS):]
+          if branch.startswith(R_HEADS):
+            branch = branch[len(R_HEADS):]
 
-        env = os.environ.copy()
+          env = os.environ.copy()
-        if (env.has_key('TARGET_PRODUCT') and
+          if (env.has_key('TARGET_PRODUCT') and
-            env.has_key('TARGET_BUILD_VARIANT')):
+              env.has_key('TARGET_BUILD_VARIANT')):
-          target = '%s-%s' % (env['TARGET_PRODUCT'],
+            target = '%s-%s' % (env['TARGET_PRODUCT'],
-                              env['TARGET_BUILD_VARIANT'])
+                                env['TARGET_BUILD_VARIANT'])
-          [success, manifest_str] = server.GetApprovedManifest(branch, target)
+            [success, manifest_str] = server.GetApprovedManifest(branch, target)
+          else:
+            [success, manifest_str] = server.GetApprovedManifest(branch)
         else:
-          [success, manifest_str] = server.GetApprovedManifest(branch)
+          assert(opt.smart_tag)
+          [success, manifest_str] = server.GetManifest(opt.smart_tag)
 
         if success:
           manifest_name = "smart_sync_override.xml"

subcmds/upload.py

@@ -120,6 +120,9 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
     p.add_option('--cc',
                  type='string',  action='append', dest='cc',
                  help='Also send email to these email addresses.')
+    p.add_option('--br',
+                 type='string',  action='store', dest='branch',
+                 help='Branch to upload.')
 
     # Options relating to upload hook.  Note that verify and no-verify are NOT
     # opposites of each other, which is why they store to different locations.
@@ -336,9 +339,13 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
     pending = []
     reviewers = []
     cc = []
+    branch = None
+
+    if opt.branch:
+      branch = opt.branch
 
     for project in project_list:
-      avail = project.GetUploadableBranches()
+      avail = project.GetUploadableBranches(branch)
       if avail:
         pending.append((project, avail))