Add a PGP key for cco3@android.com

This change adds a PGP key to allow cco3@android.com to sign releases.

Change-Id: I18a70c8b7d8f272dd1aad9d6b2e4a237ef35af33

.pydevproject

@@ -5,6 +5,6 @@
 <pydev_pathproperty name="org.python.pydev.PROJECT_SOURCE_PATH">
 <path>/repo</path>
 </pydev_pathproperty>
-<pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python 2.4</pydev_property>
+<pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python 2.6</pydev_property>
 <pydev_property name="org.python.pydev.PYTHON_PROJECT_INTERPRETER">Default</pydev_property>
 </pydev_project>

.pylintrc

@@ -0,0 +1,301 @@
+# lint Python modules using external checkers.
+#
+# This is the main checker controling the other ones and the reports
+# generation. It is itself both a raw checker and an astng checker in order
+# to:
+# * handle message activation / deactivation at the module level
+# * handle some basic but necessary stats'data (number of classes, methods...)
+#
+[MASTER]
+
+# Specify a configuration file.
+#rcfile=
+
+# Python code to execute, usually for sys.path manipulation such as
+# pygtk.require().
+#init-hook=
+
+# Profiled execution.
+profile=no
+
+# Add <file or directory> to the black list. It should be a base name, not a
+# path. You may set this option multiple times.
+ignore=SVN
+
+# Pickle collected data for later comparisons.
+persistent=yes
+
+# Set the cache size for astng objects.
+cache-size=500
+
+# List of plugins (as comma separated values of python modules names) to load,
+# usually to register additional checkers.
+load-plugins=
+
+
+[MESSAGES CONTROL]
+
+# Enable only checker(s) with the given id(s). This option conflicts with the
+# disable-checker option
+#enable-checker=
+
+# Enable all checker(s) except those with the given id(s). This option
+# conflicts with the enable-checker option
+#disable-checker=
+
+# Enable all messages in the listed categories.
+#enable-msg-cat=
+
+# Disable all messages in the listed categories.
+#disable-msg-cat=
+
+# Enable the message(s) with the given id(s).
+enable=RP0004
+
+# Disable the message(s) with the given id(s).
+disable=R0903,R0912,R0913,R0914,R0915,W0141,C0111,C0103,C0323,C0322,C0324,W0603,W0703,R0911,C0301,C0302,R0902,R0904,W0142,W0212,E1101,E1103,R0201,W0201,W0122,W0232,W0311,RP0001,RP0003,RP0101,RP0002,RP0401,RP0701,RP0801
+
+[REPORTS]
+
+# set the output format. Available formats are text, parseable, colorized, msvs
+# (visual studio) and html
+output-format=text
+
+# Include message's id in output
+include-ids=yes
+
+# Put messages in a separate file for each module / package specified on the
+# command line instead of printing them on stdout. Reports (if any) will be
+# written in a file name "pylint_global.[txt|html]".
+files-output=no
+
+# Tells whether to display a full report or only the messages
+reports=yes
+
+# Python expression which should return a note less than 10 (10 is the highest
+# note).You have access to the variables errors warning, statement which
+# respectivly contain the number of errors / warnings messages and the total
+# number of statements analyzed. This is used by the global evaluation report
+# (R0004).
+evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
+
+# Add a comment according to your evaluation note. This is used by the global
+# evaluation report (R0004).
+comment=no
+
+# checks for
+# * unused variables / imports
+# * undefined variables
+# * redefinition of variable from builtins or from an outer scope
+# * use of variable before assigment
+#
+[VARIABLES]
+
+# Tells whether we should check for unused import in __init__ files.
+init-import=no
+
+# A regular expression matching names used for dummy variables (i.e. not used).
+dummy-variables-rgx=_|dummy
+
+# List of additional names supposed to be defined in builtins. Remember that
+# you should avoid to define new builtins when possible.
+additional-builtins=
+
+
+# try to find bugs in the code using type inference
+#
+[TYPECHECK]
+
+# Tells whether missing members accessed in mixin class should be ignored. A
+# mixin class is detected if its name ends with "mixin" (case insensitive).
+ignore-mixin-members=yes
+
+# List of classes names for which member attributes should not be checked
+# (useful for classes with attributes dynamicaly set).
+ignored-classes=SQLObject
+
+# When zope mode is activated, consider the acquired-members option to ignore
+# access to some undefined attributes.
+zope=no
+
+# List of members which are usually get through zope's acquisition mecanism and
+# so shouldn't trigger E0201 when accessed (need zope=yes to be considered).
+acquired-members=REQUEST,acl_users,aq_parent
+
+
+# checks for :
+# * doc strings
+# * modules / classes / functions / methods / arguments / variables name
+# * number of arguments, local variables, branchs, returns and statements in
+# functions, methods
+# * required module attributes
+# * dangerous default values as arguments
+# * redefinition of function / method / class
+# * uses of the global statement
+#
+[BASIC]
+
+# Required attributes for module, separated by a comma
+required-attributes=
+
+# Regular expression which should only match functions or classes name which do
+# not require a docstring
+no-docstring-rgx=_main|__.*__
+
+# Regular expression which should only match correct module names
+module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
+
+# Regular expression which should only match correct module level names
+const-rgx=(([A-Z_][A-Z1-9_]*)|(__.*__))|(log)$
+
+# Regular expression which should only match correct class names
+class-rgx=[A-Z_][a-zA-Z0-9]+$
+
+# Regular expression which should only match correct function names
+function-rgx=[a-z_][a-z0-9_]{2,30}$
+
+# Regular expression which should only match correct method names
+method-rgx=[a-z_][a-z0-9_]{2,30}$
+
+# Regular expression which should only match correct instance attribute names
+attr-rgx=[a-z_][a-z0-9_]{2,30}$
+
+# Regular expression which should only match correct argument names
+argument-rgx=[a-z_][a-z0-9_]{2,30}$
+
+# Regular expression which should only match correct variable names
+variable-rgx=[a-z_][a-z0-9_]{2,30}$
+
+# Regular expression which should only match correct list comprehension /
+# generator expression variable names
+inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$
+
+# Good variable names which should always be accepted, separated by a comma
+good-names=i,j,k,ex,Run,_,e,d1,d2,v,f,l,d
+
+# Bad variable names which should always be refused, separated by a comma
+bad-names=foo,bar,baz,toto,tutu,tata
+
+# List of builtins function names that should not be used, separated by a comma
+bad-functions=map,filter,apply,input
+
+
+# checks for sign of poor/misdesign:
+# * number of methods, attributes, local variables...
+# * size, complexity of functions, methods
+#
+[DESIGN]
+
+# Maximum number of arguments for function / method
+max-args=5
+
+# Maximum number of locals for function / method body
+max-locals=15
+
+# Maximum number of return / yield for function / method body
+max-returns=6
+
+# Maximum number of branch for function / method body
+max-branchs=12
+
+# Maximum number of statements in function / method body
+max-statements=50
+
+# Maximum number of parents for a class (see R0901).
+max-parents=7
+
+# Maximum number of attributes for a class (see R0902).
+max-attributes=20
+
+# Minimum number of public methods for a class (see R0903).
+min-public-methods=2
+
+# Maximum number of public methods for a class (see R0904).
+max-public-methods=30
+
+
+# checks for
+# * external modules dependencies
+# * relative / wildcard imports
+# * cyclic imports
+# * uses of deprecated modules
+#
+[IMPORTS]
+
+# Deprecated modules which should not be used, separated by a comma
+deprecated-modules=regsub,string,TERMIOS,Bastion,rexec
+
+# Create a graph of every (i.e. internal and external) dependencies in the
+# given file (report R0402 must not be disabled)
+import-graph=
+
+# Create a graph of external dependencies in the given file (report R0402 must
+# not be disabled)
+ext-import-graph=
+
+# Create a graph of internal dependencies in the given file (report R0402 must
+# not be disabled)
+int-import-graph=
+
+
+# checks for :
+# * methods without self as first argument
+# * overridden methods signature
+# * access only to existant members via self
+# * attributes not defined in the __init__ method
+# * supported interfaces implementation
+# * unreachable code
+#
+[CLASSES]
+
+# List of interface methods to ignore, separated by a comma. This is used for
+# instance to not check methods defines in Zope's Interface base class.
+ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by
+
+# List of method names used to declare (i.e. assign) instance attributes.
+defining-attr-methods=__init__,__new__,setUp
+
+
+# checks for similarities and duplicated code. This computation may be
+# memory / CPU intensive, so you should disable it if you experiments some
+# problems.
+#
+[SIMILARITIES]
+
+# Minimum lines number of a similarity.
+min-similarity-lines=4
+
+# Ignore comments when computing similarities.
+ignore-comments=yes
+
+# Ignore docstrings when computing similarities.
+ignore-docstrings=yes
+
+
+# checks for:
+# * warning notes in the code like FIXME, XXX
+# * PEP 263: source code with non ascii character but no encoding declaration
+#
+[MISCELLANEOUS]
+
+# List of note tags to take in consideration, separated by a comma.
+notes=FIXME,XXX,TODO
+
+
+# checks for :
+# * unauthorized constructions
+# * strict indentation
+# * line length
+# * use of <> instead of !=
+#
+[FORMAT]
+
+# Maximum number of characters on a single line.
+max-line-length=80
+
+# Maximum number of lines in a module
+max-module-lines=1000
+
+# String used as indentation unit. This is usually "    " (4 spaces) or "\t" (1
+# tab).  In repo it is 2 spaces.
+indent-string='  '

SUBMITTING_PATCHES

@@ -2,6 +2,7 @@ Short Version:
 
  - Make small logical changes.
  - Provide a meaningful commit message.
+ - Check for coding errors with pylint
  - Make sure all code is under the Apache License, 2.0.
  - Publish your changes for review:
 
@@ -33,7 +34,14 @@ If your description starts to get too long, that's a sign that you
 probably need to split up your commit to finer grained pieces.
 
 
-(2) Check the license
+(2) Check for coding errors with pylint
+
+Run pylint on changed modules using the provided configuration:
+
+  pylint --rcfile=.pylintrc file.py
+
+
+(3) Check the license
 
 repo is licensed under the Apache License, 2.0.
 
@@ -49,7 +57,7 @@ your patch.  It is virtually impossible to remove a patch once it
 has been applied and pushed out.
 
 
-(3) Sending your patches.
+(4) Sending your patches.
 
 Do not email your patches to anyone.
 

color.py

@@ -38,8 +38,11 @@ ATTRS = {None     :-1,
 
 RESET = "\033[m"
 
-def is_color(s): return s in COLORS
-def is_attr(s):  return s in ATTRS
+def is_color(s):
+    return s in COLORS
+
+def is_attr(s):
+    return s in ATTRS
 
 def _Color(fg = None, bg = None, attr = None):
     fg = COLORS[fg]
@@ -80,8 +83,8 @@ def _Color(fg = None, bg = None, attr = None):
 
 
 class Coloring(object):
-  def __init__(self, config, type):
-    self._section = 'color.%s' % type
+  def __init__(self, config, section_type):
+    self._section = 'color.%s' % section_type
     self._config = config
     self._out = sys.stdout
 
@@ -126,8 +129,8 @@ class Coloring(object):
     if self._on:
       c = self._parse(opt, fg, bg, attr)
       def f(fmt, *args):
-        str = fmt % args
-        return ''.join([c, str, RESET])
+        output = fmt % args
+        return ''.join([c, output, RESET])
       return f
     else:
       def f(fmt, *args):
@@ -151,8 +154,10 @@ class Coloring(object):
     have_fg = False
     for a in v.split(' '):
       if is_color(a):
-        if have_fg: bg = a
-        else:       fg = a
+        if have_fg:
+          bg = a
+        else:
+          fg = a
       elif is_attr(a):
         attr = a
 

command.py

@@ -60,10 +60,36 @@ class Command(object):
     """
     raise NotImplementedError
 
+  def _ResetPathToProjectMap(self, projects):
+    self._by_path = dict((p.worktree, p) for p in projects)
+
+  def _UpdatePathToProjectMap(self, project):
+    self._by_path[project.worktree] = project
+
+  def _GetProjectByPath(self, path):
+    project = None
+    if os.path.exists(path):
+      oldpath = None
+      while path \
+        and path != oldpath \
+        and path != self.manifest.topdir:
+        try:
+          project = self._by_path[path]
+          break
+        except KeyError:
+          oldpath = path
+          path = os.path.dirname(path)
+    else:
+      try:
+        project = self._by_path[path]
+      except KeyError:
+        pass
+    return project
+
   def GetProjects(self, args, missing_ok=False):
     """A list of projects that match the arguments.
     """
-    all = self.manifest.projects
+    all_projects = self.manifest.projects
     result = []
 
     mp = self.manifest.manifestProject
@@ -74,40 +100,38 @@ class Command(object):
     groups = [x for x in re.split('[,\s]+', groups) if x]
 
     if not args:
-      for project in all.values():
+      all_projects_list = all_projects.values()
+      derived_projects = []
+      for project in all_projects_list:
+        if project.Registered:
+          # Do not search registered subproject for derived projects
+          # since its parent has been searched already
+          continue
+        derived_projects.extend(project.GetDerivedSubprojects())
+      all_projects_list.extend(derived_projects)
+      for project in all_projects_list:
         if ((missing_ok or project.Exists) and
             project.MatchesGroups(groups)):
           result.append(project)
     else:
-      by_path = None
+      self._ResetPathToProjectMap(all_projects.values())
 
       for arg in args:
-        project = all.get(arg)
+        project = all_projects.get(arg)
 
         if not project:
           path = os.path.abspath(arg).replace('\\', '/')
+          project = self._GetProjectByPath(path)
 
-          if not by_path:
-            by_path = dict()
-            for p in all.values():
-              by_path[p.worktree] = p
-
-          if os.path.exists(path):
-            oldpath = None
-            while path \
-              and path != oldpath \
-              and path != self.manifest.topdir:
-              try:
-                project = by_path[path]
-                break
-              except KeyError:
-                oldpath = path
-                path = os.path.dirname(path)
-          else:
-            try:
-              project = by_path[path]
-            except KeyError:
-              pass
+          # If it's not a derived project, update path->project mapping and
+          # search again, as arg might actually point to a derived subproject.
+          if project and not project.Derived:
+            search_again = False
+            for subproject in project.GetDerivedSubprojects():
+              self._UpdatePathToProjectMap(subproject)
+              search_again = True
+            if search_again:
+              project = self._GetProjectByPath(path) or project
 
         if not project:
           raise NoSuchProjectError(arg)
@@ -123,6 +147,11 @@ class Command(object):
     result.sort(key=_getpath)
     return result
 
+# pylint: disable=W0223
+# Pylint warns that the `InteractiveCommand` and `PagedCommand` classes do not
+# override method `Execute` which is abstract in `Command`.  Since that method
+# is always implemented in classes derived from `InteractiveCommand` and
+# `PagedCommand`, this warning can be suppressed.
 class InteractiveCommand(Command):
   """Command which requires user interaction on the tty and
      must not run within a pager, even if the user asks to.
@@ -137,6 +166,8 @@ class PagedCommand(Command):
   def WantPager(self, opt):
     return True
 
+# pylint: enable=W0223
+
 class MirrorSafeCommand(object):
   """Command permits itself to run within a mirror,
      and does not require a working directory.

docs/manifest-format.txt

@@ -45,7 +45,8 @@ following DTD:
     <!ELEMENT manifest-server (EMPTY)>
     <!ATTLIST url              CDATA #REQUIRED>
   
-    <!ELEMENT project (annotation?)>
+    <!ELEMENT project (annotation?,
+                       project*)>
     <!ATTLIST project name     CDATA #REQUIRED>
     <!ATTLIST project path     CDATA #IMPLIED>
     <!ATTLIST project remote   IDREF #IMPLIED>
@@ -152,7 +153,10 @@ Element project
 
 One or more project elements may be specified.  Each element
 describes a single Git repository to be cloned into the repo
-client workspace.
+client workspace.  You may specify Git-submodules by creating a
+nested project.  Git-submodules will be automatically
+recognized and inherit their parent's attributes, but those
+may be overridden by an explicitly specified project element.
 
 Attribute `name`: A unique name for this project.  The project's
 name is appended onto its remote's fetch URL to generate the actual
@@ -163,7 +167,8 @@ URL to configure the Git remote with.  The URL gets formed as:
 where ${remote_fetch} is the remote's fetch attribute and
 ${project_name} is the project's name attribute.  The suffix ".git"
 is always appended as repo assumes the upstream is a forest of
-bare Git repositories.
+bare Git repositories.  If the project has a parent element, its
+name will be prefixed by the parent's.
 
 The project name must match the name Gerrit knows, if Gerrit is
 being used for code reviews.
@@ -171,6 +176,8 @@ being used for code reviews.
 Attribute `path`: An optional path relative to the top directory
 of the repo client where the Git working directory for this project
 should be placed.  If not supplied the project name is used.
+If the project has a parent element, its path will be prefixed
+by the parent's.
 
 Attribute `remote`: Name of a previously defined remote element.
 If not supplied the remote given by the default element is used.
@@ -190,6 +197,8 @@ its name:`name` and path:`path`.  E.g. for
 definition is implicitly in the following manifest groups:
 default, name:monkeys, and path:barrel-of.  If you place a project in the
 group "notdefault", it will not be automatically downloaded by repo.
+If the project has a parent element, the `name` and `path` here
+are the prefixed ones.
 
 Element annotation
 ------------------

editor.py

@@ -91,7 +91,7 @@ least one of these before using this command."""
 
       try:
         rc = subprocess.Popen(args, shell=shell).wait()
-      except OSError, e:
+      except OSError as e:
         raise EditorError('editor failed, %s: %s %s'
           % (str(e), editor, path))
       if rc != 0:

error.py

@@ -25,6 +25,7 @@ class EditorError(Exception):
   """Unspecified error from the user's text editor.
   """
   def __init__(self, reason):
+    super(EditorError, self).__init__()
     self.reason = reason
 
   def __str__(self):
@@ -34,24 +35,17 @@ class GitError(Exception):
   """Unspecified internal error from git.
   """
   def __init__(self, command):
+    super(GitError, self).__init__()
     self.command = command
 
   def __str__(self):
     return self.command
 
-class ImportError(Exception):
-  """An import from a non-Git format cannot be performed.
-  """
-  def __init__(self, reason):
-    self.reason = reason
-
-  def __str__(self):
-    return self.reason
-
 class UploadError(Exception):
   """A bundle upload to Gerrit did not succeed.
   """
   def __init__(self, reason):
+    super(UploadError, self).__init__()
     self.reason = reason
 
   def __str__(self):
@@ -61,6 +55,7 @@ class DownloadError(Exception):
   """Cannot download a repository.
   """
   def __init__(self, reason):
+    super(DownloadError, self).__init__()
     self.reason = reason
 
   def __str__(self):
@@ -70,6 +65,7 @@ class NoSuchProjectError(Exception):
   """A specified project does not exist in the work tree.
   """
   def __init__(self, name=None):
+    super(NoSuchProjectError, self).__init__()
     self.name = name
 
   def __str__(self):
@@ -82,6 +78,7 @@ class InvalidProjectGroupsError(Exception):
   """A specified project is not suitable for the specified groups
   """
   def __init__(self, name=None):
+    super(InvalidProjectGroupsError, self).__init__()
     self.name = name
 
   def __str__(self):
@@ -94,12 +91,12 @@ class RepoChangedException(Exception):
      repo or manifest repositories.  In this special case we must
      use exec to re-execute repo with the new code and manifest.
   """
-  def __init__(self, extra_args=[]):
-    self.extra_args = extra_args
+  def __init__(self, extra_args=None):
+    super(RepoChangedException, self).__init__()
+    self.extra_args = extra_args or []
 
 class HookError(Exception):
   """Thrown if a 'repo-hook' could not be run.
 
   The common case is that the file wasn't present when we tried to run it.
   """
-  pass

git_command.py

@@ -37,11 +37,11 @@ def ssh_sock(create=True):
   if _ssh_sock_path is None:
     if not create:
       return None
-    dir = '/tmp'
-    if not os.path.exists(dir):
-      dir = tempfile.gettempdir()
+    tmp_dir = '/tmp'
+    if not os.path.exists(tmp_dir):
+      tmp_dir = tempfile.gettempdir()
     _ssh_sock_path = os.path.join(
-      tempfile.mkdtemp('', 'ssh-', dir),
+      tempfile.mkdtemp('', 'ssh-', tmp_dir),
       'master-%r@%h:%p')
   return _ssh_sock_path
 
@@ -217,7 +217,7 @@ class GitCommand(object):
                            stdin = stdin,
                            stdout = stdout,
                            stderr = stderr)
-    except Exception, e:
+    except Exception as e:
       raise GitError('%s: %s' % (command[1], e))
 
     if ssh_proxy:

git_config.py

@@ -56,16 +56,16 @@ class GitConfig(object):
   @classmethod
   def ForUser(cls):
     if cls._ForUser is None:
-      cls._ForUser = cls(file = os.path.expanduser('~/.gitconfig'))
+      cls._ForUser = cls(configfile = os.path.expanduser('~/.gitconfig'))
     return cls._ForUser
 
   @classmethod
   def ForRepository(cls, gitdir, defaults=None):
-    return cls(file = os.path.join(gitdir, 'config'),
+    return cls(configfile = os.path.join(gitdir, 'config'),
                defaults = defaults)
 
-  def __init__(self, file, defaults=None, pickleFile=None):
-    self.file = file
+  def __init__(self, configfile, defaults=None, pickleFile=None):
+    self.file = configfile
     self.defaults = defaults
     self._cache_dict = None
     self._section_dict = None
@@ -104,20 +104,20 @@ class GitConfig(object):
       return False
     return None
 
-  def GetString(self, name, all=False):
+  def GetString(self, name, all_keys=False):
     """Get the first value for a key, or None if it is not defined.
 
        This configuration file is used first, if the key is not
-       defined or all = True then the defaults are also searched.
+       defined or all_keys = True then the defaults are also searched.
     """
     try:
       v = self._cache[_key(name)]
     except KeyError:
       if self.defaults:
-        return self.defaults.GetString(name, all = all)
+        return self.defaults.GetString(name, all_keys = all_keys)
       v = []
 
-    if not all:
+    if not all_keys:
       if v:
         return v[0]
       return None
@@ -125,7 +125,7 @@ class GitConfig(object):
     r = []
     r.extend(v)
     if self.defaults:
-      r.extend(self.defaults.GetString(name, all = True))
+      r.extend(self.defaults.GetString(name, all_keys = True))
     return r
 
   def SetString(self, name, value):
@@ -449,7 +449,7 @@ def _open_ssh(host, port=None):
     try:
       Trace(': %s', ' '.join(command))
       p = subprocess.Popen(command)
-    except Exception, e:
+    except Exception as e:
       _ssh_master = False
       print >>sys.stderr, \
         '\nwarn: cannot enable ssh control master for %s:%s\n%s' \
@@ -526,7 +526,7 @@ class Remote(object):
     self.review = self._Get('review')
     self.projectname = self._Get('projectname')
     self.fetch = map(lambda x: RefSpec.FromString(x),
-                     self._Get('fetch', all=True))
+                     self._Get('fetch', all_keys=True))
     self._review_url = None
 
   def _InsteadOf(self):
@@ -537,7 +537,7 @@ class Remote(object):
 
     for url in urlList:
       key = "url." + url + ".insteadOf"
-      insteadOfList = globCfg.GetString(key, all=True)
+      insteadOfList = globCfg.GetString(key, all_keys=True)
 
       for insteadOf in insteadOfList:
         if self.url.startswith(insteadOf) \
@@ -567,7 +567,7 @@ class Remote(object):
       if u.endswith('/ssh_info'):
         u = u[:len(u) - len('/ssh_info')]
       if not u.endswith('/'):
-         u += '/'
+        u += '/'
       http_url = u
 
       if u in REVIEW_CACHE:
@@ -592,9 +592,9 @@ class Remote(object):
           else:
             host, port = info.split()
             self._review_url = self._SshReviewUrl(userEmail, host, port)
-        except urllib2.HTTPError, e:
+        except urllib2.HTTPError as e:
           raise UploadError('%s: %s' % (self.review, str(e)))
-        except urllib2.URLError, e:
+        except urllib2.URLError as e:
           raise UploadError('%s: %s' % (self.review, str(e)))
 
         REVIEW_CACHE[u] = self._review_url
@@ -651,9 +651,9 @@ class Remote(object):
     key = 'remote.%s.%s' % (self.name, key)
     return self._config.SetString(key, value)
 
-  def _Get(self, key, all=False):
+  def _Get(self, key, all_keys=False):
     key = 'remote.%s.%s' % (self.name, key)
-    return self._config.GetString(key, all = all)
+    return self._config.GetString(key, all_keys = all_keys)
 
 
 class Branch(object):
@@ -703,6 +703,6 @@ class Branch(object):
     key = 'branch.%s.%s' % (self.name, key)
     return self._config.SetString(key, value)
 
-  def _Get(self, key, all=False):
+  def _Get(self, key, all_keys=False):
     key = 'branch.%s.%s' % (self.name, key)
-    return self._config.GetString(key, all = all)
+    return self._config.GetString(key, all_keys = all_keys)

git_refs.py

@@ -115,10 +115,10 @@ class GitRefs(object):
 
         line = line[:-1]
         p = line.split(' ')
-        id = p[0]
+        ref_id = p[0]
         name = p[1]
 
-        self._phyref[name] = id
+        self._phyref[name] = ref_id
     finally:
       fd.close()
     self._mtime['packed-refs'] = mtime
@@ -144,18 +144,18 @@ class GitRefs(object):
     try:
       try:
         mtime = os.path.getmtime(path)
-        id = fd.readline()
+        ref_id = fd.readline()
       except:
         return
     finally:
       fd.close()
 
-    if not id:
+    if not ref_id:
       return
-    id = id[:-1]
+    ref_id = ref_id[:-1]
 
-    if id.startswith('ref: '):
-      self._symref[name] = id[5:]
+    if ref_id.startswith('ref: '):
+      self._symref[name] = ref_id[5:]
     else:
-      self._phyref[name] = id
+      self._phyref[name] = ref_id
     self._mtime[name] = mtime

hooks/commit-msg

@@ -1,5 +1,5 @@
 #!/bin/sh
-# From Gerrit Code Review 2.1.2-rc2-33-g7e30c72
+# From Gerrit Code Review 2.5-rc0
 #
 # Part of Gerrit Code Review (http://code.google.com/p/gerrit/)
 #
@@ -24,71 +24,144 @@ MSG="$1"
 # Check for, and add if missing, a unique Change-Id
 #
 add_ChangeId() {
-	clean_message=$(sed -e '
+	clean_message=`sed -e '
 		/^diff --git a\/.*/{
 			s///
 			q
 		}
 		/^Signed-off-by:/d
 		/^#/d
-	' "$MSG" | git stripspace)
+	' "$MSG" | git stripspace`
 	if test -z "$clean_message"
 	then
 		return
 	fi
 
+	# Does Change-Id: already exist? if so, exit (no change).
 	if grep -i '^Change-Id:' "$MSG" >/dev/null
 	then
 		return
 	fi
 
-	id=$(_gen_ChangeId)
-	perl -e '
-		$MSG = shift;
-		$id = shift;
-		$CHANGE_ID_AFTER = shift;
+	id=`_gen_ChangeId`
+	T="$MSG.tmp.$$"
+	AWK=awk
+	if [ -x /usr/xpg4/bin/awk ]; then
+		# Solaris AWK is just too broken
+		AWK=/usr/xpg4/bin/awk
+	fi
 
-		undef $/;
-		open(I, $MSG); $_ = <I>; close I;
-		s|^diff --git a/.*||ms;
-		s|^#.*$||mg;
-		exit unless $_;
+	# How this works:
+	# - parse the commit message as (textLine+ blankLine*)*
+	# - assume textLine+ to be a footer until proven otherwise
+	# - exception: the first block is not footer (as it is the title)
+	# - read textLine+ into a variable
+	# - then count blankLines
+	# - once the next textLine appears, print textLine+ blankLine* as these
+	#   aren't footer
+	# - in END, the last textLine+ block is available for footer parsing
+	$AWK '
+	BEGIN {
+		# while we start with the assumption that textLine+
+		# is a footer, the first block is not.
+		isFooter = 0
+		footerComment = 0
+		blankLines = 0
+	}
 
-		@message = split /\n/;
-		$haveFooter = 0;
-		$startFooter = @message;
-		for($line = @message - 1; $line >= 0; $line--) {
-			$_ = $message[$line];
+	# Skip lines starting with "#" without any spaces before it.
+	/^#/ { next }
 
-			($haveFooter++, next) if /^[a-zA-Z0-9-]+:/;
-			next if /^[ []/;
-			$startFooter = $line if ($haveFooter && /^\r?$/);
-			last;
+	# Skip the line starting with the diff command and everything after it,
+	# up to the end of the file, assuming it is only patch data.
+	# If more than one line before the diff was empty, strip all but one.
+	/^diff --git a/ {
+		blankLines = 0
+		while (getline) { }
+		next
+	}
+
+	# Count blank lines outside footer comments
+	/^$/ && (footerComment == 0) {
+		blankLines++
+		next
+	}
+
+	# Catch footer comment
+	/^\[[a-zA-Z0-9-]+:/ && (isFooter == 1) {
+		footerComment = 1
+	}
+
+	/]$/ && (footerComment == 1) {
+		footerComment = 2
+	}
+
+	# We have a non-blank line after blank lines. Handle this.
+	(blankLines > 0) {
+		print lines
+		for (i = 0; i < blankLines; i++) {
+			print ""
 		}
 
-		@footer = @message[$startFooter+1..@message];
-		@message = @message[0..$startFooter];
-		push(@footer, "") unless @footer;
+		lines = ""
+		blankLines = 0
+		isFooter = 1
+		footerComment = 0
+	}
 
-		for ($line = 0; $line < @footer; $line++) {
-			$_ = $footer[$line];
-			next if /^($CHANGE_ID_AFTER):/i;
-			last;
+	# Detect that the current block is not the footer
+	(footerComment == 0) && (!/^\[?[a-zA-Z0-9-]+:/ || /^[a-zA-Z0-9-]+:\/\//) {
+		isFooter = 0
+	}
+
+	{
+		# We need this information about the current last comment line
+		if (footerComment == 2) {
+			footerComment = 0
 		}
-		splice(@footer, $line, 0, "Change-Id: I$id");
+		if (lines != "") {
+			lines = lines "\n";
+		}
+		lines = lines $0
+	}
 
-		$_ = join("\n", @message, @footer);
-		open(O, ">$MSG"); print O; close O;
-	' "$MSG" "$id" "$CHANGE_ID_AFTER"
+	# Footer handling:
+	# If the last block is considered a footer, splice in the Change-Id at the
+	# right place.
+	# Look for the right place to inject Change-Id by considering
+	# CHANGE_ID_AFTER. Keys listed in it (case insensitive) come first,
+	# then Change-Id, then everything else (eg. Signed-off-by:).
+	#
+	# Otherwise just print the last block, a new line and the Change-Id as a
+	# block of its own.
+	END {
+		unprinted = 1
+		if (isFooter == 0) {
+			print lines "\n"
+			lines = ""
+		}
+		changeIdAfter = "^(" tolower("'"$CHANGE_ID_AFTER"'") "):"
+		numlines = split(lines, footer, "\n")
+		for (line = 1; line <= numlines; line++) {
+			if (unprinted && match(tolower(footer[line]), changeIdAfter) != 1) {
+				unprinted = 0
+				print "Change-Id: I'"$id"'"
+			}
+			print footer[line]
+		}
+		if (unprinted) {
+			print "Change-Id: I'"$id"'"
+		}
+	}' "$MSG" > $T && mv $T "$MSG" || rm -f $T
 }
 _gen_ChangeIdInput() {
-	echo "tree $(git write-tree)"
-	if parent=$(git rev-parse HEAD^0 2>/dev/null)
+	echo "tree `git write-tree`"
+	if parent=`git rev-parse "HEAD^0" 2>/dev/null`
 	then
 		echo "parent $parent"
 	fi
-	echo "author $(git var GIT_AUTHOR_IDENT)"
-	echo "committer $(git var GIT_COMMITTER_IDENT)"
+	echo "author `git var GIT_AUTHOR_IDENT`"
+	echo "committer `git var GIT_COMMITTER_IDENT`"
 	echo
 	printf '%s' "$clean_message"
 }

main.py

@@ -22,6 +22,8 @@ if __name__ == '__main__':
     del sys.argv[-1]
 del magic
 
+import getpass
+import imp
 import netrc
 import optparse
 import os
@@ -44,7 +46,7 @@ from error import RepoChangedException
 from manifest_xml import XmlManifest
 from pager import RunPager
 
-from subcmds import all as all_commands
+from subcmds import all_commands
 
 global_options = optparse.OptionParser(
                  usage="repo [-p|--paginate|--no-pager] COMMAND [ARGS]"
@@ -88,7 +90,7 @@ class _Repo(object):
       glob = argv
       name = 'help'
       argv = []
-    gopts, gargs = global_options.parse_args(glob)
+    gopts, _gargs = global_options.parse_args(glob)
 
     if gopts.trace:
       SetTrace()
@@ -145,13 +147,13 @@ class _Repo(object):
           else:
             print >>sys.stderr, 'real\t%dh%dm%.3fs' \
               % (hours, minutes, seconds)
-    except DownloadError, e:
+    except DownloadError as e:
       print >>sys.stderr, 'error: %s' % str(e)
       return 1
-    except ManifestInvalidRevisionError, e:
+    except ManifestInvalidRevisionError as e:
       print >>sys.stderr, 'error: %s' % str(e)
       return 1
-    except NoSuchProjectError, e:
+    except NoSuchProjectError as e:
       if e.name:
         print >>sys.stderr, 'error: project %s not found' % e.name
       else:
@@ -166,24 +168,23 @@ def _MyRepoPath():
 def _MyWrapperPath():
   return os.path.join(os.path.dirname(__file__), 'repo')
 
+_wrapper_module = None
+def WrapperModule():
+  global _wrapper_module
+  if not _wrapper_module:
+    _wrapper_module = imp.load_source('wrapper', _MyWrapperPath())
+  return _wrapper_module
+
 def _CurrentWrapperVersion():
-  VERSION = None
-  pat = re.compile(r'^VERSION *=')
-  fd = open(_MyWrapperPath())
-  for line in fd:
-    if pat.match(line):
-      fd.close()
-      exec line
-      return VERSION
-  raise NameError, 'No VERSION in repo script'
+  return WrapperModule().VERSION
 
 def _CheckWrapperVersion(ver, repo_path):
   if not repo_path:
     repo_path = '~/bin/repo'
 
   if not ver:
-     print >>sys.stderr, 'no --wrapper-version argument'
-     sys.exit(1)
+    print >>sys.stderr, 'no --wrapper-version argument'
+    sys.exit(1)
 
   exp = _CurrentWrapperVersion()
   ver = tuple(map(lambda x: int(x), ver.split('.')))
@@ -209,10 +210,10 @@ def _CheckWrapperVersion(ver, repo_path):
     cp %s %s
 """ % (exp_str, _MyWrapperPath(), repo_path)
 
-def _CheckRepoDir(dir):
-  if not dir:
-     print >>sys.stderr, 'no --repo-dir argument'
-     sys.exit(1)
+def _CheckRepoDir(repo_dir):
+  if not repo_dir:
+    print >>sys.stderr, 'no --repo-dir argument'
+    sys.exit(1)
 
 def _PruneOptions(argv, opt):
   i = 0
@@ -276,7 +277,25 @@ class _UserAgentHandler(urllib2.BaseHandler):
     req.add_header('User-Agent', _UserAgent())
     return req
 
+def _AddPasswordFromUserInput(handler, msg, req):
+    # If repo could not find auth info from netrc, try to get it from user input
+    url = req.get_full_url()
+    user, password = handler.passwd.find_user_password(None, url)
+    if user is None:
+      print msg
+      try:
+        user = raw_input('User: ')
+        password = getpass.getpass()
+      except KeyboardInterrupt:
+        return
+      handler.passwd.add_password(None, url, user, password)
+
 class _BasicAuthHandler(urllib2.HTTPBasicAuthHandler):
+  def http_error_401(self, req, fp, code, msg, headers):
+    _AddPasswordFromUserInput(self, msg, req)
+    return urllib2.HTTPBasicAuthHandler.http_error_401(
+      self, req, fp, code, msg, headers)
+
   def http_error_auth_reqed(self, authreq, host, req, headers):
     try:
       old_add_header = req.add_header
@@ -295,6 +314,11 @@ class _BasicAuthHandler(urllib2.HTTPBasicAuthHandler):
       raise
 
 class _DigestAuthHandler(urllib2.HTTPDigestAuthHandler):
+  def http_error_401(self, req, fp, code, msg, headers):
+    _AddPasswordFromUserInput(self, msg, req)
+    return urllib2.HTTPDigestAuthHandler.http_error_401(
+      self, req, fp, code, msg, headers)
+
   def http_error_auth_reqed(self, auth_header, host, req, headers):
     try:
       old_add_header = req.add_header
@@ -366,14 +390,14 @@ def _Main(argv):
       close_ssh()
   except KeyboardInterrupt:
     result = 1
-  except RepoChangedException, rce:
+  except RepoChangedException as rce:
     # If repo changed, re-exec ourselves.
     #
     argv = list(sys.argv)
     argv.extend(rce.extra_args)
     try:
       os.execv(__file__, argv)
-    except OSError, e:
+    except OSError as e:
       print >>sys.stderr, 'fatal: cannot restart repo after upgrade'
       print >>sys.stderr, 'fatal: %s' % e
       result = 128

manifest_xml.py

@@ -21,7 +21,8 @@ import urlparse
 import xml.dom.minidom
 
 from git_config import GitConfig
-from project import RemoteSpec, Project, MetaProject, R_HEADS, HEAD
+from git_refs import R_HEADS, HEAD
+from project import RemoteSpec, Project, MetaProject
 from error import ManifestParseError
 
 MANIFEST_FILE_NAME = 'manifest.xml'
@@ -112,7 +113,7 @@ class XmlManifest(object):
       if os.path.exists(self.manifestFile):
         os.remove(self.manifestFile)
       os.symlink('manifests/%s' % name, self.manifestFile)
-    except OSError, e:
+    except OSError:
       raise ManifestParseError('cannot link manifest %s' % name)
 
   def _RemoteToXml(self, r, doc, root):
@@ -123,7 +124,7 @@ class XmlManifest(object):
     if r.reviewUrl is not None:
       e.setAttribute('review', r.reviewUrl)
 
-  def Save(self, fd, peg_rev=False):
+  def Save(self, fd, peg_rev=False, peg_rev_upstream=True):
     """Write the current manifest out to the given file descriptor.
     """
     mp = self.manifestProject
@@ -179,29 +180,38 @@ class XmlManifest(object):
       root.appendChild(e)
       root.appendChild(doc.createTextNode(''))
 
-    sort_projects = list(self.projects.keys())
-    sort_projects.sort()
-
-    for p in sort_projects:
-      p = self.projects[p]
+    def output_projects(parent, parent_node, projects):
+      for p in projects:
+        output_project(parent, parent_node, self.projects[p])
 
+    def output_project(parent, parent_node, p):
       if not p.MatchesGroups(groups):
-        continue
+        return
+
+      name = p.name
+      relpath = p.relpath
+      if parent:
+        name = self._UnjoinName(parent.name, name)
+        relpath = self._UnjoinRelpath(parent.relpath, relpath)
 
       e = doc.createElement('project')
-      root.appendChild(e)
-      e.setAttribute('name', p.name)
-      if p.relpath != p.name:
-        e.setAttribute('path', p.relpath)
+      parent_node.appendChild(e)
+      e.setAttribute('name', name)
+      if relpath != name:
+        e.setAttribute('path', relpath)
       if not d.remote or p.remote.name != d.remote.name:
         e.setAttribute('remote', p.remote.name)
       if peg_rev:
         if self.IsMirror:
-          e.setAttribute('revision',
-                         p.bare_git.rev_parse(p.revisionExpr + '^0'))
+          value = p.bare_git.rev_parse(p.revisionExpr + '^0')
         else:
-          e.setAttribute('revision',
-                         p.work_git.rev_parse(HEAD + '^0'))
+          value = p.work_git.rev_parse(HEAD + '^0')
+        e.setAttribute('revision', value)
+        if peg_rev_upstream and value != p.revisionExpr:
+          # Only save the origin if the origin is not a sha1, and the default
+          # isn't our value, and the if the default doesn't already have that
+          # covered.
+          e.setAttribute('upstream', p.revisionExpr)
       elif not d.revisionExpr or p.revisionExpr != d.revisionExpr:
         e.setAttribute('revision', p.revisionExpr)
 
@@ -226,6 +236,16 @@ class XmlManifest(object):
       if p.sync_c:
         e.setAttribute('sync-c', 'true')
 
+      if p.subprojects:
+        sort_projects = [subp.name for subp in p.subprojects]
+        sort_projects.sort()
+        output_projects(p, e, sort_projects)
+
+    sort_projects = [key for key in self.projects.keys()
+                     if not self.projects[key].parent]
+    sort_projects.sort()
+    output_projects(None, root, sort_projects)
+
     if self._repo_hooks_project:
       root.appendChild(doc.createTextNode(''))
       e = doc.createElement('repo-hooks')
@@ -316,7 +336,8 @@ class XmlManifest(object):
       raise ManifestParseError("no <manifest> in %s" % (path,))
 
     nodes = []
-    for node in manifest.childNodes:
+    for node in manifest.childNodes:  # pylint:disable=W0631
+                                      # We only get here if manifest is initialised
         if node.nodeName == 'include':
             name = self._reqatt(node, 'name')
             fp = os.path.join(include_root, name)
@@ -330,7 +351,7 @@ class XmlManifest(object):
             # tricky.  actual parsing implementation may vary.
             except (KeyboardInterrupt, RuntimeError, SystemExit):
                 raise
-            except Exception, e:
+            except Exception as e:
                 raise ManifestParseError(
                     "failed parsing included manifest %s: %s", (name, e))
         else:
@@ -377,11 +398,15 @@ class XmlManifest(object):
     for node in itertools.chain(*node_list):
       if node.nodeName == 'project':
         project = self._ParseProject(node)
-        if self._projects.get(project.name):
-          raise ManifestParseError(
-              'duplicate project %s in %s' %
-              (project.name, self.manifestFile))
-        self._projects[project.name] = project
+        def recursively_add_projects(project):
+          if self._projects.get(project.name):
+            raise ManifestParseError(
+                'duplicate project %s in %s' %
+                (project.name, self.manifestFile))
+          self._projects[project.name] = project
+          for subproject in project.subprojects:
+            recursively_add_projects(subproject)
+        recursively_add_projects(project)
       if node.nodeName == 'repo-hooks':
         # Get the name of the project and the (space-separated) list of enabled.
         repo_hooks_project = self._reqatt(node, 'in-project')
@@ -531,11 +556,19 @@ class XmlManifest(object):
 
     return '\n'.join(cleanLines)
 
-  def _ParseProject(self, node):
+  def _JoinName(self, parent_name, name):
+    return os.path.join(parent_name, name)
+
+  def _UnjoinName(self, parent_name, name):
+    return os.path.relpath(name, parent_name)
+
+  def _ParseProject(self, node, parent = None):
     """
     reads a <project> element from the manifest file
     """
     name = self._reqatt(node, 'name')
+    if parent:
+      name = self._JoinName(parent.name, name)
 
     remote = self._get_remote(node)
     if remote is None:
@@ -573,42 +606,73 @@ class XmlManifest(object):
     else:
       sync_c = sync_c.lower() in ("yes", "true", "1")
 
+    upstream = node.getAttribute('upstream')
+
     groups = ''
     if node.hasAttribute('groups'):
       groups = node.getAttribute('groups')
     groups = [x for x in re.split('[,\s]+', groups) if x]
 
-    default_groups = ['default', 'name:%s' % name, 'path:%s' % path]
-    groups.extend(set(default_groups).difference(groups))
-
-    if self.IsMirror:
-      relpath = None
-      worktree = None
-      gitdir = os.path.join(self.topdir, '%s.git' % name)
+    if parent is None:
+      relpath, worktree, gitdir = self.GetProjectPaths(name, path)
     else:
-      worktree = os.path.join(self.topdir, path).replace('\\', '/')
-      gitdir = os.path.join(self.repodir, 'projects/%s.git' % path)
+      relpath, worktree, gitdir = self.GetSubprojectPaths(parent, path)
+
+    default_groups = ['all', 'name:%s' % name, 'path:%s' % relpath]
+    groups.extend(set(default_groups).difference(groups))
 
     project = Project(manifest = self,
                       name = name,
                       remote = remote.ToRemoteSpec(name),
                       gitdir = gitdir,
                       worktree = worktree,
-                      relpath = path,
+                      relpath = relpath,
                       revisionExpr = revisionExpr,
                       revisionId = None,
                       rebase = rebase,
                       groups = groups,
-                      sync_c = sync_c)
+                      sync_c = sync_c,
+                      upstream = upstream,
+                      parent = parent)
 
     for n in node.childNodes:
       if n.nodeName == 'copyfile':
         self._ParseCopyFile(project, n)
       if n.nodeName == 'annotation':
         self._ParseAnnotation(project, n)
+      if n.nodeName == 'project':
+        project.subprojects.append(self._ParseProject(n, parent = project))
 
     return project
 
+  def GetProjectPaths(self, name, path):
+    relpath = path
+    if self.IsMirror:
+      worktree = None
+      gitdir = os.path.join(self.topdir, '%s.git' % name)
+    else:
+      worktree = os.path.join(self.topdir, path).replace('\\', '/')
+      gitdir = os.path.join(self.repodir, 'projects', '%s.git' % path)
+    return relpath, worktree, gitdir
+
+  def GetSubprojectName(self, parent, submodule_path):
+    return os.path.join(parent.name, submodule_path)
+
+  def _JoinRelpath(self, parent_relpath, relpath):
+    return os.path.join(parent_relpath, relpath)
+
+  def _UnjoinRelpath(self, parent_relpath, relpath):
+    return os.path.relpath(relpath, parent_relpath)
+
+  def GetSubprojectPaths(self, parent, path):
+    relpath = self._JoinRelpath(parent.relpath, path)
+    gitdir = os.path.join(parent.gitdir, 'subprojects', '%s.git' % path)
+    if self.IsMirror:
+      worktree = None
+    else:
+      worktree = os.path.join(parent.worktree, path).replace('\\', '/')
+    return relpath, worktree, gitdir
+
   def _ParseCopyFile(self, project, node):
     src = self._reqatt(node, 'src')
     dest = self._reqatt(node, 'dest')

pager.py

@@ -50,7 +50,7 @@ def RunPager(globalConfig):
     _BecomePager(pager)
   except Exception:
     print >>sys.stderr, "fatal: cannot start pager '%s'" % pager
-    os.exit(255)
+    sys.exit(255)
 
 def _SelectPager(globalConfig):
   try:
@@ -74,11 +74,11 @@ def _BecomePager(pager):
   # ready works around a long-standing bug in popularly
   # available versions of 'less', a better 'more'.
   #
-  a, b, c = select.select([0], [], [0])
+  _a, _b, _c = select.select([0], [], [0])
 
   os.environ['LESS'] = 'FRSX'
 
   try:
     os.execvp(pager, [pager])
-  except OSError, e:
+  except OSError:
     os.execv('/bin/sh', ['sh', '-c', pager])

project.py

@@ -22,6 +22,7 @@ import shutil
 import stat
 import subprocess
 import sys
+import tempfile
 import time
 
 from color import Coloring
@@ -209,9 +210,9 @@ class _CopyFile:
         if os.path.exists(dest):
           os.remove(dest)
         else:
-          dir = os.path.dirname(dest)
-          if not os.path.isdir(dir):
-            os.makedirs(dir)
+          dest_dir = os.path.dirname(dest)
+          if not os.path.isdir(dest_dir):
+            os.makedirs(dest_dir)
         shutil.copy(src, dest)
         # make the file read-only
         mode = os.stat(dest)[stat.ST_MODE]
@@ -328,7 +329,6 @@ class RepoHook(object):
       HookError: Raised if the user doesn't approve and abort_if_user_denies
           was passed to the consturctor.
     """
-    hooks_dir = self._hooks_project.worktree
     hooks_config = self._hooks_project.config
     git_approval_key = 'repo.hooks.%s.approvedhash' % self._hook_type
 
@@ -484,7 +484,29 @@ class Project(object):
                revisionId,
                rebase = True,
                groups = None,
-               sync_c = False):
+               sync_c = False,
+               upstream = None,
+               parent = None,
+               is_derived = False):
+    """Init a Project object.
+
+    Args:
+      manifest: The XmlManifest object.
+      name: The `name` attribute of manifest.xml's project element.
+      remote: RemoteSpec object specifying its remote's properties.
+      gitdir: Absolute path of git directory.
+      worktree: Absolute path of git working tree.
+      relpath: Relative path of git working tree to repo's top directory.
+      revisionExpr: The `revision` attribute of manifest.xml's project element.
+      revisionId: git commit id for checking out.
+      rebase: The `rebase` attribute of manifest.xml's project element.
+      groups: The `groups` attribute of manifest.xml's project element.
+      sync_c: The `sync-c` attribute of manifest.xml's project element.
+      upstream: The `upstream` attribute of manifest.xml's project element.
+      parent: The parent Project object.
+      is_derived: False if the project was explicitly defined in the manifest;
+                  True if the project is a discovered submodule.
+    """
     self.manifest = manifest
     self.name = name
     self.remote = remote
@@ -506,6 +528,10 @@ class Project(object):
     self.rebase = rebase
     self.groups = groups
     self.sync_c = sync_c
+    self.upstream = upstream
+    self.parent = parent
+    self.is_derived = is_derived
+    self.subprojects = []
 
     self.snapshots = {}
     self.copyfiles = []
@@ -525,6 +551,14 @@ class Project(object):
     # project containing repo hooks.
     self.enabled_repo_hooks = []
 
+  @property
+  def Registered(self):
+    return self.parent and not self.is_derived
+
+  @property
+  def Derived(self):
+    return self.is_derived
+
   @property
   def Exists(self):
     return os.path.isdir(self.gitdir)
@@ -606,25 +640,24 @@ class Project(object):
     """Get all existing local branches.
     """
     current = self.CurrentBranch
-    all = self._allrefs
+    all_refs = self._allrefs
     heads = {}
-    pubd = {}
 
-    for name, id in all.iteritems():
+    for name, ref_id in all_refs.iteritems():
       if name.startswith(R_HEADS):
         name = name[len(R_HEADS):]
         b = self.GetBranch(name)
         b.current = name == current
         b.published = None
-        b.revision = id
+        b.revision = ref_id
         heads[name] = b
 
-    for name, id in all.iteritems():
+    for name, ref_id in all_refs.iteritems():
       if name.startswith(R_PUB):
         name = name[len(R_PUB):]
         b = heads.get(name)
         if b:
-          b.published = id
+          b.published = ref_id
 
     return heads
 
@@ -724,17 +757,25 @@ class Project(object):
     paths.sort()
 
     for p in paths:
-      try: i = di[p]
-      except KeyError: i = None
+      try:
+        i = di[p]
+      except KeyError:
+        i = None
 
-      try: f = df[p]
-      except KeyError: f = None
+      try:
+        f = df[p]
+      except KeyError:
+        f = None
 
-      if i: i_status = i.status.upper()
-      else: i_status = '-'
+      if i:
+        i_status = i.status.upper()
+      else:
+        i_status = '-'
 
-      if f: f_status = f.status.lower()
-      else: f_status = '-'
+      if f:
+        f_status = f.status.lower()
+      else:
+        f_status = '-'
 
       if i and i.src_path:
         line = ' %s%s\t%s => %s (%s%%)' % (i_status, f_status,
@@ -783,40 +824,40 @@ class Project(object):
 
 ## Publish / Upload ##
 
-  def WasPublished(self, branch, all=None):
+  def WasPublished(self, branch, all_refs=None):
     """Was the branch published (uploaded) for code review?
        If so, returns the SHA-1 hash of the last published
        state for the branch.
     """
     key = R_PUB + branch
-    if all is None:
+    if all_refs is None:
       try:
         return self.bare_git.rev_parse(key)
       except GitError:
         return None
     else:
       try:
-        return all[key]
+        return all_refs[key]
       except KeyError:
         return None
 
-  def CleanPublishedCache(self, all=None):
+  def CleanPublishedCache(self, all_refs=None):
     """Prunes any stale published refs.
     """
-    if all is None:
-      all = self._allrefs
+    if all_refs is None:
+      all_refs = self._allrefs
     heads = set()
     canrm = {}
-    for name, id in all.iteritems():
+    for name, ref_id in all_refs.iteritems():
       if name.startswith(R_HEADS):
         heads.add(name)
       elif name.startswith(R_PUB):
-        canrm[name] = id
+        canrm[name] = ref_id
 
-    for name, id in canrm.iteritems():
+    for name, ref_id in canrm.iteritems():
       n = name[len(R_PUB):]
       if R_HEADS + n not in heads:
-        self.bare_git.DeleteRef(name, id)
+        self.bare_git.DeleteRef(name, ref_id)
 
   def GetUploadableBranches(self, selected_branch=None):
     """List any branches which can be uploaded for review.
@@ -824,15 +865,15 @@ class Project(object):
     heads = {}
     pubed = {}
 
-    for name, id in self._allrefs.iteritems():
+    for name, ref_id in self._allrefs.iteritems():
       if name.startswith(R_HEADS):
-        heads[name[len(R_HEADS):]] = id
+        heads[name[len(R_HEADS):]] = ref_id
       elif name.startswith(R_PUB):
-        pubed[name[len(R_PUB):]] = id
+        pubed[name[len(R_PUB):]] = ref_id
 
     ready = []
-    for branch, id in heads.iteritems():
-      if branch in pubed and pubed[branch] == id:
+    for branch, ref_id in heads.iteritems():
+      if branch in pubed and pubed[branch] == ref_id:
         continue
       if selected_branch and branch != selected_branch:
         continue
@@ -976,18 +1017,18 @@ class Project(object):
     self._InitHooks()
 
   def _CopyFiles(self):
-    for file in self.copyfiles:
-      file._Copy()
+    for copyfile in self.copyfiles:
+      copyfile._Copy()
 
-  def GetRevisionId(self, all=None):
+  def GetRevisionId(self, all_refs=None):
     if self.revisionId:
       return self.revisionId
 
     rem = self.GetRemote(self.remote.name)
     rev = rem.ToLocal(self.revisionExpr)
 
-    if all is not None and rev in all:
-      return all[rev]
+    if all_refs is not None and rev in all_refs:
+      return all_refs[rev]
 
     try:
       return self.bare_git.rev_parse('--verify', '%s^0' % rev)
@@ -1000,16 +1041,16 @@ class Project(object):
     """Perform only the local IO portion of the sync process.
        Network access is not required.
     """
-    all = self.bare_ref.all
-    self.CleanPublishedCache(all)
-    revid = self.GetRevisionId(all)
+    all_refs = self.bare_ref.all
+    self.CleanPublishedCache(all_refs)
+    revid = self.GetRevisionId(all_refs)
 
     self._InitWorkTree()
     head = self.work_git.GetHead()
     if head.startswith(R_HEADS):
       branch = head[len(R_HEADS):]
       try:
-        head = all[head]
+        head = all_refs[head]
       except KeyError:
         head = None
     else:
@@ -1036,7 +1077,7 @@ class Project(object):
 
       try:
         self._Checkout(revid, quiet=True)
-      except GitError, e:
+      except GitError as e:
         syncbuf.fail(self, e)
         return
       self._CopyFiles()
@@ -1058,14 +1099,14 @@ class Project(object):
                    branch.name)
       try:
         self._Checkout(revid, quiet=True)
-      except GitError, e:
+      except GitError as e:
         syncbuf.fail(self, e)
         return
       self._CopyFiles()
       return
 
     upstream_gain = self._revlist(not_rev(HEAD), revid)
-    pub = self.WasPublished(branch.name, all)
+    pub = self.WasPublished(branch.name, all_refs)
     if pub:
       not_merged = self._revlist(not_rev(revid), pub)
       if not_merged:
@@ -1143,7 +1184,7 @@ class Project(object):
       try:
         self._ResetHard(revid)
         self._CopyFiles()
-      except GitError, e:
+      except GitError as e:
         syncbuf.fail(self, e)
         return
     else:
@@ -1188,8 +1229,8 @@ class Project(object):
     if head == (R_HEADS + name):
       return True
 
-    all = self.bare_ref.all
-    if (R_HEADS + name) in all:
+    all_refs = self.bare_ref.all
+    if (R_HEADS + name) in all_refs:
       return GitCommand(self,
                         ['checkout', name, '--'],
                         capture_stdout = True,
@@ -1198,11 +1239,11 @@ class Project(object):
     branch = self.GetBranch(name)
     branch.remote = self.GetRemote(self.remote.name)
     branch.merge = self.revisionExpr
-    revid = self.GetRevisionId(all)
+    revid = self.GetRevisionId(all_refs)
 
     if head.startswith(R_HEADS):
       try:
-        head = all[head]
+        head = all_refs[head]
       except KeyError:
         head = None
 
@@ -1243,9 +1284,9 @@ class Project(object):
       #
       return True
 
-    all = self.bare_ref.all
+    all_refs = self.bare_ref.all
     try:
-      revid = all[rev]
+      revid = all_refs[rev]
     except KeyError:
       # Branch does not exist in this project
       #
@@ -1253,7 +1294,7 @@ class Project(object):
 
     if head.startswith(R_HEADS):
       try:
-        head = all[head]
+        head = all_refs[head]
       except KeyError:
         head = None
 
@@ -1281,8 +1322,8 @@ class Project(object):
       didn't exist.
     """
     rev = R_HEADS + name
-    all = self.bare_ref.all
-    if rev not in all:
+    all_refs = self.bare_ref.all
+    if rev not in all_refs:
       # Doesn't exist
       return None
 
@@ -1291,9 +1332,9 @@ class Project(object):
       # We can't destroy the branch while we are sitting
       # on it.  Switch to a detached HEAD.
       #
-      head = all[head]
+      head = all_refs[head]
 
-      revid = self.GetRevisionId(all)
+      revid = self.GetRevisionId(all_refs)
       if head == revid:
         _lwrite(os.path.join(self.worktree, '.git', HEAD),
                 '%s\n' % revid)
@@ -1362,6 +1403,150 @@ class Project(object):
     return kept
 
 
+## Submodule Management ##
+
+  def GetRegisteredSubprojects(self):
+    result = []
+    def rec(subprojects):
+      if not subprojects:
+        return
+      result.extend(subprojects)
+      for p in subprojects:
+        rec(p.subprojects)
+    rec(self.subprojects)
+    return result
+
+  def _GetSubmodules(self):
+    # Unfortunately we cannot call `git submodule status --recursive` here
+    # because the working tree might not exist yet, and it cannot be used
+    # without a working tree in its current implementation.
+
+    def get_submodules(gitdir, rev):
+      # Parse .gitmodules for submodule sub_paths and sub_urls
+      sub_paths, sub_urls = parse_gitmodules(gitdir, rev)
+      if not sub_paths:
+        return []
+      # Run `git ls-tree` to read SHAs of submodule object, which happen to be
+      # revision of submodule repository
+      sub_revs = git_ls_tree(gitdir, rev, sub_paths)
+      submodules = []
+      for sub_path, sub_url in zip(sub_paths, sub_urls):
+        try:
+          sub_rev = sub_revs[sub_path]
+        except KeyError:
+          # Ignore non-exist submodules
+          continue
+        submodules.append((sub_rev, sub_path, sub_url))
+      return submodules
+
+    re_path = re.compile(r'submodule.(\w+).path')
+    re_url = re.compile(r'submodule.(\w+).url')
+    def parse_gitmodules(gitdir, rev):
+      cmd = ['cat-file', 'blob', '%s:.gitmodules' % rev]
+      try:
+        p = GitCommand(None, cmd, capture_stdout = True, capture_stderr = True,
+                       bare = True, gitdir = gitdir)
+      except GitError:
+        return [], []
+      if p.Wait() != 0:
+        return [], []
+
+      gitmodules_lines = []
+      fd, temp_gitmodules_path = tempfile.mkstemp()
+      try:
+        os.write(fd, p.stdout)
+        os.close(fd)
+        cmd = ['config', '--file', temp_gitmodules_path, '--list']
+        p = GitCommand(None, cmd, capture_stdout = True, capture_stderr = True,
+                       bare = True, gitdir = gitdir)
+        if p.Wait() != 0:
+          return [], []
+        gitmodules_lines = p.stdout.split('\n')
+      except GitError:
+        return [], []
+      finally:
+        os.remove(temp_gitmodules_path)
+
+      names = set()
+      paths = {}
+      urls = {}
+      for line in gitmodules_lines:
+        if not line:
+          continue
+        key, value = line.split('=')
+        m = re_path.match(key)
+        if m:
+          names.add(m.group(1))
+          paths[m.group(1)] = value
+          continue
+        m = re_url.match(key)
+        if m:
+          names.add(m.group(1))
+          urls[m.group(1)] = value
+          continue
+      names = sorted(names)
+      return [paths[name] for name in names], [urls[name] for name in names]
+
+    def git_ls_tree(gitdir, rev, paths):
+      cmd = ['ls-tree', rev, '--']
+      cmd.extend(paths)
+      try:
+        p = GitCommand(None, cmd, capture_stdout = True, capture_stderr = True,
+                       bare = True, gitdir = gitdir)
+      except GitError:
+        return []
+      if p.Wait() != 0:
+        return []
+      objects = {}
+      for line in p.stdout.split('\n'):
+        if not line.strip():
+          continue
+        object_rev, object_path = line.split()[2:4]
+        objects[object_path] = object_rev
+      return objects
+
+    try:
+      rev = self.GetRevisionId()
+    except GitError:
+      return []
+    return get_submodules(self.gitdir, rev)
+
+  def GetDerivedSubprojects(self):
+    result = []
+    if not self.Exists:
+      # If git repo does not exist yet, querying its submodules will
+      # mess up its states; so return here.
+      return result
+    for rev, path, url in self._GetSubmodules():
+      name = self.manifest.GetSubprojectName(self, path)
+      project = self.manifest.projects.get(name)
+      if project and project.Registered:
+        # If it has been registered, skip it because we are searching
+        # derived subprojects, but search for its derived subprojects.
+        result.extend(project.GetDerivedSubprojects())
+        continue
+      relpath, worktree, gitdir = self.manifest.GetSubprojectPaths(self, path)
+      remote = RemoteSpec(self.remote.name,
+                          url = url,
+                          review = self.remote.review)
+      subproject = Project(manifest = self.manifest,
+                           name = name,
+                           remote = remote,
+                           gitdir = gitdir,
+                           worktree = worktree,
+                           relpath = relpath,
+                           revisionExpr = self.revisionExpr,
+                           revisionId = rev,
+                           rebase = self.rebase,
+                           groups = self.groups,
+                           sync_c = self.sync_c,
+                           parent = self,
+                           is_derived = True)
+      result.append(subproject)
+      result.extend(subproject.GetDerivedSubprojects())
+    return result
+
+
 ## Direct Git Commands ##
 
   def _RemoteFetch(self, name=None,
@@ -1373,6 +1558,16 @@ class Project(object):
     is_sha1 = False
     tag_name = None
 
+    def CheckForSha1():
+        try:
+          # if revision (sha or tag) is not present then following function
+          # throws an error.
+          self.bare_git.rev_parse('--verify', '%s^0' % self.revisionExpr)
+          return True
+        except GitError:
+          # There is no such persistent revision. We have to fetch it.
+          return False
+
     if current_branch_only:
       if ID_RE.match(self.revisionExpr) is not None:
         is_sha1 = True
@@ -1381,14 +1576,10 @@ class Project(object):
         tag_name = self.revisionExpr[len(R_TAGS):]
 
       if is_sha1 or tag_name is not None:
-        try:
-          # if revision (sha or tag) is not present then following function
-          # throws an error.
-          self.bare_git.rev_parse('--verify', '%s^0' % self.revisionExpr)
+        if CheckForSha1():
           return True
-        except GitError:
-          # There is no such persistent revision. We have to fetch it.
-          pass
+      if is_sha1 and (not self.upstream or ID_RE.match(self.upstream)):
+        current_branch_only = False
 
     if not name:
       name = self.remote.name
@@ -1404,33 +1595,33 @@ class Project(object):
         packed_refs = os.path.join(self.gitdir, 'packed-refs')
         remote = self.GetRemote(name)
 
-        all = self.bare_ref.all
-        ids = set(all.values())
+        all_refs = self.bare_ref.all
+        ids = set(all_refs.values())
         tmp = set()
 
-        for r, id in GitRefs(ref_dir).all.iteritems():
-          if r not in all:
+        for r, ref_id in GitRefs(ref_dir).all.iteritems():
+          if r not in all_refs:
             if r.startswith(R_TAGS) or remote.WritesTo(r):
-              all[r] = id
-              ids.add(id)
+              all_refs[r] = ref_id
+              ids.add(ref_id)
               continue
 
-          if id in ids:
+          if ref_id in ids:
             continue
 
-          r = 'refs/_alt/%s' % id
-          all[r] = id
-          ids.add(id)
+          r = 'refs/_alt/%s' % ref_id
+          all_refs[r] = ref_id
+          ids.add(ref_id)
           tmp.add(r)
 
-        ref_names = list(all.keys())
+        ref_names = list(all_refs.keys())
         ref_names.sort()
 
         tmp_packed = ''
         old_packed = ''
 
         for r in ref_names:
-          line = '%s %s\n' % (all[r], r)
+          line = '%s %s\n' % (all_refs[r], r)
           tmp_packed += line
           if r not in tmp:
             old_packed += line
@@ -1453,7 +1644,7 @@ class Project(object):
       cmd.append('--update-head-ok')
     cmd.append(name)
 
-    if not current_branch_only or is_sha1:
+    if not current_branch_only:
       # Fetch whole repo
       cmd.append('--tags')
       cmd.append((u'+refs/heads/*:') + remote.ToLocal('refs/heads/*'))
@@ -1462,15 +1653,23 @@ class Project(object):
       cmd.append(tag_name)
     else:
       branch = self.revisionExpr
+      if is_sha1:
+        branch = self.upstream
       if branch.startswith(R_HEADS):
         branch = branch[len(R_HEADS):]
       cmd.append((u'+refs/heads/%s:' % branch) + remote.ToLocal('refs/heads/%s' % branch))
 
     ok = False
-    for i in range(2):
-      if GitCommand(self, cmd, bare=True, ssh_proxy=ssh_proxy).Wait() == 0:
+    for _i in range(2):
+      ret = GitCommand(self, cmd, bare=True, ssh_proxy=ssh_proxy).Wait()
+      if ret == 0:
         ok = True
         break
+      elif current_branch_only and is_sha1 and ret == 128:
+        # Exit code 128 means "couldn't find the ref you asked for"; if we're in sha1
+        # mode, we just tried sync'ing from the upstream field; it doesn't exist, thus
+        # abort the optimization attempt and do a full sync.
+        break
       time.sleep(random.randint(30, 45))
 
     if initial:
@@ -1480,6 +1679,15 @@ class Project(object):
         else:
           os.remove(packed_refs)
       self.bare_git.pack_refs('--all', '--prune')
+
+    if is_sha1 and current_branch_only and self.upstream:
+      # We just synced the upstream given branch; verify we
+      # got what we wanted, else trigger a second run of all
+      # refs.
+      if not CheckForSha1():
+        return self._RemoteFetch(name=name, current_branch_only=False,
+                                 initial=False, quiet=quiet, alt_dir=alt_dir)
+
     return ok
 
   def _ApplyCloneBundle(self, initial=False, quiet=False):
@@ -1692,7 +1900,7 @@ class Project(object):
           continue
       try:
         os.symlink(os.path.relpath(stock_hook, os.path.dirname(dst)), dst)
-      except OSError, e:
+      except OSError as e:
         if e.errno == errno.EPERM:
           raise GitError('filesystem must support symlinks')
         else:
@@ -1755,7 +1963,7 @@ class Project(object):
             os.symlink(os.path.relpath(src, os.path.dirname(dst)), dst)
           else:
             raise GitError('cannot overwrite a local work tree')
-        except OSError, e:
+        except OSError as e:
           if e.errno == errno.EPERM:
             raise GitError('filesystem must support symlinks')
           else:
@@ -1938,7 +2146,9 @@ class Project(object):
 
       Since we don't have a 'rev_parse' method defined, the __getattr__ will
       run.  We'll replace the '_' with a '-' and try to run a git command.
-      Any other arguments will be passed to the git command.
+      Any other positional arguments will be passed to the git command, and the
+      following keyword arguments are supported:
+        config: An optional dict of git config options to be passed with '-c'.
 
       Args:
         name: The name of the git command to call.  Any '_' characters will
@@ -1948,8 +2158,17 @@ class Project(object):
         A callable object that will try to call git with the named command.
       """
       name = name.replace('_', '-')
-      def runner(*args):
-        cmdv = [name]
+      def runner(*args, **kwargs):
+        cmdv = []
+        config = kwargs.pop('config', None)
+        for k in kwargs:
+          raise TypeError('%s() got an unexpected keyword argument %r'
+                          % (name, k))
+        if config is not None:
+          for k, v in config.iteritems():
+            cmdv.append('-c')
+            cmdv.append('%s=%s' % (k, v))
+        cmdv.append(name)
         cmdv.extend(args)
         p = GitCommand(self._project,
                        cmdv,
@@ -2009,7 +2228,7 @@ class _Later(object):
       self.action()
       out.nl()
       return True
-    except GitError, e:
+    except GitError:
       out.nl()
       return False
 
@@ -2079,7 +2298,6 @@ class MetaProject(Project):
   """A special project housed under .repo.
   """
   def __init__(self, manifest, name, gitdir, worktree):
-    repodir = manifest.repodir
     Project.__init__(self,
                      manifest = manifest,
                      name = name,
@@ -2131,12 +2349,12 @@ class MetaProject(Project):
     if not self.remote or not self.revisionExpr:
       return False
 
-    all = self.bare_ref.all
-    revid = self.GetRevisionId(all)
+    all_refs = self.bare_ref.all
+    revid = self.GetRevisionId(all_refs)
     head = self.work_git.GetHead()
     if head.startswith(R_HEADS):
       try:
-        head = all[head]
+        head = all_refs[head]
       except KeyError:
         head = None
 

repo

@@ -28,10 +28,10 @@ if __name__ == '__main__':
 del magic
 
 # increment this whenever we make important changes to this script
-VERSION = (1, 17)
+VERSION = (1, 18)
 
 # increment this if the MAINTAINER_KEYS block is modified
-KEYRING_VERSION = (1,0)
+KEYRING_VERSION = (1,1)
 MAINTAINER_KEYS = """
 
      Repo Maintainer <repo@android.kernel.org>
@@ -74,13 +74,45 @@ HTHs37+/QLMomGEGKZMWi0dShU2J5mNRQu3Hhxl3hHDVbt5CeJBb26aQcQrFz69W
 zE3GNvmJosh6leayjtI9P2A6iEkEGBECAAkFAkj3uiACGwwACgkQFlMNXpIPXGWp
 TACbBS+Up3RpfYVfd63c1cDdlru13pQAn3NQy/SN858MkxN+zym86UBgOad2
 =CMiZ
+-----END PGP PUBLIC KEY BLOCK-----
+
+     Conley Owens <cco3@android.com>
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v1.4.11 (GNU/Linux)
+
+mQENBFBiLPwBCACvISTASOgFXwADw2GYRH2I2z9RvYkYoZ6ThTTNlMXbbYYKO2Wo
+a9LQDNW0TbCEekg5UKk0FD13XOdWaqUt4Gtuvq9c43GRSjMO6NXH+0BjcQ8vUtY2
+/W4CYUevwdo4nQ1+1zsOCu1XYe/CReXq0fdugv3hgmRmh3sz1soo37Q44W2frxxg
+U7Rz3Da4FjgAL0RQ8qndD+LwRHXTY7H7wYM8V/3cYFZV7pSodd75q3MAXYQLf0ZV
+QR1XATu5l1QnXrxgHvz7MmDwb1D+jX3YPKnZveaukigQ6hDHdiVcePBiGXmk8LZC
+2jQkdXeF7Su1ZYpr2nnEHLJ6vOLcCpPGb8gDABEBAAG0H0NvbmxleSBPd2VucyA8
+Y2NvM0BhbmRyb2lkLmNvbT6JATgEEwECACIFAlBiLPwCGwMGCwkIBwMCBhUIAgkK
+CwQWAgMBAh4BAheAAAoJEBkmlFUziHGkHVkH/2Hks2Cif5i2xPtv2IFZcjL42joU
+T7lO5XFqUYS9ZNHpGa/V0eiPt7rHoO16glR83NZtwlrq2cSN89i9HfOhMYV/qLu8
+fLCHcV2muw+yCB5s5bxnI5UkToiNZyBNqFkcOt/Kbj9Hpy68A1kmc6myVEaUYebq
+2Chx/f3xuEthan099t746v1K+/6SvQGDNctHuaMr9cWdxZtHjdRf31SQRc99Phe5
+w+ZGR/ebxNDKRK9mKgZT8wVFHlXerJsRqWIqtx1fsW1UgLgbpcpe2MChm6B5wTu0
+s1ltzox3l4q71FyRRPUJxXyvGkDLZWpK7EpiHSCOYq/KP3HkKeXU3xqHpcG5AQ0E
+UGIs/AEIAKzO/7lO9cB6dshmZYo8Vy/b7aGicThE+ChcDSfhvyOXVdEM2GKAjsR+
+rlBWbTFX3It301p2HwZPFEi9nEvJxVlqqBiW0bPmNMkDRR55l2vbWg35wwkg6RyE
+Bc5/TQjhXI2w8IvlimoGoUff4t3JmMOnWrnKSvL+5iuRj12p9WmanCHzw3Ee7ztf
+/aU/q+FTpr3DLerb6S8xbv86ySgnJT6o5CyL2DCWRtnYQyGVi0ZmLzEouAYiO0hs
+z0AAu28Mj+12g2WwePRz6gfM9rHtI37ylYW3oT/9M9mO9ei/Bc/1D7Dz6qNV+0vg
+uSVJxM2Bl6GalHPZLhHntFEdIA6EdoUAEQEAAYkBHwQYAQIACQUCUGIs/AIbDAAK
+CRAZJpRVM4hxpNfkB/0W/hP5WK/NETXBlWXXW7JPaWO2c5kGwD0lnj5RRmridyo1
+vbm5PdM91jOsDQYqRu6YOoYBnDnEhB2wL2bPh34HWwwrA+LwB8hlcAV2z1bdwyfl
+3R823fReKN3QcvLHzmvZPrF4Rk97M9UIyKS0RtnfTWykRgDWHIsrtQPoNwsXrWoT
+9LrM2v+1+9mp3vuXnE473/NHxmiWEQH9Ez+O/mOxQ7rSOlqGRiKq/IBZCfioJOtV
+fTQeIu/yASZnsLBqr6SJEGwYBoWcyjG++k4fyw8ocOAo4uGDYbxgN7yYfNQ0OH7o
+V6pfUgqKLWa/aK7/N1ZHnPdFLD8Xt0Dmy4BPwrKC
+=O7am
 -----END PGP PUBLIC KEY BLOCK-----
 """
 
 GIT = 'git'                     # our git command
 MIN_GIT_VERSION = (1, 5, 4)     # minimum supported git version
 repodir = '.repo'               # name of repo's private directory
-S_repo = 'repo'                 # special repo reposiory
+S_repo = 'repo'                 # special repo repository
 S_manifests = 'manifests'       # special manifest repository
 REPO_MAIN = S_repo + '/main.py' # main script
 
@@ -88,7 +120,6 @@ REPO_MAIN = S_repo + '/main.py' # main script
 import optparse
 import os
 import re
-import readline
 import subprocess
 import sys
 import urllib2
@@ -131,7 +162,7 @@ group.add_option('-g', '--groups',
                  metavar='GROUP')
 group.add_option('-p', '--platform',
                  dest='platform', default="auto",
-                 help='restrict manifest projects to ones with a specified'
+                 help='restrict manifest projects to ones with a specified '
                       'platform group [auto|all|none|linux|darwin|...]',
                  metavar='PLATFORM')
 
@@ -186,7 +217,7 @@ def _Init(args):
   if not os.path.isdir(repodir):
     try:
       os.mkdir(repodir)
-    except OSError, e:
+    except OSError as e:
       print >>sys.stderr, \
             'fatal: cannot make %s directory: %s' % (
             repodir, e.strerror)
@@ -197,8 +228,8 @@ def _Init(args):
 
   _CheckGitVersion()
   try:
-    if _NeedSetupGnuPG():
-      can_verify = _SetupGnuPG(opt.quiet)
+    if NeedSetupGnuPG():
+      can_verify = SetupGnuPG(opt.quiet)
     else:
       can_verify = True
 
@@ -222,7 +253,7 @@ def _CheckGitVersion():
   cmd = [GIT, '--version']
   try:
     proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
-  except OSError, e:
+  except OSError as e:
     print >>sys.stderr
     print >>sys.stderr, "fatal: '%s' is not available" % GIT
     print >>sys.stderr, 'fatal: %s' % e
@@ -247,7 +278,7 @@ def _CheckGitVersion():
     raise CloneFailure()
 
 
-def _NeedSetupGnuPG():
+def NeedSetupGnuPG():
   if not os.path.isdir(home_dot_repo):
     return True
 
@@ -265,11 +296,11 @@ def _NeedSetupGnuPG():
   return False
 
 
-def _SetupGnuPG(quiet):
+def SetupGnuPG(quiet):
   if not os.path.isdir(home_dot_repo):
     try:
       os.mkdir(home_dot_repo)
-    except OSError, e:
+    except OSError as e:
       print >>sys.stderr, \
             'fatal: cannot make %s directory: %s' % (
             home_dot_repo, e.strerror)
@@ -278,7 +309,7 @@ def _SetupGnuPG(quiet):
   if not os.path.isdir(gpg_dir):
     try:
       os.mkdir(gpg_dir, 0700)
-    except OSError, e:
+    except OSError as e:
       print >>sys.stderr, \
             'fatal: cannot make %s directory: %s' % (
             gpg_dir, e.strerror)
@@ -292,7 +323,7 @@ def _SetupGnuPG(quiet):
     proc = subprocess.Popen(cmd,
                             env = env,
                             stdin = subprocess.PIPE)
-  except OSError, e:
+  except OSError as e:
     if not quiet:
       print >>sys.stderr, 'warning: gpg (GnuPG) is not available.'
       print >>sys.stderr, 'warning: Installing it is strongly encouraged.'
@@ -393,13 +424,13 @@ def _DownloadBundle(url, local, quiet):
   try:
     try:
       r = urllib2.urlopen(url)
-    except urllib2.HTTPError, e:
+    except urllib2.HTTPError as e:
       if e.code == 404:
         return False
       print >>sys.stderr, 'fatal: Cannot get %s' % url
       print >>sys.stderr, 'fatal: HTTP error %s' % e.code
       raise CloneFailure()
-    except urllib2.URLError, e:
+    except urllib2.URLError as e:
       print >>sys.stderr, 'fatal: Cannot get %s' % url
       print >>sys.stderr, 'fatal: error %s' % e.reason
       raise CloneFailure()
@@ -428,7 +459,7 @@ def _Clone(url, local, quiet):
   """
   try:
     os.mkdir(local)
-  except OSError, e:
+  except OSError as e:
     print >>sys.stderr, \
           'fatal: cannot make %s directory: %s' \
           % (local, e.strerror)
@@ -437,7 +468,7 @@ def _Clone(url, local, quiet):
   cmd = [GIT, 'init', '--quiet']
   try:
     proc = subprocess.Popen(cmd, cwd = local)
-  except OSError, e:
+  except OSError as e:
     print >>sys.stderr
     print >>sys.stderr, "fatal: '%s' is not available" % GIT
     print >>sys.stderr, 'fatal: %s' % e
@@ -539,19 +570,19 @@ def _Checkout(cwd, branch, rev, quiet):
 def _FindRepo():
   """Look for a repo installation, starting at the current directory.
   """
-  dir = os.getcwd()
+  curdir = os.getcwd()
   repo = None
 
   olddir = None
-  while dir != '/' \
-    and dir != olddir \
+  while curdir != '/' \
+    and curdir != olddir \
     and not repo:
-    repo = os.path.join(dir, repodir, REPO_MAIN)
+    repo = os.path.join(curdir, repodir, REPO_MAIN)
     if not os.path.isfile(repo):
       repo = None
-      olddir = dir
-      dir = os.path.dirname(dir)
-  return (repo, os.path.join(dir, repodir))
+      olddir = curdir
+      curdir = os.path.dirname(curdir)
+  return (repo, os.path.join(curdir, repodir))
 
 
 class _Options:
@@ -657,13 +688,13 @@ def _SetDefaultsTo(gitdir):
 
 
 def main(orig_args):
-  main, dir = _FindRepo()
+  repo_main, rel_repo_dir = _FindRepo()
   cmd, opt, args = _ParseArguments(orig_args)
 
   wrapper_path = os.path.abspath(__file__)
   my_main, my_git = _RunSelf(wrapper_path)
 
-  if not main:
+  if not repo_main:
     if opt.help:
       _Usage()
     if cmd == 'help':
@@ -683,25 +714,25 @@ def main(orig_args):
             os.rmdir(os.path.join(root, name))
         os.rmdir(repodir)
         sys.exit(1)
-      main, dir = _FindRepo()
+      repo_main, rel_repo_dir = _FindRepo()
     else:
       _NoCommands(cmd)
 
   if my_main:
-    main = my_main
+    repo_main = my_main
 
   ver_str = '.'.join(map(lambda x: str(x), VERSION))
-  me = [main,
-        '--repo-dir=%s' % dir,
+  me = [repo_main,
+        '--repo-dir=%s' % rel_repo_dir,
         '--wrapper-version=%s' % ver_str,
         '--wrapper-path=%s' % wrapper_path,
         '--']
   me.extend(orig_args)
   me.extend(extra_args)
   try:
-    os.execv(main, me)
-  except OSError, e:
-    print >>sys.stderr, "fatal: unable to start %s" % main
+    os.execv(repo_main, me)
+  except OSError as e:
+    print >>sys.stderr, "fatal: unable to start %s" % repo_main
     print >>sys.stderr, "fatal: %s" % e
     sys.exit(148)
 

subcmds/__init__.py

@@ -15,7 +15,7 @@
 
 import os
 
-all = {}
+all_commands = {}
 
 my_dir = os.path.dirname(__file__)
 for py in os.listdir(my_dir):
@@ -43,7 +43,7 @@ for py in os.listdir(my_dir):
 
     name = name.replace('_', '-')
     cmd.NAME = name
-    all[name] = cmd
+    all_commands[name] = cmd
 
-if 'help' in all:
-  all['help'].commands = all
+if 'help' in all_commands:
+  all_commands['help'].commands = all_commands

subcmds/abandon.py

@@ -42,10 +42,10 @@ It is equivalent to "git branch -D <branchname>".
     nb = args[0]
     err = []
     success = []
-    all = self.GetProjects(args[1:])
+    all_projects = self.GetProjects(args[1:])
 
-    pm = Progress('Abandon %s' % nb, len(all))
-    for project in all:
+    pm = Progress('Abandon %s' % nb, len(all_projects))
+    for project in all_projects:
       pm.update()
 
       status = project.AbandonBranch(nb)

subcmds/branches.py

@@ -93,17 +93,17 @@ is shown, then the branch appears in all projects.
   def Execute(self, opt, args):
     projects = self.GetProjects(args)
     out = BranchColoring(self.manifest.manifestProject.config)
-    all = {}
+    all_branches = {}
     project_cnt = len(projects)
 
     for project in projects:
       for name, b in project.GetBranches().iteritems():
         b.project = project
-        if name not in all:
-          all[name] = BranchInfo(name)
-        all[name].add(b)
+        if name not in all_branches:
+          all_branches[name] = BranchInfo(name)
+        all_branches[name].add(b)
 
-    names = all.keys()
+    names = all_branches.keys()
     names.sort()
 
     if not names:
@@ -116,7 +116,7 @@ is shown, then the branch appears in all projects.
         width = len(name)
 
     for name in names:
-      i = all[name]
+      i = all_branches[name]
       in_cnt = len(i.projects)
 
       if i.IsCurrent:
@@ -140,12 +140,12 @@ is shown, then the branch appears in all projects.
         fmt = out.write
         paths = []
         if in_cnt < project_cnt - in_cnt: 
-          type = 'in'
+          in_type = 'in'
           for b in i.projects:
             paths.append(b.project.relpath)
         else:
           fmt = out.notinproject
-          type = 'not in'
+          in_type = 'not in'
           have = set()
           for b in i.projects:
             have.add(b.project)
@@ -153,11 +153,11 @@ is shown, then the branch appears in all projects.
             if not p in have:
               paths.append(p.relpath)
 
-        s = ' %s %s' % (type, ', '.join(paths))
+        s = ' %s %s' % (in_type, ', '.join(paths))
         if width + 7 + len(s) < 80:
           fmt(s)
         else:
-          fmt(' %s:' % type)
+          fmt(' %s:' % in_type)
           for p in paths:
             out.nl()
             fmt(width*' ' + '          %s' % p)

subcmds/checkout.py

@@ -39,10 +39,10 @@ The command is equivalent to:
     nb = args[0]
     err = []
     success = []
-    all = self.GetProjects(args[1:])
+    all_projects = self.GetProjects(args[1:])
 
-    pm = Progress('Checkout %s' % nb, len(all))
-    for project in all:
+    pm = Progress('Checkout %s' % nb, len(all_projects))
+    for project in all_projects:
       pm.update()
 
       status = project.CheckoutBranch(nb)

subcmds/cherry_pick.py

@@ -13,7 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sys, re
+import re
+import sys
 from command import Command
 from git_command import GitCommand
 

subcmds/forall.py

@@ -141,12 +141,16 @@ terminal and are not redirected.
       for cn in cmd[1:]:
         if not cn.startswith('-'):
           break
-      if cn in _CAN_COLOR:
+      else:
+        cn = None
+      # pylint: disable=W0631
+      if cn and cn in _CAN_COLOR:
         class ColorCmd(Coloring):
           def __init__(self, config, cmd):
             Coloring.__init__(self, config, cmd)
         if ColorCmd(self.manifest.manifestProject.config, cn).is_on:
           cmd.insert(cmd.index(cn) + 1, '--color')
+      # pylint: enable=W0631
 
     mirror = self.manifest.IsMirror
     out = ForallColoring(self.manifest.manifestProject.config)
@@ -208,7 +212,6 @@ terminal and are not redirected.
             return self.fd.fileno()
 
         empty = True
-        didout = False
         errbuf = ''
 
         p.stdin.close()
@@ -220,7 +223,7 @@ terminal and are not redirected.
           fcntl.fcntl(s.fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
 
         while s_in:
-          in_ready, out_ready, err_ready = select.select(s_in, [], [])
+          in_ready, _out_ready, _err_ready = select.select(s_in, [], [])
           for s in in_ready:
             buf = s.fd.read(4096)
             if not buf:
@@ -229,9 +232,7 @@ terminal and are not redirected.
               continue
 
             if not opt.verbose:
-              if s.fd == p.stdout:
-                didout = True
-              else:
+              if s.fd != p.stdout:
                 errbuf += buf
                 continue
 

subcmds/help.py

@@ -120,8 +120,8 @@ See 'repo help --all' for a complete list of recognized commands.
           m = asciidoc_hdr.match(para)
           if m:
             title = m.group(1)
-            type = m.group(2)
-            if type[0] in ('=', '-'):
+            section_type = m.group(2)
+            if section_type[0] in ('=', '-'):
               p = self.heading
             else:
               def _p(fmt, *args):
@@ -131,7 +131,7 @@ See 'repo help --all' for a complete list of recognized commands.
 
             p('%s', title)
             self.nl()
-            p('%s', ''.ljust(len(title),type[0]))
+            p('%s', ''.ljust(len(title),section_type[0]))
             self.nl()
             continue
 

subcmds/init.py

@@ -207,14 +207,12 @@ to update the working directory files.
 
     try:
       self.manifest.Link(name)
-    except ManifestParseError, e:
+    except ManifestParseError as e:
       print >>sys.stderr, "fatal: manifest '%s' not available" % name
       print >>sys.stderr, 'fatal: %s' % str(e)
       sys.exit(1)
 
   def _Prompt(self, prompt, value):
-    mp = self.manifest.manifestProject
-
     sys.stdout.write('%-10s [%s]: ' % (prompt, value))
     a = sys.stdin.readline().strip()
     if a == '':
@@ -317,6 +315,10 @@ to update the working directory files.
 
   def Execute(self, opt, args):
     git_require(MIN_GIT_VERSION, fail=True)
+
+    if opt.reference:
+      opt.reference = os.path.expanduser(opt.reference)
+
     self._SyncManifest(opt)
     self._LinkManifest(opt.manifest_name)
 
@@ -328,9 +330,9 @@ to update the working directory files.
     self._ConfigureDepth(opt)
 
     if self.manifest.IsMirror:
-      type = 'mirror '
+      init_type = 'mirror '
     else:
-      type = ''
+      init_type = ''
 
     print ''
-    print 'repo %sinitialized in %s' % (type, self.manifest.topdir)
+    print 'repo %sinitialized in %s' % (init_type, self.manifest.topdir)

subcmds/list.py

@@ -13,13 +13,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import re
+
 from command import Command, MirrorSafeCommand
 
 class List(Command, MirrorSafeCommand):
   common = True
   helpSummary = "List projects and their associated directories"
   helpUsage = """
-%prog [<project>...]
+%prog [-f] [<project>...]
+%prog [-f] -r str1 [str2]..."
 """
   helpDescription = """
 List all projects; pass '.' to list the project for the cwd.
@@ -27,6 +30,14 @@ List all projects; pass '.' to list the project for the cwd.
 This is similar to running: repo forall -c 'echo "$REPO_PATH : $REPO_PROJECT"'.
 """
 
+  def _Options(self, p, show_smart=True):
+    p.add_option('-r', '--regex',
+                 dest='regex', action='store_true',
+                 help="Filter the project list based on regex or wildcard matching of strings")
+    p.add_option('-f', '--fullpath',
+                 dest='fullpath', action='store_true',
+                 help="Display the full work tree path instead of the relative path")
+
   def Execute(self, opt, args):
     """List all projects and the associated directories.
 
@@ -35,14 +46,33 @@ This is similar to running: repo forall -c 'echo "$REPO_PATH : $REPO_PROJECT"'.
     discoverable.
 
     Args:
-      opt: The options.  We don't take any.
+      opt: The options.
       args: Positional args.  Can be a list of projects to list, or empty.
     """
-    projects = self.GetProjects(args)
+    if not opt.regex:
+      projects = self.GetProjects(args)
+    else:
+      projects = self.FindProjects(args)
+
+    def _getpath(x):
+      if opt.fullpath:
+        return x.worktree
+      return x.relpath
 
     lines = []
     for project in projects:
-      lines.append("%s : %s" % (project.relpath, project.name))
+      lines.append("%s : %s" % (_getpath(project), project.name))
 
     lines.sort()
     print '\n'.join(lines)
+
+  def FindProjects(self, args):
+    result = []
+    for project in self.GetProjects(''):
+      for arg in args:
+        pattern = re.compile(r'%s' % arg, re.IGNORECASE)
+        if pattern.search(project.name) or pattern.search(project.relpath):
+          result.append(project)
+          break
+    result.sort(key=lambda project: project.relpath)
+    return result

subcmds/manifest.py

@@ -35,19 +35,24 @@ in a Git repository for use during future 'repo init' invocations.
 
   @property
   def helpDescription(self):
-    help = self._helpDescription + '\n'
+    helptext = self._helpDescription + '\n'
     r = os.path.dirname(__file__)
     r = os.path.dirname(r)
     fd = open(os.path.join(r, 'docs', 'manifest-format.txt'))
     for line in fd:
-      help += line
+      helptext += line
     fd.close()
-    return help
+    return helptext
 
   def _Options(self, p):
     p.add_option('-r', '--revision-as-HEAD',
                  dest='peg_rev', action='store_true',
                  help='Save revisions as current HEAD')
+    p.add_option('--suppress-upstream-revision', dest='peg_rev_upstream',
+                 default=True, action='store_false',
+                 help='If in -r mode, do not write the upstream field.  '
+                 'Only of use if the branch names for a sha1 manifest are '
+                 'sensitive.')
     p.add_option('-o', '--output-file',
                  dest='output_file',
                  default='-',
@@ -60,7 +65,8 @@ in a Git repository for use during future 'repo init' invocations.
     else:
       fd = open(opt.output_file, 'w')
     self.manifest.Save(fd,
-                       peg_rev = opt.peg_rev)
+                       peg_rev = opt.peg_rev,
+                       peg_rev_upstream = opt.peg_rev_upstream)
     fd.close()
     if opt.output_file != '-':
       print >>sys.stderr, 'Saved manifest to %s' % opt.output_file

subcmds/overview.py

@@ -38,16 +38,16 @@ are displayed.
                  help="Consider only checked out branches")
 
   def Execute(self, opt, args):
-    all = []
+    all_branches = []
     for project in self.GetProjects(args):
       br = [project.GetUploadableBranch(x)
             for x in project.GetBranches().keys()]
       br = [x for x in br if x]
       if opt.current_branch:
         br = [x for x in br if x.name == project.CurrentBranch]
-      all.extend(br)
+      all_branches.extend(br)
 
-    if not all:
+    if not all_branches:
       return
 
     class Report(Coloring):
@@ -55,13 +55,13 @@ are displayed.
         Coloring.__init__(self, config, 'status')
         self.project = self.printer('header', attr='bold')
 
-    out = Report(all[0].project.config)
+    out = Report(all_branches[0].project.config)
     out.project('Projects Overview')
     out.nl()
 
     project = None
 
-    for branch in all:
+    for branch in all_branches:
       if project != branch.project:
         project = branch.project
         out.nl()

subcmds/prune.py

@@ -24,11 +24,11 @@ class Prune(PagedCommand):
 """
 
   def Execute(self, opt, args):
-    all = []
+    all_branches = []
     for project in self.GetProjects(args):
-      all.extend(project.PruneHeads())
+      all_branches.extend(project.PruneHeads())
 
-    if not all:
+    if not all_branches:
       return
 
     class Report(Coloring):
@@ -36,13 +36,13 @@ class Prune(PagedCommand):
         Coloring.__init__(self, config, 'status')
         self.project = self.printer('header', attr='bold')
 
-    out = Report(all[0].project.config)
+    out = Report(all_branches[0].project.config)
     out.project('Pending Branches')
     out.nl()
 
     project = None
 
-    for branch in all:
+    for branch in all_branches:
       if project != branch.project:
         project = branch.project
         out.nl()

subcmds/rebase.py

@@ -55,14 +55,14 @@ branch but need to incorporate new upstream changes "underneath" them.
                  help='Stash local modifications before starting')
 
   def Execute(self, opt, args):
-    all = self.GetProjects(args)
-    one_project = len(all) == 1
+    all_projects = self.GetProjects(args)
+    one_project = len(all_projects) == 1
 
     if opt.interactive and not one_project:
       print >>sys.stderr, 'error: interactive rebase not supported with multiple projects'
       return -1
 
-    for project in all:
+    for project in all_projects:
       cb = project.CurrentBranch
       if not cb:
         if one_project:

subcmds/smartsync.py

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from sync import Sync
+from subcmds.sync import Sync
 
 class Smartsync(Sync):
   common = True

subcmds/stage.py

@@ -48,8 +48,8 @@ The '%prog' command stages files to prepare the next commit.
       self.Usage()
 
   def _Interactive(self, opt, args):
-    all = filter(lambda x: x.IsDirty(), self.GetProjects(args))
-    if not all:
+    all_projects = filter(lambda x: x.IsDirty(), self.GetProjects(args))
+    if not all_projects:
       print >>sys.stderr,'no projects have uncommitted modifications'
       return
 
@@ -58,8 +58,8 @@ The '%prog' command stages files to prepare the next commit.
       out.header('        %s', 'project')
       out.nl()
 
-      for i in xrange(0, len(all)):
-        p = all[i]
+      for i in xrange(0, len(all_projects)):
+        p = all_projects[i]
         out.write('%3d:    %s', i + 1, p.relpath + '/')
         out.nl()
       out.nl()
@@ -93,11 +93,11 @@ The '%prog' command stages files to prepare the next commit.
       if a_index is not None:
         if a_index == 0:
           break
-        if 0 < a_index and a_index <= len(all):
-          _AddI(all[a_index - 1])
+        if 0 < a_index and a_index <= len(all_projects):
+          _AddI(all_projects[a_index - 1])
           continue
 
-      p = filter(lambda x: x.name == a or x.relpath == a, all)
+      p = filter(lambda x: x.name == a or x.relpath == a, all_projects)
       if len(p) == 1:
         _AddI(p[0])
         continue

subcmds/start.py

@@ -52,10 +52,10 @@ revision specified in the manifest.
         print >>sys.stderr, "error: at least one project must be specified"
         sys.exit(1)
 
-    all = self.GetProjects(projects)
+    all_projects = self.GetProjects(projects)
 
-    pm = Progress('Starting %s' % nb, len(all))
-    for project in all:
+    pm = Progress('Starting %s' % nb, len(all_projects))
+    for project in all_projects:
       pm.update()
       # If the current revision is a specific SHA1 then we can't push back
       # to it so substitute the manifest default revision instead.

subcmds/status.py

@@ -98,18 +98,18 @@ the following meanings:
       sem.release()
 
   def Execute(self, opt, args):
-    all = self.GetProjects(args)
+    all_projects = self.GetProjects(args)
     counter = itertools.count()
 
     if opt.jobs == 1:
-      for project in all:
+      for project in all_projects:
         state = project.PrintWorkTreeStatus()
         if state == 'CLEAN':
           counter.next()
     else:
       sem = _threading.Semaphore(opt.jobs)
       threads_and_output = []
-      for project in all:
+      for project in all_projects:
         sem.acquire()
 
         class BufList(StringIO.StringIO):
@@ -128,5 +128,5 @@ the following meanings:
         t.join()
         output.dump(sys.stdout)
         output.close()
-    if len(all) == counter.next():
+    if len(all_projects) == counter.next():
       print 'nothing to commit (working directory clean)'

subcmds/sync.py

@@ -13,14 +13,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import netrc
 from optparse import SUPPRESS_HELP
 import os
+import pickle
 import re
 import shutil
 import socket
 import subprocess
 import sys
 import time
+import urlparse
 import xmlrpclib
 
 try:
@@ -36,8 +39,14 @@ except ImportError:
   def _rlimit_nofile():
     return (256, 256)
 
+try:
+  import multiprocessing
+except ImportError:
+  multiprocessing = None
+
 from git_command import GIT
 from git_refs import R_HEADS, HEAD
+from main import WrapperModule
 from project import Project
 from project import RemoteSpec
 from command import Command, MirrorSafeCommand
@@ -45,6 +54,8 @@ from error import RepoChangedException, GitError
 from project import SyncBuffer
 from progress import Progress
 
+_ONE_DAY_S = 24 * 60 * 60
+
 class _FetchError(Exception):
   """Internal error thrown in _FetchHelper() when we don't want stack trace."""
   pass
@@ -81,6 +92,18 @@ build as specified by the manifest-server element in the current
 manifest. The -t/--smart-tag option is similar and allows you to
 specify a custom tag/label.
 
+The -u/--manifest-server-username and -p/--manifest-server-password
+options can be used to specify a username and password to authenticate
+with the manifest server when using the -s or -t option.
+
+If -u and -p are not specified when using the -s or -t option, '%prog'
+will attempt to read authentication credentials for the manifest server
+from the user's .netrc file.
+
+'%prog' will not use authentication credentials from -u/-p or .netrc
+if the manifest server specified in the manifest file already includes
+credentials.
+
 The -f/--force-broken option can be used to proceed with syncing
 other projects if a project sync fails.
 
@@ -157,6 +180,12 @@ later is required to fix a server side protocol bug.
       p.add_option('-t', '--smart-tag',
                    dest='smart_tag', action='store',
                    help='smart sync using manifest from a known tag')
+      p.add_option('-u', '--manifest-server-username', action='store',
+                   dest='manifest_server_username',
+                   help='username to authenticate with the manifest server')
+      p.add_option('-p', '--manifest-server-password', action='store',
+                   dest='manifest_server_password',
+                   help='password to authenticate with the manifest server')
 
     g = p.add_option_group('repo Version options')
     g.add_option('--no-repo-verify',
@@ -192,10 +221,12 @@ later is required to fix a server side protocol bug.
       # - We always make sure we unlock the lock if we locked it.
       try:
         try:
+          start = time.time()
           success = project.Sync_NetworkHalf(
             quiet=opt.quiet,
             current_branch_only=opt.current_branch_only,
             clone_bundle=not opt.no_clone_bundle)
+          self._fetch_times.Set(project, time.time() - start)
 
           # Lock around all the rest of the code, since printing, updating a set
           # and Progress.update() are not thread safe.
@@ -273,10 +304,57 @@ later is required to fix a server side protocol bug.
         sys.exit(1)
 
     pm.end()
-    for project in projects:
-      project.bare_git.gc('--auto')
+    self._fetch_times.Save()
+
+    self._GCProjects(projects)
     return fetched
 
+  def _GCProjects(self, projects):
+    if multiprocessing:
+      cpu_count = multiprocessing.cpu_count()
+    else:
+      cpu_count = 1
+    jobs = min(self.jobs, cpu_count)
+
+    if jobs < 2:
+      for project in projects:
+        project.bare_git.gc('--auto')
+      return
+
+    config = {'pack.threads': cpu_count / jobs if cpu_count > jobs else 1}
+
+    threads = set()
+    sem = _threading.Semaphore(jobs)
+    err_event = _threading.Event()
+
+    def GC(project):
+      try:
+        try:
+          project.bare_git.gc('--auto', config=config)
+        except GitError:
+          err_event.set()
+        except:
+          err_event.set()
+          raise
+      finally:
+        sem.release()
+
+    for project in projects:
+      if err_event.isSet():
+        break
+      sem.acquire()
+      t = _threading.Thread(target=GC, args=(project,))
+      t.daemon = True
+      threads.add(t)
+      t.start()
+
+    for t in threads:
+      t.join()
+
+    if err_event.isSet():
+      print >>sys.stderr, '\nerror: Exited sync due to gc errors'
+      sys.exit(1)
+
   def UpdateProjectList(self):
     new_project_paths = []
     for project in self.GetProjects(None, missing_ok=True):
@@ -296,8 +374,7 @@ later is required to fix a server side protocol bug.
         if not path:
           continue
         if path not in new_project_paths:
-          """If the path has already been deleted, we don't need to do it
-          """
+          # If the path has already been deleted, we don't need to do it
           if os.path.exists(self.manifest.topdir + '/' + path):
               project = Project(
                              manifest = self.manifest,
@@ -320,13 +397,13 @@ uncommitted changes are present' % project.relpath
                 print >>sys.stderr, 'Deleting obsolete path %s' % project.worktree
                 shutil.rmtree(project.worktree)
                 # Try deleting parent subdirs if they are empty
-                dir = os.path.dirname(project.worktree)
-                while dir != self.manifest.topdir:
+                project_dir = os.path.dirname(project.worktree)
+                while project_dir != self.manifest.topdir:
                   try:
-                    os.rmdir(dir)
+                    os.rmdir(project_dir)
                   except OSError:
                     break
-                  dir = os.path.dirname(dir)
+                  project_dir = os.path.dirname(project_dir)
 
     new_project_paths.sort()
     fd = open(file_path, 'w')
@@ -356,6 +433,14 @@ uncommitted changes are present' % project.relpath
     if opt.manifest_name and opt.smart_tag:
       print >>sys.stderr, 'error: cannot combine -m and -t'
       sys.exit(1)
+    if opt.manifest_server_username or opt.manifest_server_password:
+      if not (opt.smart_sync or opt.smart_tag):
+        print >>sys.stderr, 'error: -u and -p may only be combined with ' \
+                            '-s or -t'
+        sys.exit(1)
+      if None in [opt.manifest_server_username, opt.manifest_server_password]:
+        print >>sys.stderr, 'error: both -u and -p must be given'
+        sys.exit(1)
 
     if opt.manifest_name:
       self.manifest.Override(opt.manifest_name)
@@ -365,8 +450,41 @@ uncommitted changes are present' % project.relpath
         print >>sys.stderr, \
             'error: cannot smart sync: no manifest server defined in manifest'
         sys.exit(1)
+
+      manifest_server = self.manifest.manifest_server
+
+      if not '@' in manifest_server:
+        username = None
+        password = None
+        if opt.manifest_server_username and opt.manifest_server_password:
+          username = opt.manifest_server_username
+          password = opt.manifest_server_password
+        else:
+          try:
+            info = netrc.netrc()
+          except IOError:
+            print >>sys.stderr, '.netrc file does not exist or could not be opened'
+          else:
+            try:
+              parse_result = urlparse.urlparse(manifest_server)
+              if parse_result.hostname:
+                username, _account, password = \
+                  info.authenticators(parse_result.hostname)
+            except TypeError:
+              # TypeError is raised when the given hostname is not present
+              # in the .netrc file.
+              print >>sys.stderr, 'No credentials found for %s in .netrc' % \
+                                  parse_result.hostname
+            except netrc.NetrcParseError as e:
+              print >>sys.stderr, 'Error parsing .netrc file: %s' % e
+
+        if (username and password):
+          manifest_server = manifest_server.replace('://', '://%s:%s@' %
+                                                    (username, password),
+                                                    1)
+
       try:
-        server = xmlrpclib.Server(self.manifest.manifest_server)
+        server = xmlrpclib.Server(manifest_server)
         if opt.smart_sync:
           p = self.manifest.manifestProject
           b = p.GetBranch(p.CurrentBranch)
@@ -404,11 +522,11 @@ uncommitted changes are present' % project.relpath
         else:
           print >>sys.stderr, 'error: %s' % manifest_str
           sys.exit(1)
-      except (socket.error, IOError, xmlrpclib.Fault), e:
+      except (socket.error, IOError, xmlrpclib.Fault) as e:
         print >>sys.stderr, 'error: cannot connect to manifest server %s:\n%s' % (
             self.manifest.manifest_server, e)
         sys.exit(1)
-      except xmlrpclib.ProtocolError, e:
+      except xmlrpclib.ProtocolError as e:
         print >>sys.stderr, 'error: cannot connect to manifest server %s:\n%d %s' % (
             self.manifest.manifest_server, e.errcode, e.errmsg)
         sys.exit(1)
@@ -420,7 +538,7 @@ uncommitted changes are present' % project.relpath
     mp.PreSync()
 
     if opt.repo_upgraded:
-      _PostRepoUpgrade(self.manifest)
+      _PostRepoUpgrade(self.manifest, opt)
 
     if not opt.local_only:
       mp.Sync_NetworkHalf(quiet=opt.quiet,
@@ -434,14 +552,16 @@ uncommitted changes are present' % project.relpath
       self.manifest._Unload()
       if opt.jobs is None:
         self.jobs = self.manifest.default.sync_j
-    all = self.GetProjects(args, missing_ok=True)
+    all_projects = self.GetProjects(args, missing_ok=True)
 
+    self._fetch_times = _FetchTimes(self.manifest)
     if not opt.local_only:
       to_fetch = []
       now = time.time()
-      if (24 * 60 * 60) <= (now - rp.LastFetch):
+      if _ONE_DAY_S <= (now - rp.LastFetch):
         to_fetch.append(rp)
-      to_fetch.extend(all)
+      to_fetch.extend(all_projects)
+      to_fetch.sort(key=self._fetch_times.Get, reverse=True)
 
       fetched = self._Fetch(to_fetch, opt)
       _PostRepoFetch(rp, opt.no_repo_verify)
@@ -449,13 +569,24 @@ uncommitted changes are present' % project.relpath
         # bail out now; the rest touches the working tree
         return
 
+      # Iteratively fetch missing and/or nested unregistered submodules
+      previously_missing_set = set()
+      while True:
         self.manifest._Unload()
-        all = self.GetProjects(args, missing_ok=True)
+        all_projects = self.GetProjects(args, missing_ok=True)
         missing = []
-        for project in all:
+        for project in all_projects:
           if project.gitdir not in fetched:
             missing.append(project)
-        self._Fetch(missing, opt)
+        if not missing:
+          break
+        # Stop us from non-stopped fetching actually-missing repos: If set of
+        # missing repos has not been changed from last fetch, we break.
+        missing_set = set(p.name for p in missing)
+        if previously_missing_set == missing_set:
+          break
+        previously_missing_set = missing_set
+        fetched.update(self._Fetch(missing, opt))
 
     if self.manifest.IsMirror:
       # bail out now, we have no working tree
@@ -466,8 +597,8 @@ uncommitted changes are present' % project.relpath
 
     syncbuf = SyncBuffer(mp.config,
                          detach_head = opt.detach_head)
-    pm = Progress('Syncing work tree', len(all))
-    for project in all:
+    pm = Progress('Syncing work tree', len(all_projects))
+    for project in all_projects:
       pm.update()
       if project.worktree:
         project.Sync_LocalHalf(syncbuf)
@@ -481,7 +612,10 @@ uncommitted changes are present' % project.relpath
     if self.manifest.notice:
       print self.manifest.notice
 
-def _PostRepoUpgrade(manifest):
+def _PostRepoUpgrade(manifest, opt):
+  wrapper = WrapperModule()
+  if wrapper.NeedSetupGnuPG():
+    wrapper.SetupGnuPG(opt.quiet)
   for project in manifest.projects.values():
     if project.Exists:
       project.PostRepoUpgrade()
@@ -550,3 +684,66 @@ warning: Cannot automatically authenticate repo."""
     print >>sys.stderr
     return False
   return True
+
+class _FetchTimes(object):
+  _ALPHA = 0.5
+
+  def __init__(self, manifest):
+    self._path = os.path.join(manifest.repodir, '.repopickle_fetchtimes')
+    self._times = None
+    self._seen = set()
+
+  def Get(self, project):
+    self._Load()
+    return self._times.get(project.name, _ONE_DAY_S)
+
+  def Set(self, project, t):
+    self._Load()
+    name = project.name
+    old = self._times.get(name, t)
+    self._seen.add(name)
+    a = self._ALPHA
+    self._times[name] = (a*t) + ((1-a) * old)
+
+  def _Load(self):
+    if self._times is None:
+      try:
+        f = open(self._path)
+      except IOError:
+        self._times = {}
+        return self._times
+      try:
+        try:
+          self._times = pickle.load(f)
+        except:
+          try:
+            os.remove(self._path)
+          except OSError:
+            pass
+          self._times = {}
+      finally:
+        f.close()
+    return self._times
+
+  def Save(self):
+    if self._times is None:
+      return
+
+    to_delete = []
+    for name in self._times:
+      if name not in self._seen:
+        to_delete.append(name)
+    for name in to_delete:
+      del self._times[name]
+
+    try:
+      f = open(self._path, 'wb')
+      try:
+        pickle.dump(self._times, f)
+      except (IOError, OSError, pickle.PickleError):
+        try:
+          os.remove(self._path)
+        except OSError:
+          pass
+    finally:
+      f.close()

subcmds/upload.py

@@ -40,8 +40,8 @@ def _die(fmt, *args):
 
 def _SplitEmails(values):
   result = []
-  for str in values:
-    result.extend([s.strip() for s in str.split(',')])
+  for value in values:
+    result.extend([s.strip() for s in value.split(',')])
   return result
 
 class Upload(InteractiveCommand):
@@ -174,15 +174,15 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
 
     if answer is None:
       date = branch.date
-      list = branch.commits
+      commit_list = branch.commits
 
       print 'Upload project %s/ to remote branch %s:' % (project.relpath, project.revisionExpr)
       print '  branch %s (%2d commit%s, %s):' % (
                     name,
-                    len(list),
-                    len(list) != 1 and 's' or '',
+                    len(commit_list),
+                    len(commit_list) != 1 and 's' or '',
                     date)
-      for commit in list:
+      for commit in commit_list:
         print '         %s' % commit
 
       sys.stdout.write('to %s (y/N)? ' % remote.review)
@@ -212,17 +212,17 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
       for branch in avail:
         name = branch.name
         date = branch.date
-        list = branch.commits
+        commit_list = branch.commits
 
         if b:
           script.append('#')
         script.append('#  branch %s (%2d commit%s, %s) to remote branch %s:' % (
                       name,
-                      len(list),
-                      len(list) != 1 and 's' or '',
+                      len(commit_list),
+                      len(commit_list) != 1 and 's' or '',
                       date,
                       project.revisionExpr))
-        for commit in list:
+        for commit in commit_list:
           script.append('#         %s' % commit)
         b[name] = branch
 
@@ -329,7 +329,7 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
 
         branch.UploadForReview(people, auto_topic=opt.auto_topic, draft=opt.draft)
         branch.uploaded = True
-      except UploadError, e:
+      except UploadError as e:
         branch.error = e
         branch.uploaded = False
         have_errors = True
@@ -384,7 +384,7 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
       pending_proj_names = [project.name for (project, avail) in pending]
       try:
         hook.Run(opt.allow_all_hooks, project_list=pending_proj_names)
-      except HookError, e:
+      except HookError as e:
         print >>sys.stderr, "ERROR: %s" % str(e)
         return
 

subcmds/version.py

@@ -16,7 +16,7 @@
 import sys
 from command import Command, MirrorSafeCommand
 from git_command import git
-from project import HEAD
+from git_refs import HEAD
 
 class Version(Command, MirrorSafeCommand):
   wrapper_version = None