Revert "Repo: fall back to http, if ssh connection fails for http repos"

This reverts commit 488bf092d5.

Issue 230

Change-Id: I3a5725301f576e1a2ac499cb6daa631895115640

.flake8

@@ -0,0 +1,3 @@
+[flake8]
+max-line-length=80
+ignore=E111,E114,E402

.gitattributes

@@ -0,0 +1,4 @@
+# Prevent /bin/sh scripts from being clobbered by autocrlf=true
+git_ssh text eol=lf
+repo text eol=lf
+hooks/* text eol=lf

.gitignore

@@ -1,2 +1,3 @@
 *.pyc
 .repopickle_*
+/repoc

.mailmap

@@ -0,0 +1,11 @@
+Anthony Newnam <anthony.newnam@garmin.com>    Anthony <anthony@bnovc.com>
+Hu Xiuyun <xiuyun.hu@hisilicon.com>           Hu xiuyun <xiuyun.hu@hisilicon.com>
+Hu Xiuyun <xiuyun.hu@hisilicon.com>           Hu Xiuyun <clouds08@qq.com>
+Jelly Chen <chenguodong@huawei.com>           chenguodong <chenguodong@huawei.com>
+Jia Bi <bijia@xiaomi.com>                     bijia <bijia@xiaomi.com>
+JoonCheol Park <jooncheol@gmail.com>          Jooncheol Park <jooncheol@gmail.com>
+Sergii Pylypenko <x.pelya.x@gmail.com>        pelya <x.pelya.x@gmail.com>
+Shawn Pearce <sop@google.com>                 Shawn O. Pearce <sop@google.com>
+Ulrik Sjölin <ulrik.sjolin@sonyericsson.com>  Ulrik Sjolin <ulrik.sjolin@gmail.com>
+Ulrik Sjölin <ulrik.sjolin@sonyericsson.com>  Ulrik Sjolin <ulrik.sjolin@sonyericsson.com>
+Ulrik Sjölin <ulrik.sjolin@sonyericsson.com>  Ulrik Sjölin <ulrik.sjolin@sonyericsson.com>

.project

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <projectDescription>
-	<name>repo</name>
+	<name>git-repo</name>
 	<comment></comment>
 	<projects>
 	</projects>

.pydevproject

@@ -3,7 +3,7 @@
 
 <pydev_project>
 <pydev_pathproperty name="org.python.pydev.PROJECT_SOURCE_PATH">
-<path>/repo</path>
+<path>/git-repo</path>
 </pydev_pathproperty>
 <pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python 2.6</pydev_property>
 <pydev_property name="org.python.pydev.PYTHON_PROJECT_INTERPRETER">Default</pydev_property>

.pylintrc

@@ -1,301 +0,0 @@
-# lint Python modules using external checkers.
-#
-# This is the main checker controling the other ones and the reports
-# generation. It is itself both a raw checker and an astng checker in order
-# to:
-# * handle message activation / deactivation at the module level
-# * handle some basic but necessary stats'data (number of classes, methods...)
-#
-[MASTER]
-
-# Specify a configuration file.
-#rcfile=
-
-# Python code to execute, usually for sys.path manipulation such as
-# pygtk.require().
-#init-hook=
-
-# Profiled execution.
-profile=no
-
-# Add <file or directory> to the black list. It should be a base name, not a
-# path. You may set this option multiple times.
-ignore=SVN
-
-# Pickle collected data for later comparisons.
-persistent=yes
-
-# Set the cache size for astng objects.
-cache-size=500
-
-# List of plugins (as comma separated values of python modules names) to load,
-# usually to register additional checkers.
-load-plugins=
-
-
-[MESSAGES CONTROL]
-
-# Enable only checker(s) with the given id(s). This option conflicts with the
-# disable-checker option
-#enable-checker=
-
-# Enable all checker(s) except those with the given id(s). This option
-# conflicts with the enable-checker option
-#disable-checker=
-
-# Enable all messages in the listed categories.
-#enable-msg-cat=
-
-# Disable all messages in the listed categories.
-#disable-msg-cat=
-
-# Enable the message(s) with the given id(s).
-enable=RP0004
-
-# Disable the message(s) with the given id(s).
-disable=R0903,R0912,R0913,R0914,R0915,W0141,C0111,C0103,C0323,C0322,C0324,W0603,W0703,R0911,C0301,C0302,R0902,R0904,W0142,W0212,E1101,E1103,R0201,W0201,W0122,W0232,W0311,RP0001,RP0003,RP0101,RP0002,RP0401,RP0701,RP0801
-
-[REPORTS]
-
-# set the output format. Available formats are text, parseable, colorized, msvs
-# (visual studio) and html
-output-format=text
-
-# Include message's id in output
-include-ids=yes
-
-# Put messages in a separate file for each module / package specified on the
-# command line instead of printing them on stdout. Reports (if any) will be
-# written in a file name "pylint_global.[txt|html]".
-files-output=no
-
-# Tells whether to display a full report or only the messages
-reports=yes
-
-# Python expression which should return a note less than 10 (10 is the highest
-# note).You have access to the variables errors warning, statement which
-# respectivly contain the number of errors / warnings messages and the total
-# number of statements analyzed. This is used by the global evaluation report
-# (R0004).
-evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
-
-# Add a comment according to your evaluation note. This is used by the global
-# evaluation report (R0004).
-comment=no
-
-# checks for
-# * unused variables / imports
-# * undefined variables
-# * redefinition of variable from builtins or from an outer scope
-# * use of variable before assigment
-#
-[VARIABLES]
-
-# Tells whether we should check for unused import in __init__ files.
-init-import=no
-
-# A regular expression matching names used for dummy variables (i.e. not used).
-dummy-variables-rgx=_|dummy
-
-# List of additional names supposed to be defined in builtins. Remember that
-# you should avoid to define new builtins when possible.
-additional-builtins=
-
-
-# try to find bugs in the code using type inference
-#
-[TYPECHECK]
-
-# Tells whether missing members accessed in mixin class should be ignored. A
-# mixin class is detected if its name ends with "mixin" (case insensitive).
-ignore-mixin-members=yes
-
-# List of classes names for which member attributes should not be checked
-# (useful for classes with attributes dynamicaly set).
-ignored-classes=SQLObject
-
-# When zope mode is activated, consider the acquired-members option to ignore
-# access to some undefined attributes.
-zope=no
-
-# List of members which are usually get through zope's acquisition mecanism and
-# so shouldn't trigger E0201 when accessed (need zope=yes to be considered).
-acquired-members=REQUEST,acl_users,aq_parent
-
-
-# checks for :
-# * doc strings
-# * modules / classes / functions / methods / arguments / variables name
-# * number of arguments, local variables, branchs, returns and statements in
-# functions, methods
-# * required module attributes
-# * dangerous default values as arguments
-# * redefinition of function / method / class
-# * uses of the global statement
-#
-[BASIC]
-
-# Required attributes for module, separated by a comma
-required-attributes=
-
-# Regular expression which should only match functions or classes name which do
-# not require a docstring
-no-docstring-rgx=_main|__.*__
-
-# Regular expression which should only match correct module names
-module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
-
-# Regular expression which should only match correct module level names
-const-rgx=(([A-Z_][A-Z1-9_]*)|(__.*__))|(log)$
-
-# Regular expression which should only match correct class names
-class-rgx=[A-Z_][a-zA-Z0-9]+$
-
-# Regular expression which should only match correct function names
-function-rgx=[a-z_][a-z0-9_]{2,30}$
-
-# Regular expression which should only match correct method names
-method-rgx=[a-z_][a-z0-9_]{2,30}$
-
-# Regular expression which should only match correct instance attribute names
-attr-rgx=[a-z_][a-z0-9_]{2,30}$
-
-# Regular expression which should only match correct argument names
-argument-rgx=[a-z_][a-z0-9_]{2,30}$
-
-# Regular expression which should only match correct variable names
-variable-rgx=[a-z_][a-z0-9_]{2,30}$
-
-# Regular expression which should only match correct list comprehension /
-# generator expression variable names
-inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$
-
-# Good variable names which should always be accepted, separated by a comma
-good-names=i,j,k,ex,Run,_,e,d1,d2,v,f,l,d
-
-# Bad variable names which should always be refused, separated by a comma
-bad-names=foo,bar,baz,toto,tutu,tata
-
-# List of builtins function names that should not be used, separated by a comma
-bad-functions=map,filter,apply,input
-
-
-# checks for sign of poor/misdesign:
-# * number of methods, attributes, local variables...
-# * size, complexity of functions, methods
-#
-[DESIGN]
-
-# Maximum number of arguments for function / method
-max-args=5
-
-# Maximum number of locals for function / method body
-max-locals=15
-
-# Maximum number of return / yield for function / method body
-max-returns=6
-
-# Maximum number of branch for function / method body
-max-branchs=12
-
-# Maximum number of statements in function / method body
-max-statements=50
-
-# Maximum number of parents for a class (see R0901).
-max-parents=7
-
-# Maximum number of attributes for a class (see R0902).
-max-attributes=20
-
-# Minimum number of public methods for a class (see R0903).
-min-public-methods=2
-
-# Maximum number of public methods for a class (see R0904).
-max-public-methods=30
-
-
-# checks for
-# * external modules dependencies
-# * relative / wildcard imports
-# * cyclic imports
-# * uses of deprecated modules
-#
-[IMPORTS]
-
-# Deprecated modules which should not be used, separated by a comma
-deprecated-modules=regsub,string,TERMIOS,Bastion,rexec
-
-# Create a graph of every (i.e. internal and external) dependencies in the
-# given file (report R0402 must not be disabled)
-import-graph=
-
-# Create a graph of external dependencies in the given file (report R0402 must
-# not be disabled)
-ext-import-graph=
-
-# Create a graph of internal dependencies in the given file (report R0402 must
-# not be disabled)
-int-import-graph=
-
-
-# checks for :
-# * methods without self as first argument
-# * overridden methods signature
-# * access only to existant members via self
-# * attributes not defined in the __init__ method
-# * supported interfaces implementation
-# * unreachable code
-#
-[CLASSES]
-
-# List of interface methods to ignore, separated by a comma. This is used for
-# instance to not check methods defines in Zope's Interface base class.
-ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by
-
-# List of method names used to declare (i.e. assign) instance attributes.
-defining-attr-methods=__init__,__new__,setUp
-
-
-# checks for similarities and duplicated code. This computation may be
-# memory / CPU intensive, so you should disable it if you experiments some
-# problems.
-#
-[SIMILARITIES]
-
-# Minimum lines number of a similarity.
-min-similarity-lines=4
-
-# Ignore comments when computing similarities.
-ignore-comments=yes
-
-# Ignore docstrings when computing similarities.
-ignore-docstrings=yes
-
-
-# checks for:
-# * warning notes in the code like FIXME, XXX
-# * PEP 263: source code with non ascii character but no encoding declaration
-#
-[MISCELLANEOUS]
-
-# List of note tags to take in consideration, separated by a comma.
-notes=FIXME,XXX,TODO
-
-
-# checks for :
-# * unauthorized constructions
-# * strict indentation
-# * line length
-# * use of <> instead of !=
-#
-[FORMAT]
-
-# Maximum number of characters on a single line.
-max-line-length=80
-
-# Maximum number of lines in a module
-max-module-lines=1000
-
-# String used as indentation unit. This is usually "    " (4 spaces) or "\t" (1
-# tab).  In repo it is 2 spaces.
-indent-string='  '

README.md

@@ -0,0 +1,14 @@
+# repo
+
+Repo is a tool built on top of Git.  Repo helps manage many Git repositories,
+does the uploads to revision control systems, and automates parts of the
+development workflow.  Repo is not meant to replace Git, only to make it
+easier to work with Git.  The repo command is an executable Python script
+that you can put anywhere in your path.
+
+* Homepage: https://code.google.com/p/git-repo/
+* Bug reports: https://code.google.com/p/git-repo/issues/
+* Source: https://code.google.com/p/git-repo/
+* Overview: https://source.android.com/source/developing.html
+* Docs: https://source.android.com/source/using-repo.html
+* [Submitting patches](./SUBMITTING_PATCHES.md)

SUBMITTING_PATCHES

@@ -1,87 +0,0 @@
-Short Version:
-
- - Make small logical changes.
- - Provide a meaningful commit message.
- - Check for coding errors with pylint
- - Make sure all code is under the Apache License, 2.0.
- - Publish your changes for review:
-
-   git push https://gerrit-review.googlesource.com/git-repo HEAD:refs/for/master
-
-
-Long Version:
-
-I wanted a file describing how to submit patches for repo,
-so I started with the one found in the core Git distribution
-(Documentation/SubmittingPatches), which itself was based on the
-patch submission guidelines for the Linux kernel.
-
-However there are some differences, so please review and familiarize
-yourself with the following relevant bits:
-
-
-(1) Make separate commits for logically separate changes.
-
-Unless your patch is really trivial, you should not be sending
-out a patch that was generated between your working tree and your
-commit head.  Instead, always make a commit with complete commit
-message and generate a series of patches from your repository.
-It is a good discipline.
-
-Describe the technical detail of the change(s).
-
-If your description starts to get too long, that's a sign that you
-probably need to split up your commit to finer grained pieces.
-
-
-(2) Check for coding errors with pylint
-
-Run pylint on changed modules using the provided configuration:
-
-  pylint --rcfile=.pylintrc file.py
-
-
-(3) Check the license
-
-repo is licensed under the Apache License, 2.0.
-
-Because of this licensing model *every* file within the project
-*must* list the license that covers it in the header of the file.
-Any new contributions to an existing file *must* be submitted under
-the current license of that file.  Any new files *must* clearly
-indicate which license they are provided under in the file header.
-
-Please verify that you are legally allowed and willing to submit your
-changes under the license covering each file *prior* to submitting
-your patch.  It is virtually impossible to remove a patch once it
-has been applied and pushed out.
-
-
-(4) Sending your patches.
-
-Do not email your patches to anyone.
-
-Instead, login to the Gerrit Code Review tool at:
-
-  https://gerrit-review.googlesource.com/
-
-Ensure you have completed one of the necessary contributor
-agreements, providing documentation to the project maintainers that
-they have right to redistribute your work under the Apache License:
-
-  https://gerrit-review.googlesource.com/#/settings/agreements
-
-Ensure you have obtained an HTTP password to authenticate:
-
-  https://gerrit-review.googlesource.com/new-password
-
-Push your patches over HTTPS to the review server, possibly through
-a remembered remote to make this easier in the future:
-
-   git config remote.review.url https://gerrit-review.googlesource.com/git-repo
-   git config remote.review.push HEAD:refs/for/master
-
-   git push review
-
-You will be automatically emailed a copy of your commits, and any
-comments made by the project maintainers.

SUBMITTING_PATCHES.md

@@ -0,0 +1,135 @@
+# Short Version
+
+ - Make small logical changes.
+ - Provide a meaningful commit message.
+ - Check for coding errors and style nits with pyflakes and flake8
+ - Make sure all code is under the Apache License, 2.0.
+ - Publish your changes for review.
+ - Make corrections if requested.
+ - Verify your changes on gerrit so they can be submitted.
+
+   `git push https://gerrit-review.googlesource.com/git-repo HEAD:refs/for/master`
+
+
+# Long Version
+
+I wanted a file describing how to submit patches for repo,
+so I started with the one found in the core Git distribution
+(Documentation/SubmittingPatches), which itself was based on the
+patch submission guidelines for the Linux kernel.
+
+However there are some differences, so please review and familiarize
+yourself with the following relevant bits.
+
+
+## Make separate commits for logically separate changes.
+
+Unless your patch is really trivial, you should not be sending
+out a patch that was generated between your working tree and your
+commit head.  Instead, always make a commit with complete commit
+message and generate a series of patches from your repository.
+It is a good discipline.
+
+Describe the technical detail of the change(s).
+
+If your description starts to get too long, that's a sign that you
+probably need to split up your commit to finer grained pieces.
+
+
+## Check for coding errors and style nits with pyflakes and flake8
+
+### Coding errors
+
+Run `pyflakes` on changed modules:
+
+    pyflakes file.py
+
+Ideally there should be no new errors or warnings introduced.
+
+### Style violations
+
+Run `flake8` on changes modules:
+
+    flake8 file.py
+
+Note that repo generally follows [Google's python style guide]
+(https://google.github.io/styleguide/pyguide.html) rather than [PEP 8]
+(https://www.python.org/dev/peps/pep-0008/), so it's possible that
+the output of `flake8` will be quite noisy. It's not mandatory to
+avoid all warnings, but at least the maximum line length should be
+followed.
+
+If there are many occurrences of the same warning that cannot be
+avoided without going against the Google style guide, these may be
+suppressed in the included `.flake8` file.
+
+## Check the license
+
+repo is licensed under the Apache License, 2.0.
+
+Because of this licensing model *every* file within the project
+*must* list the license that covers it in the header of the file.
+Any new contributions to an existing file *must* be submitted under
+the current license of that file.  Any new files *must* clearly
+indicate which license they are provided under in the file header.
+
+Please verify that you are legally allowed and willing to submit your
+changes under the license covering each file *prior* to submitting
+your patch.  It is virtually impossible to remove a patch once it
+has been applied and pushed out.
+
+
+## Sending your patches.
+
+Do not email your patches to anyone.
+
+Instead, login to the Gerrit Code Review tool at:
+
+  https://gerrit-review.googlesource.com/
+
+Ensure you have completed one of the necessary contributor
+agreements, providing documentation to the project maintainers that
+they have right to redistribute your work under the Apache License:
+
+  https://gerrit-review.googlesource.com/#/settings/agreements
+
+Ensure you have obtained an HTTP password to authenticate:
+
+  https://gerrit-review.googlesource.com/new-password
+
+Ensure that you have the local commit hook installed to automatically
+add a ChangeId to your commits:
+
+    curl -Lo `git rev-parse --git-dir`/hooks/commit-msg https://gerrit-review.googlesource.com/tools/hooks/commit-msg
+    chmod +x `git rev-parse --git-dir`/hooks/commit-msg
+
+If you have already committed your changes you will need to amend the commit
+to get the ChangeId added.
+
+    git commit --amend
+
+Push your patches over HTTPS to the review server, possibly through
+a remembered remote to make this easier in the future:
+
+    git config remote.review.url https://gerrit-review.googlesource.com/git-repo
+    git config remote.review.push HEAD:refs/for/master
+
+    git push review
+
+You will be automatically emailed a copy of your commits, and any
+comments made by the project maintainers.
+
+
+## Make changes if requested
+
+The project maintainer who reviews your changes might request changes to your
+commit. If you make the requested changes you will need to amend your commit
+and push it to the review server again.
+
+
+## Verify your changes on gerrit
+
+After you receive a Code-Review+2 from the maintainer, select the Verified
+button on the gerrit page for the change. This verifies that you have tested
+your changes and notifies the maintainer that they are ready to be submitted.
+The maintainer will then submit your changes to the repository.

color.py

@@ -18,68 +18,91 @@ import sys
 
 import pager
 
-COLORS = {None     :-1,
-          'normal' :-1,
-          'black'  : 0,
-          'red'    : 1,
-          'green'  : 2,
-          'yellow' : 3,
-          'blue'   : 4,
+COLORS = {None: -1,
+          'normal': -1,
+          'black': 0,
+          'red': 1,
+          'green': 2,
+          'yellow': 3,
+          'blue': 4,
           'magenta': 5,
-          'cyan'   : 6,
-          'white'  : 7}
+          'cyan': 6,
+          'white': 7}
 
-ATTRS = {None     :-1,
-         'bold'   : 1,
-         'dim'    : 2,
-         'ul'     : 4,
-         'blink'  : 5,
+ATTRS = {None: -1,
+         'bold': 1,
+         'dim': 2,
+         'ul': 4,
+         'blink': 5,
          'reverse': 7}
 
 RESET = "\033[m"
 
+
 def is_color(s):
-    return s in COLORS
+  return s in COLORS
+
 
 def is_attr(s):
-    return s in ATTRS
+  return s in ATTRS
 
-def _Color(fg = None, bg = None, attr = None):
-    fg = COLORS[fg]
-    bg = COLORS[bg]
-    attr = ATTRS[attr]
 
-    if attr >= 0 or fg >= 0 or bg >= 0:
-      need_sep = False
-      code = "\033["
+def _Color(fg=None, bg=None, attr=None):
+  fg = COLORS[fg]
+  bg = COLORS[bg]
+  attr = ATTRS[attr]
 
-      if attr >= 0:
-        code += chr(ord('0') + attr)
-        need_sep = True
+  if attr >= 0 or fg >= 0 or bg >= 0:
+    need_sep = False
+    code = "\033["
 
-      if fg >= 0:
-        if need_sep:
-          code += ';'
-        need_sep = True
+    if attr >= 0:
+      code += chr(ord('0') + attr)
+      need_sep = True
 
-        if fg < 8:
-          code += '3%c' % (ord('0') + fg)
-        else:
-          code += '38;5;%d' % fg
+    if fg >= 0:
+      if need_sep:
+        code += ';'
+      need_sep = True
 
-      if bg >= 0:
-        if need_sep:
-          code += ';'
-        need_sep = True
+      if fg < 8:
+        code += '3%c' % (ord('0') + fg)
+      else:
+        code += '38;5;%d' % fg
 
-        if bg < 8:
-          code += '4%c' % (ord('0') + bg)
-        else:
-          code += '48;5;%d' % bg
-      code += 'm'
-    else:
-      code = ''
-    return code
+    if bg >= 0:
+      if need_sep:
+        code += ';'
+
+      if bg < 8:
+        code += '4%c' % (ord('0') + bg)
+      else:
+        code += '48;5;%d' % bg
+    code += 'm'
+  else:
+    code = ''
+  return code
+
+DEFAULT = None
+
+
+def SetDefaultColoring(state):
+  """Set coloring behavior to |state|.
+
+  This is useful for overriding config options via the command line.
+  """
+  if state is None:
+    # Leave it alone -- return quick!
+    return
+
+  global DEFAULT
+  state = state.lower()
+  if state in ('auto',):
+    DEFAULT = state
+  elif state in ('always', 'yes', 'true', True):
+    DEFAULT = 'always'
+  elif state in ('never', 'no', 'false', False):
+    DEFAULT = 'never'
 
 
 class Coloring(object):
@@ -88,9 +111,11 @@ class Coloring(object):
     self._config = config
     self._out = sys.stdout
 
-    on = self._config.GetString(self._section)
+    on = DEFAULT
     if on is None:
-      on = self._config.GetString('color.ui')
+      on = self._config.GetString(self._section)
+      if on is None:
+        on = self._config.GetString('color.ui')
 
     if on == 'auto':
       if pager.active or os.isatty(1):
@@ -121,22 +146,45 @@ class Coloring(object):
   def printer(self, opt=None, fg=None, bg=None, attr=None):
     s = self
     c = self.colorer(opt, fg, bg, attr)
+
     def f(fmt, *args):
       s._out.write(c(fmt, *args))
     return f
 
+  def nofmt_printer(self, opt=None, fg=None, bg=None, attr=None):
+    s = self
+    c = self.nofmt_colorer(opt, fg, bg, attr)
+
+    def f(fmt):
+      s._out.write(c(fmt))
+    return f
+
   def colorer(self, opt=None, fg=None, bg=None, attr=None):
     if self._on:
       c = self._parse(opt, fg, bg, attr)
+
       def f(fmt, *args):
         output = fmt % args
         return ''.join([c, output, RESET])
       return f
     else:
+
       def f(fmt, *args):
         return fmt % args
       return f
 
+  def nofmt_colorer(self, opt=None, fg=None, bg=None, attr=None):
+    if self._on:
+      c = self._parse(opt, fg, bg, attr)
+
+      def f(fmt):
+        return ''.join([c, fmt, RESET])
+      return f
+    else:
+      def f(fmt):
+        return fmt
+      return f
+
   def _parse(self, opt, fg, bg, attr):
     if not opt:
       return _Color(fg, bg, attr)

command.py

@@ -22,6 +22,7 @@ import sys
 from error import NoSuchProjectError
 from error import InvalidProjectGroupsError
 
+
 class Command(object):
   """Base class for any command line action in repo.
   """
@@ -30,9 +31,30 @@ class Command(object):
   manifest = None
   _optparse = None
 
-  def WantPager(self, opt):
+  def WantPager(self, _opt):
     return False
 
+  def ReadEnvironmentOptions(self, opts):
+    """ Set options from environment variables. """
+
+    env_options = self._RegisteredEnvironmentOptions()
+
+    for env_key, opt_key in env_options.items():
+      # Get the user-set option value if any
+      opt_value = getattr(opts, opt_key)
+
+      # If the value is set, it means the user has passed it as a command
+      # line option, and we should use that.  Otherwise we can try to set it
+      # with the value from the corresponding environment variable.
+      if opt_value is not None:
+        continue
+
+      env_value = os.environ.get(env_key)
+      if env_value is not None:
+        setattr(opts, opt_key, env_value)
+
+    return opts
+
   @property
   def OptionParser(self):
     if self._optparse is None:
@@ -41,7 +63,7 @@ class Command(object):
         usage = self.helpUsage.strip().replace('%prog', me)
       except AttributeError:
         usage = 'repo %s' % self.NAME
-      self._optparse = optparse.OptionParser(usage = usage)
+      self._optparse = optparse.OptionParser(usage=usage)
       self._Options(self._optparse)
     return self._optparse
 
@@ -49,6 +71,24 @@ class Command(object):
     """Initialize the option parser.
     """
 
+  def _RegisteredEnvironmentOptions(self):
+    """Get options that can be set from environment variables.
+
+    Return a dictionary mapping environment variable name
+    to option key name that it can override.
+
+    Example: {'REPO_MY_OPTION': 'my_option'}
+
+    Will allow the option with key value 'my_option' to be set
+    from the value in the environment variable named 'REPO_MY_OPTION'.
+
+    Note: This does not work properly for options that are explicitly
+    set to None by the user, or options that are defined with a
+    default value other than None.
+
+    """
+    return {}
+
   def Usage(self):
     """Display usage and terminate.
     """
@@ -60,69 +100,122 @@ class Command(object):
     """
     raise NotImplementedError
 
-  def GetProjects(self, args, missing_ok=False):
+  def _ResetPathToProjectMap(self, projects):
+    self._by_path = dict((p.worktree, p) for p in projects)
+
+  def _UpdatePathToProjectMap(self, project):
+    self._by_path[project.worktree] = project
+
+  def _GetProjectByPath(self, manifest, path):
+    project = None
+    if os.path.exists(path):
+      oldpath = None
+      while path and \
+            path != oldpath and \
+            path != manifest.topdir:
+        try:
+          project = self._by_path[path]
+          break
+        except KeyError:
+          oldpath = path
+          path = os.path.dirname(path)
+      if not project and path == manifest.topdir:
+        try:
+          project = self._by_path[path]
+        except KeyError:
+          pass
+    else:
+      try:
+        project = self._by_path[path]
+      except KeyError:
+        pass
+    return project
+
+  def GetProjects(self, args, manifest=None, groups='', missing_ok=False,
+                  submodules_ok=False):
     """A list of projects that match the arguments.
     """
-    all_projects = self.manifest.projects
+    if not manifest:
+      manifest = self.manifest
+    all_projects_list = manifest.projects
     result = []
 
-    mp = self.manifest.manifestProject
+    mp = manifest.manifestProject
 
-    groups = mp.config.GetString('manifest.groups')
     if not groups:
-      groups = 'all,-notdefault,platform-' + platform.system().lower()
-    groups = [x for x in re.split('[,\s]+', groups) if x]
+      groups = mp.config.GetString('manifest.groups')
+    if not groups:
+      groups = 'default,platform-' + platform.system().lower()
+    groups = [x for x in re.split(r'[,\s]+', groups) if x]
 
     if not args:
-      for project in all_projects.values():
-        if ((missing_ok or project.Exists) and
-            project.MatchesGroups(groups)):
+      derived_projects = {}
+      for project in all_projects_list:
+        if submodules_ok or project.sync_s:
+          derived_projects.update((p.name, p)
+                                  for p in project.GetDerivedSubprojects())
+      all_projects_list.extend(derived_projects.values())
+      for project in all_projects_list:
+        if (missing_ok or project.Exists) and project.MatchesGroups(groups):
           result.append(project)
     else:
-      by_path = None
+      self._ResetPathToProjectMap(all_projects_list)
 
       for arg in args:
-        project = all_projects.get(arg)
+        projects = manifest.GetProjectsWithName(arg)
 
-        if not project:
+        if not projects:
           path = os.path.abspath(arg).replace('\\', '/')
+          project = self._GetProjectByPath(manifest, path)
 
-          if not by_path:
-            by_path = dict()
-            for p in all_projects.values():
-              by_path[p.worktree] = p
+          # If it's not a derived project, update path->project mapping and
+          # search again, as arg might actually point to a derived subproject.
+          if (project and not project.Derived and (submodules_ok or
+                                                   project.sync_s)):
+            search_again = False
+            for subproject in project.GetDerivedSubprojects():
+              self._UpdatePathToProjectMap(subproject)
+              search_again = True
+            if search_again:
+              project = self._GetProjectByPath(manifest, path) or project
 
-          if os.path.exists(path):
-            oldpath = None
-            while path \
-              and path != oldpath \
-              and path != self.manifest.topdir:
-              try:
-                project = by_path[path]
-                break
-              except KeyError:
-                oldpath = path
-                path = os.path.dirname(path)
-          else:
-            try:
-              project = by_path[path]
-            except KeyError:
-              pass
+          if project:
+            projects = [project]
 
-        if not project:
+        if not projects:
           raise NoSuchProjectError(arg)
-        if not missing_ok and not project.Exists:
-          raise NoSuchProjectError(arg)
-        if not project.MatchesGroups(groups):
-          raise InvalidProjectGroupsError(arg)
 
-        result.append(project)
+        for project in projects:
+          if not missing_ok and not project.Exists:
+            raise NoSuchProjectError(arg)
+          if not project.MatchesGroups(groups):
+            raise InvalidProjectGroupsError(arg)
+
+        result.extend(projects)
 
     def _getpath(x):
       return x.relpath
     result.sort(key=_getpath)
     return result
 
+  def FindProjects(self, args, inverse=False):
+    result = []
+    patterns = [re.compile(r'%s' % a, re.IGNORECASE) for a in args]
+    for project in self.GetProjects(''):
+      for pattern in patterns:
+        match = pattern.search(project.name) or pattern.search(project.relpath)
+        if not inverse and match:
+          result.append(project)
+          break
+        if inverse and match:
+          break
+      else:
+        if inverse:
+          result.append(project)
+    result.sort(key=lambda project: project.relpath)
+    return result
+
+
 # pylint: disable=W0223
 # Pylint warns that the `InteractiveCommand` and `PagedCommand` classes do not
 # override method `Execute` which is abstract in `Command`.  Since that method
@@ -132,19 +225,33 @@ class InteractiveCommand(Command):
   """Command which requires user interaction on the tty and
      must not run within a pager, even if the user asks to.
   """
-  def WantPager(self, opt):
+  def WantPager(self, _opt):
     return False
 
+
 class PagedCommand(Command):
   """Command which defaults to output in a pager, as its
      display tends to be larger than one screen full.
   """
-  def WantPager(self, opt):
+  def WantPager(self, _opt):
     return True
 
 # pylint: enable=W0223
 
+
 class MirrorSafeCommand(object):
   """Command permits itself to run within a mirror,
      and does not require a working directory.
   """
+
+
+class GitcAvailableCommand(object):
+  """Command that requires GITC to be available, but does
+     not require the local client to be a GITC client.
+  """
+
+
+class GitcClientCommand(object):
+  """Command that requires the local client to be a GITC
+     client.
+  """

docs/manifest-format.txt

@@ -26,38 +26,64 @@ following DTD:
                         manifest-server?,
                         remove-project*,
                         project*,
+                        extend-project*,
                         repo-hooks?)>
-  
+
     <!ELEMENT notice (#PCDATA)>
-  
+
     <!ELEMENT remote (EMPTY)>
     <!ATTLIST remote name         ID    #REQUIRED>
     <!ATTLIST remote alias        CDATA #IMPLIED>
     <!ATTLIST remote fetch        CDATA #REQUIRED>
+    <!ATTLIST remote pushurl      CDATA #IMPLIED>
     <!ATTLIST remote review       CDATA #IMPLIED>
-  
+    <!ATTLIST remote revision     CDATA #IMPLIED>
+
     <!ELEMENT default (EMPTY)>
-    <!ATTLIST default remote   IDREF #IMPLIED>
-    <!ATTLIST default revision CDATA #IMPLIED>
-    <!ATTLIST default sync-j   CDATA #IMPLIED>
-    <!ATTLIST default sync-c   CDATA #IMPLIED>
+    <!ATTLIST default remote      IDREF #IMPLIED>
+    <!ATTLIST default revision    CDATA #IMPLIED>
+    <!ATTLIST default dest-branch CDATA #IMPLIED>
+    <!ATTLIST default sync-j      CDATA #IMPLIED>
+    <!ATTLIST default sync-c      CDATA #IMPLIED>
+    <!ATTLIST default sync-s      CDATA #IMPLIED>
 
     <!ELEMENT manifest-server (EMPTY)>
-    <!ATTLIST url              CDATA #REQUIRED>
-  
-    <!ELEMENT project (annotation?)>
-    <!ATTLIST project name     CDATA #REQUIRED>
-    <!ATTLIST project path     CDATA #IMPLIED>
-    <!ATTLIST project remote   IDREF #IMPLIED>
-    <!ATTLIST project revision CDATA #IMPLIED>
-    <!ATTLIST project groups   CDATA #IMPLIED>
-    <!ATTLIST project sync-c   CDATA #IMPLIED>
+    <!ATTLIST manifest-server url CDATA #REQUIRED>
+
+    <!ELEMENT project (annotation*,
+                       project*,
+                       copyfile*,
+                       linkfile*)>
+    <!ATTLIST project name        CDATA #REQUIRED>
+    <!ATTLIST project path        CDATA #IMPLIED>
+    <!ATTLIST project remote      IDREF #IMPLIED>
+    <!ATTLIST project revision    CDATA #IMPLIED>
+    <!ATTLIST project dest-branch CDATA #IMPLIED>
+    <!ATTLIST project groups      CDATA #IMPLIED>
+    <!ATTLIST project sync-c      CDATA #IMPLIED>
+    <!ATTLIST project sync-s      CDATA #IMPLIED>
+    <!ATTLIST project upstream CDATA #IMPLIED>
+    <!ATTLIST project clone-depth CDATA #IMPLIED>
+    <!ATTLIST project force-path CDATA #IMPLIED>
 
     <!ELEMENT annotation (EMPTY)>
     <!ATTLIST annotation name  CDATA #REQUIRED>
     <!ATTLIST annotation value CDATA #REQUIRED>
     <!ATTLIST annotation keep  CDATA "true">
-  
+
+    <!ELEMENT copyfile (EMPTY)>
+    <!ATTLIST copyfile src  CDATA #REQUIRED>
+    <!ATTLIST copyfile dest CDATA #REQUIRED>
+
+    <!ELEMENT linkfile (EMPTY)>
+    <!ATTLIST linkfile src CDATA #REQUIRED>
+    <!ATTLIST linkfile dest CDATA #REQUIRED>
+
+    <!ELEMENT extend-project (EMPTY)>
+    <!ATTLIST extend-project name CDATA #REQUIRED>
+    <!ATTLIST extend-project path CDATA #IMPLIED>
+    <!ATTLIST extend-project groups CDATA #IMPLIED>
+
     <!ELEMENT remove-project (EMPTY)>
     <!ATTLIST remove-project name  CDATA #REQUIRED>
 
@@ -100,10 +126,20 @@ Attribute `fetch`: The Git URL prefix for all projects which use
 this remote.  Each project's name is appended to this prefix to
 form the actual URL used to clone the project.
 
+Attribute `pushurl`: The Git "push" URL prefix for all projects
+which use this remote.  Each project's name is appended to this
+prefix to form the actual URL used to "git push" the project.
+This attribute is optional; if not specified then "git push"
+will use the same URL as the `fetch` attribute.
+
 Attribute `review`: Hostname of the Gerrit server where reviews
 are uploaded to by `repo upload`.  This attribute is optional;
 if not specified then `repo upload` will not function.
 
+Attribute `revision`: Name of a Git branch (e.g. `master` or
+`refs/heads/master`). Remotes with their own revision will override
+the default revision.
+
 Element default
 ---------------
 
@@ -119,6 +155,20 @@ Attribute `revision`: Name of a Git branch (e.g. `master` or
 `refs/heads/master`).  Project elements lacking their own
 revision attribute will use this revision.
 
+Attribute `dest-branch`: Name of a Git branch (e.g. `master`).
+Project elements not setting their own `dest-branch` will inherit
+this value. If this value is not set, projects will use `revision`
+by default instead.
+
+Attribute `sync-j`: Number of parallel jobs to use when synching.
+
+Attribute `sync-c`: Set to true to only sync the given Git
+branch (specified in the `revision` attribute) rather than the
+whole ref space.  Project elements lacking a sync-c element of
+their own will use this value.
+
+Attribute `sync-s`: Set to true to also sync sub-projects.
+
 
 Element manifest-server
 -----------------------
@@ -132,7 +182,8 @@ The manifest server should implement the following RPC methods:
   GetApprovedManifest(branch, target)
 
 Return a manifest in which each project is pegged to a known good revision
-for the current branch and target.
+for the current branch and target. This is used by repo sync when the
+--smart-sync option is given.
 
 The target to use is defined by environment variables TARGET_PRODUCT
 and TARGET_BUILD_VARIANT. These variables are used to create a string
@@ -144,7 +195,8 @@ should choose a reasonable default target.
   GetManifest(tag)
 
 Return a manifest in which each project is pegged to the revision at
-the specified tag.
+the specified tag. This is used by repo sync when the --smart-tag option
+is given.
 
 
 Element project
@@ -152,7 +204,10 @@ Element project
 
 One or more project elements may be specified.  Each element
 describes a single Git repository to be cloned into the repo
-client workspace.
+client workspace.  You may specify Git-submodules by creating a
+nested project.  Git-submodules will be automatically
+recognized and inherit their parent's attributes, but those
+may be overridden by an explicitly specified project element.
 
 Attribute `name`: A unique name for this project.  The project's
 name is appended onto its remote's fetch URL to generate the actual
@@ -163,7 +218,8 @@ URL to configure the Git remote with.  The URL gets formed as:
 where ${remote_fetch} is the remote's fetch attribute and
 ${project_name} is the project's name attribute.  The suffix ".git"
 is always appended as repo assumes the upstream is a forest of
-bare Git repositories.
+bare Git repositories.  If the project has a parent element, its
+name will be prefixed by the parent's.
 
 The project name must match the name Gerrit knows, if Gerrit is
 being used for code reviews.
@@ -171,6 +227,8 @@ being used for code reviews.
 Attribute `path`: An optional path relative to the top directory
 of the repo client where the Git working directory for this project
 should be placed.  If not supplied the project name is used.
+If the project has a parent element, its path will be prefixed
+by the parent's.
 
 Attribute `remote`: Name of a previously defined remote element.
 If not supplied the remote given by the default element is used.
@@ -180,7 +238,13 @@ to track for this project.  Names can be relative to refs/heads
 (e.g. just "master") or absolute (e.g. "refs/heads/master").
 Tags and/or explicit SHA-1s should work in theory, but have not
 been extensively tested.  If not supplied the revision given by
-the default element is used.
+the remote element is used if applicable, else the default
+element is used.
+
+Attribute `dest-branch`: Name of a Git branch (e.g. `master`).
+When using `repo upload`, changes will be submitted for code
+review on this branch. If unspecified both here and in the
+default element, `revision` is used instead.
 
 Attribute `groups`: List of groups to which this project belongs,
 whitespace or comma separated.  All projects belong to the group
@@ -190,6 +254,44 @@ its name:`name` and path:`path`.  E.g. for
 definition is implicitly in the following manifest groups:
 default, name:monkeys, and path:barrel-of.  If you place a project in the
 group "notdefault", it will not be automatically downloaded by repo.
+If the project has a parent element, the `name` and `path` here
+are the prefixed ones.
+
+Attribute `sync-c`: Set to true to only sync the given Git
+branch (specified in the `revision` attribute) rather than the
+whole ref space.
+
+Attribute `sync-s`: Set to true to also sync sub-projects.
+
+Attribute `upstream`: Name of the Git ref in which a sha1
+can be found.  Used when syncing a revision locked manifest in
+-c mode to avoid having to sync the entire ref space.
+
+Attribute `clone-depth`: Set the depth to use when fetching this
+project.  If specified, this value will override any value given
+to repo init with the --depth option on the command line.
+
+Attribute `force-path`: Set to true to force this project to create the
+local mirror repository according to its `path` attribute (if supplied)
+rather than the `name` attribute.  This attribute only applies to the
+local mirrors syncing, it will be ignored when syncing the projects in a
+client working directory.
+
+Element extend-project
+----------------------
+
+Modify the attributes of the named project.
+
+This element is mostly useful in a local manifest file, to modify the
+attributes of an existing project without completely replacing the
+existing project definition.  This makes the local manifest more robust
+against changes to the original manifest.
+
+Attribute `path`: If specified, limit the change to projects checked out
+at the specified path, rather than all projects with the given name.
+
+Attribute `groups`: List of additional groups to which this project
+belongs.  Same syntax as the corresponding element of `project`.
 
 Element annotation
 ------------------
@@ -202,6 +304,21 @@ prefixed with REPO__.  In addition, there is an optional attribute
 "false".  This attribute determines whether or not the annotation will
 be kept when exported with the manifest subcommand.
 
+Element copyfile
+----------------
+
+Zero or more copyfile elements may be specified as children of a
+project element. Each element describes a src-dest pair of files;
+the "src" file will be copied to the "dest" place during 'repo sync'
+command.
+"src" is project relative, "dest" is relative to the top of the tree.
+
+Element linkfile
+----------------
+
+It's just like copyfile and runs at the same time as copyfile but
+instead of copying it creates a symlink.
+
 Element remove-project
 ----------------------
 
@@ -209,7 +326,7 @@ Deletes the named project from the internal manifest table, possibly
 allowing a subsequent project element in the same manifest file to
 replace the project with a different source.
 
-This element is mostly useful in the local_manifest.xml, where
+This element is mostly useful in a local manifest file, where
 the user can remove a project, and possibly replace it with their
 own definition.
 
@@ -218,21 +335,25 @@ Element include
 
 This element provides the capability of including another manifest
 file into the originating manifest.  Normal rules apply for the
-target manifest to include- it must be a usable manifest on it's own.
+target manifest to include - it must be a usable manifest on its own.
 
-Attribute `name`; the manifest to include, specified relative to
-the manifest repositories root.
+Attribute `name`: the manifest to include, specified relative to
+the manifest repository's root.
 
 
-Local Manifest
-==============
+Local Manifests
+===============
 
-Additional remotes and projects may be added through a local
-manifest, stored in `$TOP_DIR/.repo/local_manifest.xml`.
+Additional remotes and projects may be added through local manifest
+files stored in `$TOP_DIR/.repo/local_manifests/*.xml`.
 
 For example:
 
-  $ cat .repo/local_manifest.xml
+  $ ls .repo/local_manifests
+  local_manifest.xml
+  another_local_manifest.xml
+
+  $ cat .repo/local_manifests/local_manifest.xml
   <?xml version="1.0" encoding="UTF-8"?>
   <manifest>
     <project path="manifest"
@@ -241,6 +362,17 @@ For example:
              name="platform/manifest" />
   </manifest>
 
-Users may add projects to the local manifest prior to a `repo sync`
+Users may add projects to the local manifest(s) prior to a `repo sync`
 invocation, instructing repo to automatically download and manage
 these extra projects.
+
+Manifest files stored in `$TOP_DIR/.repo/local_manifests/*.xml` will
+be loaded in alphabetical order.
+
+Additional remotes and projects may also be added through a local
+manifest, stored in `$TOP_DIR/.repo/local_manifest.xml`. This method
+is deprecated in favor of using multiple manifest files as mentioned
+above.
+
+If `$TOP_DIR/.repo/local_manifest.xml` exists, it will be loaded before
+any manifest files stored in `$TOP_DIR/.repo/local_manifests/*.xml`.

editor.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import os
 import re
 import sys
@@ -53,10 +54,10 @@ class Editor(object):
       return e
 
     if os.getenv('TERM') == 'dumb':
-      print >>sys.stderr,\
+      print(
 """No editor specified in GIT_EDITOR, core.editor, VISUAL or EDITOR.
 Tried to fall back to vi but terminal is dumb.  Please configure at
-least one of these before using this command."""
+least one of these before using this command.""", file=sys.stderr)
       sys.exit(1)
 
     return 'vi'
@@ -67,7 +68,7 @@ least one of these before using this command."""
 
        Args:
          data        : the text to edit
-  
+
       Returns:
         new value of edited text; None if editing did not succeed
     """

error.py

@@ -21,6 +21,17 @@ class ManifestInvalidRevisionError(Exception):
   """The revision value in a project is incorrect.
   """
 
+class NoManifestException(Exception):
+  """The required manifest does not exist.
+  """
+  def __init__(self, path, reason):
+    super(NoManifestException, self).__init__()
+    self.path = path
+    self.reason = reason
+
+  def __str__(self):
+    return self.reason
+
 class EditorError(Exception):
   """Unspecified error from the user's text editor.
   """
@@ -69,7 +80,7 @@ class NoSuchProjectError(Exception):
     self.name = name
 
   def __str__(self):
-    if self.Name is None:
+    if self.name is None:
       return 'in current directory'
     return self.name
 
@@ -82,7 +93,7 @@ class InvalidProjectGroupsError(Exception):
     self.name = name
 
   def __str__(self):
-    if self.Name is None:
+    if self.name is None:
       return 'in current directory'
     return self.name
 

git_command.py

@@ -13,13 +13,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
+import fcntl
 import os
+import select
 import sys
 import subprocess
 import tempfile
 from signal import SIGTERM
 from error import GitError
 from trace import REPO_TRACE, IsTrace, Trace
+from wrapper import Wrapper
 
 GIT = 'git'
 MIN_GIT_VERSION = (1, 5, 4)
@@ -74,25 +78,33 @@ def terminate_ssh_clients():
 
 _git_version = None
 
+class _sfd(object):
+  """select file descriptor class"""
+  def __init__(self, fd, dest, std_name):
+    assert std_name in ('stdout', 'stderr')
+    self.fd = fd
+    self.dest = dest
+    self.std_name = std_name
+  def fileno(self):
+    return self.fd.fileno()
+
 class _GitCall(object):
   def version(self):
     p = GitCommand(None, ['--version'], capture_stdout=True)
     if p.Wait() == 0:
-      return p.stdout
+      if hasattr(p.stdout, 'decode'):
+        return p.stdout.decode('utf-8')
+      else:
+        return p.stdout
     return None
 
   def version_tuple(self):
     global _git_version
-
     if _git_version is None:
       ver_str = git.version()
-      if ver_str.startswith('git version '):
-        _git_version = tuple(
-          map(lambda x: int(x),
-            ver_str[len('git version '):].strip().split('-')[0].split('.')[0:3]
-          ))
-      else:
-        print >>sys.stderr, 'fatal: "%s" unsupported' % ver_str
+      _git_version = Wrapper().ParseGitVersion(ver_str)
+      if _git_version is None:
+        print('fatal: "%s" unsupported' % ver_str, file=sys.stderr)
         sys.exit(1)
     return _git_version
 
@@ -110,8 +122,8 @@ def git_require(min_version, fail=False):
   if min_version <= git_version:
     return True
   if fail:
-    need = '.'.join(map(lambda x: str(x), min_version))
-    print >>sys.stderr, 'fatal: git %s or later required' % need
+    need = '.'.join(map(str, min_version))
+    print('fatal: git %s or later required' % need, file=sys.stderr)
     sys.exit(1)
   return False
 
@@ -132,15 +144,18 @@ class GitCommand(object):
                gitdir = None):
     env = os.environ.copy()
 
-    for e in [REPO_TRACE,
+    for key in [REPO_TRACE,
               GIT_DIR,
               'GIT_ALTERNATE_OBJECT_DIRECTORIES',
               'GIT_OBJECT_DIRECTORY',
               'GIT_WORK_TREE',
               'GIT_GRAFT_FILE',
               'GIT_INDEX_FILE']:
-      if e in env:
-        del env[e]
+      if key in env:
+        del env[key]
+
+    # If we are not capturing std* then need to print it.
+    self.tee = {'stdout': not capture_stdout, 'stderr': not capture_stderr}
 
     if disable_editor:
       _setenv(env, 'GIT_EDITOR', ':')
@@ -153,6 +168,9 @@ class GitCommand(object):
       if p is not None:
         s = p + ' ' + s
       _setenv(env, 'GIT_CONFIG_PARAMETERS', s)
+    if 'GIT_ALLOW_PROTOCOL' not in env:
+      _setenv(env, 'GIT_ALLOW_PROTOCOL',
+              'file:git:http:https:ssh:persistent-http:persistent-https:sso:rpc')
 
     if project:
       if not cwd:
@@ -165,22 +183,21 @@ class GitCommand(object):
       if gitdir:
         _setenv(env, GIT_DIR, gitdir)
       cwd = None
-    command.extend(cmdv)
+    command.append(cmdv[0])
+    # Need to use the --progress flag for fetch/clone so output will be
+    # displayed as by default git only does progress output if stderr is a TTY.
+    if sys.stderr.isatty() and cmdv[0] in ('fetch', 'clone'):
+      if '--progress' not in cmdv and '--quiet' not in cmdv:
+        command.append('--progress')
+    command.extend(cmdv[1:])
 
     if provide_stdin:
       stdin = subprocess.PIPE
     else:
       stdin = None
 
-    if capture_stdout:
-      stdout = subprocess.PIPE
-    else:
-      stdout = None
-
-    if capture_stderr:
-      stderr = subprocess.PIPE
-    else:
-      stderr = None
+    stdout = subprocess.PIPE
+    stderr = subprocess.PIPE
 
     if IsTrace():
       global LAST_CWD
@@ -229,8 +246,36 @@ class GitCommand(object):
   def Wait(self):
     try:
       p = self.process
-      (self.stdout, self.stderr) = p.communicate()
-      rc = p.returncode
+      rc = self._CaptureOutput()
     finally:
       _remove_ssh_client(p)
     return rc
+
+  def _CaptureOutput(self):
+    p = self.process
+    s_in = [_sfd(p.stdout, sys.stdout, 'stdout'),
+            _sfd(p.stderr, sys.stderr, 'stderr')]
+    self.stdout = ''
+    self.stderr = ''
+
+    for s in s_in:
+      flags = fcntl.fcntl(s.fd, fcntl.F_GETFL)
+      fcntl.fcntl(s.fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
+
+    while s_in:
+      in_ready, _, _ = select.select(s_in, [], [])
+      for s in in_ready:
+        buf = s.fd.read(4096)
+        if not buf:
+          s_in.remove(s)
+          continue
+        if not hasattr(buf, 'encode'):
+          buf = buf.decode()
+        if s.std_name == 'stdout':
+          self.stdout += buf
+        else:
+          self.stderr += buf
+        if self.tee[s.std_name]:
+          s.dest.write(buf)
+          s.dest.flush()
+    return p.wait()

git_config.py

@@ -13,7 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import cPickle
+from __future__ import print_function
+
+import contextlib
+import errno
+import json
 import os
 import re
 import subprocess
@@ -23,11 +27,25 @@ try:
 except ImportError:
   import dummy_threading as _threading
 import time
-import urllib2
+
+from pyversion import is_python3
+if is_python3():
+  import urllib.request
+  import urllib.error
+else:
+  import urllib2
+  import imp
+  urllib = imp.new_module('urllib')
+  urllib.request = urllib2
+  urllib.error = urllib2
 
 from signal import SIGTERM
 from error import GitError, UploadError
 from trace import Trace
+if is_python3():
+  from http.client import HTTPException
+else:
+  from httplib import HTTPException
 
 from git_command import GitCommand
 from git_command import ssh_sock
@@ -35,7 +53,7 @@ from git_command import terminate_ssh_clients
 
 R_HEADS = 'refs/heads/'
 R_TAGS  = 'refs/tags/'
-ID_RE = re.compile('^[0-9a-f]{40}$')
+ID_RE = re.compile(r'^[0-9a-f]{40}$')
 
 REVIEW_CACHE = dict()
 
@@ -64,7 +82,7 @@ class GitConfig(object):
     return cls(configfile = os.path.join(gitdir, 'config'),
                defaults = defaults)
 
-  def __init__(self, configfile, defaults=None, pickleFile=None):
+  def __init__(self, configfile, defaults=None, jsonFile=None):
     self.file = configfile
     self.defaults = defaults
     self._cache_dict = None
@@ -72,12 +90,11 @@ class GitConfig(object):
     self._remotes = {}
     self._branches = {}
 
-    if pickleFile is None:
-      self._pickle = os.path.join(
+    self._json = jsonFile
+    if self._json is None:
+      self._json = os.path.join(
         os.path.dirname(self.file),
-        '.repopickle_' + os.path.basename(self.file))
-    else:
-      self._pickle = pickleFile
+        '.repo_' + os.path.basename(self.file) + '.json')
 
   def Has(self, name, include_defaults = True):
     """Return true if this configuration file has the key.
@@ -157,7 +174,7 @@ class GitConfig(object):
       elif old != value:
         self._cache[key] = list(value)
         self._do('--replace-all', name, value[0])
-        for i in xrange(1, len(value)):
+        for i in range(1, len(value)):
           self._do('--add', name, value[i])
 
     elif len(old) != 1 or old[0] != value:
@@ -201,9 +218,9 @@ class GitConfig(object):
     """Resolve any url.*.insteadof references.
     """
     for new_url in self.GetSubSections('url'):
-      old_url = self.GetString('url.%s.insteadof' % new_url)
-      if old_url is not None and url.startswith(old_url):
-        return new_url + url[len(old_url):]
+      for old_url in self.GetString('url.%s.insteadof' % new_url, True):
+        if old_url is not None and url.startswith(old_url):
+          return new_url + url[len(old_url):]
     return url
 
   @property
@@ -232,50 +249,41 @@ class GitConfig(object):
     return self._cache_dict
 
   def _Read(self):
-    d = self._ReadPickle()
+    d = self._ReadJson()
     if d is None:
       d = self._ReadGit()
-      self._SavePickle(d)
+      self._SaveJson(d)
     return d
 
-  def _ReadPickle(self):
+  def _ReadJson(self):
     try:
-      if os.path.getmtime(self._pickle) \
+      if os.path.getmtime(self._json) \
       <= os.path.getmtime(self.file):
-        os.remove(self._pickle)
+        os.remove(self._json)
         return None
     except OSError:
       return None
     try:
-      Trace(': unpickle %s', self.file)
-      fd = open(self._pickle, 'rb')
+      Trace(': parsing %s', self.file)
+      fd = open(self._json)
       try:
-        return cPickle.load(fd)
+        return json.load(fd)
       finally:
         fd.close()
-    except EOFError:
-      os.remove(self._pickle)
-      return None
-    except IOError:
-      os.remove(self._pickle)
-      return None
-    except cPickle.PickleError:
-      os.remove(self._pickle)
+    except (IOError, ValueError):
+      os.remove(self._json)
       return None
 
-  def _SavePickle(self, cache):
+  def _SaveJson(self, cache):
     try:
-      fd = open(self._pickle, 'wb')
+      fd = open(self._json, 'w')
       try:
-        cPickle.dump(cache, fd, cPickle.HIGHEST_PROTOCOL)
+        json.dump(cache, fd, indent=2)
       finally:
         fd.close()
-    except IOError:
-      if os.path.exists(self._pickle):
-        os.remove(self._pickle)
-    except cPickle.PickleError:
-      if os.path.exists(self._pickle):
-        os.remove(self._pickle)
+    except (IOError, TypeError):
+      if os.path.exists(self._json):
+        os.remove(self._json)
 
   def _ReadGit(self):
     """
@@ -288,12 +296,13 @@ class GitConfig(object):
     d = self._do('--null', '--list')
     if d is None:
       return c
-    for line in d.rstrip('\0').split('\0'):
+    for line in d.decode('utf-8').rstrip('\0').split('\0'):  # pylint: disable=W1401
+                                                             # Backslash is not anomalous
       if '\n' in line:
-          key, val = line.split('\n', 1)
+        key, val = line.split('\n', 1)
       else:
-          key = line
-          val = None
+        key = line
+        val = None
 
       if key in c:
         c[key].append(val)
@@ -418,7 +427,7 @@ def _open_ssh(host, port=None):
                      '-o','ControlPath %s' % ssh_sock(),
                      host]
     if port is not None:
-      command_base[1:1] = ['-p',str(port)]
+      command_base[1:1] = ['-p', str(port)]
 
     # Since the key wasn't in _master_keys, we think that master isn't running.
     # ...but before actually starting a master, we'll double-check.  This can
@@ -451,14 +460,17 @@ def _open_ssh(host, port=None):
       p = subprocess.Popen(command)
     except Exception as e:
       _ssh_master = False
-      print >>sys.stderr, \
-        '\nwarn: cannot enable ssh control master for %s:%s\n%s' \
-        % (host,port, str(e))
+      print('\nwarn: cannot enable ssh control master for %s:%s\n%s'
+             % (host,port, str(e)), file=sys.stderr)
+      return False
+
+    time.sleep(1)
+    ssh_died = (p.poll() is not None)
+    if ssh_died:
       return False
 
     _master_processes.append(p)
     _master_keys.add(key)
-    time.sleep(1)
     return True
   finally:
     _master_keys_lock.release()
@@ -496,6 +508,43 @@ def GetSchemeFromUrl(url):
     return m.group(1)
   return None
 
+@contextlib.contextmanager
+def GetUrlCookieFile(url, quiet):
+  if url.startswith('persistent-'):
+    try:
+      p = subprocess.Popen(
+          ['git-remote-persistent-https', '-print_config', url],
+          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+          stderr=subprocess.PIPE)
+      try:
+        cookieprefix = 'http.cookiefile='
+        proxyprefix = 'http.proxy='
+        cookiefile = None
+        proxy = None
+        for line in p.stdout:
+          line = line.strip()
+          if line.startswith(cookieprefix):
+            cookiefile = line[len(cookieprefix):]
+          if line.startswith(proxyprefix):
+            proxy = line[len(proxyprefix):]
+        # Leave subprocess open, as cookie file may be transient.
+        if cookiefile or proxy:
+          yield cookiefile, proxy
+          return
+      finally:
+        p.stdin.close()
+        if p.wait():
+          err_msg = p.stderr.read()
+          if ' -print_config' in err_msg:
+            pass  # Persistent proxy doesn't support -print_config.
+          elif not quiet:
+            print(err_msg, file=sys.stderr)
+    except OSError as e:
+      if e.errno == errno.ENOENT:
+        pass  # No persistent proxy.
+      raise
+  yield GitConfig.ForUser().GetString('http.cookiefile'), None
+
 def _preconnect(url):
   m = URI_ALL.match(url)
   if m:
@@ -523,10 +572,11 @@ class Remote(object):
     self._config = config
     self.name = name
     self.url = self._Get('url')
+    self.pushUrl = self._Get('pushurl')
     self.review = self._Get('review')
     self.projectname = self._Get('projectname')
-    self.fetch = map(lambda x: RefSpec.FromString(x),
-                     self._Get('fetch', all_keys=True))
+    self.fetch = list(map(RefSpec.FromString,
+                      self._Get('fetch', all_keys=True)))
     self._review_url = None
 
   def _InsteadOf(self):
@@ -560,7 +610,9 @@ class Remote(object):
         return None
 
       u = self.review
-      if not u.startswith('http:') and not u.startswith('https:'):
+      if u.startswith('persistent-'):
+        u = u[len('persistent-'):]
+      if u.split(':')[0] not in ('http', 'https', 'sso'):
         u = 'http://%s' % u
       if u.endswith('/Gerrit'):
         u = u[:len(u) - len('/Gerrit')]
@@ -576,26 +628,28 @@ class Remote(object):
         host, port = os.environ['REPO_HOST_PORT_INFO'].split()
         self._review_url = self._SshReviewUrl(userEmail, host, port)
         REVIEW_CACHE[u] = self._review_url
+      elif u.startswith('sso:'):
+        self._review_url = u  # Assume it's right
+        REVIEW_CACHE[u] = self._review_url
       else:
         try:
           info_url = u + 'ssh_info'
-          info = urllib2.urlopen(info_url).read()
-          if '<' in info:
-            # Assume the server gave us some sort of HTML
-            # response back, like maybe a login page.
+          info = urllib.request.urlopen(info_url).read()
+          if info == 'NOT_AVAILABLE' or '<' in info:
+            # If `info` contains '<', we assume the server gave us some sort
+            # of HTML response back, like maybe a login page.
             #
-            raise UploadError('%s: Cannot parse response' % info_url)
-
-          if info == 'NOT_AVAILABLE':
-            # Assume HTTP if SSH is not enabled.
-            self._review_url = http_url + 'p/'
+            # Assume HTTP if SSH is not enabled or ssh_info doesn't look right.
+            self._review_url = http_url
           else:
             host, port = info.split()
             self._review_url = self._SshReviewUrl(userEmail, host, port)
-        except urllib2.HTTPError as e:
+        except urllib.error.HTTPError as e:
           raise UploadError('%s: %s' % (self.review, str(e)))
-        except urllib2.URLError as e:
+        except urllib.error.URLError as e:
           raise UploadError('%s: %s' % (self.review, str(e)))
+        except HTTPException as e:
+          raise UploadError('%s: %s' % (self.review, e.__class__.__name__))
 
         REVIEW_CACHE[u] = self._review_url
     return self._review_url + self.projectname
@@ -609,9 +663,7 @@ class Remote(object):
   def ToLocal(self, rev):
     """Convert a remote revision string to something we have locally.
     """
-    if IsId(rev):
-      return rev
-    if rev.startswith(R_TAGS):
+    if self.name == '.' or IsId(rev):
       return rev
 
     if not rev.startswith('refs/'):
@@ -620,6 +672,10 @@ class Remote(object):
     for spec in self.fetch:
       if spec.SourceMatches(rev):
         return spec.MapSource(rev)
+
+    if not rev.startswith(R_HEADS):
+      return rev
+
     raise GitError('remote %s does not have %s' % (self.name, rev))
 
   def WritesTo(self, ref):
@@ -643,9 +699,13 @@ class Remote(object):
     """Save this remote to the configuration.
     """
     self._Set('url', self.url)
+    if self.pushUrl is not None:
+      self._Set('pushurl', self.pushUrl + '/' + self.projectname)
+    else:
+      self._Set('pushurl', self.pushUrl)
     self._Set('review', self.review)
     self._Set('projectname', self.projectname)
-    self._Set('fetch', map(lambda x: str(x), self.fetch))
+    self._Set('fetch', list(map(str, self.fetch)))
 
   def _Set(self, key, value):
     key = 'remote.%s.%s' % (self.name, key)
@@ -689,7 +749,7 @@ class Branch(object):
       self._Set('merge', self.merge)
 
     else:
-      fd = open(self._config.file, 'ab')
+      fd = open(self._config.file, 'a')
       try:
         fd.write('[branch "%s"]\n' % self.name)
         if self.remote:

git_refs.py

@@ -66,7 +66,7 @@ class GitRefs(object):
   def _NeedUpdate(self):
     Trace(': scan refs %s', self._gitdir)
 
-    for name, mtime in self._mtime.iteritems():
+    for name, mtime in self._mtime.items():
       try:
         if mtime != os.path.getmtime(os.path.join(self._gitdir, name)):
           return True
@@ -89,7 +89,7 @@ class GitRefs(object):
     attempts = 0
     while scan and attempts < 5:
       scan_next = {}
-      for name, dest in scan.iteritems():
+      for name, dest in scan.items():
         if dest in self._phyref:
           self._phyref[name] = self._phyref[dest]
         else:
@@ -100,7 +100,7 @@ class GitRefs(object):
   def _ReadPackedRefs(self):
     path = os.path.join(self._gitdir, 'packed-refs')
     try:
-      fd = open(path, 'rb')
+      fd = open(path, 'r')
       mtime = os.path.getmtime(path)
     except IOError:
       return
@@ -108,6 +108,7 @@ class GitRefs(object):
       return
     try:
       for line in fd:
+        line = str(line)
         if line[0] == '#':
           continue
         if line[0] == '^':
@@ -138,18 +139,22 @@ class GitRefs(object):
   def _ReadLoose1(self, path, name):
     try:
       fd = open(path, 'rb')
-    except:
+    except IOError:
       return
 
     try:
       try:
         mtime = os.path.getmtime(path)
         ref_id = fd.readline()
-      except:
+      except (IOError, OSError):
         return
     finally:
       fd.close()
 
+    try:
+      ref_id = ref_id.decode()
+    except AttributeError:
+      pass
     if not ref_id:
       return
     ref_id = ref_id[:-1]

gitc_utils.py

@@ -0,0 +1,154 @@
+#
+# Copyright (C) 2015 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+import os
+import platform
+import re
+import sys
+import time
+
+import git_command
+import git_config
+import wrapper
+
+from error import ManifestParseError
+
+NUM_BATCH_RETRIEVE_REVISIONID = 32
+
+def get_gitc_manifest_dir():
+  return wrapper.Wrapper().get_gitc_manifest_dir()
+
+def parse_clientdir(gitc_fs_path):
+  return wrapper.Wrapper().gitc_parse_clientdir(gitc_fs_path)
+
+def _set_project_revisions(projects):
+  """Sets the revisionExpr for a list of projects.
+
+  Because of the limit of open file descriptors allowed, length of projects
+  should not be overly large. Recommend calling this function multiple times
+  with each call not exceeding NUM_BATCH_RETRIEVE_REVISIONID projects.
+
+  @param projects: List of project objects to set the revionExpr for.
+  """
+  # Retrieve the commit id for each project based off of it's current
+  # revisionExpr and it is not already a commit id.
+  project_gitcmds = [(
+      project, git_command.GitCommand(None,
+                                      ['ls-remote',
+                                       project.remote.url,
+                                       project.revisionExpr],
+                                      capture_stdout=True, cwd='/tmp'))
+      for project in projects if not git_config.IsId(project.revisionExpr)]
+  for proj, gitcmd in project_gitcmds:
+    if gitcmd.Wait():
+      print('FATAL: Failed to retrieve revisionExpr for %s' % proj)
+      sys.exit(1)
+    revisionExpr = gitcmd.stdout.split('\t')[0]
+    if not revisionExpr:
+      raise(ManifestParseError('Invalid SHA-1 revision project %s (%s)' %
+                               (proj.remote.url, proj.revisionExpr)))
+    proj.revisionExpr = revisionExpr
+
+def _manifest_groups(manifest):
+  """Returns the manifest group string that should be synced
+
+  This is the same logic used by Command.GetProjects(), which is used during
+  repo sync
+
+  @param manifest: The XmlManifest object
+  """
+  mp = manifest.manifestProject
+  groups = mp.config.GetString('manifest.groups')
+  if not groups:
+    groups = 'default,platform-' + platform.system().lower()
+  return groups
+
+def generate_gitc_manifest(gitc_manifest, manifest, paths=None):
+  """Generate a manifest for shafsd to use for this GITC client.
+
+  @param gitc_manifest: Current gitc manifest, or None if there isn't one yet.
+  @param manifest: A GitcManifest object loaded with the current repo manifest.
+  @param paths: List of project paths we want to update.
+  """
+
+  print('Generating GITC Manifest by fetching revision SHAs for each '
+        'project.')
+  if paths is None:
+    paths = manifest.paths.keys()
+
+  groups = [x for x in re.split(r'[,\s]+', _manifest_groups(manifest)) if x]
+
+  # Convert the paths to projects, and filter them to the matched groups.
+  projects = [manifest.paths[p] for p in paths]
+  projects = [p for p in projects if p.MatchesGroups(groups)]
+
+  if gitc_manifest is not None:
+    for path, proj in manifest.paths.iteritems():
+      if not proj.MatchesGroups(groups):
+        continue
+
+      if not proj.upstream and not git_config.IsId(proj.revisionExpr):
+        proj.upstream = proj.revisionExpr
+
+      if not path in gitc_manifest.paths:
+        # Any new projects need their first revision, even if we weren't asked
+        # for them.
+        projects.append(proj)
+      elif not path in paths:
+        # And copy revisions from the previous manifest if we're not updating
+        # them now.
+        gitc_proj = gitc_manifest.paths[path]
+        if gitc_proj.old_revision:
+          proj.revisionExpr = None
+          proj.old_revision = gitc_proj.old_revision
+        else:
+          proj.revisionExpr = gitc_proj.revisionExpr
+
+  index = 0
+  while index < len(projects):
+    _set_project_revisions(
+        projects[index:(index+NUM_BATCH_RETRIEVE_REVISIONID)])
+    index += NUM_BATCH_RETRIEVE_REVISIONID
+
+  if gitc_manifest is not None:
+    for path, proj in gitc_manifest.paths.iteritems():
+      if proj.old_revision and path in paths:
+        # If we updated a project that has been started, keep the old-revision
+        # updated.
+        repo_proj = manifest.paths[path]
+        repo_proj.old_revision = repo_proj.revisionExpr
+        repo_proj.revisionExpr = None
+
+  # Convert URLs from relative to absolute.
+  for _name, remote in manifest.remotes.iteritems():
+    remote.fetchUrl = remote.resolvedFetchUrl
+
+  # Save the manifest.
+  save_manifest(manifest)
+
+def save_manifest(manifest, client_dir=None):
+  """Save the manifest file in the client_dir.
+
+  @param client_dir: Client directory to save the manifest in.
+  @param manifest: Manifest object to save.
+  """
+  if not client_dir:
+    client_dir = manifest.gitc_client_dir
+  with open(os.path.join(client_dir, '.manifest'), 'w') as f:
+    manifest.Save(f, groups=_manifest_groups(manifest))
+  # TODO(sbasi/jorg): Come up with a solution to remove the sleep below.
+  # Give the GITC filesystem time to register the manifest changes.
+  time.sleep(3)

hooks/commit-msg

@@ -1,7 +1,7 @@
 #!/bin/sh
-# From Gerrit Code Review 2.5-rc0
+# From Gerrit Code Review 2.12.1
 #
-# Part of Gerrit Code Review (http://code.google.com/p/gerrit/)
+# Part of Gerrit Code Review (https://www.gerritcodereview.com/)
 #
 # Copyright (C) 2009 The Android Open Source Project
 #
@@ -18,14 +18,16 @@
 # limitations under the License.
 #
 
-CHANGE_ID_AFTER="Bug|Issue"
+unset GREP_OPTIONS
+
+CHANGE_ID_AFTER="Bug|Issue|Test"
 MSG="$1"
 
 # Check for, and add if missing, a unique Change-Id
 #
 add_ChangeId() {
 	clean_message=`sed -e '
-		/^diff --git a\/.*/{
+		/^diff --git .*/{
 			s///
 			q
 		}
@@ -37,6 +39,17 @@ add_ChangeId() {
 		return
 	fi
 
+	# Do not add Change-Id to temp commits
+	if echo "$clean_message" | head -1 | grep -q '^\(fixup\|squash\)!'
+	then
+		return
+	fi
+
+	if test "false" = "`git config --bool --get gerrit.createChangeId`"
+	then
+		return
+	fi
+
 	# Does Change-Id: already exist? if so, exit (no change).
 	if grep -i '^Change-Id:' "$MSG" >/dev/null
 	then
@@ -51,6 +64,10 @@ add_ChangeId() {
 		AWK=/usr/xpg4/bin/awk
 	fi
 
+	# Get core.commentChar from git config or use default symbol
+	commentChar=`git config --get core.commentChar`
+	commentChar=${commentChar:-#}
+
 	# How this works:
 	# - parse the commit message as (textLine+ blankLine*)*
 	# - assume textLine+ to be a footer until proven otherwise
@@ -69,13 +86,13 @@ add_ChangeId() {
 		blankLines = 0
 	}
 
-	# Skip lines starting with "#" without any spaces before it.
-	/^#/ { next }
+	# Skip lines starting with commentChar without any spaces before it.
+	/^'"$commentChar"'/ { next }
 
 	# Skip the line starting with the diff command and everything after it,
 	# up to the end of the file, assuming it is only patch data.
 	# If more than one line before the diff was empty, strip all but one.
-	/^diff --git a/ {
+	/^diff --git / {
 		blankLines = 0
 		while (getline) { }
 		next
@@ -152,7 +169,7 @@ add_ChangeId() {
 		if (unprinted) {
 			print "Change-Id: I'"$id"'"
 		}
-	}' "$MSG" > $T && mv $T "$MSG" || rm -f $T
+	}' "$MSG" > "$T" && mv "$T" "$MSG" || rm -f "$T"
 }
 _gen_ChangeIdInput() {
 	echo "tree `git write-tree`"

hooks/pre-auto-gc

@@ -35,7 +35,7 @@ elif grep -q "AC Power \+: 1" /proc/pmu/info 2>/dev/null
 then
 	exit 0
 elif test -x /usr/bin/pmset && /usr/bin/pmset -g batt |
-	grep -q "Currently drawing from 'AC Power'"
+	grep -q "drawing from 'AC Power'"
 then
 	exit 0
 elif test -d /sys/bus/acpi/drivers/battery && test 0 = \

main.py

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env python
 #
 # Copyright (C) 2008 The Android Open Source Project
 #
@@ -14,40 +14,56 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-magic='--calling-python-from-/bin/sh--'
-"""exec" python -E "$0" "$@" """#$magic"
-if __name__ == '__main__':
-  import sys
-  if sys.argv[-1] == '#%s' % magic:
-    del sys.argv[-1]
-del magic
-
+from __future__ import print_function
 import getpass
 import imp
 import netrc
 import optparse
 import os
-import re
 import sys
 import time
-import urllib2
 
+from pyversion import is_python3
+if is_python3():
+  import urllib.request
+else:
+  import urllib2
+  urllib = imp.new_module('urllib')
+  urllib.request = urllib2
+
+try:
+  import kerberos
+except ImportError:
+  kerberos = None
+
+from color import SetDefaultColoring
 from trace import SetTrace
 from git_command import git, GitCommand
 from git_config import init_ssh, close_ssh
 from command import InteractiveCommand
 from command import MirrorSafeCommand
+from command import GitcAvailableCommand, GitcClientCommand
 from subcmds.version import Version
 from editor import Editor
 from error import DownloadError
+from error import InvalidProjectGroupsError
 from error import ManifestInvalidRevisionError
+from error import ManifestParseError
+from error import NoManifestException
 from error import NoSuchProjectError
 from error import RepoChangedException
-from manifest_xml import XmlManifest
+import gitc_utils
+from manifest_xml import GitcManifest, XmlManifest
 from pager import RunPager
+from wrapper import WrapperPath, Wrapper
 
 from subcmds import all_commands
 
+if not is_python3():
+  # pylint:disable=W0622
+  input = raw_input
+  # pylint:enable=W0622
+
 global_options = optparse.OptionParser(
                  usage="repo [-p|--paginate|--no-pager] COMMAND [ARGS]"
                  )
@@ -57,6 +73,9 @@ global_options.add_option('-p', '--paginate',
 global_options.add_option('--no-pager',
                           dest='no_pager', action='store_true',
                           help='disable the pager')
+global_options.add_option('--color',
+                          choices=('auto', 'always', 'never'), default=None,
+                          help='control color usage: auto, always, never')
 global_options.add_option('--trace',
                           dest='trace', action='store_true',
                           help='trace git command execution')
@@ -79,7 +98,7 @@ class _Repo(object):
     name = None
     glob = []
 
-    for i in xrange(0, len(argv)):
+    for i in range(len(argv)):
       if not argv[i].startswith('-'):
         name = argv[i]
         if i > 0:
@@ -98,28 +117,52 @@ class _Repo(object):
       if name == 'help':
         name = 'version'
       else:
-        print >>sys.stderr, 'fatal: invalid usage of --version'
+        print('fatal: invalid usage of --version', file=sys.stderr)
         return 1
 
+    SetDefaultColoring(gopts.color)
+
     try:
       cmd = self.commands[name]
     except KeyError:
-      print >>sys.stderr,\
-            "repo: '%s' is not a repo command.  See 'repo help'."\
-            % name
+      print("repo: '%s' is not a repo command.  See 'repo help'." % name,
+            file=sys.stderr)
       return 1
 
     cmd.repodir = self.repodir
     cmd.manifest = XmlManifest(cmd.repodir)
+    cmd.gitc_manifest = None
+    gitc_client_name = gitc_utils.parse_clientdir(os.getcwd())
+    if gitc_client_name:
+      cmd.gitc_manifest = GitcManifest(cmd.repodir, gitc_client_name)
+      cmd.manifest.isGitcClient = True
+
     Editor.globalConfig = cmd.manifest.globalConfig
 
     if not isinstance(cmd, MirrorSafeCommand) and cmd.manifest.IsMirror:
-      print >>sys.stderr, \
-            "fatal: '%s' requires a working directory"\
-            % name
+      print("fatal: '%s' requires a working directory" % name,
+            file=sys.stderr)
       return 1
 
-    copts, cargs = cmd.OptionParser.parse_args(argv)
+    if isinstance(cmd, GitcAvailableCommand) and not gitc_utils.get_gitc_manifest_dir():
+      print("fatal: '%s' requires GITC to be available" % name,
+            file=sys.stderr)
+      return 1
+
+    if isinstance(cmd, GitcClientCommand) and not gitc_client_name:
+      print("fatal: '%s' requires a GITC client" % name,
+            file=sys.stderr)
+      return 1
+
+    try:
+      copts, cargs = cmd.OptionParser.parse_args(argv)
+      copts = cmd.ReadEnvironmentOptions(copts)
+    except NoManifestException as e:
+      print('error: in `%s`: %s' % (' '.join([name] + argv), str(e)),
+        file=sys.stderr)
+      print('error: manifest missing or unreadable -- please run init',
+            file=sys.stderr)
+      return 1
 
     if not gopts.no_pager and not isinstance(cmd, InteractiveCommand):
       config = cmd.manifest.globalConfig
@@ -132,87 +175,81 @@ class _Repo(object):
       if use_pager:
         RunPager(config)
 
+    start = time.time()
     try:
-      start = time.time()
-      try:
-        result = cmd.Execute(copts, cargs)
-      finally:
-        elapsed = time.time() - start
-        hours, remainder = divmod(elapsed, 3600)
-        minutes, seconds = divmod(remainder, 60)
-        if gopts.time:
-          if hours == 0:
-            print >>sys.stderr, 'real\t%dm%.3fs' \
-              % (minutes, seconds)
-          else:
-            print >>sys.stderr, 'real\t%dh%dm%.3fs' \
-              % (hours, minutes, seconds)
-    except DownloadError as e:
-      print >>sys.stderr, 'error: %s' % str(e)
-      return 1
-    except ManifestInvalidRevisionError as e:
-      print >>sys.stderr, 'error: %s' % str(e)
-      return 1
+      result = cmd.Execute(copts, cargs)
+    except (DownloadError, ManifestInvalidRevisionError,
+        NoManifestException) as e:
+      print('error: in `%s`: %s' % (' '.join([name] + argv), str(e)),
+        file=sys.stderr)
+      if isinstance(e, NoManifestException):
+        print('error: manifest missing or unreadable -- please run init',
+              file=sys.stderr)
+      result = 1
     except NoSuchProjectError as e:
       if e.name:
-        print >>sys.stderr, 'error: project %s not found' % e.name
+        print('error: project %s not found' % e.name, file=sys.stderr)
       else:
-        print >>sys.stderr, 'error: no project in current directory'
-      return 1
+        print('error: no project in current directory', file=sys.stderr)
+      result = 1
+    except InvalidProjectGroupsError as e:
+      if e.name:
+        print('error: project group must be enabled for project %s' % e.name, file=sys.stderr)
+      else:
+        print('error: project group must be enabled for the project in the current directory', file=sys.stderr)
+      result = 1
+    finally:
+      elapsed = time.time() - start
+      hours, remainder = divmod(elapsed, 3600)
+      minutes, seconds = divmod(remainder, 60)
+      if gopts.time:
+        if hours == 0:
+          print('real\t%dm%.3fs' % (minutes, seconds), file=sys.stderr)
+        else:
+          print('real\t%dh%dm%.3fs' % (hours, minutes, seconds),
+                file=sys.stderr)
 
     return result
 
+
 def _MyRepoPath():
   return os.path.dirname(__file__)
 
-def _MyWrapperPath():
-  return os.path.join(os.path.dirname(__file__), 'repo')
-
-_wrapper_module = None
-def WrapperModule():
-  global _wrapper_module
-  if not _wrapper_module:
-    _wrapper_module = imp.load_source('wrapper', _MyWrapperPath())
-  return _wrapper_module
-
-def _CurrentWrapperVersion():
-  return WrapperModule().VERSION
 
 def _CheckWrapperVersion(ver, repo_path):
   if not repo_path:
     repo_path = '~/bin/repo'
 
   if not ver:
-    print >>sys.stderr, 'no --wrapper-version argument'
+    print('no --wrapper-version argument', file=sys.stderr)
     sys.exit(1)
 
-  exp = _CurrentWrapperVersion()
-  ver = tuple(map(lambda x: int(x), ver.split('.')))
+  exp = Wrapper().VERSION
+  ver = tuple(map(int, ver.split('.')))
   if len(ver) == 1:
     ver = (0, ver[0])
 
+  exp_str = '.'.join(map(str, exp))
   if exp[0] > ver[0] or ver < (0, 4):
-    exp_str = '.'.join(map(lambda x: str(x), exp))
-    print >>sys.stderr, """
+    print("""
 !!! A new repo command (%5s) is available.    !!!
 !!! You must upgrade before you can continue:   !!!
 
     cp %s %s
-""" % (exp_str, _MyWrapperPath(), repo_path)
+""" % (exp_str, WrapperPath(), repo_path), file=sys.stderr)
     sys.exit(1)
 
   if exp > ver:
-    exp_str = '.'.join(map(lambda x: str(x), exp))
-    print >>sys.stderr, """
+    print("""
 ... A new repo command (%5s) is available.
 ... You should upgrade soon:
 
     cp %s %s
-""" % (exp_str, _MyWrapperPath(), repo_path)
+""" % (exp_str, WrapperPath(), repo_path), file=sys.stderr)
 
 def _CheckRepoDir(repo_dir):
   if not repo_dir:
-    print >>sys.stderr, 'no --repo-dir argument'
+    print('no --repo-dir argument', file=sys.stderr)
     sys.exit(1)
 
 def _PruneOptions(argv, opt):
@@ -264,11 +301,11 @@ def _UserAgent():
     _user_agent = 'git-repo/%s (%s) git/%s Python/%d.%d.%d' % (
       repo_version,
       os_name,
-      '.'.join(map(lambda d: str(d), git.version_tuple())),
+      '.'.join(map(str, git.version_tuple())),
       py_version[0], py_version[1], py_version[2])
   return _user_agent
 
-class _UserAgentHandler(urllib2.BaseHandler):
+class _UserAgentHandler(urllib.request.BaseHandler):
   def http_request(self, req):
     req.add_header('User-Agent', _UserAgent())
     return req
@@ -278,22 +315,22 @@ class _UserAgentHandler(urllib2.BaseHandler):
     return req
 
 def _AddPasswordFromUserInput(handler, msg, req):
-    # If repo could not find auth info from netrc, try to get it from user input
-    url = req.get_full_url()
-    user, password = handler.passwd.find_user_password(None, url)
-    if user is None:
-      print msg
-      try:
-        user = raw_input('User: ')
-        password = getpass.getpass()
-      except KeyboardInterrupt:
-        return
-      handler.passwd.add_password(None, url, user, password)
+  # If repo could not find auth info from netrc, try to get it from user input
+  url = req.get_full_url()
+  user, password = handler.passwd.find_user_password(None, url)
+  if user is None:
+    print(msg)
+    try:
+      user = input('User: ')
+      password = getpass.getpass()
+    except KeyboardInterrupt:
+      return
+    handler.passwd.add_password(None, url, user, password)
 
-class _BasicAuthHandler(urllib2.HTTPBasicAuthHandler):
+class _BasicAuthHandler(urllib.request.HTTPBasicAuthHandler):
   def http_error_401(self, req, fp, code, msg, headers):
     _AddPasswordFromUserInput(self, msg, req)
-    return urllib2.HTTPBasicAuthHandler.http_error_401(
+    return urllib.request.HTTPBasicAuthHandler.http_error_401(
       self, req, fp, code, msg, headers)
 
   def http_error_auth_reqed(self, authreq, host, req, headers):
@@ -303,7 +340,7 @@ class _BasicAuthHandler(urllib2.HTTPBasicAuthHandler):
         val = val.replace('\n', '')
         old_add_header(name, val)
       req.add_header = _add_header
-      return urllib2.AbstractBasicAuthHandler.http_error_auth_reqed(
+      return urllib.request.AbstractBasicAuthHandler.http_error_auth_reqed(
         self, authreq, host, req, headers)
     except:
       reset = getattr(self, 'reset_retry_count', None)
@@ -313,10 +350,10 @@ class _BasicAuthHandler(urllib2.HTTPBasicAuthHandler):
         self.retried = 0
       raise
 
-class _DigestAuthHandler(urllib2.HTTPDigestAuthHandler):
+class _DigestAuthHandler(urllib.request.HTTPDigestAuthHandler):
   def http_error_401(self, req, fp, code, msg, headers):
     _AddPasswordFromUserInput(self, msg, req)
-    return urllib2.HTTPDigestAuthHandler.http_error_401(
+    return urllib.request.HTTPDigestAuthHandler.http_error_401(
       self, req, fp, code, msg, headers)
 
   def http_error_auth_reqed(self, auth_header, host, req, headers):
@@ -326,7 +363,7 @@ class _DigestAuthHandler(urllib2.HTTPDigestAuthHandler):
         val = val.replace('\n', '')
         old_add_header(name, val)
       req.add_header = _add_header
-      return urllib2.AbstractDigestAuthHandler.http_error_auth_reqed(
+      return urllib.request.AbstractDigestAuthHandler.http_error_auth_reqed(
         self, auth_header, host, req, headers)
     except:
       reset = getattr(self, 'reset_retry_count', None)
@@ -336,10 +373,90 @@ class _DigestAuthHandler(urllib2.HTTPDigestAuthHandler):
         self.retried = 0
       raise
 
+class _KerberosAuthHandler(urllib.request.BaseHandler):
+  def __init__(self):
+    self.retried = 0
+    self.context = None
+    self.handler_order = urllib.request.BaseHandler.handler_order - 50
+
+  def http_error_401(self, req, fp, code, msg, headers): # pylint:disable=unused-argument
+    host = req.get_host()
+    retry = self.http_error_auth_reqed('www-authenticate', host, req, headers)
+    return retry
+
+  def http_error_auth_reqed(self, auth_header, host, req, headers):
+    try:
+      spn = "HTTP@%s" % host
+      authdata = self._negotiate_get_authdata(auth_header, headers)
+
+      if self.retried > 3:
+        raise urllib.request.HTTPError(req.get_full_url(), 401,
+          "Negotiate auth failed", headers, None)
+      else:
+        self.retried += 1
+
+      neghdr = self._negotiate_get_svctk(spn, authdata)
+      if neghdr is None:
+        return None
+
+      req.add_unredirected_header('Authorization', neghdr)
+      response = self.parent.open(req)
+
+      srvauth = self._negotiate_get_authdata(auth_header, response.info())
+      if self._validate_response(srvauth):
+        return response
+    except kerberos.GSSError:
+      return None
+    except:
+      self.reset_retry_count()
+      raise
+    finally:
+      self._clean_context()
+
+  def reset_retry_count(self):
+    self.retried = 0
+
+  def _negotiate_get_authdata(self, auth_header, headers):
+    authhdr = headers.get(auth_header, None)
+    if authhdr is not None:
+      for mech_tuple in authhdr.split(","):
+        mech, __, authdata = mech_tuple.strip().partition(" ")
+        if mech.lower() == "negotiate":
+          return authdata.strip()
+    return None
+
+  def _negotiate_get_svctk(self, spn, authdata):
+    if authdata is None:
+      return None
+
+    result, self.context = kerberos.authGSSClientInit(spn)
+    if result < kerberos.AUTH_GSS_COMPLETE:
+      return None
+
+    result = kerberos.authGSSClientStep(self.context, authdata)
+    if result < kerberos.AUTH_GSS_CONTINUE:
+      return None
+
+    response = kerberos.authGSSClientResponse(self.context)
+    return "Negotiate %s" % response
+
+  def _validate_response(self, authdata):
+    if authdata is None:
+      return None
+    result = kerberos.authGSSClientStep(self.context, authdata)
+    if result == kerberos.AUTH_GSS_COMPLETE:
+      return True
+    return None
+
+  def _clean_context(self):
+    if self.context is not None:
+      kerberos.authGSSClientClean(self.context)
+      self.context = None
+
 def init_http():
   handlers = [_UserAgentHandler()]
 
-  mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
+  mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
   try:
     n = netrc.netrc()
     for host in n.hosts:
@@ -352,14 +469,16 @@ def init_http():
     pass
   handlers.append(_BasicAuthHandler(mgr))
   handlers.append(_DigestAuthHandler(mgr))
+  if kerberos:
+    handlers.append(_KerberosAuthHandler())
 
   if 'http_proxy' in os.environ:
     url = os.environ['http_proxy']
-    handlers.append(urllib2.ProxyHandler({'http': url, 'https': url}))
+    handlers.append(urllib.request.ProxyHandler({'http': url, 'https': url}))
   if 'REPO_CURL_VERBOSE' in os.environ:
-    handlers.append(urllib2.HTTPHandler(debuglevel=1))
-    handlers.append(urllib2.HTTPSHandler(debuglevel=1))
-  urllib2.install_opener(urllib2.build_opener(*handlers))
+    handlers.append(urllib.request.HTTPHandler(debuglevel=1))
+    handlers.append(urllib.request.HTTPSHandler(debuglevel=1))
+  urllib.request.install_opener(urllib.request.build_opener(*handlers))
 
 def _Main(argv):
   result = 0
@@ -389,6 +508,10 @@ def _Main(argv):
     finally:
       close_ssh()
   except KeyboardInterrupt:
+    print('aborted by user', file=sys.stderr)
+    result = 1
+  except ManifestParseError as mpe:
+    print('fatal: %s' % mpe, file=sys.stderr)
     result = 1
   except RepoChangedException as rce:
     # If repo changed, re-exec ourselves.
@@ -398,8 +521,8 @@ def _Main(argv):
     try:
       os.execv(__file__, argv)
     except OSError as e:
-      print >>sys.stderr, 'fatal: cannot restart repo after upgrade'
-      print >>sys.stderr, 'fatal: %s' % e
+      print('fatal: cannot restart repo after upgrade', file=sys.stderr)
+      print('fatal: %s' % e, file=sys.stderr)
       result = 128
 
   sys.exit(result)

manifest_xml.py

@@ -13,62 +13,102 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import itertools
 import os
 import re
 import sys
-import urlparse
 import xml.dom.minidom
 
+from pyversion import is_python3
+if is_python3():
+  import urllib.parse
+else:
+  import imp
+  import urlparse
+  urllib = imp.new_module('urllib')
+  urllib.parse = urlparse
+
+import gitc_utils
 from git_config import GitConfig
 from git_refs import R_HEADS, HEAD
 from project import RemoteSpec, Project, MetaProject
-from error import ManifestParseError
+from error import ManifestParseError, ManifestInvalidRevisionError
 
 MANIFEST_FILE_NAME = 'manifest.xml'
 LOCAL_MANIFEST_NAME = 'local_manifest.xml'
+LOCAL_MANIFESTS_DIR_NAME = 'local_manifests'
 
-urlparse.uses_relative.extend(['ssh', 'git'])
-urlparse.uses_netloc.extend(['ssh', 'git'])
+# urljoin gets confused if the scheme is not known.
+urllib.parse.uses_relative.extend(['ssh', 'git', 'persistent-https', 'rpc'])
+urllib.parse.uses_netloc.extend(['ssh', 'git', 'persistent-https', 'rpc'])
 
 class _Default(object):
   """Project defaults within the manifest."""
 
   revisionExpr = None
+  destBranchExpr = None
   remote = None
   sync_j = 1
   sync_c = False
+  sync_s = False
+
+  def __eq__(self, other):
+    return self.__dict__ == other.__dict__
+
+  def __ne__(self, other):
+    return self.__dict__ != other.__dict__
 
 class _XmlRemote(object):
   def __init__(self,
                name,
                alias=None,
                fetch=None,
+               pushUrl=None,
                manifestUrl=None,
-               review=None):
+               review=None,
+               revision=None):
     self.name = name
     self.fetchUrl = fetch
+    self.pushUrl = pushUrl
     self.manifestUrl = manifestUrl
     self.remoteAlias = alias
     self.reviewUrl = review
+    self.revision = revision
     self.resolvedFetchUrl = self._resolveFetchUrl()
 
+  def __eq__(self, other):
+    return self.__dict__ == other.__dict__
+
+  def __ne__(self, other):
+    return self.__dict__ != other.__dict__
+
   def _resolveFetchUrl(self):
     url = self.fetchUrl.rstrip('/')
     manifestUrl = self.manifestUrl.rstrip('/')
-    # urljoin will get confused if there is no scheme in the base url
-    # ie, if manifestUrl is of the form <hostname:port>
+    # urljoin will gets confused over quite a few things.  The ones we care
+    # about here are:
+    # * no scheme in the base url, like <hostname:port>
+    # We handle no scheme by replacing it with an obscure protocol, gopher
+    # and then replacing it with the original when we are done.
+
     if manifestUrl.find(':') != manifestUrl.find('/') - 1:
-        manifestUrl = 'gopher://' + manifestUrl
-    url = urlparse.urljoin(manifestUrl, url)
-    return re.sub(r'^gopher://', '', url)
+      url = urllib.parse.urljoin('gopher://' + manifestUrl, url)
+      url = re.sub(r'^gopher://', '', url)
+    else:
+      url = urllib.parse.urljoin(manifestUrl, url)
+    return url
 
   def ToRemoteSpec(self, projectName):
     url = self.resolvedFetchUrl.rstrip('/') + '/' + projectName
     remoteName = self.name
     if self.remoteAlias:
       remoteName = self.remoteAlias
-    return RemoteSpec(remoteName, url, self.reviewUrl)
+    return RemoteSpec(remoteName,
+                      url=url,
+                      pushUrl=self.pushUrl,
+                      review=self.reviewUrl,
+                      orig_name=self.name)
 
 class XmlManifest(object):
   """manages the repo configuration file"""
@@ -78,6 +118,8 @@ class XmlManifest(object):
     self.topdir = os.path.dirname(self.repodir)
     self.manifestFile = os.path.join(self.repodir, MANIFEST_FILE_NAME)
     self.globalConfig = GitConfig.ForUser()
+    self.localManifestWarning = False
+    self.isGitcClient = False
 
     self.repoProject = MetaProject(self, 'repo',
       gitdir   = os.path.join(repodir, 'repo/.git'),
@@ -110,29 +152,38 @@ class XmlManifest(object):
     self.Override(name)
 
     try:
-      if os.path.exists(self.manifestFile):
+      if os.path.lexists(self.manifestFile):
         os.remove(self.manifestFile)
       os.symlink('manifests/%s' % name, self.manifestFile)
-    except OSError:
-      raise ManifestParseError('cannot link manifest %s' % name)
+    except OSError as e:
+      raise ManifestParseError('cannot link manifest %s: %s' % (name, str(e)))
 
   def _RemoteToXml(self, r, doc, root):
     e = doc.createElement('remote')
     root.appendChild(e)
     e.setAttribute('name', r.name)
     e.setAttribute('fetch', r.fetchUrl)
+    if r.pushUrl is not None:
+      e.setAttribute('pushurl', r.pushUrl)
+    if r.remoteAlias is not None:
+      e.setAttribute('alias', r.remoteAlias)
     if r.reviewUrl is not None:
       e.setAttribute('review', r.reviewUrl)
+    if r.revision is not None:
+      e.setAttribute('revision', r.revision)
 
-  def Save(self, fd, peg_rev=False, peg_rev_upstream=True):
+  def _ParseGroups(self, groups):
+    return [x for x in re.split(r'[,\s]+', groups) if x]
+
+  def Save(self, fd, peg_rev=False, peg_rev_upstream=True, groups=None):
     """Write the current manifest out to the given file descriptor.
     """
     mp = self.manifestProject
 
-    groups = mp.config.GetString('manifest.groups')
-    if not groups:
-      groups = 'all'
-    groups = [x for x in re.split(r'[,\s]+', groups) if x]
+    if groups is None:
+      groups = mp.config.GetString('manifest.groups')
+    if groups:
+      groups = self._ParseGroups(groups)
 
     doc = xml.dom.minidom.Document()
     root = doc.createElement('manifest')
@@ -148,10 +199,8 @@ class XmlManifest(object):
       notice_element.appendChild(doc.createTextNode(indented_notice))
 
     d = self.default
-    sort_remotes = list(self.remotes.keys())
-    sort_remotes.sort()
 
-    for r in sort_remotes:
+    for r in sorted(self.remotes):
       self._RemoteToXml(self.remotes[r], doc, root)
     if self.remotes:
       root.appendChild(doc.createTextNode(''))
@@ -164,12 +213,18 @@ class XmlManifest(object):
     if d.revisionExpr:
       have_default = True
       e.setAttribute('revision', d.revisionExpr)
+    if d.destBranchExpr:
+      have_default = True
+      e.setAttribute('dest-branch', d.destBranchExpr)
     if d.sync_j > 1:
       have_default = True
       e.setAttribute('sync-j', '%d' % d.sync_j)
     if d.sync_c:
       have_default = True
       e.setAttribute('sync-c', 'true')
+    if d.sync_s:
+      have_default = True
+      e.setAttribute('sync-s', 'true')
     if have_default:
       root.appendChild(e)
       root.appendChild(doc.createTextNode(''))
@@ -180,35 +235,54 @@ class XmlManifest(object):
       root.appendChild(e)
       root.appendChild(doc.createTextNode(''))
 
-    sort_projects = list(self.projects.keys())
-    sort_projects.sort()
-
-    for p in sort_projects:
-      p = self.projects[p]
+    def output_projects(parent, parent_node, projects):
+      for project_name in projects:
+        for project in self._projects[project_name]:
+          output_project(parent, parent_node, project)
 
+    def output_project(parent, parent_node, p):
       if not p.MatchesGroups(groups):
-        continue
+        return
+
+      name = p.name
+      relpath = p.relpath
+      if parent:
+        name = self._UnjoinName(parent.name, name)
+        relpath = self._UnjoinRelpath(parent.relpath, relpath)
 
       e = doc.createElement('project')
-      root.appendChild(e)
-      e.setAttribute('name', p.name)
-      if p.relpath != p.name:
-        e.setAttribute('path', p.relpath)
-      if not d.remote or p.remote.name != d.remote.name:
-        e.setAttribute('remote', p.remote.name)
+      parent_node.appendChild(e)
+      e.setAttribute('name', name)
+      if relpath != name:
+        e.setAttribute('path', relpath)
+      remoteName = None
+      if d.remote:
+        remoteName = d.remote.name
+      if not d.remote or p.remote.orig_name != remoteName:
+        remoteName = p.remote.orig_name
+        e.setAttribute('remote', remoteName)
       if peg_rev:
         if self.IsMirror:
           value = p.bare_git.rev_parse(p.revisionExpr + '^0')
         else:
           value = p.work_git.rev_parse(HEAD + '^0')
         e.setAttribute('revision', value)
-        if peg_rev_upstream and value != p.revisionExpr:
-          # Only save the origin if the origin is not a sha1, and the default
-          # isn't our value, and the if the default doesn't already have that
-          # covered.
-          e.setAttribute('upstream', p.revisionExpr)
-      elif not d.revisionExpr or p.revisionExpr != d.revisionExpr:
-        e.setAttribute('revision', p.revisionExpr)
+        if peg_rev_upstream:
+          if p.upstream:
+            e.setAttribute('upstream', p.upstream)
+          elif value != p.revisionExpr:
+            # Only save the origin if the origin is not a sha1, and the default
+            # isn't our value
+            e.setAttribute('upstream', p.revisionExpr)
+      else:
+        revision = self.remotes[p.remote.orig_name].revision or d.revisionExpr
+        if not revision or revision != p.revisionExpr:
+          e.setAttribute('revision', p.revisionExpr)
+        if p.upstream and p.upstream != p.revisionExpr:
+          e.setAttribute('upstream', p.upstream)
+
+      if p.dest_branch and p.dest_branch != d.destBranchExpr:
+        e.setAttribute('dest-branch', p.dest_branch)
 
       for c in p.copyfiles:
         ce = doc.createElement('copyfile')
@@ -216,6 +290,12 @@ class XmlManifest(object):
         ce.setAttribute('dest', c.dest)
         e.appendChild(ce)
 
+      for l in p.linkfiles:
+        le = doc.createElement('linkfile')
+        le.setAttribute('src', l.src)
+        le.setAttribute('dest', l.dest)
+        e.appendChild(le)
+
       default_groups = ['all', 'name:%s' % p.name, 'path:%s' % p.relpath]
       egroups = [g for g in p.groups if g not in default_groups]
       if egroups:
@@ -231,6 +311,21 @@ class XmlManifest(object):
       if p.sync_c:
         e.setAttribute('sync-c', 'true')
 
+      if p.sync_s:
+        e.setAttribute('sync-s', 'true')
+
+      if p.clone_depth:
+        e.setAttribute('clone-depth', str(p.clone_depth))
+
+      self._output_manifest_project_extras(p, e)
+
+      if p.subprojects:
+        subprojects = set(subp.name for subp in p.subprojects)
+        output_projects(p, e, list(sorted(subprojects)))
+
+    projects = set(p.name for p in self._paths.values() if not p.parent)
+    output_projects(None, root, list(sorted(projects)))
+
     if self._repo_hooks_project:
       root.appendChild(doc.createTextNode(''))
       e = doc.createElement('repo-hooks')
@@ -241,10 +336,19 @@ class XmlManifest(object):
 
     doc.writexml(fd, '', '  ', '\n', 'UTF-8')
 
+  def _output_manifest_project_extras(self, p, e):
+    """Manifests can modify e if they support extra project attributes."""
+    pass
+
+  @property
+  def paths(self):
+    self._Load()
+    return self._paths
+
   @property
   def projects(self):
     self._Load()
-    return self._projects
+    return list(self._paths.values())
 
   @property
   def remotes(self):
@@ -275,9 +379,14 @@ class XmlManifest(object):
   def IsMirror(self):
     return self.manifestProject.config.GetBoolean('repo.mirror')
 
+  @property
+  def IsArchive(self):
+    return self.manifestProject.config.GetBoolean('repo.archive')
+
   def _Unload(self):
     self._loaded = False
     self._projects = {}
+    self._paths = {}
     self._remotes = {}
     self._default = None
     self._repo_hooks_project = None
@@ -299,9 +408,29 @@ class XmlManifest(object):
 
       local = os.path.join(self.repodir, LOCAL_MANIFEST_NAME)
       if os.path.exists(local):
+        if not self.localManifestWarning:
+          self.localManifestWarning = True
+          print('warning: %s is deprecated; put local manifests in `%s` instead'
+                % (LOCAL_MANIFEST_NAME, os.path.join(self.repodir, LOCAL_MANIFESTS_DIR_NAME)),
+                file=sys.stderr)
         nodes.append(self._ParseManifestXml(local, self.repodir))
 
-      self._ParseManifest(nodes)
+      local_dir = os.path.abspath(os.path.join(self.repodir, LOCAL_MANIFESTS_DIR_NAME))
+      try:
+        for local_file in sorted(os.listdir(local_dir)):
+          if local_file.endswith('.xml'):
+            local = os.path.join(local_dir, local_file)
+            nodes.append(self._ParseManifestXml(local, self.repodir))
+      except OSError:
+        pass
+
+      try:
+        self._ParseManifest(nodes)
+      except ManifestParseError as e:
+        # There was a problem parsing, unload ourselves in case they catch
+        # this error and try again later, we will show the correct error
+        self._Unload()
+        raise e
 
       if self.IsMirror:
         self._AddMetaProjectMirror(self.repoProject)
@@ -310,7 +439,11 @@ class XmlManifest(object):
       self._loaded = True
 
   def _ParseManifestXml(self, path, include_root):
-    root = xml.dom.minidom.parse(path)
+    try:
+      root = xml.dom.minidom.parse(path)
+    except (OSError, xml.parsers.expat.ExpatError) as e:
+      raise ManifestParseError("error parsing manifest %s: %s" % (path, e))
+
     if not root or not root.childNodes:
       raise ManifestParseError("no root node in %s" % (path,))
 
@@ -323,43 +456,47 @@ class XmlManifest(object):
     nodes = []
     for node in manifest.childNodes:  # pylint:disable=W0631
                                       # We only get here if manifest is initialised
-        if node.nodeName == 'include':
-            name = self._reqatt(node, 'name')
-            fp = os.path.join(include_root, name)
-            if not os.path.isfile(fp):
-                raise ManifestParseError, \
-                    "include %s doesn't exist or isn't a file" % \
-                    (name,)
-            try:
-                nodes.extend(self._ParseManifestXml(fp, include_root))
-            # should isolate this to the exact exception, but that's
-            # tricky.  actual parsing implementation may vary.
-            except (KeyboardInterrupt, RuntimeError, SystemExit):
-                raise
-            except Exception as e:
-                raise ManifestParseError(
-                    "failed parsing included manifest %s: %s", (name, e))
-        else:
-          nodes.append(node)
+      if node.nodeName == 'include':
+        name = self._reqatt(node, 'name')
+        fp = os.path.join(include_root, name)
+        if not os.path.isfile(fp):
+          raise ManifestParseError("include %s doesn't exist or isn't a file"
+              % (name,))
+        try:
+          nodes.extend(self._ParseManifestXml(fp, include_root))
+        # should isolate this to the exact exception, but that's
+        # tricky.  actual parsing implementation may vary.
+        except (KeyboardInterrupt, RuntimeError, SystemExit):
+          raise
+        except Exception as e:
+          raise ManifestParseError(
+              "failed parsing included manifest %s: %s", (name, e))
+      else:
+        nodes.append(node)
     return nodes
 
   def _ParseManifest(self, node_list):
     for node in itertools.chain(*node_list):
       if node.nodeName == 'remote':
         remote = self._ParseRemote(node)
-        if self._remotes.get(remote.name):
-          raise ManifestParseError(
-              'duplicate remote %s in %s' %
-              (remote.name, self.manifestFile))
-        self._remotes[remote.name] = remote
+        if remote:
+          if remote.name in self._remotes:
+            if remote != self._remotes[remote.name]:
+              raise ManifestParseError(
+                  'remote %s already exists with different attributes' %
+                  (remote.name))
+          else:
+            self._remotes[remote.name] = remote
 
     for node in itertools.chain(*node_list):
       if node.nodeName == 'default':
-        if self._default is not None:
-          raise ManifestParseError(
-              'duplicate default in %s' %
-              (self.manifestFile))
-        self._default = self._ParseDefault(node)
+        new_default = self._ParseDefault(node)
+        if self._default is None:
+          self._default = new_default
+        elif new_default != self._default:
+          raise ManifestParseError('duplicate default in %s' %
+                                   (self.manifestFile))
+
     if self._default is None:
       self._default = _Default()
 
@@ -375,19 +512,47 @@ class XmlManifest(object):
       if node.nodeName == 'manifest-server':
         url = self._reqatt(node, 'url')
         if self._manifest_server is not None:
-            raise ManifestParseError(
-                'duplicate manifest-server in %s' %
-                (self.manifestFile))
+          raise ManifestParseError(
+              'duplicate manifest-server in %s' %
+              (self.manifestFile))
         self._manifest_server = url
 
+    def recursively_add_projects(project):
+      projects = self._projects.setdefault(project.name, [])
+      if project.relpath is None:
+        raise ManifestParseError(
+            'missing path for %s in %s' %
+            (project.name, self.manifestFile))
+      if project.relpath in self._paths:
+        raise ManifestParseError(
+            'duplicate path %s in %s' %
+            (project.relpath, self.manifestFile))
+      self._paths[project.relpath] = project
+      projects.append(project)
+      for subproject in project.subprojects:
+        recursively_add_projects(subproject)
+
     for node in itertools.chain(*node_list):
       if node.nodeName == 'project':
         project = self._ParseProject(node)
-        if self._projects.get(project.name):
-          raise ManifestParseError(
-              'duplicate project %s in %s' %
-              (project.name, self.manifestFile))
-        self._projects[project.name] = project
+        recursively_add_projects(project)
+      if node.nodeName == 'extend-project':
+        name = self._reqatt(node, 'name')
+
+        if name not in self._projects:
+          raise ManifestParseError('extend-project element specifies non-existent '
+                                   'project: %s' % name)
+
+        path = node.getAttribute('path')
+        groups = node.getAttribute('groups')
+        if groups:
+          groups = self._ParseGroups(groups)
+
+        for p in self._projects[name]:
+          if path and p.relpath != path:
+            continue
+          if groups:
+            p.groups.extend(groups)
       if node.nodeName == 'repo-hooks':
         # Get the name of the project and the (space-separated) list of enabled.
         repo_hooks_project = self._reqatt(node, 'in-project')
@@ -401,22 +566,30 @@ class XmlManifest(object):
 
         # Store a reference to the Project.
         try:
-          self._repo_hooks_project = self._projects[repo_hooks_project]
+          repo_hooks_projects = self._projects[repo_hooks_project]
         except KeyError:
           raise ManifestParseError(
               'project %s not found for repo-hooks' %
               (repo_hooks_project))
 
+        if len(repo_hooks_projects) != 1:
+          raise ManifestParseError(
+              'internal error parsing repo-hooks in %s' %
+              (self.manifestFile))
+        self._repo_hooks_project = repo_hooks_projects[0]
+
         # Store the enabled hooks in the Project object.
         self._repo_hooks_project.enabled_repo_hooks = enabled_repo_hooks
       if node.nodeName == 'remove-project':
         name = self._reqatt(node, 'name')
-        try:
-          del self._projects[name]
-        except KeyError:
-          raise ManifestParseError(
-              'project %s not found' %
-              (name))
+
+        if name not in self._projects:
+          raise ManifestParseError('remove-project element specifies non-existent '
+                                   'project: %s' % name)
+
+        for p in self._projects[name]:
+          del self._paths[p.relpath]
+        del self._projects[name]
 
         # If the manifest removes the hooks project, treat it as if it deleted
         # the repo-hooks element too.
@@ -428,7 +601,7 @@ class XmlManifest(object):
     name = None
     m_url = m.GetRemote(m.remote.name).url
     if m_url.endswith('/.git'):
-      raise ManifestParseError, 'refusing to mirror %s' % m_url
+      raise ManifestParseError('refusing to mirror %s' % m_url)
 
     if self._default and self._default.remote:
       url = self._default.remote.resolvedFetchUrl
@@ -454,11 +627,13 @@ class XmlManifest(object):
                         name = name,
                         remote = remote.ToRemoteSpec(name),
                         gitdir = gitdir,
+                        objdir = gitdir,
                         worktree = None,
-                        relpath = None,
+                        relpath = name or None,
                         revisionExpr = m.revisionExpr,
                         revisionId = None)
-      self._projects[project.name] = project
+      self._projects[project.name] = [project]
+      self._paths[project.relpath] = project
 
   def _ParseRemote(self, node):
     """
@@ -469,11 +644,17 @@ class XmlManifest(object):
     if alias == '':
       alias = None
     fetch = self._reqatt(node, 'fetch')
+    pushUrl = node.getAttribute('pushurl')
+    if pushUrl == '':
+      pushUrl = None
     review = node.getAttribute('review')
     if review == '':
       review = None
+    revision = node.getAttribute('revision')
+    if revision == '':
+      revision = None
     manifestUrl = self.manifestProject.config.GetString('remote.origin.url')
-    return _XmlRemote(name, alias, fetch, manifestUrl, review)
+    return _XmlRemote(name, alias, fetch, pushUrl, manifestUrl, review, revision)
 
   def _ParseDefault(self, node):
     """
@@ -485,6 +666,8 @@ class XmlManifest(object):
     if d.revisionExpr == '':
       d.revisionExpr = None
 
+    d.destBranchExpr = node.getAttribute('dest-branch') or None
+
     sync_j = node.getAttribute('sync-j')
     if sync_j == '' or sync_j is None:
       d.sync_j = 1
@@ -496,6 +679,12 @@ class XmlManifest(object):
       d.sync_c = False
     else:
       d.sync_c = sync_c.lower() in ("yes", "true", "1")
+
+    sync_s = node.getAttribute('sync-s')
+    if not sync_s:
+      d.sync_s = False
+    else:
+      d.sync_s = sync_s.lower() in ("yes", "true", "1")
     return d
 
   def _ParseNotice(self, node):
@@ -516,7 +705,7 @@ class XmlManifest(object):
 
     # Figure out minimum indentation, skipping the first line (the same line
     # as the <notice> tag)...
-    minIndent = sys.maxint
+    minIndent = sys.maxsize
     lines = notice.splitlines()
     for line in lines[1:]:
       lstrippedLine = line.lstrip()
@@ -537,35 +726,40 @@ class XmlManifest(object):
 
     return '\n'.join(cleanLines)
 
-  def _ParseProject(self, node):
+  def _JoinName(self, parent_name, name):
+    return os.path.join(parent_name, name)
+
+  def _UnjoinName(self, parent_name, name):
+    return os.path.relpath(name, parent_name)
+
+  def _ParseProject(self, node, parent = None, **extra_proj_attrs):
     """
     reads a <project> element from the manifest file
     """
     name = self._reqatt(node, 'name')
+    if parent:
+      name = self._JoinName(parent.name, name)
 
     remote = self._get_remote(node)
     if remote is None:
       remote = self._default.remote
     if remote is None:
-      raise ManifestParseError, \
-            "no remote for project %s within %s" % \
-            (name, self.manifestFile)
+      raise ManifestParseError("no remote for project %s within %s" %
+            (name, self.manifestFile))
 
-    revisionExpr = node.getAttribute('revision')
+    revisionExpr = node.getAttribute('revision') or remote.revision
     if not revisionExpr:
       revisionExpr = self._default.revisionExpr
     if not revisionExpr:
-      raise ManifestParseError, \
-            "no revision for project %s within %s" % \
-            (name, self.manifestFile)
+      raise ManifestParseError("no revision for project %s within %s" %
+            (name, self.manifestFile))
 
     path = node.getAttribute('path')
     if not path:
       path = name
     if path.startswith('/'):
-      raise ManifestParseError, \
-            "project %s path cannot be absolute in %s" % \
-            (name, self.manifestFile)
+      raise ManifestParseError("project %s path cannot be absolute in %s" %
+            (name, self.manifestFile))
 
     rebase = node.getAttribute('rebase')
     if not rebase:
@@ -579,44 +773,109 @@ class XmlManifest(object):
     else:
       sync_c = sync_c.lower() in ("yes", "true", "1")
 
+    sync_s = node.getAttribute('sync-s')
+    if not sync_s:
+      sync_s = self._default.sync_s
+    else:
+      sync_s = sync_s.lower() in ("yes", "true", "1")
+
+    clone_depth = node.getAttribute('clone-depth')
+    if clone_depth:
+      try:
+        clone_depth = int(clone_depth)
+        if  clone_depth <= 0:
+          raise ValueError()
+      except ValueError:
+        raise ManifestParseError('invalid clone-depth %s in %s' %
+                                 (clone_depth, self.manifestFile))
+
+    dest_branch = node.getAttribute('dest-branch') or self._default.destBranchExpr
+
     upstream = node.getAttribute('upstream')
 
     groups = ''
     if node.hasAttribute('groups'):
       groups = node.getAttribute('groups')
-    groups = [x for x in re.split('[,\s]+', groups) if x]
+    groups = self._ParseGroups(groups)
 
-    default_groups = ['all', 'name:%s' % name, 'path:%s' % path]
+    if parent is None:
+      relpath, worktree, gitdir, objdir = self.GetProjectPaths(name, path)
+    else:
+      relpath, worktree, gitdir, objdir = \
+          self.GetSubprojectPaths(parent, name, path)
+
+    default_groups = ['all', 'name:%s' % name, 'path:%s' % relpath]
     groups.extend(set(default_groups).difference(groups))
 
-    if self.IsMirror:
-      worktree = None
-      gitdir = os.path.join(self.topdir, '%s.git' % name)
-    else:
-      worktree = os.path.join(self.topdir, path).replace('\\', '/')
-      gitdir = os.path.join(self.repodir, 'projects/%s.git' % path)
+    if self.IsMirror and node.hasAttribute('force-path'):
+      if node.getAttribute('force-path').lower() in ("yes", "true", "1"):
+        gitdir = os.path.join(self.topdir, '%s.git' % path)
 
     project = Project(manifest = self,
                       name = name,
                       remote = remote.ToRemoteSpec(name),
                       gitdir = gitdir,
+                      objdir = objdir,
                       worktree = worktree,
-                      relpath = path,
+                      relpath = relpath,
                       revisionExpr = revisionExpr,
                       revisionId = None,
                       rebase = rebase,
                       groups = groups,
                       sync_c = sync_c,
-                      upstream = upstream)
+                      sync_s = sync_s,
+                      clone_depth = clone_depth,
+                      upstream = upstream,
+                      parent = parent,
+                      dest_branch = dest_branch,
+                      **extra_proj_attrs)
 
     for n in node.childNodes:
       if n.nodeName == 'copyfile':
         self._ParseCopyFile(project, n)
+      if n.nodeName == 'linkfile':
+        self._ParseLinkFile(project, n)
       if n.nodeName == 'annotation':
         self._ParseAnnotation(project, n)
+      if n.nodeName == 'project':
+        project.subprojects.append(self._ParseProject(n, parent = project))
 
     return project
 
+  def GetProjectPaths(self, name, path):
+    relpath = path
+    if self.IsMirror:
+      worktree = None
+      gitdir = os.path.join(self.topdir, '%s.git' % name)
+      objdir = gitdir
+    else:
+      worktree = os.path.join(self.topdir, path).replace('\\', '/')
+      gitdir = os.path.join(self.repodir, 'projects', '%s.git' % path)
+      objdir = os.path.join(self.repodir, 'project-objects', '%s.git' % name)
+    return relpath, worktree, gitdir, objdir
+
+  def GetProjectsWithName(self, name):
+    return self._projects.get(name, [])
+
+  def GetSubprojectName(self, parent, submodule_path):
+    return os.path.join(parent.name, submodule_path)
+
+  def _JoinRelpath(self, parent_relpath, relpath):
+    return os.path.join(parent_relpath, relpath)
+
+  def _UnjoinRelpath(self, parent_relpath, relpath):
+    return os.path.relpath(relpath, parent_relpath)
+
+  def GetSubprojectPaths(self, parent, name, path):
+    relpath = self._JoinRelpath(parent.relpath, path)
+    gitdir = os.path.join(parent.gitdir, 'subprojects', '%s.git' % path)
+    objdir = os.path.join(parent.gitdir, 'subproject-objects', '%s.git' % name)
+    if self.IsMirror:
+      worktree = None
+    else:
+      worktree = os.path.join(parent.worktree, path).replace('\\', '/')
+    return relpath, worktree, gitdir, objdir
+
   def _ParseCopyFile(self, project, node):
     src = self._reqatt(node, 'src')
     dest = self._reqatt(node, 'dest')
@@ -625,6 +884,14 @@ class XmlManifest(object):
       # dest is relative to the top of the tree
       project.AddCopyFile(src, dest, os.path.join(self.topdir, dest))
 
+  def _ParseLinkFile(self, project, node):
+    src = self._reqatt(node, 'src')
+    dest = self._reqatt(node, 'dest')
+    if not self.IsMirror:
+      # src is project relative;
+      # dest is relative to the top of the tree
+      project.AddLinkFile(src, dest, os.path.join(self.topdir, dest))
+
   def _ParseAnnotation(self, project, node):
     name = self._reqatt(node, 'name')
     value = self._reqatt(node, 'value')
@@ -633,7 +900,8 @@ class XmlManifest(object):
     except ManifestParseError:
       keep = "true"
     if keep != "true" and keep != "false":
-      raise ManifestParseError, "optional \"keep\" attribute must be \"true\" or \"false\""
+      raise ManifestParseError('optional "keep" attribute must be '
+            '"true" or "false"')
     project.AddAnnotation(name, value, keep)
 
   def _get_remote(self, node):
@@ -643,9 +911,8 @@ class XmlManifest(object):
 
     v = self._remotes.get(name)
     if not v:
-      raise ManifestParseError, \
-            "remote %s not defined in %s" % \
-            (name, self.manifestFile)
+      raise ManifestParseError("remote %s not defined in %s" %
+            (name, self.manifestFile))
     return v
 
   def _reqatt(self, node, attname):
@@ -654,7 +921,64 @@ class XmlManifest(object):
     """
     v = node.getAttribute(attname)
     if not v:
-      raise ManifestParseError, \
-            "no %s in <%s> within %s" % \
-            (attname, node.nodeName, self.manifestFile)
+      raise ManifestParseError("no %s in <%s> within %s" %
+            (attname, node.nodeName, self.manifestFile))
     return v
+
+  def projectsDiff(self, manifest):
+    """return the projects differences between two manifests.
+
+    The diff will be from self to given manifest.
+
+    """
+    fromProjects = self.paths
+    toProjects = manifest.paths
+
+    fromKeys = sorted(fromProjects.keys())
+    toKeys = sorted(toProjects.keys())
+
+    diff = {'added': [], 'removed': [], 'changed': [], 'unreachable': []}
+
+    for proj in fromKeys:
+      if not proj in toKeys:
+        diff['removed'].append(fromProjects[proj])
+      else:
+        fromProj = fromProjects[proj]
+        toProj = toProjects[proj]
+        try:
+          fromRevId = fromProj.GetCommitRevisionId()
+          toRevId = toProj.GetCommitRevisionId()
+        except ManifestInvalidRevisionError:
+          diff['unreachable'].append((fromProj, toProj))
+        else:
+          if fromRevId != toRevId:
+            diff['changed'].append((fromProj, toProj))
+        toKeys.remove(proj)
+
+    for proj in toKeys:
+      diff['added'].append(toProjects[proj])
+
+    return diff
+
+
+class GitcManifest(XmlManifest):
+
+  def __init__(self, repodir, gitc_client_name):
+    """Initialize the GitcManifest object."""
+    super(GitcManifest, self).__init__(repodir)
+    self.isGitcClient = True
+    self.gitc_client_name = gitc_client_name
+    self.gitc_client_dir = os.path.join(gitc_utils.get_gitc_manifest_dir(),
+                                        gitc_client_name)
+    self.manifestFile = os.path.join(self.gitc_client_dir, '.manifest')
+
+  def _ParseProject(self, node, parent = None):
+    """Override _ParseProject and add support for GITC specific attributes."""
+    return super(GitcManifest, self)._ParseProject(
+        node, parent=parent, old_revision=node.getAttribute('old-revision'))
+
+  def _output_manifest_project_extras(self, p, e):
+    """Output GITC Specific Project attributes"""
+    if p.old_revision:
+      e.setAttribute('old-revision', str(p.old_revision))
+

pager.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import os
 import select
 import sys
@@ -49,7 +50,7 @@ def RunPager(globalConfig):
 
     _BecomePager(pager)
   except Exception:
-    print >>sys.stderr, "fatal: cannot start pager '%s'" % pager
+    print("fatal: cannot start pager '%s'" % pager, file=sys.stderr)
     sys.exit(255)
 
 def _SelectPager(globalConfig):

pyversion.py

@@ -0,0 +1,19 @@
+#
+# Copyright (C) 2013 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+def is_python3():
+  return sys.version_info[0] == 3

repo

@@ -1,9 +1,12 @@
-#!/bin/sh
+#!/usr/bin/env python
 
-## repo default configuration
-##
-REPO_URL='https://gerrit.googlesource.com/git-repo'
-REPO_REV='stable'
+# repo default configuration
+#
+import os
+REPO_URL = os.environ.get('REPO_URL', None)
+if not REPO_URL:
+  REPO_URL = 'https://gerrit.googlesource.com/git-repo'
+REPO_REV = 'stable'
 
 # Copyright (C) 2008 Google Inc.
 #
@@ -19,19 +22,14 @@ REPO_REV='stable'
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-magic='--calling-python-from-/bin/sh--'
-"""exec" python -E "$0" "$@" """#$magic"
-if __name__ == '__main__':
-  import sys
-  if sys.argv[-1] == '#%s' % magic:
-    del sys.argv[-1]
-del magic
-
 # increment this whenever we make important changes to this script
-VERSION = (1, 18)
+VERSION = (1, 23)
 
 # increment this if the MAINTAINER_KEYS block is modified
-KEYRING_VERSION = (1,1)
+KEYRING_VERSION = (1, 2)
+
+# Each individual key entry is created by using:
+# gpg --armor --export keyid
 MAINTAINER_KEYS = """
 
      Repo Maintainer <repo@android.kernel.org>
@@ -80,49 +78,79 @@ TACbBS+Up3RpfYVfd63c1cDdlru13pQAn3NQy/SN858MkxN+zym86UBgOad2
 -----BEGIN PGP PUBLIC KEY BLOCK-----
 Version: GnuPG v1.4.11 (GNU/Linux)
 
-mQENBFBiLPwBCACvISTASOgFXwADw2GYRH2I2z9RvYkYoZ6ThTTNlMXbbYYKO2Wo
-a9LQDNW0TbCEekg5UKk0FD13XOdWaqUt4Gtuvq9c43GRSjMO6NXH+0BjcQ8vUtY2
-/W4CYUevwdo4nQ1+1zsOCu1XYe/CReXq0fdugv3hgmRmh3sz1soo37Q44W2frxxg
-U7Rz3Da4FjgAL0RQ8qndD+LwRHXTY7H7wYM8V/3cYFZV7pSodd75q3MAXYQLf0ZV
-QR1XATu5l1QnXrxgHvz7MmDwb1D+jX3YPKnZveaukigQ6hDHdiVcePBiGXmk8LZC
-2jQkdXeF7Su1ZYpr2nnEHLJ6vOLcCpPGb8gDABEBAAG0H0NvbmxleSBPd2VucyA8
-Y2NvM0BhbmRyb2lkLmNvbT6JATgEEwECACIFAlBiLPwCGwMGCwkIBwMCBhUIAgkK
-CwQWAgMBAh4BAheAAAoJEBkmlFUziHGkHVkH/2Hks2Cif5i2xPtv2IFZcjL42joU
-T7lO5XFqUYS9ZNHpGa/V0eiPt7rHoO16glR83NZtwlrq2cSN89i9HfOhMYV/qLu8
-fLCHcV2muw+yCB5s5bxnI5UkToiNZyBNqFkcOt/Kbj9Hpy68A1kmc6myVEaUYebq
-2Chx/f3xuEthan099t746v1K+/6SvQGDNctHuaMr9cWdxZtHjdRf31SQRc99Phe5
-w+ZGR/ebxNDKRK9mKgZT8wVFHlXerJsRqWIqtx1fsW1UgLgbpcpe2MChm6B5wTu0
-s1ltzox3l4q71FyRRPUJxXyvGkDLZWpK7EpiHSCOYq/KP3HkKeXU3xqHpcG5AQ0E
-UGIs/AEIAKzO/7lO9cB6dshmZYo8Vy/b7aGicThE+ChcDSfhvyOXVdEM2GKAjsR+
-rlBWbTFX3It301p2HwZPFEi9nEvJxVlqqBiW0bPmNMkDRR55l2vbWg35wwkg6RyE
-Bc5/TQjhXI2w8IvlimoGoUff4t3JmMOnWrnKSvL+5iuRj12p9WmanCHzw3Ee7ztf
-/aU/q+FTpr3DLerb6S8xbv86ySgnJT6o5CyL2DCWRtnYQyGVi0ZmLzEouAYiO0hs
-z0AAu28Mj+12g2WwePRz6gfM9rHtI37ylYW3oT/9M9mO9ei/Bc/1D7Dz6qNV+0vg
-uSVJxM2Bl6GalHPZLhHntFEdIA6EdoUAEQEAAYkBHwQYAQIACQUCUGIs/AIbDAAK
-CRAZJpRVM4hxpNfkB/0W/hP5WK/NETXBlWXXW7JPaWO2c5kGwD0lnj5RRmridyo1
-vbm5PdM91jOsDQYqRu6YOoYBnDnEhB2wL2bPh34HWwwrA+LwB8hlcAV2z1bdwyfl
-3R823fReKN3QcvLHzmvZPrF4Rk97M9UIyKS0RtnfTWykRgDWHIsrtQPoNwsXrWoT
-9LrM2v+1+9mp3vuXnE473/NHxmiWEQH9Ez+O/mOxQ7rSOlqGRiKq/IBZCfioJOtV
-fTQeIu/yASZnsLBqr6SJEGwYBoWcyjG++k4fyw8ocOAo4uGDYbxgN7yYfNQ0OH7o
-V6pfUgqKLWa/aK7/N1ZHnPdFLD8Xt0Dmy4BPwrKC
-=O7am
+mQENBFHRvc8BCADFg45Xx/y6QDC+T7Y/gGc7vx0ww7qfOwIKlAZ9xG3qKunMxo+S
+hPCnzEl3cq+6I1Ww/ndop/HB3N3toPXRCoN8Vs4/Hc7by+SnaLFnacrm+tV5/OgT
+V37Lzt8lhay1Kl+YfpFwHYYpIEBLFV9knyfRXS/428W2qhdzYfvB15/AasRmwmor
+py4NIzSs8UD/SPr1ihqNCdZM76+MQyN5HMYXW/ALZXUFG0pwluHFA7hrfPG74i8C
+zMiP7qvMWIl/r/jtzHioH1dRKgbod+LZsrDJ8mBaqsZaDmNJMhss9g76XvfMyLra
+9DI9/iFuBpGzeqBv0hwOGQspLRrEoyTeR6n1ABEBAAG0H0NvbmxleSBPd2VucyA8
+Y2NvM0BhbmRyb2lkLmNvbT6JATgEEwECACIFAlHRvc8CGwMGCwkIBwMCBhUIAgkK
+CwQWAgMBAh4BAheAAAoJEGe35EhpKzgsP6AIAJKJmNtn4l7hkYHKHFSo3egb6RjQ
+zEIP3MFTcu8HFX1kF1ZFbrp7xqurLaE53kEkKuAAvjJDAgI8mcZHP1JyplubqjQA
+xvv84gK+OGP3Xk+QK1ZjUQSbjOpjEiSZpRhWcHci3dgOUH4blJfByHw25hlgHowd
+a/2PrNKZVcJ92YienaxxGjcXEUcd0uYEG2+rwllQigFcnMFDhr9B71MfalRHjFKE
+fmdoypqLrri61YBc59P88Rw2/WUpTQjgNubSqa3A2+CKdaRyaRw+2fdF4TdR0h8W
+zbg+lbaPtJHsV+3mJC7fq26MiJDRJa5ZztpMn8su20gbLgi2ShBOaHAYDDi5AQ0E
+UdG9zwEIAMoOBq+QLNozAhxOOl5GL3StTStGRgPRXINfmViTsihrqGCWBBUfXlUE
+OytC0mYcrDUQev/8ToVoyqw+iGSwDkcSXkrEUCKFtHV/GECWtk1keyHgR10YKI1R
+mquSXoubWGqPeG1PAI74XWaRx8UrL8uCXUtmD8Q5J7mDjKR5NpxaXrwlA0bKsf2E
+Gp9tu1kKauuToZhWHMRMqYSOGikQJwWSFYKT1KdNcOXLQF6+bfoJ6sjVYdwfmNQL
+Ixn8QVhoTDedcqClSWB17VDEFDFa7MmqXZz2qtM3X1R/MUMHqPtegQzBGNhRdnI2
+V45+1Nnx/uuCxDbeI4RbHzujnxDiq70AEQEAAYkBHwQYAQIACQUCUdG9zwIbDAAK
+CRBnt+RIaSs4LNVeB/0Y2pZ8I7gAAcEM0Xw8drr4omg2fUoK1J33ozlA/RxeA/lJ
+I3KnyCDTpXuIeBKPGkdL8uMATC9Z8DnBBajRlftNDVZS3Hz4G09G9QpMojvJkFJV
+By+01Flw/X+eeN8NpqSuLV4W+AjEO8at/VvgKr1AFvBRdZ7GkpI1o6DgPe7ZqX+1
+dzQZt3e13W0rVBb/bUgx9iSLoeWP3aq/k+/GRGOR+S6F6BBSl0SQ2EF2+dIywb1x
+JuinEP+AwLAUZ1Bsx9ISC0Agpk2VeHXPL3FGhroEmoMvBzO0kTFGyoeT7PR/BfKv
++H/g3HsL2LOB9uoIm8/5p2TTU5ttYCXMHhQZ81AY
+=AUp4
 -----END PGP PUBLIC KEY BLOCK-----
 """
 
-GIT = 'git'                     # our git command
-MIN_GIT_VERSION = (1, 5, 4)     # minimum supported git version
-repodir = '.repo'               # name of repo's private directory
-S_repo = 'repo'                 # special repo repository
-S_manifests = 'manifests'       # special manifest repository
-REPO_MAIN = S_repo + '/main.py' # main script
+GIT = 'git'                      # our git command
+MIN_GIT_VERSION = (1, 7, 2)      # minimum supported git version
+repodir = '.repo'                # name of repo's private directory
+S_repo = 'repo'                  # special repo repository
+S_manifests = 'manifests'        # special manifest repository
+REPO_MAIN = S_repo + '/main.py'  # main script
+MIN_PYTHON_VERSION = (2, 6)      # minimum supported python version
+GITC_CONFIG_FILE = '/gitc/.config'
+GITC_FS_ROOT_DIR = '/gitc/manifest-rw/'
 
 
+import errno
 import optparse
-import os
 import re
+import shutil
+import stat
 import subprocess
 import sys
-import urllib2
+
+if sys.version_info[0] == 3:
+  import urllib.request
+  import urllib.error
+else:
+  import imp
+  import urllib2
+  urllib = imp.new_module('urllib')
+  urllib.request = urllib2
+  urllib.error = urllib2
+
+
+def _print(*objects, **kwargs):
+  sep = kwargs.get('sep', ' ')
+  end = kwargs.get('end', '\n')
+  out = kwargs.get('file', sys.stdout)
+  out.write(sep.join(objects) + end)
+
+
+# Python version check
+ver = sys.version_info
+if (ver[0], ver[1]) < MIN_PYTHON_VERSION:
+  _print('error: Python version %s unsupported.\n'
+         'Please use Python 2.6 - 2.7 instead.'
+         % sys.version.split(' ')[0], file=sys.stderr)
+  sys.exit(1)
 
 home_dot_repo = os.path.expanduser('~/.repoconfig')
 gpg_dir = os.path.join(home_dot_repo, 'gnupg')
@@ -149,22 +177,31 @@ group.add_option('-m', '--manifest-name',
                  help='initial manifest file', metavar='NAME.xml')
 group.add_option('--mirror',
                  dest='mirror', action='store_true',
-                 help='mirror the forrest')
+                 help='create a replica of the remote repositories '
+                      'rather than a client working directory')
 group.add_option('--reference',
                  dest='reference',
                  help='location of mirror directory', metavar='DIR')
 group.add_option('--depth', type='int', default=None,
                  dest='depth',
                  help='create a shallow clone with given depth; see git clone')
+group.add_option('--archive',
+                 dest='archive', action='store_true',
+                 help='checkout an archive instead of a git repository for '
+                      'each project. See git archive.')
 group.add_option('-g', '--groups',
                  dest='groups', default='default',
-                 help='restrict manifest projects to ones with a specified group',
+                 help='restrict manifest projects to ones with specified '
+                      'group(s) [default|all|G1,G2,G3|G4,-G5,-G6]',
                  metavar='GROUP')
 group.add_option('-p', '--platform',
                  dest='platform', default="auto",
                  help='restrict manifest projects to ones with a specified '
                       'platform group [auto|all|none|linux|darwin|...]',
                  metavar='PLATFORM')
+group.add_option('--no-clone-bundle',
+                 dest='no_clone_bundle', action='store_true',
+                 help='disable use of /clone.bundle on HTTP/HTTPS')
 
 
 # Tool
@@ -185,14 +222,69 @@ group.add_option('--config-name',
                  dest='config_name', action="store_true", default=False,
                  help='Always prompt for name/e-mail')
 
+
+def _GitcInitOptions(init_optparse_arg):
+  init_optparse_arg.set_usage("repo gitc-init -u url -c client [options]")
+  g = init_optparse_arg.add_option_group('GITC options')
+  g.add_option('-f', '--manifest-file',
+               dest='manifest_file',
+               help='Optional manifest file to use for this GITC client.')
+  g.add_option('-c', '--gitc-client',
+               dest='gitc_client',
+               help='The name of the gitc_client instance to create or modify.')
+
+_gitc_manifest_dir = None
+
+
+def get_gitc_manifest_dir():
+  global _gitc_manifest_dir
+  if _gitc_manifest_dir is None:
+    _gitc_manifest_dir = ''
+    try:
+      with open(GITC_CONFIG_FILE, 'r') as gitc_config:
+        for line in gitc_config:
+          match = re.match('gitc_dir=(?P<gitc_manifest_dir>.*)', line)
+          if match:
+            _gitc_manifest_dir = match.group('gitc_manifest_dir')
+    except IOError:
+      pass
+  return _gitc_manifest_dir
+
+
+def gitc_parse_clientdir(gitc_fs_path):
+  """Parse a path in the GITC FS and return its client name.
+
+  @param gitc_fs_path: A subdirectory path within the GITC_FS_ROOT_DIR.
+
+  @returns: The GITC client name
+  """
+  if gitc_fs_path == GITC_FS_ROOT_DIR:
+    return None
+  if not gitc_fs_path.startswith(GITC_FS_ROOT_DIR):
+    manifest_dir = get_gitc_manifest_dir()
+    if manifest_dir == '':
+      return None
+    if manifest_dir[-1] != '/':
+      manifest_dir += '/'
+    if gitc_fs_path == manifest_dir:
+      return None
+    if not gitc_fs_path.startswith(manifest_dir):
+      return None
+    return gitc_fs_path.split(manifest_dir)[1].split('/')[0]
+  return gitc_fs_path.split(GITC_FS_ROOT_DIR)[1].split('/')[0]
+
+
 class CloneFailure(Exception):
+
   """Indicate the remote clone of repo itself failed.
   """
 
 
-def _Init(args):
+def _Init(args, gitc_init=False):
   """Installs repo by cloning it over the network.
   """
+  if gitc_init:
+    _GitcInitOptions(init_optparse)
   opt, args = init_optparse.parse_args(args)
   if args:
     init_optparse.print_usage()
@@ -211,17 +303,36 @@ def _Init(args):
   if branch.startswith('refs/heads/'):
     branch = branch[len('refs/heads/'):]
   if branch.startswith('refs/'):
-    print >>sys.stderr, "fatal: invalid branch name '%s'" % branch
+    _print("fatal: invalid branch name '%s'" % branch, file=sys.stderr)
     raise CloneFailure()
 
-  if not os.path.isdir(repodir):
-    try:
-      os.mkdir(repodir)
-    except OSError as e:
-      print >>sys.stderr, \
-            'fatal: cannot make %s directory: %s' % (
-            repodir, e.strerror)
-      # Don't faise CloneFailure; that would delete the
+  try:
+    if gitc_init:
+      gitc_manifest_dir = get_gitc_manifest_dir()
+      if not gitc_manifest_dir:
+        _print('fatal: GITC filesystem is not available. Exiting...',
+               file=sys.stderr)
+        sys.exit(1)
+      gitc_client = opt.gitc_client
+      if not gitc_client:
+        gitc_client = gitc_parse_clientdir(os.getcwd())
+      if not gitc_client:
+        _print('fatal: GITC client (-c) is required.', file=sys.stderr)
+        sys.exit(1)
+      client_dir = os.path.join(gitc_manifest_dir, gitc_client)
+      if not os.path.exists(client_dir):
+        os.makedirs(client_dir)
+      os.chdir(client_dir)
+      if os.path.exists(repodir):
+        # This GITC Client has already initialized repo so continue.
+        return
+
+    os.mkdir(repodir)
+  except OSError as e:
+    if e.errno != errno.EEXIST:
+      _print('fatal: cannot make %s directory: %s'
+             % (repodir, e.strerror), file=sys.stderr)
+      # Don't raise CloneFailure; that would delete the
       # name. Instead exit immediately.
       #
       sys.exit(1)
@@ -234,7 +345,7 @@ def _Init(args):
       can_verify = True
 
     dst = os.path.abspath(os.path.join(repodir, S_repo))
-    _Clone(url, dst, opt.quiet)
+    _Clone(url, dst, opt.quiet, not opt.no_clone_bundle)
 
     if can_verify and not opt.no_repo_verify:
       rev = _Verify(dst, branch, opt.quiet)
@@ -244,37 +355,50 @@ def _Init(args):
     _Checkout(dst, branch, rev, opt.quiet)
   except CloneFailure:
     if opt.quiet:
-      print >>sys.stderr, \
-        'fatal: repo init failed; run without --quiet to see why'
+      _print('fatal: repo init failed; run without --quiet to see why',
+             file=sys.stderr)
     raise
 
 
+def ParseGitVersion(ver_str):
+  if not ver_str.startswith('git version '):
+    return None
+
+  num_ver_str = ver_str[len('git version '):].strip().split('-')[0]
+  to_tuple = []
+  for num_str in num_ver_str.split('.')[:3]:
+    if num_str.isdigit():
+      to_tuple.append(int(num_str))
+    else:
+      to_tuple.append(0)
+  return tuple(to_tuple)
+
+
 def _CheckGitVersion():
   cmd = [GIT, '--version']
   try:
     proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
   except OSError as e:
-    print >>sys.stderr
-    print >>sys.stderr, "fatal: '%s' is not available" % GIT
-    print >>sys.stderr, 'fatal: %s' % e
-    print >>sys.stderr
-    print >>sys.stderr, 'Please make sure %s is installed'\
-                        ' and in your path.' % GIT
+    _print(file=sys.stderr)
+    _print("fatal: '%s' is not available" % GIT, file=sys.stderr)
+    _print('fatal: %s' % e, file=sys.stderr)
+    _print(file=sys.stderr)
+    _print('Please make sure %s is installed and in your path.' % GIT,
+           file=sys.stderr)
     raise CloneFailure()
 
   ver_str = proc.stdout.read().strip()
   proc.stdout.close()
   proc.wait()
 
-  if not ver_str.startswith('git version '):
-    print >>sys.stderr, 'error: "%s" unsupported' % ver_str
+  ver_act = ParseGitVersion(ver_str)
+  if ver_act is None:
+    _print('error: "%s" unsupported' % ver_str, file=sys.stderr)
     raise CloneFailure()
 
-  ver_str = ver_str[len('git version '):].strip()
-  ver_act = tuple(map(lambda x: int(x), ver_str.split('.')[0:3]))
   if ver_act < MIN_GIT_VERSION:
-    need = '.'.join(map(lambda x: str(x), MIN_GIT_VERSION))
-    print >>sys.stderr, 'fatal: git %s or later required' % need
+    need = '.'.join(map(str, MIN_GIT_VERSION))
+    _print('fatal: git %s or later required' % need, file=sys.stderr)
     raise CloneFailure()
 
 
@@ -290,56 +414,57 @@ def NeedSetupGnuPG():
   if not kv:
     return True
 
-  kv = tuple(map(lambda x: int(x), kv.split('.')))
+  kv = tuple(map(int, kv.split('.')))
   if kv < KEYRING_VERSION:
     return True
   return False
 
 
 def SetupGnuPG(quiet):
-  if not os.path.isdir(home_dot_repo):
-    try:
-      os.mkdir(home_dot_repo)
-    except OSError as e:
-      print >>sys.stderr, \
-            'fatal: cannot make %s directory: %s' % (
-            home_dot_repo, e.strerror)
+  try:
+    os.mkdir(home_dot_repo)
+  except OSError as e:
+    if e.errno != errno.EEXIST:
+      _print('fatal: cannot make %s directory: %s'
+             % (home_dot_repo, e.strerror), file=sys.stderr)
       sys.exit(1)
 
-  if not os.path.isdir(gpg_dir):
-    try:
-      os.mkdir(gpg_dir, 0700)
-    except OSError as e:
-      print >>sys.stderr, \
-            'fatal: cannot make %s directory: %s' % (
-            gpg_dir, e.strerror)
+  try:
+    os.mkdir(gpg_dir, stat.S_IRWXU)
+  except OSError as e:
+    if e.errno != errno.EEXIST:
+      _print('fatal: cannot make %s directory: %s' % (gpg_dir, e.strerror),
+             file=sys.stderr)
       sys.exit(1)
 
   env = os.environ.copy()
-  env['GNUPGHOME'] = gpg_dir.encode()
+  try:
+    env['GNUPGHOME'] = gpg_dir
+  except UnicodeEncodeError:
+    env['GNUPGHOME'] = gpg_dir.encode()
 
   cmd = ['gpg', '--import']
   try:
     proc = subprocess.Popen(cmd,
-                            env = env,
-                            stdin = subprocess.PIPE)
+                            env=env,
+                            stdin=subprocess.PIPE)
   except OSError as e:
     if not quiet:
-      print >>sys.stderr, 'warning: gpg (GnuPG) is not available.'
-      print >>sys.stderr, 'warning: Installing it is strongly encouraged.'
-      print >>sys.stderr
+      _print('warning: gpg (GnuPG) is not available.', file=sys.stderr)
+      _print('warning: Installing it is strongly encouraged.', file=sys.stderr)
+      _print(file=sys.stderr)
     return False
 
   proc.stdin.write(MAINTAINER_KEYS)
   proc.stdin.close()
 
   if proc.wait() != 0:
-    print >>sys.stderr, 'fatal: registering repo maintainer keys failed'
+    _print('fatal: registering repo maintainer keys failed', file=sys.stderr)
     sys.exit(1)
-  print
+  _print()
 
   fd = open(os.path.join(home_dot_repo, 'keyring-version'), 'w')
-  fd.write('.'.join(map(lambda x: str(x), KEYRING_VERSION)) + '\n')
+  fd.write('.'.join(map(str, KEYRING_VERSION)) + '\n')
   fd.close()
   return True
 
@@ -348,37 +473,38 @@ def _SetConfig(local, name, value):
   """Set a git configuration option to the specified value.
   """
   cmd = [GIT, 'config', name, value]
-  if subprocess.Popen(cmd, cwd = local).wait() != 0:
+  if subprocess.Popen(cmd, cwd=local).wait() != 0:
     raise CloneFailure()
 
 
 def _InitHttp():
   handlers = []
 
-  mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
+  mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
   try:
     import netrc
     n = netrc.netrc()
     for host in n.hosts:
       p = n.hosts[host]
-      mgr.add_password(p[1], 'http://%s/'  % host, p[0], p[2])
+      mgr.add_password(p[1], 'http://%s/' % host, p[0], p[2])
       mgr.add_password(p[1], 'https://%s/' % host, p[0], p[2])
-  except:
+  except:  # pylint: disable=bare-except
     pass
-  handlers.append(urllib2.HTTPBasicAuthHandler(mgr))
-  handlers.append(urllib2.HTTPDigestAuthHandler(mgr))
+  handlers.append(urllib.request.HTTPBasicAuthHandler(mgr))
+  handlers.append(urllib.request.HTTPDigestAuthHandler(mgr))
 
   if 'http_proxy' in os.environ:
     url = os.environ['http_proxy']
-    handlers.append(urllib2.ProxyHandler({'http': url, 'https': url}))
+    handlers.append(urllib.request.ProxyHandler({'http': url, 'https': url}))
   if 'REPO_CURL_VERBOSE' in os.environ:
-    handlers.append(urllib2.HTTPHandler(debuglevel=1))
-    handlers.append(urllib2.HTTPSHandler(debuglevel=1))
-  urllib2.install_opener(urllib2.build_opener(*handlers))
+    handlers.append(urllib.request.HTTPHandler(debuglevel=1))
+    handlers.append(urllib.request.HTTPSHandler(debuglevel=1))
+  urllib.request.install_opener(urllib.request.build_opener(*handlers))
+
 
 def _Fetch(url, local, src, quiet):
   if not quiet:
-    print >>sys.stderr, 'Get %s' % url
+    _print('Get %s' % url, file=sys.stderr)
 
   cmd = [GIT, 'fetch']
   if quiet:
@@ -390,22 +516,23 @@ def _Fetch(url, local, src, quiet):
   cmd.append('+refs/heads/*:refs/remotes/origin/*')
   cmd.append('refs/tags/*:refs/tags/*')
 
-  proc = subprocess.Popen(cmd, cwd = local, stderr = err)
+  proc = subprocess.Popen(cmd, cwd=local, stderr=err)
   if err:
     proc.stderr.read()
     proc.stderr.close()
   if proc.wait() != 0:
     raise CloneFailure()
 
+
 def _DownloadBundle(url, local, quiet):
   if not url.endswith('/'):
     url += '/'
   url += 'clone.bundle'
 
   proc = subprocess.Popen(
-    [GIT, 'config', '--get-regexp', 'url.*.insteadof'],
-    cwd = local,
-    stdout = subprocess.PIPE)
+      [GIT, 'config', '--get-regexp', 'url.*.insteadof'],
+      cwd=local,
+      stdout=subprocess.PIPE)
   for line in proc.stdout:
     m = re.compile(r'^url\.(.*)\.insteadof (.*)$').match(line)
     if m:
@@ -423,20 +550,20 @@ def _DownloadBundle(url, local, quiet):
   dest = open(os.path.join(local, '.git', 'clone.bundle'), 'w+b')
   try:
     try:
-      r = urllib2.urlopen(url)
-    except urllib2.HTTPError as e:
-      if e.code == 404:
+      r = urllib.request.urlopen(url)
+    except urllib.error.HTTPError as e:
+      if e.code in [401, 403, 404, 501]:
         return False
-      print >>sys.stderr, 'fatal: Cannot get %s' % url
-      print >>sys.stderr, 'fatal: HTTP error %s' % e.code
+      _print('fatal: Cannot get %s' % url, file=sys.stderr)
+      _print('fatal: HTTP error %s' % e.code, file=sys.stderr)
       raise CloneFailure()
-    except urllib2.URLError as e:
-      print >>sys.stderr, 'fatal: Cannot get %s' % url
-      print >>sys.stderr, 'fatal: error %s' % e.reason
+    except urllib.error.URLError as e:
+      _print('fatal: Cannot get %s' % url, file=sys.stderr)
+      _print('fatal: error %s' % e.reason, file=sys.stderr)
       raise CloneFailure()
     try:
       if not quiet:
-        print >>sys.stderr, 'Get %s' % url
+        _print('Get %s' % url, file=sys.stderr)
       while True:
         buf = r.read(8192)
         if buf == '':
@@ -447,6 +574,7 @@ def _DownloadBundle(url, local, quiet):
   finally:
     dest.close()
 
+
 def _ImportBundle(local):
   path = os.path.join(local, '.git', 'clone.bundle')
   try:
@@ -454,40 +582,40 @@ def _ImportBundle(local):
   finally:
     os.remove(path)
 
-def _Clone(url, local, quiet):
+
+def _Clone(url, local, quiet, clone_bundle):
   """Clones a git repository to a new subdirectory of repodir
   """
   try:
     os.mkdir(local)
   except OSError as e:
-    print >>sys.stderr, \
-          'fatal: cannot make %s directory: %s' \
-          % (local, e.strerror)
+    _print('fatal: cannot make %s directory: %s' % (local, e.strerror),
+           file=sys.stderr)
     raise CloneFailure()
 
   cmd = [GIT, 'init', '--quiet']
   try:
-    proc = subprocess.Popen(cmd, cwd = local)
+    proc = subprocess.Popen(cmd, cwd=local)
   except OSError as e:
-    print >>sys.stderr
-    print >>sys.stderr, "fatal: '%s' is not available" % GIT
-    print >>sys.stderr, 'fatal: %s' % e
-    print >>sys.stderr
-    print >>sys.stderr, 'Please make sure %s is installed'\
-                        ' and in your path.' % GIT
+    _print(file=sys.stderr)
+    _print("fatal: '%s' is not available" % GIT, file=sys.stderr)
+    _print('fatal: %s' % e, file=sys.stderr)
+    _print(file=sys.stderr)
+    _print('Please make sure %s is installed and in your path.' % GIT,
+           file=sys.stderr)
     raise CloneFailure()
   if proc.wait() != 0:
-    print >>sys.stderr, 'fatal: could not create %s' % local
+    _print('fatal: could not create %s' % local, file=sys.stderr)
     raise CloneFailure()
 
   _InitHttp()
   _SetConfig(local, 'remote.origin.url', url)
-  _SetConfig(local, 'remote.origin.fetch',
-                    '+refs/heads/*:refs/remotes/origin/*')
-  if _DownloadBundle(url, local, quiet):
+  _SetConfig(local,
+             'remote.origin.fetch',
+             '+refs/heads/*:refs/remotes/origin/*')
+  if clone_bundle and _DownloadBundle(url, local, quiet):
     _ImportBundle(local)
-  else:
-    _Fetch(url, local, 'origin', quiet)
+  _Fetch(url, local, 'origin', quiet)
 
 
 def _Verify(cwd, branch, quiet):
@@ -497,7 +625,7 @@ def _Verify(cwd, branch, quiet):
   proc = subprocess.Popen(cmd,
                           stdout=subprocess.PIPE,
                           stderr=subprocess.PIPE,
-                          cwd = cwd)
+                          cwd=cwd)
   cur = proc.stdout.read().strip()
   proc.stdout.close()
 
@@ -505,31 +633,31 @@ def _Verify(cwd, branch, quiet):
   proc.stderr.close()
 
   if proc.wait() != 0 or not cur:
-    print >>sys.stderr
-    print >>sys.stderr,\
-      "fatal: branch '%s' has not been signed" \
-      % branch
+    _print(file=sys.stderr)
+    _print("fatal: branch '%s' has not been signed" % branch, file=sys.stderr)
     raise CloneFailure()
 
   m = re.compile(r'^(.*)-[0-9]{1,}-g[0-9a-f]{1,}$').match(cur)
   if m:
     cur = m.group(1)
     if not quiet:
-      print >>sys.stderr
-      print >>sys.stderr, \
-        "info: Ignoring branch '%s'; using tagged release '%s'" \
-        % (branch, cur)
-      print >>sys.stderr
+      _print(file=sys.stderr)
+      _print("info: Ignoring branch '%s'; using tagged release '%s'"
+             % (branch, cur), file=sys.stderr)
+      _print(file=sys.stderr)
 
   env = os.environ.copy()
-  env['GNUPGHOME'] = gpg_dir.encode()
+  try:
+    env['GNUPGHOME'] = gpg_dir
+  except UnicodeEncodeError:
+    env['GNUPGHOME'] = gpg_dir.encode()
 
   cmd = [GIT, 'tag', '-v', cur]
   proc = subprocess.Popen(cmd,
-                          stdout = subprocess.PIPE,
-                          stderr = subprocess.PIPE,
-                          cwd = cwd,
-                          env = env)
+                          stdout=subprocess.PIPE,
+                          stderr=subprocess.PIPE,
+                          cwd=cwd,
+                          env=env)
   out = proc.stdout.read()
   proc.stdout.close()
 
@@ -537,10 +665,10 @@ def _Verify(cwd, branch, quiet):
   proc.stderr.close()
 
   if proc.wait() != 0:
-    print >>sys.stderr
-    print >>sys.stderr, out
-    print >>sys.stderr, err
-    print >>sys.stderr
+    _print(file=sys.stderr)
+    _print(out, file=sys.stderr)
+    _print(err, file=sys.stderr)
+    _print(file=sys.stderr)
     raise CloneFailure()
   return '%s^0' % cur
 
@@ -549,21 +677,21 @@ def _Checkout(cwd, branch, rev, quiet):
   """Checkout an upstream branch into the repository and track it.
   """
   cmd = [GIT, 'update-ref', 'refs/heads/default', rev]
-  if subprocess.Popen(cmd, cwd = cwd).wait() != 0:
+  if subprocess.Popen(cmd, cwd=cwd).wait() != 0:
     raise CloneFailure()
 
   _SetConfig(cwd, 'branch.default.remote', 'origin')
   _SetConfig(cwd, 'branch.default.merge', 'refs/heads/%s' % branch)
 
   cmd = [GIT, 'symbolic-ref', 'HEAD', 'refs/heads/default']
-  if subprocess.Popen(cmd, cwd = cwd).wait() != 0:
+  if subprocess.Popen(cmd, cwd=cwd).wait() != 0:
     raise CloneFailure()
 
   cmd = [GIT, 'read-tree', '--reset', '-u']
   if not quiet:
     cmd.append('-v')
   cmd.append('HEAD')
-  if subprocess.Popen(cmd, cwd = cwd).wait() != 0:
+  if subprocess.Popen(cmd, cwd=cwd).wait() != 0:
     raise CloneFailure()
 
 
@@ -575,8 +703,8 @@ def _FindRepo():
 
   olddir = None
   while curdir != '/' \
-    and curdir != olddir \
-    and not repo:
+          and curdir != olddir \
+          and not repo:
     repo = os.path.join(curdir, repodir, REPO_MAIN)
     if not os.path.isfile(repo):
       repo = None
@@ -585,7 +713,7 @@ def _FindRepo():
   return (repo, os.path.join(curdir, repodir))
 
 
-class _Options:
+class _Options(object):
   help = False
 
 
@@ -594,7 +722,7 @@ def _ParseArguments(args):
   opt = _Options()
   arg = []
 
-  for i in xrange(0, len(args)):
+  for i in range(len(args)):
     a = args[i]
     if a == '-h' or a == '--help':
       opt.help = True
@@ -607,18 +735,23 @@ def _ParseArguments(args):
 
 
 def _Usage():
-  print >>sys.stderr,\
-"""usage: repo COMMAND [ARGS]
+  gitc_usage = ""
+  if get_gitc_manifest_dir():
+    gitc_usage = "  gitc-init Initialize a GITC Client.\n"
+
+  _print(
+      """usage: repo COMMAND [ARGS]
 
 repo is not yet installed.  Use "repo init" to install it here.
 
 The most commonly used repo commands are:
 
   init      Install repo in the current working directory
-  help      Display detailed help on a command
+""" + gitc_usage +
+      """  help      Display detailed help on a command
 
 For access to the full online help, install repo ("repo init").
-"""
+""", file=sys.stderr)
   sys.exit(1)
 
 
@@ -627,26 +760,28 @@ def _Help(args):
     if args[0] == 'init':
       init_optparse.print_help()
       sys.exit(0)
+    elif args[0] == 'gitc-init':
+      _GitcInitOptions(init_optparse)
+      init_optparse.print_help()
+      sys.exit(0)
     else:
-      print >>sys.stderr,\
-      "error: '%s' is not a bootstrap command.\n"\
-      '        For access to online help, install repo ("repo init").'\
-      % args[0]
+      _print("error: '%s' is not a bootstrap command.\n"
+             '        For access to online help, install repo ("repo init").'
+             % args[0], file=sys.stderr)
   else:
     _Usage()
   sys.exit(1)
 
 
 def _NotInstalled():
-  print >>sys.stderr,\
-'error: repo is not installed.  Use "repo init" to install it here.'
+  _print('error: repo is not installed.  Use "repo init" to install it here.',
+         file=sys.stderr)
   sys.exit(1)
 
 
 def _NoCommands(cmd):
-  print >>sys.stderr,\
-"""error: command '%s' requires repo to be installed first.
-       Use "repo init" to install it here.""" % cmd
+  _print("""error: command '%s' requires repo to be installed first.
+         Use "repo init" to install it here.""" % cmd, file=sys.stderr)
   sys.exit(1)
 
 
@@ -674,8 +809,8 @@ def _SetDefaultsTo(gitdir):
                            '--git-dir=%s' % gitdir,
                            'symbolic-ref',
                            'HEAD'],
-                          stdout = subprocess.PIPE,
-                          stderr = subprocess.PIPE)
+                          stdout=subprocess.PIPE,
+                          stderr=subprocess.PIPE)
   REPO_REV = proc.stdout.read().strip()
   proc.stdout.close()
 
@@ -683,17 +818,28 @@ def _SetDefaultsTo(gitdir):
   proc.stderr.close()
 
   if proc.wait() != 0:
-    print >>sys.stderr, 'fatal: %s has no current branch' % gitdir
+    _print('fatal: %s has no current branch' % gitdir, file=sys.stderr)
     sys.exit(1)
 
 
 def main(orig_args):
-  repo_main, rel_repo_dir = _FindRepo()
   cmd, opt, args = _ParseArguments(orig_args)
 
+  repo_main, rel_repo_dir = None, None
+  # Don't use the local repo copy, make sure to switch to the gitc client first.
+  if cmd != 'gitc-init':
+    repo_main, rel_repo_dir = _FindRepo()
+
   wrapper_path = os.path.abspath(__file__)
   my_main, my_git = _RunSelf(wrapper_path)
 
+  cwd = os.getcwd()
+  if get_gitc_manifest_dir() and cwd.startswith(get_gitc_manifest_dir()):
+    _print('error: repo cannot be used in the GITC local manifest directory.'
+           '\nIf you want to work on this GITC client please rerun this '
+           'command from the corresponding client under /gitc/',
+           file=sys.stderr)
+    sys.exit(1)
   if not repo_main:
     if opt.help:
       _Usage()
@@ -701,18 +847,13 @@ def main(orig_args):
       _Help(args)
     if not cmd:
       _NotInstalled()
-    if cmd == 'init':
+    if cmd == 'init' or cmd == 'gitc-init':
       if my_git:
         _SetDefaultsTo(my_git)
       try:
-        _Init(args)
+        _Init(args, gitc_init=(cmd == 'gitc-init'))
       except CloneFailure:
-        for root, dirs, files in os.walk(repodir, topdown=False):
-          for name in files:
-            os.remove(os.path.join(root, name))
-          for name in dirs:
-            os.rmdir(os.path.join(root, name))
-        os.rmdir(repodir)
+        shutil.rmtree(os.path.join(repodir, S_repo), ignore_errors=True)
         sys.exit(1)
       repo_main, rel_repo_dir = _FindRepo()
     else:
@@ -721,8 +862,8 @@ def main(orig_args):
   if my_main:
     repo_main = my_main
 
-  ver_str = '.'.join(map(lambda x: str(x), VERSION))
-  me = [repo_main,
+  ver_str = '.'.join(map(str, VERSION))
+  me = [sys.executable, repo_main,
         '--repo-dir=%s' % rel_repo_dir,
         '--wrapper-version=%s' % ver_str,
         '--wrapper-path=%s' % wrapper_path,
@@ -730,12 +871,16 @@ def main(orig_args):
   me.extend(orig_args)
   me.extend(extra_args)
   try:
-    os.execv(repo_main, me)
+    os.execv(sys.executable, me)
   except OSError as e:
-    print >>sys.stderr, "fatal: unable to start %s" % repo_main
-    print >>sys.stderr, "fatal: %s" % e
+    _print("fatal: unable to start %s" % repo_main, file=sys.stderr)
+    _print("fatal: %s" % e, file=sys.stderr)
     sys.exit(148)
 
 
 if __name__ == '__main__':
+  if ver[0] == 3:
+    _print('warning: Python 3 support is currently experimental. YMMV.\n'
+           'Please use Python 2.6 - 2.7 instead.',
+           file=sys.stderr)
   main(sys.argv[1:])

subcmds/__init__.py

@@ -38,8 +38,8 @@ for py in os.listdir(my_dir):
     try:
       cmd = getattr(mod, clsn)()
     except AttributeError:
-      raise SyntaxError, '%s/%s does not define class %s' % (
-                         __name__, py, clsn)
+      raise SyntaxError('%s/%s does not define class %s' % (
+                         __name__, py, clsn))
 
     name = name.replace('_', '-')
     cmd.NAME = name

subcmds/abandon.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import sys
 from command import Command
 from git_command import git
@@ -36,7 +37,7 @@ It is equivalent to "git branch -D <branchname>".
 
     nb = args[0]
     if not git.check_ref_format('heads/%s' % nb):
-      print >>sys.stderr, "error: '%s' is not a valid name" % nb
+      print("error: '%s' is not a valid name" % nb, file=sys.stderr)
       sys.exit(1)
 
     nb = args[0]
@@ -58,13 +59,13 @@ It is equivalent to "git branch -D <branchname>".
 
     if err:
       for p in err:
-        print >>sys.stderr,\
-          "error: %s/: cannot abandon %s" \
-          % (p.relpath, nb)
+        print("error: %s/: cannot abandon %s" % (p.relpath, nb),
+              file=sys.stderr)
       sys.exit(1)
     elif not success:
-      print >>sys.stderr, 'error: no project has branch %s' % nb
+      print('error: no project has branch %s' % nb, file=sys.stderr)
       sys.exit(1)
     else:
-      print >>sys.stderr, 'Abandoned in %d project(s):\n  %s' % (
-            len(success), '\n  '.join(p.relpath for p in success))
+      print('Abandoned in %d project(s):\n  %s'
+            % (len(success), '\n  '.join(p.relpath for p in success)),
+            file=sys.stderr)

subcmds/branches.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import sys
 from color import Coloring
 from command import Command
@@ -45,6 +46,10 @@ class BranchInfo(object):
   def IsCurrent(self):
     return self.current > 0
 
+  @property
+  def IsSplitCurrent(self):
+    return self.current != 0 and self.current != len(self.projects)
+
   @property
   def IsPublished(self):
     return self.published > 0
@@ -97,17 +102,16 @@ is shown, then the branch appears in all projects.
     project_cnt = len(projects)
 
     for project in projects:
-      for name, b in project.GetBranches().iteritems():
+      for name, b in project.GetBranches().items():
         b.project = project
         if name not in all_branches:
           all_branches[name] = BranchInfo(name)
         all_branches[name].add(b)
 
-    names = all_branches.keys()
-    names.sort()
+    names = list(sorted(all_branches))
 
     if not names:
-      print >>sys.stderr, '   (no branches)'
+      print('   (no branches)', file=sys.stderr)
       return
 
     width = 25
@@ -139,10 +143,14 @@ is shown, then the branch appears in all projects.
       if in_cnt < project_cnt:
         fmt = out.write
         paths = []
-        if in_cnt < project_cnt - in_cnt: 
+        non_cur_paths = []
+        if i.IsSplitCurrent or (in_cnt < project_cnt - in_cnt):
           in_type = 'in'
           for b in i.projects:
-            paths.append(b.project.relpath)
+            if not i.IsSplitCurrent or b.current:
+              paths.append(b.project.relpath)
+            else:
+              non_cur_paths.append(b.project.relpath)
         else:
           fmt = out.notinproject
           in_type = 'not in'
@@ -154,13 +162,19 @@ is shown, then the branch appears in all projects.
               paths.append(p.relpath)
 
         s = ' %s %s' % (in_type, ', '.join(paths))
-        if width + 7 + len(s) < 80:
+        if not i.IsSplitCurrent and (width + 7 + len(s) < 80):
+          fmt = out.current if i.IsCurrent else fmt
           fmt(s)
         else:
           fmt(' %s:' % in_type)
+          fmt = out.current if i.IsCurrent else out.write
           for p in paths:
             out.nl()
             fmt(width*' ' + '          %s' % p)
+          fmt = out.write
+          for p in non_cur_paths:
+            out.nl()
+            fmt(width*' ' + '          %s' % p)
       else:
         out.write(' in all projects')
       out.nl()

subcmds/checkout.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import sys
 from command import Command
 from progress import Progress
@@ -55,10 +56,9 @@ The command is equivalent to:
 
     if err:
       for p in err:
-        print >>sys.stderr,\
-          "error: %s/: cannot checkout %s" \
-          % (p.relpath, nb)
+        print("error: %s/: cannot checkout %s" % (p.relpath, nb),
+              file=sys.stderr)
       sys.exit(1)
     elif not success:
-      print >>sys.stderr, 'error: no project has branch %s' % nb
+      print('error: no project has branch %s' % nb, file=sys.stderr)
       sys.exit(1)

subcmds/cherry_pick.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import re
 import sys
 from command import Command
@@ -46,13 +47,13 @@ change id will be added.
                    capture_stdout = True,
                    capture_stderr = True)
     if p.Wait() != 0:
-      print >>sys.stderr, p.stderr
+      print(p.stderr, file=sys.stderr)
       sys.exit(1)
     sha1 = p.stdout.strip()
 
     p = GitCommand(None, ['cat-file', 'commit', sha1], capture_stdout=True)
     if p.Wait() != 0:
-      print >>sys.stderr, "error: Failed to retrieve old commit message"
+      print("error: Failed to retrieve old commit message", file=sys.stderr)
       sys.exit(1)
     old_msg = self._StripHeader(p.stdout)
 
@@ -62,8 +63,8 @@ change id will be added.
                    capture_stderr = True)
     status = p.Wait()
 
-    print >>sys.stdout, p.stdout
-    print >>sys.stderr, p.stderr
+    print(p.stdout, file=sys.stdout)
+    print(p.stderr, file=sys.stderr)
 
     if status == 0:
       # The cherry-pick was applied correctly. We just need to edit the
@@ -75,17 +76,16 @@ change id will be added.
                      capture_stdout = True,
                      capture_stderr = True)
       p.stdin.write(new_msg)
+      p.stdin.close()
       if p.Wait() != 0:
-        print >>sys.stderr, "error: Failed to update commit message"
+        print("error: Failed to update commit message", file=sys.stderr)
         sys.exit(1)
 
     else:
-      print >>sys.stderr, """\
-NOTE: When committing (please see above) and editing the commit message,
-please remove the old Change-Id-line and add:
-"""
-      print >>sys.stderr, self._GetReference(sha1)
-      print >>sys.stderr
+      print('NOTE: When committing (please see above) and editing the commit '
+            'message, please remove the old Change-Id-line and add:')
+      print(self._GetReference(sha1), file=sys.stderr)
+      print(file=sys.stderr)
 
   def _IsChangeId(self, line):
     return CHANGE_ID_RE.match(line)

subcmds/diffmanifests.py

@@ -0,0 +1,204 @@
+#
+# Copyright (C) 2014 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from color import Coloring
+from command import PagedCommand
+from manifest_xml import XmlManifest
+
+class _Coloring(Coloring):
+  def __init__(self, config):
+    Coloring.__init__(self, config, "status")
+
+class Diffmanifests(PagedCommand):
+  """ A command to see logs in projects represented by manifests
+
+  This is used to see deeper differences between manifests. Where a simple
+  diff would only show a diff of sha1s for example, this command will display
+  the logs of the project between both sha1s, allowing user to see diff at a
+  deeper level.
+  """
+
+  common = True
+  helpSummary = "Manifest diff utility"
+  helpUsage = """%prog manifest1.xml [manifest2.xml] [options]"""
+
+  helpDescription = """
+The %prog command shows differences between project revisions of manifest1 and
+manifest2. if manifest2 is not specified, current manifest.xml will be used
+instead. Both absolute and relative paths may be used for manifests. Relative
+paths start from project's ".repo/manifests" folder.
+
+The --raw option Displays the diff in a way that facilitates parsing, the
+project pattern will be <status> <path> <revision from> [<revision to>] and the
+commit pattern will be <status> <onelined log> with status values respectively :
+
+  A = Added project
+  R = Removed project
+  C = Changed project
+  U = Project with unreachable revision(s) (revision(s) not found)
+
+for project, and
+
+   A = Added commit
+   R = Removed commit
+
+for a commit.
+
+Only changed projects may contain commits, and commit status always starts with
+a space, and are part of last printed project.
+Unreachable revisions may occur if project is not up to date or if repo has not
+been initialized with all the groups, in which case some projects won't be
+synced and their revisions won't be found.
+
+"""
+
+  def _Options(self, p):
+    p.add_option('--raw',
+                 dest='raw', action='store_true',
+                 help='Display raw diff.')
+    p.add_option('--no-color',
+                 dest='color', action='store_false', default=True,
+                 help='does not display the diff in color.')
+    p.add_option('--pretty-format',
+                 dest='pretty_format', action='store',
+                 metavar='<FORMAT>',
+                 help='print the log using a custom git pretty format string')
+
+  def _printRawDiff(self, diff):
+    for project in diff['added']:
+      self.printText("A %s %s" % (project.relpath, project.revisionExpr))
+      self.out.nl()
+
+    for project in diff['removed']:
+      self.printText("R %s %s" % (project.relpath, project.revisionExpr))
+      self.out.nl()
+
+    for project, otherProject in diff['changed']:
+      self.printText("C %s %s %s" % (project.relpath, project.revisionExpr,
+                                     otherProject.revisionExpr))
+      self.out.nl()
+      self._printLogs(project, otherProject, raw=True, color=False)
+
+    for project, otherProject in diff['unreachable']:
+      self.printText("U %s %s %s" % (project.relpath, project.revisionExpr,
+                                     otherProject.revisionExpr))
+      self.out.nl()
+
+  def _printDiff(self, diff, color=True, pretty_format=None):
+    if diff['added']:
+      self.out.nl()
+      self.printText('added projects : \n')
+      self.out.nl()
+      for project in diff['added']:
+        self.printProject('\t%s' % (project.relpath))
+        self.printText(' at revision ')
+        self.printRevision(project.revisionExpr)
+        self.out.nl()
+
+    if diff['removed']:
+      self.out.nl()
+      self.printText('removed projects : \n')
+      self.out.nl()
+      for project in diff['removed']:
+        self.printProject('\t%s' % (project.relpath))
+        self.printText(' at revision ')
+        self.printRevision(project.revisionExpr)
+        self.out.nl()
+
+    if diff['changed']:
+      self.out.nl()
+      self.printText('changed projects : \n')
+      self.out.nl()
+      for project, otherProject in diff['changed']:
+        self.printProject('\t%s' % (project.relpath))
+        self.printText(' changed from ')
+        self.printRevision(project.revisionExpr)
+        self.printText(' to ')
+        self.printRevision(otherProject.revisionExpr)
+        self.out.nl()
+        self._printLogs(project, otherProject, raw=False, color=color,
+                        pretty_format=pretty_format)
+        self.out.nl()
+
+    if diff['unreachable']:
+      self.out.nl()
+      self.printText('projects with unreachable revisions : \n')
+      self.out.nl()
+      for project, otherProject in diff['unreachable']:
+        self.printProject('\t%s ' % (project.relpath))
+        self.printRevision(project.revisionExpr)
+        self.printText(' or ')
+        self.printRevision(otherProject.revisionExpr)
+        self.printText(' not found')
+        self.out.nl()
+
+  def _printLogs(self, project, otherProject, raw=False, color=True,
+                 pretty_format=None):
+
+    logs = project.getAddedAndRemovedLogs(otherProject,
+                                          oneline=(pretty_format is None),
+                                          color=color,
+                                          pretty_format=pretty_format)
+    if logs['removed']:
+      removedLogs = logs['removed'].split('\n')
+      for log in removedLogs:
+        if log.strip():
+          if raw:
+            self.printText(' R ' + log)
+            self.out.nl()
+          else:
+            self.printRemoved('\t\t[-] ')
+            self.printText(log)
+            self.out.nl()
+
+    if logs['added']:
+      addedLogs = logs['added'].split('\n')
+      for log in addedLogs:
+        if log.strip():
+          if raw:
+            self.printText(' A ' + log)
+            self.out.nl()
+          else:
+            self.printAdded('\t\t[+] ')
+            self.printText(log)
+            self.out.nl()
+
+  def Execute(self, opt, args):
+    if not args or len(args) > 2:
+      self.Usage()
+
+    self.out = _Coloring(self.manifest.globalConfig)
+    self.printText = self.out.nofmt_printer('text')
+    if opt.color:
+      self.printProject = self.out.nofmt_printer('project', attr = 'bold')
+      self.printAdded = self.out.nofmt_printer('green', fg = 'green', attr = 'bold')
+      self.printRemoved = self.out.nofmt_printer('red', fg = 'red', attr = 'bold')
+      self.printRevision = self.out.nofmt_printer('revision', fg = 'yellow')
+    else:
+      self.printProject = self.printAdded = self.printRemoved = self.printRevision = self.printText
+
+    manifest1 = XmlManifest(self.manifest.repodir)
+    manifest1.Override(args[0])
+    if len(args) == 1:
+      manifest2 = self.manifest
+    else:
+      manifest2 = XmlManifest(self.manifest.repodir)
+      manifest2.Override(args[1])
+
+    diff = manifest1.projectsDiff(manifest2)
+    if opt.raw:
+      self._printRawDiff(diff)
+    else:
+      self._printDiff(diff, color=opt.color, pretty_format=opt.pretty_format)

subcmds/download.py

@@ -13,10 +13,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import re
 import sys
 
 from command import Command
+from error import GitError
 
 CHANGE_RE = re.compile(r'^([1-9][0-9]*)(?:[/\.-]([1-9][0-9]*))?$')
 
@@ -32,13 +34,13 @@ makes it available in your project's local working directory.
 """
 
   def _Options(self, p):
-    p.add_option('-c','--cherry-pick',
+    p.add_option('-c', '--cherry-pick',
                  dest='cherrypick', action='store_true',
                  help="cherry-pick instead of checkout")
-    p.add_option('-r','--revert',
+    p.add_option('-r', '--revert',
                  dest='revert', action='store_true',
                  help="revert instead of checkout")
-    p.add_option('-f','--ff-only',
+    p.add_option('-f', '--ff-only',
                  dest='ffonly', action='store_true',
                  help="force fast-forward merge")
 
@@ -68,25 +70,31 @@ makes it available in your project's local working directory.
     for project, change_id, ps_id in self._ParseChangeIds(args):
       dl = project.DownloadPatchSet(change_id, ps_id)
       if not dl:
-        print >>sys.stderr, \
-          '[%s] change %d/%d not found' \
-          % (project.name, change_id, ps_id)
+        print('[%s] change %d/%d not found'
+              % (project.name, change_id, ps_id),
+              file=sys.stderr)
         sys.exit(1)
 
       if not opt.revert and not dl.commits:
-        print >>sys.stderr, \
-          '[%s] change %d/%d has already been merged' \
-          % (project.name, change_id, ps_id)
+        print('[%s] change %d/%d has already been merged'
+              % (project.name, change_id, ps_id),
+              file=sys.stderr)
         continue
 
       if len(dl.commits) > 1:
-        print >>sys.stderr, \
-          '[%s] %d/%d depends on %d unmerged changes:' \
-          % (project.name, change_id, ps_id, len(dl.commits))
+        print('[%s] %d/%d depends on %d unmerged changes:' \
+              % (project.name, change_id, ps_id, len(dl.commits)),
+              file=sys.stderr)
         for c in dl.commits:
-          print >>sys.stderr, '  %s' % (c)
+          print('  %s' % (c), file=sys.stderr)
       if opt.cherrypick:
-        project._CherryPick(dl.commit)
+        try:
+          project._CherryPick(dl.commit)
+        except GitError:
+          print('[%s] Could not complete the cherry-pick of %s' \
+                % (project.name, dl.commit), file=sys.stderr)
+          sys.exit(1)
+
       elif opt.revert:
         project._Revert(dl.commit)
       elif opt.ffonly:

subcmds/forall.py

@@ -13,10 +13,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
+import errno
 import fcntl
+import multiprocessing
 import re
 import os
 import select
+import signal
 import sys
 import subprocess
 
@@ -30,6 +34,7 @@ _CAN_COLOR = [
   'log',
 ]
 
+
 class ForallColoring(Coloring):
   def __init__(self, config):
     Coloring.__init__(self, config, 'forall')
@@ -41,10 +46,14 @@ class Forall(Command, MirrorSafeCommand):
   helpSummary = "Run a shell command in each project"
   helpUsage = """
 %prog [<project>...] -c <command> [<arg>...]
+%prog -r str1 [str2] ... -c <command> [<arg>...]"
 """
   helpDescription = """
 Executes the same shell command in each project.
 
+The -r option allows running the command only on projects matching
+regex or wildcard expression.
+
 Output Formatting
 -----------------
 
@@ -82,6 +91,12 @@ revision to a locally executed git command, use REPO_LREV.
 REPO_RREV is the name of the revision from the manifest, exactly
 as written in the manifest.
 
+REPO_COUNT is the total number of projects being iterated.
+
+REPO_I is the current (1-based) iteration count. Can be used in
+conjunction with REPO_COUNT to add a simple progress indicator to your
+command.
+
 REPO__* are any extra environment variables, specified by the
 "annotation" element under any project element.  This can be useful
 for differentiating trees based on user-specific criteria, or simply
@@ -92,6 +107,9 @@ following <command>.
 
 Unless -p is used, stdin, stdout, stderr are inherited from the
 terminal and are not redirected.
+
+If -e is used, when a command exits unsuccessfully, '%prog' will abort
+without iterating through the remaining projects.
 """
 
   def _Options(self, p):
@@ -99,11 +117,23 @@ terminal and are not redirected.
       setattr(parser.values, option.dest, list(parser.rargs))
       while parser.rargs:
         del parser.rargs[0]
+    p.add_option('-r', '--regex',
+                 dest='regex', action='store_true',
+                 help="Execute the command only on projects matching regex or wildcard expression")
+    p.add_option('-i', '--inverse-regex',
+                 dest='inverse_regex', action='store_true',
+                 help="Execute the command only on projects not matching regex or wildcard expression")
+    p.add_option('-g', '--groups',
+                 dest='groups',
+                 help="Execute the command only on projects matching the specified groups")
     p.add_option('-c', '--command',
                  help='Command (and arguments) to execute',
                  dest='command',
                  action='callback',
                  callback=cmd)
+    p.add_option('-e', '--abort-on-errors',
+                 dest='abort_on_errors', action='store_true',
+                 help='Abort if a command exits unsuccessfully')
 
     g = p.add_option_group('Output')
     g.add_option('-p',
@@ -112,9 +142,35 @@ terminal and are not redirected.
     g.add_option('-v', '--verbose',
                  dest='verbose', action='store_true',
                  help='Show command error messages')
+    g.add_option('-j', '--jobs',
+                 dest='jobs', action='store', type='int', default=1,
+                 help='number of commands to execute simultaneously')
 
   def WantPager(self, opt):
-    return opt.project_header
+    return opt.project_header and opt.jobs == 1
+
+  def _SerializeProject(self, project):
+    """ Serialize a project._GitGetByExec instance.
+
+    project._GitGetByExec is not pickle-able. Instead of trying to pass it
+    around between processes, make a dict ourselves containing only the
+    attributes that we need.
+
+    """
+    if not self.manifest.IsMirror:
+      lrev = project.GetRevisionId()
+    else:
+      lrev = None
+    return {
+      'name': project.name,
+      'relpath': project.relpath,
+      'remote_name': project.remote.name,
+      'lrev': lrev,
+      'rrev': project.revisionExpr,
+      'annotations': dict((a.name, a.value) for a in project.annotations),
+      'gitdir': project.gitdir,
+      'worktree': project.worktree,
+    }
 
   def Execute(self, opt, args):
     if not opt.command:
@@ -153,108 +209,192 @@ terminal and are not redirected.
       # pylint: enable=W0631
 
     mirror = self.manifest.IsMirror
-    out = ForallColoring(self.manifest.manifestProject.config)
-    out.redirect(sys.stdout)
-
     rc = 0
-    first = True
 
-    for project in self.GetProjects(args):
-      env = os.environ.copy()
-      def setenv(name, val):
-        if val is None:
-          val = ''
-        env[name] = val.encode()
+    smart_sync_manifest_name = "smart_sync_override.xml"
+    smart_sync_manifest_path = os.path.join(
+      self.manifest.manifestProject.worktree, smart_sync_manifest_name)
 
-      setenv('REPO_PROJECT', project.name)
-      setenv('REPO_PATH', project.relpath)
-      setenv('REPO_REMOTE', project.remote.name)
-      setenv('REPO_LREV', project.GetRevisionId())
-      setenv('REPO_RREV', project.revisionExpr)
-      for a in project.annotations:
-        setenv("REPO__%s" % (a.name), a.value)
+    if os.path.isfile(smart_sync_manifest_path):
+      self.manifest.Override(smart_sync_manifest_path)
 
-      if mirror:
-        setenv('GIT_DIR', project.gitdir)
-        cwd = project.gitdir
-      else:
-        cwd = project.worktree
+    if opt.regex:
+      projects = self.FindProjects(args)
+    elif opt.inverse_regex:
+      projects = self.FindProjects(args, inverse=True)
+    else:
+      projects = self.GetProjects(args, groups=opt.groups)
 
-      if not os.path.exists(cwd):
-        if (opt.project_header and opt.verbose) \
-        or not opt.project_header:
-          print >>sys.stderr, 'skipping %s/' % project.relpath
-        continue
+    os.environ['REPO_COUNT'] = str(len(projects))
 
-      if opt.project_header:
-        stdin = subprocess.PIPE
-        stdout = subprocess.PIPE
-        stderr = subprocess.PIPE
-      else:
-        stdin = None
-        stdout = None
-        stderr = None
-
-      p = subprocess.Popen(cmd,
-                           cwd = cwd,
-                           shell = shell,
-                           env = env,
-                           stdin = stdin,
-                           stdout = stdout,
-                           stderr = stderr)
-
-      if opt.project_header:
-        class sfd(object):
-          def __init__(self, fd, dest):
-            self.fd = fd
-            self.dest = dest
-          def fileno(self):
-            return self.fd.fileno()
-
-        empty = True
-        errbuf = ''
-
-        p.stdin.close()
-        s_in = [sfd(p.stdout, sys.stdout),
-                sfd(p.stderr, sys.stderr)]
-
-        for s in s_in:
-          flags = fcntl.fcntl(s.fd, fcntl.F_GETFL)
-          fcntl.fcntl(s.fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
-
-        while s_in:
-          in_ready, _out_ready, _err_ready = select.select(s_in, [], [])
-          for s in in_ready:
-            buf = s.fd.read(4096)
-            if not buf:
-              s.fd.close()
-              s_in.remove(s)
-              continue
-
-            if not opt.verbose:
-              if s.fd != p.stdout:
-                errbuf += buf
-                continue
-
-            if empty:
-              if first:
-                first = False
-              else:
-                out.nl()
-              out.project('project %s/', project.relpath)
-              out.nl()
-              out.flush()
-              if errbuf:
-                sys.stderr.write(errbuf)
-                sys.stderr.flush()
-                errbuf = ''
-              empty = False
-
-            s.dest.write(buf)
-            s.dest.flush()
-
-      r = p.wait()
-      if r != 0 and r != rc:
-        rc = r
+    pool = multiprocessing.Pool(opt.jobs, InitWorker)
+    try:
+      config = self.manifest.manifestProject.config
+      results_it = pool.imap(
+         DoWorkWrapper,
+         self.ProjectArgs(projects, mirror, opt, cmd, shell, config))
+      pool.close()
+      for r in results_it:
+        rc = rc or r
+        if r != 0 and opt.abort_on_errors:
+          raise Exception('Aborting due to previous error')
+    except (KeyboardInterrupt, WorkerKeyboardInterrupt):
+      # Catch KeyboardInterrupt raised inside and outside of workers
+      print('Interrupted - terminating the pool')
+      pool.terminate()
+      rc = rc or errno.EINTR
+    except Exception as e:
+      # Catch any other exceptions raised
+      print('Got an error, terminating the pool: %s: %s' %
+              (type(e).__name__, e),
+            file=sys.stderr)
+      pool.terminate()
+      rc = rc or getattr(e, 'errno', 1)
+    finally:
+      pool.join()
     if rc != 0:
       sys.exit(rc)
+
+  def ProjectArgs(self, projects, mirror, opt, cmd, shell, config):
+    for cnt, p in enumerate(projects):
+      try:
+        project = self._SerializeProject(p)
+      except Exception as e:
+        print('Project list error on project %s: %s: %s' %
+                (p.name, type(e).__name__, e),
+              file=sys.stderr)
+        return
+      except KeyboardInterrupt:
+        print('Project list interrupted',
+              file=sys.stderr)
+        return
+      yield [mirror, opt, cmd, shell, cnt, config, project]
+
+class WorkerKeyboardInterrupt(Exception):
+  """ Keyboard interrupt exception for worker processes. """
+  pass
+
+
+def InitWorker():
+  signal.signal(signal.SIGINT, signal.SIG_IGN)
+
+def DoWorkWrapper(args):
+  """ A wrapper around the DoWork() method.
+
+  Catch the KeyboardInterrupt exceptions here and re-raise them as a different,
+  ``Exception``-based exception to stop it flooding the console with stacktraces
+  and making the parent hang indefinitely.
+
+  """
+  project = args.pop()
+  try:
+    return DoWork(project, *args)
+  except KeyboardInterrupt:
+    print('%s: Worker interrupted' % project['name'])
+    raise WorkerKeyboardInterrupt()
+
+
+def DoWork(project, mirror, opt, cmd, shell, cnt, config):
+  env = os.environ.copy()
+  def setenv(name, val):
+    if val is None:
+      val = ''
+    if hasattr(val, 'encode'):
+      val = val.encode()
+    env[name] = val
+
+  setenv('REPO_PROJECT', project['name'])
+  setenv('REPO_PATH', project['relpath'])
+  setenv('REPO_REMOTE', project['remote_name'])
+  setenv('REPO_LREV', project['lrev'])
+  setenv('REPO_RREV', project['rrev'])
+  setenv('REPO_I', str(cnt + 1))
+  for name in project['annotations']:
+    setenv("REPO__%s" % (name), project['annotations'][name])
+
+  if mirror:
+    setenv('GIT_DIR', project['gitdir'])
+    cwd = project['gitdir']
+  else:
+    cwd = project['worktree']
+
+  if not os.path.exists(cwd):
+    if (opt.project_header and opt.verbose) \
+    or not opt.project_header:
+      print('skipping %s/' % project['relpath'], file=sys.stderr)
+    return
+
+  if opt.project_header:
+    stdin = subprocess.PIPE
+    stdout = subprocess.PIPE
+    stderr = subprocess.PIPE
+  else:
+    stdin = None
+    stdout = None
+    stderr = None
+
+  p = subprocess.Popen(cmd,
+                       cwd=cwd,
+                       shell=shell,
+                       env=env,
+                       stdin=stdin,
+                       stdout=stdout,
+                       stderr=stderr)
+
+  if opt.project_header:
+    out = ForallColoring(config)
+    out.redirect(sys.stdout)
+    class sfd(object):
+      def __init__(self, fd, dest):
+        self.fd = fd
+        self.dest = dest
+      def fileno(self):
+        return self.fd.fileno()
+
+    empty = True
+    errbuf = ''
+
+    p.stdin.close()
+    s_in = [sfd(p.stdout, sys.stdout),
+            sfd(p.stderr, sys.stderr)]
+
+    for s in s_in:
+      flags = fcntl.fcntl(s.fd, fcntl.F_GETFL)
+      fcntl.fcntl(s.fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
+
+    while s_in:
+      in_ready, _out_ready, _err_ready = select.select(s_in, [], [])
+      for s in in_ready:
+        buf = s.fd.read(4096)
+        if not buf:
+          s.fd.close()
+          s_in.remove(s)
+          continue
+
+        if not opt.verbose:
+          if s.fd != p.stdout:
+            errbuf += buf
+            continue
+
+        if empty and out:
+          if not cnt == 0:
+            out.nl()
+
+          if mirror:
+            project_header_path = project['name']
+          else:
+            project_header_path = project['relpath']
+          out.project('project %s/', project_header_path)
+          out.nl()
+          out.flush()
+          if errbuf:
+            sys.stderr.write(errbuf)
+            sys.stderr.flush()
+            errbuf = ''
+          empty = False
+
+        s.dest.write(buf)
+        s.dest.flush()
+
+  r = p.wait()
+  return r

subcmds/gitc_delete.py

@@ -0,0 +1,55 @@
+#
+# Copyright (C) 2015 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+import os
+import shutil
+import sys
+
+from command import Command, GitcClientCommand
+import gitc_utils
+
+from pyversion import is_python3
+if not is_python3():
+  # pylint:disable=W0622
+  input = raw_input
+  # pylint:enable=W0622
+
+class GitcDelete(Command, GitcClientCommand):
+  common = True
+  visible_everywhere = False
+  helpSummary = "Delete a GITC Client."
+  helpUsage = """
+%prog
+"""
+  helpDescription = """
+This subcommand deletes the current GITC client, deleting the GITC manifest
+and all locally downloaded sources.
+"""
+
+  def _Options(self, p):
+    p.add_option('-f', '--force',
+                 dest='force', action='store_true',
+                 help='Force the deletion (no prompt).')
+
+  def Execute(self, opt, args):
+    if not opt.force:
+      prompt = ('This will delete GITC client: %s\nAre you sure? (yes/no) ' %
+                self.gitc_manifest.gitc_client_name)
+      response = input(prompt).lower()
+      if not response == 'yes':
+        print('Response was not "yes"\n Exiting...')
+        sys.exit(1)
+    shutil.rmtree(self.gitc_manifest.gitc_client_dir)

subcmds/gitc_init.py

@@ -0,0 +1,82 @@
+#
+# Copyright (C) 2015 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+import os
+import sys
+
+import gitc_utils
+from command import GitcAvailableCommand
+from manifest_xml import GitcManifest
+from subcmds import init
+import wrapper
+
+
+class GitcInit(init.Init, GitcAvailableCommand):
+  common = True
+  helpSummary = "Initialize a GITC Client."
+  helpUsage = """
+%prog [options] [client name]
+"""
+  helpDescription = """
+The '%prog' command is ran to initialize a new GITC client for use
+with the GITC file system.
+
+This command will setup the client directory, initialize repo, just
+like repo init does, and then downloads the manifest collection
+and installs it in the .repo/directory of the GITC client.
+
+Once this is done, a GITC manifest is generated by pulling the HEAD
+SHA for each project and generates the properly formatted XML file
+and installs it as .manifest in the GITC client directory.
+
+The -c argument is required to specify the GITC client name.
+
+The optional -f argument can be used to specify the manifest file to
+use for this GITC client.
+"""
+
+  def _Options(self, p):
+    super(GitcInit, self)._Options(p)
+    g = p.add_option_group('GITC options')
+    g.add_option('-f', '--manifest-file',
+                 dest='manifest_file',
+                 help='Optional manifest file to use for this GITC client.')
+    g.add_option('-c', '--gitc-client',
+                 dest='gitc_client',
+                 help='The name of the gitc_client instance to create or modify.')
+
+  def Execute(self, opt, args):
+    gitc_client = gitc_utils.parse_clientdir(os.getcwd())
+    if not gitc_client or (opt.gitc_client and gitc_client != opt.gitc_client):
+      print('fatal: Please update your repo command. See go/gitc for instructions.', file=sys.stderr)
+      sys.exit(1)
+    self.client_dir = os.path.join(gitc_utils.get_gitc_manifest_dir(),
+                                   gitc_client)
+    super(GitcInit, self).Execute(opt, args)
+
+    manifest_file = self.manifest.manifestFile
+    if opt.manifest_file:
+      if not os.path.exists(opt.manifest_file):
+        print('fatal: Specified manifest file %s does not exist.' %
+              opt.manifest_file)
+        sys.exit(1)
+      manifest_file = opt.manifest_file
+
+    manifest = GitcManifest(self.repodir, gitc_client)
+    manifest.Override(manifest_file)
+    gitc_utils.generate_gitc_manifest(None, manifest)
+    print('Please run `cd %s` to view your GITC client.' %
+          os.path.join(wrapper.Wrapper().GITC_FS_ROOT_DIR, gitc_client))

subcmds/grep.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import sys
 from color import Coloring
 from command import PagedCommand
@@ -51,7 +52,7 @@ Examples
 
 Look for a line that has '#define' and either 'MAX_PATH or 'PATH_MAX':
 
-  repo grep -e '#define' --and -\( -e MAX_PATH -e PATH_MAX \)
+  repo grep -e '#define' --and -\\( -e MAX_PATH -e PATH_MAX \\)
 
 Look for a line that has 'NODE' or 'Unexpected' in files that
 contain a line that matches both expressions:
@@ -84,7 +85,7 @@ contain a line that matches both expressions:
     g.add_option('--cached',
                  action='callback', callback=carry,
                  help='Search the index, instead of the work tree')
-    g.add_option('-r','--revision',
+    g.add_option('-r', '--revision',
                  dest='revision', action='append', metavar='TREEish',
                  help='Search TREEish, instead of the work tree')
 
@@ -96,7 +97,7 @@ contain a line that matches both expressions:
     g.add_option('-i', '--ignore-case',
                  action='callback', callback=carry,
                  help='Ignore case differences')
-    g.add_option('-a','--text',
+    g.add_option('-a', '--text',
                  action='callback', callback=carry,
                  help="Process binary files as if they were text")
     g.add_option('-I',
@@ -125,7 +126,7 @@ contain a line that matches both expressions:
     g.add_option('--and', '--or', '--not',
                  action='callback', callback=carry,
                  help='Boolean operators to combine patterns')
-    g.add_option('-(','-)',
+    g.add_option('-(', '-)',
                  action='callback', callback=carry,
                  help='Boolean operator grouping')
 
@@ -145,10 +146,10 @@ contain a line that matches both expressions:
                  action='callback', callback=carry,
                  metavar='CONTEXT', type='str',
                  help='Show CONTEXT lines after match')
-    g.add_option('-l','--name-only','--files-with-matches',
+    g.add_option('-l', '--name-only', '--files-with-matches',
                  action='callback', callback=carry,
                  help='Show only file names containing matching lines')
-    g.add_option('-L','--files-without-match',
+    g.add_option('-L', '--files-without-match',
                  action='callback', callback=carry,
                  help='Show only file names not containing matching lines')
 
@@ -157,9 +158,9 @@ contain a line that matches both expressions:
     out = GrepColoring(self.manifest.manifestProject.config)
 
     cmd_argv = ['grep']
-    if out.is_on and git_require((1,6,3)):
+    if out.is_on and git_require((1, 6, 3)):
       cmd_argv.append('--color')
-    cmd_argv.extend(getattr(opt,'cmd_argv',[]))
+    cmd_argv.extend(getattr(opt, 'cmd_argv', []))
 
     if '-e' not in cmd_argv:
       if not args:
@@ -178,8 +179,7 @@ contain a line that matches both expressions:
     have_rev = False
     if opt.revision:
       if '--cached' in cmd_argv:
-        print >>sys.stderr,\
-          'fatal: cannot combine --cached and --revision'
+        print('fatal: cannot combine --cached and --revision', file=sys.stderr)
         sys.exit(1)
       have_rev = True
       cmd_argv.extend(opt.revision)
@@ -230,13 +230,13 @@ contain a line that matches both expressions:
           out.nl()
       else:
         for line in r:
-          print line
+          print(line)
 
     if have_match:
       sys.exit(0)
     elif have_rev and bad_rev:
       for r in opt.revision:
-        print >>sys.stderr, "error: can't search revision %s" % r
+        print("error: can't search revision %s" % r, file=sys.stderr)
       sys.exit(1)
     else:
       sys.exit(1)

subcmds/help.py

@@ -13,12 +13,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import re
 import sys
 from formatter import AbstractFormatter, DumbWriter
 
 from color import Coloring
-from command import PagedCommand, MirrorSafeCommand
+from command import PagedCommand, MirrorSafeCommand, GitcAvailableCommand, GitcClientCommand
+import gitc_utils
 
 class Help(PagedCommand, MirrorSafeCommand):
   common = False
@@ -31,12 +33,9 @@ Displays detailed usage information about a command.
 """
 
   def _PrintAllCommands(self):
-    print 'usage: repo COMMAND [ARGS]'
-    print """
-The complete list of recognized repo commands are:
-"""
-    commandNames = self.commands.keys()
-    commandNames.sort()
+    print('usage: repo COMMAND [ARGS]')
+    print('The complete list of recognized repo commands are:')
+    commandNames = list(sorted(self.commands))
 
     maxlen = 0
     for name in commandNames:
@@ -49,20 +48,28 @@ The complete list of recognized repo commands are:
         summary = command.helpSummary.strip()
       except AttributeError:
         summary = ''
-      print fmt % (name, summary)
-    print """
-See 'repo help <command>' for more information on a specific command.
-"""
+      print(fmt % (name, summary))
+    print("See 'repo help <command>' for more information on a "
+          'specific command.')
 
   def _PrintCommonCommands(self):
-    print 'usage: repo COMMAND [ARGS]'
-    print """
-The most commonly used repo commands are:
-"""
-    commandNames = [name 
-                    for name in self.commands.keys()
-                    if self.commands[name].common]
-    commandNames.sort()
+    print('usage: repo COMMAND [ARGS]')
+    print('The most commonly used repo commands are:')
+
+    def gitc_supported(cmd):
+      if not isinstance(cmd, GitcAvailableCommand) and not isinstance(cmd, GitcClientCommand):
+        return True
+      if self.manifest.isGitcClient:
+        return True
+      if isinstance(cmd, GitcClientCommand):
+        return False
+      if gitc_utils.get_gitc_manifest_dir():
+        return True
+      return False
+
+    commandNames = list(sorted([name
+                    for name, command in self.commands.items()
+                    if command.common and gitc_supported(command)]))
 
     maxlen = 0
     for name in commandNames:
@@ -75,11 +82,10 @@ The most commonly used repo commands are:
         summary = command.helpSummary.strip()
       except AttributeError:
         summary = ''
-      print fmt % (name, summary)
-    print """
-See 'repo help <command>' for more information on a specific command.
-See 'repo help --all' for a complete list of recognized commands.
-"""
+      print(fmt % (name, summary))
+    print(
+"See 'repo help <command>' for more information on a specific command.\n"
+"See 'repo help --all' for a complete list of recognized commands.")
 
   def _PrintCommandHelp(self, cmd):
     class _Out(Coloring):
@@ -131,7 +137,7 @@ See 'repo help --all' for a complete list of recognized commands.
 
             p('%s', title)
             self.nl()
-            p('%s', ''.ljust(len(title),section_type[0]))
+            p('%s', ''.ljust(len(title), section_type[0]))
             self.nl()
             continue
 
@@ -162,7 +168,7 @@ See 'repo help --all' for a complete list of recognized commands.
       try:
         cmd = self.commands[name]
       except KeyError:
-        print >>sys.stderr, "repo: '%s' is not a repo command." % name
+        print("repo: '%s' is not a repo command." % name, file=sys.stderr)
         sys.exit(1)
 
       cmd.manifest = self.manifest

subcmds/info.py

@@ -0,0 +1,202 @@
+#
+# Copyright (C) 2012 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from command import PagedCommand
+from color import Coloring
+from error import NoSuchProjectError
+from git_refs import R_M
+
+class _Coloring(Coloring):
+  def __init__(self, config):
+    Coloring.__init__(self, config, "status")
+
+class Info(PagedCommand):
+  common = True
+  helpSummary = "Get info on the manifest branch, current branch or unmerged branches"
+  helpUsage = "%prog [-dl] [-o [-b]] [<project>...]"
+
+  def _Options(self, p):
+    p.add_option('-d', '--diff',
+                 dest='all', action='store_true',
+                 help="show full info and commit diff including remote branches")
+    p.add_option('-o', '--overview',
+                 dest='overview', action='store_true',
+                 help='show overview of all local commits')
+    p.add_option('-b', '--current-branch',
+                 dest="current_branch", action="store_true",
+                 help="consider only checked out branches")
+    p.add_option('-l', '--local-only',
+                 dest="local", action="store_true",
+                 help="Disable all remote operations")
+
+
+  def Execute(self, opt, args):
+    self.out = _Coloring(self.manifest.globalConfig)
+    self.heading = self.out.printer('heading', attr = 'bold')
+    self.headtext = self.out.printer('headtext', fg = 'yellow')
+    self.redtext = self.out.printer('redtext', fg = 'red')
+    self.sha = self.out.printer("sha", fg = 'yellow')
+    self.text = self.out.nofmt_printer('text')
+    self.dimtext = self.out.printer('dimtext', attr = 'dim')
+
+    self.opt = opt
+
+    manifestConfig = self.manifest.manifestProject.config
+    mergeBranch = manifestConfig.GetBranch("default").merge
+    manifestGroups = (manifestConfig.GetString('manifest.groups')
+                      or 'all,-notdefault')
+
+    self.heading("Manifest branch: ")
+    if self.manifest.default.revisionExpr:
+        self.headtext(self.manifest.default.revisionExpr)
+    self.out.nl()
+    self.heading("Manifest merge branch: ")
+    self.headtext(mergeBranch)
+    self.out.nl()
+    self.heading("Manifest groups: ")
+    self.headtext(manifestGroups)
+    self.out.nl()
+
+    self.printSeparator()
+
+    if not opt.overview:
+      self.printDiffInfo(args)
+    else:
+      self.printCommitOverview(args)
+
+  def printSeparator(self):
+    self.text("----------------------------")
+    self.out.nl()
+
+  def printDiffInfo(self, args):
+    try:
+      projs = self.GetProjects(args)
+    except NoSuchProjectError:
+      return
+
+    for p in projs:
+      self.heading("Project: ")
+      self.headtext(p.name)
+      self.out.nl()
+
+      self.heading("Mount path: ")
+      self.headtext(p.worktree)
+      self.out.nl()
+
+      self.heading("Current revision: ")
+      self.headtext(p.revisionExpr)
+      self.out.nl()
+
+      localBranches = p.GetBranches().keys()
+      self.heading("Local Branches: ")
+      self.redtext(str(len(localBranches)))
+      if len(localBranches) > 0:
+        self.text(" [")
+        self.text(", ".join(localBranches))
+        self.text("]")
+      self.out.nl()
+
+      if self.opt.all:
+        self.findRemoteLocalDiff(p)
+
+      self.printSeparator()
+
+  def findRemoteLocalDiff(self, project):
+    #Fetch all the latest commits
+    if not self.opt.local:
+      project.Sync_NetworkHalf(quiet=True, current_branch_only=True)
+
+    logTarget = R_M + self.manifest.manifestProject.config.GetBranch("default").merge
+
+    bareTmp = project.bare_git._bare
+    project.bare_git._bare = False
+    localCommits = project.bare_git.rev_list(
+        '--abbrev=8',
+        '--abbrev-commit',
+        '--pretty=oneline',
+        logTarget + "..",
+        '--')
+
+    originCommits = project.bare_git.rev_list(
+        '--abbrev=8',
+        '--abbrev-commit',
+        '--pretty=oneline',
+        ".." + logTarget,
+        '--')
+    project.bare_git._bare = bareTmp
+
+    self.heading("Local Commits: ")
+    self.redtext(str(len(localCommits)))
+    self.dimtext(" (on current branch)")
+    self.out.nl()
+
+    for c in localCommits:
+      split = c.split()
+      self.sha(split[0] + " ")
+      self.text(" ".join(split[1:]))
+      self.out.nl()
+
+    self.printSeparator()
+
+    self.heading("Remote Commits: ")
+    self.redtext(str(len(originCommits)))
+    self.out.nl()
+
+    for c in originCommits:
+      split = c.split()
+      self.sha(split[0] + " ")
+      self.text(" ".join(split[1:]))
+      self.out.nl()
+
+  def printCommitOverview(self, args):
+    all_branches = []
+    for project in self.GetProjects(args):
+      br = [project.GetUploadableBranch(x)
+            for x in project.GetBranches()]
+      br = [x for x in br if x]
+      if self.opt.current_branch:
+        br = [x for x in br if x.name == project.CurrentBranch]
+      all_branches.extend(br)
+
+    if not all_branches:
+      return
+
+    self.out.nl()
+    self.heading('Projects Overview')
+    project = None
+
+    for branch in all_branches:
+      if project != branch.project:
+        project = branch.project
+        self.out.nl()
+        self.headtext(project.relpath)
+        self.out.nl()
+
+      commits = branch.commits
+      date = branch.date
+      self.text('%s %-33s (%2d commit%s, %s)' % (
+        branch.name == project.CurrentBranch and '*' or ' ',
+        branch.name,
+        len(commits),
+        len(commits) != 1 and 's' or '',
+        date))
+      self.out.nl()
+
+      for commit in commits:
+        split = commit.split()
+        self.text('{0:38}{1} '.format('','-'))
+        self.sha(split[0] + " ")
+        self.text(" ".join(split[1:]))
+        self.out.nl()

subcmds/init.py

@@ -13,12 +13,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import os
 import platform
 import re
 import shutil
 import sys
 
+from pyversion import is_python3
+if is_python3():
+  import urllib.parse
+else:
+  import imp
+  import urlparse
+  urllib = imp.new_module('urllib')
+  urllib.parse = urlparse
+
 from color import Coloring
 from command import InteractiveCommand, MirrorSafeCommand
 from error import ManifestParseError
@@ -51,6 +61,11 @@ directory use as much data as possible from the local reference
 directory when fetching from the server. This will make the sync
 go a lot faster by reducing data traffic on the network.
 
+The --no-clone-bundle option disables any attempt to use
+$URL/clone.bundle to bootstrap a new Git repository from a
+resumeable bundle file on a content delivery network. This
+may be necessary if there are problems with the local Python
+HTTP client or proxy configuration, but the Git binary works.
 
 Switching Manifest Branches
 ---------------------------
@@ -89,15 +104,23 @@ to update the working directory files.
     g.add_option('--depth', type='int', default=None,
                  dest='depth',
                  help='create a shallow clone with given depth; see git clone')
+    g.add_option('--archive',
+                 dest='archive', action='store_true',
+                 help='checkout an archive instead of a git repository for '
+                      'each project. See git archive.')
     g.add_option('-g', '--groups',
-                 dest='groups', default='all,-notdefault',
-                 help='restrict manifest projects to ones with a specified group',
+                 dest='groups', default='default',
+                 help='restrict manifest projects to ones with specified '
+                      'group(s) [default|all|G1,G2,G3|G4,-G5,-G6]',
                  metavar='GROUP')
     g.add_option('-p', '--platform',
                  dest='platform', default='auto',
                  help='restrict manifest projects to ones with a specified '
                       'platform group [auto|all|none|linux|darwin|...]',
                  metavar='PLATFORM')
+    g.add_option('--no-clone-bundle',
+                 dest='no_clone_bundle', action='store_true',
+                 help='disable use of /clone.bundle on HTTP/HTTPS')
 
     # Tool
     g = p.add_option_group('repo Version options')
@@ -117,19 +140,35 @@ to update the working directory files.
                  dest='config_name', action="store_true", default=False,
                  help='Always prompt for name/e-mail')
 
+  def _RegisteredEnvironmentOptions(self):
+    return {'REPO_MANIFEST_URL': 'manifest_url',
+            'REPO_MIRROR_LOCATION': 'reference'}
+
   def _SyncManifest(self, opt):
     m = self.manifest.manifestProject
     is_new = not m.Exists
 
     if is_new:
       if not opt.manifest_url:
-        print >>sys.stderr, 'fatal: manifest url (-u) is required.'
+        print('fatal: manifest url (-u) is required.', file=sys.stderr)
         sys.exit(1)
 
       if not opt.quiet:
-        print >>sys.stderr, 'Get %s' \
-          % GitConfig.ForUser().UrlInsteadOf(opt.manifest_url)
-      m._InitGitDir()
+        print('Get %s' % GitConfig.ForUser().UrlInsteadOf(opt.manifest_url),
+              file=sys.stderr)
+
+      # The manifest project object doesn't keep track of the path on the
+      # server where this git is located, so let's save that here.
+      mirrored_manifest_git = None
+      if opt.reference:
+        manifest_git_path = urllib.parse.urlparse(opt.manifest_url).path[1:]
+        mirrored_manifest_git = os.path.join(opt.reference, manifest_git_path)
+        if not mirrored_manifest_git.endswith(".git"):
+          mirrored_manifest_git += ".git"
+        if not os.path.exists(mirrored_manifest_git):
+          mirrored_manifest_git = os.path.join(opt.reference + '/.repo/manifests.git')
+
+      m._InitGitDir(mirror_git=mirrored_manifest_git)
 
       if opt.manifest_branch:
         m.revisionExpr = opt.manifest_branch
@@ -147,8 +186,8 @@ to update the working directory files.
       r.ResetFetch()
       r.Save()
 
-    groups = re.split('[,\s]+', opt.groups)
-    all_platforms = ['linux', 'darwin']
+    groups = re.split(r'[,\s]+', opt.groups)
+    all_platforms = ['linux', 'darwin', 'windows']
     platformize = lambda x: 'platform-' + x
     if opt.platform == 'auto':
       if (not opt.mirror and
@@ -157,30 +196,44 @@ to update the working directory files.
     elif opt.platform == 'all':
       groups.extend(map(platformize, all_platforms))
     elif opt.platform in all_platforms:
-      groups.extend(platformize(opt.platform))
+      groups.append(platformize(opt.platform))
     elif opt.platform != 'none':
-      print >>sys.stderr, 'fatal: invalid platform flag'
+      print('fatal: invalid platform flag', file=sys.stderr)
       sys.exit(1)
 
     groups = [x for x in groups if x]
     groupstr = ','.join(groups)
-    if opt.platform == 'auto' and groupstr == 'all,-notdefault,platform-' + platform.system().lower():
+    if opt.platform == 'auto' and groupstr == 'default,platform-' + platform.system().lower():
       groupstr = None
     m.config.SetString('manifest.groups', groupstr)
 
     if opt.reference:
       m.config.SetString('repo.reference', opt.reference)
 
+    if opt.archive:
+      if is_new:
+        m.config.SetString('repo.archive', 'true')
+      else:
+        print('fatal: --archive is only supported when initializing a new '
+              'workspace.', file=sys.stderr)
+        print('Either delete the .repo folder in this workspace, or initialize '
+              'in another location.', file=sys.stderr)
+        sys.exit(1)
+
     if opt.mirror:
       if is_new:
         m.config.SetString('repo.mirror', 'true')
       else:
-        print >>sys.stderr, 'fatal: --mirror not supported on existing client'
+        print('fatal: --mirror is only supported when initializing a new '
+              'workspace.', file=sys.stderr)
+        print('Either delete the .repo folder in this workspace, or initialize '
+              'in another location.', file=sys.stderr)
         sys.exit(1)
 
-    if not m.Sync_NetworkHalf(is_new=is_new):
+    if not m.Sync_NetworkHalf(is_new=is_new, quiet=opt.quiet,
+        clone_bundle=not opt.no_clone_bundle):
       r = m.GetRemote(m.remote.name)
-      print >>sys.stderr, 'fatal: cannot obtain manifest %s' % r.url
+      print('fatal: cannot obtain manifest %s' % r.url, file=sys.stderr)
 
       # Better delete the manifest git dir if we created it; otherwise next
       # time (when user fixes problems) we won't go through the "is_new" logic.
@@ -189,7 +242,7 @@ to update the working directory files.
       sys.exit(1)
 
     if opt.manifest_branch:
-      m.MetaBranchSwitch(opt.manifest_branch)
+      m.MetaBranchSwitch()
 
     syncbuf = SyncBuffer(m.config)
     m.Sync_LocalHalf(syncbuf)
@@ -197,19 +250,19 @@ to update the working directory files.
 
     if is_new or m.CurrentBranch is None:
       if not m.StartBranch('default'):
-        print >>sys.stderr, 'fatal: cannot create default in manifest'
+        print('fatal: cannot create default in manifest', file=sys.stderr)
         sys.exit(1)
 
   def _LinkManifest(self, name):
     if not name:
-      print >>sys.stderr, 'fatal: manifest name (-m) is required.'
+      print('fatal: manifest name (-m) is required.', file=sys.stderr)
       sys.exit(1)
 
     try:
       self.manifest.Link(name)
     except ManifestParseError as e:
-      print >>sys.stderr, "fatal: manifest '%s' not available" % name
-      print >>sys.stderr, 'fatal: %s' % str(e)
+      print("fatal: manifest '%s' not available" % name, file=sys.stderr)
+      print('fatal: %s' % str(e), file=sys.stderr)
       sys.exit(1)
 
   def _Prompt(self, prompt, value):
@@ -231,24 +284,24 @@ to update the working directory files.
       mp.config.SetString('user.name', gc.GetString('user.name'))
       mp.config.SetString('user.email', gc.GetString('user.email'))
 
-    print ''
-    print 'Your identity is: %s <%s>' % (mp.config.GetString('user.name'),
-                                         mp.config.GetString('user.email'))
-    print 'If you want to change this, please re-run \'repo init\' with --config-name'
+    print()
+    print('Your identity is: %s <%s>' % (mp.config.GetString('user.name'),
+                                         mp.config.GetString('user.email')))
+    print('If you want to change this, please re-run \'repo init\' with --config-name')
     return False
 
   def _ConfigureUser(self):
     mp = self.manifest.manifestProject
 
     while True:
-      print ''
+      print()
       name  = self._Prompt('Your Name', mp.UserName)
       email = self._Prompt('Your Email', mp.UserEmail)
 
-      print ''
-      print 'Your identity is: %s <%s>' % (name, email)
+      print()
+      print('Your identity is: %s <%s>' % (name, email))
       sys.stdout.write('is this correct [y/N]? ')
-      a = sys.stdin.readline().strip()
+      a = sys.stdin.readline().strip().lower()
       if a in ('yes', 'y', 't', 'true'):
         break
 
@@ -274,17 +327,17 @@ to update the working directory files.
         self._on = True
     out = _Test()
 
-    print ''
-    print "Testing colorized output (for 'repo diff', 'repo status'):"
+    print()
+    print("Testing colorized output (for 'repo diff', 'repo status'):")
 
-    for c in ['black','red','green','yellow','blue','magenta','cyan']:
+    for c in ['black', 'red', 'green', 'yellow', 'blue', 'magenta', 'cyan']:
       out.write(' ')
       out.printer(fg=c)(' %-6s ', c)
     out.write(' ')
     out.printer(fg='white', bg='black')(' %s ' % 'white')
     out.nl()
 
-    for c in ['bold','dim','ul','reverse']:
+    for c in ['bold', 'dim', 'ul', 'reverse']:
       out.write(' ')
       out.printer(fg='black', attr=c)(' %-6s ', c)
     out.nl()
@@ -313,12 +366,36 @@ to update the working directory files.
       # We store the depth in the main manifest project.
       self.manifest.manifestProject.config.SetString('repo.depth', depth)
 
+  def _DisplayResult(self):
+    if self.manifest.IsMirror:
+      init_type = 'mirror '
+    else:
+      init_type = ''
+
+    print()
+    print('repo %shas been initialized in %s'
+          % (init_type, self.manifest.topdir))
+
+    current_dir = os.getcwd()
+    if current_dir != self.manifest.topdir:
+      print('If this is not the directory in which you want to initialize '
+            'repo, please run:')
+      print('   rm -r %s/.repo' % self.manifest.topdir)
+      print('and try again.')
+
   def Execute(self, opt, args):
     git_require(MIN_GIT_VERSION, fail=True)
 
     if opt.reference:
       opt.reference = os.path.expanduser(opt.reference)
 
+    # Check this here, else manifest will be tagged "not new" and init won't be
+    # possible anymore without removing the .repo/manifests directory.
+    if opt.archive and opt.mirror:
+      print('fatal: --mirror and --archive cannot be used together.',
+            file=sys.stderr)
+      sys.exit(1)
+
     self._SyncManifest(opt)
     self._LinkManifest(opt.manifest_name)
 
@@ -329,10 +406,4 @@ to update the working directory files.
 
     self._ConfigureDepth(opt)
 
-    if self.manifest.IsMirror:
-      init_type = 'mirror '
-    else:
-      init_type = ''
-
-    print ''
-    print 'repo %sinitialized in %s' % (init_type, self.manifest.topdir)
+    self._DisplayResult()

subcmds/list.py

@@ -13,7 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import re
+from __future__ import print_function
+import sys
 
 from command import Command, MirrorSafeCommand
 
@@ -30,13 +31,22 @@ List all projects; pass '.' to list the project for the cwd.
 This is similar to running: repo forall -c 'echo "$REPO_PATH : $REPO_PROJECT"'.
 """
 
-  def _Options(self, p, show_smart=True):
+  def _Options(self, p):
     p.add_option('-r', '--regex',
                  dest='regex', action='store_true',
                  help="Filter the project list based on regex or wildcard matching of strings")
+    p.add_option('-g', '--groups',
+                 dest='groups',
+                 help="Filter the project list based on the groups the project is in")
     p.add_option('-f', '--fullpath',
                  dest='fullpath', action='store_true',
                  help="Display the full work tree path instead of the relative path")
+    p.add_option('-n', '--name-only',
+                 dest='name_only', action='store_true',
+                 help="Display only the name of the repository")
+    p.add_option('-p', '--path-only',
+                 dest='path_only', action='store_true',
+                 help="Display only the path of the repository")
 
   def Execute(self, opt, args):
     """List all projects and the associated directories.
@@ -49,8 +59,13 @@ This is similar to running: repo forall -c 'echo "$REPO_PATH : $REPO_PROJECT"'.
       opt: The options.
       args: Positional args.  Can be a list of projects to list, or empty.
     """
+
+    if opt.fullpath and opt.name_only:
+      print('error: cannot combine -f and -n', file=sys.stderr)
+      sys.exit(1)
+
     if not opt.regex:
-      projects = self.GetProjects(args)
+      projects = self.GetProjects(args, groups=opt.groups)
     else:
       projects = self.FindProjects(args)
 
@@ -61,18 +76,12 @@ This is similar to running: repo forall -c 'echo "$REPO_PATH : $REPO_PROJECT"'.
 
     lines = []
     for project in projects:
-      lines.append("%s : %s" % (_getpath(project), project.name))
+      if opt.name_only and not opt.path_only:
+        lines.append("%s" % ( project.name))
+      elif opt.path_only and not opt.name_only:
+        lines.append("%s" % (_getpath(project)))
+      else:
+        lines.append("%s : %s" % (_getpath(project), project.name))
 
     lines.sort()
-    print '\n'.join(lines)
-
-  def FindProjects(self, args):
-    result = []
-    for project in self.GetProjects(''):
-      for arg in args:
-        pattern = re.compile(r'%s' % arg, re.IGNORECASE)
-        if pattern.search(project.name) or pattern.search(project.relpath):
-          result.append(project)
-          break
-    result.sort(key=lambda project: project.relpath)
-    return result
+    print('\n'.join(lines))

subcmds/manifest.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import os
 import sys
 
@@ -69,7 +70,7 @@ in a Git repository for use during future 'repo init' invocations.
                        peg_rev_upstream = opt.peg_rev_upstream)
     fd.close()
     if opt.output_file != '-':
-      print >>sys.stderr, 'Saved manifest to %s' % opt.output_file
+      print('Saved manifest to %s' % opt.output_file, file=sys.stderr)
 
   def Execute(self, opt, args):
     if args:
@@ -79,6 +80,6 @@ in a Git repository for use during future 'repo init' invocations.
       self._Output(opt)
       return
 
-    print >>sys.stderr, 'error: no operation to perform'
-    print >>sys.stderr, 'error: see repo help manifest'
+    print('error: no operation to perform', file=sys.stderr)
+    print('error: see repo help manifest', file=sys.stderr)
     sys.exit(1)

subcmds/overview.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 from color import Coloring
 from command import PagedCommand
 
@@ -41,7 +42,7 @@ are displayed.
     all_branches = []
     for project in self.GetProjects(args):
       br = [project.GetUploadableBranch(x)
-            for x in project.GetBranches().keys()]
+            for x in project.GetBranches()]
       br = [x for x in br if x]
       if opt.current_branch:
         br = [x for x in br if x.name == project.CurrentBranch]
@@ -54,8 +55,11 @@ are displayed.
       def __init__(self, config):
         Coloring.__init__(self, config, 'status')
         self.project = self.printer('header', attr='bold')
+        self.text = self.printer('text')
 
     out = Report(all_branches[0].project.config)
+    out.text("Deprecated. See repo info -o.")
+    out.nl()
     out.project('Projects Overview')
     out.nl()
 
@@ -70,11 +74,11 @@ are displayed.
 
       commits = branch.commits
       date = branch.date
-      print '%s %-33s (%2d commit%s, %s)' % (
+      print('%s %-33s (%2d commit%s, %s)' % (
             branch.name == project.CurrentBranch and '*' or ' ',
             branch.name,
             len(commits),
             len(commits) != 1 and 's' or ' ',
-            date)
+            date))
       for commit in commits:
-        print '%-35s   - %s' % ('', commit)
+        print('%-35s   - %s' % ('', commit))

subcmds/prune.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 from color import Coloring
 from command import PagedCommand
 
@@ -51,9 +52,9 @@ class Prune(PagedCommand):
 
       commits = branch.commits
       date = branch.date
-      print '%s %-33s (%2d commit%s, %s)' % (
+      print('%s %-33s (%2d commit%s, %s)' % (
             branch.name == project.CurrentBranch and '*' or ' ',
             branch.name,
             len(commits),
             len(commits) != 1 and 's' or ' ',
-            date)
+            date))

subcmds/rebase.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import sys
 
 from command import Command
@@ -53,20 +54,30 @@ branch but need to incorporate new upstream changes "underneath" them.
     p.add_option('--auto-stash',
                  dest='auto_stash', action='store_true',
                  help='Stash local modifications before starting')
+    p.add_option('-m', '--onto-manifest',
+                 dest='onto_manifest', action='store_true',
+                 help='Rebase onto the manifest version instead of upstream '
+                      'HEAD.  This helps to make sure the local tree stays '
+                      'consistent if you previously synced to a manifest.')
 
   def Execute(self, opt, args):
     all_projects = self.GetProjects(args)
     one_project = len(all_projects) == 1
 
     if opt.interactive and not one_project:
-      print >>sys.stderr, 'error: interactive rebase not supported with multiple projects'
+      print('error: interactive rebase not supported with multiple projects',
+            file=sys.stderr)
+      if len(args) == 1:
+        print('note: project %s is mapped to more than one path' % (args[0],),
+            file=sys.stderr)
       return -1
 
     for project in all_projects:
       cb = project.CurrentBranch
       if not cb:
         if one_project:
-          print >>sys.stderr, "error: project %s has a detatched HEAD" % project.relpath
+          print("error: project %s has a detached HEAD" % project.relpath,
+                file=sys.stderr)
           return -1
         # ignore branches with detatched HEADs
         continue
@@ -74,7 +85,8 @@ branch but need to incorporate new upstream changes "underneath" them.
       upbranch = project.GetBranch(cb)
       if not upbranch.LocalMerge:
         if one_project:
-          print >>sys.stderr, "error: project %s does not track any remote branches" % project.relpath
+          print("error: project %s does not track any remote branches"
+                % project.relpath, file=sys.stderr)
           return -1
         # ignore branches without remotes
         continue
@@ -99,10 +111,14 @@ branch but need to incorporate new upstream changes "underneath" them.
       if opt.interactive:
         args.append("-i")
 
+      if opt.onto_manifest:
+        args.append('--onto')
+        args.append(project.revisionExpr)
+
       args.append(upbranch.LocalMerge)
 
-      print >>sys.stderr, '# %s: rebasing %s -> %s' % \
-        (project.relpath, cb, upbranch.LocalMerge)
+      print('# %s: rebasing %s -> %s'
+            % (project.relpath, cb, upbranch.LocalMerge), file=sys.stderr)
 
       needs_stash = False
       if opt.auto_stash:

subcmds/selfupdate.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 from optparse import SUPPRESS_HELP
 import sys
 
@@ -52,7 +53,7 @@ need to be performed by an end-user.
 
     else:
       if not rp.Sync_NetworkHalf():
-        print >>sys.stderr, "error: can't update repo"
+        print("error: can't update repo", file=sys.stderr)
         sys.exit(1)
 
       rp.bare_git.gc('--auto')

subcmds/stage.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import sys
 
 from color import Coloring
@@ -48,9 +49,9 @@ The '%prog' command stages files to prepare the next commit.
       self.Usage()
 
   def _Interactive(self, opt, args):
-    all_projects = filter(lambda x: x.IsDirty(), self.GetProjects(args))
+    all_projects = [p for p in self.GetProjects(args) if p.IsDirty()]
     if not all_projects:
-      print >>sys.stderr,'no projects have uncommitted modifications'
+      print('no projects have uncommitted modifications', file=sys.stderr)
       return
 
     out = _ProjectList(self.manifest.manifestProject.config)
@@ -58,7 +59,7 @@ The '%prog' command stages files to prepare the next commit.
       out.header('        %s', 'project')
       out.nl()
 
-      for i in xrange(0, len(all_projects)):
+      for i in range(len(all_projects)):
         p = all_projects[i]
         out.write('%3d:    %s', i + 1, p.relpath + '/')
         out.nl()
@@ -97,11 +98,11 @@ The '%prog' command stages files to prepare the next commit.
           _AddI(all_projects[a_index - 1])
           continue
 
-      p = filter(lambda x: x.name == a or x.relpath == a, all_projects)
-      if len(p) == 1:
-        _AddI(p[0])
+      projects = [p for p in all_projects if a in [p.name, p.relpath]]
+      if len(projects) == 1:
+        _AddI(projects[0])
         continue
-    print 'Bye.'
+    print('Bye.')
 
 def _AddI(project):
   p = GitCommand(project, ['add', '--interactive'], bare=False)

subcmds/start.py

@@ -13,11 +13,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
+import os
 import sys
+
 from command import Command
 from git_config import IsId
 from git_command import git
+import gitc_utils
 from progress import Progress
+from project import SyncBuffer
 
 class Start(Command):
   common = True
@@ -41,7 +46,7 @@ revision specified in the manifest.
 
     nb = args[0]
     if not git.check_ref_format('heads/%s' % nb):
-      print >>sys.stderr, "error: '%s' is not a valid name" % nb
+      print("error: '%s' is not a valid name" % nb, file=sys.stderr)
       sys.exit(1)
 
     err = []
@@ -49,25 +54,64 @@ revision specified in the manifest.
     if not opt.all:
       projects = args[1:]
       if len(projects) < 1:
-        print >>sys.stderr, "error: at least one project must be specified"
-        sys.exit(1)
+        projects = ['.',]  # start it in the local project by default
 
-    all_projects = self.GetProjects(projects)
+    all_projects = self.GetProjects(projects,
+                                    missing_ok=bool(self.gitc_manifest))
+
+    # This must happen after we find all_projects, since GetProjects may need
+    # the local directory, which will disappear once we save the GITC manifest.
+    if self.gitc_manifest:
+      gitc_projects = self.GetProjects(projects, manifest=self.gitc_manifest,
+                                       missing_ok=True)
+      for project in gitc_projects:
+        if project.old_revision:
+          project.already_synced = True
+        else:
+          project.already_synced = False
+          project.old_revision = project.revisionExpr
+        project.revisionExpr = None
+      # Save the GITC manifest.
+      gitc_utils.save_manifest(self.gitc_manifest)
+
+      # Make sure we have a valid CWD
+      if not os.path.exists(os.getcwd()):
+        os.chdir(self.manifest.topdir)
 
     pm = Progress('Starting %s' % nb, len(all_projects))
     for project in all_projects:
       pm.update()
+
+      if self.gitc_manifest:
+        gitc_project = self.gitc_manifest.paths[project.relpath]
+        # Sync projects that have not been opened.
+        if not gitc_project.already_synced:
+          proj_localdir = os.path.join(self.gitc_manifest.gitc_client_dir,
+                                       project.relpath)
+          project.worktree = proj_localdir
+          if not os.path.exists(proj_localdir):
+            os.makedirs(proj_localdir)
+          project.Sync_NetworkHalf()
+          sync_buf = SyncBuffer(self.manifest.manifestProject.config)
+          project.Sync_LocalHalf(sync_buf)
+          project.revisionId = gitc_project.old_revision
+
       # If the current revision is a specific SHA1 then we can't push back
-      # to it so substitute the manifest default revision instead.
+      # to it; so substitute with dest_branch if defined, or with manifest
+      # default revision instead.
+      branch_merge = ''
       if IsId(project.revisionExpr):
-        project.revisionExpr = self.manifest.default.revisionExpr
-      if not project.StartBranch(nb):
+        if project.dest_branch:
+          branch_merge = project.dest_branch
+        else:
+          branch_merge = self.manifest.default.revisionExpr
+
+      if not project.StartBranch(nb, branch_merge=branch_merge):
         err.append(project)
     pm.end()
 
     if err:
       for p in err:
-        print >>sys.stderr,\
-          "error: %s/: cannot start %s" \
-          % (p.relpath, nb)
+        print("error: %s/: cannot start %s" % (p.relpath, nb),
+              file=sys.stderr)
       sys.exit(1)

subcmds/status.py

@@ -20,9 +20,12 @@ try:
 except ImportError:
   import dummy_threading as _threading
 
+import glob
+
 import itertools
-import sys
-import StringIO
+import os
+
+from color import Coloring
 
 class Status(PagedCommand):
   common = True
@@ -39,6 +42,13 @@ is a difference between these three states.
 The -j/--jobs option can be used to run multiple status queries
 in parallel.
 
+The -o/--orphans option can be used to show objects that are in
+the working directory, but not associated with a repo project.
+This includes unmanaged top-level files and directories, but also
+includes deeper items.  For example, if dir/subdir/proj1 and
+dir/subdir/proj2 are repo projects, dir/subdir/proj3 will be shown
+if it is not known to repo.
+
 Status Display
 --------------
 
@@ -76,8 +86,11 @@ the following meanings:
     p.add_option('-j', '--jobs',
                  dest='jobs', action='store', type='int', default=2,
                  help="number of projects to check simultaneously")
+    p.add_option('-o', '--orphans',
+                 dest='orphans', action='store_true',
+                 help="include objects in working directory outside of repo projects")
 
-  def _StatusHelper(self, project, clean_counter, sem, output):
+  def _StatusHelper(self, project, clean_counter, sem):
     """Obtains the status for a specific project.
 
     Obtains the status for a project, redirecting the output to
@@ -91,12 +104,28 @@ the following meanings:
       output: Where to output the status.
     """
     try:
-      state = project.PrintWorkTreeStatus(output)
+      state = project.PrintWorkTreeStatus()
       if state == 'CLEAN':
-        clean_counter.next()
+        next(clean_counter)
     finally:
       sem.release()
 
+  def _FindOrphans(self, dirs, proj_dirs, proj_dirs_parents, outstring):
+    """find 'dirs' that are present in 'proj_dirs_parents' but not in 'proj_dirs'"""
+    status_header = ' --\t'
+    for item in dirs:
+      if not os.path.isdir(item):
+        outstring.append(''.join([status_header, item]))
+        continue
+      if item in proj_dirs:
+        continue
+      if item in proj_dirs_parents:
+        self._FindOrphans(glob.glob('%s/.*' % item) +
+            glob.glob('%s/*' % item),
+            proj_dirs, proj_dirs_parents, outstring)
+        continue
+      outstring.append(''.join([status_header, item, '/']))
+
   def Execute(self, opt, args):
     all_projects = self.GetProjects(args)
     counter = itertools.count()
@@ -105,28 +134,59 @@ the following meanings:
       for project in all_projects:
         state = project.PrintWorkTreeStatus()
         if state == 'CLEAN':
-          counter.next()
+          next(counter)
     else:
       sem = _threading.Semaphore(opt.jobs)
-      threads_and_output = []
+      threads = []
       for project in all_projects:
         sem.acquire()
 
-        class BufList(StringIO.StringIO):
-          def dump(self, ostream):
-            for entry in self.buflist:
-              ostream.write(entry)
-
-        output = BufList()
-
         t = _threading.Thread(target=self._StatusHelper,
-                              args=(project, counter, sem, output))
-        threads_and_output.append((t, output))
+                              args=(project, counter, sem))
+        threads.append(t)
         t.daemon = True
         t.start()
-      for (t, output) in threads_and_output:
+      for t in threads:
         t.join()
-        output.dump(sys.stdout)
-        output.close()
-    if len(all_projects) == counter.next():
-      print 'nothing to commit (working directory clean)'
+    if len(all_projects) == next(counter):
+      print('nothing to commit (working directory clean)')
+
+    if opt.orphans:
+      proj_dirs = set()
+      proj_dirs_parents = set()
+      for project in self.GetProjects(None, missing_ok=True):
+        proj_dirs.add(project.relpath)
+        (head, _tail) = os.path.split(project.relpath)
+        while head != "":
+          proj_dirs_parents.add(head)
+          (head, _tail) = os.path.split(head)
+      proj_dirs.add('.repo')
+
+      class StatusColoring(Coloring):
+        def __init__(self, config):
+          Coloring.__init__(self, config, 'status')
+          self.project = self.printer('header', attr = 'bold')
+          self.untracked = self.printer('untracked', fg = 'red')
+
+      orig_path = os.getcwd()
+      try:
+        os.chdir(self.manifest.topdir)
+
+        outstring = []
+        self._FindOrphans(glob.glob('.*') +
+            glob.glob('*'),
+            proj_dirs, proj_dirs_parents, outstring)
+
+        if outstring:
+          output = StatusColoring(self.manifest.globalConfig)
+          output.project('Objects not within a project (orphans)')
+          output.nl()
+          for entry in outstring:
+            output.untracked(entry)
+            output.nl()
+        else:
+          print('No orphan files or directories')
+
+      finally:
+        # Restore CWD.
+        os.chdir(orig_path)

subcmds/upload.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import copy
 import re
 import sys
@@ -20,22 +21,33 @@ import sys
 from command import InteractiveCommand
 from editor import Editor
 from error import HookError, UploadError
+from git_command import GitCommand
 from project import RepoHook
 
+from pyversion import is_python3
+# pylint:disable=W0622
+if not is_python3():
+  input = raw_input
+else:
+  unicode = str
+# pylint:enable=W0622
+
 UNUSUAL_COMMIT_THRESHOLD = 5
 
 def _ConfirmManyUploads(multiple_branches=False):
   if multiple_branches:
-    print "ATTENTION: One or more branches has an unusually high number of commits."
+    print('ATTENTION: One or more branches has an unusually high number '
+          'of commits.')
   else:
-    print "ATTENTION: You are uploading an unusually high number of commits."
-  print "YOU PROBABLY DO NOT MEAN TO DO THIS. (Did you rebase across branches?)"
-  answer = raw_input("If you are sure you intend to do this, type 'yes': ").strip()
+    print('ATTENTION: You are uploading an unusually high number of commits.')
+  print('YOU PROBABLY DO NOT MEAN TO DO THIS. (Did you rebase across '
+        'branches?)')
+  answer = input("If you are sure you intend to do this, type 'yes': ").strip()
   return answer == "yes"
 
 def _die(fmt, *args):
   msg = fmt % args
-  print >>sys.stderr, 'error: %s' % msg
+  print('error: %s' % msg, file=sys.stderr)
   sys.exit(1)
 
 def _SplitEmails(values):
@@ -47,7 +59,7 @@ def _SplitEmails(values):
 class Upload(InteractiveCommand):
   common = True
   helpSummary = "Upload changes for code review"
-  helpUsage="""
+  helpUsage = """
 %prog [--re --cc] [<project>]...
 """
   helpDescription = """
@@ -79,6 +91,11 @@ to "true" then repo will assume you always answer "y" at the prompt,
 and will not prompt you further.  If it is set to "false" then repo
 will assume you always answer "n", and will abort.
 
+review.URL.autoreviewer:
+
+To automatically append a user or mailing list to reviews, you can set
+a per-project or global Git option to do so.
+
 review.URL.autocopy:
 
 To automatically copy a user or mailing list to all uploaded reviews,
@@ -137,6 +154,10 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
     p.add_option('-d', '--draft',
                  action='store_true', dest='draft', default=False,
                  help='If specified, upload as a draft.')
+    p.add_option('-D', '--destination', '--dest',
+                 type='string', action='store', dest='dest_branch',
+                 metavar='BRANCH',
+                 help='Submit for review on this target branch.')
 
     # Options relating to upload hook.  Note that verify and no-verify are NOT
     # opposites of each other, which is why they store to different locations.
@@ -176,18 +197,19 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
       date = branch.date
       commit_list = branch.commits
 
-      print 'Upload project %s/ to remote branch %s:' % (project.relpath, project.revisionExpr)
-      print '  branch %s (%2d commit%s, %s):' % (
+      destination = opt.dest_branch or project.dest_branch or project.revisionExpr
+      print('Upload project %s/ to remote branch %s:' % (project.relpath, destination))
+      print('  branch %s (%2d commit%s, %s):' % (
                     name,
                     len(commit_list),
                     len(commit_list) != 1 and 's' or '',
-                    date)
+                    date))
       for commit in commit_list:
-        print '         %s' % commit
+        print('         %s' % commit)
 
       sys.stdout.write('to %s (y/N)? ' % remote.review)
-      answer = sys.stdin.readline().strip()
-      answer = answer in ('y', 'Y', 'yes', '1', 'true', 't')
+      answer = sys.stdin.readline().strip().lower()
+      answer = answer in ('y', 'yes', '1', 'true', 't')
 
     if answer:
       if len(branch.commits) > UNUSUAL_COMMIT_THRESHOLD:
@@ -210,18 +232,21 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
 
       b = {}
       for branch in avail:
+        if branch is None:
+          continue
         name = branch.name
         date = branch.date
         commit_list = branch.commits
 
         if b:
           script.append('#')
+        destination = opt.dest_branch or project.dest_branch or project.revisionExpr
         script.append('#  branch %s (%2d commit%s, %s) to remote branch %s:' % (
                       name,
                       len(commit_list),
                       len(commit_list) != 1 and 's' or '',
                       date,
-                      project.revisionExpr))
+                      destination))
         for commit in commit_list:
           script.append('#         %s' % commit)
         b[name] = branch
@@ -275,14 +300,20 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
 
     self._UploadAndReport(opt, todo, people)
 
-  def _AppendAutoCcList(self, branch, people):
+  def _AppendAutoList(self, branch, people):
     """
+    Appends the list of reviewers in the git project's config.
     Appends the list of users in the CC list in the git project's config if a
     non-empty reviewer list was found.
     """
-
     name = branch.name
     project = branch.project
+
+    key = 'review.%s.autoreviewer' % project.GetBranch(name).remote.review
+    raw_list = project.config.GetString(key)
+    if not raw_list is None:
+      people[0].extend([entry.strip() for entry in raw_list.split(',')])
+
     key = 'review.%s.autocopy' % project.GetBranch(name).remote.review
     raw_list = project.config.GetString(key)
     if not raw_list is None and len(people[0]) > 0:
@@ -297,7 +328,7 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
     try:
       # refs/changes/XYZ/N --> XYZ
       return refs.get(last_pub).split('/')[-2]
-    except:
+    except (AttributeError, IndexError):
       return ""
 
   def _UploadAndReport(self, opt, todo, original_people):
@@ -305,37 +336,56 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
     for branch in todo:
       try:
         people = copy.deepcopy(original_people)
-        self._AppendAutoCcList(branch, people)
+        self._AppendAutoList(branch, people)
 
         # Check if there are local changes that may have been forgotten
-        if branch.project.HasChanges():
-            key = 'review.%s.autoupload' % branch.project.remote.review
-            answer = branch.project.config.GetBoolean(key)
+        changes = branch.project.UncommitedFiles()
+        if changes:
+          key = 'review.%s.autoupload' % branch.project.remote.review
+          answer = branch.project.config.GetBoolean(key)
 
-            # if they want to auto upload, let's not ask because it could be automated
-            if answer is None:
-                sys.stdout.write('Uncommitted changes in ' + branch.project.name + ' (did you forget to amend?). Continue uploading? (y/N) ')
-                a = sys.stdin.readline().strip().lower()
-                if a not in ('y', 'yes', 't', 'true', 'on'):
-                    print >>sys.stderr, "skipping upload"
-                    branch.uploaded = False
-                    branch.error = 'User aborted'
-                    continue
+          # if they want to auto upload, let's not ask because it could be automated
+          if answer is None:
+            sys.stdout.write('Uncommitted changes in ' + branch.project.name)
+            sys.stdout.write(' (did you forget to amend?):\n')
+            sys.stdout.write('\n'.join(changes) + '\n')
+            sys.stdout.write('Continue uploading? (y/N) ')
+            a = sys.stdin.readline().strip().lower()
+            if a not in ('y', 'yes', 't', 'true', 'on'):
+              print("skipping upload", file=sys.stderr)
+              branch.uploaded = False
+              branch.error = 'User aborted'
+              continue
 
         # Check if topic branches should be sent to the server during upload
         if opt.auto_topic is not True:
-           key = 'review.%s.uploadtopic' % branch.project.remote.review
-           opt.auto_topic = branch.project.config.GetBoolean(key)
+          key = 'review.%s.uploadtopic' % branch.project.remote.review
+          opt.auto_topic = branch.project.config.GetBoolean(key)
 
-        branch.UploadForReview(people, auto_topic=opt.auto_topic, draft=opt.draft)
+        destination = opt.dest_branch or branch.project.dest_branch
+
+        # Make sure our local branch is not setup to track a different remote branch
+        merge_branch = self._GetMergeBranch(branch.project)
+        if destination:
+          full_dest = 'refs/heads/%s' % destination
+          if not opt.dest_branch and merge_branch and merge_branch != full_dest:
+            print('merge branch %s does not match destination branch %s'
+                  % (merge_branch, full_dest))
+            print('skipping upload.')
+            print('Please use `--destination %s` if this is intentional'
+                  % destination)
+            branch.uploaded = False
+            continue
+
+        branch.UploadForReview(people, auto_topic=opt.auto_topic, draft=opt.draft, dest_branch=destination)
         branch.uploaded = True
       except UploadError as e:
         branch.error = e
         branch.uploaded = False
         have_errors = True
 
-    print >>sys.stderr, ''
-    print >>sys.stderr, '----------------------------------------------------------------------'
+    print(file=sys.stderr)
+    print('----------------------------------------------------------------------', file=sys.stderr)
 
     if have_errors:
       for branch in todo:
@@ -344,21 +394,38 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
             fmt = ' (%s)'
           else:
             fmt = '\n       (%s)'
-          print >>sys.stderr, ('[FAILED] %-15s %-15s' + fmt) % (
+          print(('[FAILED] %-15s %-15s' + fmt) % (
                  branch.project.relpath + '/', \
                  branch.name, \
-                 str(branch.error))
-      print >>sys.stderr, ''
+                 str(branch.error)),
+                 file=sys.stderr)
+      print()
 
     for branch in todo:
-        if branch.uploaded:
-          print >>sys.stderr, '[OK    ] %-15s %s' % (
-                 branch.project.relpath + '/',
-                 branch.name)
+      if branch.uploaded:
+        print('[OK    ] %-15s %s' % (
+               branch.project.relpath + '/',
+               branch.name),
+               file=sys.stderr)
 
     if have_errors:
       sys.exit(1)
 
+  def _GetMergeBranch(self, project):
+    p = GitCommand(project,
+                   ['rev-parse', '--abbrev-ref', 'HEAD'],
+                   capture_stdout = True,
+                   capture_stderr = True)
+    p.Wait()
+    local_branch = p.stdout.strip()
+    p = GitCommand(project,
+                   ['config', '--get', 'branch.%s.merge' % local_branch],
+                   capture_stdout = True,
+                   capture_stderr = True)
+    p.Wait()
+    merge_branch = p.stdout.strip()
+    return merge_branch
+
   def Execute(self, opt, args):
     project_list = self.GetProjects(args)
     pending = []
@@ -372,31 +439,46 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
     for project in project_list:
       if opt.current_branch:
         cbr = project.CurrentBranch
-        avail = [project.GetUploadableBranch(cbr)] if cbr else None
+        up_branch = project.GetUploadableBranch(cbr)
+        if up_branch:
+          avail = [up_branch]
+        else:
+          avail = None
+          print('ERROR: Current branch (%s) not uploadable. '
+                'You may be able to type '
+                '"git branch --set-upstream-to m/master" to fix '
+                'your branch.' % str(cbr),
+                file=sys.stderr)
       else:
         avail = project.GetUploadableBranches(branch)
       if avail:
         pending.append((project, avail))
 
-    if pending and (not opt.bypass_hooks):
+    if not pending:
+      print("no branches ready for upload", file=sys.stderr)
+      return
+
+    if not opt.bypass_hooks:
       hook = RepoHook('pre-upload', self.manifest.repo_hooks_project,
-                      self.manifest.topdir, abort_if_user_denies=True)
+                      self.manifest.topdir,
+                      self.manifest.manifestProject.GetRemote('origin').url,
+                      abort_if_user_denies=True)
       pending_proj_names = [project.name for (project, avail) in pending]
+      pending_worktrees = [project.worktree for (project, avail) in pending]
       try:
-        hook.Run(opt.allow_all_hooks, project_list=pending_proj_names)
+        hook.Run(opt.allow_all_hooks, project_list=pending_proj_names,
+                 worktree_list=pending_worktrees)
       except HookError as e:
-        print >>sys.stderr, "ERROR: %s" % str(e)
+        print("ERROR: %s" % str(e), file=sys.stderr)
         return
 
     if opt.reviewers:
       reviewers = _SplitEmails(opt.reviewers)
     if opt.cc:
       cc = _SplitEmails(opt.cc)
-    people = (reviewers,cc)
+    people = (reviewers, cc)
 
-    if not pending:
-      print >>sys.stdout, "no branches ready for upload"
-    elif len(pending) == 1 and len(pending[0][1]) == 1:
+    if len(pending) == 1 and len(pending[0][1]) == 1:
       self._SingleBranch(opt, pending[0][1][0], people)
     else:
       self._MultipleBranches(opt, pending, people)

subcmds/version.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import sys
 from command import Command, MirrorSafeCommand
 from git_command import git
@@ -32,12 +33,12 @@ class Version(Command, MirrorSafeCommand):
     rp = self.manifest.repoProject
     rem = rp.GetRemote(rp.remote.name)
 
-    print 'repo version %s' % rp.work_git.describe(HEAD)
-    print '       (from %s)' % rem.url
+    print('repo version %s' % rp.work_git.describe(HEAD))
+    print('       (from %s)' % rem.url)
 
     if Version.wrapper_path is not None:
-      print 'repo launcher version %s' % Version.wrapper_version
-      print '       (from %s)' % Version.wrapper_path
+      print('repo launcher version %s' % Version.wrapper_version)
+      print('       (from %s)' % Version.wrapper_path)
 
-    print git.version().strip()
-    print 'Python %s' % sys.version
+    print(git.version().strip())
+    print('Python %s' % sys.version)

tests/fixtures/gitc_config

@@ -0,0 +1 @@
+gitc_dir=/test/usr/local/google/gitc

tests/test_git_config.py

@@ -4,49 +4,49 @@ import unittest
 import git_config
 
 def fixture(*paths):
-    """Return a path relative to test/fixtures.
-    """
-    return os.path.join(os.path.dirname(__file__), 'fixtures', *paths)
+  """Return a path relative to test/fixtures.
+  """
+  return os.path.join(os.path.dirname(__file__), 'fixtures', *paths)
 
 class GitConfigUnitTest(unittest.TestCase):
-    """Tests the GitConfig class.
+  """Tests the GitConfig class.
+  """
+  def setUp(self):
+    """Create a GitConfig object using the test.gitconfig fixture.
     """
-    def setUp(self):
-        """Create a GitConfig object using the test.gitconfig fixture.
-        """
-        config_fixture = fixture('test.gitconfig')
-        self.config = git_config.GitConfig(config_fixture)
+    config_fixture = fixture('test.gitconfig')
+    self.config = git_config.GitConfig(config_fixture)
 
-    def test_GetString_with_empty_config_values(self):
-        """
-        Test config entries with no value.
+  def test_GetString_with_empty_config_values(self):
+    """
+    Test config entries with no value.
 
-        [section]
-            empty
+    [section]
+        empty
 
-        """
-        val = self.config.GetString('section.empty')
-        self.assertEqual(val, None)
+    """
+    val = self.config.GetString('section.empty')
+    self.assertEqual(val, None)
 
-    def test_GetString_with_true_value(self):
-        """
-        Test config entries with a string value.
+  def test_GetString_with_true_value(self):
+    """
+    Test config entries with a string value.
 
-        [section]
-            nonempty = true
+    [section]
+        nonempty = true
 
-        """
-        val = self.config.GetString('section.nonempty')
-        self.assertEqual(val, 'true')
+    """
+    val = self.config.GetString('section.nonempty')
+    self.assertEqual(val, 'true')
 
-    def test_GetString_from_missing_file(self):
-        """
-        Test missing config file
-        """
-        config_fixture = fixture('not.present.gitconfig')
-        config = git_config.GitConfig(config_fixture)
-        val = config.GetString('empty')
-        self.assertEqual(val, None)
+  def test_GetString_from_missing_file(self):
+    """
+    Test missing config file
+    """
+    config_fixture = fixture('not.present.gitconfig')
+    config = git_config.GitConfig(config_fixture)
+    val = config.GetString('empty')
+    self.assertEqual(val, None)
 
 if __name__ == '__main__':
-    unittest.main()
+  unittest.main()

tests/test_wrapper.py

@@ -0,0 +1,75 @@
+#
+# Copyright (C) 2015 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import unittest
+
+import wrapper
+
+def fixture(*paths):
+  """Return a path relative to tests/fixtures.
+  """
+  return os.path.join(os.path.dirname(__file__), 'fixtures', *paths)
+
+class RepoWrapperUnitTest(unittest.TestCase):
+  """Tests helper functions in the repo wrapper
+  """
+  def setUp(self):
+    """Load the wrapper module every time
+    """
+    wrapper._wrapper_module = None
+    self.wrapper = wrapper.Wrapper()
+
+  def test_get_gitc_manifest_dir_no_gitc(self):
+    """
+    Test reading a missing gitc config file
+    """
+    self.wrapper.GITC_CONFIG_FILE = fixture('missing_gitc_config')
+    val = self.wrapper.get_gitc_manifest_dir()
+    self.assertEqual(val, '')
+
+  def test_get_gitc_manifest_dir(self):
+    """
+    Test reading the gitc config file and parsing the directory
+    """
+    self.wrapper.GITC_CONFIG_FILE = fixture('gitc_config')
+    val = self.wrapper.get_gitc_manifest_dir()
+    self.assertEqual(val, '/test/usr/local/google/gitc')
+
+  def test_gitc_parse_clientdir_no_gitc(self):
+    """
+    Test parsing the gitc clientdir without gitc running
+    """
+    self.wrapper.GITC_CONFIG_FILE = fixture('missing_gitc_config')
+    self.assertEqual(self.wrapper.gitc_parse_clientdir('/something'), None)
+    self.assertEqual(self.wrapper.gitc_parse_clientdir('/gitc/manifest-rw/test'), 'test')
+
+  def test_gitc_parse_clientdir(self):
+    """
+    Test parsing the gitc clientdir
+    """
+    self.wrapper.GITC_CONFIG_FILE = fixture('gitc_config')
+    self.assertEqual(self.wrapper.gitc_parse_clientdir('/something'), None)
+    self.assertEqual(self.wrapper.gitc_parse_clientdir('/gitc/manifest-rw/test'), 'test')
+    self.assertEqual(self.wrapper.gitc_parse_clientdir('/gitc/manifest-rw/test/'), 'test')
+    self.assertEqual(self.wrapper.gitc_parse_clientdir('/gitc/manifest-rw/test/extra'), 'test')
+    self.assertEqual(self.wrapper.gitc_parse_clientdir('/test/usr/local/google/gitc/test'), 'test')
+    self.assertEqual(self.wrapper.gitc_parse_clientdir('/test/usr/local/google/gitc/test/'), 'test')
+    self.assertEqual(self.wrapper.gitc_parse_clientdir('/test/usr/local/google/gitc/test/extra'), 'test')
+    self.assertEqual(self.wrapper.gitc_parse_clientdir('/gitc/manifest-rw/'), None)
+    self.assertEqual(self.wrapper.gitc_parse_clientdir('/test/usr/local/google/gitc/'), None)
+
+if __name__ == '__main__':
+  unittest.main()

trace.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import sys
 import os
 REPO_TRACE = 'REPO_TRACE'
@@ -31,4 +32,4 @@ def SetTrace():
 
 def Trace(fmt, *args):
   if IsTrace():
-    print >>sys.stderr, fmt % args
+    print(fmt % args, file=sys.stderr)

wrapper.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2014 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+import imp
+import os
+
+
+def WrapperPath():
+  return os.path.join(os.path.dirname(__file__), 'repo')
+
+_wrapper_module = None
+def Wrapper():
+  global _wrapper_module
+  if not _wrapper_module:
+    _wrapper_module = imp.load_source('wrapper', WrapperPath())
+  return _wrapper_module