Fix tag clobbering when -c is used.

Bug: b/140189154
Change-Id: I8861a6115b20c9a3d88ddec5344c75326ae44823
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/237572
Reviewed-by: Mike Frysinger <vapier@google.com>
Tested-by: Xin Li <delphij@google.com>

.flake8

@@ -0,0 +1,3 @@
+[flake8]
+max-line-length=80
+ignore=E111,E114,E402

.gitattributes

@@ -1,4 +1,4 @@
 # Prevent /bin/sh scripts from being clobbered by autocrlf=true
 git_ssh text eol=lf
-main.py text eol=lf
 repo text eol=lf
+hooks/* text eol=lf

.mailmap

@@ -0,0 +1,12 @@
+Anthony Newnam <anthony.newnam@garmin.com>    Anthony <anthony@bnovc.com>
+He Ping <tdihp@hotmail.com>                   heping <tdihp@hotmail.com>
+Hu Xiuyun <xiuyun.hu@hisilicon.com>           Hu xiuyun <xiuyun.hu@hisilicon.com>
+Hu Xiuyun <xiuyun.hu@hisilicon.com>           Hu Xiuyun <clouds08@qq.com>
+Jelly Chen <chenguodong@huawei.com>           chenguodong <chenguodong@huawei.com>
+Jia Bi <bijia@xiaomi.com>                     bijia <bijia@xiaomi.com>
+JoonCheol Park <jooncheol@gmail.com>          Jooncheol Park <jooncheol@gmail.com>
+Sergii Pylypenko <x.pelya.x@gmail.com>        pelya <x.pelya.x@gmail.com>
+Shawn Pearce <sop@google.com>                 Shawn O. Pearce <sop@google.com>
+Ulrik Sjölin <ulrik.sjolin@sonyericsson.com>  Ulrik Sjolin <ulrik.sjolin@gmail.com>
+Ulrik Sjölin <ulrik.sjolin@sonyericsson.com>  Ulrik Sjolin <ulrik.sjolin@sonyericsson.com>
+Ulrik Sjölin <ulrik.sjolin@sonyericsson.com>  Ulrik Sjölin <ulrik.sjolin@sonyericsson.com>

.pydevproject

@@ -5,6 +5,6 @@
 <pydev_pathproperty name="org.python.pydev.PROJECT_SOURCE_PATH">
 <path>/git-repo</path>
 </pydev_pathproperty>
-<pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python 2.6</pydev_property>
+<pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python 2.7</pydev_property>
 <pydev_property name="org.python.pydev.PYTHON_PROJECT_INTERPRETER">Default</pydev_property>
 </pydev_project>

.pylintrc

@@ -1,298 +0,0 @@
-# lint Python modules using external checkers.
-#
-# This is the main checker controling the other ones and the reports
-# generation. It is itself both a raw checker and an astng checker in order
-# to:
-# * handle message activation / deactivation at the module level
-# * handle some basic but necessary stats'data (number of classes, methods...)
-#
-[MASTER]
-
-# Specify a configuration file.
-#rcfile=
-
-# Python code to execute, usually for sys.path manipulation such as
-# pygtk.require().
-#init-hook=
-
-# Profiled execution.
-profile=no
-
-# Add <file or directory> to the black list. It should be a base name, not a
-# path. You may set this option multiple times.
-ignore=SVN
-
-# Pickle collected data for later comparisons.
-persistent=yes
-
-# Set the cache size for astng objects.
-cache-size=500
-
-# List of plugins (as comma separated values of python modules names) to load,
-# usually to register additional checkers.
-load-plugins=
-
-
-[MESSAGES CONTROL]
-
-# Enable only checker(s) with the given id(s). This option conflicts with the
-# disable-checker option
-#enable-checker=
-
-# Enable all checker(s) except those with the given id(s). This option
-# conflicts with the enable-checker option
-#disable-checker=
-
-# Enable all messages in the listed categories.
-#enable-msg-cat=
-
-# Disable all messages in the listed categories.
-#disable-msg-cat=
-
-# Enable the message(s) with the given id(s).
-enable=RP0004
-
-# Disable the message(s) with the given id(s).
-disable=R0903,R0912,R0913,R0914,R0915,W0141,C0111,C0103,W0603,W0703,R0911,C0301,C0302,R0902,R0904,W0142,W0212,E1101,E1103,R0201,W0201,W0122,W0232,RP0001,RP0003,RP0101,RP0002,RP0401,RP0701,RP0801,F0401,E0611,R0801,I0011
-
-[REPORTS]
-
-# set the output format. Available formats are text, parseable, colorized, msvs
-# (visual studio) and html
-output-format=text
-
-# Put messages in a separate file for each module / package specified on the
-# command line instead of printing them on stdout. Reports (if any) will be
-# written in a file name "pylint_global.[txt|html]".
-files-output=no
-
-# Tells whether to display a full report or only the messages
-reports=yes
-
-# Python expression which should return a note less than 10 (10 is the highest
-# note).You have access to the variables errors warning, statement which
-# respectivly contain the number of errors / warnings messages and the total
-# number of statements analyzed. This is used by the global evaluation report
-# (R0004).
-evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
-
-# Add a comment according to your evaluation note. This is used by the global
-# evaluation report (R0004).
-comment=no
-
-# checks for
-# * unused variables / imports
-# * undefined variables
-# * redefinition of variable from builtins or from an outer scope
-# * use of variable before assigment
-#
-[VARIABLES]
-
-# Tells whether we should check for unused import in __init__ files.
-init-import=no
-
-# A regular expression matching names used for dummy variables (i.e. not used).
-dummy-variables-rgx=_|dummy
-
-# List of additional names supposed to be defined in builtins. Remember that
-# you should avoid to define new builtins when possible.
-additional-builtins=
-
-
-# try to find bugs in the code using type inference
-#
-[TYPECHECK]
-
-# Tells whether missing members accessed in mixin class should be ignored. A
-# mixin class is detected if its name ends with "mixin" (case insensitive).
-ignore-mixin-members=yes
-
-# List of classes names for which member attributes should not be checked
-# (useful for classes with attributes dynamicaly set).
-ignored-classes=SQLObject
-
-# When zope mode is activated, consider the acquired-members option to ignore
-# access to some undefined attributes.
-zope=no
-
-# List of members which are usually get through zope's acquisition mecanism and
-# so shouldn't trigger E0201 when accessed (need zope=yes to be considered).
-acquired-members=REQUEST,acl_users,aq_parent
-
-
-# checks for :
-# * doc strings
-# * modules / classes / functions / methods / arguments / variables name
-# * number of arguments, local variables, branchs, returns and statements in
-# functions, methods
-# * required module attributes
-# * dangerous default values as arguments
-# * redefinition of function / method / class
-# * uses of the global statement
-#
-[BASIC]
-
-# Required attributes for module, separated by a comma
-required-attributes=
-
-# Regular expression which should only match functions or classes name which do
-# not require a docstring
-no-docstring-rgx=_main|__.*__
-
-# Regular expression which should only match correct module names
-module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
-
-# Regular expression which should only match correct module level names
-const-rgx=(([A-Z_][A-Z1-9_]*)|(__.*__))|(log)$
-
-# Regular expression which should only match correct class names
-class-rgx=[A-Z_][a-zA-Z0-9]+$
-
-# Regular expression which should only match correct function names
-function-rgx=[a-z_][a-z0-9_]{2,30}$
-
-# Regular expression which should only match correct method names
-method-rgx=[a-z_][a-z0-9_]{2,30}$
-
-# Regular expression which should only match correct instance attribute names
-attr-rgx=[a-z_][a-z0-9_]{2,30}$
-
-# Regular expression which should only match correct argument names
-argument-rgx=[a-z_][a-z0-9_]{2,30}$
-
-# Regular expression which should only match correct variable names
-variable-rgx=[a-z_][a-z0-9_]{2,30}$
-
-# Regular expression which should only match correct list comprehension /
-# generator expression variable names
-inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$
-
-# Good variable names which should always be accepted, separated by a comma
-good-names=i,j,k,ex,Run,_,e,d1,d2,v,f,l,d
-
-# Bad variable names which should always be refused, separated by a comma
-bad-names=foo,bar,baz,toto,tutu,tata
-
-# List of builtins function names that should not be used, separated by a comma
-bad-functions=map,filter,apply,input
-
-
-# checks for sign of poor/misdesign:
-# * number of methods, attributes, local variables...
-# * size, complexity of functions, methods
-#
-[DESIGN]
-
-# Maximum number of arguments for function / method
-max-args=5
-
-# Maximum number of locals for function / method body
-max-locals=15
-
-# Maximum number of return / yield for function / method body
-max-returns=6
-
-# Maximum number of branch for function / method body
-max-branchs=12
-
-# Maximum number of statements in function / method body
-max-statements=50
-
-# Maximum number of parents for a class (see R0901).
-max-parents=7
-
-# Maximum number of attributes for a class (see R0902).
-max-attributes=20
-
-# Minimum number of public methods for a class (see R0903).
-min-public-methods=2
-
-# Maximum number of public methods for a class (see R0904).
-max-public-methods=30
-
-
-# checks for
-# * external modules dependencies
-# * relative / wildcard imports
-# * cyclic imports
-# * uses of deprecated modules
-#
-[IMPORTS]
-
-# Deprecated modules which should not be used, separated by a comma
-deprecated-modules=regsub,string,TERMIOS,Bastion,rexec
-
-# Create a graph of every (i.e. internal and external) dependencies in the
-# given file (report R0402 must not be disabled)
-import-graph=
-
-# Create a graph of external dependencies in the given file (report R0402 must
-# not be disabled)
-ext-import-graph=
-
-# Create a graph of internal dependencies in the given file (report R0402 must
-# not be disabled)
-int-import-graph=
-
-
-# checks for :
-# * methods without self as first argument
-# * overridden methods signature
-# * access only to existant members via self
-# * attributes not defined in the __init__ method
-# * supported interfaces implementation
-# * unreachable code
-#
-[CLASSES]
-
-# List of interface methods to ignore, separated by a comma. This is used for
-# instance to not check methods defines in Zope's Interface base class.
-ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by
-
-# List of method names used to declare (i.e. assign) instance attributes.
-defining-attr-methods=__init__,__new__,setUp
-
-
-# checks for similarities and duplicated code. This computation may be
-# memory / CPU intensive, so you should disable it if you experiments some
-# problems.
-#
-[SIMILARITIES]
-
-# Minimum lines number of a similarity.
-min-similarity-lines=4
-
-# Ignore comments when computing similarities.
-ignore-comments=yes
-
-# Ignore docstrings when computing similarities.
-ignore-docstrings=yes
-
-
-# checks for:
-# * warning notes in the code like FIXME, XXX
-# * PEP 263: source code with non ascii character but no encoding declaration
-#
-[MISCELLANEOUS]
-
-# List of note tags to take in consideration, separated by a comma.
-notes=FIXME,XXX,TODO
-
-
-# checks for :
-# * unauthorized constructions
-# * strict indentation
-# * line length
-# * use of <> instead of !=
-#
-[FORMAT]
-
-# Maximum number of characters on a single line.
-max-line-length=80
-
-# Maximum number of lines in a module
-max-module-lines=1000
-
-# String used as indentation unit. This is usually "    " (4 spaces) or "\t" (1
-# tab).  In repo it is 2 spaces.
-indent-string='  '

README.md

@@ -0,0 +1,16 @@
+# repo
+
+Repo is a tool built on top of Git.  Repo helps manage many Git repositories,
+does the uploads to revision control systems, and automates parts of the
+development workflow.  Repo is not meant to replace Git, only to make it
+easier to work with Git.  The repo command is an executable Python script
+that you can put anywhere in your path.
+
+* Homepage: https://gerrit.googlesource.com/git-repo/
+* Bug reports: https://bugs.chromium.org/p/gerrit/issues/list?q=component:repo
+* Source: https://gerrit.googlesource.com/git-repo/
+* Overview: https://source.android.com/source/developing.html
+* Docs: https://source.android.com/source/using-repo.html
+* [repo Manifest Format](./docs/manifest-format.md)
+* [repo Hooks](./docs/repo-hooks.md)
+* [Submitting patches](./SUBMITTING_PATCHES.md)

SUBMITTING_PATCHES.md

@@ -1,17 +1,19 @@
-Short Version:
+[TOC]
+
+# Short Version
 
  - Make small logical changes.
  - Provide a meaningful commit message.
- - Check for coding errors with pylint
+ - Check for coding errors and style nits with pyflakes and flake8
  - Make sure all code is under the Apache License, 2.0.
  - Publish your changes for review.
  - Make corrections if requested.
  - Verify your changes on gerrit so they can be submitted.
 
-   git push https://gerrit-review.googlesource.com/git-repo HEAD:refs/for/master
+   `git push https://gerrit-review.googlesource.com/git-repo HEAD:refs/for/master`
 
 
-Long Version:
+# Long Version
 
 I wanted a file describing how to submit patches for repo,
 so I started with the one found in the core Git distribution
@@ -19,10 +21,10 @@ so I started with the one found in the core Git distribution
 patch submission guidelines for the Linux kernel.
 
 However there are some differences, so please review and familiarize
-yourself with the following relevant bits:
+yourself with the following relevant bits.
 
 
-(1) Make separate commits for logically separate changes.
+## Make separate commits for logically separate changes.
 
 Unless your patch is really trivial, you should not be sending
 out a patch that was generated between your working tree and your
@@ -36,14 +38,46 @@ If your description starts to get too long, that's a sign that you
 probably need to split up your commit to finer grained pieces.
 
 
-(2) Check for coding errors with pylint
+## Check for coding errors and style nits with pyflakes and flake8
 
-Run pylint on changed modules using the provided configuration:
+### Coding errors
 
-  pylint --rcfile=.pylintrc file.py
+Run `pyflakes` on changed modules:
+
+    pyflakes file.py
+
+Ideally there should be no new errors or warnings introduced.
+
+### Style violations
+
+Run `flake8` on changes modules:
+
+    flake8 file.py
+
+Note that repo generally follows [Google's python style guide] rather than
+[PEP 8], so it's possible that the output of `flake8` will be quite noisy.
+It's not mandatory to avoid all warnings, but at least the maximum line
+length should be followed.
+
+If there are many occurrences of the same warning that cannot be
+avoided without going against the Google style guide, these may be
+suppressed in the included `.flake8` file.
+
+[Google's python style guide]: https://google.github.io/styleguide/pyguide.html
+[PEP 8]: https://www.python.org/dev/peps/pep-0008/
 
 
-(3) Check the license
+## Running tests
+
+There is a [`./run_tests`](./run_tests) helper script for quickly invoking all
+of our unittests.  The coverage isn't great currently, but it should still be
+run for all commits.
+
+Adding more unittests for changes you make would be greatly appreciated :).
+Check out the [tests/](./tests/) subdirectory for more details.
+
+
+## Check the license
 
 repo is licensed under the Apache License, 2.0.
 
@@ -59,7 +93,7 @@ your patch.  It is virtually impossible to remove a patch once it
 has been applied and pushed out.
 
 
-(4) Sending your patches.
+## Sending your patches.
 
 Do not email your patches to anyone.
 
@@ -91,23 +125,23 @@ to get the ChangeId added.
 Push your patches over HTTPS to the review server, possibly through
 a remembered remote to make this easier in the future:
 
-   git config remote.review.url https://gerrit-review.googlesource.com/git-repo
+    git config remote.review.url https://gerrit-review.googlesource.com/git-repo
-   git config remote.review.push HEAD:refs/for/master
+    git config remote.review.push HEAD:refs/for/master
 
-   git push review
+    git push review
 
 You will be automatically emailed a copy of your commits, and any
 comments made by the project maintainers.
 
 
-(5) Make changes if requested
+## Make changes if requested
 
 The project maintainer who reviews your changes might request changes to your
 commit. If you make the requested changes you will need to amend your commit
 and push it to the review server again.
 
 
-(6) Verify your changes on gerrit
+## Verify your changes on gerrit
 
 After you receive a Code-Review+2 from the maintainer, select the Verified
 button on the gerrit page for the change. This verifies that you have tested

color.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #

command.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #
@@ -19,6 +20,7 @@ import platform
 import re
 import sys
 
+from event_log import EventLog
 from error import NoSuchProjectError
 from error import InvalidProjectGroupsError
 
@@ -28,10 +30,11 @@ class Command(object):
   """
 
   common = False
+  event_log = EventLog()
   manifest = None
   _optparse = None
 
-  def WantPager(self, opt):
+  def WantPager(self, _opt):
     return False
 
   def ReadEnvironmentOptions(self, opts):
@@ -63,7 +66,7 @@ class Command(object):
         usage = self.helpUsage.strip().replace('%prog', me)
       except AttributeError:
         usage = 'repo %s' % self.NAME
-      self._optparse = optparse.OptionParser(usage = usage)
+      self._optparse = optparse.OptionParser(usage=usage)
       self._Options(self._optparse)
     return self._optparse
 
@@ -95,6 +98,16 @@ class Command(object):
     self.OptionParser.print_usage()
     sys.exit(1)
 
+  def ValidateOptions(self, opt, args):
+    """Validate the user options & arguments before executing.
+
+    This is meant to help break the code up into logical steps.  Some tips:
+    * Use self.OptionParser.error to display CLI related errors.
+    * Adjust opt member defaults as makes sense.
+    * Adjust the args list, but do so inplace so the caller sees updates.
+    * Try to avoid updating self state.  Leave that to Execute.
+    """
+
   def Execute(self, opt, args):
     """Perform the action, after option parsing is complete.
     """
@@ -110,15 +123,20 @@ class Command(object):
     project = None
     if os.path.exists(path):
       oldpath = None
-      while path \
+      while path and \
-        and path != oldpath \
+            path != oldpath and \
-        and path != manifest.topdir:
+            path != manifest.topdir:
         try:
           project = self._by_path[path]
           break
         except KeyError:
           oldpath = path
           path = os.path.dirname(path)
+      if not project and path == manifest.topdir:
+        try:
+          project = self._by_path[path]
+        except KeyError:
+          pass
     else:
       try:
         project = self._by_path[path]
@@ -138,7 +156,7 @@ class Command(object):
     mp = manifest.manifestProject
 
     if not groups:
-        groups = mp.config.GetString('manifest.groups')
+      groups = mp.config.GetString('manifest.groups')
     if not groups:
       groups = 'default,platform-' + platform.system().lower()
     groups = [x for x in re.split(r'[,\s]+', groups) if x]
@@ -151,8 +169,7 @@ class Command(object):
                                   for p in project.GetDerivedSubprojects())
       all_projects_list.extend(derived_projects.values())
       for project in all_projects_list:
-        if ((missing_ok or project.Exists) and
+        if (missing_ok or project.Exists) and project.MatchesGroups(groups):
-            project.MatchesGroups(groups)):
           result.append(project)
     else:
       self._ResetPathToProjectMap(all_projects_list)
@@ -166,8 +183,8 @@ class Command(object):
 
           # If it's not a derived project, update path->project mapping and
           # search again, as arg might actually point to a derived subproject.
-          if (project and not project.Derived and
+          if (project and not project.Derived and (submodules_ok or
-              (submodules_ok or project.sync_s)):
+                                                   project.sync_s)):
             search_again = False
             for subproject in project.GetDerivedSubprojects():
               self._UpdatePathToProjectMap(subproject)
@@ -194,48 +211,52 @@ class Command(object):
     result.sort(key=_getpath)
     return result
 
-  def FindProjects(self, args):
+  def FindProjects(self, args, inverse=False):
     result = []
     patterns = [re.compile(r'%s' % a, re.IGNORECASE) for a in args]
     for project in self.GetProjects(''):
       for pattern in patterns:
-        if pattern.search(project.name) or pattern.search(project.relpath):
+        match = pattern.search(project.name) or pattern.search(project.relpath)
+        if not inverse and match:
           result.append(project)
           break
+        if inverse and match:
+          break
+      else:
+        if inverse:
+          result.append(project)
     result.sort(key=lambda project: project.relpath)
     return result
 
-# pylint: disable=W0223
+
-# Pylint warns that the `InteractiveCommand` and `PagedCommand` classes do not
-# override method `Execute` which is abstract in `Command`.  Since that method
-# is always implemented in classes derived from `InteractiveCommand` and
-# `PagedCommand`, this warning can be suppressed.
 class InteractiveCommand(Command):
   """Command which requires user interaction on the tty and
      must not run within a pager, even if the user asks to.
   """
-  def WantPager(self, opt):
+  def WantPager(self, _opt):
     return False
 
+
 class PagedCommand(Command):
   """Command which defaults to output in a pager, as its
      display tends to be larger than one screen full.
   """
-  def WantPager(self, opt):
+  def WantPager(self, _opt):
     return True
 
-# pylint: enable=W0223
 
 class MirrorSafeCommand(object):
   """Command permits itself to run within a mirror,
      and does not require a working directory.
   """
 
+
 class GitcAvailableCommand(object):
   """Command that requires GITC to be available, but does
      not require the local client to be a GITC client.
   """
 
+
 class GitcClientCommand(object):
   """Command that requires the local client to be a GITC
      client.

docs/manifest-format.md

@@ -1,110 +1,116 @@
-repo Manifest Format
+# repo Manifest Format
-====================
 
 A repo manifest describes the structure of a repo client; that is
 the directories that are visible and where they should be obtained
 from with git.
 
 The basic structure of a manifest is a bare Git repository holding
-a single 'default.xml' XML file in the top level directory.
+a single `default.xml` XML file in the top level directory.
 
 Manifests are inherently version controlled, since they are kept
 within a Git repository.  Updates to manifests are automatically
 obtained by clients during `repo sync`.
 
+[TOC]
 
-XML File Format
----------------
 
-A manifest XML file (e.g. 'default.xml') roughly conforms to the
+## XML File Format
+
+A manifest XML file (e.g. `default.xml`) roughly conforms to the
 following DTD:
 
-  <!DOCTYPE manifest [
+```xml
-    <!ELEMENT manifest (notice?,
+<!DOCTYPE manifest [
-                        remote*,
+  <!ELEMENT manifest (notice?,
-                        default?,
+                      remote*,
-                        manifest-server?,
+                      default?,
-                        remove-project*,
+                      manifest-server?,
-                        project*,
+                      remove-project*,
-                        extend-project*,
+                      project*,
-                        repo-hooks?)>
+                      extend-project*,
+                      repo-hooks?,
+                      include*)>
 
-    <!ELEMENT notice (#PCDATA)>
+  <!ELEMENT notice (#PCDATA)>
 
-    <!ELEMENT remote (EMPTY)>
+  <!ELEMENT remote EMPTY>
-    <!ATTLIST remote name         ID    #REQUIRED>
+  <!ATTLIST remote name         ID    #REQUIRED>
-    <!ATTLIST remote alias        CDATA #IMPLIED>
+  <!ATTLIST remote alias        CDATA #IMPLIED>
-    <!ATTLIST remote fetch        CDATA #REQUIRED>
+  <!ATTLIST remote fetch        CDATA #REQUIRED>
-    <!ATTLIST remote review       CDATA #IMPLIED>
+  <!ATTLIST remote pushurl      CDATA #IMPLIED>
-    <!ATTLIST remote revision     CDATA #IMPLIED>
+  <!ATTLIST remote review       CDATA #IMPLIED>
+  <!ATTLIST remote revision     CDATA #IMPLIED>
 
-    <!ELEMENT default (EMPTY)>
+  <!ELEMENT default EMPTY>
-    <!ATTLIST default remote      IDREF #IMPLIED>
+  <!ATTLIST default remote      IDREF #IMPLIED>
-    <!ATTLIST default revision    CDATA #IMPLIED>
+  <!ATTLIST default revision    CDATA #IMPLIED>
-    <!ATTLIST default dest-branch CDATA #IMPLIED>
+  <!ATTLIST default dest-branch CDATA #IMPLIED>
-    <!ATTLIST default sync-j      CDATA #IMPLIED>
+  <!ATTLIST default upstream    CDATA #IMPLIED>
-    <!ATTLIST default sync-c      CDATA #IMPLIED>
+  <!ATTLIST default sync-j      CDATA #IMPLIED>
-    <!ATTLIST default sync-s      CDATA #IMPLIED>
+  <!ATTLIST default sync-c      CDATA #IMPLIED>
+  <!ATTLIST default sync-s      CDATA #IMPLIED>
+  <!ATTLIST default sync-tags   CDATA #IMPLIED>
 
-    <!ELEMENT manifest-server (EMPTY)>
+  <!ELEMENT manifest-server EMPTY>
-    <!ATTLIST manifest-server url CDATA #REQUIRED>
+  <!ATTLIST manifest-server url CDATA #REQUIRED>
 
-    <!ELEMENT project (annotation*,
+  <!ELEMENT project (annotation*,
-                       project*,
+                     project*,
-                       copyfile*,
+                     copyfile*,
-                       linkfile*)>
+                     linkfile*)>
-    <!ATTLIST project name        CDATA #REQUIRED>
+  <!ATTLIST project name        CDATA #REQUIRED>
-    <!ATTLIST project path        CDATA #IMPLIED>
+  <!ATTLIST project path        CDATA #IMPLIED>
-    <!ATTLIST project remote      IDREF #IMPLIED>
+  <!ATTLIST project remote      IDREF #IMPLIED>
-    <!ATTLIST project revision    CDATA #IMPLIED>
+  <!ATTLIST project revision    CDATA #IMPLIED>
-    <!ATTLIST project dest-branch CDATA #IMPLIED>
+  <!ATTLIST project dest-branch CDATA #IMPLIED>
-    <!ATTLIST project groups      CDATA #IMPLIED>
+  <!ATTLIST project groups      CDATA #IMPLIED>
-    <!ATTLIST project sync-c      CDATA #IMPLIED>
+  <!ATTLIST project sync-c      CDATA #IMPLIED>
-    <!ATTLIST project sync-s      CDATA #IMPLIED>
+  <!ATTLIST project sync-s      CDATA #IMPLIED>
-    <!ATTLIST project upstream CDATA #IMPLIED>
+  <!ATTLIST project sync-tags   CDATA #IMPLIED>
-    <!ATTLIST project clone-depth CDATA #IMPLIED>
+  <!ATTLIST project upstream CDATA #IMPLIED>
-    <!ATTLIST project force-path CDATA #IMPLIED>
+  <!ATTLIST project clone-depth CDATA #IMPLIED>
+  <!ATTLIST project force-path CDATA #IMPLIED>
 
-    <!ELEMENT annotation (EMPTY)>
+  <!ELEMENT annotation EMPTY>
-    <!ATTLIST annotation name  CDATA #REQUIRED>
+  <!ATTLIST annotation name  CDATA #REQUIRED>
-    <!ATTLIST annotation value CDATA #REQUIRED>
+  <!ATTLIST annotation value CDATA #REQUIRED>
-    <!ATTLIST annotation keep  CDATA "true">
+  <!ATTLIST annotation keep  CDATA "true">
 
-    <!ELEMENT copyfile (EMPTY)>
+  <!ELEMENT copyfile EMPTY>
-    <!ATTLIST copyfile src  CDATA #REQUIRED>
+  <!ATTLIST copyfile src  CDATA #REQUIRED>
-    <!ATTLIST copyfile dest CDATA #REQUIRED>
+  <!ATTLIST copyfile dest CDATA #REQUIRED>
 
-    <!ELEMENT linkfile (EMPTY)>
+  <!ELEMENT linkfile EMPTY>
-    <!ATTLIST linkfile src CDATA #REQUIRED>
+  <!ATTLIST linkfile src CDATA #REQUIRED>
-    <!ATTLIST linkfile dest CDATA #REQUIRED>
+  <!ATTLIST linkfile dest CDATA #REQUIRED>
 
-    <!ELEMENT extend-project (EMPTY)>
+  <!ELEMENT extend-project EMPTY>
-    <!ATTLIST extend-project name CDATA #REQUIRED>
+  <!ATTLIST extend-project name CDATA #REQUIRED>
-    <!ATTLIST extend-project path CDATA #IMPLIED>
+  <!ATTLIST extend-project path CDATA #IMPLIED>
-    <!ATTLIST extend-project groups CDATA #IMPLIED>
+  <!ATTLIST extend-project groups CDATA #IMPLIED>
+  <!ATTLIST extend-project revision CDATA #IMPLIED>
 
-    <!ELEMENT remove-project (EMPTY)>
+  <!ELEMENT remove-project EMPTY>
-    <!ATTLIST remove-project name  CDATA #REQUIRED>
+  <!ATTLIST remove-project name  CDATA #REQUIRED>
 
-    <!ELEMENT repo-hooks (EMPTY)>
+  <!ELEMENT repo-hooks EMPTY>
-    <!ATTLIST repo-hooks in-project CDATA #REQUIRED>
+  <!ATTLIST repo-hooks in-project CDATA #REQUIRED>
-    <!ATTLIST repo-hooks enabled-list CDATA #REQUIRED>
+  <!ATTLIST repo-hooks enabled-list CDATA #REQUIRED>
 
-    <!ELEMENT include      (EMPTY)>
+  <!ELEMENT include EMPTY>
-    <!ATTLIST include name CDATA #REQUIRED>
+  <!ATTLIST include name CDATA #REQUIRED>
-  ]>
+]>
+```
 
 A description of the elements and their attributes follows.
 
 
-Element manifest
+### Element manifest
-----------------
 
 The root element of the file.
 
 
-Element remote
+### Element remote
---------------
 
 One or more remote elements may be specified.  Each remote element
 specifies a Git URL shared by one or more projects and (optionally)
@@ -125,6 +131,12 @@ Attribute `fetch`: The Git URL prefix for all projects which use
 this remote.  Each project's name is appended to this prefix to
 form the actual URL used to clone the project.
 
+Attribute `pushurl`: The Git "push" URL prefix for all projects
+which use this remote.  Each project's name is appended to this
+prefix to form the actual URL used to "git push" the project.
+This attribute is optional; if not specified then "git push"
+will use the same URL as the `fetch` attribute.
+
 Attribute `review`: Hostname of the Gerrit server where reviews
 are uploaded to by `repo upload`.  This attribute is optional;
 if not specified then `repo upload` will not function.
@@ -133,8 +145,7 @@ Attribute `revision`: Name of a Git branch (e.g. `master` or
 `refs/heads/master`). Remotes with their own revision will override
 the default revision.
 
-Element default
+### Element default
----------------
 
 At most one default element may be specified.  Its remote and
 revision attributes are used when a project element does not
@@ -153,6 +164,11 @@ Project elements not setting their own `dest-branch` will inherit
 this value. If this value is not set, projects will use `revision`
 by default instead.
 
+Attribute `upstream`: Name of the Git ref in which a sha1
+can be found.  Used when syncing a revision locked manifest in
+-c mode to avoid having to sync the entire ref space. Project elements
+not setting their own `upstream` will inherit this value.
+
 Attribute `sync-j`: Number of parallel jobs to use when synching.
 
 Attribute `sync-c`: Set to true to only sync the given Git
@@ -162,9 +178,12 @@ their own will use this value.
 
 Attribute `sync-s`: Set to true to also sync sub-projects.
 
+Attribute `sync-tags`: Set to false to only sync the given Git
+branch (specified in the `revision` attribute) rather than
+the other ref tags.
 
-Element manifest-server
+
------------------------
+### Element manifest-server
 
 At most one manifest-server may be specified. The url attribute
 is used to specify the URL of a manifest server, which is an
@@ -172,10 +191,11 @@ XML RPC service.
 
 The manifest server should implement the following RPC methods:
 
-  GetApprovedManifest(branch, target)
+    GetApprovedManifest(branch, target)
 
 Return a manifest in which each project is pegged to a known good revision
-for the current branch and target.
+for the current branch and target. This is used by repo sync when the
+--smart-sync option is given.
 
 The target to use is defined by environment variables TARGET_PRODUCT
 and TARGET_BUILD_VARIANT. These variables are used to create a string
@@ -184,14 +204,14 @@ If one of those variables or both are not present, the program will call
 GetApprovedManifest without the target parameter and the manifest server
 should choose a reasonable default target.
 
-  GetManifest(tag)
+    GetManifest(tag)
 
 Return a manifest in which each project is pegged to the revision at
-the specified tag.
+the specified tag. This is used by repo sync when the --smart-tag option
+is given.
 
 
-Element project
+### Element project
----------------
 
 One or more project elements may be specified.  Each element
 describes a single Git repository to be cloned into the repo
@@ -204,7 +224,7 @@ Attribute `name`: A unique name for this project.  The project's
 name is appended onto its remote's fetch URL to generate the actual
 URL to configure the Git remote with.  The URL gets formed as:
 
-  ${remote_fetch}/${project_name}.git
+    ${remote_fetch}/${project_name}.git
 
 where ${remote_fetch} is the remote's fetch attribute and
 ${project_name} is the project's name attribute.  The suffix ".git"
@@ -268,8 +288,7 @@ rather than the `name` attribute.  This attribute only applies to the
 local mirrors syncing, it will be ignored when syncing the projects in a
 client working directory.
 
-Element extend-project
+### Element extend-project
-----------------------
 
 Modify the attributes of the named project.
 
@@ -284,8 +303,10 @@ at the specified path, rather than all projects with the given name.
 Attribute `groups`: List of additional groups to which this project
 belongs.  Same syntax as the corresponding element of `project`.
 
-Element annotation
+Attribute `revision`: If specified, overrides the revision of the original
-------------------
+project.  Same syntax as the corresponding element of `project`.
+
+### Element annotation
 
 Zero or more annotation elements may be specified as children of a
 project element. Each element describes a name-value pair that will be
@@ -295,23 +316,36 @@ prefixed with REPO__.  In addition, there is an optional attribute
 "false".  This attribute determines whether or not the annotation will
 be kept when exported with the manifest subcommand.
 
-Element copyfile
+### Element copyfile
-----------------
 
 Zero or more copyfile elements may be specified as children of a
 project element. Each element describes a src-dest pair of files;
-the "src" file will be copied to the "dest" place during 'repo sync'
+the "src" file will be copied to the "dest" place during `repo sync`
 command.
-"src" is project relative, "dest" is relative to the top of the tree.
 
-Element linkfile
+"src" is project relative, "dest" is relative to the top of the tree.
-----------------
+Copying from paths outside of the project or to paths outside of the repo
+client is not allowed.
+
+"src" and "dest" must be files.  Directories or symlinks are not allowed.
+Intermediate paths must not be symlinks either.
+
+Parent directories of "dest" will be automatically created if missing.
+
+### Element linkfile
 
 It's just like copyfile and runs at the same time as copyfile but
 instead of copying it creates a symlink.
 
-Element remove-project
+The symlink is created at "dest" (relative to the top of the tree) and
-----------------------
+points to the path specified by "src".
+
+Parent directories of "dest" will be automatically created if missing.
+
+The symlink target may be a file or directory, but it may not point outside
+of the repo client.
+
+### Element remove-project
 
 Deletes the named project from the internal manifest table, possibly
 allowing a subsequent project element in the same manifest file to
@@ -321,8 +355,7 @@ This element is mostly useful in a local manifest file, where
 the user can remove a project, and possibly replace it with their
 own definition.
 
-Element include
+### Element include
----------------
 
 This element provides the capability of including another manifest
 file into the originating manifest.  Normal rules apply for the
@@ -332,26 +365,25 @@ Attribute `name`: the manifest to include, specified relative to
 the manifest repository's root.
 
 
-Local Manifests
+## Local Manifests
-===============
 
 Additional remotes and projects may be added through local manifest
 files stored in `$TOP_DIR/.repo/local_manifests/*.xml`.
 
 For example:
 
-  $ ls .repo/local_manifests
+    $ ls .repo/local_manifests
-  local_manifest.xml
+    local_manifest.xml
-  another_local_manifest.xml
+    another_local_manifest.xml
 
-  $ cat .repo/local_manifests/local_manifest.xml
+    $ cat .repo/local_manifests/local_manifest.xml
-  <?xml version="1.0" encoding="UTF-8"?>
+    <?xml version="1.0" encoding="UTF-8"?>
-  <manifest>
+    <manifest>
-    <project path="manifest"
+      <project path="manifest"
-             name="tools/manifest" />
+               name="tools/manifest" />
-    <project path="platform-manifest"
+      <project path="platform-manifest"
-             name="platform/manifest" />
+               name="platform/manifest" />
-  </manifest>
+    </manifest>
 
 Users may add projects to the local manifest(s) prior to a `repo sync`
 invocation, instructing repo to automatically download and manage

docs/python-support.md

@@ -0,0 +1,47 @@
+# Supported Python Versions
+
+With Python 2.7 officially going EOL on [01 Jan 2020](https://pythonclock.org/),
+we need a support plan for the repo project itself.
+Inevitably, there will be a long tail of users who still want to use Python 2 on
+their old LTS/corp systems and have little power to change the system.
+
+## Summary
+
+* Python 3.6 (released Dec 2016) is required by default starting with repo-1.14.
+* Older versions of Python (e.g. v2.7) may use the legacy feature-frozen branch
+  based on repo-1.13.
+
+## Overview
+
+We provide a branch for Python 2 users that is feature-frozen.
+Bugfixes may be added on a best-effort basis or from the community, but largely
+no new features will be added, nor is support guaranteed.
+
+Users can select this during `repo init` time via the [repo launcher].
+Otherwise the default branches (e.g. stable & master) will be used which will
+require Python 3.
+
+This means the [repo launcher] needs to support both Python 2 & Python 3, but
+since it doesn't import any other repo code, this shouldn't be too problematic.
+
+The master branch will require Python 3.6 at a minimum.
+If the system has an older version of Python 3, then users will have to select
+the legacy Python 2 branch instead.
+
+### repo hooks
+
+Projects that use [repo hooks] run on independent schedules.
+They might migrate to Python 3 earlier or later than us.
+To support them, we'll probe the shebang of the hook script and if we find an
+interpreter in there that indicates a different version than repo is currently
+running under, we'll attempt to reexec ourselves under that.
+
+For example, a hook with a header like `#!/usr/bin/python2` will have repo
+execute `/usr/bin/python2` to execute the hook code specifically if repo is
+currently running Python 3.
+
+For more details, consult the [repo hooks] documentation.
+
+
+[repo hooks]: ./repo-hooks.md
+[repo launcher]: ../repo

docs/release-process.md

@@ -0,0 +1,167 @@
+# repo release process
+
+This is the process for creating a new release of repo, as well as all the
+related topics and flows.
+
+[TOC]
+
+## Launcher script
+
+The main repo script serves as a standalone program and is often referred to as
+the "launcher script".
+This makes it easy to copy around and install as you don't have to install any
+other files from the git repo.
+
+Whenever major changes are made to the launcher script, you should increment the
+`VERSION` variable in the launcher itself.
+At runtime, repo will check this to see if it needs to be updated (and notify
+the user automatically).
+
+## Key management
+
+Every release has a git tag that is signed with a key that repo recognizes.
+Those keys are hardcoded inside of the repo launcher itself -- look for the
+`KEYRING_VERSION` and `MAINTAINER_KEYS` settings.
+
+Adding new keys to the repo launcher will allow tags to be recognized by new
+keys, but only people using that updated version will be able to.
+Since the majority of users will be using an official launcher version, their
+version will simply ignore any new signed tags.
+
+If you want to add new keys, it's best to register them long ahead of time,
+and then wait for that updated launcher to make its way out to everyone.
+Even then, there will be a long tail of users with outdated launchers, so be
+prepared for people asking questions.
+
+### Registering a new key
+
+The process of actually adding a new key is quite simple.
+
+1.  Add the public half of the key to `MAINTAINER_KEYS`.
+2.  Increment `KEYRING_VERSION` so repo knows it needs to update.
+3.  Wait a long time after that version is in a release (~months) before trying
+    to create a new release using those new keys.
+
+## Self update algorithm
+
+When creating a new repo checkout with `repo init`, there are a few options that
+control how repo finds updates:
+
+*   `--repo-url`: This tells repo where to clone the full repo project itself.
+    It defaults to the official project (`REPO_URL` in the launcher script).
+*   `--repo-branch`: This tells repo which branch to use for the full project.
+    It defaults to the `stable` branch (`REPO_REV` in the launcher script).
+
+Whenever `repo sync` is run, repo will check to see if an update is available.
+It fetches the latest repo-branch from the repo-url.
+Then it verifies that the latest commit in the branch has a valid signed tag
+using `git tag -v` (which uses gpg).
+If the tag is valid, then repo will update its internal checkout to it.
+
+If the latest commit doesn't have a signed tag, repo will fall back to the
+most recent tag it can find (via `git describe`).
+If that tag is valid, then repo will warn and use that commit instead.
+
+If that tag cannot be verified, it gives up and forces the user to resolve.
+
+## Branch management
+
+All development happens on the `master` branch and should generally be stable.
+
+Since the repo launcher defaults to tracking the `stable` branch, it is not
+normally updated until a new release is available.
+If something goes wrong with a new release, an older release can be force pushed
+and clients will automatically downgrade.
+
+The `maint` branch is used to track the previous major release of repo.
+It is not normally meant to be used by people as `stable` should be good enough.
+Once a new major release is pushed to the `stable` branch, then the previous
+major release can be pushed to `maint`.
+For example, when `stable` moves from `v1.10.x` to `v1.11.x`, then the `maint`
+branch will be updated from `v1.9.x` to `v1.10.x`.
+
+We don't have parallel release branches/series.
+Typically all tags are made against the `master` branch and then pushed to the
+`stable` branch to make it available to the rest of the world.
+Since repo doesn't typically see a lot of changes, this tends to be OK.
+
+## Creating a new release
+
+When you want to create a new release, you'll need to select a good version and
+create a signed tag using a key registered in repo itself.
+Typically we just tag the latest version of the `master` branch.
+The tag could be pushed now, but it won't be used by clients normally (since the
+default `repo-branch` setting is `stable`).
+This would allow some early testing on systems who explicitly select `master`.
+
+### Creating a signed tag
+
+Lets assume your keys live in a dedicated directory, e.g. `~/.gnupg/repo/`.
+
+*** note
+If you need access to the official keys, check out the internal documentation
+at [go/repo-release].
+Note that only official maintainers of repo will have access as it describes
+internal processes for accessing the restricted keys.
+***
+
+```sh
+# Set the gpg key directory.
+$ export GNUPGHOME=~/.gnupg/repo/
+
+# Verify the listed key is “Repo Maintainer”.
+$ gpg -K
+
+# Pick whatever branch or commit you want to tag.
+$ r=master
+
+# Pick the new version.
+$ t=1.12.10
+
+# Create the signed tag.
+$ git tag -s v$t -u "Repo Maintainer <repo@android.kernel.org>" -m "repo $t" $r
+
+# Verify the signed tag.
+$ git show v$t
+```
+
+### Push the new release
+
+Once you're ready to make the release available to everyone, push it to the
+`stable` branch.
+
+Make sure you never push the tag itself to the stable branch!
+Only push the commit -- notice the use of `$t` and `$r` below.
+
+```sh
+$ git push https://gerrit-review.googlesource.com/git-repo v$t
+$ git push https://gerrit-review.googlesource.com/git-repo $r:stable
+```
+
+If something goes horribly wrong, you can force push the previous version to the
+`stable` branch and people should automatically recover.
+Again, make sure you never push the tag itself!
+
+```sh
+$ oldrev="whatever-old-commit"
+$ git push https://gerrit-review.googlesource.com/git-repo $oldrev:stable --force
+```
+
+### Announce the release
+
+Once you do push a new release to `stable`, make sure to announce it on the
+[repo-discuss@googlegroups.com] group.
+Here is an [example announcement].
+
+You can create a short changelog using the command:
+
+```sh
+# If you haven't pushed to the stable branch yet, you can use origin/stable.
+# If you have pushed, change origin/stable to the previous release tag.
+$ git log --format="%h (%aN) %s" --no-merges origin/stable..$r
+```
+
+
+[example announcement]: https://groups.google.com/d/topic/repo-discuss/UGBNismWo1M/discussion
+[repo-discuss@googlegroups.com]: https://groups.google.com/forum/#!forum/repo-discuss
+[go/repo-release]: https://goto.google.com/repo-release

docs/repo-hooks.md

@@ -0,0 +1,135 @@
+# repo hooks
+
+[TOC]
+
+Repo provides a mechanism to hook specific stages of the runtime with custom
+python modules.  All the hooks live in one git project which is checked out by
+the manifest (specified during `repo init`), and the manifest itself defines
+which hooks are registered.
+
+These are useful to run linters, check formatting, and run quick unittests
+before allowing a step to proceed (e.g. before uploading a commit to Gerrit).
+
+A complete example can be found in the Android project.  It can be easily
+re-used by any repo based project and is not specific to Android.<br>
+https://android.googlesource.com/platform/tools/repohooks
+
+## Approvals
+
+When a hook is processed the first time, the user is prompted for approval.
+We don't want to execute arbitrary code without explicit consent.  For manifests
+fetched via secure protocols (e.g. https://), the user is prompted once.  For
+insecure protocols (e.g. http://), the user is prompted whenever the registered
+repohooks project is updated and a hook is triggered.
+
+## Manifest Settings
+
+For the full syntax, see the [repo manifest format](./manifest-format.md).
+
+Here's a short example from
+[Android](https://android.googlesource.com/platform/manifest/+/master/default.xml).
+The `<project>` line checks out the repohooks git repo to the local
+`tools/repohooks/` path.  The `<repo-hooks>` line says to look in the project
+with the name `platform/tools/repohooks` for hooks to run during the
+`pre-upload` phase.
+
+```xml
+<project path="tools/repohooks" name="platform/tools/repohooks" />
+<repo-hooks in-project="platform/tools/repohooks" enabled-list="pre-upload" />
+```
+
+## Source Layout
+
+The repohooks git repo should have a python file with the same name as the hook.
+So if you want to support the `pre-upload` hook, you'll need to create a file
+named `pre-upload.py`.  Repo will dynamically load that module when processing
+the hook and then call the `main` function in it.
+
+Hooks should have their `main` accept `**kwargs` for future compatibility.
+
+## Runtime
+
+Hook return values are ignored.
+
+Any uncaught exceptions from the hook will cause the step to fail.  This is
+intended as a fallback safety check though rather than the normal flow.  If
+you want your hook to trigger a failure, it should call `sys.exit()` (after
+displaying relevant diagnostics).
+
+Output (stdout & stderr) are not filtered in any way.  Hooks should generally
+not be too verbose.  A short summary is nice, and some status information when
+long running operations occur, but long/verbose output should be used only if
+the hook ultimately fails.
+
+The hook runs from the top level of the repo client where the operation is
+started.
+For example, if the repo client is under `~/tree/`, then that is where the hook
+runs, even if you ran repo in a git repository at `~/tree/src/foo/`, or in a
+subdirectory of that git repository in `~/tree/src/foo/bar/`.
+Hooks frequently start off by doing a `os.chdir` to the specific project they're
+called on (see below) and then changing back to the original dir when they're
+finished.
+
+Python's `sys.path` is modified so that the top of repohooks directory comes
+first.  This should help simplify the hook logic to easily allow importing of
+local modules.
+
+Repo does not modify the state of the git checkout.  This means that the hooks
+might be running in a dirty git repo with many commits and checked out to the
+latest one.  If the hook wants to operate on specific git commits, it needs to
+manually discover the list of pending commits, extract the diff/commit, and
+then check it directly.  Hooks should not normally modify the active git repo
+(such as checking out a specific commit to run checks) without first prompting
+the user.  Although user interaction is discouraged in the common case, it can
+be useful when deploying automatic fixes.
+
+### Shebang Handling
+
+*** note
+This is intended as a transitional feature.  Hooks are expected to eventually
+migrate to Python 3 only as Python 2 is EOL & deprecated.
+***
+
+If the hook is written against a specific version of Python (either 2 or 3),
+the script can declare that explicitly.  Repo will then attempt to execute it
+under the right version of Python regardless of the version repo itself might
+be executing under.
+
+Here are the shebangs that are recognized.
+
+* `#!/usr/bin/env python` & `#!/usr/bin/python`: The hook is compatible with
+  Python 2 & Python 3.  For maximum compatibility, these are recommended.
+* `#!/usr/bin/env python2` & `#!/usr/bin/python2`: The hook requires Python 2.
+  Version specific names like `python2.7` are also recognized.
+* `#!/usr/bin/env python3` & `#!/usr/bin/python3`: The hook requires Python 3.
+  Version specific names like `python3.6` are also recognized.
+
+If no shebang is detected, or does not match the forms above, we assume that the
+hook is compatible with both Python 2 & Python 3 as if `#!/usr/bin/python` was
+used.
+
+## Hooks
+
+Here are all the points available for hooking.
+
+### pre-upload
+
+This hook runs when people run `repo upload`.
+
+The `pre-upload.py` file should be defined like:
+
+```py
+def main(project_list, worktree_list=None, **kwargs):
+    """Main function invoked directly by repo.
+
+    We must use the name "main" as that is what repo requires.
+
+    Args:
+      project_list: List of projects to run on.
+      worktree_list: A list of directories.  It should be the same length as
+          project_list, so that each entry in project_list matches with a
+          directory in worktree_list.  If None, we will attempt to calculate
+          the directories automatically.
+      kwargs: Leave this here for forward-compatibility.
+    """
+```

editor.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #
@@ -21,6 +22,7 @@ import subprocess
 import tempfile
 
 from error import EditorError
+import platform_utils
 
 class Editor(object):
   """Manages the user's preferred text editor."""
@@ -82,7 +84,12 @@ least one of these before using this command.""", file=sys.stderr)
       os.close(fd)
       fd = None
 
-      if re.compile("^.*[$ \t'].*$").match(editor):
+      if platform_utils.isWindows():
+        # Split on spaces, respecting quoted strings
+        import shlex
+        args = shlex.split(editor)
+        shell = False
+      elif re.compile("^.*[$ \t'].*$").match(editor):
         args = [editor + ' "$@"', 'sh']
         shell = True
       else:
@@ -107,4 +114,4 @@ least one of these before using this command.""", file=sys.stderr)
     finally:
       if fd:
         os.close(fd)
-      os.remove(path)
+      platform_utils.remove(path)

error.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #

event_log.py

@@ -0,0 +1,177 @@
+# -*- coding:utf-8 -*-
+#
+# Copyright (C) 2017 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+
+import json
+import multiprocessing
+
+TASK_COMMAND = 'command'
+TASK_SYNC_NETWORK = 'sync-network'
+TASK_SYNC_LOCAL = 'sync-local'
+
+class EventLog(object):
+  """Event log that records events that occurred during a repo invocation.
+
+  Events are written to the log as a consecutive JSON entries, one per line.
+  Each entry contains the following keys:
+  - id: A ('RepoOp', ID) tuple, suitable for storing in a datastore.
+        The ID is only unique for the invocation of the repo command.
+  - name: Name of the object being operated upon.
+  - task_name: The task that was performed.
+  - start: Timestamp of when the operation started.
+  - finish: Timestamp of when the operation finished.
+  - success: Boolean indicating if the operation was successful.
+  - try_count: A counter indicating the try count of this task.
+
+  Optionally:
+  - parent: A ('RepoOp', ID) tuple indicating the parent event for nested
+            events.
+
+  Valid task_names include:
+  - command: The invocation of a subcommand.
+  - sync-network: The network component of a sync command.
+  - sync-local: The local component of a sync command.
+
+  Specific tasks may include additional informational properties.
+  """
+
+  def __init__(self):
+    """Initializes the event log."""
+    self._log = []
+    self._parent = None
+
+  def Add(self, name, task_name, start, finish=None, success=None,
+          try_count=1, kind='RepoOp'):
+    """Add an event to the log.
+
+    Args:
+      name: Name of the object being operated upon.
+      task_name: A sub-task that was performed for name.
+      start: Timestamp of when the operation started.
+      finish: Timestamp of when the operation finished.
+      success: Boolean indicating if the operation was successful.
+      try_count: A counter indicating the try count of this task.
+      kind: The kind of the object for the unique identifier.
+
+    Returns:
+      A dictionary of the event added to the log.
+    """
+    event = {
+        'id': (kind, _NextEventId()),
+        'name': name,
+        'task_name': task_name,
+        'start_time': start,
+        'try': try_count,
+    }
+
+    if self._parent:
+      event['parent'] = self._parent['id']
+
+    if success is not None or finish is not None:
+        self.FinishEvent(event, finish, success)
+
+    self._log.append(event)
+    return event
+
+  def AddSync(self, project, task_name, start, finish, success):
+    """Add a event to the log for a sync command.
+
+    Args:
+      project: Project being synced.
+      task_name: A sub-task that was performed for name.
+                 One of (TASK_SYNC_NETWORK, TASK_SYNC_LOCAL)
+      start: Timestamp of when the operation started.
+      finish: Timestamp of when the operation finished.
+      success: Boolean indicating if the operation was successful.
+
+    Returns:
+      A dictionary of the event added to the log.
+    """
+    event = self.Add(project.relpath, task_name, start, finish, success)
+    if event is not None:
+      event['project'] = project.name
+      if project.revisionExpr:
+        event['revision'] = project.revisionExpr
+      if project.remote.url:
+        event['project_url'] = project.remote.url
+      if project.remote.fetchUrl:
+        event['remote_url'] = project.remote.fetchUrl
+      try:
+        event['git_hash'] = project.GetCommitRevisionId()
+      except Exception:
+        pass
+    return event
+
+  def GetStatusString(self, success):
+    """Converst a boolean success to a status string.
+
+    Args:
+      success: Boolean indicating if the operation was successful.
+
+    Returns:
+      status string.
+    """
+    return 'pass' if success else 'fail'
+
+  def FinishEvent(self, event, finish, success):
+    """Finishes an incomplete event.
+
+    Args:
+      event: An event that has been added to the log.
+      finish: Timestamp of when the operation finished.
+      success: Boolean indicating if the operation was successful.
+
+    Returns:
+      A dictionary of the event added to the log.
+    """
+    event['status'] =  self.GetStatusString(success)
+    event['finish_time'] = finish
+    return event
+
+  def SetParent(self, event):
+    """Set a parent event for all new entities.
+
+    Args:
+      event: The event to use as a parent.
+    """
+    self._parent = event
+
+  def Write(self, filename):
+    """Writes the log out to a file.
+
+    Args:
+      filename: The file to write the log to.
+    """
+    with open(filename, 'w+') as f:
+      for e in self._log:
+        json.dump(e, f, sort_keys=True)
+        f.write('\n')
+
+
+# An integer id that is unique across this invocation of the program.
+_EVENT_ID = multiprocessing.Value('i', 1)
+
+def _NextEventId():
+  """Helper function for grabbing the next unique id.
+
+  Returns:
+    A unique, to this invocation of the program, integer id.
+  """
+  with _EVENT_ID.get_lock():
+    val = _EVENT_ID.value
+    _EVENT_ID.value += 1
+  return val

git_command.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #
@@ -14,15 +15,15 @@
 # limitations under the License.
 
 from __future__ import print_function
-import fcntl
 import os
-import select
 import sys
 import subprocess
 import tempfile
 from signal import SIGTERM
+
 from error import GitError
-from trace import REPO_TRACE, IsTrace, Trace
+import platform_utils
+from repo_trace import REPO_TRACE, IsTrace, Trace
 from wrapper import Wrapper
 
 GIT = 'git'
@@ -78,33 +79,13 @@ def terminate_ssh_clients():
 
 _git_version = None
 
-class _sfd(object):
-  """select file descriptor class"""
-  def __init__(self, fd, dest, std_name):
-    assert std_name in ('stdout', 'stderr')
-    self.fd = fd
-    self.dest = dest
-    self.std_name = std_name
-  def fileno(self):
-    return self.fd.fileno()
-
 class _GitCall(object):
-  def version(self):
-    p = GitCommand(None, ['--version'], capture_stdout=True)
-    if p.Wait() == 0:
-      if hasattr(p.stdout, 'decode'):
-        return p.stdout.decode('utf-8')
-      else:
-        return p.stdout
-    return None
-
   def version_tuple(self):
     global _git_version
     if _git_version is None:
-      ver_str = git.version()
+      _git_version = Wrapper().ParseGitVersion()
-      _git_version = Wrapper().ParseGitVersion(ver_str)
       if _git_version is None:
-        print('fatal: "%s" unsupported' % ver_str, file=sys.stderr)
+        print('fatal: unable to detect git version', file=sys.stderr)
         sys.exit(1)
     return _git_version
 
@@ -117,13 +98,15 @@ class _GitCall(object):
     return fun
 git = _GitCall()
 
-def git_require(min_version, fail=False):
+def git_require(min_version, fail=False, msg=''):
   git_version = git.version_tuple()
   if min_version <= git_version:
     return True
   if fail:
     need = '.'.join(map(str, min_version))
-    print('fatal: git %s or later required' % need, file=sys.stderr)
+    if msg:
+      msg = ' for ' + msg
+    print('fatal: git %s or later required%s' % (need, msg), file=sys.stderr)
     sys.exit(1)
   return False
 
@@ -162,6 +145,7 @@ class GitCommand(object):
     if ssh_proxy:
       _setenv(env, 'REPO_SSH_SOCK', ssh_sock())
       _setenv(env, 'GIT_SSH', _ssh_proxy())
+      _setenv(env, 'GIT_SSH_VARIANT', 'ssh')
     if 'http_proxy' in env and 'darwin' == sys.platform:
       s = "'http.proxy=%s'" % (env['http_proxy'],)
       p = env.get('GIT_CONFIG_PARAMETERS')
@@ -253,19 +237,16 @@ class GitCommand(object):
 
   def _CaptureOutput(self):
     p = self.process
-    s_in = [_sfd(p.stdout, sys.stdout, 'stdout'),
+    s_in = platform_utils.FileDescriptorStreams.create()
-            _sfd(p.stderr, sys.stderr, 'stderr')]
+    s_in.add(p.stdout, sys.stdout, 'stdout')
+    s_in.add(p.stderr, sys.stderr, 'stderr')
     self.stdout = ''
     self.stderr = ''
 
-    for s in s_in:
+    while not s_in.is_done:
-      flags = fcntl.fcntl(s.fd, fcntl.F_GETFL)
+      in_ready = s_in.select()
-      fcntl.fcntl(s.fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
-
-    while s_in:
-      in_ready, _, _ = select.select(s_in, [], [])
       for s in in_ready:
-        buf = s.fd.read(4096)
+        buf = s.read()
         if not buf:
           s_in.remove(s)
           continue

git_config.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #
@@ -20,6 +21,7 @@ import errno
 import json
 import os
 import re
+import ssl
 import subprocess
 import sys
 try:
@@ -41,7 +43,8 @@ else:
 
 from signal import SIGTERM
 from error import GitError, UploadError
-from trace import Trace
+import platform_utils
+from repo_trace import Trace
 if is_python3():
   from http.client import HTTPException
 else:
@@ -50,16 +53,24 @@ else:
 from git_command import GitCommand
 from git_command import ssh_sock
 from git_command import terminate_ssh_clients
+from git_refs import R_CHANGES, R_HEADS, R_TAGS
 
-R_HEADS = 'refs/heads/'
-R_TAGS  = 'refs/tags/'
 ID_RE = re.compile(r'^[0-9a-f]{40}$')
 
 REVIEW_CACHE = dict()
 
+def IsChange(rev):
+  return rev.startswith(R_CHANGES)
+
 def IsId(rev):
   return ID_RE.match(rev)
 
+def IsTag(rev):
+  return rev.startswith(R_TAGS)
+
+def IsImmutable(rev):
+    return IsChange(rev) or IsId(rev) or IsTag(rev)
+
 def _key(name):
   parts = name.split('.')
   if len(parts) < 2:
@@ -259,7 +270,7 @@ class GitConfig(object):
     try:
       if os.path.getmtime(self._json) \
       <= os.path.getmtime(self.file):
-        os.remove(self._json)
+        platform_utils.remove(self._json)
         return None
     except OSError:
       return None
@@ -271,7 +282,7 @@ class GitConfig(object):
       finally:
         fd.close()
     except (IOError, ValueError):
-      os.remove(self._json)
+      platform_utils.remove(self._json)
       return None
 
   def _SaveJson(self, cache):
@@ -283,7 +294,7 @@ class GitConfig(object):
         fd.close()
     except (IOError, TypeError):
       if os.path.exists(self._json):
-        os.remove(self._json)
+        platform_utils.remove(self._json)
 
   def _ReadGit(self):
     """
@@ -296,8 +307,9 @@ class GitConfig(object):
     d = self._do('--null', '--list')
     if d is None:
       return c
-    for line in d.decode('utf-8').rstrip('\0').split('\0'):  # pylint: disable=W1401
+    if not is_python3():
-                                                             # Backslash is not anomalous
+      d = d.decode('utf-8')
+    for line in d.rstrip('\0').split('\0'):
       if '\n' in line:
         key, val = line.split('\n', 1)
       else:
@@ -464,9 +476,13 @@ def _open_ssh(host, port=None):
              % (host,port, str(e)), file=sys.stderr)
       return False
 
+    time.sleep(1)
+    ssh_died = (p.poll() is not None)
+    if ssh_died:
+      return False
+
     _master_processes.append(p)
     _master_keys.add(key)
-    time.sleep(1)
     return True
   finally:
     _master_keys_lock.release()
@@ -488,7 +504,7 @@ def close_ssh():
   d = ssh_sock(create=False)
   if d:
     try:
-      os.rmdir(os.path.dirname(d))
+      platform_utils.rmdir(os.path.dirname(d))
     except OSError:
       pass
 
@@ -520,7 +536,7 @@ def GetUrlCookieFile(url, quiet):
         for line in p.stdout:
           line = line.strip()
           if line.startswith(cookieprefix):
-            cookiefile = line[len(cookieprefix):]
+            cookiefile = os.path.expanduser(line[len(cookieprefix):])
           if line.startswith(proxyprefix):
             proxy = line[len(proxyprefix):]
         # Leave subprocess open, as cookie file may be transient.
@@ -539,7 +555,10 @@ def GetUrlCookieFile(url, quiet):
       if e.errno == errno.ENOENT:
         pass  # No persistent proxy.
       raise
-  yield GitConfig.ForUser().GetString('http.cookiefile'), None
+  cookiefile = GitConfig.ForUser().GetString('http.cookiefile')
+  if cookiefile:
+    cookiefile = os.path.expanduser(cookiefile)
+  yield cookiefile, None
 
 def _preconnect(url):
   m = URI_ALL.match(url)
@@ -568,6 +587,7 @@ class Remote(object):
     self._config = config
     self.name = name
     self.url = self._Get('url')
+    self.pushUrl = self._Get('pushurl')
     self.review = self._Get('review')
     self.projectname = self._Get('projectname')
     self.fetch = list(map(RefSpec.FromString,
@@ -599,7 +619,7 @@ class Remote(object):
     connectionUrl = self._InsteadOf()
     return _preconnect(connectionUrl)
 
-  def ReviewUrl(self, userEmail):
+  def ReviewUrl(self, userEmail, validate_certs):
     if self._review_url is None:
       if self.review is None:
         return None
@@ -607,7 +627,7 @@ class Remote(object):
       u = self.review
       if u.startswith('persistent-'):
         u = u[len('persistent-'):]
-      if u.split(':')[0] not in ('http', 'https', 'sso'):
+      if u.split(':')[0] not in ('http', 'https', 'sso', 'ssh'):
         u = 'http://%s' % u
       if u.endswith('/Gerrit'):
         u = u[:len(u) - len('/Gerrit')]
@@ -623,20 +643,28 @@ class Remote(object):
         host, port = os.environ['REPO_HOST_PORT_INFO'].split()
         self._review_url = self._SshReviewUrl(userEmail, host, port)
         REVIEW_CACHE[u] = self._review_url
-      elif u.startswith('sso:'):
+      elif u.startswith('sso:') or u.startswith('ssh:'):
         self._review_url = u  # Assume it's right
         REVIEW_CACHE[u] = self._review_url
+      elif 'REPO_IGNORE_SSH_INFO' in os.environ:
+        self._review_url = http_url
+        REVIEW_CACHE[u] = self._review_url
       else:
         try:
           info_url = u + 'ssh_info'
-          info = urllib.request.urlopen(info_url).read()
+          if not validate_certs:
-          if info == 'NOT_AVAILABLE' or '<' in info:
+              context = ssl._create_unverified_context()
+              info = urllib.request.urlopen(info_url, context=context).read()
+          else:
+              info = urllib.request.urlopen(info_url).read()
+          if info == b'NOT_AVAILABLE' or b'<' in info:
             # If `info` contains '<', we assume the server gave us some sort
             # of HTML response back, like maybe a login page.
             #
             # Assume HTTP if SSH is not enabled or ssh_info doesn't look right.
             self._review_url = http_url
           else:
+            info = info.decode('utf-8')
             host, port = info.split()
             self._review_url = self._SshReviewUrl(userEmail, host, port)
         except urllib.error.HTTPError as e:
@@ -671,7 +699,8 @@ class Remote(object):
     if not rev.startswith(R_HEADS):
       return rev
 
-    raise GitError('remote %s does not have %s' % (self.name, rev))
+    raise GitError('%s: remote %s does not have %s' %
+                   (self.projectname, self.name, rev))
 
   def WritesTo(self, ref):
     """True if the remote stores to the tracking ref.
@@ -694,6 +723,10 @@ class Remote(object):
     """Save this remote to the configuration.
     """
     self._Set('url', self.url)
+    if self.pushUrl is not None:
+      self._Set('pushurl', self.pushUrl + '/' + self.projectname)
+    else:
+      self._Set('pushurl', self.pushUrl)
     self._Set('review', self.review)
     self._Set('projectname', self.projectname)
     self._Set('fetch', list(map(str, self.fetch)))

git_refs.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2009 The Android Open Source Project
 #
@@ -14,13 +15,15 @@
 # limitations under the License.
 
 import os
-from trace import Trace
+from repo_trace import Trace
+import platform_utils
 
-HEAD    = 'HEAD'
+HEAD      = 'HEAD'
-R_HEADS = 'refs/heads/'
+R_CHANGES = 'refs/changes/'
-R_TAGS  = 'refs/tags/'
+R_HEADS   = 'refs/heads/'
-R_PUB   = 'refs/published/'
+R_TAGS    = 'refs/tags/'
-R_M     = 'refs/remotes/m/'
+R_PUB     = 'refs/published/'
+R_M       = 'refs/remotes/m/'
 
 
 class GitRefs(object):
@@ -126,9 +129,9 @@ class GitRefs(object):
 
   def _ReadLoose(self, prefix):
     base = os.path.join(self._gitdir, prefix)
-    for name in os.listdir(base):
+    for name in platform_utils.listdir(base):
       p = os.path.join(base, name)
-      if os.path.isdir(p):
+      if platform_utils.isdir(p):
         self._mtime[prefix] = os.path.getmtime(base)
         self._ReadLoose(prefix + name + '/')
       elif name.endswith('.lock'):
@@ -138,7 +141,7 @@ class GitRefs(object):
 
   def _ReadLoose1(self, path, name):
     try:
-      fd = open(path, 'rb')
+      fd = open(path)
     except IOError:
       return
 

git_ssh

@@ -1,2 +1,17 @@
 #!/bin/sh
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 exec ssh -o "ControlMaster no" -o "ControlPath $REPO_SSH_SOCK" "$@"

gitc_utils.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2015 The Android Open Source Project
 #
@@ -24,7 +25,9 @@ import git_command
 import git_config
 import wrapper
 
-NUM_BATCH_RETRIEVE_REVISIONID = 300
+from error import ManifestParseError
+
+NUM_BATCH_RETRIEVE_REVISIONID = 32
 
 def get_gitc_manifest_dir():
   return wrapper.Wrapper().get_gitc_manifest_dir()
@@ -54,7 +57,11 @@ def _set_project_revisions(projects):
     if gitcmd.Wait():
       print('FATAL: Failed to retrieve revisionExpr for %s' % proj)
       sys.exit(1)
-    proj.revisionExpr = gitcmd.stdout.split('\t')[0]
+    revisionExpr = gitcmd.stdout.split('\t')[0]
+    if not revisionExpr:
+      raise ManifestParseError('Invalid SHA-1 revision project %s (%s)' %
+                               (proj.remote.url, proj.revisionExpr))
+    proj.revisionExpr = revisionExpr
 
 def _manifest_groups(manifest):
   """Returns the manifest group string that should be synced
@@ -81,7 +88,7 @@ def generate_gitc_manifest(gitc_manifest, manifest, paths=None):
   print('Generating GITC Manifest by fetching revision SHAs for each '
         'project.')
   if paths is None:
-    paths = manifest.paths.keys()
+    paths = list(manifest.paths.keys())
 
   groups = [x for x in re.split(r'[,\s]+', _manifest_groups(manifest)) if x]
 
@@ -90,7 +97,7 @@ def generate_gitc_manifest(gitc_manifest, manifest, paths=None):
   projects = [p for p in projects if p.MatchesGroups(groups)]
 
   if gitc_manifest is not None:
-    for path, proj in manifest.paths.iteritems():
+    for path, proj in manifest.paths.items():
       if not proj.MatchesGroups(groups):
         continue
 
@@ -118,7 +125,7 @@ def generate_gitc_manifest(gitc_manifest, manifest, paths=None):
     index += NUM_BATCH_RETRIEVE_REVISIONID
 
   if gitc_manifest is not None:
-    for path, proj in gitc_manifest.paths.iteritems():
+    for path, proj in gitc_manifest.paths.items():
       if proj.old_revision and path in paths:
         # If we updated a project that has been started, keep the old-revision
         # updated.
@@ -127,7 +134,7 @@ def generate_gitc_manifest(gitc_manifest, manifest, paths=None):
         repo_proj.revisionExpr = None
 
   # Convert URLs from relative to absolute.
-  for name, remote in manifest.remotes.iteritems():
+  for _name, remote in manifest.remotes.items():
     remote.fetchUrl = remote.resolvedFetchUrl
 
   # Save the manifest.

hooks/commit-msg

@@ -1,6 +1,7 @@
 #!/bin/sh
+# From Gerrit Code Review 2.14.6
 #
-# Part of Gerrit Code Review (http://code.google.com/p/gerrit/)
+# Part of Gerrit Code Review (https://www.gerritcodereview.com/)
 #
 # Copyright (C) 2009 The Android Open Source Project
 #
@@ -19,7 +20,7 @@
 
 unset GREP_OPTIONS
 
-CHANGE_ID_AFTER="Bug|Issue"
+CHANGE_ID_AFTER="Bug|Depends-On|Issue|Test|Feature|Fixes|Fixed"
 MSG="$1"
 
 # Check for, and add if missing, a unique Change-Id
@@ -38,6 +39,12 @@ add_ChangeId() {
 		return
 	fi
 
+	# Do not add Change-Id to temp commits
+	if echo "$clean_message" | head -1 | grep -q '^\(fixup\|squash\)!'
+	then
+		return
+	fi
+
 	if test "false" = "`git config --bool --get gerrit.createChangeId`"
 	then
 		return
@@ -57,6 +64,10 @@ add_ChangeId() {
 		AWK=/usr/xpg4/bin/awk
 	fi
 
+	# Get core.commentChar from git config or use default symbol
+	commentChar=`git config --get core.commentChar`
+	commentChar=${commentChar:-#}
+
 	# How this works:
 	# - parse the commit message as (textLine+ blankLine*)*
 	# - assume textLine+ to be a footer until proven otherwise
@@ -75,8 +86,8 @@ add_ChangeId() {
 		blankLines = 0
 	}
 
-	# Skip lines starting with "#" without any spaces before it.
+	# Skip lines starting with commentChar without any spaces before it.
-	/^#/ { next }
+	/^'"$commentChar"'/ { next }
 
 	# Skip the line starting with the diff command and everything after it,
 	# up to the end of the file, assuming it is only patch data.

hooks/pre-auto-gc

@@ -1,9 +1,9 @@
 #!/bin/sh
 #
 # An example hook script to verify if you are on battery, in case you
-# are running Linux or OS X. Called by git-gc --auto with no arguments.
+# are running Windows, Linux or OS X. Called by git-gc --auto with no
-# The hook should exit with non-zero status after issuing an appropriate
+# arguments. The hook should exit with non-zero status after issuing an
-# message if it wants to stop the auto repacking.
+# appropriate message if it wants to stop the auto repacking.
 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -19,7 +19,17 @@
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
-if test -x /sbin/on_ac_power && /sbin/on_ac_power
+if uname -s | grep -q "_NT-"
+then
+	if test -x $SYSTEMROOT/System32/Wbem/wmic
+	then
+		STATUS=$(wmic path win32_battery get batterystatus /format:list | tr -d '\r\n')
+		[ "$STATUS" = "BatteryStatus=2" ] && exit 0 || exit 1
+	fi
+	exit 0
+fi
+
+if test -x /sbin/on_ac_power && (/sbin/on_ac_power;test $? -ne 1)
 then
 	exit 0
 elif test "$(cat /sys/class/power_supply/AC/online 2>/dev/null)" = 1

main.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #
@@ -14,6 +15,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+"""The repo tool.
+
+People shouldn't run this directly; instead, they should use the `repo` wrapper
+which takes care of execing this entry point.
+"""
+
 from __future__ import print_function
 import getpass
 import imp
@@ -37,7 +44,8 @@ except ImportError:
   kerberos = None
 
 from color import SetDefaultColoring
-from trace import SetTrace
+import event_log
+from repo_trace import SetTrace
 from git_command import git, GitCommand
 from git_config import init_ssh, close_ssh
 from command import InteractiveCommand
@@ -54,15 +62,13 @@ from error import NoSuchProjectError
 from error import RepoChangedException
 import gitc_utils
 from manifest_xml import GitcManifest, XmlManifest
-from pager import RunPager
+from pager import RunPager, TerminatePager
 from wrapper import WrapperPath, Wrapper
 
 from subcmds import all_commands
 
 if not is_python3():
-  # pylint:disable=W0622
   input = raw_input
-  # pylint:enable=W0622
 
 global_options = optparse.OptionParser(
                  usage="repo [-p|--paginate|--no-pager] COMMAND [ARGS]"
@@ -78,13 +84,19 @@ global_options.add_option('--color',
                           help='control color usage: auto, always, never')
 global_options.add_option('--trace',
                           dest='trace', action='store_true',
-                          help='trace git command execution')
+                          help='trace git command execution (REPO_TRACE=1)')
+global_options.add_option('--trace-python',
+                          dest='trace_python', action='store_true',
+                          help='trace python command execution')
 global_options.add_option('--time',
                           dest='time', action='store_true',
                           help='time repo command execution')
 global_options.add_option('--version',
                           dest='show_version', action='store_true',
                           help='display this version of repo')
+global_options.add_option('--event-log',
+                          dest='event_log', action='store',
+                          help='filename of event log to append timeline to')
 
 class _Repo(object):
   def __init__(self, repodir):
@@ -93,8 +105,8 @@ class _Repo(object):
     # add 'branch' as an alias for 'branches'
     all_commands['branch'] = all_commands['branches']
 
-  def _Run(self, argv):
+  def _ParseArgs(self, argv):
-    result = 0
+    """Parse the main `repo` command line options."""
     name = None
     glob = []
 
@@ -111,6 +123,12 @@ class _Repo(object):
       argv = []
     gopts, _gargs = global_options.parse_args(glob)
 
+    return (name, gopts, argv)
+
+  def _Run(self, name, gopts, argv):
+    """Execute the requested subcommand."""
+    result = 0
+
     if gopts.trace:
       SetTrace()
     if gopts.show_version:
@@ -176,7 +194,10 @@ class _Repo(object):
         RunPager(config)
 
     start = time.time()
+    cmd_event = cmd.event_log.Add(name, event_log.TASK_COMMAND, start)
+    cmd.event_log.SetParent(cmd_event)
     try:
+      cmd.ValidateOptions(copts, cargs)
       result = cmd.Execute(copts, cargs)
     except (DownloadError, ManifestInvalidRevisionError,
         NoManifestException) as e:
@@ -198,8 +219,13 @@ class _Repo(object):
       else:
         print('error: project group must be enabled for the project in the current directory', file=sys.stderr)
       result = 1
+    except SystemExit as e:
+      if e.code:
+        result = e.code
+      raise
     finally:
-      elapsed = time.time() - start
+      finish = time.time()
+      elapsed = finish - start
       hours, remainder = divmod(elapsed, 3600)
       minutes, seconds = divmod(remainder, 60)
       if gopts.time:
@@ -209,6 +235,12 @@ class _Repo(object):
           print('real\t%dh%dm%.3fs' % (hours, minutes, seconds),
                 file=sys.stderr)
 
+      cmd.event_log.FinishEvent(cmd_event, finish,
+                                result is None or result == 0)
+      if gopts.event_log:
+        cmd.event_log.Write(os.path.abspath(
+                            os.path.expanduser(gopts.event_log)))
+
     return result
 
 
@@ -301,7 +333,7 @@ def _UserAgent():
     _user_agent = 'git-repo/%s (%s) git/%s Python/%d.%d.%d' % (
       repo_version,
       os_name,
-      '.'.join(map(str, git.version_tuple())),
+      git.version_tuple().full,
       py_version[0], py_version[1], py_version[2])
   return _user_agent
 
@@ -504,7 +536,15 @@ def _Main(argv):
     try:
       init_ssh()
       init_http()
-      result = repo._Run(argv) or 0
+      name, gopts, argv = repo._ParseArgs(argv)
+      run = lambda: repo._Run(name, gopts, argv) or 0
+      if gopts.trace_python:
+        import trace
+        tracer = trace.Trace(count=False, trace=True, timing=True,
+                             ignoredirs=set(sys.path[1:]))
+        result = tracer.runfunc(run)
+      else:
+        result = run()
     finally:
       close_ssh()
   except KeyboardInterrupt:
@@ -525,6 +565,7 @@ def _Main(argv):
       print('fatal: %s' % e, file=sys.stderr)
       result = 128
 
+  TerminatePager()
   sys.exit(result)
 
 if __name__ == '__main__':

manifest_xml.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #
@@ -32,6 +33,7 @@ else:
 import gitc_utils
 from git_config import GitConfig
 from git_refs import R_HEADS, HEAD
+import platform_utils
 from project import RemoteSpec, Project, MetaProject
 from error import ManifestParseError, ManifestInvalidRevisionError
 
@@ -40,18 +42,30 @@ LOCAL_MANIFEST_NAME = 'local_manifest.xml'
 LOCAL_MANIFESTS_DIR_NAME = 'local_manifests'
 
 # urljoin gets confused if the scheme is not known.
-urllib.parse.uses_relative.extend(['ssh', 'git', 'persistent-https', 'rpc'])
+urllib.parse.uses_relative.extend([
-urllib.parse.uses_netloc.extend(['ssh', 'git', 'persistent-https', 'rpc'])
+    'ssh',
+    'git',
+    'persistent-https',
+    'sso',
+    'rpc'])
+urllib.parse.uses_netloc.extend([
+    'ssh',
+    'git',
+    'persistent-https',
+    'sso',
+    'rpc'])
 
 class _Default(object):
   """Project defaults within the manifest."""
 
   revisionExpr = None
   destBranchExpr = None
+  upstreamExpr = None
   remote = None
   sync_j = 1
   sync_c = False
   sync_s = False
+  sync_tags = True
 
   def __eq__(self, other):
     return self.__dict__ == other.__dict__
@@ -64,11 +78,13 @@ class _XmlRemote(object):
                name,
                alias=None,
                fetch=None,
+               pushUrl=None,
                manifestUrl=None,
                review=None,
                revision=None):
     self.name = name
     self.fetchUrl = fetch
+    self.pushUrl = pushUrl
     self.manifestUrl = manifestUrl
     self.remoteAlias = alias
     self.reviewUrl = review
@@ -98,11 +114,17 @@ class _XmlRemote(object):
     return url
 
   def ToRemoteSpec(self, projectName):
-    url = self.resolvedFetchUrl.rstrip('/') + '/' + projectName
+    fetchUrl = self.resolvedFetchUrl.rstrip('/')
+    url = fetchUrl + '/' + projectName
     remoteName = self.name
     if self.remoteAlias:
       remoteName = self.remoteAlias
-    return RemoteSpec(remoteName, url, self.reviewUrl)
+    return RemoteSpec(remoteName,
+                      url=url,
+                      pushUrl=self.pushUrl,
+                      review=self.reviewUrl,
+                      orig_name=self.name,
+                      fetchUrl=self.fetchUrl)
 
 class XmlManifest(object):
   """manages the repo configuration file"""
@@ -114,6 +136,7 @@ class XmlManifest(object):
     self.globalConfig = GitConfig.ForUser()
     self.localManifestWarning = False
     self.isGitcClient = False
+    self._load_local_manifests = True
 
     self.repoProject = MetaProject(self, 'repo',
       gitdir   = os.path.join(repodir, 'repo/.git'),
@@ -125,15 +148,26 @@ class XmlManifest(object):
 
     self._Unload()
 
-  def Override(self, name):
+  def Override(self, name, load_local_manifests=True):
     """Use a different manifest, just for the current instantiation.
     """
-    path = os.path.join(self.manifestProject.worktree, name)
+    path = None
-    if not os.path.isfile(path):
+
-      raise ManifestParseError('manifest %s not found' % name)
+    # Look for a manifest by path in the filesystem (including the cwd).
+    if not load_local_manifests:
+      local_path = os.path.abspath(name)
+      if os.path.isfile(local_path):
+        path = local_path
+
+    # Look for manifests by name from the manifests repo.
+    if path is None:
+      path = os.path.join(self.manifestProject.worktree, name)
+      if not os.path.isfile(path):
+        raise ManifestParseError('manifest %s not found' % name)
 
     old = self.manifestFile
     try:
+      self._load_local_manifests = load_local_manifests
       self.manifestFile = path
       self._Unload()
       self._Load()
@@ -147,8 +181,8 @@ class XmlManifest(object):
 
     try:
       if os.path.lexists(self.manifestFile):
-        os.remove(self.manifestFile)
+        platform_utils.remove(self.manifestFile)
-      os.symlink('manifests/%s' % name, self.manifestFile)
+      platform_utils.symlink(os.path.join('manifests', name), self.manifestFile)
     except OSError as e:
       raise ManifestParseError('cannot link manifest %s: %s' % (name, str(e)))
 
@@ -157,6 +191,8 @@ class XmlManifest(object):
     root.appendChild(e)
     e.setAttribute('name', r.name)
     e.setAttribute('fetch', r.fetchUrl)
+    if r.pushUrl is not None:
+      e.setAttribute('pushurl', r.pushUrl)
     if r.remoteAlias is not None:
       e.setAttribute('alias', r.remoteAlias)
     if r.reviewUrl is not None:
@@ -208,6 +244,9 @@ class XmlManifest(object):
     if d.destBranchExpr:
       have_default = True
       e.setAttribute('dest-branch', d.destBranchExpr)
+    if d.upstreamExpr:
+      have_default = True
+      e.setAttribute('upstream', d.upstreamExpr)
     if d.sync_j > 1:
       have_default = True
       e.setAttribute('sync-j', '%d' % d.sync_j)
@@ -217,6 +256,9 @@ class XmlManifest(object):
     if d.sync_s:
       have_default = True
       e.setAttribute('sync-s', 'true')
+    if not d.sync_tags:
+      have_default = True
+      e.setAttribute('sync-tags', 'false')
     if have_default:
       root.appendChild(e)
       root.appendChild(doc.createTextNode(''))
@@ -249,9 +291,9 @@ class XmlManifest(object):
         e.setAttribute('path', relpath)
       remoteName = None
       if d.remote:
-        remoteName = d.remote.remoteAlias or d.remote.name
+        remoteName = d.remote.name
-      if not d.remote or p.remote.name != remoteName:
+      if not d.remote or p.remote.orig_name != remoteName:
-        remoteName = p.remote.name
+        remoteName = p.remote.orig_name
         e.setAttribute('remote', remoteName)
       if peg_rev:
         if self.IsMirror:
@@ -267,10 +309,11 @@ class XmlManifest(object):
             # isn't our value
             e.setAttribute('upstream', p.revisionExpr)
       else:
-        revision = self.remotes[remoteName].revision or d.revisionExpr
+        revision = self.remotes[p.remote.orig_name].revision or d.revisionExpr
         if not revision or revision != p.revisionExpr:
           e.setAttribute('revision', p.revisionExpr)
-        if p.upstream and p.upstream != p.revisionExpr:
+        if (p.upstream and (p.upstream != p.revisionExpr or
+                            p.upstream != d.upstreamExpr)):
           e.setAttribute('upstream', p.upstream)
 
       if p.dest_branch and p.dest_branch != d.destBranchExpr:
@@ -306,6 +349,9 @@ class XmlManifest(object):
       if p.sync_s:
         e.setAttribute('sync-s', 'true')
 
+      if not p.sync_tags:
+        e.setAttribute('sync-tags', 'false')
+
       if p.clone_depth:
         e.setAttribute('clone-depth', str(p.clone_depth))
 
@@ -367,6 +413,12 @@ class XmlManifest(object):
     self._Load()
     return self._manifest_server
 
+  @property
+  def CloneFilter(self):
+    if self.manifestProject.config.GetBoolean('repo.partialclone'):
+      return self.manifestProject.config.GetString('repo.clonefilter')
+    return None
+
   @property
   def IsMirror(self):
     return self.manifestProject.config.GetBoolean('repo.mirror')
@@ -375,6 +427,10 @@ class XmlManifest(object):
   def IsArchive(self):
     return self.manifestProject.config.GetBoolean('repo.archive')
 
+  @property
+  def HasSubmodules(self):
+    return self.manifestProject.config.GetBoolean('repo.submodules')
+
   def _Unload(self):
     self._loaded = False
     self._projects = {}
@@ -398,23 +454,26 @@ class XmlManifest(object):
       nodes.append(self._ParseManifestXml(self.manifestFile,
                                           self.manifestProject.worktree))
 
-      local = os.path.join(self.repodir, LOCAL_MANIFEST_NAME)
+      if self._load_local_manifests:
-      if os.path.exists(local):
+        local = os.path.join(self.repodir, LOCAL_MANIFEST_NAME)
-        if not self.localManifestWarning:
+        if os.path.exists(local):
-          self.localManifestWarning = True
+          if not self.localManifestWarning:
-          print('warning: %s is deprecated; put local manifests in `%s` instead'
+            self.localManifestWarning = True
-                % (LOCAL_MANIFEST_NAME, os.path.join(self.repodir, LOCAL_MANIFESTS_DIR_NAME)),
+            print('warning: %s is deprecated; put local manifests '
-                file=sys.stderr)
+                  'in `%s` instead' % (LOCAL_MANIFEST_NAME,
-        nodes.append(self._ParseManifestXml(local, self.repodir))
+                  os.path.join(self.repodir, LOCAL_MANIFESTS_DIR_NAME)),
+                  file=sys.stderr)
+          nodes.append(self._ParseManifestXml(local, self.repodir))
 
-      local_dir = os.path.abspath(os.path.join(self.repodir, LOCAL_MANIFESTS_DIR_NAME))
+        local_dir = os.path.abspath(os.path.join(self.repodir,
-      try:
+                                    LOCAL_MANIFESTS_DIR_NAME))
-        for local_file in sorted(os.listdir(local_dir)):
+        try:
-          if local_file.endswith('.xml'):
+          for local_file in sorted(platform_utils.listdir(local_dir)):
-            local = os.path.join(local_dir, local_file)
+            if local_file.endswith('.xml'):
-            nodes.append(self._ParseManifestXml(local, self.repodir))
+              local = os.path.join(local_dir, local_file)
-      except OSError:
+              nodes.append(self._ParseManifestXml(local, self.repodir))
-        pass
+        except OSError:
+          pass
 
       try:
         self._ParseManifest(nodes)
@@ -446,8 +505,7 @@ class XmlManifest(object):
       raise ManifestParseError("no <manifest> in %s" % (path,))
 
     nodes = []
-    for node in manifest.childNodes:  # pylint:disable=W0631
+    for node in manifest.childNodes:
-                                      # We only get here if manifest is initialised
       if node.nodeName == 'include':
         name = self._reqatt(node, 'name')
         fp = os.path.join(include_root, name)
@@ -462,7 +520,7 @@ class XmlManifest(object):
           raise
         except Exception as e:
           raise ManifestParseError(
-              "failed parsing included manifest %s: %s", (name, e))
+              "failed parsing included manifest %s: %s" % (name, e))
       else:
         nodes.append(node)
     return nodes
@@ -539,12 +597,15 @@ class XmlManifest(object):
         groups = node.getAttribute('groups')
         if groups:
           groups = self._ParseGroups(groups)
+        revision = node.getAttribute('revision')
 
         for p in self._projects[name]:
           if path and p.relpath != path:
             continue
           if groups:
             p.groups.extend(groups)
+          if revision:
+            p.revisionExpr = revision
       if node.nodeName == 'repo-hooks':
         # Get the name of the project and the (space-separated) list of enabled.
         repo_hooks_project = self._reqatt(node, 'in-project')
@@ -636,6 +697,9 @@ class XmlManifest(object):
     if alias == '':
       alias = None
     fetch = self._reqatt(node, 'fetch')
+    pushUrl = node.getAttribute('pushurl')
+    if pushUrl == '':
+      pushUrl = None
     review = node.getAttribute('review')
     if review == '':
       review = None
@@ -643,7 +707,7 @@ class XmlManifest(object):
     if revision == '':
       revision = None
     manifestUrl = self.manifestProject.config.GetString('remote.origin.url')
-    return _XmlRemote(name, alias, fetch, manifestUrl, review, revision)
+    return _XmlRemote(name, alias, fetch, pushUrl, manifestUrl, review, revision)
 
   def _ParseDefault(self, node):
     """
@@ -656,6 +720,7 @@ class XmlManifest(object):
       d.revisionExpr = None
 
     d.destBranchExpr = node.getAttribute('dest-branch') or None
+    d.upstreamExpr = node.getAttribute('upstream') or None
 
     sync_j = node.getAttribute('sync-j')
     if sync_j == '' or sync_j is None:
@@ -674,6 +739,12 @@ class XmlManifest(object):
       d.sync_s = False
     else:
       d.sync_s = sync_s.lower() in ("yes", "true", "1")
+
+    sync_tags = node.getAttribute('sync-tags')
+    if not sync_tags:
+      d.sync_tags = True
+    else:
+      d.sync_tags = sync_tags.lower() in ("yes", "true", "1")
     return d
 
   def _ParseNotice(self, node):
@@ -768,6 +839,12 @@ class XmlManifest(object):
     else:
       sync_s = sync_s.lower() in ("yes", "true", "1")
 
+    sync_tags = node.getAttribute('sync-tags')
+    if not sync_tags:
+      sync_tags = self._default.sync_tags
+    else:
+      sync_tags = sync_tags.lower() in ("yes", "true", "1")
+
     clone_depth = node.getAttribute('clone-depth')
     if clone_depth:
       try:
@@ -780,7 +857,7 @@ class XmlManifest(object):
 
     dest_branch = node.getAttribute('dest-branch') or self._default.destBranchExpr
 
-    upstream = node.getAttribute('upstream')
+    upstream = node.getAttribute('upstream') or self._default.upstreamExpr
 
     groups = ''
     if node.hasAttribute('groups'):
@@ -813,6 +890,7 @@ class XmlManifest(object):
                       groups = groups,
                       sync_c = sync_c,
                       sync_s = sync_s,
+                      sync_tags = sync_tags,
                       clone_depth = clone_depth,
                       upstream = upstream,
                       parent = parent,
@@ -969,5 +1047,5 @@ class GitcManifest(XmlManifest):
   def _output_manifest_project_extras(self, p, e):
     """Output GITC Specific Project attributes"""
     if p.old_revision:
-        e.setAttribute('old-revision', str(p.old_revision))
+      e.setAttribute('old-revision', str(p.old_revision))
 

pager.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #
@@ -16,19 +17,53 @@
 from __future__ import print_function
 import os
 import select
+import subprocess
 import sys
 
+import platform_utils
+
 active = False
+pager_process = None
+old_stdout = None
+old_stderr = None
 
 def RunPager(globalConfig):
-  global active
-
   if not os.isatty(0) or not os.isatty(1):
     return
   pager = _SelectPager(globalConfig)
   if pager == '' or pager == 'cat':
     return
 
+  if platform_utils.isWindows():
+    _PipePager(pager);
+  else:
+    _ForkPager(pager)
+
+def TerminatePager():
+  global pager_process, old_stdout, old_stderr
+  if pager_process:
+    sys.stdout.flush()
+    sys.stderr.flush()
+    pager_process.stdin.close()
+    pager_process.wait();
+    pager_process = None
+    # Restore initial stdout/err in case there is more output in this process
+    # after shutting down the pager process
+    sys.stdout = old_stdout
+    sys.stderr = old_stderr
+
+def _PipePager(pager):
+  global pager_process, old_stdout, old_stderr
+  assert pager_process is None, "Only one active pager process at a time"
+  # Create pager process, piping stdout/err into its stdin
+  pager_process = subprocess.Popen([pager], stdin=subprocess.PIPE, stdout=sys.stdout, stderr=sys.stderr)
+  old_stdout = sys.stdout
+  old_stderr = sys.stderr
+  sys.stdout = pager_process.stdin
+  sys.stderr = pager_process.stdin
+
+def _ForkPager(pager):
+  global active
   # This process turns into the pager; a child it forks will
   # do the real processing and output back to the pager. This
   # is necessary to keep the pager in control of the tty.

platform_utils.py

@@ -0,0 +1,415 @@
+# -*- coding:utf-8 -*-
+#
+# Copyright (C) 2016 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import errno
+import os
+import platform
+import select
+import shutil
+import stat
+
+from pyversion import is_python3
+if is_python3():
+  from queue import Queue
+else:
+  from Queue import Queue
+
+from threading import Thread
+
+
+def isWindows():
+  """ Returns True when running with the native port of Python for Windows,
+  False when running on any other platform (including the Cygwin port of
+  Python).
+  """
+  # Note: The cygwin port of Python returns "CYGWIN_NT_xxx"
+  return platform.system() == "Windows"
+
+
+class FileDescriptorStreams(object):
+  """ Platform agnostic abstraction enabling non-blocking I/O over a
+  collection of file descriptors. This abstraction is required because
+  fctnl(os.O_NONBLOCK) is not supported on Windows.
+  """
+  @classmethod
+  def create(cls):
+    """ Factory method: instantiates the concrete class according to the
+    current platform.
+    """
+    if isWindows():
+      return _FileDescriptorStreamsThreads()
+    else:
+      return _FileDescriptorStreamsNonBlocking()
+
+  def __init__(self):
+    self.streams = []
+
+  def add(self, fd, dest, std_name):
+    """ Wraps an existing file descriptor as a stream.
+    """
+    self.streams.append(self._create_stream(fd, dest, std_name))
+
+  def remove(self, stream):
+    """ Removes a stream, when done with it.
+    """
+    self.streams.remove(stream)
+
+  @property
+  def is_done(self):
+    """ Returns True when all streams have been processed.
+    """
+    return len(self.streams) == 0
+
+  def select(self):
+    """ Returns the set of streams that have data available to read.
+    The returned streams each expose a read() and a close() method.
+    When done with a stream, call the remove(stream) method.
+    """
+    raise NotImplementedError
+
+  def _create_stream(fd, dest, std_name):
+    """ Creates a new stream wrapping an existing file descriptor.
+    """
+    raise NotImplementedError
+
+
+class _FileDescriptorStreamsNonBlocking(FileDescriptorStreams):
+  """ Implementation of FileDescriptorStreams for platforms that support
+  non blocking I/O.
+  """
+  class Stream(object):
+    """ Encapsulates a file descriptor """
+    def __init__(self, fd, dest, std_name):
+      self.fd = fd
+      self.dest = dest
+      self.std_name = std_name
+      self.set_non_blocking()
+
+    def set_non_blocking(self):
+      import fcntl
+      flags = fcntl.fcntl(self.fd, fcntl.F_GETFL)
+      fcntl.fcntl(self.fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
+
+    def fileno(self):
+      return self.fd.fileno()
+
+    def read(self):
+      return self.fd.read(4096)
+
+    def close(self):
+      self.fd.close()
+
+  def _create_stream(self, fd, dest, std_name):
+    return self.Stream(fd, dest, std_name)
+
+  def select(self):
+    ready_streams, _, _ = select.select(self.streams, [], [])
+    return ready_streams
+
+
+class _FileDescriptorStreamsThreads(FileDescriptorStreams):
+  """ Implementation of FileDescriptorStreams for platforms that don't support
+  non blocking I/O. This implementation requires creating threads issuing
+  blocking read operations on file descriptors.
+  """
+  def __init__(self):
+    super(_FileDescriptorStreamsThreads, self).__init__()
+    # The queue is shared accross all threads so we can simulate the
+    # behavior of the select() function
+    self.queue = Queue(10)  # Limit incoming data from streams
+
+  def _create_stream(self, fd, dest, std_name):
+    return self.Stream(fd, dest, std_name, self.queue)
+
+  def select(self):
+    # Return only one stream at a time, as it is the most straighforward
+    # thing to do and it is compatible with the select() function.
+    item = self.queue.get()
+    stream = item.stream
+    stream.data = item.data
+    return [stream]
+
+  class QueueItem(object):
+    """ Item put in the shared queue """
+    def __init__(self, stream, data):
+      self.stream = stream
+      self.data = data
+
+  class Stream(object):
+    """ Encapsulates a file descriptor """
+    def __init__(self, fd, dest, std_name, queue):
+      self.fd = fd
+      self.dest = dest
+      self.std_name = std_name
+      self.queue = queue
+      self.data = None
+      self.thread = Thread(target=self.read_to_queue)
+      self.thread.daemon = True
+      self.thread.start()
+
+    def close(self):
+      self.fd.close()
+
+    def read(self):
+      data = self.data
+      self.data = None
+      return data
+
+    def read_to_queue(self):
+      """ The thread function: reads everything from the file descriptor into
+      the shared queue and terminates when reaching EOF.
+      """
+      for line in iter(self.fd.readline, b''):
+        self.queue.put(_FileDescriptorStreamsThreads.QueueItem(self, line))
+      self.fd.close()
+      self.queue.put(_FileDescriptorStreamsThreads.QueueItem(self, None))
+
+
+def symlink(source, link_name):
+  """Creates a symbolic link pointing to source named link_name.
+  Note: On Windows, source must exist on disk, as the implementation needs
+  to know whether to create a "File" or a "Directory" symbolic link.
+  """
+  if isWindows():
+    import platform_utils_win32
+    source = _validate_winpath(source)
+    link_name = _validate_winpath(link_name)
+    target = os.path.join(os.path.dirname(link_name), source)
+    if isdir(target):
+      platform_utils_win32.create_dirsymlink(_makelongpath(source), link_name)
+    else:
+      platform_utils_win32.create_filesymlink(_makelongpath(source), link_name)
+  else:
+    return os.symlink(source, link_name)
+
+
+def _validate_winpath(path):
+  path = os.path.normpath(path)
+  if _winpath_is_valid(path):
+    return path
+  raise ValueError("Path \"%s\" must be a relative path or an absolute "
+                   "path starting with a drive letter".format(path))
+
+
+def _winpath_is_valid(path):
+  """Windows only: returns True if path is relative (e.g. ".\\foo") or is
+  absolute including a drive letter (e.g. "c:\\foo"). Returns False if path
+  is ambiguous (e.g. "x:foo" or "\\foo").
+  """
+  assert isWindows()
+  path = os.path.normpath(path)
+  drive, tail = os.path.splitdrive(path)
+  if tail:
+    if not drive:
+      return tail[0] != os.sep  # "\\foo" is invalid
+    else:
+      return tail[0] == os.sep  # "x:foo" is invalid
+  else:
+    return not drive  # "x:" is invalid
+
+
+def _makelongpath(path):
+  """Return the input path normalized to support the Windows long path syntax
+  ("\\\\?\\" prefix) if needed, i.e. if the input path is longer than the
+  MAX_PATH limit.
+  """
+  if isWindows():
+    # Note: MAX_PATH is 260, but, for directories, the maximum value is actually 246.
+    if len(path) < 246:
+      return path
+    if path.startswith(u"\\\\?\\"):
+      return path
+    if not os.path.isabs(path):
+      return path
+    # Append prefix and ensure unicode so that the special longpath syntax
+    # is supported by underlying Win32 API calls
+    return u"\\\\?\\" + os.path.normpath(path)
+  else:
+    return path
+
+
+def rmtree(path):
+  """shutil.rmtree(path) wrapper with support for long paths on Windows.
+
+  Availability: Unix, Windows."""
+  if isWindows():
+    shutil.rmtree(_makelongpath(path), onerror=handle_rmtree_error)
+  else:
+    shutil.rmtree(path)
+
+
+def handle_rmtree_error(function, path, excinfo):
+  # Allow deleting read-only files
+  os.chmod(path, stat.S_IWRITE)
+  function(path)
+
+
+def rename(src, dst):
+  """os.rename(src, dst) wrapper with support for long paths on Windows.
+
+  Availability: Unix, Windows."""
+  if isWindows():
+    # On Windows, rename fails if destination exists, see
+    # https://docs.python.org/2/library/os.html#os.rename
+    try:
+      os.rename(_makelongpath(src), _makelongpath(dst))
+    except OSError as e:
+      if e.errno == errno.EEXIST:
+        os.remove(_makelongpath(dst))
+        os.rename(_makelongpath(src), _makelongpath(dst))
+      else:
+        raise
+  else:
+    os.rename(src, dst)
+
+
+def remove(path):
+  """Remove (delete) the file path. This is a replacement for os.remove that
+  allows deleting read-only files on Windows, with support for long paths and
+  for deleting directory symbolic links.
+
+  Availability: Unix, Windows."""
+  if isWindows():
+    longpath = _makelongpath(path)
+    try:
+      os.remove(longpath)
+    except OSError as e:
+      if e.errno == errno.EACCES:
+        os.chmod(longpath, stat.S_IWRITE)
+        # Directory symbolic links must be deleted with 'rmdir'.
+        if islink(longpath) and isdir(longpath):
+          os.rmdir(longpath)
+        else:
+          os.remove(longpath)
+      else:
+        raise
+  else:
+    os.remove(path)
+
+
+def walk(top, topdown=True, onerror=None, followlinks=False):
+  """os.walk(path) wrapper with support for long paths on Windows.
+
+  Availability: Windows, Unix.
+  """
+  if isWindows():
+    return _walk_windows_impl(top, topdown, onerror, followlinks)
+  else:
+    return os.walk(top, topdown, onerror, followlinks)
+
+
+def _walk_windows_impl(top, topdown, onerror, followlinks):
+  try:
+    names = listdir(top)
+  except Exception as err:
+    if onerror is not None:
+      onerror(err)
+    return
+
+  dirs, nondirs = [], []
+  for name in names:
+    if isdir(os.path.join(top, name)):
+      dirs.append(name)
+    else:
+      nondirs.append(name)
+
+  if topdown:
+    yield top, dirs, nondirs
+  for name in dirs:
+    new_path = os.path.join(top, name)
+    if followlinks or not islink(new_path):
+      for x in _walk_windows_impl(new_path, topdown, onerror, followlinks):
+        yield x
+  if not topdown:
+    yield top, dirs, nondirs
+
+
+def listdir(path):
+  """os.listdir(path) wrapper with support for long paths on Windows.
+
+  Availability: Windows, Unix.
+  """
+  return os.listdir(_makelongpath(path))
+
+
+def rmdir(path):
+  """os.rmdir(path) wrapper with support for long paths on Windows.
+
+  Availability: Windows, Unix.
+  """
+  os.rmdir(_makelongpath(path))
+
+
+def isdir(path):
+  """os.path.isdir(path) wrapper with support for long paths on Windows.
+
+  Availability: Windows, Unix.
+  """
+  return os.path.isdir(_makelongpath(path))
+
+
+def islink(path):
+  """os.path.islink(path) wrapper with support for long paths on Windows.
+
+  Availability: Windows, Unix.
+  """
+  if isWindows():
+    import platform_utils_win32
+    return platform_utils_win32.islink(_makelongpath(path))
+  else:
+    return os.path.islink(path)
+
+
+def readlink(path):
+  """Return a string representing the path to which the symbolic link
+  points. The result may be either an absolute or relative pathname;
+  if it is relative, it may be converted to an absolute pathname using
+  os.path.join(os.path.dirname(path), result).
+
+  Availability: Windows, Unix.
+  """
+  if isWindows():
+    import platform_utils_win32
+    return platform_utils_win32.readlink(_makelongpath(path))
+  else:
+    return os.readlink(path)
+
+
+def realpath(path):
+  """Return the canonical path of the specified filename, eliminating
+  any symbolic links encountered in the path.
+
+  Availability: Windows, Unix.
+  """
+  if isWindows():
+    current_path = os.path.abspath(path)
+    path_tail = []
+    for c in range(0, 100):  # Avoid cycles
+      if islink(current_path):
+        target = readlink(current_path)
+        current_path = os.path.join(os.path.dirname(current_path), target)
+      else:
+        basename = os.path.basename(current_path)
+        if basename == '':
+          path_tail.append(current_path)
+          break
+        path_tail.append(basename)
+        current_path = os.path.dirname(current_path)
+    path_tail.reverse()
+    result = os.path.normpath(os.path.join(*path_tail))
+    return result
+  else:
+    return os.path.realpath(path)

platform_utils_win32.py

@@ -0,0 +1,219 @@
+# -*- coding:utf-8 -*-
+#
+# Copyright (C) 2016 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import errno
+
+from ctypes import WinDLL, get_last_error, FormatError, WinError, addressof
+from ctypes import c_buffer
+from ctypes.wintypes import BOOL, BOOLEAN, LPCWSTR, DWORD, HANDLE, POINTER, c_ubyte
+from ctypes.wintypes import WCHAR, USHORT, LPVOID, Structure, Union, ULONG
+from ctypes.wintypes import byref
+
+kernel32 = WinDLL('kernel32', use_last_error=True)
+
+LPDWORD = POINTER(DWORD)
+UCHAR = c_ubyte
+
+# Win32 error codes
+ERROR_SUCCESS = 0
+ERROR_NOT_SUPPORTED = 50
+ERROR_PRIVILEGE_NOT_HELD = 1314
+
+# Win32 API entry points
+CreateSymbolicLinkW = kernel32.CreateSymbolicLinkW
+CreateSymbolicLinkW.restype = BOOLEAN
+CreateSymbolicLinkW.argtypes = (LPCWSTR,  # lpSymlinkFileName In
+                                LPCWSTR,  # lpTargetFileName In
+                                DWORD)    # dwFlags In
+
+# Symbolic link creation flags
+SYMBOLIC_LINK_FLAG_FILE = 0x00
+SYMBOLIC_LINK_FLAG_DIRECTORY = 0x01
+# symlink support for CreateSymbolicLink() starting with Windows 10 (1703, v10.0.14972)
+SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE = 0x02
+
+GetFileAttributesW = kernel32.GetFileAttributesW
+GetFileAttributesW.restype = DWORD
+GetFileAttributesW.argtypes = (LPCWSTR,)  # lpFileName In
+
+INVALID_FILE_ATTRIBUTES = 0xFFFFFFFF
+FILE_ATTRIBUTE_REPARSE_POINT = 0x00400
+
+CreateFileW = kernel32.CreateFileW
+CreateFileW.restype = HANDLE
+CreateFileW.argtypes = (LPCWSTR,  # lpFileName In
+                        DWORD,    # dwDesiredAccess In
+                        DWORD,    # dwShareMode In
+                        LPVOID,   # lpSecurityAttributes In_opt
+                        DWORD,    # dwCreationDisposition In
+                        DWORD,    # dwFlagsAndAttributes In
+                        HANDLE)   # hTemplateFile In_opt
+
+CloseHandle = kernel32.CloseHandle
+CloseHandle.restype = BOOL
+CloseHandle.argtypes = (HANDLE,)  # hObject In
+
+INVALID_HANDLE_VALUE = HANDLE(-1).value
+OPEN_EXISTING = 3
+FILE_FLAG_BACKUP_SEMANTICS = 0x02000000
+FILE_FLAG_OPEN_REPARSE_POINT = 0x00200000
+
+DeviceIoControl = kernel32.DeviceIoControl
+DeviceIoControl.restype = BOOL
+DeviceIoControl.argtypes = (HANDLE,   # hDevice In
+                            DWORD,    # dwIoControlCode In
+                            LPVOID,   # lpInBuffer In_opt
+                            DWORD,    # nInBufferSize In
+                            LPVOID,   # lpOutBuffer Out_opt
+                            DWORD,    # nOutBufferSize In
+                            LPDWORD,  # lpBytesReturned Out_opt
+                            LPVOID)   # lpOverlapped Inout_opt
+
+# Device I/O control flags and options
+FSCTL_GET_REPARSE_POINT = 0x000900A8
+IO_REPARSE_TAG_MOUNT_POINT = 0xA0000003
+IO_REPARSE_TAG_SYMLINK = 0xA000000C
+MAXIMUM_REPARSE_DATA_BUFFER_SIZE = 0x4000
+
+
+class GENERIC_REPARSE_BUFFER(Structure):
+  _fields_ = (('DataBuffer', UCHAR * 1),)
+
+
+class SYMBOLIC_LINK_REPARSE_BUFFER(Structure):
+  _fields_ = (('SubstituteNameOffset', USHORT),
+              ('SubstituteNameLength', USHORT),
+              ('PrintNameOffset', USHORT),
+              ('PrintNameLength', USHORT),
+              ('Flags', ULONG),
+              ('PathBuffer', WCHAR * 1))
+
+  @property
+  def PrintName(self):
+    arrayt = WCHAR * (self.PrintNameLength // 2)
+    offset = type(self).PathBuffer.offset + self.PrintNameOffset
+    return arrayt.from_address(addressof(self) + offset).value
+
+
+class MOUNT_POINT_REPARSE_BUFFER(Structure):
+  _fields_ = (('SubstituteNameOffset', USHORT),
+              ('SubstituteNameLength', USHORT),
+              ('PrintNameOffset', USHORT),
+              ('PrintNameLength', USHORT),
+              ('PathBuffer', WCHAR * 1))
+
+  @property
+  def PrintName(self):
+    arrayt = WCHAR * (self.PrintNameLength // 2)
+    offset = type(self).PathBuffer.offset + self.PrintNameOffset
+    return arrayt.from_address(addressof(self) + offset).value
+
+
+class REPARSE_DATA_BUFFER(Structure):
+  class REPARSE_BUFFER(Union):
+    _fields_ = (('SymbolicLinkReparseBuffer', SYMBOLIC_LINK_REPARSE_BUFFER),
+                ('MountPointReparseBuffer', MOUNT_POINT_REPARSE_BUFFER),
+                ('GenericReparseBuffer', GENERIC_REPARSE_BUFFER))
+  _fields_ = (('ReparseTag', ULONG),
+              ('ReparseDataLength', USHORT),
+              ('Reserved', USHORT),
+              ('ReparseBuffer', REPARSE_BUFFER))
+  _anonymous_ = ('ReparseBuffer',)
+
+
+def create_filesymlink(source, link_name):
+  """Creates a Windows file symbolic link source pointing to link_name."""
+  _create_symlink(source, link_name, SYMBOLIC_LINK_FLAG_FILE)
+
+
+def create_dirsymlink(source, link_name):
+  """Creates a Windows directory symbolic link source pointing to link_name.
+  """
+  _create_symlink(source, link_name, SYMBOLIC_LINK_FLAG_DIRECTORY)
+
+
+def _create_symlink(source, link_name, dwFlags):
+  if not CreateSymbolicLinkW(link_name, source, dwFlags | SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE):
+    # See https://github.com/golang/go/pull/24307/files#diff-b87bc12e4da2497308f9ef746086e4f0
+    # "the unprivileged create flag is unsupported below Windows 10 (1703, v10.0.14972).
+    # retry without it."
+    if not CreateSymbolicLinkW(link_name, source, dwFlags):
+      code = get_last_error()
+      error_desc = FormatError(code).strip()
+      if code == ERROR_PRIVILEGE_NOT_HELD:
+        raise OSError(errno.EPERM, error_desc, link_name)
+      _raise_winerror(
+          code,
+          'Error creating symbolic link \"%s\"'.format(link_name))
+
+
+def islink(path):
+  result = GetFileAttributesW(path)
+  if result == INVALID_FILE_ATTRIBUTES:
+    return False
+  return bool(result & FILE_ATTRIBUTE_REPARSE_POINT)
+
+
+def readlink(path):
+  reparse_point_handle = CreateFileW(path,
+                                     0,
+                                     0,
+                                     None,
+                                     OPEN_EXISTING,
+                                     FILE_FLAG_OPEN_REPARSE_POINT |
+                                     FILE_FLAG_BACKUP_SEMANTICS,
+                                     None)
+  if reparse_point_handle == INVALID_HANDLE_VALUE:
+    _raise_winerror(
+        get_last_error(),
+        'Error opening symblic link \"%s\"'.format(path))
+  target_buffer = c_buffer(MAXIMUM_REPARSE_DATA_BUFFER_SIZE)
+  n_bytes_returned = DWORD()
+  io_result = DeviceIoControl(reparse_point_handle,
+                              FSCTL_GET_REPARSE_POINT,
+                              None,
+                              0,
+                              target_buffer,
+                              len(target_buffer),
+                              byref(n_bytes_returned),
+                              None)
+  CloseHandle(reparse_point_handle)
+  if not io_result:
+    _raise_winerror(
+        get_last_error(),
+        'Error reading symblic link \"%s\"'.format(path))
+  rdb = REPARSE_DATA_BUFFER.from_buffer(target_buffer)
+  if rdb.ReparseTag == IO_REPARSE_TAG_SYMLINK:
+    return _preserve_encoding(path, rdb.SymbolicLinkReparseBuffer.PrintName)
+  elif rdb.ReparseTag == IO_REPARSE_TAG_MOUNT_POINT:
+    return _preserve_encoding(path, rdb.MountPointReparseBuffer.PrintName)
+  # Unsupported reparse point type
+  _raise_winerror(
+      ERROR_NOT_SUPPORTED,
+      'Error reading symblic link \"%s\"'.format(path))
+
+
+def _preserve_encoding(source, target):
+  """Ensures target is the same string type (i.e. unicode or str) as source."""
+  if isinstance(source, unicode):
+    return unicode(target)
+  return str(target)
+
+
+def _raise_winerror(code, error_desc):
+  win_error_desc = FormatError(code).strip()
+  error_desc = "%s: %s".format(error_desc, win_error_desc)
+  raise WinError(code, error_desc)

progress.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2009 The Android Open Source Project
 #
@@ -16,12 +17,18 @@
 import os
 import sys
 from time import time
-from trace import IsTrace
+from repo_trace import IsTrace
 
 _NOT_TTY = not os.isatty(2)
 
+# This will erase all content in the current line (wherever the cursor is).
+# It does not move the cursor, so this is usually followed by \r to move to
+# column 0.
+CSI_ERASE_LINE = '\x1b[2K'
+
 class Progress(object):
-  def __init__(self, title, total=0, units=''):
+  def __init__(self, title, total=0, units='', print_newline=False,
+               always_print_percentage=False):
     self._title = title
     self._total = total
     self._done = 0
@@ -29,6 +36,8 @@ class Progress(object):
     self._start = time()
     self._show = False
     self._units = units
+    self._print_newline = print_newline
+    self._always_print_percentage = always_print_percentage
 
   def update(self, inc=1):
     self._done += inc
@@ -43,20 +52,23 @@ class Progress(object):
         return
 
     if self._total <= 0:
-      sys.stderr.write('\r%s: %d, ' % (
+      sys.stderr.write('%s\r%s: %d,' % (
+        CSI_ERASE_LINE,
         self._title,
         self._done))
       sys.stderr.flush()
     else:
       p = (100 * self._done) / self._total
 
-      if self._lastp != p:
+      if self._lastp != p or self._always_print_percentage:
         self._lastp = p
-        sys.stderr.write('\r%s: %3d%% (%d%s/%d%s)  ' % (
+        sys.stderr.write('%s\r%s: %3d%% (%d%s/%d%s)%s' % (
+          CSI_ERASE_LINE,
           self._title,
           p,
           self._done, self._units,
-          self._total, self._units))
+          self._total, self._units,
+          "\n" if self._print_newline else ""))
         sys.stderr.flush()
 
   def end(self):
@@ -64,13 +76,15 @@ class Progress(object):
       return
 
     if self._total <= 0:
-      sys.stderr.write('\r%s: %d, done.  \n' % (
+      sys.stderr.write('%s\r%s: %d, done.\n' % (
+        CSI_ERASE_LINE,
         self._title,
         self._done))
       sys.stderr.flush()
     else:
       p = (100 * self._done) / self._total
-      sys.stderr.write('\r%s: %3d%% (%d%s/%d%s), done.  \n' % (
+      sys.stderr.write('%s\r%s: %3d%% (%d%s/%d%s), done.\n' % (
+        CSI_ERASE_LINE,
         self._title,
         p,
         self._done, self._units,

pyversion.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2013 The Android Open Source Project
 #

repo

@@ -1,4 +1,14 @@
 #!/usr/bin/env python
+# -*- coding:utf-8 -*-
+
+"""Repo launcher.
+
+This is a standalone tool that people may copy to anywhere in their system.
+It is used to get an initial repo client checkout, and after that it runs the
+copy of repo in the checkout.
+"""
+
+from __future__ import print_function
 
 # repo default configuration
 #
@@ -23,10 +33,13 @@ REPO_REV = 'stable'
 # limitations under the License.
 
 # increment this whenever we make important changes to this script
-VERSION = (1, 22)
+VERSION = (1, 25)
 
 # increment this if the MAINTAINER_KEYS block is modified
 KEYRING_VERSION = (1, 2)
+
+# Each individual key entry is created by using:
+# gpg --armor --export keyid
 MAINTAINER_KEYS = """
 
      Repo Maintainer <repo@android.kernel.org>
@@ -110,13 +123,15 @@ repodir = '.repo'                # name of repo's private directory
 S_repo = 'repo'                  # special repo repository
 S_manifests = 'manifests'        # special manifest repository
 REPO_MAIN = S_repo + '/main.py'  # main script
-MIN_PYTHON_VERSION = (2, 6)      # minimum supported python version
+MIN_PYTHON_VERSION = (2, 7)      # minimum supported python version
 GITC_CONFIG_FILE = '/gitc/.config'
 GITC_FS_ROOT_DIR = '/gitc/manifest-rw/'
 
 
+import collections
 import errno
 import optparse
+import platform
 import re
 import shutil
 import stat
@@ -134,19 +149,15 @@ else:
   urllib.error = urllib2
 
 
-def _print(*objects, **kwargs):
-  sep = kwargs.get('sep', ' ')
-  end = kwargs.get('end', '\n')
-  out = kwargs.get('file', sys.stdout)
-  out.write(sep.join(objects) + end)
-
-
 # Python version check
 ver = sys.version_info
 if (ver[0], ver[1]) < MIN_PYTHON_VERSION:
-  _print('error: Python version %s unsupported.\n'
+  print('error: Python version {} unsupported.\n'
-         'Please use Python 2.6 - 2.7 instead.'
+        'Please use Python {}.{} instead.'.format(
-         % sys.version.split(' ')[0], file=sys.stderr)
+            sys.version.split(' ')[0],
+            MIN_PYTHON_VERSION[0],
+            MIN_PYTHON_VERSION[1],
+        ), file=sys.stderr)
   sys.exit(1)
 
 home_dot_repo = os.path.expanduser('~/.repoconfig')
@@ -172,6 +183,9 @@ group.add_option('-b', '--manifest-branch',
 group.add_option('-m', '--manifest-name',
                  dest='manifest_name',
                  help='initial manifest file', metavar='NAME.xml')
+group.add_option('--current-branch',
+                 dest='current_branch_only', action='store_true',
+                 help='fetch only current manifest branch from server')
 group.add_option('--mirror',
                  dest='mirror', action='store_true',
                  help='create a replica of the remote repositories '
@@ -179,13 +193,26 @@ group.add_option('--mirror',
 group.add_option('--reference',
                  dest='reference',
                  help='location of mirror directory', metavar='DIR')
+group.add_option('--dissociate',
+                 dest='dissociate', action='store_true',
+                 help='dissociate from reference mirrors after clone')
 group.add_option('--depth', type='int', default=None,
                  dest='depth',
                  help='create a shallow clone with given depth; see git clone')
+group.add_option('--partial-clone', action='store_true',
+                 dest='partial_clone',
+                 help='perform partial clone (https://git-scm.com/'
+                 'docs/gitrepository-layout#_code_partialclone_code)')
+group.add_option('--clone-filter', action='store', default='blob:none',
+                 dest='clone_filter',
+                 help='filter for use with --partial-clone [default: %default]')
 group.add_option('--archive',
                  dest='archive', action='store_true',
                  help='checkout an archive instead of a git repository for '
                       'each project. See git archive.')
+group.add_option('--submodules',
+                 dest='submodules', action='store_true',
+                 help='sync any submodules associated with the manifest repo')
 group.add_option('-g', '--groups',
                  dest='groups', default='default',
                  help='restrict manifest projects to ones with specified '
@@ -196,6 +223,12 @@ group.add_option('-p', '--platform',
                  help='restrict manifest projects to ones with a specified '
                       'platform group [auto|all|none|linux|darwin|...]',
                  metavar='PLATFORM')
+group.add_option('--no-clone-bundle',
+                 dest='no_clone_bundle', action='store_true',
+                 help='disable use of /clone.bundle on HTTP/HTTPS')
+group.add_option('--no-tags',
+                 dest='no_tags', action='store_true',
+                 help="don't fetch tags in the manifest")
 
 
 # Tool
@@ -297,21 +330,21 @@ def _Init(args, gitc_init=False):
   if branch.startswith('refs/heads/'):
     branch = branch[len('refs/heads/'):]
   if branch.startswith('refs/'):
-    _print("fatal: invalid branch name '%s'" % branch, file=sys.stderr)
+    print("fatal: invalid branch name '%s'" % branch, file=sys.stderr)
     raise CloneFailure()
 
   try:
     if gitc_init:
       gitc_manifest_dir = get_gitc_manifest_dir()
       if not gitc_manifest_dir:
-        _print('fatal: GITC filesystem is not available. Exiting...',
+        print('fatal: GITC filesystem is not available. Exiting...',
-               file=sys.stderr)
+              file=sys.stderr)
         sys.exit(1)
       gitc_client = opt.gitc_client
       if not gitc_client:
         gitc_client = gitc_parse_clientdir(os.getcwd())
       if not gitc_client:
-        _print('fatal: GITC client (-c) is required.', file=sys.stderr)
+        print('fatal: GITC client (-c) is required.', file=sys.stderr)
         sys.exit(1)
       client_dir = os.path.join(gitc_manifest_dir, gitc_client)
       if not os.path.exists(client_dir):
@@ -324,8 +357,8 @@ def _Init(args, gitc_init=False):
     os.mkdir(repodir)
   except OSError as e:
     if e.errno != errno.EEXIST:
-      _print('fatal: cannot make %s directory: %s'
+      print('fatal: cannot make %s directory: %s'
-             % (repodir, e.strerror), file=sys.stderr)
+            % (repodir, e.strerror), file=sys.stderr)
       # Don't raise CloneFailure; that would delete the
       # name. Instead exit immediately.
       #
@@ -339,7 +372,7 @@ def _Init(args, gitc_init=False):
       can_verify = True
 
     dst = os.path.abspath(os.path.join(repodir, S_repo))
-    _Clone(url, dst, opt.quiet)
+    _Clone(url, dst, opt.quiet, not opt.no_clone_bundle)
 
     if can_verify and not opt.no_repo_verify:
       rev = _Verify(dst, branch, opt.quiet)
@@ -347,52 +380,75 @@ def _Init(args, gitc_init=False):
       rev = 'refs/remotes/origin/%s^0' % branch
 
     _Checkout(dst, branch, rev, opt.quiet)
+
+    if not os.path.isfile(os.path.join(dst, 'repo')):
+      print("warning: '%s' does not look like a git-repo repository, is "
+            "REPO_URL set correctly?" % url, file=sys.stderr)
+
   except CloneFailure:
     if opt.quiet:
-      _print('fatal: repo init failed; run without --quiet to see why',
+      print('fatal: repo init failed; run without --quiet to see why',
-             file=sys.stderr)
+            file=sys.stderr)
     raise
 
 
-def ParseGitVersion(ver_str):
+# The git version info broken down into components for easy analysis.
+# Similar to Python's sys.version_info.
+GitVersion = collections.namedtuple(
+    'GitVersion', ('major', 'minor', 'micro', 'full'))
+
+def ParseGitVersion(ver_str=None):
+  if ver_str is None:
+    # Load the version ourselves.
+    ver_str = _GetGitVersion()
+
   if not ver_str.startswith('git version '):
     return None
 
-  num_ver_str = ver_str[len('git version '):].strip().split('-')[0]
+  full_version = ver_str[len('git version '):].strip()
+  num_ver_str = full_version.split('-')[0]
   to_tuple = []
   for num_str in num_ver_str.split('.')[:3]:
     if num_str.isdigit():
       to_tuple.append(int(num_str))
     else:
       to_tuple.append(0)
-  return tuple(to_tuple)
+  to_tuple.append(full_version)
+  return GitVersion(*to_tuple)
 
 
-def _CheckGitVersion():
+def _GetGitVersion():
   cmd = [GIT, '--version']
   try:
     proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
   except OSError as e:
-    _print(file=sys.stderr)
+    print(file=sys.stderr)
-    _print("fatal: '%s' is not available" % GIT, file=sys.stderr)
+    print("fatal: '%s' is not available" % GIT, file=sys.stderr)
-    _print('fatal: %s' % e, file=sys.stderr)
+    print('fatal: %s' % e, file=sys.stderr)
-    _print(file=sys.stderr)
+    print(file=sys.stderr)
-    _print('Please make sure %s is installed and in your path.' % GIT,
+    print('Please make sure %s is installed and in your path.' % GIT,
-           file=sys.stderr)
+          file=sys.stderr)
-    raise CloneFailure()
+    raise
 
   ver_str = proc.stdout.read().strip()
   proc.stdout.close()
   proc.wait()
+  return ver_str.decode('utf-8')
+
+
+def _CheckGitVersion():
+  try:
+    ver_act = ParseGitVersion()
+  except OSError:
+    raise CloneFailure()
 
-  ver_act = ParseGitVersion(ver_str)
   if ver_act is None:
-    _print('error: "%s" unsupported' % ver_str, file=sys.stderr)
+    print('fatal: unable to detect git version', file=sys.stderr)
     raise CloneFailure()
 
   if ver_act < MIN_GIT_VERSION:
     need = '.'.join(map(str, MIN_GIT_VERSION))
-    _print('fatal: git %s or later required' % need, file=sys.stderr)
+    print('fatal: git %s or later required' % need, file=sys.stderr)
     raise CloneFailure()
 
 
@@ -419,20 +475,23 @@ def SetupGnuPG(quiet):
     os.mkdir(home_dot_repo)
   except OSError as e:
     if e.errno != errno.EEXIST:
-      _print('fatal: cannot make %s directory: %s'
+      print('fatal: cannot make %s directory: %s'
-             % (home_dot_repo, e.strerror), file=sys.stderr)
+            % (home_dot_repo, e.strerror), file=sys.stderr)
       sys.exit(1)
 
   try:
     os.mkdir(gpg_dir, stat.S_IRWXU)
   except OSError as e:
     if e.errno != errno.EEXIST:
-      _print('fatal: cannot make %s directory: %s' % (gpg_dir, e.strerror),
+      print('fatal: cannot make %s directory: %s' % (gpg_dir, e.strerror),
-             file=sys.stderr)
+            file=sys.stderr)
       sys.exit(1)
 
   env = os.environ.copy()
-  env['GNUPGHOME'] = gpg_dir.encode()
+  try:
+    env['GNUPGHOME'] = gpg_dir
+  except UnicodeEncodeError:
+    env['GNUPGHOME'] = gpg_dir.encode()
 
   cmd = ['gpg', '--import']
   try:
@@ -441,18 +500,18 @@ def SetupGnuPG(quiet):
                             stdin=subprocess.PIPE)
   except OSError as e:
     if not quiet:
-      _print('warning: gpg (GnuPG) is not available.', file=sys.stderr)
+      print('warning: gpg (GnuPG) is not available.', file=sys.stderr)
-      _print('warning: Installing it is strongly encouraged.', file=sys.stderr)
+      print('warning: Installing it is strongly encouraged.', file=sys.stderr)
-      _print(file=sys.stderr)
+      print(file=sys.stderr)
     return False
 
-  proc.stdin.write(MAINTAINER_KEYS)
+  proc.stdin.write(MAINTAINER_KEYS.encode('utf-8'))
   proc.stdin.close()
 
   if proc.wait() != 0:
-    _print('fatal: registering repo maintainer keys failed', file=sys.stderr)
+    print('fatal: registering repo maintainer keys failed', file=sys.stderr)
     sys.exit(1)
-  _print()
+  print()
 
   fd = open(os.path.join(home_dot_repo, 'keyring-version'), 'w')
   fd.write('.'.join(map(str, KEYRING_VERSION)) + '\n')
@@ -479,7 +538,7 @@ def _InitHttp():
       p = n.hosts[host]
       mgr.add_password(p[1], 'http://%s/' % host, p[0], p[2])
       mgr.add_password(p[1], 'https://%s/' % host, p[0], p[2])
-  except:  # pylint: disable=bare-except
+  except:
     pass
   handlers.append(urllib.request.HTTPBasicAuthHandler(mgr))
   handlers.append(urllib.request.HTTPDigestAuthHandler(mgr))
@@ -495,7 +554,7 @@ def _InitHttp():
 
 def _Fetch(url, local, src, quiet):
   if not quiet:
-    _print('Get %s' % url, file=sys.stderr)
+    print('Get %s' % url, file=sys.stderr)
 
   cmd = [GIT, 'fetch']
   if quiet:
@@ -505,7 +564,7 @@ def _Fetch(url, local, src, quiet):
     err = None
   cmd.append(src)
   cmd.append('+refs/heads/*:refs/remotes/origin/*')
-  cmd.append('refs/tags/*:refs/tags/*')
+  cmd.append('+refs/tags/*:refs/tags/*')
 
   proc = subprocess.Popen(cmd, cwd=local, stderr=err)
   if err:
@@ -525,6 +584,7 @@ def _DownloadBundle(url, local, quiet):
       cwd=local,
       stdout=subprocess.PIPE)
   for line in proc.stdout:
+    line = line.decode('utf-8')
     m = re.compile(r'^url\.(.*)\.insteadof (.*)$').match(line)
     if m:
       new_url = m.group(1)
@@ -543,21 +603,21 @@ def _DownloadBundle(url, local, quiet):
     try:
       r = urllib.request.urlopen(url)
     except urllib.error.HTTPError as e:
-      if e.code in [401, 403, 404]:
+      if e.code in [401, 403, 404, 501]:
         return False
-      _print('fatal: Cannot get %s' % url, file=sys.stderr)
+      print('fatal: Cannot get %s' % url, file=sys.stderr)
-      _print('fatal: HTTP error %s' % e.code, file=sys.stderr)
+      print('fatal: HTTP error %s' % e.code, file=sys.stderr)
       raise CloneFailure()
     except urllib.error.URLError as e:
-      _print('fatal: Cannot get %s' % url, file=sys.stderr)
+      print('fatal: Cannot get %s' % url, file=sys.stderr)
-      _print('fatal: error %s' % e.reason, file=sys.stderr)
+      print('fatal: error %s' % e.reason, file=sys.stderr)
       raise CloneFailure()
     try:
       if not quiet:
-        _print('Get %s' % url, file=sys.stderr)
+        print('Get %s' % url, file=sys.stderr)
       while True:
         buf = r.read(8192)
-        if buf == '':
+        if not buf:
           return True
         dest.write(buf)
     finally:
@@ -574,29 +634,29 @@ def _ImportBundle(local):
     os.remove(path)
 
 
-def _Clone(url, local, quiet):
+def _Clone(url, local, quiet, clone_bundle):
   """Clones a git repository to a new subdirectory of repodir
   """
   try:
     os.mkdir(local)
   except OSError as e:
-    _print('fatal: cannot make %s directory: %s' % (local, e.strerror),
+    print('fatal: cannot make %s directory: %s' % (local, e.strerror),
-           file=sys.stderr)
+          file=sys.stderr)
     raise CloneFailure()
 
   cmd = [GIT, 'init', '--quiet']
   try:
     proc = subprocess.Popen(cmd, cwd=local)
   except OSError as e:
-    _print(file=sys.stderr)
+    print(file=sys.stderr)
-    _print("fatal: '%s' is not available" % GIT, file=sys.stderr)
+    print("fatal: '%s' is not available" % GIT, file=sys.stderr)
-    _print('fatal: %s' % e, file=sys.stderr)
+    print('fatal: %s' % e, file=sys.stderr)
-    _print(file=sys.stderr)
+    print(file=sys.stderr)
-    _print('Please make sure %s is installed and in your path.' % GIT,
+    print('Please make sure %s is installed and in your path.' % GIT,
-           file=sys.stderr)
+          file=sys.stderr)
     raise CloneFailure()
   if proc.wait() != 0:
-    _print('fatal: could not create %s' % local, file=sys.stderr)
+    print('fatal: could not create %s' % local, file=sys.stderr)
     raise CloneFailure()
 
   _InitHttp()
@@ -604,7 +664,7 @@ def _Clone(url, local, quiet):
   _SetConfig(local,
              'remote.origin.fetch',
              '+refs/heads/*:refs/remotes/origin/*')
-  if _DownloadBundle(url, local, quiet):
+  if clone_bundle and _DownloadBundle(url, local, quiet):
     _ImportBundle(local)
   _Fetch(url, local, 'origin', quiet)
 
@@ -617,28 +677,31 @@ def _Verify(cwd, branch, quiet):
                           stdout=subprocess.PIPE,
                           stderr=subprocess.PIPE,
                           cwd=cwd)
-  cur = proc.stdout.read().strip()
+  cur = proc.stdout.read().strip().decode('utf-8')
   proc.stdout.close()
 
   proc.stderr.read()
   proc.stderr.close()
 
   if proc.wait() != 0 or not cur:
-    _print(file=sys.stderr)
+    print(file=sys.stderr)
-    _print("fatal: branch '%s' has not been signed" % branch, file=sys.stderr)
+    print("fatal: branch '%s' has not been signed" % branch, file=sys.stderr)
     raise CloneFailure()
 
   m = re.compile(r'^(.*)-[0-9]{1,}-g[0-9a-f]{1,}$').match(cur)
   if m:
     cur = m.group(1)
     if not quiet:
-      _print(file=sys.stderr)
+      print(file=sys.stderr)
-      _print("info: Ignoring branch '%s'; using tagged release '%s'"
+      print("info: Ignoring branch '%s'; using tagged release '%s'"
-             % (branch, cur), file=sys.stderr)
+            % (branch, cur), file=sys.stderr)
-      _print(file=sys.stderr)
+      print(file=sys.stderr)
 
   env = os.environ.copy()
-  env['GNUPGHOME'] = gpg_dir.encode()
+  try:
+    env['GNUPGHOME'] = gpg_dir
+  except UnicodeEncodeError:
+    env['GNUPGHOME'] = gpg_dir.encode()
 
   cmd = [GIT, 'tag', '-v', cur]
   proc = subprocess.Popen(cmd,
@@ -646,17 +709,17 @@ def _Verify(cwd, branch, quiet):
                           stderr=subprocess.PIPE,
                           cwd=cwd,
                           env=env)
-  out = proc.stdout.read()
+  out = proc.stdout.read().decode('utf-8')
   proc.stdout.close()
 
-  err = proc.stderr.read()
+  err = proc.stderr.read().decode('utf-8')
   proc.stderr.close()
 
   if proc.wait() != 0:
-    _print(file=sys.stderr)
+    print(file=sys.stderr)
-    _print(out, file=sys.stderr)
+    print(out, file=sys.stderr)
-    _print(err, file=sys.stderr)
+    print(err, file=sys.stderr)
-    _print(file=sys.stderr)
+    print(file=sys.stderr)
     raise CloneFailure()
   return '%s^0' % cur
 
@@ -727,7 +790,7 @@ def _Usage():
   if get_gitc_manifest_dir():
     gitc_usage = "  gitc-init Initialize a GITC Client.\n"
 
-  _print(
+  print(
       """usage: repo COMMAND [ARGS]
 
 repo is not yet installed.  Use "repo init" to install it here.
@@ -739,8 +802,8 @@ The most commonly used repo commands are:
       """  help      Display detailed help on a command
 
 For access to the full online help, install repo ("repo init").
-""", file=sys.stderr)
+""")
-  sys.exit(1)
+  sys.exit(0)
 
 
 def _Help(args):
@@ -753,23 +816,23 @@ def _Help(args):
       init_optparse.print_help()
       sys.exit(0)
     else:
-      _print("error: '%s' is not a bootstrap command.\n"
+      print("error: '%s' is not a bootstrap command.\n"
-             '        For access to online help, install repo ("repo init").'
+            '        For access to online help, install repo ("repo init").'
-             % args[0], file=sys.stderr)
+            % args[0], file=sys.stderr)
   else:
     _Usage()
   sys.exit(1)
 
 
 def _NotInstalled():
-  _print('error: repo is not installed.  Use "repo init" to install it here.',
+  print('error: repo is not installed.  Use "repo init" to install it here.',
-         file=sys.stderr)
+        file=sys.stderr)
   sys.exit(1)
 
 
 def _NoCommands(cmd):
-  _print("""error: command '%s' requires repo to be installed first.
+  print("""error: command '%s' requires repo to be installed first.
-         Use "repo init" to install it here.""" % cmd, file=sys.stderr)
+        Use "repo init" to install it here.""" % cmd, file=sys.stderr)
   sys.exit(1)
 
 
@@ -799,14 +862,14 @@ def _SetDefaultsTo(gitdir):
                            'HEAD'],
                           stdout=subprocess.PIPE,
                           stderr=subprocess.PIPE)
-  REPO_REV = proc.stdout.read().strip()
+  REPO_REV = proc.stdout.read().strip().decode('utf-8')
   proc.stdout.close()
 
   proc.stderr.read()
   proc.stderr.close()
 
   if proc.wait() != 0:
-    _print('fatal: %s has no current branch' % gitdir, file=sys.stderr)
+    print('fatal: %s has no current branch' % gitdir, file=sys.stderr)
     sys.exit(1)
 
 
@@ -823,10 +886,10 @@ def main(orig_args):
 
   cwd = os.getcwd()
   if get_gitc_manifest_dir() and cwd.startswith(get_gitc_manifest_dir()):
-    _print('error: repo cannot be used in the GITC local manifest directory.'
+    print('error: repo cannot be used in the GITC local manifest directory.'
-           '\nIf you want to work on this GITC client please rerun this '
+          '\nIf you want to work on this GITC client please rerun this '
-           'command from the corresponding client under /gitc/',
+          'command from the corresponding client under /gitc/',
-           file=sys.stderr)
+          file=sys.stderr)
     sys.exit(1)
   if not repo_main:
     if opt.help:
@@ -841,7 +904,10 @@ def main(orig_args):
       try:
         _Init(args, gitc_init=(cmd == 'gitc-init'))
       except CloneFailure:
-        shutil.rmtree(os.path.join(repodir, S_repo), ignore_errors=True)
+        path = os.path.join(repodir, S_repo)
+        print("fatal: cloning the git-repo repository failed, will remove "
+              "'%s' " % path, file=sys.stderr)
+        shutil.rmtree(path, ignore_errors=True)
         sys.exit(1)
       repo_main, rel_repo_dir = _FindRepo()
     else:
@@ -859,16 +925,15 @@ def main(orig_args):
   me.extend(orig_args)
   me.extend(extra_args)
   try:
-    os.execv(sys.executable, me)
+    if platform.system() == "Windows":
+      sys.exit(subprocess.call(me))
+    else:
+      os.execv(sys.executable, me)
   except OSError as e:
-    _print("fatal: unable to start %s" % repo_main, file=sys.stderr)
+    print("fatal: unable to start %s" % repo_main, file=sys.stderr)
-    _print("fatal: %s" % e, file=sys.stderr)
+    print("fatal: %s" % e, file=sys.stderr)
     sys.exit(148)
 
 
 if __name__ == '__main__':
-  if ver[0] == 3:
-    _print('warning: Python 3 support is currently experimental. YMMV.\n'
-           'Please use Python 2.6 - 2.7 instead.',
-           file=sys.stderr)
   main(sys.argv[1:])

repo_trace.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #
@@ -13,15 +14,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+"""Logic for tracing repo interactions.
+
+Activated via `repo --trace ...` or `REPO_TRACE=1 repo ...`.
+"""
+
 from __future__ import print_function
 import sys
 import os
+
+# Env var to implicitly turn on tracing.
 REPO_TRACE = 'REPO_TRACE'
 
-try:
+_TRACE = os.environ.get(REPO_TRACE) == '1'
-  _TRACE = os.environ[REPO_TRACE] == '1'
-except KeyError:
-  _TRACE = False
 
 def IsTrace():
   return _TRACE

run_tests

@@ -0,0 +1,54 @@
+#!/usr/bin/python
+# -*- coding:utf-8 -*-
+# Copyright 2019 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Wrapper to run pytest with the right settings."""
+
+from __future__ import print_function
+
+import errno
+import os
+import subprocess
+import sys
+
+
+def run_pytest(cmd, argv):
+  """Run the unittests via |cmd|."""
+  try:
+    subprocess.check_call([cmd] + argv)
+    return 0
+  except OSError as e:
+    if e.errno == errno.ENOENT:
+      print('%s: unable to run `%s`: %s' % (__file__, cmd, e), file=sys.stderr)
+      print('%s: Try installing pytest: sudo apt-get install python-pytest' %
+            (__file__,), file=sys.stderr)
+      return 1
+    else:
+      raise
+
+
+def main(argv):
+  """The main entry."""
+  # Add the repo tree to PYTHONPATH as the tests expect to be able to import
+  # modules directly.
+  topdir = os.path.dirname(os.path.realpath(__file__))
+  pythonpath = os.environ.get('PYTHONPATH', '')
+  os.environ['PYTHONPATH'] = '%s:%s' % (topdir, pythonpath)
+
+  return run_pytest('pytest', argv)
+
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv[1:]))

subcmds/__init__.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #

subcmds/abandon.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #
@@ -16,6 +17,7 @@
 from __future__ import print_function
 import sys
 from command import Command
+from collections import defaultdict
 from git_command import git
 from progress import Progress
 
@@ -23,49 +25,75 @@ class Abandon(Command):
   common = True
   helpSummary = "Permanently abandon a development branch"
   helpUsage = """
-%prog <branchname> [<project>...]
+%prog [--all | <branchname>] [<project>...]
 
 This subcommand permanently abandons a development branch by
 deleting it (and all its history) from your local repository.
 
 It is equivalent to "git branch -D <branchname>".
 """
+  def _Options(self, p):
+    p.add_option('--all',
+                 dest='all', action='store_true',
+                 help='delete all branches in all projects')
 
-  def Execute(self, opt, args):
+  def ValidateOptions(self, opt, args):
-    if not args:
+    if not opt.all and not args:
       self.Usage()
 
-    nb = args[0]
+    if not opt.all:
-    if not git.check_ref_format('heads/%s' % nb):
+      nb = args[0]
-      print("error: '%s' is not a valid name" % nb, file=sys.stderr)
+      if not git.check_ref_format('heads/%s' % nb):
-      sys.exit(1)
+        self.OptionParser.error("'%s' is not a valid branch name" % nb)
+    else:
+      args.insert(0, "'All local branches'")
 
+  def Execute(self, opt, args):
     nb = args[0]
-    err = []
+    err = defaultdict(list)
-    success = []
+    success = defaultdict(list)
     all_projects = self.GetProjects(args[1:])
 
     pm = Progress('Abandon %s' % nb, len(all_projects))
     for project in all_projects:
       pm.update()
 
-      status = project.AbandonBranch(nb)
+      if opt.all:
-      if status is not None:
+        branches = list(project.GetBranches().keys())
-        if status:
+      else:
-          success.append(project)
+        branches = [nb]
-        else:
+
-          err.append(project)
+      for name in branches:
+        status = project.AbandonBranch(name)
+        if status is not None:
+          if status:
+            success[name].append(project)
+          else:
+            err[name].append(project)
     pm.end()
 
+    width = 25
+    for name in branches:
+      if width < len(name):
+        width = len(name)
+
     if err:
-      for p in err:
+      for br in err.keys():
-        print("error: %s/: cannot abandon %s" % (p.relpath, nb),
+        err_msg = "error: cannot abandon %s" %br
-              file=sys.stderr)
+        print(err_msg, file=sys.stderr)
+        for proj in err[br]:
+          print(' '*len(err_msg) + " | %s" % proj.relpath, file=sys.stderr)
       sys.exit(1)
     elif not success:
-      print('error: no project has branch %s' % nb, file=sys.stderr)
+      print('error: no project has local branch(es) : %s' % nb,
+            file=sys.stderr)
       sys.exit(1)
     else:
-      print('Abandoned in %d project(s):\n  %s'
+      print('Abandoned branches:', file=sys.stderr)
-            % (len(success), '\n  '.join(p.relpath for p in success)),
+      for br in success.keys():
-            file=sys.stderr)
+        if len(all_projects) > 1 and len(all_projects) == len(success[br]):
+          result = "all project"
+        else:
+          result = "%s" % (
+            ('\n'+' '*width + '| ').join(p.relpath for p in success[br]))
+        print("%s%s| %s\n" % (br,' '*(width-len(br)), result),file=sys.stderr)

subcmds/branches.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2009 The Android Open Source Project
 #
@@ -67,8 +68,7 @@ class Branches(Command):
 
 Summarizes the currently available topic branches.
 
-Branch Display
+# Branch Display
---------------
 
 The branch display output by this command is organized into four
 columns of information; for example:

subcmds/checkout.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2009 The Android Open Source Project
 #
@@ -33,10 +34,11 @@ The command is equivalent to:
   repo forall [<project>...] -c git checkout <branchname>
 """
 
-  def Execute(self, opt, args):
+  def ValidateOptions(self, opt, args):
     if not args:
       self.Usage()
 
+  def Execute(self, opt, args):
     nb = args[0]
     err = []
     success = []

subcmds/cherry_pick.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2010 The Android Open Source Project
 #
@@ -36,10 +37,11 @@ change id will be added.
   def _Options(self, p):
     pass
 
-  def Execute(self, opt, args):
+  def ValidateOptions(self, opt, args):
     if len(args) != 1:
       self.Usage()
 
+  def Execute(self, opt, args):
     reference = args[0]
 
     p = GitCommand(None,

subcmds/diff.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #

subcmds/diffmanifests.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2014 The Android Open Source Project
 #
@@ -71,6 +72,10 @@ synced and their revisions won't be found.
     p.add_option('--no-color',
                  dest='color', action='store_false', default=True,
                  help='does not display the diff in color.')
+    p.add_option('--pretty-format',
+                 dest='pretty_format', action='store',
+                 metavar='<FORMAT>',
+                 help='print the log using a custom git pretty format string')
 
   def _printRawDiff(self, diff):
     for project in diff['added']:
@@ -92,7 +97,7 @@ synced and their revisions won't be found.
                                      otherProject.revisionExpr))
       self.out.nl()
 
-  def _printDiff(self, diff, color=True):
+  def _printDiff(self, diff, color=True, pretty_format=None):
     if diff['added']:
       self.out.nl()
       self.printText('added projects : \n')
@@ -124,7 +129,8 @@ synced and their revisions won't be found.
         self.printText(' to ')
         self.printRevision(otherProject.revisionExpr)
         self.out.nl()
-        self._printLogs(project, otherProject, raw=False, color=color)
+        self._printLogs(project, otherProject, raw=False, color=color,
+                        pretty_format=pretty_format)
         self.out.nl()
 
     if diff['unreachable']:
@@ -139,9 +145,13 @@ synced and their revisions won't be found.
         self.printText(' not found')
         self.out.nl()
 
-  def _printLogs(self, project, otherProject, raw=False, color=True):
+  def _printLogs(self, project, otherProject, raw=False, color=True,
-    logs = project.getAddedAndRemovedLogs(otherProject, oneline=True,
+                 pretty_format=None):
-                                          color=color)
+
+    logs = project.getAddedAndRemovedLogs(otherProject,
+                                          oneline=(pretty_format is None),
+                                          color=color,
+                                          pretty_format=pretty_format)
     if logs['removed']:
       removedLogs = logs['removed'].split('\n')
       for log in removedLogs:
@@ -166,10 +176,11 @@ synced and their revisions won't be found.
             self.printText(log)
             self.out.nl()
 
-  def Execute(self, opt, args):
+  def ValidateOptions(self, opt, args):
     if not args or len(args) > 2:
-      self.Usage()
+      self.OptionParser.error('missing manifests to diff')
 
+  def Execute(self, opt, args):
     self.out = _Coloring(self.manifest.globalConfig)
     self.printText = self.out.nofmt_printer('text')
     if opt.color:
@@ -181,15 +192,15 @@ synced and their revisions won't be found.
       self.printProject = self.printAdded = self.printRemoved = self.printRevision = self.printText
 
     manifest1 = XmlManifest(self.manifest.repodir)
-    manifest1.Override(args[0])
+    manifest1.Override(args[0], load_local_manifests=False)
     if len(args) == 1:
       manifest2 = self.manifest
     else:
       manifest2 = XmlManifest(self.manifest.repodir)
-      manifest2.Override(args[1])
+      manifest2.Override(args[1], load_local_manifests=False)
 
     diff = manifest1.projectsDiff(manifest2)
     if opt.raw:
       self._printRawDiff(diff)
     else:
-      self._printDiff(diff, color=opt.color)
+      self._printDiff(diff, color=opt.color, pretty_format=opt.pretty_format)

subcmds/download.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #
@@ -26,11 +27,12 @@ class Download(Command):
   common = True
   helpSummary = "Download and checkout a change"
   helpUsage = """
-%prog {project change[/patchset]}...
+%prog {[project] change[/patchset]}...
 """
   helpDescription = """
 The '%prog' command downloads a change from the review system and
 makes it available in your project's local working directory.
+If no project is specified try to use current directory as a project.
 """
 
   def _Options(self, p):
@@ -55,12 +57,21 @@ makes it available in your project's local working directory.
       m = CHANGE_RE.match(a)
       if m:
         if not project:
-          self.Usage()
+          project = self.GetProjects(".")[0]
         chg_id = int(m.group(1))
         if m.group(2):
           ps_id = int(m.group(2))
         else:
           ps_id = 1
+          refs = 'refs/changes/%2.2d/%d/' % (chg_id % 100, chg_id)
+          output = project._LsRemote(refs + '*')
+          if output:
+            regex = refs + r'(\d+)'
+            rcomp = re.compile(regex, re.I)
+            for line in output.splitlines():
+              match = rcomp.search(line)
+              if match:
+                ps_id = max(int(match.group(1)), ps_id)
         to_get.append((project, chg_id, ps_id))
       else:
         project = self.GetProjects([a])[0]

subcmds/forall.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #
@@ -15,17 +16,16 @@
 
 from __future__ import print_function
 import errno
-import fcntl
 import multiprocessing
 import re
 import os
-import select
 import signal
 import sys
 import subprocess
 
 from color import Coloring
 from command import Command, MirrorSafeCommand
+import platform_utils
 
 _CAN_COLOR = [
   'branch',
@@ -54,8 +54,7 @@ Executes the same shell command in each project.
 The -r option allows running the command only on projects matching
 regex or wildcard expression.
 
-Output Formatting
+# Output Formatting
------------------
 
 The -p option causes '%prog' to bind pipes to the command's stdin,
 stdout and stderr streams, and pipe all output into a continuous
@@ -72,8 +71,7 @@ command produces output only on stderr.  Normally the -p option
 causes command output to be suppressed until the command produces
 at least one byte of output on stdout.
 
-Environment
+# Environment
------------
 
 pwd is the project's working directory.  If the current client is
 a mirror client, then pwd is the Git repository.
@@ -105,6 +103,13 @@ annotating tree details.
 shell positional arguments ($1, $2, .., $#) are set to any arguments
 following <command>.
 
+Example: to list projects:
+
+  %prog -c 'echo $REPO_PROJECT'
+
+Notice that $REPO_PROJECT is quoted to ensure it is expanded in
+the context of running <command> instead of in the calling shell.
+
 Unless -p is used, stdin, stdout, stderr are inherited from the
 terminal and are not redirected.
 
@@ -120,6 +125,9 @@ without iterating through the remaining projects.
     p.add_option('-r', '--regex',
                  dest='regex', action='store_true',
                  help="Execute the command only on projects matching regex or wildcard expression")
+    p.add_option('-i', '--inverse-regex',
+                 dest='inverse_regex', action='store_true',
+                 help="Execute the command only on projects not matching regex or wildcard expression")
     p.add_option('-g', '--groups',
                  dest='groups',
                  help="Execute the command only on projects matching the specified groups")
@@ -169,10 +177,11 @@ without iterating through the remaining projects.
       'worktree': project.worktree,
     }
 
-  def Execute(self, opt, args):
+  def ValidateOptions(self, opt, args):
     if not opt.command:
       self.Usage()
 
+  def Execute(self, opt, args):
     cmd = [opt.command[0]]
 
     shell = True
@@ -196,14 +205,12 @@ without iterating through the remaining projects.
           break
       else:
         cn = None
-      # pylint: disable=W0631
       if cn and cn in _CAN_COLOR:
         class ColorCmd(Coloring):
           def __init__(self, config, cmd):
             Coloring.__init__(self, config, cmd)
         if ColorCmd(self.manifest.manifestProject.config, cn).is_on:
           cmd.insert(cmd.index(cn) + 1, '--color')
-      # pylint: enable=W0631
 
     mirror = self.manifest.IsMirror
     rc = 0
@@ -215,10 +222,12 @@ without iterating through the remaining projects.
     if os.path.isfile(smart_sync_manifest_path):
       self.manifest.Override(smart_sync_manifest_path)
 
-    if not opt.regex:
+    if opt.regex:
-      projects = self.GetProjects(args, groups=opt.groups)
-    else:
       projects = self.FindProjects(args)
+    elif opt.inverse_regex:
+      projects = self.FindProjects(args, inverse=True)
+    else:
+      projects = self.GetProjects(args, groups=opt.groups)
 
     os.environ['REPO_COUNT'] = str(len(projects))
 
@@ -339,35 +348,25 @@ def DoWork(project, mirror, opt, cmd, shell, cnt, config):
   if opt.project_header:
     out = ForallColoring(config)
     out.redirect(sys.stdout)
-    class sfd(object):
-      def __init__(self, fd, dest):
-        self.fd = fd
-        self.dest = dest
-      def fileno(self):
-        return self.fd.fileno()
-
     empty = True
     errbuf = ''
 
     p.stdin.close()
-    s_in = [sfd(p.stdout, sys.stdout),
+    s_in = platform_utils.FileDescriptorStreams.create()
-            sfd(p.stderr, sys.stderr)]
+    s_in.add(p.stdout, sys.stdout, 'stdout')
+    s_in.add(p.stderr, sys.stderr, 'stderr')
 
-    for s in s_in:
+    while not s_in.is_done:
-      flags = fcntl.fcntl(s.fd, fcntl.F_GETFL)
+      in_ready = s_in.select()
-      fcntl.fcntl(s.fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
-
-    while s_in:
-      in_ready, _out_ready, _err_ready = select.select(s_in, [], [])
       for s in in_ready:
-        buf = s.fd.read(4096)
+        buf = s.read()
         if not buf:
-          s.fd.close()
+          s.close()
           s_in.remove(s)
           continue
 
         if not opt.verbose:
-          if s.fd != p.stdout:
+          if s.std_name == 'stderr':
             errbuf += buf
             continue
 

subcmds/gitc_delete.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2015 The Android Open Source Project
 #
@@ -14,18 +15,14 @@
 # limitations under the License.
 
 from __future__ import print_function
-import os
-import shutil
 import sys
 
 from command import Command, GitcClientCommand
-import gitc_utils
+import platform_utils
 
 from pyversion import is_python3
 if not is_python3():
-  # pylint:disable=W0622
   input = raw_input
-  # pylint:enable=W0622
 
 class GitcDelete(Command, GitcClientCommand):
   common = True
@@ -52,4 +49,4 @@ and all locally downloaded sources.
       if not response == 'yes':
         print('Response was not "yes"\n Exiting...')
         sys.exit(1)
-    shutil.rmtree(self.gitc_manifest.gitc_client_dir)
+    platform_utils.rmtree(self.gitc_manifest.gitc_client_dir)

subcmds/gitc_init.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2015 The Android Open Source Project
 #

subcmds/grep.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2009 The Android Open Source Project
 #
@@ -33,8 +34,7 @@ class Grep(PagedCommand):
   helpDescription = """
 Search for the specified patterns in all project files.
 
-Boolean Options
+# Boolean Options
----------------
 
 The following options can appear as often as necessary to express
 the pattern to locate:
@@ -47,8 +47,7 @@ in order to scan multiple trees.  If the same file matches in more
 than one tree, only the first result is reported, prefixed by the
 revision name it was found under.
 
-Examples
+# Examples
--------
 
 Look for a line that has '#define' and either 'MAX_PATH or 'PATH_MAX':
 

subcmds/help.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #
@@ -87,7 +88,7 @@ Displays detailed usage information about a command.
 "See 'repo help <command>' for more information on a specific command.\n"
 "See 'repo help --all' for a complete list of recognized commands.")
 
-  def _PrintCommandHelp(self, cmd):
+  def _PrintCommandHelp(self, cmd, header_prefix=''):
     class _Out(Coloring):
       def __init__(self, gc):
         Coloring.__init__(self, gc, 'help')
@@ -105,17 +106,15 @@ Displays detailed usage information about a command.
 
         self.nl()
 
-        self.heading('%s', heading)
+        self.heading('%s%s', header_prefix, heading)
         self.nl()
-
-        self.heading('%s', ''.ljust(len(heading), '-'))
         self.nl()
 
         me = 'repo %s' % cmd.NAME
         body = body.strip()
         body = body.replace('%prog', me)
 
-        asciidoc_hdr = re.compile(r'^\n?([^\n]{1,})\n([=~-]{2,})$')
+        asciidoc_hdr = re.compile(r'^\n?#+ (.+)$')
         for para in body.split("\n\n"):
           if para.startswith(' '):
             self.write('%s', para)
@@ -125,19 +124,8 @@ Displays detailed usage information about a command.
 
           m = asciidoc_hdr.match(para)
           if m:
-            title = m.group(1)
+            self.heading('%s%s', header_prefix, m.group(1))
-            section_type = m.group(2)
-            if section_type[0] in ('=', '-'):
-              p = self.heading
-            else:
-              def _p(fmt, *args):
-                self.write('  ')
-                self.heading(fmt, *args)
-              p = _p
-
-            p('%s', title)
             self.nl()
-            p('%s', ''.ljust(len(title), section_type[0]))
             self.nl()
             continue
 
@@ -150,14 +138,25 @@ Displays detailed usage information about a command.
     cmd.OptionParser.print_help()
     out._PrintSection('Description', 'helpDescription')
 
+  def _PrintAllCommandHelp(self):
+    for name in sorted(self.commands):
+      cmd = self.commands[name]
+      cmd.manifest = self.manifest
+      self._PrintCommandHelp(cmd, header_prefix='[%s] ' % (name,))
+
   def _Options(self, p):
     p.add_option('-a', '--all',
                  dest='show_all', action='store_true',
                  help='show the complete list of commands')
+    p.add_option('--help-all',
+                 dest='show_all_help', action='store_true',
+                 help='show the --help of all commands')
 
   def Execute(self, opt, args):
     if len(args) == 0:
-      if opt.show_all:
+      if opt.show_all_help:
+        self._PrintAllCommandHelp()
+      elif opt.show_all:
         self._PrintAllCommands()
       else:
         self._PrintCommonCommands()

subcmds/info.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2012 The Android Open Source Project
 #
@@ -45,7 +46,7 @@ class Info(PagedCommand):
   def Execute(self, opt, args):
     self.out = _Coloring(self.manifest.globalConfig)
     self.heading = self.out.printer('heading', attr = 'bold')
-    self.headtext = self.out.printer('headtext', fg = 'yellow')
+    self.headtext = self.out.nofmt_printer('headtext', fg = 'yellow')
     self.redtext = self.out.printer('redtext', fg = 'red')
     self.sha = self.out.printer("sha", fg = 'yellow')
     self.text = self.out.nofmt_printer('text')
@@ -99,7 +100,7 @@ class Info(PagedCommand):
       self.headtext(p.revisionExpr)
       self.out.nl()
 
-      localBranches = p.GetBranches().keys()
+      localBranches = list(p.GetBranches().keys())
       self.heading("Local Branches: ")
       self.redtext(str(len(localBranches)))
       if len(localBranches) > 0:

subcmds/init.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #
@@ -17,7 +18,6 @@ from __future__ import print_function
 import os
 import platform
 import re
-import shutil
 import sys
 
 from pyversion import is_python3
@@ -35,6 +35,7 @@ from error import ManifestParseError
 from project import SyncBuffer
 from git_config import GitConfig
 from git_command import git_require, MIN_GIT_VERSION
+import platform_utils
 
 class Init(InteractiveCommand, MirrorSafeCommand):
   common = True
@@ -61,9 +62,18 @@ directory use as much data as possible from the local reference
 directory when fetching from the server. This will make the sync
 go a lot faster by reducing data traffic on the network.
 
+The --dissociate option can be used to borrow the objects from
+the directory specified with the --reference option only to reduce
+network transfer, and stop borrowing from them after a first clone
+is made by making necessary local copies of borrowed objects.
 
-Switching Manifest Branches
+The --no-clone-bundle option disables any attempt to use
----------------------------
+$URL/clone.bundle to bootstrap a new Git repository from a
+resumeable bundle file on a content delivery network. This
+may be necessary if there are problems with the local Python
+HTTP client or proxy configuration, but the Git binary works.
+
+# Switching Manifest Branches
 
 To switch to another manifest branch, `repo init -b otherbranch`
 may be used in an existing client.  However, as this only updates the
@@ -86,6 +96,9 @@ to update the working directory files.
     g.add_option('-b', '--manifest-branch',
                  dest='manifest_branch',
                  help='manifest branch or revision', metavar='REVISION')
+    g.add_option('--current-branch',
+                 dest='current_branch_only', action='store_true',
+                 help='fetch only current manifest branch from server')
     g.add_option('-m', '--manifest-name',
                  dest='manifest_name', default='default.xml',
                  help='initial manifest file', metavar='NAME.xml')
@@ -96,13 +109,26 @@ to update the working directory files.
     g.add_option('--reference',
                  dest='reference',
                  help='location of mirror directory', metavar='DIR')
+    g.add_option('--dissociate',
+                 dest='dissociate', action='store_true',
+                 help='dissociate from reference mirrors after clone')
     g.add_option('--depth', type='int', default=None,
                  dest='depth',
                  help='create a shallow clone with given depth; see git clone')
+    g.add_option('--partial-clone', action='store_true',
+                 dest='partial_clone',
+                 help='perform partial clone (https://git-scm.com/'
+                 'docs/gitrepository-layout#_code_partialclone_code)')
+    g.add_option('--clone-filter', action='store', default='blob:none',
+                 dest='clone_filter',
+                 help='filter for use with --partial-clone [default: %default]')
     g.add_option('--archive',
                  dest='archive', action='store_true',
                  help='checkout an archive instead of a git repository for '
                       'each project. See git archive.')
+    g.add_option('--submodules',
+                 dest='submodules', action='store_true',
+                 help='sync any submodules associated with the manifest repo')
     g.add_option('-g', '--groups',
                  dest='groups', default='default',
                  help='restrict manifest projects to ones with specified '
@@ -113,6 +139,12 @@ to update the working directory files.
                  help='restrict manifest projects to ones with a specified '
                       'platform group [auto|all|none|linux|darwin|...]',
                  metavar='PLATFORM')
+    g.add_option('--no-clone-bundle',
+                 dest='no_clone_bundle', action='store_true',
+                 help='disable use of /clone.bundle on HTTP/HTTPS')
+    g.add_option('--no-tags',
+                 dest='no_tags', action='store_true',
+                 help="don't fetch tags in the manifest")
 
     # Tool
     g = p.add_option_group('repo Version options')
@@ -158,7 +190,8 @@ to update the working directory files.
         if not mirrored_manifest_git.endswith(".git"):
           mirrored_manifest_git += ".git"
         if not os.path.exists(mirrored_manifest_git):
-          mirrored_manifest_git = os.path.join(opt.reference + '/.repo/manifests.git')
+          mirrored_manifest_git = os.path.join(opt.reference,
+                                               '.repo/manifests.git')
 
       m._InitGitDir(mirror_git=mirrored_manifest_git)
 
@@ -172,6 +205,8 @@ to update the working directory files.
       else:
         m.PreSync()
 
+    self._ConfigureDepth(opt)
+
     if opt.manifest_url:
       r = m.GetRemote(m.remote.name)
       r.url = opt.manifest_url
@@ -202,6 +237,9 @@ to update the working directory files.
     if opt.reference:
       m.config.SetString('repo.reference', opt.reference)
 
+    if opt.dissociate:
+      m.config.SetString('repo.dissociate', 'true')
+
     if opt.archive:
       if is_new:
         m.config.SetString('repo.archive', 'true')
@@ -222,21 +260,39 @@ to update the working directory files.
               'in another location.', file=sys.stderr)
         sys.exit(1)
 
-    if not m.Sync_NetworkHalf(is_new=is_new):
+    if opt.partial_clone:
+      if opt.mirror:
+        print('fatal: --mirror and --partial-clone are mutually exclusive',
+              file=sys.stderr)
+        sys.exit(1)
+      m.config.SetString('repo.partialclone', 'true')
+      if opt.clone_filter:
+        m.config.SetString('repo.clonefilter', opt.clone_filter)
+    else:
+      opt.clone_filter = None
+
+    if opt.submodules:
+      m.config.SetString('repo.submodules', 'true')
+
+    if not m.Sync_NetworkHalf(is_new=is_new, quiet=opt.quiet,
+        clone_bundle=not opt.no_clone_bundle,
+        current_branch_only=opt.current_branch_only,
+        no_tags=opt.no_tags, submodules=opt.submodules,
+        clone_filter=opt.clone_filter):
       r = m.GetRemote(m.remote.name)
       print('fatal: cannot obtain manifest %s' % r.url, file=sys.stderr)
 
       # Better delete the manifest git dir if we created it; otherwise next
       # time (when user fixes problems) we won't go through the "is_new" logic.
       if is_new:
-        shutil.rmtree(m.gitdir)
+        platform_utils.rmtree(m.gitdir)
       sys.exit(1)
 
     if opt.manifest_branch:
-      m.MetaBranchSwitch()
+      m.MetaBranchSwitch(submodules=opt.submodules)
 
     syncbuf = SyncBuffer(m.config)
-    m.Sync_LocalHalf(syncbuf)
+    m.Sync_LocalHalf(syncbuf, submodules=opt.submodules)
     syncbuf.Finish()
 
     if is_new or m.CurrentBranch is None:
@@ -257,7 +313,9 @@ to update the working directory files.
       sys.exit(1)
 
   def _Prompt(self, prompt, value):
-    sys.stdout.write('%-10s [%s]: ' % (prompt, value))
+    print('%-10s [%s]: ' % (prompt, value), end='')
+    # TODO: When we require Python 3, use flush=True w/print above.
+    sys.stdout.flush()
     a = sys.stdin.readline().strip()
     if a == '':
       return value
@@ -291,7 +349,9 @@ to update the working directory files.
 
       print()
       print('Your identity is: %s <%s>' % (name, email))
-      sys.stdout.write('is this correct [y/N]? ')
+      print('is this correct [y/N]? ', end='')
+      # TODO: When we require Python 3, use flush=True w/print above.
+      sys.stdout.flush()
       a = sys.stdin.readline().strip().lower()
       if a in ('yes', 'y', 't', 'true'):
         break
@@ -333,7 +393,9 @@ to update the working directory files.
       out.printer(fg='black', attr=c)(' %-6s ', c)
     out.nl()
 
-    sys.stdout.write('Enable color display in this user account (y/N)? ')
+    print('Enable color display in this user account (y/N)? ', end='')
+    # TODO: When we require Python 3, use flush=True w/print above.
+    sys.stdout.flush()
     a = sys.stdin.readline().strip().lower()
     if a in ('y', 'yes', 't', 'true', 'on'):
       gc.SetString('color.ui', 'auto')
@@ -374,18 +436,17 @@ to update the working directory files.
       print('   rm -r %s/.repo' % self.manifest.topdir)
       print('and try again.')
 
-  def Execute(self, opt, args):
+  def ValidateOptions(self, opt, args):
-    git_require(MIN_GIT_VERSION, fail=True)
-
     if opt.reference:
       opt.reference = os.path.expanduser(opt.reference)
 
     # Check this here, else manifest will be tagged "not new" and init won't be
     # possible anymore without removing the .repo/manifests directory.
     if opt.archive and opt.mirror:
-      print('fatal: --mirror and --archive cannot be used together.',
+      self.OptionParser.error('--mirror and --archive cannot be used together.')
-            file=sys.stderr)
+
-      sys.exit(1)
+  def Execute(self, opt, args):
+    git_require(MIN_GIT_VERSION, fail=True)
 
     self._SyncManifest(opt)
     self._LinkManifest(opt.manifest_name)
@@ -395,6 +456,4 @@ to update the working directory files.
         self._ConfigureUser()
       self._ConfigureColor()
 
-    self._ConfigureDepth(opt)
-
     self._DisplayResult()

subcmds/list.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2011 The Android Open Source Project
 #
@@ -48,6 +49,10 @@ This is similar to running: repo forall -c 'echo "$REPO_PATH : $REPO_PROJECT"'.
                  dest='path_only', action='store_true',
                  help="Display only the path of the repository")
 
+  def ValidateOptions(self, opt, args):
+    if opt.fullpath and opt.name_only:
+      self.OptionParser.error('cannot combine -f and -n')
+
   def Execute(self, opt, args):
     """List all projects and the associated directories.
 
@@ -59,11 +64,6 @@ This is similar to running: repo forall -c 'echo "$REPO_PATH : $REPO_PROJECT"'.
       opt: The options.
       args: Positional args.  Can be a list of projects to list, or empty.
     """
-
-    if opt.fullpath and opt.name_only:
-      print('error: cannot combine -f and -n', file=sys.stderr)
-      sys.exit(1)
-
     if not opt.regex:
       projects = self.GetProjects(args, groups=opt.groups)
     else:

subcmds/manifest.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2009 The Android Open Source Project
 #
@@ -39,7 +40,7 @@ in a Git repository for use during future 'repo init' invocations.
     helptext = self._helpDescription + '\n'
     r = os.path.dirname(__file__)
     r = os.path.dirname(r)
-    fd = open(os.path.join(r, 'docs', 'manifest-format.txt'))
+    fd = open(os.path.join(r, 'docs', 'manifest-format.md'))
     for line in fd:
       helptext += line
     fd.close()
@@ -72,14 +73,9 @@ in a Git repository for use during future 'repo init' invocations.
     if opt.output_file != '-':
       print('Saved manifest to %s' % opt.output_file, file=sys.stderr)
 
-  def Execute(self, opt, args):
+  def ValidateOptions(self, opt, args):
     if args:
       self.Usage()
 
-    if opt.output_file is not None:
+  def Execute(self, opt, args):
-      self._Output(opt)
+    self._Output(opt)
-      return
-
-    print('error: no operation to perform', file=sys.stderr)
-    print('error: see repo help manifest', file=sys.stderr)
-    sys.exit(1)

subcmds/overview.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2012 The Android Open Source Project
 #

subcmds/prune.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #

subcmds/rebase.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2010 The Android Open Source Project
 #
@@ -70,7 +71,22 @@ branch but need to incorporate new upstream changes "underneath" them.
       if len(args) == 1:
         print('note: project %s is mapped to more than one path' % (args[0],),
             file=sys.stderr)
-      return -1
+      return 1
+
+    # Setup the common git rebase args that we use for all projects.
+    common_args = ['rebase']
+    if opt.whitespace:
+      common_args.append('--whitespace=%s' % opt.whitespace)
+    if opt.quiet:
+      common_args.append('--quiet')
+    if opt.force_rebase:
+      common_args.append('--force-rebase')
+    if opt.no_ff:
+      common_args.append('--no-ff')
+    if opt.autosquash:
+      common_args.append('--autosquash')
+    if opt.interactive:
+      common_args.append('-i')
 
     for project in all_projects:
       cb = project.CurrentBranch
@@ -78,7 +94,7 @@ branch but need to incorporate new upstream changes "underneath" them.
         if one_project:
           print("error: project %s has a detached HEAD" % project.relpath,
                 file=sys.stderr)
-          return -1
+          return 1
         # ignore branches with detatched HEADs
         continue
 
@@ -87,30 +103,11 @@ branch but need to incorporate new upstream changes "underneath" them.
         if one_project:
           print("error: project %s does not track any remote branches"
                 % project.relpath, file=sys.stderr)
-          return -1
+          return 1
         # ignore branches without remotes
         continue
 
-      args = ["rebase"]
+      args = common_args[:]
-
-      if opt.whitespace:
-        args.append('--whitespace=%s' % opt.whitespace)
-
-      if opt.quiet:
-        args.append('--quiet')
-
-      if opt.force_rebase:
-        args.append('--force-rebase')
-
-      if opt.no_ff:
-        args.append('--no-ff')
-
-      if opt.autosquash:
-        args.append('--autosquash')
-
-      if opt.interactive:
-        args.append("-i")
-
       if opt.onto_manifest:
         args.append('--onto')
         args.append(project.revisionExpr)
@@ -130,13 +127,13 @@ branch but need to incorporate new upstream changes "underneath" them.
           stash_args = ["stash"]
 
           if GitCommand(project, stash_args).Wait() != 0:
-            return -1
+            return 1
 
       if GitCommand(project, args).Wait() != 0:
-        return -1
+        return 1
 
       if needs_stash:
         stash_args.append('pop')
         stash_args.append('--quiet')
         if GitCommand(project, stash_args).Wait() != 0:
-          return -1
+          return 1

subcmds/selfupdate.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2009 The Android Open Source Project
 #

subcmds/smartsync.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2010 The Android Open Source Project
 #

subcmds/stage.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #
@@ -60,8 +61,8 @@ The '%prog' command stages files to prepare the next commit.
       out.nl()
 
       for i in range(len(all_projects)):
-        p = all_projects[i]
+        project = all_projects[i]
-        out.write('%3d:    %s', i + 1, p.relpath + '/')
+        out.write('%3d:    %s', i + 1, project.relpath + '/')
         out.nl()
       out.nl()
 

subcmds/start.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #
@@ -18,7 +19,7 @@ import os
 import sys
 
 from command import Command
-from git_config import IsId
+from git_config import IsImmutable
 from git_command import git
 import gitc_utils
 from progress import Progress
@@ -39,23 +40,27 @@ revision specified in the manifest.
     p.add_option('--all',
                  dest='all', action='store_true',
                  help='begin branch in all projects')
+    p.add_option('-r', '--rev', '--revision', dest='revision',
+                 help='point branch at this revision instead of upstream')
+    p.add_option('--head', dest='revision', action='store_const', const='HEAD',
+                 help='abbreviation for --rev HEAD')
 
-  def Execute(self, opt, args):
+  def ValidateOptions(self, opt, args):
     if not args:
       self.Usage()
 
     nb = args[0]
     if not git.check_ref_format('heads/%s' % nb):
-      print("error: '%s' is not a valid name" % nb, file=sys.stderr)
+      self.OptionParser.error("'%s' is not a valid name" % nb)
-      sys.exit(1)
 
+  def Execute(self, opt, args):
+    nb = args[0]
     err = []
     projects = []
     if not opt.all:
       projects = args[1:]
       if len(projects) < 1:
-        print("error: at least one project must be specified", file=sys.stderr)
+        projects = ['.',]  # start it in the local project by default
-        sys.exit(1)
 
     all_projects = self.GetProjects(projects,
                                     missing_ok=bool(self.gitc_manifest))
@@ -97,17 +102,18 @@ revision specified in the manifest.
           project.Sync_LocalHalf(sync_buf)
           project.revisionId = gitc_project.old_revision
 
-      # If the current revision is a specific SHA1 then we can't push back
+      # If the current revision is immutable, such as a SHA1, a tag or
-      # to it; so substitute with dest_branch if defined, or with manifest
+      # a change, then we can't push back to it. Substitute with
-      # default revision instead.
+      # dest_branch, if defined; or with manifest default revision instead.
       branch_merge = ''
-      if IsId(project.revisionExpr):
+      if IsImmutable(project.revisionExpr):
         if project.dest_branch:
           branch_merge = project.dest_branch
         else:
           branch_merge = self.manifest.default.revisionExpr
 
-      if not project.StartBranch(nb, branch_merge=branch_merge):
+      if not project.StartBranch(
+          nb, branch_merge=branch_merge, revision=opt.revision):
         err.append(project)
     pm.end()
 

subcmds/status.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #
@@ -13,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
+
 from command import PagedCommand
 
 try:
@@ -26,6 +29,7 @@ import itertools
 import os
 
 from color import Coloring
+import platform_utils
 
 class Status(PagedCommand):
   common = True
@@ -49,8 +53,7 @@ includes deeper items.  For example, if dir/subdir/proj1 and
 dir/subdir/proj2 are repo projects, dir/subdir/proj3 will be shown
 if it is not known to repo.
 
-Status Display
+# Status Display
---------------
 
 The status display is organized into three columns of information,
 for example if the file 'subcmds/status.py' is modified in the
@@ -89,8 +92,10 @@ the following meanings:
     p.add_option('-o', '--orphans',
                  dest='orphans', action='store_true',
                  help="include objects in working directory outside of repo projects")
+    p.add_option('-q', '--quiet', action='store_true',
+                 help="only print the name of modified projects")
 
-  def _StatusHelper(self, project, clean_counter, sem):
+  def _StatusHelper(self, project, clean_counter, sem, quiet):
     """Obtains the status for a specific project.
 
     Obtains the status for a project, redirecting the output to
@@ -104,7 +109,7 @@ the following meanings:
       output: Where to output the status.
     """
     try:
-      state = project.PrintWorkTreeStatus()
+      state = project.PrintWorkTreeStatus(quiet=quiet)
       if state == 'CLEAN':
         next(clean_counter)
     finally:
@@ -114,7 +119,7 @@ the following meanings:
     """find 'dirs' that are present in 'proj_dirs_parents' but not in 'proj_dirs'"""
     status_header = ' --\t'
     for item in dirs:
-      if not os.path.isdir(item):
+      if not platform_utils.isdir(item):
         outstring.append(''.join([status_header, item]))
         continue
       if item in proj_dirs:
@@ -132,7 +137,7 @@ the following meanings:
 
     if opt.jobs == 1:
       for project in all_projects:
-        state = project.PrintWorkTreeStatus()
+        state = project.PrintWorkTreeStatus(quiet=opt.quiet)
         if state == 'CLEAN':
           next(counter)
     else:
@@ -142,13 +147,13 @@ the following meanings:
         sem.acquire()
 
         t = _threading.Thread(target=self._StatusHelper,
-                              args=(project, counter, sem))
+                              args=(project, counter, sem, opt.quiet))
         threads.append(t)
         t.daemon = True
         t.start()
       for t in threads:
         t.join()
-    if len(all_projects) == next(counter):
+    if not opt.quiet and len(all_projects) == next(counter):
       print('nothing to commit (working directory clean)')
 
     if opt.orphans:

subcmds/sync.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #
@@ -19,7 +20,6 @@ import netrc
 from optparse import SUPPRESS_HELP
 import os
 import re
-import shutil
 import socket
 import subprocess
 import sys
@@ -64,6 +64,7 @@ try:
 except ImportError:
   multiprocessing = None
 
+import event_log
 from git_command import GIT, git_require
 from git_config import GetUrlCookieFile
 from git_refs import R_HEADS, HEAD
@@ -72,6 +73,7 @@ from project import Project
 from project import RemoteSpec
 from command import Command, MirrorSafeCommand
 from error import RepoChangedException, GitError, ManifestParseError
+import platform_utils
 from project import SyncBuffer
 from progress import Progress
 from wrapper import Wrapper
@@ -83,6 +85,9 @@ class _FetchError(Exception):
   """Internal error thrown in _FetchHelper() when we don't want stack trace."""
   pass
 
+class _CheckoutError(Exception):
+  """Internal error thrown in _CheckoutOne() when we don't want stack trace."""
+
 class Sync(Command, MirrorSafeCommand):
   jobs = 1
   common = True
@@ -127,14 +132,19 @@ from the user's .netrc file.
 if the manifest server specified in the manifest file already includes
 credentials.
 
-The -f/--force-broken option can be used to proceed with syncing
+By default, all projects will be synced. The --fail-fast option can be used
-other projects if a project sync fails.
+to halt syncing as soon as possible when the the first project fails to sync.
 
 The --force-sync option can be used to overwrite existing git
 directories if they have previously been linked to a different
 object direcotry. WARNING: This may cause data to be lost since
 refs may be removed when overwriting.
 
+The --force-remove-dirty option can be used to remove previously used
+projects with uncommitted changes. WARNING: This may cause data to be
+lost since uncommitted changes may be removed with projects that no longer
+exist in the manifest.
+
 The --no-clone-bundle option disables any attempt to use
 $URL/clone.bundle to bootstrap a new Git repository from a
 resumeable bundle file on a content delivery network. This
@@ -154,8 +164,7 @@ exist locally.
 The --prune option can be used to remove any refs that no longer
 exist on the remote.
 
-SSH Connections
+# SSH Connections
----------------
 
 If at least one project remote URL uses an SSH connection (ssh://,
 git+ssh://, or user@host:path syntax) repo will automatically
@@ -169,8 +178,7 @@ environment variable to 'ssh'.  For example:
   export GIT_SSH=ssh
   %prog
 
-Compatibility
+# Compatibility
-~~~~~~~~~~~~~
 
 This feature is automatically disabled on Windows, due to the lack
 of UNIX domain socket support.
@@ -192,12 +200,20 @@ later is required to fix a server side protocol bug.
 
     p.add_option('-f', '--force-broken',
                  dest='force_broken', action='store_true',
-                 help="continue sync even if a project fails to sync")
+                 help='obsolete option (to be deleted in the future)')
+    p.add_option('--fail-fast',
+                 dest='fail_fast', action='store_true',
+                 help='stop syncing after first error is hit')
     p.add_option('--force-sync',
                  dest='force_sync', action='store_true',
                  help="overwrite an existing git directory if it needs to "
                       "point to a different object directory. WARNING: this "
                       "may cause loss of data")
+    p.add_option('--force-remove-dirty',
+                 dest='force_remove_dirty', action='store_true',
+                 help="force remove projects with uncommitted modifications if "
+                      "projects no longer exist in the manifest. "
+                      "WARNING: this may cause loss of data")
     p.add_option('-l', '--local-only',
                  dest='local_only', action='store_true',
                  help="only update working tree, don't fetch")
@@ -242,7 +258,7 @@ later is required to fix a server side protocol bug.
     if show_smart:
       p.add_option('-s', '--smart-sync',
                    dest='smart_sync', action='store_true',
-                   help='smart sync using manifest from a known good build')
+                   help='smart sync using manifest from the latest known good build')
       p.add_option('-t', '--smart-tag',
                    dest='smart_tag', action='store',
                    help='smart sync using manifest from a known tag')
@@ -255,23 +271,29 @@ later is required to fix a server side protocol bug.
                  dest='repo_upgraded', action='store_true',
                  help=SUPPRESS_HELP)
 
-  def _FetchProjectList(self, opt, projects, *args, **kwargs):
+  def _FetchProjectList(self, opt, projects, sem, *args, **kwargs):
-    """Main function of the fetch threads when jobs are > 1.
+    """Main function of the fetch threads.
 
     Delegates most of the work to _FetchHelper.
 
     Args:
       opt: Program options returned from optparse.  See _Options().
       projects: Projects to fetch.
+      sem: We'll release() this semaphore when we exit so that another thread
+          can be started up.
       *args, **kwargs: Remaining arguments to pass to _FetchHelper. See the
           _FetchHelper docstring for details.
     """
-    for project in projects:
+    try:
-      success = self._FetchHelper(opt, project, *args, **kwargs)
+        for project in projects:
-      if not success and not opt.force_broken:
+          success = self._FetchHelper(opt, project, *args, **kwargs)
-        break
+          if not success and opt.fail_fast:
+            break
+    finally:
+        sem.release()
 
-  def _FetchHelper(self, opt, project, lock, fetched, pm, sem, err_event):
+  def _FetchHelper(self, opt, project, lock, fetched, pm, err_event,
+                   clone_filter):
     """Fetch git objects for a single project.
 
     Args:
@@ -283,10 +305,9 @@ later is required to fix a server side protocol bug.
           (with our lock held).
       pm: Instance of a Project object.  We will call pm.update() (with our
           lock held).
-      sem: We'll release() this semaphore when we exit so that another thread
-          can be started up.
       err_event: We'll set this event in the case of an error (after printing
           out info about the error).
+      clone_filter: Filter for use in a partial clone.
 
     Returns:
       Whether the fetch was successful.
@@ -299,11 +320,11 @@ later is required to fix a server side protocol bug.
 
     # Encapsulate everything in a try/except/finally so that:
     # - We always set err_event in the case of an exception.
-    # - We always make sure we call sem.release().
     # - We always make sure we unlock the lock if we locked it.
+    start = time.time()
+    success = False
     try:
       try:
-        start = time.time()
         success = project.Sync_NetworkHalf(
           quiet=opt.quiet,
           current_branch_only=opt.current_branch_only,
@@ -311,7 +332,8 @@ later is required to fix a server side protocol bug.
           clone_bundle=not opt.no_clone_bundle,
           no_tags=opt.no_tags, archive=self.manifest.IsArchive,
           optimized_fetch=opt.optimized_fetch,
-          prune=opt.prune)
+          prune=opt.prune,
+          clone_filter=clone_filter)
         self._fetch_times.Set(project, time.time() - start)
 
         # Lock around all the rest of the code, since printing, updating a set
@@ -321,11 +343,10 @@ later is required to fix a server side protocol bug.
 
         if not success:
           err_event.set()
-          print('error: Cannot fetch %s' % project.name, file=sys.stderr)
+          print('error: Cannot fetch %s from %s'
-          if opt.force_broken:
+                % (project.name, project.remote.url),
-            print('warn: --force-broken, continuing to sync',
+                file=sys.stderr)
-                  file=sys.stderr)
+          if opt.fail_fast:
-          else:
             raise _FetchError()
 
         fetched.add(project.gitdir)
@@ -340,14 +361,18 @@ later is required to fix a server side protocol bug.
     finally:
       if did_lock:
         lock.release()
-      sem.release()
+      finish = time.time()
+      self.event_log.AddSync(project, event_log.TASK_SYNC_NETWORK,
+                             start, finish, success)
 
     return success
 
   def _Fetch(self, projects, opt):
     fetched = set()
     lock = _threading.Lock()
-    pm = Progress('Fetching projects', len(projects))
+    pm = Progress('Fetching projects', len(projects),
+                  print_newline=not(opt.quiet),
+                  always_print_percentage=opt.quiet)
 
     objdir_project_map = dict()
     for project in projects:
@@ -359,17 +384,18 @@ later is required to fix a server side protocol bug.
     for project_list in objdir_project_map.values():
       # Check for any errors before running any more tasks.
       # ...we'll let existing threads finish, though.
-      if err_event.isSet() and not opt.force_broken:
+      if err_event.isSet() and opt.fail_fast:
         break
 
       sem.acquire()
       kwargs = dict(opt=opt,
                     projects=project_list,
+                    sem=sem,
                     lock=lock,
                     fetched=fetched,
                     pm=pm,
-                    sem=sem,
+                    err_event=err_event,
-                    err_event=err_event)
+                    clone_filter=self.manifest.CloneFilter)
       if self.jobs > 1:
         t = _threading.Thread(target = self._FetchProjectList,
                               kwargs = kwargs)
@@ -384,7 +410,7 @@ later is required to fix a server side protocol bug.
       t.join()
 
     # If we saw an error, exit with code 1 so that other scripts can check.
-    if err_event.isSet():
+    if err_event.isSet() and opt.fail_fast:
       print('\nerror: Exited sync due to fetch errors', file=sys.stderr)
       sys.exit(1)
 
@@ -396,10 +422,153 @@ later is required to fix a server side protocol bug.
 
     return fetched
 
+  def _CheckoutWorker(self, opt, sem, project, *args, **kwargs):
+    """Main function of the fetch threads.
+
+    Delegates most of the work to _CheckoutOne.
+
+    Args:
+      opt: Program options returned from optparse.  See _Options().
+      projects: Projects to fetch.
+      sem: We'll release() this semaphore when we exit so that another thread
+          can be started up.
+      *args, **kwargs: Remaining arguments to pass to _CheckoutOne. See the
+          _CheckoutOne docstring for details.
+    """
+    try:
+      return self._CheckoutOne(opt, project, *args, **kwargs)
+    finally:
+      sem.release()
+
+  def _CheckoutOne(self, opt, project, lock, pm, err_event):
+    """Checkout work tree for one project
+
+    Args:
+      opt: Program options returned from optparse.  See _Options().
+      project: Project object for the project to checkout.
+      lock: Lock for accessing objects that are shared amongst multiple
+          _CheckoutWorker() threads.
+      pm: Instance of a Project object.  We will call pm.update() (with our
+          lock held).
+      err_event: We'll set this event in the case of an error (after printing
+          out info about the error).
+
+    Returns:
+      Whether the fetch was successful.
+    """
+    # We'll set to true once we've locked the lock.
+    did_lock = False
+
+    if not opt.quiet:
+      print('Checking out project %s' % project.name)
+
+    # Encapsulate everything in a try/except/finally so that:
+    # - We always set err_event in the case of an exception.
+    # - We always make sure we unlock the lock if we locked it.
+    start = time.time()
+    syncbuf = SyncBuffer(self.manifest.manifestProject.config,
+                         detach_head=opt.detach_head)
+    success = False
+    try:
+      try:
+        project.Sync_LocalHalf(syncbuf, force_sync=opt.force_sync)
+        success = syncbuf.Finish()
+
+        # Lock around all the rest of the code, since printing, updating a set
+        # and Progress.update() are not thread safe.
+        lock.acquire()
+        did_lock = True
+
+        if not success:
+          err_event.set()
+          print('error: Cannot checkout %s' % (project.name),
+                file=sys.stderr)
+          raise _CheckoutError()
+
+        pm.update()
+      except _CheckoutError:
+        pass
+      except Exception as e:
+        print('error: Cannot checkout %s: %s: %s' %
+              (project.name, type(e).__name__, str(e)),
+              file=sys.stderr)
+        err_event.set()
+        raise
+    finally:
+      if did_lock:
+        lock.release()
+      finish = time.time()
+      self.event_log.AddSync(project, event_log.TASK_SYNC_LOCAL,
+                             start, finish, success)
+
+    return success
+
+  def _Checkout(self, all_projects, opt):
+    """Checkout projects listed in all_projects
+
+    Args:
+      all_projects: List of all projects that should be checked out.
+      opt: Program options returned from optparse.  See _Options().
+    """
+
+    # Perform checkouts in multiple threads when we are using partial clone.
+    # Without partial clone, all needed git objects are already downloaded,
+    # in this situation it's better to use only one process because the checkout
+    # would be mostly disk I/O; with partial clone, the objects are only
+    # downloaded when demanded (at checkout time), which is similar to the
+    # Sync_NetworkHalf case and parallelism would be helpful.
+    if self.manifest.CloneFilter:
+      syncjobs = self.jobs
+    else:
+      syncjobs = 1
+
+    lock = _threading.Lock()
+    pm = Progress('Syncing work tree', len(all_projects))
+
+    threads = set()
+    sem = _threading.Semaphore(syncjobs)
+    err_event = _threading.Event()
+
+    for project in all_projects:
+      # Check for any errors before running any more tasks.
+      # ...we'll let existing threads finish, though.
+      if err_event.isSet() and opt.fail_fast:
+        break
+
+      sem.acquire()
+      if project.worktree:
+        kwargs = dict(opt=opt,
+                      sem=sem,
+                      project=project,
+                      lock=lock,
+                      pm=pm,
+                      err_event=err_event)
+        if syncjobs > 1:
+          t = _threading.Thread(target=self._CheckoutWorker,
+                                kwargs=kwargs)
+          # Ensure that Ctrl-C will not freeze the repo process.
+          t.daemon = True
+          threads.add(t)
+          t.start()
+        else:
+          self._CheckoutWorker(**kwargs)
+
+    for t in threads:
+      t.join()
+
+    pm.end()
+    # If we saw an error, exit with code 1 so that other scripts can check.
+    if err_event.isSet():
+      print('\nerror: Exited sync due to checkout errors', file=sys.stderr)
+      sys.exit(1)
+
   def _GCProjects(self, projects):
-    gitdirs = {}
+    gc_gitdirs = {}
     for project in projects:
-      gitdirs[project.gitdir] = project.bare_git
+      if len(project.manifest.GetProjectsWithName(project.name)) > 1:
+        print('Shared project %s found, disabling pruning.' % project.name)
+        project.bare_git.config('--replace-all', 'gc.pruneExpire', 'never')
+      gc_gitdirs[project.gitdir] = project.bare_git
 
     has_dash_c = git_require((1, 7, 2))
     if multiprocessing and has_dash_c:
@@ -409,11 +578,11 @@ later is required to fix a server side protocol bug.
     jobs = min(self.jobs, cpu_count)
 
     if jobs < 2:
-      for bare_git in gitdirs.values():
+      for bare_git in gc_gitdirs.values():
         bare_git.gc('--auto')
       return
 
-    config = {'pack.threads': cpu_count / jobs if cpu_count > jobs else 1}
+    config = {'pack.threads': cpu_count // jobs if cpu_count > jobs else 1}
 
     threads = set()
     sem = _threading.Semaphore(jobs)
@@ -431,7 +600,7 @@ later is required to fix a server side protocol bug.
       finally:
         sem.release()
 
-    for bare_git in gitdirs.values():
+    for bare_git in gc_gitdirs.values():
       if err_event.isSet():
         break
       sem.acquire()
@@ -454,7 +623,66 @@ later is required to fix a server side protocol bug.
     else:
       self.manifest._Unload()
 
-  def UpdateProjectList(self):
+  def _DeleteProject(self, path):
+    print('Deleting obsolete path %s' % path, file=sys.stderr)
+
+    # Delete the .git directory first, so we're less likely to have a partially
+    # working git repository around. There shouldn't be any git projects here,
+    # so rmtree works.
+    try:
+      platform_utils.rmtree(os.path.join(path, '.git'))
+    except OSError as e:
+      print('Failed to remove %s (%s)' % (os.path.join(path, '.git'), str(e)), file=sys.stderr)
+      print('error: Failed to delete obsolete path %s' % path, file=sys.stderr)
+      print('       remove manually, then run sync again', file=sys.stderr)
+      return 1
+
+    # Delete everything under the worktree, except for directories that contain
+    # another git project
+    dirs_to_remove = []
+    failed = False
+    for root, dirs, files in platform_utils.walk(path):
+      for f in files:
+        try:
+          platform_utils.remove(os.path.join(root, f))
+        except OSError as e:
+          print('Failed to remove %s (%s)' % (os.path.join(root, f), str(e)), file=sys.stderr)
+          failed = True
+      dirs[:] = [d for d in dirs
+                 if not os.path.lexists(os.path.join(root, d, '.git'))]
+      dirs_to_remove += [os.path.join(root, d) for d in dirs
+                         if os.path.join(root, d) not in dirs_to_remove]
+    for d in reversed(dirs_to_remove):
+      if platform_utils.islink(d):
+        try:
+          platform_utils.remove(d)
+        except OSError as e:
+          print('Failed to remove %s (%s)' % (os.path.join(root, d), str(e)), file=sys.stderr)
+          failed = True
+      elif len(platform_utils.listdir(d)) == 0:
+        try:
+          platform_utils.rmdir(d)
+        except OSError as e:
+          print('Failed to remove %s (%s)' % (os.path.join(root, d), str(e)), file=sys.stderr)
+          failed = True
+          continue
+    if failed:
+      print('error: Failed to delete obsolete path %s' % path, file=sys.stderr)
+      print('       remove manually, then run sync again', file=sys.stderr)
+      return 1
+
+    # Try deleting parent dirs if they are empty
+    project_dir = path
+    while project_dir != self.manifest.topdir:
+      if len(platform_utils.listdir(project_dir)) == 0:
+        platform_utils.rmdir(project_dir)
+      else:
+        break
+      project_dir = os.path.dirname(project_dir)
+
+    return 0
+
+  def UpdateProjectList(self, opt):
     new_project_paths = []
     for project in self.GetProjects(None, missing_ok=True):
       if project.relpath:
@@ -469,13 +697,14 @@ later is required to fix a server side protocol bug.
         old_project_paths = fd.read().split('\n')
       finally:
         fd.close()
-      for path in old_project_paths:
+      # In reversed order, so subfolders are deleted before parent folder.
+      for path in sorted(old_project_paths, reverse=True):
         if not path:
           continue
         if path not in new_project_paths:
           # If the path has already been deleted, we don't need to do it
-          if os.path.exists(self.manifest.topdir + '/' + path):
+          gitdir = os.path.join(self.manifest.topdir, path, '.git')
-            gitdir = os.path.join(self.manifest.topdir, path, '.git')
+          if os.path.exists(gitdir):
             project = Project(
                            manifest = self.manifest,
                            name = path,
@@ -488,24 +717,18 @@ later is required to fix a server side protocol bug.
                            revisionId = None,
                            groups = None)
 
-            if project.IsDirty():
+            if project.IsDirty() and opt.force_remove_dirty:
+              print('WARNING: Removing dirty project "%s": uncommitted changes '
+                    'erased' % project.relpath, file=sys.stderr)
+              self._DeleteProject(project.worktree)
+            elif project.IsDirty():
               print('error: Cannot remove project "%s": uncommitted changes '
                     'are present' % project.relpath, file=sys.stderr)
               print('       commit changes, then run sync again',
                     file=sys.stderr)
-              return -1
+              return 1
-            else:
+            elif self._DeleteProject(project.worktree):
-              print('Deleting obsolete path %s' % project.worktree,
+              return 1
-                    file=sys.stderr)
-              shutil.rmtree(project.worktree)
-              # Try deleting parent subdirs if they are empty
-              project_dir = os.path.dirname(project.worktree)
-              while project_dir != self.manifest.topdir:
-                try:
-                  os.rmdir(project_dir)
-                except OSError:
-                  break
-                project_dir = os.path.dirname(project_dir)
 
     new_project_paths.sort()
     fd = open(file_path, 'w')
@@ -516,140 +739,141 @@ later is required to fix a server side protocol bug.
       fd.close()
     return 0
 
+  def _SmartSyncSetup(self, opt, smart_sync_manifest_path):
+    if not self.manifest.manifest_server:
+      print('error: cannot smart sync: no manifest server defined in '
+            'manifest', file=sys.stderr)
+      sys.exit(1)
+
+    manifest_server = self.manifest.manifest_server
+    if not opt.quiet:
+      print('Using manifest server %s' % manifest_server)
+
+    if not '@' in manifest_server:
+      username = None
+      password = None
+      if opt.manifest_server_username and opt.manifest_server_password:
+        username = opt.manifest_server_username
+        password = opt.manifest_server_password
+      else:
+        try:
+          info = netrc.netrc()
+        except IOError:
+          # .netrc file does not exist or could not be opened
+          pass
+        else:
+          try:
+            parse_result = urllib.parse.urlparse(manifest_server)
+            if parse_result.hostname:
+              auth = info.authenticators(parse_result.hostname)
+              if auth:
+                username, _account, password = auth
+              else:
+                print('No credentials found for %s in .netrc'
+                      % parse_result.hostname, file=sys.stderr)
+          except netrc.NetrcParseError as e:
+            print('Error parsing .netrc file: %s' % e, file=sys.stderr)
+
+      if (username and password):
+        manifest_server = manifest_server.replace('://', '://%s:%s@' %
+                                                  (username, password),
+                                                  1)
+
+    transport = PersistentTransport(manifest_server)
+    if manifest_server.startswith('persistent-'):
+      manifest_server = manifest_server[len('persistent-'):]
+
+    try:
+      server = xmlrpc.client.Server(manifest_server, transport=transport)
+      if opt.smart_sync:
+        p = self.manifest.manifestProject
+        b = p.GetBranch(p.CurrentBranch)
+        branch = b.merge
+        if branch.startswith(R_HEADS):
+          branch = branch[len(R_HEADS):]
+
+        env = os.environ.copy()
+        if 'SYNC_TARGET' in env:
+          target = env['SYNC_TARGET']
+          [success, manifest_str] = server.GetApprovedManifest(branch, target)
+        elif 'TARGET_PRODUCT' in env and 'TARGET_BUILD_VARIANT' in env:
+          target = '%s-%s' % (env['TARGET_PRODUCT'],
+                              env['TARGET_BUILD_VARIANT'])
+          [success, manifest_str] = server.GetApprovedManifest(branch, target)
+        else:
+          [success, manifest_str] = server.GetApprovedManifest(branch)
+      else:
+        assert(opt.smart_tag)
+        [success, manifest_str] = server.GetManifest(opt.smart_tag)
+
+      if success:
+        manifest_name = os.path.basename(smart_sync_manifest_path)
+        try:
+          f = open(smart_sync_manifest_path, 'w')
+          try:
+            f.write(manifest_str)
+          finally:
+            f.close()
+        except IOError as e:
+          print('error: cannot write manifest to %s:\n%s'
+                % (smart_sync_manifest_path, e),
+                file=sys.stderr)
+          sys.exit(1)
+        self._ReloadManifest(manifest_name)
+      else:
+        print('error: manifest server RPC call failed: %s' %
+              manifest_str, file=sys.stderr)
+        sys.exit(1)
+    except (socket.error, IOError, xmlrpc.client.Fault) as e:
+      print('error: cannot connect to manifest server %s:\n%s'
+            % (self.manifest.manifest_server, e), file=sys.stderr)
+      sys.exit(1)
+    except xmlrpc.client.ProtocolError as e:
+      print('error: cannot connect to manifest server %s:\n%d %s'
+            % (self.manifest.manifest_server, e.errcode, e.errmsg),
+            file=sys.stderr)
+      sys.exit(1)
+
+    return manifest_name
+
+  def ValidateOptions(self, opt, args):
+    if opt.force_broken:
+      print('warning: -f/--force-broken is now the default behavior, and the '
+            'options are deprecated', file=sys.stderr)
+    if opt.network_only and opt.detach_head:
+      self.OptionParser.error('cannot combine -n and -d')
+    if opt.network_only and opt.local_only:
+      self.OptionParser.error('cannot combine -n and -l')
+    if opt.manifest_name and opt.smart_sync:
+      self.OptionParser.error('cannot combine -m and -s')
+    if opt.manifest_name and opt.smart_tag:
+      self.OptionParser.error('cannot combine -m and -t')
+    if opt.manifest_server_username or opt.manifest_server_password:
+      if not (opt.smart_sync or opt.smart_tag):
+        self.OptionParser.error('-u and -p may only be combined with -s or -t')
+      if None in [opt.manifest_server_username, opt.manifest_server_password]:
+        self.OptionParser.error('both -u and -p must be given')
+
   def Execute(self, opt, args):
     if opt.jobs:
       self.jobs = opt.jobs
     if self.jobs > 1:
       soft_limit, _ = _rlimit_nofile()
-      self.jobs = min(self.jobs, (soft_limit - 5) / 3)
+      self.jobs = min(self.jobs, (soft_limit - 5) // 3)
-
-    if opt.network_only and opt.detach_head:
-      print('error: cannot combine -n and -d', file=sys.stderr)
-      sys.exit(1)
-    if opt.network_only and opt.local_only:
-      print('error: cannot combine -n and -l', file=sys.stderr)
-      sys.exit(1)
-    if opt.manifest_name and opt.smart_sync:
-      print('error: cannot combine -m and -s', file=sys.stderr)
-      sys.exit(1)
-    if opt.manifest_name and opt.smart_tag:
-      print('error: cannot combine -m and -t', file=sys.stderr)
-      sys.exit(1)
-    if opt.manifest_server_username or opt.manifest_server_password:
-      if not (opt.smart_sync or opt.smart_tag):
-        print('error: -u and -p may only be combined with -s or -t',
-              file=sys.stderr)
-        sys.exit(1)
-      if None in [opt.manifest_server_username, opt.manifest_server_password]:
-        print('error: both -u and -p must be given', file=sys.stderr)
-        sys.exit(1)
 
     if opt.manifest_name:
       self.manifest.Override(opt.manifest_name)
 
     manifest_name = opt.manifest_name
-    smart_sync_manifest_name = "smart_sync_override.xml"
     smart_sync_manifest_path = os.path.join(
-      self.manifest.manifestProject.worktree, smart_sync_manifest_name)
+      self.manifest.manifestProject.worktree, 'smart_sync_override.xml')
 
     if opt.smart_sync or opt.smart_tag:
-      if not self.manifest.manifest_server:
+      manifest_name = self._SmartSyncSetup(opt, smart_sync_manifest_path)
-        print('error: cannot smart sync: no manifest server defined in '
+    else:
-              'manifest', file=sys.stderr)
-        sys.exit(1)
-
-      manifest_server = self.manifest.manifest_server
-      if not opt.quiet:
-        print('Using manifest server %s' % manifest_server)
-
-      if not '@' in manifest_server:
-        username = None
-        password = None
-        if opt.manifest_server_username and opt.manifest_server_password:
-          username = opt.manifest_server_username
-          password = opt.manifest_server_password
-        else:
-          try:
-            info = netrc.netrc()
-          except IOError:
-            # .netrc file does not exist or could not be opened
-            pass
-          else:
-            try:
-              parse_result = urllib.parse.urlparse(manifest_server)
-              if parse_result.hostname:
-                auth = info.authenticators(parse_result.hostname)
-                if auth:
-                  username, _account, password = auth
-                else:
-                  print('No credentials found for %s in .netrc'
-                        % parse_result.hostname, file=sys.stderr)
-            except netrc.NetrcParseError as e:
-              print('Error parsing .netrc file: %s' % e, file=sys.stderr)
-
-        if (username and password):
-          manifest_server = manifest_server.replace('://', '://%s:%s@' %
-                                                    (username, password),
-                                                    1)
-
-      transport = PersistentTransport(manifest_server)
-      if manifest_server.startswith('persistent-'):
-        manifest_server = manifest_server[len('persistent-'):]
-
-      try:
-        server = xmlrpc.client.Server(manifest_server, transport=transport)
-        if opt.smart_sync:
-          p = self.manifest.manifestProject
-          b = p.GetBranch(p.CurrentBranch)
-          branch = b.merge
-          if branch.startswith(R_HEADS):
-            branch = branch[len(R_HEADS):]
-
-          env = os.environ.copy()
-          if 'SYNC_TARGET' in env:
-            target = env['SYNC_TARGET']
-            [success, manifest_str] = server.GetApprovedManifest(branch, target)
-          elif 'TARGET_PRODUCT' in env and 'TARGET_BUILD_VARIANT' in env:
-            target = '%s-%s' % (env['TARGET_PRODUCT'],
-                                env['TARGET_BUILD_VARIANT'])
-            [success, manifest_str] = server.GetApprovedManifest(branch, target)
-          else:
-            [success, manifest_str] = server.GetApprovedManifest(branch)
-        else:
-          assert(opt.smart_tag)
-          [success, manifest_str] = server.GetManifest(opt.smart_tag)
-
-        if success:
-          manifest_name = smart_sync_manifest_name
-          try:
-            f = open(smart_sync_manifest_path, 'w')
-            try:
-              f.write(manifest_str)
-            finally:
-              f.close()
-          except IOError as e:
-            print('error: cannot write manifest to %s:\n%s'
-                  % (smart_sync_manifest_path, e),
-                  file=sys.stderr)
-            sys.exit(1)
-          self._ReloadManifest(manifest_name)
-        else:
-          print('error: manifest server RPC call failed: %s' %
-                manifest_str, file=sys.stderr)
-          sys.exit(1)
-      except (socket.error, IOError, xmlrpc.client.Fault) as e:
-        print('error: cannot connect to manifest server %s:\n%s'
-              % (self.manifest.manifest_server, e), file=sys.stderr)
-        sys.exit(1)
-      except xmlrpc.client.ProtocolError as e:
-        print('error: cannot connect to manifest server %s:\n%d %s'
-              % (self.manifest.manifest_server, e.errcode, e.errmsg),
-              file=sys.stderr)
-        sys.exit(1)
-    else:  # Not smart sync or smart tag mode
       if os.path.isfile(smart_sync_manifest_path):
         try:
-          os.remove(smart_sync_manifest_path)
+          platform_utils.remove(smart_sync_manifest_path)
         except OSError as e:
           print('error: failed to remove existing smart sync override manifest: %s' %
                 e, file=sys.stderr)
@@ -664,15 +888,25 @@ later is required to fix a server side protocol bug.
       _PostRepoUpgrade(self.manifest, quiet=opt.quiet)
 
     if not opt.local_only:
-      mp.Sync_NetworkHalf(quiet=opt.quiet,
+      start = time.time()
-                          current_branch_only=opt.current_branch_only,
+      success = mp.Sync_NetworkHalf(quiet=opt.quiet,
-                          no_tags=opt.no_tags,
+                                    current_branch_only=opt.current_branch_only,
-                          optimized_fetch=opt.optimized_fetch)
+                                    no_tags=opt.no_tags,
+                                    optimized_fetch=opt.optimized_fetch,
+                                    submodules=self.manifest.HasSubmodules,
+                                    clone_filter=self.manifest.CloneFilter)
+      finish = time.time()
+      self.event_log.AddSync(mp, event_log.TASK_SYNC_NETWORK,
+                             start, finish, success)
 
     if mp.HasChanges:
       syncbuf = SyncBuffer(mp.config)
-      mp.Sync_LocalHalf(syncbuf)
+      start = time.time()
-      if not syncbuf.Finish():
+      mp.Sync_LocalHalf(syncbuf, submodules=self.manifest.HasSubmodules)
+      clean = syncbuf.Finish()
+      self.event_log.AddSync(mp, event_log.TASK_SYNC_LOCAL,
+                             start, time.time(), clean)
+      if not clean:
         sys.exit(1)
       self._ReloadManifest(manifest_name)
       if opt.jobs is None:
@@ -709,8 +943,8 @@ later is required to fix a server side protocol bug.
       # generate a new args list to represent the opened projects.
       # TODO: make this more reliable -- if there's a project name/path overlap,
       # this may choose the wrong project.
-      args = [os.path.relpath(self.manifest.paths[p].worktree, os.getcwd())
+      args = [os.path.relpath(self.manifest.paths[path].worktree, os.getcwd())
-              for p in opened_projects]
+              for path in opened_projects]
       if not args:
         return
     all_projects = self.GetProjects(args,
@@ -757,20 +991,10 @@ later is required to fix a server side protocol bug.
       # bail out now, we have no working tree
       return
 
-    if self.UpdateProjectList():
+    if self.UpdateProjectList(opt):
       sys.exit(1)
 
-    syncbuf = SyncBuffer(mp.config,
+    self._Checkout(all_projects, opt)
-                         detach_head = opt.detach_head)
-    pm = Progress('Syncing work tree', len(all_projects))
-    for project in all_projects:
-      pm.update()
-      if project.worktree:
-        project.Sync_LocalHalf(syncbuf, force_sync=opt.force_sync)
-    pm.end()
-    print(file=sys.stderr)
-    if not syncbuf.Finish():
-      sys.exit(1)
 
     # If there's a notice that's supposed to print at the end of the sync, print
     # it now...
@@ -850,6 +1074,7 @@ def _VerifyTag(project):
     return False
   return True
 
+
 class _FetchTimes(object):
   _ALPHA = 0.5
 
@@ -880,7 +1105,7 @@ class _FetchTimes(object):
           f.close()
       except (IOError, ValueError):
         try:
-          os.remove(self._path)
+          platform_utils.remove(self._path)
         except OSError:
           pass
         self._times = {}
@@ -904,7 +1129,7 @@ class _FetchTimes(object):
         f.close()
     except (IOError, TypeError):
       try:
-        os.remove(self._path)
+        platform_utils.remove(self._path)
       except OSError:
         pass
 

subcmds/upload.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #
@@ -25,12 +26,10 @@ from git_command import GitCommand
 from project import RepoHook
 
 from pyversion import is_python3
-# pylint:disable=W0622
 if not is_python3():
   input = raw_input
 else:
   unicode = str
-# pylint:enable=W0622
 
 UNUSUAL_COMMIT_THRESHOLD = 5
 
@@ -80,8 +79,7 @@ added to the respective list of users, and emails are sent to any
 new users.  Users passed as --reviewers must already be registered
 with the code review system, or the upload will fail.
 
-Configuration
+# Configuration
--------------
 
 review.URL.autoupload:
 
@@ -128,10 +126,9 @@ is set to "true" then repo will assume you always want the equivalent
 of the -t option to the repo command. If unset or set to "false" then
 repo will make use of only the command line option.
 
-References
+# References
-----------
 
-Gerrit Code Review:  http://code.google.com/p/gerrit/
+Gerrit Code Review:  https://www.gerritcodereview.com/
 
 """
 
@@ -154,6 +151,19 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
     p.add_option('-d', '--draft',
                  action='store_true', dest='draft', default=False,
                  help='If specified, upload as a draft.')
+    p.add_option('--ne', '--no-emails',
+                 action='store_false', dest='notify', default=True,
+                 help='If specified, do not send emails on upload.')
+    p.add_option('-p', '--private',
+                 action='store_true', dest='private', default=False,
+                 help='If specified, upload as a private change.')
+    p.add_option('-w', '--wip',
+                 action='store_true', dest='wip', default=False,
+                 help='If specified, upload as a work-in-progress change.')
+    p.add_option('-o', '--push-option',
+                 type='string', action='append', dest='push_options',
+                 default=[],
+                 help='Additional push options to transmit')
     p.add_option('-D', '--destination', '--dest',
                  type='string', action='store', dest='dest_branch',
                  metavar='BRANCH',
@@ -175,6 +185,9 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
     #   Never run upload hooks, but upload anyway (AKA bypass hooks).
     # - no-verify=True, verify=True:
     #   Invalid
+    p.add_option('--no-cert-checks',
+                 dest='validate_certs', action='store_false', default=True,
+                 help='Disable verifying ssl certs (unsafe).')
     p.add_option('--no-verify',
                  dest='bypass_hooks', action='store_true',
                  help='Do not run the upload hook.')
@@ -198,7 +211,8 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
       commit_list = branch.commits
 
       destination = opt.dest_branch or project.dest_branch or project.revisionExpr
-      print('Upload project %s/ to remote branch %s:' % (project.relpath, destination))
+      print('Upload project %s/ to remote branch %s%s:' %
+            (project.relpath, destination, ' (draft)' if opt.draft else ''))
       print('  branch %s (%2d commit%s, %s):' % (
                     name,
                     len(commit_list),
@@ -207,7 +221,9 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
       for commit in commit_list:
         print('         %s' % commit)
 
-      sys.stdout.write('to %s (y/N)? ' % remote.review)
+      print('to %s (y/N)? ' % remote.review, end='')
+      # TODO: When we require Python 3, use flush=True w/print above.
+      sys.stdout.flush()
       answer = sys.stdin.readline().strip().lower()
       answer = answer in ('y', 'yes', '1', 'true', 't')
 
@@ -346,10 +362,13 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
 
           # if they want to auto upload, let's not ask because it could be automated
           if answer is None:
-            sys.stdout.write('Uncommitted changes in ' + branch.project.name)
+            print()
-            sys.stdout.write(' (did you forget to amend?):\n')
+            print('Uncommitted changes in %s (did you forget to amend?):'
-            sys.stdout.write('\n'.join(changes) + '\n')
+                  % branch.project.name)
-            sys.stdout.write('Continue uploading? (y/N) ')
+            print('\n'.join(changes))
+            print('Continue uploading? (y/N) ', end='')
+            # TODO: When we require Python 3, use flush=True w/print above.
+            sys.stdout.flush()
             a = sys.stdin.readline().strip().lower()
             if a not in ('y', 'yes', 't', 'true', 'on'):
               print("skipping upload", file=sys.stderr)
@@ -377,7 +396,16 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
             branch.uploaded = False
             continue
 
-        branch.UploadForReview(people, auto_topic=opt.auto_topic, draft=opt.draft, dest_branch=destination)
+        branch.UploadForReview(people,
+                               auto_topic=opt.auto_topic,
+                               draft=opt.draft,
+                               private=opt.private,
+                               notify=None if opt.notify else 'NONE',
+                               wip=opt.wip,
+                               dest_branch=destination,
+                               validate_certs=opt.validate_certs,
+                               push_options=opt.push_options)
+
         branch.uploaded = True
       except UploadError as e:
         branch.error = e
@@ -454,11 +482,17 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
       if avail:
         pending.append((project, avail))
 
-    if pending and (not opt.bypass_hooks):
+    if not pending:
+      print("no branches ready for upload", file=sys.stderr)
+      return
+
+    if not opt.bypass_hooks:
       hook = RepoHook('pre-upload', self.manifest.repo_hooks_project,
-                      self.manifest.topdir, abort_if_user_denies=True)
+                      self.manifest.topdir,
-      pending_proj_names = [project.name for (project, avail) in pending]
+                      self.manifest.manifestProject.GetRemote('origin').url,
-      pending_worktrees = [project.worktree for (project, avail) in pending]
+                      abort_if_user_denies=True)
+      pending_proj_names = [project.name for (project, available) in pending]
+      pending_worktrees = [project.worktree for (project, available) in pending]
       try:
         hook.Run(opt.allow_all_hooks, project_list=pending_proj_names,
                  worktree_list=pending_worktrees)
@@ -472,9 +506,7 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
       cc = _SplitEmails(opt.cc)
     people = (reviewers, cc)
 
-    if not pending:
+    if len(pending) == 1 and len(pending[0][1]) == 1:
-      print("no branches ready for upload", file=sys.stderr)
-    elif len(pending) == 1 and len(pending[0][1]) == 1:
       self._SingleBranch(opt, pending[0][1][0], people)
     else:
       self._MultipleBranches(opt, pending, people)

subcmds/version.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2009 The Android Open Source Project
 #
@@ -40,5 +41,5 @@ class Version(Command, MirrorSafeCommand):
       print('repo launcher version %s' % Version.wrapper_version)
       print('       (from %s)' % Version.wrapper_path)
 
-    print(git.version().strip())
+    print('git %s' % git.version_tuple().full)
     print('Python %s' % sys.version)

tests/fixtures/.gitignore

@@ -0,0 +1,2 @@
+/.repo_not.present.gitconfig.json
+/.repo_test.gitconfig.json

tests/test_git_command.py

@@ -0,0 +1,49 @@
+# -*- coding:utf-8 -*-
+#
+# Copyright 2019 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Unittests for the git_command.py module."""
+
+from __future__ import print_function
+
+import unittest
+
+import git_command
+
+
+class GitCallUnitTest(unittest.TestCase):
+  """Tests the _GitCall class (via git_command.git)."""
+
+  def test_version_tuple(self):
+    """Check git.version_tuple() handling."""
+    ver = git_command.git.version_tuple()
+    self.assertIsNotNone(ver)
+
+    # We don't dive too deep into the values here to avoid having to update
+    # whenever git versions change.  We do check relative to this min version
+    # as this is what `repo` itself requires via MIN_GIT_VERSION.
+    MIN_GIT_VERSION = (1, 7, 2)
+    self.assertTrue(isinstance(ver.major, int))
+    self.assertTrue(isinstance(ver.minor, int))
+    self.assertTrue(isinstance(ver.micro, int))
+
+    self.assertGreater(ver.major, MIN_GIT_VERSION[0] - 1)
+    self.assertGreaterEqual(ver.micro, 0)
+    self.assertGreaterEqual(ver.major, 0)
+
+    self.assertGreaterEqual(ver, MIN_GIT_VERSION)
+    self.assertLess(ver, (9999, 9999, 9999))
+
+    self.assertNotEqual('', ver.full)

tests/test_git_config.py

@@ -1,3 +1,23 @@
+# -*- coding:utf-8 -*-
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Unittests for the git_config.py module."""
+
+from __future__ import print_function
+
 import os
 import unittest
 

tests/test_project.py

@@ -0,0 +1,62 @@
+# -*- coding:utf-8 -*-
+#
+# Copyright (C) 2019 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Unittests for the project.py module."""
+
+from __future__ import print_function
+
+import unittest
+
+import project
+
+
+class RepoHookShebang(unittest.TestCase):
+  """Check shebang parsing in RepoHook."""
+
+  def test_no_shebang(self):
+    """Lines w/out shebangs should be rejected."""
+    DATA = (
+        '',
+        '# -*- coding:utf-8 -*-\n',
+        '#\n# foo\n',
+        '# Bad shebang in script\n#!/foo\n'
+    )
+    for data in DATA:
+      self.assertIsNone(project.RepoHook._ExtractInterpFromShebang(data))
+
+  def test_direct_interp(self):
+    """Lines whose shebang points directly to the interpreter."""
+    DATA = (
+        ('#!/foo', '/foo'),
+        ('#! /foo', '/foo'),
+        ('#!/bin/foo ', '/bin/foo'),
+        ('#! /usr/foo ', '/usr/foo'),
+        ('#! /usr/foo -args', '/usr/foo'),
+    )
+    for shebang, interp in DATA:
+      self.assertEqual(project.RepoHook._ExtractInterpFromShebang(shebang),
+                       interp)
+
+  def test_env_interp(self):
+    """Lines whose shebang launches through `env`."""
+    DATA = (
+        ('#!/usr/bin/env foo', 'foo'),
+        ('#!/bin/env foo', 'foo'),
+        ('#! /bin/env /bin/foo ', '/bin/foo'),
+    )
+    for shebang, interp in DATA:
+      self.assertEqual(project.RepoHook._ExtractInterpFromShebang(shebang),
+                       interp)

tests/test_wrapper.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2015 The Android Open Source Project
 #
@@ -13,6 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+"""Unittests for the wrapper.py module."""
+
+from __future__ import print_function
+
 import os
 import unittest
 

wrapper.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2014 The Android Open Source Project
 #
@@ -15,7 +15,12 @@
 # limitations under the License.
 
 from __future__ import print_function
-import imp
+try:
+  from importlib.machinery import SourceFileLoader
+  _loader = lambda *args: SourceFileLoader(*args).load_module()
+except ImportError:
+  import imp
+  _loader = lambda *args: imp.load_source(*args)
 import os
 
 
@@ -26,5 +31,5 @@ _wrapper_module = None
 def Wrapper():
   global _wrapper_module
   if not _wrapper_module:
-    _wrapper_module = imp.load_source('wrapper', WrapperPath())
+    _wrapper_module = _loader('wrapper', WrapperPath())
   return _wrapper_module