README: add install details

Change-Id: I57043449a7927068fa5735cb71633353e1039532
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/245816
Reviewed-by: David Pursehouse <dpursehouse@collab.net>
Tested-by: Mike Frysinger <vapier@google.com>

.pydevproject

@@ -5,6 +5,6 @@
 <pydev_pathproperty name="org.python.pydev.PROJECT_SOURCE_PATH">
 <path>/git-repo</path>
 </pydev_pathproperty>
-<pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python 2.6</pydev_property>
+<pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python 2.7</pydev_property>
 <pydev_property name="org.python.pydev.PYTHON_PROJECT_INTERPRETER">Default</pydev_property>
 </pydev_project>

README.md

@@ -7,10 +7,29 @@ easier to work with Git.  The repo command is an executable Python script
 that you can put anywhere in your path.
 
 * Homepage: https://gerrit.googlesource.com/git-repo/
-* Bug reports: https://code.google.com/p/git-repo/issues/
+* Bug reports: https://bugs.chromium.org/p/gerrit/issues/list?q=component:repo
 * Source: https://gerrit.googlesource.com/git-repo/
 * Overview: https://source.android.com/source/developing.html
 * Docs: https://source.android.com/source/using-repo.html
 * [repo Manifest Format](./docs/manifest-format.md)
 * [repo Hooks](./docs/repo-hooks.md)
 * [Submitting patches](./SUBMITTING_PATCHES.md)
+
+## Install
+
+Many distros include repo, so you might be able to install from there.
+```sh
+# Debian/Ubuntu.
+$ sudo apt-get install repo
+
+# Gentoo.
+$ sudo emerge dev-vcs/repo
+```
+
+You can install it manually as well as it's a single script.
+```sh
+$ mkdir -p ~/.bin
+$ PATH="${HOME}/.bin:${PATH}"
+$ curl https://storage.googleapis.com/git-repo-downloads/repo > ~/.bin/repo
+$ chmod a+rx ~/.bin/repo
+```

SUBMITTING_PATCHES.md

@@ -1,3 +1,5 @@
+[TOC]
+
 # Short Version
 
  - Make small logical changes.
@@ -52,17 +54,29 @@ Run `flake8` on changes modules:
 
     flake8 file.py
 
-Note that repo generally follows [Google's python style guide]
-(https://google.github.io/styleguide/pyguide.html) rather than [PEP 8]
-(https://www.python.org/dev/peps/pep-0008/), so it's possible that
-the output of `flake8` will be quite noisy. It's not mandatory to
-avoid all warnings, but at least the maximum line length should be
-followed.
+Note that repo generally follows [Google's python style guide] rather than
+[PEP 8], so it's possible that the output of `flake8` will be quite noisy.
+It's not mandatory to avoid all warnings, but at least the maximum line
+length should be followed.
 
 If there are many occurrences of the same warning that cannot be
 avoided without going against the Google style guide, these may be
 suppressed in the included `.flake8` file.
 
+[Google's python style guide]: https://google.github.io/styleguide/pyguide.html
+[PEP 8]: https://www.python.org/dev/peps/pep-0008/
+
+
+## Running tests
+
+There is a [`./run_tests`](./run_tests) helper script for quickly invoking all
+of our unittests.  The coverage isn't great currently, but it should still be
+run for all commits.
+
+Adding more unittests for changes you make would be greatly appreciated :).
+Check out the [tests/](./tests/) subdirectory for more details.
+
+
 ## Check the license
 
 repo is licensed under the Apache License, 2.0.

color.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #

command.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #
@@ -97,6 +98,16 @@ class Command(object):
     self.OptionParser.print_usage()
     sys.exit(1)
 
+  def ValidateOptions(self, opt, args):
+    """Validate the user options & arguments before executing.
+
+    This is meant to help break the code up into logical steps.  Some tips:
+    * Use self.OptionParser.error to display CLI related errors.
+    * Adjust opt member defaults as makes sense.
+    * Adjust the args list, but do so inplace so the caller sees updates.
+    * Try to avoid updating self state.  Leave that to Execute.
+    """
+
   def Execute(self, opt, args):
     """Perform the action, after option parsing is complete.
     """
@@ -164,7 +175,10 @@ class Command(object):
       self._ResetPathToProjectMap(all_projects_list)
 
       for arg in args:
-        projects = manifest.GetProjectsWithName(arg)
+        # We have to filter by manifest groups in case the requested project is
+        # checked out multiple times or differently based on them.
+        projects = [project for project in manifest.GetProjectsWithName(arg)
+                    if project.MatchesGroups(groups)]
 
         if not projects:
           path = os.path.abspath(arg).replace('\\', '/')
@@ -189,7 +203,7 @@ class Command(object):
 
         for project in projects:
           if not missing_ok and not project.Exists:
-            raise NoSuchProjectError(arg)
+            raise NoSuchProjectError('%s (%s)' % (arg, project.relpath))
           if not project.MatchesGroups(groups):
             raise InvalidProjectGroupsError(arg)
 

docs/manifest-format.md

@@ -66,7 +66,7 @@ following DTD:
   <!ATTLIST project groups      CDATA #IMPLIED>
   <!ATTLIST project sync-c      CDATA #IMPLIED>
   <!ATTLIST project sync-s      CDATA #IMPLIED>
-  <!ATTLIST default sync-tags   CDATA #IMPLIED>
+  <!ATTLIST project sync-tags   CDATA #IMPLIED>
   <!ATTLIST project upstream CDATA #IMPLIED>
   <!ATTLIST project clone-depth CDATA #IMPLIED>
   <!ATTLIST project force-path CDATA #IMPLIED>
@@ -322,13 +322,29 @@ Zero or more copyfile elements may be specified as children of a
 project element. Each element describes a src-dest pair of files;
 the "src" file will be copied to the "dest" place during `repo sync`
 command.
+
 "src" is project relative, "dest" is relative to the top of the tree.
+Copying from paths outside of the project or to paths outside of the repo
+client is not allowed.
+
+"src" and "dest" must be files.  Directories or symlinks are not allowed.
+Intermediate paths must not be symlinks either.
+
+Parent directories of "dest" will be automatically created if missing.
 
 ### Element linkfile
 
 It's just like copyfile and runs at the same time as copyfile but
 instead of copying it creates a symlink.
 
+The symlink is created at "dest" (relative to the top of the tree) and
+points to the path specified by "src".
+
+Parent directories of "dest" will be automatically created if missing.
+
+The symlink target may be a file or directory, but it may not point outside
+of the repo client.
+
 ### Element remove-project
 
 Deletes the named project from the internal manifest table, possibly

docs/python-support.md

@@ -0,0 +1,47 @@
+# Supported Python Versions
+
+With Python 2.7 officially going EOL on [01 Jan 2020](https://pythonclock.org/),
+we need a support plan for the repo project itself.
+Inevitably, there will be a long tail of users who still want to use Python 2 on
+their old LTS/corp systems and have little power to change the system.
+
+## Summary
+
+* Python 3.6 (released Dec 2016) is required by default starting with repo-2.x.
+* Older versions of Python (e.g. v2.7) may use the legacy feature-frozen branch
+  based on repo-1.x.
+
+## Overview
+
+We provide a branch for Python 2 users that is feature-frozen.
+Bugfixes may be added on a best-effort basis or from the community, but largely
+no new features will be added, nor is support guaranteed.
+
+Users can select this during `repo init` time via the [repo launcher].
+Otherwise the default branches (e.g. stable & master) will be used which will
+require Python 3.
+
+This means the [repo launcher] needs to support both Python 2 & Python 3, but
+since it doesn't import any other repo code, this shouldn't be too problematic.
+
+The master branch will require Python 3.6 at a minimum.
+If the system has an older version of Python 3, then users will have to select
+the legacy Python 2 branch instead.
+
+### repo hooks
+
+Projects that use [repo hooks] run on independent schedules.
+They might migrate to Python 3 earlier or later than us.
+To support them, we'll probe the shebang of the hook script and if we find an
+interpreter in there that indicates a different version than repo is currently
+running under, we'll attempt to reexec ourselves under that.
+
+For example, a hook with a header like `#!/usr/bin/python2` will have repo
+execute `/usr/bin/python2` to execute the hook code specifically if repo is
+currently running Python 3.
+
+For more details, consult the [repo hooks] documentation.
+
+
+[repo hooks]: ./repo-hooks.md
+[repo launcher]: ../repo

docs/release-process.md

@@ -0,0 +1,167 @@
+# repo release process
+
+This is the process for creating a new release of repo, as well as all the
+related topics and flows.
+
+[TOC]
+
+## Launcher script
+
+The main repo script serves as a standalone program and is often referred to as
+the "launcher script".
+This makes it easy to copy around and install as you don't have to install any
+other files from the git repo.
+
+Whenever major changes are made to the launcher script, you should increment the
+`VERSION` variable in the launcher itself.
+At runtime, repo will check this to see if it needs to be updated (and notify
+the user automatically).
+
+## Key management
+
+Every release has a git tag that is signed with a key that repo recognizes.
+Those keys are hardcoded inside of the repo launcher itself -- look for the
+`KEYRING_VERSION` and `MAINTAINER_KEYS` settings.
+
+Adding new keys to the repo launcher will allow tags to be recognized by new
+keys, but only people using that updated version will be able to.
+Since the majority of users will be using an official launcher version, their
+version will simply ignore any new signed tags.
+
+If you want to add new keys, it's best to register them long ahead of time,
+and then wait for that updated launcher to make its way out to everyone.
+Even then, there will be a long tail of users with outdated launchers, so be
+prepared for people asking questions.
+
+### Registering a new key
+
+The process of actually adding a new key is quite simple.
+
+1.  Add the public half of the key to `MAINTAINER_KEYS`.
+2.  Increment `KEYRING_VERSION` so repo knows it needs to update.
+3.  Wait a long time after that version is in a release (~months) before trying
+    to create a new release using those new keys.
+
+## Self update algorithm
+
+When creating a new repo checkout with `repo init`, there are a few options that
+control how repo finds updates:
+
+*   `--repo-url`: This tells repo where to clone the full repo project itself.
+    It defaults to the official project (`REPO_URL` in the launcher script).
+*   `--repo-branch`: This tells repo which branch to use for the full project.
+    It defaults to the `stable` branch (`REPO_REV` in the launcher script).
+
+Whenever `repo sync` is run, repo will check to see if an update is available.
+It fetches the latest repo-branch from the repo-url.
+Then it verifies that the latest commit in the branch has a valid signed tag
+using `git tag -v` (which uses gpg).
+If the tag is valid, then repo will update its internal checkout to it.
+
+If the latest commit doesn't have a signed tag, repo will fall back to the
+most recent tag it can find (via `git describe`).
+If that tag is valid, then repo will warn and use that commit instead.
+
+If that tag cannot be verified, it gives up and forces the user to resolve.
+
+## Branch management
+
+All development happens on the `master` branch and should generally be stable.
+
+Since the repo launcher defaults to tracking the `stable` branch, it is not
+normally updated until a new release is available.
+If something goes wrong with a new release, an older release can be force pushed
+and clients will automatically downgrade.
+
+The `maint` branch is used to track the previous major release of repo.
+It is not normally meant to be used by people as `stable` should be good enough.
+Once a new major release is pushed to the `stable` branch, then the previous
+major release can be pushed to `maint`.
+For example, when `stable` moves from `v1.10.x` to `v1.11.x`, then the `maint`
+branch will be updated from `v1.9.x` to `v1.10.x`.
+
+We don't have parallel release branches/series.
+Typically all tags are made against the `master` branch and then pushed to the
+`stable` branch to make it available to the rest of the world.
+Since repo doesn't typically see a lot of changes, this tends to be OK.
+
+## Creating a new release
+
+When you want to create a new release, you'll need to select a good version and
+create a signed tag using a key registered in repo itself.
+Typically we just tag the latest version of the `master` branch.
+The tag could be pushed now, but it won't be used by clients normally (since the
+default `repo-branch` setting is `stable`).
+This would allow some early testing on systems who explicitly select `master`.
+
+### Creating a signed tag
+
+Lets assume your keys live in a dedicated directory, e.g. `~/.gnupg/repo/`.
+
+*** note
+If you need access to the official keys, check out the internal documentation
+at [go/repo-release].
+Note that only official maintainers of repo will have access as it describes
+internal processes for accessing the restricted keys.
+***
+
+```sh
+# Set the gpg key directory.
+$ export GNUPGHOME=~/.gnupg/repo/
+
+# Verify the listed key is “Repo Maintainer”.
+$ gpg -K
+
+# Pick whatever branch or commit you want to tag.
+$ r=master
+
+# Pick the new version.
+$ t=1.12.10
+
+# Create the signed tag.
+$ git tag -s v$t -u "Repo Maintainer <repo@android.kernel.org>" -m "repo $t" $r
+
+# Verify the signed tag.
+$ git show v$t
+```
+
+### Push the new release
+
+Once you're ready to make the release available to everyone, push it to the
+`stable` branch.
+
+Make sure you never push the tag itself to the stable branch!
+Only push the commit -- notice the use of `$t` and `$r` below.
+
+```sh
+$ git push https://gerrit-review.googlesource.com/git-repo v$t
+$ git push https://gerrit-review.googlesource.com/git-repo $r:stable
+```
+
+If something goes horribly wrong, you can force push the previous version to the
+`stable` branch and people should automatically recover.
+Again, make sure you never push the tag itself!
+
+```sh
+$ oldrev="whatever-old-commit"
+$ git push https://gerrit-review.googlesource.com/git-repo $oldrev:stable --force
+```
+
+### Announce the release
+
+Once you do push a new release to `stable`, make sure to announce it on the
+[repo-discuss@googlegroups.com] group.
+Here is an [example announcement].
+
+You can create a short changelog using the command:
+
+```sh
+# If you haven't pushed to the stable branch yet, you can use origin/stable.
+# If you have pushed, change origin/stable to the previous release tag.
+$ git log --format="%h (%aN) %s" --no-merges origin/stable..$r
+```
+
+
+[example announcement]: https://groups.google.com/d/topic/repo-discuss/UGBNismWo1M/discussion
+[repo-discuss@googlegroups.com]: https://groups.google.com/forum/#!forum/repo-discuss
+[go/repo-release]: https://goto.google.com/repo-release

docs/repo-hooks.md

@@ -83,6 +83,31 @@ then check it directly.  Hooks should not normally modify the active git repo
 the user.  Although user interaction is discouraged in the common case, it can
 be useful when deploying automatic fixes.
 
+### Shebang Handling
+
+*** note
+This is intended as a transitional feature.  Hooks are expected to eventually
+migrate to Python 3 only as Python 2 is EOL & deprecated.
+***
+
+If the hook is written against a specific version of Python (either 2 or 3),
+the script can declare that explicitly.  Repo will then attempt to execute it
+under the right version of Python regardless of the version repo itself might
+be executing under.
+
+Here are the shebangs that are recognized.
+
+* `#!/usr/bin/env python` & `#!/usr/bin/python`: The hook is compatible with
+  Python 2 & Python 3.  For maximum compatibility, these are recommended.
+* `#!/usr/bin/env python2` & `#!/usr/bin/python2`: The hook requires Python 2.
+  Version specific names like `python2.7` are also recognized.
+* `#!/usr/bin/env python3` & `#!/usr/bin/python3`: The hook requires Python 3.
+  Version specific names like `python3.6` are also recognized.
+
+If no shebang is detected, or does not match the forms above, we assume that the
+hook is compatible with both Python 2 & Python 3 as if `#!/usr/bin/python` was
+used.
+
 ## Hooks
 
 Here are all the points available for hooking.

editor.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #
@@ -67,11 +68,14 @@ least one of these before using this command.""", file=sys.stderr)
   def EditString(cls, data):
     """Opens an editor to edit the given content.
 
-       Args:
-         data        : the text to edit
+    Args:
+      data: The text to edit.
 
-      Returns:
-        new value of edited text; None if editing did not succeed
+    Returns:
+      New value of edited text.
+
+    Raises:
+      EditorError: The editor failed to run.
     """
     editor = cls._GetEditor()
     if editor == ':':
@@ -79,7 +83,7 @@ least one of these before using this command.""", file=sys.stderr)
 
     fd, path = tempfile.mkstemp()
     try:
-      os.write(fd, data)
+      os.write(fd, data.encode('utf-8'))
       os.close(fd)
       fd = None
 
@@ -105,11 +109,8 @@ least one of these before using this command.""", file=sys.stderr)
         raise EditorError('editor failed with exit status %d: %s %s'
           % (rc, editor, path))
 
-      fd2 = open(path)
-      try:
-        return fd2.read()
-      finally:
-        fd2.close()
+      with open(path, mode='rb') as fd2:
+        return fd2.read().decode('utf-8')
     finally:
       if fd:
         os.close(fd)

error.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #

event_log.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2017 The Android Open Source Project
 #
@@ -18,8 +19,6 @@ from __future__ import print_function
 import json
 import multiprocessing
 
-from pyversion import is_python3
-
 TASK_COMMAND = 'command'
 TASK_SYNC_NETWORK = 'sync-network'
 TASK_SYNC_LOCAL = 'sync-local'
@@ -53,7 +52,6 @@ class EventLog(object):
   def __init__(self):
     """Initializes the event log."""
     self._log = []
-    self._next_id = _EventIdGenerator()
     self._parent = None
 
   def Add(self, name, task_name, start, finish=None, success=None,
@@ -73,7 +71,7 @@ class EventLog(object):
       A dictionary of the event added to the log.
     """
     event = {
-        'id': (kind, self._next_id.__next__() if is_python3() else self._next_id.next()),
+        'id': (kind, _NextEventId()),
         'name': name,
         'task_name': task_name,
         'start_time': start,
@@ -164,16 +162,16 @@ class EventLog(object):
         f.write('\n')
 
 
-def _EventIdGenerator():
-  """Returns multi-process safe iterator that generates locally unique id.
+# An integer id that is unique across this invocation of the program.
+_EVENT_ID = multiprocessing.Value('i', 1)
 
-  Yields:
+def _NextEventId():
+  """Helper function for grabbing the next unique id.
+
+  Returns:
     A unique, to this invocation of the program, integer id.
   """
-  eid = multiprocessing.Value('i', 1)
-
-  while True:
-    with eid.get_lock():
-      val = eid.value
-      eid.value += 1
-    yield val
+  with _EVENT_ID.get_lock():
+    val = _EVENT_ID.value
+    _EVENT_ID.value += 1
+  return val

git_command.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #
@@ -21,8 +22,9 @@ import tempfile
 from signal import SIGTERM
 
 from error import GitError
+from git_refs import HEAD
 import platform_utils
-from trace import REPO_TRACE, IsTrace, Trace
+from repo_trace import REPO_TRACE, IsTrace, Trace
 from wrapper import Wrapper
 
 GIT = 'git'
@@ -79,22 +81,12 @@ def terminate_ssh_clients():
 _git_version = None
 
 class _GitCall(object):
-  def version(self):
-    p = GitCommand(None, ['--version'], capture_stdout=True)
-    if p.Wait() == 0:
-      if hasattr(p.stdout, 'decode'):
-        return p.stdout.decode('utf-8')
-      else:
-        return p.stdout
-    return None
-
   def version_tuple(self):
     global _git_version
     if _git_version is None:
-      ver_str = git.version()
-      _git_version = Wrapper().ParseGitVersion(ver_str)
+      _git_version = Wrapper().ParseGitVersion()
       if _git_version is None:
-        print('fatal: "%s" unsupported' % ver_str, file=sys.stderr)
+        print('fatal: unable to detect git version', file=sys.stderr)
         sys.exit(1)
     return _git_version
 
@@ -107,13 +99,95 @@ class _GitCall(object):
     return fun
 git = _GitCall()
 
-def git_require(min_version, fail=False):
+
+def RepoSourceVersion():
+  """Return the version of the repo.git tree."""
+  ver = getattr(RepoSourceVersion, 'version', None)
+
+  # We avoid GitCommand so we don't run into circular deps -- GitCommand needs
+  # to initialize version info we provide.
+  if ver is None:
+    env = GitCommand._GetBasicEnv()
+
+    proj = os.path.dirname(os.path.abspath(__file__))
+    env[GIT_DIR] = os.path.join(proj, '.git')
+
+    p = subprocess.Popen([GIT, 'describe', HEAD], stdout=subprocess.PIPE,
+                         env=env)
+    if p.wait() == 0:
+      ver = p.stdout.read().strip().decode('utf-8')
+      if ver.startswith('v'):
+        ver = ver[1:]
+    else:
+      ver = 'unknown'
+    setattr(RepoSourceVersion, 'version', ver)
+
+  return ver
+
+
+class UserAgent(object):
+  """Mange User-Agent settings when talking to external services
+
+  We follow the style as documented here:
+  https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/User-Agent
+  """
+
+  _os = None
+  _repo_ua = None
+  _git_ua = None
+
+  @property
+  def os(self):
+    """The operating system name."""
+    if self._os is None:
+      os_name = sys.platform
+      if os_name.lower().startswith('linux'):
+        os_name = 'Linux'
+      elif os_name == 'win32':
+        os_name = 'Win32'
+      elif os_name == 'cygwin':
+        os_name = 'Cygwin'
+      elif os_name == 'darwin':
+        os_name = 'Darwin'
+      self._os = os_name
+
+    return self._os
+
+  @property
+  def repo(self):
+    """The UA when connecting directly from repo."""
+    if self._repo_ua is None:
+      py_version = sys.version_info
+      self._repo_ua = 'git-repo/%s (%s) git/%s Python/%d.%d.%d' % (
+          RepoSourceVersion(),
+          self.os,
+          git.version_tuple().full,
+          py_version.major, py_version.minor, py_version.micro)
+
+    return self._repo_ua
+
+  @property
+  def git(self):
+    """The UA when running git."""
+    if self._git_ua is None:
+      self._git_ua = 'git/%s (%s) git-repo/%s' % (
+          git.version_tuple().full,
+          self.os,
+          RepoSourceVersion())
+
+    return self._git_ua
+
+user_agent = UserAgent()
+
+def git_require(min_version, fail=False, msg=''):
   git_version = git.version_tuple()
   if min_version <= git_version:
     return True
   if fail:
     need = '.'.join(map(str, min_version))
-    print('fatal: git %s or later required' % need, file=sys.stderr)
+    if msg:
+      msg = ' for ' + msg
+    print('fatal: git %s or later required%s' % (need, msg), file=sys.stderr)
     sys.exit(1)
   return False
 
@@ -132,17 +206,7 @@ class GitCommand(object):
                ssh_proxy = False,
                cwd = None,
                gitdir = None):
-    env = os.environ.copy()
-
-    for key in [REPO_TRACE,
-              GIT_DIR,
-              'GIT_ALTERNATE_OBJECT_DIRECTORIES',
-              'GIT_OBJECT_DIRECTORY',
-              'GIT_WORK_TREE',
-              'GIT_GRAFT_FILE',
-              'GIT_INDEX_FILE']:
-      if key in env:
-        del env[key]
+    env = self._GetBasicEnv()
 
     # If we are not capturing std* then need to print it.
     self.tee = {'stdout': not capture_stdout, 'stderr': not capture_stderr}
@@ -162,6 +226,7 @@ class GitCommand(object):
     if 'GIT_ALLOW_PROTOCOL' not in env:
       _setenv(env, 'GIT_ALLOW_PROTOCOL',
               'file:git:http:https:ssh:persistent-http:persistent-https:sso:rpc')
+    _setenv(env, 'GIT_HTTP_USER_AGENT', user_agent.git)
 
     if project:
       if not cwd:
@@ -234,6 +299,23 @@ class GitCommand(object):
     self.process = p
     self.stdin = p.stdin
 
+  @staticmethod
+  def _GetBasicEnv():
+    """Return a basic env for running git under.
+
+    This is guaranteed to be side-effect free.
+    """
+    env = os.environ.copy()
+    for key in (REPO_TRACE,
+                GIT_DIR,
+                'GIT_ALTERNATE_OBJECT_DIRECTORIES',
+                'GIT_OBJECT_DIRECTORY',
+                'GIT_WORK_TREE',
+                'GIT_GRAFT_FILE',
+                'GIT_INDEX_FILE'):
+      env.pop(key, None)
+    return env
+
   def Wait(self):
     try:
       p = self.process

git_config.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #
@@ -43,7 +44,7 @@ else:
 from signal import SIGTERM
 from error import GitError, UploadError
 import platform_utils
-from trace import Trace
+from repo_trace import Trace
 if is_python3():
   from http.client import HTTPException
 else:
@@ -275,22 +276,16 @@ class GitConfig(object):
       return None
     try:
       Trace(': parsing %s', self.file)
-      fd = open(self._json)
-      try:
+      with open(self._json) as fd:
         return json.load(fd)
-      finally:
-        fd.close()
     except (IOError, ValueError):
       platform_utils.remove(self._json)
       return None
 
   def _SaveJson(self, cache):
     try:
-      fd = open(self._json, 'w')
-      try:
+      with open(self._json, 'w') as fd:
         json.dump(cache, fd, indent=2)
-      finally:
-        fd.close()
     except (IOError, TypeError):
       if os.path.exists(self._json):
         platform_utils.remove(self._json)
@@ -656,13 +651,14 @@ class Remote(object):
               info = urllib.request.urlopen(info_url, context=context).read()
           else:
               info = urllib.request.urlopen(info_url).read()
-          if info == 'NOT_AVAILABLE' or '<' in info:
+          if info == b'NOT_AVAILABLE' or b'<' in info:
             # If `info` contains '<', we assume the server gave us some sort
             # of HTML response back, like maybe a login page.
             #
             # Assume HTTP if SSH is not enabled or ssh_info doesn't look right.
             self._review_url = http_url
           else:
+            info = info.decode('utf-8')
             host, port = info.split()
             self._review_url = self._SshReviewUrl(userEmail, host, port)
         except urllib.error.HTTPError as e:
@@ -697,7 +693,8 @@ class Remote(object):
     if not rev.startswith(R_HEADS):
       return rev
 
-    raise GitError('remote %s does not have %s' % (self.name, rev))
+    raise GitError('%s: remote %s does not have %s' %
+                   (self.projectname, self.name, rev))
 
   def WritesTo(self, ref):
     """True if the remote stores to the tracking ref.
@@ -770,15 +767,12 @@ class Branch(object):
       self._Set('merge', self.merge)
 
     else:
-      fd = open(self._config.file, 'a')
-      try:
+      with open(self._config.file, 'a') as fd:
         fd.write('[branch "%s"]\n' % self.name)
         if self.remote:
           fd.write('\tremote = %s\n' % self.remote.name)
         if self.merge:
           fd.write('\tmerge = %s\n' % self.merge)
-      finally:
-        fd.close()
 
   def _Set(self, key, value):
     key = 'branch.%s.%s' % (self.name, key)

git_refs.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2009 The Android Open Source Project
 #
@@ -14,7 +15,7 @@
 # limitations under the License.
 
 import os
-from trace import Trace
+from repo_trace import Trace
 import platform_utils
 
 HEAD      = 'HEAD'
@@ -140,18 +141,11 @@ class GitRefs(object):
 
   def _ReadLoose1(self, path, name):
     try:
-      fd = open(path)
-    except IOError:
-      return
-
-    try:
-      try:
+      with open(path) as fd:
         mtime = os.path.getmtime(path)
         ref_id = fd.readline()
-      except (IOError, OSError):
-        return
-    finally:
-      fd.close()
+    except (IOError, OSError):
+      return
 
     try:
       ref_id = ref_id.decode()

git_ssh

@@ -1,2 +1,17 @@
 #!/bin/sh
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 exec ssh -o "ControlMaster no" -o "ControlPath $REPO_SSH_SOCK" "$@"

gitc_utils.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2015 The Android Open Source Project
 #
@@ -58,8 +59,8 @@ def _set_project_revisions(projects):
       sys.exit(1)
     revisionExpr = gitcmd.stdout.split('\t')[0]
     if not revisionExpr:
-      raise(ManifestParseError('Invalid SHA-1 revision project %s (%s)' %
-                               (proj.remote.url, proj.revisionExpr)))
+      raise ManifestParseError('Invalid SHA-1 revision project %s (%s)' %
+                               (proj.remote.url, proj.revisionExpr))
     proj.revisionExpr = revisionExpr
 
 def _manifest_groups(manifest):
@@ -87,7 +88,7 @@ def generate_gitc_manifest(gitc_manifest, manifest, paths=None):
   print('Generating GITC Manifest by fetching revision SHAs for each '
         'project.')
   if paths is None:
-    paths = manifest.paths.keys()
+    paths = list(manifest.paths.keys())
 
   groups = [x for x in re.split(r'[,\s]+', _manifest_groups(manifest)) if x]
 
@@ -96,7 +97,7 @@ def generate_gitc_manifest(gitc_manifest, manifest, paths=None):
   projects = [p for p in projects if p.MatchesGroups(groups)]
 
   if gitc_manifest is not None:
-    for path, proj in manifest.paths.iteritems():
+    for path, proj in manifest.paths.items():
       if not proj.MatchesGroups(groups):
         continue
 
@@ -124,7 +125,7 @@ def generate_gitc_manifest(gitc_manifest, manifest, paths=None):
     index += NUM_BATCH_RETRIEVE_REVISIONID
 
   if gitc_manifest is not None:
-    for path, proj in gitc_manifest.paths.iteritems():
+    for path, proj in gitc_manifest.paths.items():
       if proj.old_revision and path in paths:
         # If we updated a project that has been started, keep the old-revision
         # updated.
@@ -133,7 +134,7 @@ def generate_gitc_manifest(gitc_manifest, manifest, paths=None):
         repo_proj.revisionExpr = None
 
   # Convert URLs from relative to absolute.
-  for _name, remote in manifest.remotes.iteritems():
+  for _name, remote in manifest.remotes.items():
     remote.fetchUrl = remote.resolvedFetchUrl
 
   # Save the manifest.

main.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #
@@ -14,9 +15,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+"""The repo tool.
+
+People shouldn't run this directly; instead, they should use the `repo` wrapper
+which takes care of execing this entry point.
+"""
+
 from __future__ import print_function
 import getpass
-import imp
 import netrc
 import optparse
 import os
@@ -27,6 +33,7 @@ from pyversion import is_python3
 if is_python3():
   import urllib.request
 else:
+  import imp
   import urllib2
   urllib = imp.new_module('urllib')
   urllib.request = urllib2
@@ -38,8 +45,8 @@ except ImportError:
 
 from color import SetDefaultColoring
 import event_log
-from trace import SetTrace
-from git_command import git, GitCommand
+from repo_trace import SetTrace
+from git_command import git, GitCommand, user_agent
 from git_config import init_ssh, close_ssh
 from command import InteractiveCommand
 from command import MirrorSafeCommand
@@ -77,7 +84,10 @@ global_options.add_option('--color',
                           help='control color usage: auto, always, never')
 global_options.add_option('--trace',
                           dest='trace', action='store_true',
-                          help='trace git command execution')
+                          help='trace git command execution (REPO_TRACE=1)')
+global_options.add_option('--trace-python',
+                          dest='trace_python', action='store_true',
+                          help='trace python command execution')
 global_options.add_option('--time',
                           dest='time', action='store_true',
                           help='time repo command execution')
@@ -95,8 +105,8 @@ class _Repo(object):
     # add 'branch' as an alias for 'branches'
     all_commands['branch'] = all_commands['branches']
 
-  def _Run(self, argv):
-    result = 0
+  def _ParseArgs(self, argv):
+    """Parse the main `repo` command line options."""
     name = None
     glob = []
 
@@ -113,6 +123,12 @@ class _Repo(object):
       argv = []
     gopts, _gargs = global_options.parse_args(glob)
 
+    return (name, gopts, argv)
+
+  def _Run(self, name, gopts, argv):
+    """Execute the requested subcommand."""
+    result = 0
+
     if gopts.trace:
       SetTrace()
     if gopts.show_version:
@@ -181,6 +197,7 @@ class _Repo(object):
     cmd_event = cmd.event_log.Add(name, event_log.TASK_COMMAND, start)
     cmd.event_log.SetParent(cmd_event)
     try:
+      cmd.ValidateOptions(copts, cargs)
       result = cmd.Execute(copts, cargs)
     except (DownloadError, ManifestInvalidRevisionError,
         NoManifestException) as e:
@@ -227,10 +244,6 @@ class _Repo(object):
     return result
 
 
-def _MyRepoPath():
-  return os.path.dirname(__file__)
-
-
 def _CheckWrapperVersion(ver, repo_path):
   if not repo_path:
     repo_path = '~/bin/repo'
@@ -282,51 +295,13 @@ def _PruneOptions(argv, opt):
       continue
     i += 1
 
-_user_agent = None
-
-def _UserAgent():
-  global _user_agent
-
-  if _user_agent is None:
-    py_version = sys.version_info
-
-    os_name = sys.platform
-    if os_name == 'linux2':
-      os_name = 'Linux'
-    elif os_name == 'win32':
-      os_name = 'Win32'
-    elif os_name == 'cygwin':
-      os_name = 'Cygwin'
-    elif os_name == 'darwin':
-      os_name = 'Darwin'
-
-    p = GitCommand(
-      None, ['describe', 'HEAD'],
-      cwd = _MyRepoPath(),
-      capture_stdout = True)
-    if p.Wait() == 0:
-      repo_version = p.stdout
-      if len(repo_version) > 0 and repo_version[-1] == '\n':
-        repo_version = repo_version[0:-1]
-      if len(repo_version) > 0 and repo_version[0] == 'v':
-        repo_version = repo_version[1:]
-    else:
-      repo_version = 'unknown'
-
-    _user_agent = 'git-repo/%s (%s) git/%s Python/%d.%d.%d' % (
-      repo_version,
-      os_name,
-      '.'.join(map(str, git.version_tuple())),
-      py_version[0], py_version[1], py_version[2])
-  return _user_agent
-
 class _UserAgentHandler(urllib.request.BaseHandler):
   def http_request(self, req):
-    req.add_header('User-Agent', _UserAgent())
+    req.add_header('User-Agent', user_agent.repo)
     return req
 
   def https_request(self, req):
-    req.add_header('User-Agent', _UserAgent())
+    req.add_header('User-Agent', user_agent.repo)
     return req
 
 def _AddPasswordFromUserInput(handler, msg, req):
@@ -519,7 +494,15 @@ def _Main(argv):
     try:
       init_ssh()
       init_http()
-      result = repo._Run(argv) or 0
+      name, gopts, argv = repo._ParseArgs(argv)
+      run = lambda: repo._Run(name, gopts, argv) or 0
+      if gopts.trace_python:
+        import trace
+        tracer = trace.Trace(count=False, trace=True, timing=True,
+                             ignoredirs=set(sys.path[1:]))
+        result = tracer.runfunc(run)
+      else:
+        result = run()
     finally:
       close_ssh()
   except KeyboardInterrupt:

manifest_xml.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #
@@ -135,6 +136,7 @@ class XmlManifest(object):
     self.globalConfig = GitConfig.ForUser()
     self.localManifestWarning = False
     self.isGitcClient = False
+    self._load_local_manifests = True
 
     self.repoProject = MetaProject(self, 'repo',
       gitdir   = os.path.join(repodir, 'repo/.git'),
@@ -146,15 +148,26 @@ class XmlManifest(object):
 
     self._Unload()
 
-  def Override(self, name):
+  def Override(self, name, load_local_manifests=True):
     """Use a different manifest, just for the current instantiation.
     """
-    path = os.path.join(self.manifestProject.worktree, name)
-    if not os.path.isfile(path):
-      raise ManifestParseError('manifest %s not found' % name)
+    path = None
+
+    # Look for a manifest by path in the filesystem (including the cwd).
+    if not load_local_manifests:
+      local_path = os.path.abspath(name)
+      if os.path.isfile(local_path):
+        path = local_path
+
+    # Look for manifests by name from the manifests repo.
+    if path is None:
+      path = os.path.join(self.manifestProject.worktree, name)
+      if not os.path.isfile(path):
+        raise ManifestParseError('manifest %s not found' % name)
 
     old = self.manifestFile
     try:
+      self._load_local_manifests = load_local_manifests
       self.manifestFile = path
       self._Unload()
       self._Load()
@@ -400,6 +413,12 @@ class XmlManifest(object):
     self._Load()
     return self._manifest_server
 
+  @property
+  def CloneFilter(self):
+    if self.manifestProject.config.GetBoolean('repo.partialclone'):
+      return self.manifestProject.config.GetString('repo.clonefilter')
+    return None
+
   @property
   def IsMirror(self):
     return self.manifestProject.config.GetBoolean('repo.mirror')
@@ -435,23 +454,26 @@ class XmlManifest(object):
       nodes.append(self._ParseManifestXml(self.manifestFile,
                                           self.manifestProject.worktree))
 
-      local = os.path.join(self.repodir, LOCAL_MANIFEST_NAME)
-      if os.path.exists(local):
-        if not self.localManifestWarning:
-          self.localManifestWarning = True
-          print('warning: %s is deprecated; put local manifests in `%s` instead'
-                % (LOCAL_MANIFEST_NAME, os.path.join(self.repodir, LOCAL_MANIFESTS_DIR_NAME)),
-                file=sys.stderr)
-        nodes.append(self._ParseManifestXml(local, self.repodir))
+      if self._load_local_manifests:
+        local = os.path.join(self.repodir, LOCAL_MANIFEST_NAME)
+        if os.path.exists(local):
+          if not self.localManifestWarning:
+            self.localManifestWarning = True
+            print('warning: %s is deprecated; put local manifests '
+                  'in `%s` instead' % (LOCAL_MANIFEST_NAME,
+                  os.path.join(self.repodir, LOCAL_MANIFESTS_DIR_NAME)),
+                  file=sys.stderr)
+          nodes.append(self._ParseManifestXml(local, self.repodir))
 
-      local_dir = os.path.abspath(os.path.join(self.repodir, LOCAL_MANIFESTS_DIR_NAME))
-      try:
-        for local_file in sorted(platform_utils.listdir(local_dir)):
-          if local_file.endswith('.xml'):
-            local = os.path.join(local_dir, local_file)
-            nodes.append(self._ParseManifestXml(local, self.repodir))
-      except OSError:
-        pass
+        local_dir = os.path.abspath(os.path.join(self.repodir,
+                                    LOCAL_MANIFESTS_DIR_NAME))
+        try:
+          for local_file in sorted(platform_utils.listdir(local_dir)):
+            if local_file.endswith('.xml'):
+              local = os.path.join(local_dir, local_file)
+              nodes.append(self._ParseManifestXml(local, self.repodir))
+        except OSError:
+          pass
 
       try:
         self._ParseManifest(nodes)
@@ -498,7 +520,7 @@ class XmlManifest(object):
           raise
         except Exception as e:
           raise ManifestParseError(
-              "failed parsing included manifest %s: %s", (name, e))
+              "failed parsing included manifest %s: %s" % (name, e))
       else:
         nodes.append(node)
     return nodes

pager.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #

platform_utils.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2016 The Android Open Source Project
 #
@@ -240,14 +241,15 @@ def _makelongpath(path):
     return path
 
 
-def rmtree(path):
+def rmtree(path, ignore_errors=False):
   """shutil.rmtree(path) wrapper with support for long paths on Windows.
 
   Availability: Unix, Windows."""
+  onerror = None
   if isWindows():
-    shutil.rmtree(_makelongpath(path), onerror=handle_rmtree_error)
-  else:
-    shutil.rmtree(path)
+    path = _makelongpath(path)
+    onerror = handle_rmtree_error
+  shutil.rmtree(path, ignore_errors=ignore_errors, onerror=onerror)
 
 
 def handle_rmtree_error(function, path, excinfo):

platform_utils_win32.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2016 The Android Open Source Project
 #
@@ -17,7 +18,7 @@ import errno
 
 from ctypes import WinDLL, get_last_error, FormatError, WinError, addressof
 from ctypes import c_buffer
-from ctypes.wintypes import BOOL, LPCWSTR, DWORD, HANDLE, POINTER, c_ubyte
+from ctypes.wintypes import BOOL, BOOLEAN, LPCWSTR, DWORD, HANDLE, POINTER, c_ubyte
 from ctypes.wintypes import WCHAR, USHORT, LPVOID, Structure, Union, ULONG
 from ctypes.wintypes import byref
 
@@ -33,7 +34,7 @@ ERROR_PRIVILEGE_NOT_HELD = 1314
 
 # Win32 API entry points
 CreateSymbolicLinkW = kernel32.CreateSymbolicLinkW
-CreateSymbolicLinkW.restype = BOOL
+CreateSymbolicLinkW.restype = BOOLEAN
 CreateSymbolicLinkW.argtypes = (LPCWSTR,  # lpSymlinkFileName In
                                 LPCWSTR,  # lpTargetFileName In
                                 DWORD)    # dwFlags In
@@ -145,19 +146,12 @@ def create_dirsymlink(source, link_name):
 
 
 def _create_symlink(source, link_name, dwFlags):
-  # Note: Win32 documentation for CreateSymbolicLink is incorrect.
-  # On success, the function returns "1".
-  # On error, the function returns some random value (e.g. 1280).
-  # The best bet seems to use "GetLastError" and check for error/success.
-  CreateSymbolicLinkW(link_name, source, dwFlags | SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE)
-  code = get_last_error()
-  if code != ERROR_SUCCESS:
+  if not CreateSymbolicLinkW(link_name, source, dwFlags | SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE):
     # See https://github.com/golang/go/pull/24307/files#diff-b87bc12e4da2497308f9ef746086e4f0
     # "the unprivileged create flag is unsupported below Windows 10 (1703, v10.0.14972).
     # retry without it."
-    CreateSymbolicLinkW(link_name, source, dwFlags)
-    code = get_last_error()
-    if code != ERROR_SUCCESS:
+    if not CreateSymbolicLinkW(link_name, source, dwFlags):
+      code = get_last_error()
       error_desc = FormatError(code).strip()
       if code == ERROR_PRIVILEGE_NOT_HELD:
         raise OSError(errno.EPERM, error_desc, link_name)

progress.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2009 The Android Open Source Project
 #
@@ -16,10 +17,15 @@
 import os
 import sys
 from time import time
-from trace import IsTrace
+from repo_trace import IsTrace
 
 _NOT_TTY = not os.isatty(2)
 
+# This will erase all content in the current line (wherever the cursor is).
+# It does not move the cursor, so this is usually followed by \r to move to
+# column 0.
+CSI_ERASE_LINE = '\x1b[2K'
+
 class Progress(object):
   def __init__(self, title, total=0, units='', print_newline=False,
                always_print_percentage=False):
@@ -33,7 +39,7 @@ class Progress(object):
     self._print_newline = print_newline
     self._always_print_percentage = always_print_percentage
 
-  def update(self, inc=1):
+  def update(self, inc=1, msg=''):
     self._done += inc
 
     if _NOT_TTY or IsTrace():
@@ -46,7 +52,8 @@ class Progress(object):
         return
 
     if self._total <= 0:
-      sys.stderr.write('\r%s: %d, ' % (
+      sys.stderr.write('%s\r%s: %d,' % (
+        CSI_ERASE_LINE,
         self._title,
         self._done))
       sys.stderr.flush()
@@ -55,11 +62,13 @@ class Progress(object):
 
       if self._lastp != p or self._always_print_percentage:
         self._lastp = p
-        sys.stderr.write('\r%s: %3d%% (%d%s/%d%s)%s' % (
+        sys.stderr.write('%s\r%s: %3d%% (%d%s/%d%s)%s%s%s' % (
+          CSI_ERASE_LINE,
           self._title,
           p,
           self._done, self._units,
           self._total, self._units,
+          ' ' if msg else '', msg,
           "\n" if self._print_newline else ""))
         sys.stderr.flush()
 
@@ -68,13 +77,15 @@ class Progress(object):
       return
 
     if self._total <= 0:
-      sys.stderr.write('\r%s: %d, done.  \n' % (
+      sys.stderr.write('%s\r%s: %d, done.\n' % (
+        CSI_ERASE_LINE,
         self._title,
         self._done))
       sys.stderr.flush()
     else:
       p = (100 * self._done) / self._total
-      sys.stderr.write('\r%s: %3d%% (%d%s/%d%s), done.  \n' % (
+      sys.stderr.write('%s\r%s: %3d%% (%d%s/%d%s), done.\n' % (
+        CSI_ERASE_LINE,
         self._title,
         p,
         self._done, self._units,

project.py

@@ -1,3 +1,5 @@
+# -*- coding:utf-8 -*-
+#
 # Copyright (C) 2008 The Android Open Source Project
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,6 +18,7 @@ from __future__ import print_function
 import errno
 import filecmp
 import glob
+import json
 import os
 import random
 import re
@@ -36,7 +39,8 @@ from error import GitError, HookError, UploadError, DownloadError
 from error import ManifestInvalidRevisionError
 from error import NoManifestException
 import platform_utils
-from trace import IsTrace, Trace
+import progress
+from repo_trace import IsTrace, Trace
 
 from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M
 
@@ -54,11 +58,8 @@ else:
 def _lwrite(path, content):
   lock = '%s.lock' % path
 
-  fd = open(lock, 'w')
-  try:
+  with open(lock, 'w') as fd:
     fd.write(content)
-  finally:
-    fd.close()
 
   try:
     platform_utils.rename(lock, path)
@@ -133,6 +134,7 @@ class DownloadedChange(object):
 
 class ReviewableBranch(object):
   _commit_cache = None
+  _base_exists = None
 
   def __init__(self, project, branch, base):
     self.project = project
@@ -146,14 +148,19 @@ class ReviewableBranch(object):
   @property
   def commits(self):
     if self._commit_cache is None:
-      self._commit_cache = self.project.bare_git.rev_list('--abbrev=8',
-                                                          '--abbrev-commit',
-                                                          '--pretty=oneline',
-                                                          '--reverse',
-                                                          '--date-order',
-                                                          not_rev(self.base),
-                                                          R_HEADS + self.name,
-                                                          '--')
+      args = ('--abbrev=8', '--abbrev-commit', '--pretty=oneline', '--reverse',
+              '--date-order', not_rev(self.base), R_HEADS + self.name, '--')
+      try:
+        self._commit_cache = self.project.bare_git.rev_list(*args)
+      except GitError:
+        # We weren't able to probe the commits for this branch.  Was it tracking
+        # a branch that no longer exists?  If so, return no commits.  Otherwise,
+        # rethrow the error as we don't know what's going on.
+        if self.base_exists:
+          raise
+
+        self._commit_cache = []
+
     return self._commit_cache
 
   @property
@@ -172,6 +179,23 @@ class ReviewableBranch(object):
                                      R_HEADS + self.name,
                                      '--')
 
+  @property
+  def base_exists(self):
+    """Whether the branch we're tracking exists.
+
+    Normally it should, but sometimes branches we track can get deleted.
+    """
+    if self._base_exists is None:
+      try:
+        self.project.bare_git.rev_parse('--verify', not_rev(self.base))
+        # If we're still here, the base branch exists.
+        self._base_exists = True
+      except GitError:
+        # If we failed to verify, the base branch doesn't exist.
+        self._base_exists = False
+
+    return self._base_exists
+
   def UploadForReview(self, people,
                       auto_topic=False,
                       draft=False,
@@ -226,6 +250,7 @@ class DiffColoring(Coloring):
   def __init__(self, config):
     Coloring.__init__(self, config, 'diff')
     self.project = self.printer('header', attr='bold')
+    self.fail = self.printer('fail', fg='red')
 
 
 class _Annotation(object):
@@ -542,6 +567,105 @@ class RepoHook(object):
         prompt % (self._GetMustVerb(), self._script_fullpath),
         'Scripts have changed since %s was allowed.' % (self._hook_type,))
 
+  @staticmethod
+  def _ExtractInterpFromShebang(data):
+    """Extract the interpreter used in the shebang.
+
+    Try to locate the interpreter the script is using (ignoring `env`).
+
+    Args:
+      data: The file content of the script.
+
+    Returns:
+      The basename of the main script interpreter, or None if a shebang is not
+      used or could not be parsed out.
+    """
+    firstline = data.splitlines()[:1]
+    if not firstline:
+      return None
+
+    # The format here can be tricky.
+    shebang = firstline[0].strip()
+    m = re.match(r'^#!\s*([^\s]+)(?:\s+([^\s]+))?', shebang)
+    if not m:
+      return None
+
+    # If the using `env`, find the target program.
+    interp = m.group(1)
+    if os.path.basename(interp) == 'env':
+      interp = m.group(2)
+
+    return interp
+
+  def _ExecuteHookViaReexec(self, interp, context, **kwargs):
+    """Execute the hook script through |interp|.
+
+    Note: Support for this feature should be dropped ~Jun 2021.
+
+    Args:
+      interp: The Python program to run.
+      context: Basic Python context to execute the hook inside.
+      kwargs: Arbitrary arguments to pass to the hook script.
+
+    Raises:
+      HookError: When the hooks failed for any reason.
+    """
+    # This logic needs to be kept in sync with _ExecuteHookViaImport below.
+    script = """
+import json, os, sys
+path = '''%(path)s'''
+kwargs = json.loads('''%(kwargs)s''')
+context = json.loads('''%(context)s''')
+sys.path.insert(0, os.path.dirname(path))
+data = open(path).read()
+exec(compile(data, path, 'exec'), context)
+context['main'](**kwargs)
+""" % {
+        'path': self._script_fullpath,
+        'kwargs': json.dumps(kwargs),
+        'context': json.dumps(context),
+    }
+
+    # We pass the script via stdin to avoid OS argv limits.  It also makes
+    # unhandled exception tracebacks less verbose/confusing for users.
+    cmd = [interp, '-c', 'import sys; exec(sys.stdin.read())']
+    proc = subprocess.Popen(cmd, stdin=subprocess.PIPE)
+    proc.communicate(input=script.encode('utf-8'))
+    if proc.returncode:
+      raise HookError('Failed to run %s hook.' % (self._hook_type,))
+
+  def _ExecuteHookViaImport(self, data, context, **kwargs):
+    """Execute the hook code in |data| directly.
+
+    Args:
+      data: The code of the hook to execute.
+      context: Basic Python context to execute the hook inside.
+      kwargs: Arbitrary arguments to pass to the hook script.
+
+    Raises:
+      HookError: When the hooks failed for any reason.
+    """
+    # Exec, storing global context in the context dict.  We catch exceptions
+    # and convert to a HookError w/ just the failing traceback.
+    try:
+      exec(compile(data, self._script_fullpath, 'exec'), context)
+    except Exception:
+      raise HookError('%s\nFailed to import %s hook; see traceback above.' %
+                      (traceback.format_exc(), self._hook_type))
+
+    # Running the script should have defined a main() function.
+    if 'main' not in context:
+      raise HookError('Missing main() in: "%s"' % self._script_fullpath)
+
+    # Call the main function in the hook.  If the hook should cause the
+    # build to fail, it will raise an Exception.  We'll catch that convert
+    # to a HookError w/ just the failing traceback.
+    try:
+      context['main'](**kwargs)
+    except Exception:
+      raise HookError('%s\nFailed to run main() for %s hook; see traceback '
+                      'above.' % (traceback.format_exc(), self._hook_type))
+
   def _ExecuteHook(self, **kwargs):
     """Actually execute the given hook.
 
@@ -566,19 +690,8 @@ class RepoHook(object):
       # hooks can't import repo files.
       sys.path = [os.path.dirname(self._script_fullpath)] + sys.path[1:]
 
-      # Exec, storing global context in the context dict.  We catch exceptions
-      # and  convert to a HookError w/ just the failing traceback.
+      # Initial global context for the hook to run within.
       context = {'__file__': self._script_fullpath}
-      try:
-        exec(compile(open(self._script_fullpath).read(),
-                     self._script_fullpath, 'exec'), context)
-      except Exception:
-        raise HookError('%s\nFailed to import %s hook; see traceback above.' %
-                        (traceback.format_exc(), self._hook_type))
-
-      # Running the script should have defined a main() function.
-      if 'main' not in context:
-        raise HookError('Missing main() in: "%s"' % self._script_fullpath)
 
       # Add 'hook_should_take_kwargs' to the arguments to be passed to main.
       # We don't actually want hooks to define their main with this argument--
@@ -590,15 +703,31 @@ class RepoHook(object):
       kwargs = kwargs.copy()
       kwargs['hook_should_take_kwargs'] = True
 
-      # Call the main function in the hook.  If the hook should cause the
-      # build to fail, it will raise an Exception.  We'll catch that convert
-      # to a HookError w/ just the failing traceback.
-      try:
-        context['main'](**kwargs)
-      except Exception:
-        raise HookError('%s\nFailed to run main() for %s hook; see traceback '
-                        'above.' % (traceback.format_exc(),
-                                    self._hook_type))
+      # See what version of python the hook has been written against.
+      data = open(self._script_fullpath).read()
+      interp = self._ExtractInterpFromShebang(data)
+      reexec = False
+      if interp:
+        prog = os.path.basename(interp)
+        if prog.startswith('python2') and sys.version_info.major != 2:
+          reexec = True
+        elif prog.startswith('python3') and sys.version_info.major == 2:
+          reexec = True
+
+      # Attempt to execute the hooks through the requested version of Python.
+      if reexec:
+        try:
+          self._ExecuteHookViaReexec(interp, context, **kwargs)
+        except OSError as e:
+          if e.errno == errno.ENOENT:
+            # We couldn't find the interpreter, so fallback to importing.
+            reexec = False
+          else:
+            raise
+
+      # Run the hook by importing directly.
+      if not reexec:
+        self._ExecuteHookViaImport(data, context, **kwargs)
     finally:
       # Restore sys.path and CWD.
       sys.path = orig_syspath
@@ -757,10 +886,17 @@ class Project(object):
   @property
   def CurrentBranch(self):
     """Obtain the name of the currently checked out branch.
-       The branch name omits the 'refs/heads/' prefix.
-       None is returned if the project is on a detached HEAD.
+
+    The branch name omits the 'refs/heads/' prefix.
+    None is returned if the project is on a detached HEAD, or if the work_git is
+    otheriwse inaccessible (e.g. an incomplete sync).
     """
-    b = self.work_git.GetHead()
+    try:
+      b = self.work_git.GetHead()
+    except NoManifestException:
+      # If the local checkout is in a bad state, don't barf.  Let the callers
+      # process this like the head is unreadable.
+      return None
     if b.startswith(R_HEADS):
       return b[len(R_HEADS):]
     return None
@@ -1028,19 +1164,29 @@ class Project(object):
       cmd.append('--src-prefix=a/%s/' % self.relpath)
       cmd.append('--dst-prefix=b/%s/' % self.relpath)
     cmd.append('--')
-    p = GitCommand(self,
-                   cmd,
-                   capture_stdout=True,
-                   capture_stderr=True)
+    try:
+      p = GitCommand(self,
+                     cmd,
+                     capture_stdout=True,
+                     capture_stderr=True)
+    except GitError as e:
+      out.nl()
+      out.project('project %s/' % self.relpath)
+      out.nl()
+      out.fail('%s', str(e))
+      out.nl()
+      return False
     has_diff = False
     for line in p.process.stdout:
+      if not hasattr(line, 'encode'):
+        line = line.decode()
       if not has_diff:
         out.nl()
         out.project('project %s/' % self.relpath)
         out.nl()
         has_diff = True
       print(line[:-1])
-    p.Wait()
+    return p.Wait() == 0
 
 
 # Publish / Upload ##
@@ -1172,10 +1318,11 @@ class Project(object):
 
     ref_spec = '%s:refs/%s/%s' % (R_HEADS + branch.name, upload_type,
                                   dest_branch)
+    opts = []
     if auto_topic:
-      ref_spec = ref_spec + '/' + branch.name
+      opts += ['topic=' + branch.name]
 
-    opts = ['r=%s' % p for p in people[0]]
+    opts += ['r=%s' % p for p in people[0]]
     opts += ['cc=%s' % p for p in people[1]]
     if notify:
       opts += ['notify=' + notify]
@@ -1223,7 +1370,8 @@ class Project(object):
                        archive=False,
                        optimized_fetch=False,
                        prune=False,
-                       submodules=False):
+                       submodules=False,
+                       clone_filter=None):
     """Perform only the network IO portion of the sync process.
        Local working directory/branch state is not affected.
     """
@@ -1265,12 +1413,9 @@ class Project(object):
     if is_new:
       alt = os.path.join(self.gitdir, 'objects/info/alternates')
       try:
-        fd = open(alt)
-        try:
+        with open(alt) as fd:
           # This works for both absolute and relative alternate directories.
           alt_dir = os.path.join(self.objdir, 'objects', fd.readline().rstrip())
-        finally:
-          fd.close()
       except IOError:
         alt_dir = None
     else:
@@ -1306,7 +1451,8 @@ class Project(object):
         not self._RemoteFetch(initial=is_new, quiet=quiet, alt_dir=alt_dir,
                               current_branch_only=current_branch_only,
                               no_tags=no_tags, prune=prune, depth=depth,
-                              submodules=submodules)):
+                              submodules=submodules, force_sync=force_sync,
+                              clone_filter=clone_filter)):
       return False
 
     mp = self.manifest.manifestProject
@@ -1376,6 +1522,13 @@ class Project(object):
     """Perform only the local IO portion of the sync process.
        Network access is not required.
     """
+    if not os.path.exists(self.gitdir):
+      syncbuf.fail(self,
+                   'Cannot checkout %s due to missing network sync; Run '
+                   '`repo sync -n %s` first.' %
+                   (self.name, self.name))
+      return
+
     self._InitWorkTree(force_sync=force_sync, submodules=submodules)
     all_refs = self.bare_ref.all
     self.CleanPublishedCache(all_refs)
@@ -1456,7 +1609,16 @@ class Project(object):
       return
 
     upstream_gain = self._revlist(not_rev(HEAD), revid)
-    pub = self.WasPublished(branch.name, all_refs)
+
+    # See if we can perform a fast forward merge.  This can happen if our
+    # branch isn't in the exact same state as we last published.
+    try:
+      self.work_git.merge_base('--is-ancestor', HEAD, revid)
+      # Skip the published logic.
+      pub = False
+    except GitError:
+      pub = self.WasPublished(branch.name, all_refs)
+
     if pub:
       not_merged = self._revlist(not_rev(revid), pub)
       if not_merged:
@@ -1485,7 +1647,7 @@ class Project(object):
     last_mine = None
     cnt_mine = 0
     for commit in local_changes:
-      commit_id, committer_email = commit.decode('utf-8').split(' ', 1)
+      commit_id, committer_email = commit.split(' ', 1)
       if committer_email == self.UserEmail:
         last_mine = commit_id
         cnt_mine += 1
@@ -1585,7 +1747,7 @@ class Project(object):
 
 # Branch Management ##
 
-  def StartBranch(self, name, branch_merge=''):
+  def StartBranch(self, name, branch_merge='', revision=None):
     """Create a new branch off the manifest's revision.
     """
     if not branch_merge:
@@ -1606,7 +1768,11 @@ class Project(object):
     branch.merge = branch_merge
     if not branch.merge.startswith('refs/') and not ID_RE.match(branch_merge):
       branch.merge = R_HEADS + branch_merge
-    revid = self.GetRevisionId(all_refs)
+
+    if revision is None:
+      revid = self.GetRevisionId(all_refs)
+    else:
+      revid = self.work_git.rev_parse(revision)
 
     if head.startswith(R_HEADS):
       try:
@@ -1807,8 +1973,8 @@ class Project(object):
         submodules.append((sub_rev, sub_path, sub_url))
       return submodules
 
-    re_path = re.compile(r'^submodule\.([^.]+)\.path=(.*)$')
-    re_url = re.compile(r'^submodule\.([^.]+)\.url=(.*)$')
+    re_path = re.compile(r'^submodule\.(.+)\.path=(.*)$')
+    re_url = re.compile(r'^submodule\.(.+)\.url=(.*)$')
 
     def parse_gitmodules(gitdir, rev):
       cmd = ['cat-file', 'blob', '%s:.gitmodules' % rev]
@@ -1955,7 +2121,9 @@ class Project(object):
                    no_tags=False,
                    prune=False,
                    depth=None,
-                   submodules=False):
+                   submodules=False,
+                   force_sync=False,
+                   clone_filter=None):
 
     is_sha1 = False
     tag_name = None
@@ -1979,8 +2147,9 @@ class Project(object):
 
       if is_sha1 or tag_name is not None:
         if self._CheckForImmutableRevision():
-          print('Skipped fetching project %s (already have persistent ref)'
-                % self.name)
+          if not quiet:
+            print('Skipped fetching project %s (already have persistent ref)'
+                  % self.name)
           return True
       if is_sha1 and not depth:
         # When syncing a specific commit and --depth is not set:
@@ -2045,6 +2214,11 @@ class Project(object):
 
     cmd = ['fetch']
 
+    if clone_filter:
+      git_require((2, 19, 0), fail=True, msg='partial clones')
+      cmd.append('--filter=%s' % clone_filter)
+      self.config.SetString('extensions.partialclone', self.remote.name)
+
     if depth:
       cmd.append('--depth=%s' % depth)
     else:
@@ -2061,12 +2235,18 @@ class Project(object):
       cmd.append('--update-head-ok')
     cmd.append(name)
 
+    spec = []
+
     # If using depth then we should not get all the tags since they may
     # be outside of the depth.
     if no_tags or depth:
       cmd.append('--no-tags')
     else:
       cmd.append('--tags')
+      spec.append(str((u'+refs/tags/*:') + remote.ToLocal('refs/tags/*')))
+
+    if force_sync:
+      cmd.append('--force')
 
     if prune:
       cmd.append('--prune')
@@ -2074,7 +2254,6 @@ class Project(object):
     if submodules:
       cmd.append('--recurse-submodules=on-demand')
 
-    spec = []
     if not current_branch_only:
       # Fetch whole repo
       spec.append(str((u'+refs/heads/*:') + remote.ToLocal('refs/heads/*')))
@@ -2142,12 +2321,12 @@ class Project(object):
           return self._RemoteFetch(name=name,
                                    current_branch_only=current_branch_only,
                                    initial=False, quiet=quiet, alt_dir=alt_dir,
-                                   depth=None)
+                                   depth=None, clone_filter=clone_filter)
         else:
           # Avoid infinite recursion: sync all branches with depth set to None
           return self._RemoteFetch(name=name, current_branch_only=False,
                                    initial=False, quiet=quiet, alt_dir=alt_dir,
-                                   depth=None)
+                                   depth=None, clone_filter=clone_filter)
 
     return ok
 
@@ -2209,13 +2388,17 @@ class Project(object):
         cmd += ['--continue-at', '%d' % (size,)]
       else:
         platform_utils.remove(tmpPath)
-    if 'http_proxy' in os.environ and 'darwin' == sys.platform:
-      cmd += ['--proxy', os.environ['http_proxy']]
-    with GetUrlCookieFile(srcUrl, quiet) as (cookiefile, _proxy):
+    with GetUrlCookieFile(srcUrl, quiet) as (cookiefile, proxy):
       if cookiefile:
         cmd += ['--cookie', cookiefile, '--cookie-jar', cookiefile]
-      if srcUrl.startswith('persistent-'):
-        srcUrl = srcUrl[len('persistent-'):]
+      if proxy:
+        cmd += ['--proxy', proxy]
+      elif 'http_proxy' in os.environ and 'darwin' == sys.platform:
+        cmd += ['--proxy', os.environ['http_proxy']]
+      if srcUrl.startswith('persistent-https'):
+        srcUrl = 'http' + srcUrl[len('persistent-https'):]
+      elif srcUrl.startswith('persistent-http'):
+        srcUrl = 'http' + srcUrl[len('persistent-http'):]
       cmd += [srcUrl]
 
       if IsTrace():
@@ -2249,8 +2432,8 @@ class Project(object):
 
   def _IsValidBundle(self, path, quiet):
     try:
-      with open(path) as f:
-        if f.read(16) == '# v2 git bundle\n':
+      with open(path, 'rb') as f:
+        if f.read(16) == b'# v2 git bundle\n':
           return True
         else:
           if not quiet:
@@ -2281,10 +2464,7 @@ class Project(object):
     cmd = ['ls-remote', self.remote.name, refs]
     p = GitCommand(self, cmd, capture_stdout=True)
     if p.Wait() == 0:
-      if hasattr(p.stdout, 'decode'):
-        return p.stdout.decode('utf-8')
-      else:
-        return p.stdout
+      return p.stdout
     return None
 
   def _Revert(self, rev):
@@ -2393,6 +2573,7 @@ class Project(object):
           if m.Has(key, include_defaults=False):
             self.config.SetString(key, m.GetString(key))
         self.config.SetString('filter.lfs.smudge', 'git-lfs smudge --skip -- %f')
+        self.config.SetString('filter.lfs.process', 'git-lfs filter-process --skip')
         if self.manifest.IsMirror:
           self.config.SetString('core.bare', 'true')
         else:
@@ -2558,41 +2739,45 @@ class Project(object):
           raise
 
   def _InitWorkTree(self, force_sync=False, submodules=False):
-    dotgit = os.path.join(self.worktree, '.git')
-    init_dotgit = not os.path.exists(dotgit)
+    realdotgit = os.path.join(self.worktree, '.git')
+    tmpdotgit = realdotgit + '.tmp'
+    init_dotgit = not os.path.exists(realdotgit)
+    if init_dotgit:
+      dotgit = tmpdotgit
+      platform_utils.rmtree(tmpdotgit, ignore_errors=True)
+      os.makedirs(tmpdotgit)
+      self._ReferenceGitDir(self.gitdir, tmpdotgit, share_refs=True,
+                            copy_all=False)
+    else:
+      dotgit = realdotgit
+
     try:
-      if init_dotgit:
-        os.makedirs(dotgit)
-        self._ReferenceGitDir(self.gitdir, dotgit, share_refs=True,
-                              copy_all=False)
+      self._CheckDirReference(self.gitdir, dotgit, share_refs=True)
+    except GitError as e:
+      if force_sync and not init_dotgit:
+        try:
+          platform_utils.rmtree(dotgit)
+          return self._InitWorkTree(force_sync=False, submodules=submodules)
+        except:
+          raise e
+      raise e
 
-      try:
-        self._CheckDirReference(self.gitdir, dotgit, share_refs=True)
-      except GitError as e:
-        if force_sync:
-          try:
-            platform_utils.rmtree(dotgit)
-            return self._InitWorkTree(force_sync=False, submodules=submodules)
-          except:
-            raise e
-        raise e
+    if init_dotgit:
+      _lwrite(os.path.join(tmpdotgit, HEAD), '%s\n' % self.GetRevisionId())
 
-      if init_dotgit:
-        _lwrite(os.path.join(dotgit, HEAD), '%s\n' % self.GetRevisionId())
+      # Now that the .git dir is fully set up, move it to its final home.
+      platform_utils.rename(tmpdotgit, realdotgit)
 
-        cmd = ['read-tree', '--reset', '-u']
-        cmd.append('-v')
-        cmd.append(HEAD)
-        if GitCommand(self, cmd).Wait() != 0:
-          raise GitError("cannot initialize work tree")
+      # Finish checking out the worktree.
+      cmd = ['read-tree', '--reset', '-u']
+      cmd.append('-v')
+      cmd.append(HEAD)
+      if GitCommand(self, cmd).Wait() != 0:
+        raise GitError('Cannot initialize work tree for ' + self.name)
 
-        if submodules:
-          self._SyncSubmodules(quiet=True)
-        self._CopyAndLinkFiles()
-    except Exception:
-      if init_dotgit:
-        platform_utils.rmtree(dotgit)
-      raise
+      if submodules:
+        self._SyncSubmodules(quiet=True)
+      self._CopyAndLinkFiles()
 
   def _get_symlink_error_message(self):
     if platform_utils.isWindows():
@@ -2684,6 +2869,7 @@ class Project(object):
     def DiffZ(self, name, *args):
       cmd = [name]
       cmd.append('-z')
+      cmd.append('--ignore-submodules')
       cmd.extend(args)
       p = GitCommand(self._project,
                      cmd,
@@ -2693,6 +2879,8 @@ class Project(object):
                      capture_stderr=True)
       try:
         out = p.process.stdout.read()
+        if not hasattr(out, 'encode'):
+          out = out.decode()
         r = {}
         if out:
           out = iter(out[:-1].split('\0'))
@@ -2738,13 +2926,10 @@ class Project(object):
       else:
         path = os.path.join(self._project.worktree, '.git', HEAD)
       try:
-        fd = open(path)
+        with open(path) as fd:
+          line = fd.readline()
       except IOError as e:
         raise NoManifestException(path, str(e))
-      try:
-        line = fd.readline()
-      finally:
-        fd.close()
       try:
         line = line.decode()
       except AttributeError:
@@ -2801,15 +2986,10 @@ class Project(object):
                      gitdir=self._gitdir,
                      capture_stdout=True,
                      capture_stderr=True)
-      r = []
-      for line in p.process.stdout:
-        if line[-1] == '\n':
-          line = line[:-1]
-        r.append(line)
       if p.Wait() != 0:
         raise GitError('%s rev-list %s: %s' %
                        (self._project.name, str(args), p.stderr))
-      return r
+      return p.stdout.splitlines()
 
     def __getattr__(self, name):
       """Allow arbitrary git commands using pythonic syntax.
@@ -2857,10 +3037,6 @@ class Project(object):
           raise GitError('%s %s: %s' %
                          (self._project.name, name, p.stderr))
         r = p.stdout
-        try:
-          r = r.decode('utf-8')
-        except AttributeError:
-          pass
         if r.endswith('\n') and r.index('\n') == len(r) - 1:
           return r[:-1]
         return r
@@ -2988,6 +3164,11 @@ class SyncBuffer(object):
     return True
 
   def _PrintMessages(self):
+    if self._messages or self._failures:
+      if os.isatty(2):
+        self.out.write(progress.CSI_ERASE_LINE)
+      self.out.write('\r')
+
     for m in self._messages:
       m.Print(self)
     for m in self._failures:

pyversion.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2013 The Android Open Source Project
 #

repo

@@ -1,4 +1,14 @@
 #!/usr/bin/env python
+# -*- coding:utf-8 -*-
+
+"""Repo launcher.
+
+This is a standalone tool that people may copy to anywhere in their system.
+It is used to get an initial repo client checkout, and after that it runs the
+copy of repo in the checkout.
+"""
+
+from __future__ import print_function
 
 # repo default configuration
 #
@@ -23,7 +33,7 @@ REPO_REV = 'stable'
 # limitations under the License.
 
 # increment this whenever we make important changes to this script
-VERSION = (1, 24)
+VERSION = (1, 26)
 
 # increment this if the MAINTAINER_KEYS block is modified
 KEYRING_VERSION = (1, 2)
@@ -113,11 +123,12 @@ repodir = '.repo'                # name of repo's private directory
 S_repo = 'repo'                  # special repo repository
 S_manifests = 'manifests'        # special manifest repository
 REPO_MAIN = S_repo + '/main.py'  # main script
-MIN_PYTHON_VERSION = (2, 6)      # minimum supported python version
+MIN_PYTHON_VERSION = (2, 7)      # minimum supported python version
 GITC_CONFIG_FILE = '/gitc/.config'
 GITC_FS_ROOT_DIR = '/gitc/manifest-rw/'
 
 
+import collections
 import errno
 import optparse
 import platform
@@ -138,23 +149,15 @@ else:
   urllib.error = urllib2
 
 
-def _print(*objects, **kwargs):
-  sep = kwargs.get('sep', ' ')
-  end = kwargs.get('end', '\n')
-  out = kwargs.get('file', sys.stdout)
-  out.write(sep.join(objects) + end)
-
-  # On Windows stderr is buffered, so flush to maintain the order of error messages.
-  if out == sys.stderr and platform.system() == "Windows":
-    out.flush()
-
-
 # Python version check
 ver = sys.version_info
 if (ver[0], ver[1]) < MIN_PYTHON_VERSION:
-  _print('error: Python version %s unsupported.\n'
-         'Please use Python 2.6 - 2.7 instead.'
-         % sys.version.split(' ')[0], file=sys.stderr)
+  print('error: Python version {} unsupported.\n'
+        'Please use Python {}.{} instead.'.format(
+            sys.version.split(' ')[0],
+            MIN_PYTHON_VERSION[0],
+            MIN_PYTHON_VERSION[1],
+        ), file=sys.stderr)
   sys.exit(1)
 
 home_dot_repo = os.path.expanduser('~/.repoconfig')
@@ -180,7 +183,7 @@ group.add_option('-b', '--manifest-branch',
 group.add_option('-m', '--manifest-name',
                  dest='manifest_name',
                  help='initial manifest file', metavar='NAME.xml')
-group.add_option('-c', '--current-branch',
+group.add_option('--current-branch',
                  dest='current_branch_only', action='store_true',
                  help='fetch only current manifest branch from server')
 group.add_option('--mirror',
@@ -196,6 +199,13 @@ group.add_option('--dissociate',
 group.add_option('--depth', type='int', default=None,
                  dest='depth',
                  help='create a shallow clone with given depth; see git clone')
+group.add_option('--partial-clone', action='store_true',
+                 dest='partial_clone',
+                 help='perform partial clone (https://git-scm.com/'
+                 'docs/gitrepository-layout#_code_partialclone_code)')
+group.add_option('--clone-filter', action='store', default='blob:none',
+                 dest='clone_filter',
+                 help='filter for use with --partial-clone [default: %default]')
 group.add_option('--archive',
                  dest='archive', action='store_true',
                  help='checkout an archive instead of a git repository for '
@@ -320,21 +330,21 @@ def _Init(args, gitc_init=False):
   if branch.startswith('refs/heads/'):
     branch = branch[len('refs/heads/'):]
   if branch.startswith('refs/'):
-    _print("fatal: invalid branch name '%s'" % branch, file=sys.stderr)
+    print("fatal: invalid branch name '%s'" % branch, file=sys.stderr)
     raise CloneFailure()
 
   try:
     if gitc_init:
       gitc_manifest_dir = get_gitc_manifest_dir()
       if not gitc_manifest_dir:
-        _print('fatal: GITC filesystem is not available. Exiting...',
-               file=sys.stderr)
+        print('fatal: GITC filesystem is not available. Exiting...',
+              file=sys.stderr)
         sys.exit(1)
       gitc_client = opt.gitc_client
       if not gitc_client:
         gitc_client = gitc_parse_clientdir(os.getcwd())
       if not gitc_client:
-        _print('fatal: GITC client (-c) is required.', file=sys.stderr)
+        print('fatal: GITC client (-c) is required.', file=sys.stderr)
         sys.exit(1)
       client_dir = os.path.join(gitc_manifest_dir, gitc_client)
       if not os.path.exists(client_dir):
@@ -347,8 +357,8 @@ def _Init(args, gitc_init=False):
     os.mkdir(repodir)
   except OSError as e:
     if e.errno != errno.EEXIST:
-      _print('fatal: cannot make %s directory: %s'
-             % (repodir, e.strerror), file=sys.stderr)
+      print('fatal: cannot make %s directory: %s'
+            % (repodir, e.strerror), file=sys.stderr)
       # Don't raise CloneFailure; that would delete the
       # name. Instead exit immediately.
       #
@@ -372,55 +382,73 @@ def _Init(args, gitc_init=False):
     _Checkout(dst, branch, rev, opt.quiet)
 
     if not os.path.isfile(os.path.join(dst, 'repo')):
-      _print("warning: '%s' does not look like a git-repo repository, is "
-             "REPO_URL set correctly?" % url, file=sys.stderr)
+      print("warning: '%s' does not look like a git-repo repository, is "
+            "REPO_URL set correctly?" % url, file=sys.stderr)
 
   except CloneFailure:
     if opt.quiet:
-      _print('fatal: repo init failed; run without --quiet to see why',
-             file=sys.stderr)
+      print('fatal: repo init failed; run without --quiet to see why',
+            file=sys.stderr)
     raise
 
 
-def ParseGitVersion(ver_str):
+# The git version info broken down into components for easy analysis.
+# Similar to Python's sys.version_info.
+GitVersion = collections.namedtuple(
+    'GitVersion', ('major', 'minor', 'micro', 'full'))
+
+def ParseGitVersion(ver_str=None):
+  if ver_str is None:
+    # Load the version ourselves.
+    ver_str = _GetGitVersion()
+
   if not ver_str.startswith('git version '):
     return None
 
-  num_ver_str = ver_str[len('git version '):].strip().split('-')[0]
+  full_version = ver_str[len('git version '):].strip()
+  num_ver_str = full_version.split('-')[0]
   to_tuple = []
   for num_str in num_ver_str.split('.')[:3]:
     if num_str.isdigit():
       to_tuple.append(int(num_str))
     else:
       to_tuple.append(0)
-  return tuple(to_tuple)
+  to_tuple.append(full_version)
+  return GitVersion(*to_tuple)
 
 
-def _CheckGitVersion():
+def _GetGitVersion():
   cmd = [GIT, '--version']
   try:
     proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
   except OSError as e:
-    _print(file=sys.stderr)
-    _print("fatal: '%s' is not available" % GIT, file=sys.stderr)
-    _print('fatal: %s' % e, file=sys.stderr)
-    _print(file=sys.stderr)
-    _print('Please make sure %s is installed and in your path.' % GIT,
-           file=sys.stderr)
-    raise CloneFailure()
+    print(file=sys.stderr)
+    print("fatal: '%s' is not available" % GIT, file=sys.stderr)
+    print('fatal: %s' % e, file=sys.stderr)
+    print(file=sys.stderr)
+    print('Please make sure %s is installed and in your path.' % GIT,
+          file=sys.stderr)
+    raise
 
   ver_str = proc.stdout.read().strip()
   proc.stdout.close()
   proc.wait()
+  return ver_str.decode('utf-8')
+
+
+def _CheckGitVersion():
+  try:
+    ver_act = ParseGitVersion()
+  except OSError:
+    raise CloneFailure()
 
-  ver_act = ParseGitVersion(ver_str)
   if ver_act is None:
-    _print('error: "%s" unsupported' % ver_str, file=sys.stderr)
+    print('fatal: unable to detect git version', file=sys.stderr)
     raise CloneFailure()
 
   if ver_act < MIN_GIT_VERSION:
     need = '.'.join(map(str, MIN_GIT_VERSION))
-    _print('fatal: git %s or later required' % need, file=sys.stderr)
+    print('fatal: git %s or later required' % need, file=sys.stderr)
     raise CloneFailure()
 
 
@@ -447,16 +475,16 @@ def SetupGnuPG(quiet):
     os.mkdir(home_dot_repo)
   except OSError as e:
     if e.errno != errno.EEXIST:
-      _print('fatal: cannot make %s directory: %s'
-             % (home_dot_repo, e.strerror), file=sys.stderr)
+      print('fatal: cannot make %s directory: %s'
+            % (home_dot_repo, e.strerror), file=sys.stderr)
       sys.exit(1)
 
   try:
     os.mkdir(gpg_dir, stat.S_IRWXU)
   except OSError as e:
     if e.errno != errno.EEXIST:
-      _print('fatal: cannot make %s directory: %s' % (gpg_dir, e.strerror),
-             file=sys.stderr)
+      print('fatal: cannot make %s directory: %s' % (gpg_dir, e.strerror),
+            file=sys.stderr)
       sys.exit(1)
 
   env = os.environ.copy()
@@ -472,22 +500,21 @@ def SetupGnuPG(quiet):
                             stdin=subprocess.PIPE)
   except OSError as e:
     if not quiet:
-      _print('warning: gpg (GnuPG) is not available.', file=sys.stderr)
-      _print('warning: Installing it is strongly encouraged.', file=sys.stderr)
-      _print(file=sys.stderr)
+      print('warning: gpg (GnuPG) is not available.', file=sys.stderr)
+      print('warning: Installing it is strongly encouraged.', file=sys.stderr)
+      print(file=sys.stderr)
     return False
 
-  proc.stdin.write(MAINTAINER_KEYS)
+  proc.stdin.write(MAINTAINER_KEYS.encode('utf-8'))
   proc.stdin.close()
 
   if proc.wait() != 0:
-    _print('fatal: registering repo maintainer keys failed', file=sys.stderr)
+    print('fatal: registering repo maintainer keys failed', file=sys.stderr)
     sys.exit(1)
-  _print()
+  print()
 
-  fd = open(os.path.join(home_dot_repo, 'keyring-version'), 'w')
-  fd.write('.'.join(map(str, KEYRING_VERSION)) + '\n')
-  fd.close()
+  with open(os.path.join(home_dot_repo, 'keyring-version'), 'w') as fd:
+    fd.write('.'.join(map(str, KEYRING_VERSION)) + '\n')
   return True
 
 
@@ -526,7 +553,7 @@ def _InitHttp():
 
 def _Fetch(url, local, src, quiet):
   if not quiet:
-    _print('Get %s' % url, file=sys.stderr)
+    print('Get %s' % url, file=sys.stderr)
 
   cmd = [GIT, 'fetch']
   if quiet:
@@ -556,6 +583,7 @@ def _DownloadBundle(url, local, quiet):
       cwd=local,
       stdout=subprocess.PIPE)
   for line in proc.stdout:
+    line = line.decode('utf-8')
     m = re.compile(r'^url\.(.*)\.insteadof (.*)$').match(line)
     if m:
       new_url = m.group(1)
@@ -576,19 +604,19 @@ def _DownloadBundle(url, local, quiet):
     except urllib.error.HTTPError as e:
       if e.code in [401, 403, 404, 501]:
         return False
-      _print('fatal: Cannot get %s' % url, file=sys.stderr)
-      _print('fatal: HTTP error %s' % e.code, file=sys.stderr)
+      print('fatal: Cannot get %s' % url, file=sys.stderr)
+      print('fatal: HTTP error %s' % e.code, file=sys.stderr)
       raise CloneFailure()
     except urllib.error.URLError as e:
-      _print('fatal: Cannot get %s' % url, file=sys.stderr)
-      _print('fatal: error %s' % e.reason, file=sys.stderr)
+      print('fatal: Cannot get %s' % url, file=sys.stderr)
+      print('fatal: error %s' % e.reason, file=sys.stderr)
       raise CloneFailure()
     try:
       if not quiet:
-        _print('Get %s' % url, file=sys.stderr)
+        print('Get %s' % url, file=sys.stderr)
       while True:
         buf = r.read(8192)
-        if buf == '':
+        if not buf:
           return True
         dest.write(buf)
     finally:
@@ -611,23 +639,23 @@ def _Clone(url, local, quiet, clone_bundle):
   try:
     os.mkdir(local)
   except OSError as e:
-    _print('fatal: cannot make %s directory: %s' % (local, e.strerror),
-           file=sys.stderr)
+    print('fatal: cannot make %s directory: %s' % (local, e.strerror),
+          file=sys.stderr)
     raise CloneFailure()
 
   cmd = [GIT, 'init', '--quiet']
   try:
     proc = subprocess.Popen(cmd, cwd=local)
   except OSError as e:
-    _print(file=sys.stderr)
-    _print("fatal: '%s' is not available" % GIT, file=sys.stderr)
-    _print('fatal: %s' % e, file=sys.stderr)
-    _print(file=sys.stderr)
-    _print('Please make sure %s is installed and in your path.' % GIT,
-           file=sys.stderr)
+    print(file=sys.stderr)
+    print("fatal: '%s' is not available" % GIT, file=sys.stderr)
+    print('fatal: %s' % e, file=sys.stderr)
+    print(file=sys.stderr)
+    print('Please make sure %s is installed and in your path.' % GIT,
+          file=sys.stderr)
     raise CloneFailure()
   if proc.wait() != 0:
-    _print('fatal: could not create %s' % local, file=sys.stderr)
+    print('fatal: could not create %s' % local, file=sys.stderr)
     raise CloneFailure()
 
   _InitHttp()
@@ -648,25 +676,25 @@ def _Verify(cwd, branch, quiet):
                           stdout=subprocess.PIPE,
                           stderr=subprocess.PIPE,
                           cwd=cwd)
-  cur = proc.stdout.read().strip()
+  cur = proc.stdout.read().strip().decode('utf-8')
   proc.stdout.close()
 
   proc.stderr.read()
   proc.stderr.close()
 
   if proc.wait() != 0 or not cur:
-    _print(file=sys.stderr)
-    _print("fatal: branch '%s' has not been signed" % branch, file=sys.stderr)
+    print(file=sys.stderr)
+    print("fatal: branch '%s' has not been signed" % branch, file=sys.stderr)
     raise CloneFailure()
 
   m = re.compile(r'^(.*)-[0-9]{1,}-g[0-9a-f]{1,}$').match(cur)
   if m:
     cur = m.group(1)
     if not quiet:
-      _print(file=sys.stderr)
-      _print("info: Ignoring branch '%s'; using tagged release '%s'"
-             % (branch, cur), file=sys.stderr)
-      _print(file=sys.stderr)
+      print(file=sys.stderr)
+      print("info: Ignoring branch '%s'; using tagged release '%s'"
+            % (branch, cur), file=sys.stderr)
+      print(file=sys.stderr)
 
   env = os.environ.copy()
   try:
@@ -680,17 +708,17 @@ def _Verify(cwd, branch, quiet):
                           stderr=subprocess.PIPE,
                           cwd=cwd,
                           env=env)
-  out = proc.stdout.read()
+  out = proc.stdout.read().decode('utf-8')
   proc.stdout.close()
 
-  err = proc.stderr.read()
+  err = proc.stderr.read().decode('utf-8')
   proc.stderr.close()
 
   if proc.wait() != 0:
-    _print(file=sys.stderr)
-    _print(out, file=sys.stderr)
-    _print(err, file=sys.stderr)
-    _print(file=sys.stderr)
+    print(file=sys.stderr)
+    print(out, file=sys.stderr)
+    print(err, file=sys.stderr)
+    print(file=sys.stderr)
     raise CloneFailure()
   return '%s^0' % cur
 
@@ -761,7 +789,7 @@ def _Usage():
   if get_gitc_manifest_dir():
     gitc_usage = "  gitc-init Initialize a GITC Client.\n"
 
-  _print(
+  print(
       """usage: repo COMMAND [ARGS]
 
 repo is not yet installed.  Use "repo init" to install it here.
@@ -773,8 +801,8 @@ The most commonly used repo commands are:
       """  help      Display detailed help on a command
 
 For access to the full online help, install repo ("repo init").
-""", file=sys.stderr)
-  sys.exit(1)
+""")
+  sys.exit(0)
 
 
 def _Help(args):
@@ -787,23 +815,23 @@ def _Help(args):
       init_optparse.print_help()
       sys.exit(0)
     else:
-      _print("error: '%s' is not a bootstrap command.\n"
-             '        For access to online help, install repo ("repo init").'
-             % args[0], file=sys.stderr)
+      print("error: '%s' is not a bootstrap command.\n"
+            '        For access to online help, install repo ("repo init").'
+            % args[0], file=sys.stderr)
   else:
     _Usage()
   sys.exit(1)
 
 
 def _NotInstalled():
-  _print('error: repo is not installed.  Use "repo init" to install it here.',
-         file=sys.stderr)
+  print('error: repo is not installed.  Use "repo init" to install it here.',
+        file=sys.stderr)
   sys.exit(1)
 
 
 def _NoCommands(cmd):
-  _print("""error: command '%s' requires repo to be installed first.
-         Use "repo init" to install it here.""" % cmd, file=sys.stderr)
+  print("""error: command '%s' requires repo to be installed first.
+        Use "repo init" to install it here.""" % cmd, file=sys.stderr)
   sys.exit(1)
 
 
@@ -833,14 +861,14 @@ def _SetDefaultsTo(gitdir):
                            'HEAD'],
                           stdout=subprocess.PIPE,
                           stderr=subprocess.PIPE)
-  REPO_REV = proc.stdout.read().strip()
+  REPO_REV = proc.stdout.read().strip().decode('utf-8')
   proc.stdout.close()
 
   proc.stderr.read()
   proc.stderr.close()
 
   if proc.wait() != 0:
-    _print('fatal: %s has no current branch' % gitdir, file=sys.stderr)
+    print('fatal: %s has no current branch' % gitdir, file=sys.stderr)
     sys.exit(1)
 
 
@@ -857,10 +885,10 @@ def main(orig_args):
 
   cwd = os.getcwd()
   if get_gitc_manifest_dir() and cwd.startswith(get_gitc_manifest_dir()):
-    _print('error: repo cannot be used in the GITC local manifest directory.'
-           '\nIf you want to work on this GITC client please rerun this '
-           'command from the corresponding client under /gitc/',
-           file=sys.stderr)
+    print('error: repo cannot be used in the GITC local manifest directory.'
+          '\nIf you want to work on this GITC client please rerun this '
+          'command from the corresponding client under /gitc/',
+          file=sys.stderr)
     sys.exit(1)
   if not repo_main:
     if opt.help:
@@ -876,8 +904,8 @@ def main(orig_args):
         _Init(args, gitc_init=(cmd == 'gitc-init'))
       except CloneFailure:
         path = os.path.join(repodir, S_repo)
-        _print("fatal: cloning the git-repo repository failed, will remove "
-               "'%s' " % path, file=sys.stderr)
+        print("fatal: cloning the git-repo repository failed, will remove "
+              "'%s' " % path, file=sys.stderr)
         shutil.rmtree(path, ignore_errors=True)
         sys.exit(1)
       repo_main, rel_repo_dir = _FindRepo()
@@ -901,14 +929,10 @@ def main(orig_args):
     else:
       os.execv(sys.executable, me)
   except OSError as e:
-    _print("fatal: unable to start %s" % repo_main, file=sys.stderr)
-    _print("fatal: %s" % e, file=sys.stderr)
+    print("fatal: unable to start %s" % repo_main, file=sys.stderr)
+    print("fatal: %s" % e, file=sys.stderr)
     sys.exit(148)
 
 
 if __name__ == '__main__':
-  if ver[0] == 3:
-    _print('warning: Python 3 support is currently experimental. YMMV.\n'
-           'Please use Python 2.6 - 2.7 instead.',
-           file=sys.stderr)
   main(sys.argv[1:])

repo_trace.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #
@@ -13,15 +14,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+"""Logic for tracing repo interactions.
+
+Activated via `repo --trace ...` or `REPO_TRACE=1 repo ...`.
+"""
+
 from __future__ import print_function
 import sys
 import os
+
+# Env var to implicitly turn on tracing.
 REPO_TRACE = 'REPO_TRACE'
 
-try:
-  _TRACE = os.environ[REPO_TRACE] == '1'
-except KeyError:
-  _TRACE = False
+_TRACE = os.environ.get(REPO_TRACE) == '1'
 
 def IsTrace():
   return _TRACE

run_tests

@@ -0,0 +1,54 @@
+#!/usr/bin/python
+# -*- coding:utf-8 -*-
+# Copyright 2019 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Wrapper to run pytest with the right settings."""
+
+from __future__ import print_function
+
+import errno
+import os
+import subprocess
+import sys
+
+
+def run_pytest(cmd, argv):
+  """Run the unittests via |cmd|."""
+  try:
+    subprocess.check_call([cmd] + argv)
+    return 0
+  except OSError as e:
+    if e.errno == errno.ENOENT:
+      print('%s: unable to run `%s`: %s' % (__file__, cmd, e), file=sys.stderr)
+      print('%s: Try installing pytest: sudo apt-get install python-pytest' %
+            (__file__,), file=sys.stderr)
+      return 1
+    else:
+      raise
+
+
+def main(argv):
+  """The main entry."""
+  # Add the repo tree to PYTHONPATH as the tests expect to be able to import
+  # modules directly.
+  topdir = os.path.dirname(os.path.realpath(__file__))
+  pythonpath = os.environ.get('PYTHONPATH', '')
+  os.environ['PYTHONPATH'] = '%s:%s' % (topdir, pythonpath)
+
+  return run_pytest('pytest', argv)
+
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv[1:]))

subcmds/__init__.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #

subcmds/abandon.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #
@@ -36,19 +37,19 @@ It is equivalent to "git branch -D <branchname>".
                  dest='all', action='store_true',
                  help='delete all branches in all projects')
 
-  def Execute(self, opt, args):
+  def ValidateOptions(self, opt, args):
     if not opt.all and not args:
       self.Usage()
 
     if not opt.all:
       nb = args[0]
       if not git.check_ref_format('heads/%s' % nb):
-        print("error: '%s' is not a valid name" % nb, file=sys.stderr)
-        sys.exit(1)
+        self.OptionParser.error("'%s' is not a valid branch name" % nb)
     else:
-      args.insert(0,None)
-      nb = "'All local branches'"
+      args.insert(0, "'All local branches'")
 
+  def Execute(self, opt, args):
+    nb = args[0]
     err = defaultdict(list)
     success = defaultdict(list)
     all_projects = self.GetProjects(args[1:])
@@ -58,7 +59,7 @@ It is equivalent to "git branch -D <branchname>".
       pm.update()
 
       if opt.all:
-        branches = project.GetBranches().keys()
+        branches = list(project.GetBranches().keys())
       else:
         branches = [nb]
 

subcmds/branches.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2009 The Android Open Source Project
 #

subcmds/checkout.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2009 The Android Open Source Project
 #
@@ -33,10 +34,11 @@ The command is equivalent to:
   repo forall [<project>...] -c git checkout <branchname>
 """
 
-  def Execute(self, opt, args):
+  def ValidateOptions(self, opt, args):
     if not args:
       self.Usage()
 
+  def Execute(self, opt, args):
     nb = args[0]
     err = []
     success = []

subcmds/cherry_pick.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2010 The Android Open Source Project
 #
@@ -36,10 +37,11 @@ change id will be added.
   def _Options(self, p):
     pass
 
-  def Execute(self, opt, args):
+  def ValidateOptions(self, opt, args):
     if len(args) != 1:
       self.Usage()
 
+  def Execute(self, opt, args):
     reference = args[0]
 
     p = GitCommand(None,

subcmds/diff.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #
@@ -36,5 +37,8 @@ to the Unix 'patch' command.
                  help='Paths are relative to the repository root')
 
   def Execute(self, opt, args):
+    ret = 0
     for project in self.GetProjects(args):
-      project.PrintWorkTreeDiff(opt.absolute)
+      if not project.PrintWorkTreeDiff(opt.absolute):
+        ret = 1
+    return ret

subcmds/diffmanifests.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2014 The Android Open Source Project
 #
@@ -175,10 +176,11 @@ synced and their revisions won't be found.
             self.printText(log)
             self.out.nl()
 
-  def Execute(self, opt, args):
+  def ValidateOptions(self, opt, args):
     if not args or len(args) > 2:
-      self.Usage()
+      self.OptionParser.error('missing manifests to diff')
 
+  def Execute(self, opt, args):
     self.out = _Coloring(self.manifest.globalConfig)
     self.printText = self.out.nofmt_printer('text')
     if opt.color:
@@ -190,12 +192,12 @@ synced and their revisions won't be found.
       self.printProject = self.printAdded = self.printRemoved = self.printRevision = self.printText
 
     manifest1 = XmlManifest(self.manifest.repodir)
-    manifest1.Override(args[0])
+    manifest1.Override(args[0], load_local_manifests=False)
     if len(args) == 1:
       manifest2 = self.manifest
     else:
       manifest2 = XmlManifest(self.manifest.repodir)
-      manifest2.Override(args[1])
+      manifest2.Override(args[1], load_local_manifests=False)
 
     diff = manifest1.projectsDiff(manifest2)
     if opt.raw:

subcmds/download.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #

subcmds/forall.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #
@@ -104,7 +105,7 @@ following <command>.
 
 Example: to list projects:
 
-  %prog% forall -c 'echo $REPO_PROJECT'
+  %prog -c 'echo $REPO_PROJECT'
 
 Notice that $REPO_PROJECT is quoted to ensure it is expanded in
 the context of running <command> instead of in the calling shell.
@@ -138,6 +139,9 @@ without iterating through the remaining projects.
     p.add_option('-e', '--abort-on-errors',
                  dest='abort_on_errors', action='store_true',
                  help='Abort if a command exits unsuccessfully')
+    p.add_option('--ignore-missing', action='store_true',
+                 help='Silently skip & do not exit non-zero due missing '
+                      'checkouts')
 
     g = p.add_option_group('Output')
     g.add_option('-p',
@@ -176,10 +180,11 @@ without iterating through the remaining projects.
       'worktree': project.worktree,
     }
 
-  def Execute(self, opt, args):
+  def ValidateOptions(self, opt, args):
     if not opt.command:
       self.Usage()
 
+  def Execute(self, opt, args):
     cmd = [opt.command[0]]
 
     shell = True
@@ -321,10 +326,14 @@ def DoWork(project, mirror, opt, cmd, shell, cnt, config):
     cwd = project['worktree']
 
   if not os.path.exists(cwd):
-    if (opt.project_header and opt.verbose) \
-    or not opt.project_header:
+    # Allow the user to silently ignore missing checkouts so they can run on
+    # partial checkouts (good for infra recovery tools).
+    if opt.ignore_missing:
+      return 0
+    if ((opt.project_header and opt.verbose)
+        or not opt.project_header):
       print('skipping %s/' % project['relpath'], file=sys.stderr)
-    return
+    return 1
 
   if opt.project_header:
     stdin = subprocess.PIPE

subcmds/gitc_delete.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2015 The Android Open Source Project
 #

subcmds/gitc_init.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2015 The Android Open Source Project
 #

subcmds/grep.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2009 The Android Open Source Project
 #
@@ -14,15 +15,19 @@
 # limitations under the License.
 
 from __future__ import print_function
+
 import sys
+
 from color import Coloring
 from command import PagedCommand
+from error import GitError
 from git_command import git_require, GitCommand
 
 class GrepColoring(Coloring):
   def __init__(self, config):
     Coloring.__init__(self, config, 'grep')
     self.project = self.printer('project', attr='bold')
+    self.fail = self.printer('fail', fg='red')
 
 class Grep(PagedCommand):
   common = True
@@ -183,15 +188,25 @@ contain a line that matches both expressions:
       cmd_argv.extend(opt.revision)
     cmd_argv.append('--')
 
+    git_failed = False
     bad_rev = False
     have_match = False
 
     for project in projects:
-      p = GitCommand(project,
-                     cmd_argv,
-                     bare = False,
-                     capture_stdout = True,
-                     capture_stderr = True)
+      try:
+        p = GitCommand(project,
+                       cmd_argv,
+                       bare=False,
+                       capture_stdout=True,
+                       capture_stderr=True)
+      except GitError as e:
+        git_failed = True
+        out.project('--- project %s ---' % project.relpath)
+        out.nl()
+        out.fail('%s', str(e))
+        out.nl()
+        continue
+
       if p.Wait() != 0:
         # no results
         #
@@ -201,7 +216,7 @@ contain a line that matches both expressions:
           else:
             out.project('--- project %s ---' % project.relpath)
             out.nl()
-            out.write("%s", p.stderr)
+            out.fail('%s', p.stderr.strip())
             out.nl()
         continue
       have_match = True
@@ -230,7 +245,9 @@ contain a line that matches both expressions:
         for line in r:
           print(line)
 
-    if have_match:
+    if git_failed:
+      sys.exit(1)
+    elif have_match:
       sys.exit(0)
     elif have_rev and bad_rev:
       for r in opt.revision:

subcmds/help.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #
@@ -87,7 +88,7 @@ Displays detailed usage information about a command.
 "See 'repo help <command>' for more information on a specific command.\n"
 "See 'repo help --all' for a complete list of recognized commands.")
 
-  def _PrintCommandHelp(self, cmd):
+  def _PrintCommandHelp(self, cmd, header_prefix=''):
     class _Out(Coloring):
       def __init__(self, gc):
         Coloring.__init__(self, gc, 'help')
@@ -105,7 +106,7 @@ Displays detailed usage information about a command.
 
         self.nl()
 
-        self.heading('%s', heading)
+        self.heading('%s%s', header_prefix, heading)
         self.nl()
         self.nl()
 
@@ -123,7 +124,7 @@ Displays detailed usage information about a command.
 
           m = asciidoc_hdr.match(para)
           if m:
-            self.heading(m.group(1))
+            self.heading('%s%s', header_prefix, m.group(1))
             self.nl()
             self.nl()
             continue
@@ -137,14 +138,25 @@ Displays detailed usage information about a command.
     cmd.OptionParser.print_help()
     out._PrintSection('Description', 'helpDescription')
 
+  def _PrintAllCommandHelp(self):
+    for name in sorted(self.commands):
+      cmd = self.commands[name]
+      cmd.manifest = self.manifest
+      self._PrintCommandHelp(cmd, header_prefix='[%s] ' % (name,))
+
   def _Options(self, p):
     p.add_option('-a', '--all',
                  dest='show_all', action='store_true',
                  help='show the complete list of commands')
+    p.add_option('--help-all',
+                 dest='show_all_help', action='store_true',
+                 help='show the --help of all commands')
 
   def Execute(self, opt, args):
     if len(args) == 0:
-      if opt.show_all:
+      if opt.show_all_help:
+        self._PrintAllCommandHelp()
+      elif opt.show_all:
         self._PrintAllCommands()
       else:
         self._PrintCommonCommands()

subcmds/info.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2012 The Android Open Source Project
 #
@@ -15,7 +16,6 @@
 
 from command import PagedCommand
 from color import Coloring
-from error import NoSuchProjectError
 from git_refs import R_M
 
 class _Coloring(Coloring):
@@ -45,7 +45,7 @@ class Info(PagedCommand):
   def Execute(self, opt, args):
     self.out = _Coloring(self.manifest.globalConfig)
     self.heading = self.out.printer('heading', attr = 'bold')
-    self.headtext = self.out.printer('headtext', fg = 'yellow')
+    self.headtext = self.out.nofmt_printer('headtext', fg = 'yellow')
     self.redtext = self.out.printer('redtext', fg = 'red')
     self.sha = self.out.printer("sha", fg = 'yellow')
     self.text = self.out.nofmt_printer('text')
@@ -81,10 +81,8 @@ class Info(PagedCommand):
     self.out.nl()
 
   def printDiffInfo(self, args):
-    try:
-      projs = self.GetProjects(args)
-    except NoSuchProjectError:
-      return
+    # We let exceptions bubble up to main as they'll be well structured.
+    projs = self.GetProjects(args)
 
     for p in projs:
       self.heading("Project: ")
@@ -96,13 +94,19 @@ class Info(PagedCommand):
       self.out.nl()
 
       self.heading("Current revision: ")
-      self.headtext(p.revisionExpr)
+      self.headtext(p.GetRevisionId())
       self.out.nl()
 
-      localBranches = p.GetBranches().keys()
+      currentBranch = p.CurrentBranch
+      if currentBranch:
+        self.heading('Current branch: ')
+        self.headtext(currentBranch)
+        self.out.nl()
+
+      localBranches = list(p.GetBranches().keys())
       self.heading("Local Branches: ")
       self.redtext(str(len(localBranches)))
-      if len(localBranches) > 0:
+      if localBranches:
         self.text(" [")
         self.text(", ".join(localBranches))
         self.text("]")

subcmds/init.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #
@@ -95,7 +96,7 @@ to update the working directory files.
     g.add_option('-b', '--manifest-branch',
                  dest='manifest_branch',
                  help='manifest branch or revision', metavar='REVISION')
-    g.add_option('-c', '--current-branch',
+    g.add_option('--current-branch',
                  dest='current_branch_only', action='store_true',
                  help='fetch only current manifest branch from server')
     g.add_option('-m', '--manifest-name',
@@ -114,6 +115,13 @@ to update the working directory files.
     g.add_option('--depth', type='int', default=None,
                  dest='depth',
                  help='create a shallow clone with given depth; see git clone')
+    g.add_option('--partial-clone', action='store_true',
+                 dest='partial_clone',
+                 help='perform partial clone (https://git-scm.com/'
+                 'docs/gitrepository-layout#_code_partialclone_code)')
+    g.add_option('--clone-filter', action='store', default='blob:none',
+                 dest='clone_filter',
+                 help='filter for use with --partial-clone [default: %default]')
     g.add_option('--archive',
                  dest='archive', action='store_true',
                  help='checkout an archive instead of a git repository for '
@@ -197,6 +205,8 @@ to update the working directory files.
       else:
         m.PreSync()
 
+    self._ConfigureDepth(opt)
+
     if opt.manifest_url:
       r = m.GetRemote(m.remote.name)
       r.url = opt.manifest_url
@@ -250,13 +260,25 @@ to update the working directory files.
               'in another location.', file=sys.stderr)
         sys.exit(1)
 
+    if opt.partial_clone:
+      if opt.mirror:
+        print('fatal: --mirror and --partial-clone are mutually exclusive',
+              file=sys.stderr)
+        sys.exit(1)
+      m.config.SetString('repo.partialclone', 'true')
+      if opt.clone_filter:
+        m.config.SetString('repo.clonefilter', opt.clone_filter)
+    else:
+      opt.clone_filter = None
+
     if opt.submodules:
       m.config.SetString('repo.submodules', 'true')
 
     if not m.Sync_NetworkHalf(is_new=is_new, quiet=opt.quiet,
         clone_bundle=not opt.no_clone_bundle,
         current_branch_only=opt.current_branch_only,
-        no_tags=opt.no_tags, submodules=opt.submodules):
+        no_tags=opt.no_tags, submodules=opt.submodules,
+        clone_filter=opt.clone_filter):
       r = m.GetRemote(m.remote.name)
       print('fatal: cannot obtain manifest %s' % r.url, file=sys.stderr)
 
@@ -291,7 +313,9 @@ to update the working directory files.
       sys.exit(1)
 
   def _Prompt(self, prompt, value):
-    sys.stdout.write('%-10s [%s]: ' % (prompt, value))
+    print('%-10s [%s]: ' % (prompt, value), end='')
+    # TODO: When we require Python 3, use flush=True w/print above.
+    sys.stdout.flush()
     a = sys.stdin.readline().strip()
     if a == '':
       return value
@@ -325,7 +349,9 @@ to update the working directory files.
 
       print()
       print('Your identity is: %s <%s>' % (name, email))
-      sys.stdout.write('is this correct [y/N]? ')
+      print('is this correct [y/N]? ', end='')
+      # TODO: When we require Python 3, use flush=True w/print above.
+      sys.stdout.flush()
       a = sys.stdin.readline().strip().lower()
       if a in ('yes', 'y', 't', 'true'):
         break
@@ -367,7 +393,9 @@ to update the working directory files.
       out.printer(fg='black', attr=c)(' %-6s ', c)
     out.nl()
 
-    sys.stdout.write('Enable color display in this user account (y/N)? ')
+    print('Enable color display in this user account (y/N)? ', end='')
+    # TODO: When we require Python 3, use flush=True w/print above.
+    sys.stdout.flush()
     a = sys.stdin.readline().strip().lower()
     if a in ('y', 'yes', 't', 'true', 'on'):
       gc.SetString('color.ui', 'auto')
@@ -408,18 +436,17 @@ to update the working directory files.
       print('   rm -r %s/.repo' % self.manifest.topdir)
       print('and try again.')
 
-  def Execute(self, opt, args):
-    git_require(MIN_GIT_VERSION, fail=True)
-
+  def ValidateOptions(self, opt, args):
     if opt.reference:
       opt.reference = os.path.expanduser(opt.reference)
 
     # Check this here, else manifest will be tagged "not new" and init won't be
     # possible anymore without removing the .repo/manifests directory.
     if opt.archive and opt.mirror:
-      print('fatal: --mirror and --archive cannot be used together.',
-            file=sys.stderr)
-      sys.exit(1)
+      self.OptionParser.error('--mirror and --archive cannot be used together.')
+
+  def Execute(self, opt, args):
+    git_require(MIN_GIT_VERSION, fail=True)
 
     self._SyncManifest(opt)
     self._LinkManifest(opt.manifest_name)
@@ -429,6 +456,4 @@ to update the working directory files.
         self._ConfigureUser()
       self._ConfigureColor()
 
-    self._ConfigureDepth(opt)
-
     self._DisplayResult()

subcmds/list.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2011 The Android Open Source Project
 #
@@ -48,6 +49,10 @@ This is similar to running: repo forall -c 'echo "$REPO_PATH : $REPO_PROJECT"'.
                  dest='path_only', action='store_true',
                  help="Display only the path of the repository")
 
+  def ValidateOptions(self, opt, args):
+    if opt.fullpath and opt.name_only:
+      self.OptionParser.error('cannot combine -f and -n')
+
   def Execute(self, opt, args):
     """List all projects and the associated directories.
 
@@ -59,11 +64,6 @@ This is similar to running: repo forall -c 'echo "$REPO_PATH : $REPO_PROJECT"'.
       opt: The options.
       args: Positional args.  Can be a list of projects to list, or empty.
     """
-
-    if opt.fullpath and opt.name_only:
-      print('error: cannot combine -f and -n', file=sys.stderr)
-      sys.exit(1)
-
     if not opt.regex:
       projects = self.GetProjects(args, groups=opt.groups)
     else:

subcmds/manifest.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2009 The Android Open Source Project
 #
@@ -39,10 +40,9 @@ in a Git repository for use during future 'repo init' invocations.
     helptext = self._helpDescription + '\n'
     r = os.path.dirname(__file__)
     r = os.path.dirname(r)
-    fd = open(os.path.join(r, 'docs', 'manifest-format.md'))
-    for line in fd:
-      helptext += line
-    fd.close()
+    with open(os.path.join(r, 'docs', 'manifest-format.md')) as fd:
+      for line in fd:
+        helptext += line
     return helptext
 
   def _Options(self, p):
@@ -72,14 +72,9 @@ in a Git repository for use during future 'repo init' invocations.
     if opt.output_file != '-':
       print('Saved manifest to %s' % opt.output_file, file=sys.stderr)
 
-  def Execute(self, opt, args):
+  def ValidateOptions(self, opt, args):
     if args:
       self.Usage()
 
-    if opt.output_file is not None:
-      self._Output(opt)
-      return
-
-    print('error: no operation to perform', file=sys.stderr)
-    print('error: see repo help manifest', file=sys.stderr)
-    sys.exit(1)
+  def Execute(self, opt, args):
+    self._Output(opt)

subcmds/overview.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2012 The Android Open Source Project
 #

subcmds/prune.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #
@@ -50,11 +51,16 @@ class Prune(PagedCommand):
         out.project('project %s/' % project.relpath)
         out.nl()
 
-      commits = branch.commits
-      date = branch.date
-      print('%s %-33s (%2d commit%s, %s)' % (
+      print('%s %-33s ' % (
             branch.name == project.CurrentBranch and '*' or ' ',
-            branch.name,
+            branch.name), end='')
+
+      if not branch.base_exists:
+        print('(ignoring: tracking branch is gone: %s)' % (branch.base,))
+      else:
+        commits = branch.commits
+        date = branch.date
+        print('(%2d commit%s, %s)' % (
             len(commits),
             len(commits) != 1 and 's' or ' ',
             date))

subcmds/rebase.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2010 The Android Open Source Project
 #
@@ -16,9 +17,18 @@
 from __future__ import print_function
 import sys
 
+from color import Coloring
 from command import Command
 from git_command import GitCommand
 
+
+class RebaseColoring(Coloring):
+  def __init__(self, config):
+    Coloring.__init__(self, config, 'rebase')
+    self.project = self.printer('project', attr='bold')
+    self.fail = self.printer('fail', fg='red')
+
+
 class Rebase(Command):
   common = True
   helpSummary = "Rebase local branches on upstream branch"
@@ -36,6 +46,9 @@ branch but need to incorporate new upstream changes "underneath" them.
                 dest="interactive", action="store_true",
                 help="interactive rebase (single project only)")
 
+    p.add_option('--fail-fast',
+                 dest='fail_fast', action='store_true',
+                 help='Stop rebasing after first error is hit')
     p.add_option('-f', '--force-rebase',
                  dest='force_rebase', action='store_true',
                  help='Pass --force-rebase to git rebase')
@@ -70,15 +83,38 @@ branch but need to incorporate new upstream changes "underneath" them.
       if len(args) == 1:
         print('note: project %s is mapped to more than one path' % (args[0],),
             file=sys.stderr)
-      return -1
+      return 1
 
+    # Setup the common git rebase args that we use for all projects.
+    common_args = ['rebase']
+    if opt.whitespace:
+      common_args.append('--whitespace=%s' % opt.whitespace)
+    if opt.quiet:
+      common_args.append('--quiet')
+    if opt.force_rebase:
+      common_args.append('--force-rebase')
+    if opt.no_ff:
+      common_args.append('--no-ff')
+    if opt.autosquash:
+      common_args.append('--autosquash')
+    if opt.interactive:
+      common_args.append('-i')
+
+    config = self.manifest.manifestProject.config
+    out = RebaseColoring(config)
+    out.redirect(sys.stdout)
+
+    ret = 0
     for project in all_projects:
+      if ret and opt.fail_fast:
+        break
+
       cb = project.CurrentBranch
       if not cb:
         if one_project:
           print("error: project %s has a detached HEAD" % project.relpath,
                 file=sys.stderr)
-          return -1
+          return 1
         # ignore branches with detatched HEADs
         continue
 
@@ -87,38 +123,21 @@ branch but need to incorporate new upstream changes "underneath" them.
         if one_project:
           print("error: project %s does not track any remote branches"
                 % project.relpath, file=sys.stderr)
-          return -1
+          return 1
         # ignore branches without remotes
         continue
 
-      args = ["rebase"]
-
-      if opt.whitespace:
-        args.append('--whitespace=%s' % opt.whitespace)
-
-      if opt.quiet:
-        args.append('--quiet')
-
-      if opt.force_rebase:
-        args.append('--force-rebase')
-
-      if opt.no_ff:
-        args.append('--no-ff')
-
-      if opt.autosquash:
-        args.append('--autosquash')
-
-      if opt.interactive:
-        args.append("-i")
-
+      args = common_args[:]
       if opt.onto_manifest:
         args.append('--onto')
         args.append(project.revisionExpr)
 
       args.append(upbranch.LocalMerge)
 
-      print('# %s: rebasing %s -> %s'
-            % (project.relpath, cb, upbranch.LocalMerge), file=sys.stderr)
+      out.project('project %s: rebasing %s -> %s',
+                  project.relpath, cb, upbranch.LocalMerge)
+      out.nl()
+      out.flush()
 
       needs_stash = False
       if opt.auto_stash:
@@ -130,13 +149,21 @@ branch but need to incorporate new upstream changes "underneath" them.
           stash_args = ["stash"]
 
           if GitCommand(project, stash_args).Wait() != 0:
-            return -1
+            ret += 1
+            continue
 
       if GitCommand(project, args).Wait() != 0:
-        return -1
+        ret += 1
+        continue
 
       if needs_stash:
         stash_args.append('pop')
         stash_args.append('--quiet')
         if GitCommand(project, stash_args).Wait() != 0:
-          return -1
+          ret += 1
+
+    if ret:
+      out.fail('%i projects had errors', ret)
+      out.nl()
+
+    return ret

subcmds/selfupdate.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2009 The Android Open Source Project
 #

subcmds/smartsync.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2010 The Android Open Source Project
 #

subcmds/stage.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #

subcmds/start.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #
@@ -39,16 +40,21 @@ revision specified in the manifest.
     p.add_option('--all',
                  dest='all', action='store_true',
                  help='begin branch in all projects')
+    p.add_option('-r', '--rev', '--revision', dest='revision',
+                 help='point branch at this revision instead of upstream')
+    p.add_option('--head', dest='revision', action='store_const', const='HEAD',
+                 help='abbreviation for --rev HEAD')
 
-  def Execute(self, opt, args):
+  def ValidateOptions(self, opt, args):
     if not args:
       self.Usage()
 
     nb = args[0]
     if not git.check_ref_format('heads/%s' % nb):
-      print("error: '%s' is not a valid name" % nb, file=sys.stderr)
-      sys.exit(1)
+      self.OptionParser.error("'%s' is not a valid name" % nb)
 
+  def Execute(self, opt, args):
+    nb = args[0]
     err = []
     projects = []
     if not opt.all:
@@ -106,7 +112,8 @@ revision specified in the manifest.
         else:
           branch_merge = self.manifest.default.revisionExpr
 
-      if not project.StartBranch(nb, branch_merge=branch_merge):
+      if not project.StartBranch(
+          nb, branch_merge=branch_merge, revision=opt.revision):
         err.append(project)
     pm.end()
 

subcmds/status.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #
@@ -13,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
+
 from command import PagedCommand
 
 try:

subcmds/sync.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #
@@ -84,6 +85,9 @@ class _FetchError(Exception):
   """Internal error thrown in _FetchHelper() when we don't want stack trace."""
   pass
 
+class _CheckoutError(Exception):
+  """Internal error thrown in _CheckoutOne() when we don't want stack trace."""
+
 class Sync(Command, MirrorSafeCommand):
   jobs = 1
   common = True
@@ -128,14 +132,19 @@ from the user's .netrc file.
 if the manifest server specified in the manifest file already includes
 credentials.
 
-The -f/--force-broken option can be used to proceed with syncing
-other projects if a project sync fails.
+By default, all projects will be synced. The --fail-fast option can be used
+to halt syncing as soon as possible when the the first project fails to sync.
 
 The --force-sync option can be used to overwrite existing git
 directories if they have previously been linked to a different
 object direcotry. WARNING: This may cause data to be lost since
 refs may be removed when overwriting.
 
+The --force-remove-dirty option can be used to remove previously used
+projects with uncommitted changes. WARNING: This may cause data to be
+lost since uncommitted changes may be removed with projects that no longer
+exist in the manifest.
+
 The --no-clone-bundle option disables any attempt to use
 $URL/clone.bundle to bootstrap a new Git repository from a
 resumeable bundle file on a content delivery network. This
@@ -191,12 +200,20 @@ later is required to fix a server side protocol bug.
 
     p.add_option('-f', '--force-broken',
                  dest='force_broken', action='store_true',
-                 help="continue sync even if a project fails to sync")
+                 help='obsolete option (to be deleted in the future)')
+    p.add_option('--fail-fast',
+                 dest='fail_fast', action='store_true',
+                 help='stop syncing after first error is hit')
     p.add_option('--force-sync',
                  dest='force_sync', action='store_true',
                  help="overwrite an existing git directory if it needs to "
                       "point to a different object directory. WARNING: this "
                       "may cause loss of data")
+    p.add_option('--force-remove-dirty',
+                 dest='force_remove_dirty', action='store_true',
+                 help="force remove projects with uncommitted modifications if "
+                      "projects no longer exist in the manifest. "
+                      "WARNING: this may cause loss of data")
     p.add_option('-l', '--local-only',
                  dest='local_only', action='store_true',
                  help="only update working tree, don't fetch")
@@ -255,7 +272,7 @@ later is required to fix a server side protocol bug.
                  help=SUPPRESS_HELP)
 
   def _FetchProjectList(self, opt, projects, sem, *args, **kwargs):
-    """Main function of the fetch threads when jobs are > 1.
+    """Main function of the fetch threads.
 
     Delegates most of the work to _FetchHelper.
 
@@ -270,12 +287,13 @@ later is required to fix a server side protocol bug.
     try:
         for project in projects:
           success = self._FetchHelper(opt, project, *args, **kwargs)
-          if not success and not opt.force_broken:
+          if not success and opt.fail_fast:
             break
     finally:
         sem.release()
 
-  def _FetchHelper(self, opt, project, lock, fetched, pm, err_event):
+  def _FetchHelper(self, opt, project, lock, fetched, pm, err_event,
+                   clone_filter):
     """Fetch git objects for a single project.
 
     Args:
@@ -289,6 +307,7 @@ later is required to fix a server side protocol bug.
           lock held).
       err_event: We'll set this event in the case of an error (after printing
           out info about the error).
+      clone_filter: Filter for use in a partial clone.
 
     Returns:
       Whether the fetch was successful.
@@ -296,12 +315,8 @@ later is required to fix a server side protocol bug.
     # We'll set to true once we've locked the lock.
     did_lock = False
 
-    if not opt.quiet:
-      print('Fetching project %s' % project.name)
-
     # Encapsulate everything in a try/except/finally so that:
     # - We always set err_event in the case of an exception.
-    # - We always make sure we call sem.release().
     # - We always make sure we unlock the lock if we locked it.
     start = time.time()
     success = False
@@ -314,7 +329,8 @@ later is required to fix a server side protocol bug.
           clone_bundle=not opt.no_clone_bundle,
           no_tags=opt.no_tags, archive=self.manifest.IsArchive,
           optimized_fetch=opt.optimized_fetch,
-          prune=opt.prune)
+          prune=opt.prune,
+          clone_filter=clone_filter)
         self._fetch_times.Set(project, time.time() - start)
 
         # Lock around all the rest of the code, since printing, updating a set
@@ -327,14 +343,11 @@ later is required to fix a server side protocol bug.
           print('error: Cannot fetch %s from %s'
                 % (project.name, project.remote.url),
                 file=sys.stderr)
-          if opt.force_broken:
-            print('warn: --force-broken, continuing to sync',
-                  file=sys.stderr)
-          else:
+          if opt.fail_fast:
             raise _FetchError()
 
         fetched.add(project.gitdir)
-        pm.update()
+        pm.update(msg=project.name)
       except _FetchError:
         pass
       except Exception as e:
@@ -355,7 +368,6 @@ later is required to fix a server side protocol bug.
     fetched = set()
     lock = _threading.Lock()
     pm = Progress('Fetching projects', len(projects),
-                  print_newline=not(opt.quiet),
                   always_print_percentage=opt.quiet)
 
     objdir_project_map = dict()
@@ -368,7 +380,7 @@ later is required to fix a server side protocol bug.
     for project_list in objdir_project_map.values():
       # Check for any errors before running any more tasks.
       # ...we'll let existing threads finish, though.
-      if err_event.isSet() and not opt.force_broken:
+      if err_event.isSet() and opt.fail_fast:
         break
 
       sem.acquire()
@@ -378,7 +390,8 @@ later is required to fix a server side protocol bug.
                     lock=lock,
                     fetched=fetched,
                     pm=pm,
-                    err_event=err_event)
+                    err_event=err_event,
+                    clone_filter=self.manifest.CloneFilter)
       if self.jobs > 1:
         t = _threading.Thread(target = self._FetchProjectList,
                               kwargs = kwargs)
@@ -393,7 +406,7 @@ later is required to fix a server side protocol bug.
       t.join()
 
     # If we saw an error, exit with code 1 so that other scripts can check.
-    if err_event.isSet() and not opt.force_broken:
+    if err_event.isSet() and opt.fail_fast:
       print('\nerror: Exited sync due to fetch errors', file=sys.stderr)
       sys.exit(1)
 
@@ -405,6 +418,152 @@ later is required to fix a server side protocol bug.
 
     return fetched
 
+  def _CheckoutWorker(self, opt, sem, project, *args, **kwargs):
+    """Main function of the fetch threads.
+
+    Delegates most of the work to _CheckoutOne.
+
+    Args:
+      opt: Program options returned from optparse.  See _Options().
+      projects: Projects to fetch.
+      sem: We'll release() this semaphore when we exit so that another thread
+          can be started up.
+      *args, **kwargs: Remaining arguments to pass to _CheckoutOne. See the
+          _CheckoutOne docstring for details.
+    """
+    try:
+      return self._CheckoutOne(opt, project, *args, **kwargs)
+    finally:
+      sem.release()
+
+  def _CheckoutOne(self, opt, project, lock, pm, err_event, err_results):
+    """Checkout work tree for one project
+
+    Args:
+      opt: Program options returned from optparse.  See _Options().
+      project: Project object for the project to checkout.
+      lock: Lock for accessing objects that are shared amongst multiple
+          _CheckoutWorker() threads.
+      pm: Instance of a Project object.  We will call pm.update() (with our
+          lock held).
+      err_event: We'll set this event in the case of an error (after printing
+          out info about the error).
+      err_results: A list of strings, paths to git repos where checkout
+          failed.
+
+    Returns:
+      Whether the fetch was successful.
+    """
+    # We'll set to true once we've locked the lock.
+    did_lock = False
+
+    # Encapsulate everything in a try/except/finally so that:
+    # - We always set err_event in the case of an exception.
+    # - We always make sure we unlock the lock if we locked it.
+    start = time.time()
+    syncbuf = SyncBuffer(self.manifest.manifestProject.config,
+                         detach_head=opt.detach_head)
+    success = False
+    try:
+      try:
+        project.Sync_LocalHalf(syncbuf, force_sync=opt.force_sync)
+
+        # Lock around all the rest of the code, since printing, updating a set
+        # and Progress.update() are not thread safe.
+        lock.acquire()
+        success = syncbuf.Finish()
+        did_lock = True
+
+        if not success:
+          err_event.set()
+          print('error: Cannot checkout %s' % (project.name),
+                file=sys.stderr)
+          raise _CheckoutError()
+
+        pm.update(msg=project.name)
+      except _CheckoutError:
+        pass
+      except Exception as e:
+        print('error: Cannot checkout %s: %s: %s' %
+              (project.name, type(e).__name__, str(e)),
+              file=sys.stderr)
+        err_event.set()
+        raise
+    finally:
+      if did_lock:
+        if not success:
+          err_results.append(project.relpath)
+        lock.release()
+      finish = time.time()
+      self.event_log.AddSync(project, event_log.TASK_SYNC_LOCAL,
+                             start, finish, success)
+
+    return success
+
+  def _Checkout(self, all_projects, opt):
+    """Checkout projects listed in all_projects
+
+    Args:
+      all_projects: List of all projects that should be checked out.
+      opt: Program options returned from optparse.  See _Options().
+    """
+
+    # Perform checkouts in multiple threads when we are using partial clone.
+    # Without partial clone, all needed git objects are already downloaded,
+    # in this situation it's better to use only one process because the checkout
+    # would be mostly disk I/O; with partial clone, the objects are only
+    # downloaded when demanded (at checkout time), which is similar to the
+    # Sync_NetworkHalf case and parallelism would be helpful.
+    if self.manifest.CloneFilter:
+      syncjobs = self.jobs
+    else:
+      syncjobs = 1
+
+    lock = _threading.Lock()
+    pm = Progress('Checking out projects', len(all_projects))
+
+    threads = set()
+    sem = _threading.Semaphore(syncjobs)
+    err_event = _threading.Event()
+    err_results = []
+
+    for project in all_projects:
+      # Check for any errors before running any more tasks.
+      # ...we'll let existing threads finish, though.
+      if err_event.isSet() and opt.fail_fast:
+        break
+
+      sem.acquire()
+      if project.worktree:
+        kwargs = dict(opt=opt,
+                      sem=sem,
+                      project=project,
+                      lock=lock,
+                      pm=pm,
+                      err_event=err_event,
+                      err_results=err_results)
+        if syncjobs > 1:
+          t = _threading.Thread(target=self._CheckoutWorker,
+                                kwargs=kwargs)
+          # Ensure that Ctrl-C will not freeze the repo process.
+          t.daemon = True
+          threads.add(t)
+          t.start()
+        else:
+          self._CheckoutWorker(**kwargs)
+
+    for t in threads:
+      t.join()
+
+    pm.end()
+    # If we saw an error, exit with code 1 so that other scripts can check.
+    if err_event.isSet():
+      print('\nerror: Exited sync due to checkout errors', file=sys.stderr)
+      if err_results:
+        print('Failing repos:\n%s' % '\n'.join(err_results),
+              file=sys.stderr)
+      sys.exit(1)
+
   def _GCProjects(self, projects):
     gc_gitdirs = {}
     for project in projects:
@@ -425,7 +584,7 @@ later is required to fix a server side protocol bug.
         bare_git.gc('--auto')
       return
 
-    config = {'pack.threads': cpu_count / jobs if cpu_count > jobs else 1}
+    config = {'pack.threads': cpu_count // jobs if cpu_count > jobs else 1}
 
     threads = set()
     sem = _threading.Semaphore(jobs)
@@ -478,7 +637,7 @@ later is required to fix a server side protocol bug.
       print('Failed to remove %s (%s)' % (os.path.join(path, '.git'), str(e)), file=sys.stderr)
       print('error: Failed to delete obsolete path %s' % path, file=sys.stderr)
       print('       remove manually, then run sync again', file=sys.stderr)
-      return -1
+      return 1
 
     # Delete everything under the worktree, except for directories that contain
     # another git project
@@ -512,7 +671,7 @@ later is required to fix a server side protocol bug.
     if failed:
       print('error: Failed to delete obsolete path %s' % path, file=sys.stderr)
       print('       remove manually, then run sync again', file=sys.stderr)
-      return -1
+      return 1
 
     # Try deleting parent dirs if they are empty
     project_dir = path
@@ -525,7 +684,7 @@ later is required to fix a server side protocol bug.
 
     return 0
 
-  def UpdateProjectList(self):
+  def UpdateProjectList(self, opt):
     new_project_paths = []
     for project in self.GetProjects(None, missing_ok=True):
       if project.relpath:
@@ -535,12 +694,10 @@ later is required to fix a server side protocol bug.
     old_project_paths = []
 
     if os.path.exists(file_path):
-      fd = open(file_path, 'r')
-      try:
+      with open(file_path, 'r') as fd:
         old_project_paths = fd.read().split('\n')
-      finally:
-        fd.close()
-      for path in old_project_paths:
+      # In reversed order, so subfolders are deleted before parent folder.
+      for path in sorted(old_project_paths, reverse=True):
         if not path:
           continue
         if path not in new_project_paths:
@@ -559,155 +716,181 @@ later is required to fix a server side protocol bug.
                            revisionId = None,
                            groups = None)
 
-            if project.IsDirty():
+            if project.IsDirty() and opt.force_remove_dirty:
+              print('WARNING: Removing dirty project "%s": uncommitted changes '
+                    'erased' % project.relpath, file=sys.stderr)
+              self._DeleteProject(project.worktree)
+            elif project.IsDirty():
               print('error: Cannot remove project "%s": uncommitted changes '
                     'are present' % project.relpath, file=sys.stderr)
               print('       commit changes, then run sync again',
                     file=sys.stderr)
-              return -1
+              return 1
             elif self._DeleteProject(project.worktree):
-              return -1
+              return 1
 
     new_project_paths.sort()
-    fd = open(file_path, 'w')
-    try:
+    with open(file_path, 'w') as fd:
       fd.write('\n'.join(new_project_paths))
       fd.write('\n')
-    finally:
-      fd.close()
     return 0
 
+  def _SmartSyncSetup(self, opt, smart_sync_manifest_path):
+    if not self.manifest.manifest_server:
+      print('error: cannot smart sync: no manifest server defined in '
+            'manifest', file=sys.stderr)
+      sys.exit(1)
+
+    manifest_server = self.manifest.manifest_server
+    if not opt.quiet:
+      print('Using manifest server %s' % manifest_server)
+
+    if not '@' in manifest_server:
+      username = None
+      password = None
+      if opt.manifest_server_username and opt.manifest_server_password:
+        username = opt.manifest_server_username
+        password = opt.manifest_server_password
+      else:
+        try:
+          info = netrc.netrc()
+        except IOError:
+          # .netrc file does not exist or could not be opened
+          pass
+        else:
+          try:
+            parse_result = urllib.parse.urlparse(manifest_server)
+            if parse_result.hostname:
+              auth = info.authenticators(parse_result.hostname)
+              if auth:
+                username, _account, password = auth
+              else:
+                print('No credentials found for %s in .netrc'
+                      % parse_result.hostname, file=sys.stderr)
+          except netrc.NetrcParseError as e:
+            print('Error parsing .netrc file: %s' % e, file=sys.stderr)
+
+      if (username and password):
+        manifest_server = manifest_server.replace('://', '://%s:%s@' %
+                                                  (username, password),
+                                                  1)
+
+    transport = PersistentTransport(manifest_server)
+    if manifest_server.startswith('persistent-'):
+      manifest_server = manifest_server[len('persistent-'):]
+
+    try:
+      server = xmlrpc.client.Server(manifest_server, transport=transport)
+      if opt.smart_sync:
+        p = self.manifest.manifestProject
+        b = p.GetBranch(p.CurrentBranch)
+        branch = b.merge
+        if branch.startswith(R_HEADS):
+          branch = branch[len(R_HEADS):]
+
+        env = os.environ.copy()
+        if 'SYNC_TARGET' in env:
+          target = env['SYNC_TARGET']
+          [success, manifest_str] = server.GetApprovedManifest(branch, target)
+        elif 'TARGET_PRODUCT' in env and 'TARGET_BUILD_VARIANT' in env:
+          target = '%s-%s' % (env['TARGET_PRODUCT'],
+                              env['TARGET_BUILD_VARIANT'])
+          [success, manifest_str] = server.GetApprovedManifest(branch, target)
+        else:
+          [success, manifest_str] = server.GetApprovedManifest(branch)
+      else:
+        assert(opt.smart_tag)
+        [success, manifest_str] = server.GetManifest(opt.smart_tag)
+
+      if success:
+        manifest_name = os.path.basename(smart_sync_manifest_path)
+        try:
+          with open(smart_sync_manifest_path, 'w') as f:
+            f.write(manifest_str)
+        except IOError as e:
+          print('error: cannot write manifest to %s:\n%s'
+                % (smart_sync_manifest_path, e),
+                file=sys.stderr)
+          sys.exit(1)
+        self._ReloadManifest(manifest_name)
+      else:
+        print('error: manifest server RPC call failed: %s' %
+              manifest_str, file=sys.stderr)
+        sys.exit(1)
+    except (socket.error, IOError, xmlrpc.client.Fault) as e:
+      print('error: cannot connect to manifest server %s:\n%s'
+            % (self.manifest.manifest_server, e), file=sys.stderr)
+      sys.exit(1)
+    except xmlrpc.client.ProtocolError as e:
+      print('error: cannot connect to manifest server %s:\n%d %s'
+            % (self.manifest.manifest_server, e.errcode, e.errmsg),
+            file=sys.stderr)
+      sys.exit(1)
+
+    return manifest_name
+
+  def _UpdateManifestProject(self, opt, mp, manifest_name):
+    """Fetch & update the local manifest project."""
+    if not opt.local_only:
+      start = time.time()
+      success = mp.Sync_NetworkHalf(quiet=opt.quiet,
+                                    current_branch_only=opt.current_branch_only,
+                                    no_tags=opt.no_tags,
+                                    optimized_fetch=opt.optimized_fetch,
+                                    submodules=self.manifest.HasSubmodules,
+                                    clone_filter=self.manifest.CloneFilter)
+      finish = time.time()
+      self.event_log.AddSync(mp, event_log.TASK_SYNC_NETWORK,
+                             start, finish, success)
+
+    if mp.HasChanges:
+      syncbuf = SyncBuffer(mp.config)
+      start = time.time()
+      mp.Sync_LocalHalf(syncbuf, submodules=self.manifest.HasSubmodules)
+      clean = syncbuf.Finish()
+      self.event_log.AddSync(mp, event_log.TASK_SYNC_LOCAL,
+                             start, time.time(), clean)
+      if not clean:
+        sys.exit(1)
+      self._ReloadManifest(opt.manifest_name)
+      if opt.jobs is None:
+        self.jobs = self.manifest.default.sync_j
+
+  def ValidateOptions(self, opt, args):
+    if opt.force_broken:
+      print('warning: -f/--force-broken is now the default behavior, and the '
+            'options are deprecated', file=sys.stderr)
+    if opt.network_only and opt.detach_head:
+      self.OptionParser.error('cannot combine -n and -d')
+    if opt.network_only and opt.local_only:
+      self.OptionParser.error('cannot combine -n and -l')
+    if opt.manifest_name and opt.smart_sync:
+      self.OptionParser.error('cannot combine -m and -s')
+    if opt.manifest_name and opt.smart_tag:
+      self.OptionParser.error('cannot combine -m and -t')
+    if opt.manifest_server_username or opt.manifest_server_password:
+      if not (opt.smart_sync or opt.smart_tag):
+        self.OptionParser.error('-u and -p may only be combined with -s or -t')
+      if None in [opt.manifest_server_username, opt.manifest_server_password]:
+        self.OptionParser.error('both -u and -p must be given')
+
   def Execute(self, opt, args):
     if opt.jobs:
       self.jobs = opt.jobs
     if self.jobs > 1:
       soft_limit, _ = _rlimit_nofile()
-      self.jobs = min(self.jobs, (soft_limit - 5) / 3)
-
-    if opt.network_only and opt.detach_head:
-      print('error: cannot combine -n and -d', file=sys.stderr)
-      sys.exit(1)
-    if opt.network_only and opt.local_only:
-      print('error: cannot combine -n and -l', file=sys.stderr)
-      sys.exit(1)
-    if opt.manifest_name and opt.smart_sync:
-      print('error: cannot combine -m and -s', file=sys.stderr)
-      sys.exit(1)
-    if opt.manifest_name and opt.smart_tag:
-      print('error: cannot combine -m and -t', file=sys.stderr)
-      sys.exit(1)
-    if opt.manifest_server_username or opt.manifest_server_password:
-      if not (opt.smart_sync or opt.smart_tag):
-        print('error: -u and -p may only be combined with -s or -t',
-              file=sys.stderr)
-        sys.exit(1)
-      if None in [opt.manifest_server_username, opt.manifest_server_password]:
-        print('error: both -u and -p must be given', file=sys.stderr)
-        sys.exit(1)
+      self.jobs = min(self.jobs, (soft_limit - 5) // 3)
 
     if opt.manifest_name:
       self.manifest.Override(opt.manifest_name)
 
     manifest_name = opt.manifest_name
-    smart_sync_manifest_name = "smart_sync_override.xml"
     smart_sync_manifest_path = os.path.join(
-      self.manifest.manifestProject.worktree, smart_sync_manifest_name)
+      self.manifest.manifestProject.worktree, 'smart_sync_override.xml')
 
     if opt.smart_sync or opt.smart_tag:
-      if not self.manifest.manifest_server:
-        print('error: cannot smart sync: no manifest server defined in '
-              'manifest', file=sys.stderr)
-        sys.exit(1)
-
-      manifest_server = self.manifest.manifest_server
-      if not opt.quiet:
-        print('Using manifest server %s' % manifest_server)
-
-      if not '@' in manifest_server:
-        username = None
-        password = None
-        if opt.manifest_server_username and opt.manifest_server_password:
-          username = opt.manifest_server_username
-          password = opt.manifest_server_password
-        else:
-          try:
-            info = netrc.netrc()
-          except IOError:
-            # .netrc file does not exist or could not be opened
-            pass
-          else:
-            try:
-              parse_result = urllib.parse.urlparse(manifest_server)
-              if parse_result.hostname:
-                auth = info.authenticators(parse_result.hostname)
-                if auth:
-                  username, _account, password = auth
-                else:
-                  print('No credentials found for %s in .netrc'
-                        % parse_result.hostname, file=sys.stderr)
-            except netrc.NetrcParseError as e:
-              print('Error parsing .netrc file: %s' % e, file=sys.stderr)
-
-        if (username and password):
-          manifest_server = manifest_server.replace('://', '://%s:%s@' %
-                                                    (username, password),
-                                                    1)
-
-      transport = PersistentTransport(manifest_server)
-      if manifest_server.startswith('persistent-'):
-        manifest_server = manifest_server[len('persistent-'):]
-
-      try:
-        server = xmlrpc.client.Server(manifest_server, transport=transport)
-        if opt.smart_sync:
-          p = self.manifest.manifestProject
-          b = p.GetBranch(p.CurrentBranch)
-          branch = b.merge
-          if branch.startswith(R_HEADS):
-            branch = branch[len(R_HEADS):]
-
-          env = os.environ.copy()
-          if 'SYNC_TARGET' in env:
-            target = env['SYNC_TARGET']
-            [success, manifest_str] = server.GetApprovedManifest(branch, target)
-          elif 'TARGET_PRODUCT' in env and 'TARGET_BUILD_VARIANT' in env:
-            target = '%s-%s' % (env['TARGET_PRODUCT'],
-                                env['TARGET_BUILD_VARIANT'])
-            [success, manifest_str] = server.GetApprovedManifest(branch, target)
-          else:
-            [success, manifest_str] = server.GetApprovedManifest(branch)
-        else:
-          assert(opt.smart_tag)
-          [success, manifest_str] = server.GetManifest(opt.smart_tag)
-
-        if success:
-          manifest_name = smart_sync_manifest_name
-          try:
-            f = open(smart_sync_manifest_path, 'w')
-            try:
-              f.write(manifest_str)
-            finally:
-              f.close()
-          except IOError as e:
-            print('error: cannot write manifest to %s:\n%s'
-                  % (smart_sync_manifest_path, e),
-                  file=sys.stderr)
-            sys.exit(1)
-          self._ReloadManifest(manifest_name)
-        else:
-          print('error: manifest server RPC call failed: %s' %
-                manifest_str, file=sys.stderr)
-          sys.exit(1)
-      except (socket.error, IOError, xmlrpc.client.Fault) as e:
-        print('error: cannot connect to manifest server %s:\n%s'
-              % (self.manifest.manifest_server, e), file=sys.stderr)
-        sys.exit(1)
-      except xmlrpc.client.ProtocolError as e:
-        print('error: cannot connect to manifest server %s:\n%d %s'
-              % (self.manifest.manifest_server, e.errcode, e.errmsg),
-              file=sys.stderr)
-        sys.exit(1)
-    else:  # Not smart sync or smart tag mode
+      manifest_name = self._SmartSyncSetup(opt, smart_sync_manifest_path)
+    else:
       if os.path.isfile(smart_sync_manifest_path):
         try:
           platform_utils.remove(smart_sync_manifest_path)
@@ -724,29 +907,7 @@ later is required to fix a server side protocol bug.
     if opt.repo_upgraded:
       _PostRepoUpgrade(self.manifest, quiet=opt.quiet)
 
-    if not opt.local_only:
-      start = time.time()
-      success = mp.Sync_NetworkHalf(quiet=opt.quiet,
-                                    current_branch_only=opt.current_branch_only,
-                                    no_tags=opt.no_tags,
-                                    optimized_fetch=opt.optimized_fetch,
-                                    submodules=self.manifest.HasSubmodules)
-      finish = time.time()
-      self.event_log.AddSync(mp, event_log.TASK_SYNC_NETWORK,
-                             start, finish, success)
-
-    if mp.HasChanges:
-      syncbuf = SyncBuffer(mp.config)
-      start = time.time()
-      mp.Sync_LocalHalf(syncbuf, submodules=self.manifest.HasSubmodules)
-      clean = syncbuf.Finish()
-      self.event_log.AddSync(mp, event_log.TASK_SYNC_LOCAL,
-                             start, time.time(), clean)
-      if not clean:
-        sys.exit(1)
-      self._ReloadManifest(manifest_name)
-      if opt.jobs is None:
-        self.jobs = self.manifest.default.sync_j
+    self._UpdateManifestProject(opt, mp, manifest_name)
 
     if self.gitc_manifest:
       gitc_manifest_projects = self.GetProjects(args,
@@ -827,23 +988,10 @@ later is required to fix a server side protocol bug.
       # bail out now, we have no working tree
       return
 
-    if self.UpdateProjectList():
+    if self.UpdateProjectList(opt):
       sys.exit(1)
 
-    syncbuf = SyncBuffer(mp.config,
-                         detach_head = opt.detach_head)
-    pm = Progress('Syncing work tree', len(all_projects))
-    for project in all_projects:
-      pm.update()
-      if project.worktree:
-        start = time.time()
-        project.Sync_LocalHalf(syncbuf, force_sync=opt.force_sync)
-        self.event_log.AddSync(project, event_log.TASK_SYNC_LOCAL,
-                               start, time.time(), syncbuf.Recently())
-    pm.end()
-    print(file=sys.stderr)
-    if not syncbuf.Finish():
-      sys.exit(1)
+    self._Checkout(all_projects, opt)
 
     # If there's a notice that's supposed to print at the end of the sync, print
     # it now...
@@ -947,11 +1095,8 @@ class _FetchTimes(object):
   def _Load(self):
     if self._times is None:
       try:
-        f = open(self._path)
-        try:
+        with open(self._path) as f:
           self._times = json.load(f)
-        finally:
-          f.close()
       except (IOError, ValueError):
         try:
           platform_utils.remove(self._path)
@@ -971,11 +1116,8 @@ class _FetchTimes(object):
       del self._times[name]
 
     try:
-      f = open(self._path, 'w')
-      try:
+      with open(self._path, 'w') as f:
         json.dump(self._times, f, indent=2)
-      finally:
-        f.close()
     except (IOError, TypeError):
       try:
         platform_utils.remove(self._path)

subcmds/upload.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2008 The Android Open Source Project
 #
@@ -220,7 +221,9 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
       for commit in commit_list:
         print('         %s' % commit)
 
-      sys.stdout.write('to %s (y/N)? ' % remote.review)
+      print('to %s (y/N)? ' % remote.review, end='')
+      # TODO: When we require Python 3, use flush=True w/print above.
+      sys.stdout.flush()
       answer = sys.stdin.readline().strip().lower()
       answer = answer in ('y', 'yes', '1', 'true', 't')
 
@@ -268,11 +271,6 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
       branches[project.name] = b
     script.append('')
 
-    script = [ x.encode('utf-8')
-             if issubclass(type(x), unicode)
-             else x
-             for x in script ]
-
     script = Editor.EditString("\n".join(script)).split("\n")
 
     project_re = re.compile(r'^#?\s*project\s*([^\s]+)/:$')
@@ -359,10 +357,13 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
 
           # if they want to auto upload, let's not ask because it could be automated
           if answer is None:
-            sys.stdout.write('Uncommitted changes in ' + branch.project.name)
-            sys.stdout.write(' (did you forget to amend?):\n')
-            sys.stdout.write('\n'.join(changes) + '\n')
-            sys.stdout.write('Continue uploading? (y/N) ')
+            print()
+            print('Uncommitted changes in %s (did you forget to amend?):'
+                  % branch.project.name)
+            print('\n'.join(changes))
+            print('Continue uploading? (y/N) ', end='')
+            # TODO: When we require Python 3, use flush=True w/print above.
+            sys.stdout.flush()
             a = sys.stdin.readline().strip().lower()
             if a not in ('y', 'yes', 't', 'true', 'on'):
               print("skipping upload", file=sys.stderr)

subcmds/version.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2009 The Android Open Source Project
 #
@@ -16,7 +17,7 @@
 from __future__ import print_function
 import sys
 from command import Command, MirrorSafeCommand
-from git_command import git
+from git_command import git, RepoSourceVersion, user_agent
 from git_refs import HEAD
 
 class Version(Command, MirrorSafeCommand):
@@ -33,12 +34,20 @@ class Version(Command, MirrorSafeCommand):
     rp = self.manifest.repoProject
     rem = rp.GetRemote(rp.remote.name)
 
-    print('repo version %s' % rp.work_git.describe(HEAD))
+    # These might not be the same.  Report them both.
+    src_ver = RepoSourceVersion()
+    rp_ver = rp.bare_git.describe(HEAD)
+    print('repo version %s' % rp_ver)
     print('       (from %s)' % rem.url)
 
     if Version.wrapper_path is not None:
       print('repo launcher version %s' % Version.wrapper_version)
       print('       (from %s)' % Version.wrapper_path)
 
-    print(git.version().strip())
+      if src_ver != rp_ver:
+        print('       (currently at %s)' % src_ver)
+
+    print('repo User-Agent %s' % user_agent.repo)
+    print('git %s' % git.version_tuple().full)
+    print('git User-Agent %s' % user_agent.git)
     print('Python %s' % sys.version)

tests/fixtures/.gitignore

@@ -0,0 +1,2 @@
+/.repo_not.present.gitconfig.json
+/.repo_test.gitconfig.json

tests/test_editor.py

@@ -0,0 +1,60 @@
+# -*- coding:utf-8 -*-
+#
+# Copyright (C) 2019 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Unittests for the editor.py module."""
+
+from __future__ import print_function
+
+import unittest
+
+from editor import Editor
+
+
+class EditorTestCase(unittest.TestCase):
+  """Take care of resetting Editor state across tests."""
+
+  def setUp(self):
+    self.setEditor(None)
+
+  def tearDown(self):
+    self.setEditor(None)
+
+  @staticmethod
+  def setEditor(editor):
+    Editor._editor = editor
+
+
+class GetEditor(EditorTestCase):
+  """Check GetEditor behavior."""
+
+  def test_basic(self):
+    """Basic checking of _GetEditor."""
+    self.setEditor(':')
+    self.assertEqual(':', Editor._GetEditor())
+
+
+class EditString(EditorTestCase):
+  """Check EditString behavior."""
+
+  def test_no_editor(self):
+    """Check behavior when no editor is available."""
+    self.setEditor(':')
+    self.assertEqual('foo', Editor.EditString('foo'))
+
+  def test_cat_editor(self):
+    """Check behavior when editor is `cat`."""
+    self.setEditor('cat')
+    self.assertEqual('foo', Editor.EditString('foo'))

tests/test_git_command.py

@@ -0,0 +1,78 @@
+# -*- coding:utf-8 -*-
+#
+# Copyright 2019 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Unittests for the git_command.py module."""
+
+from __future__ import print_function
+
+import re
+import unittest
+
+import git_command
+
+
+class GitCallUnitTest(unittest.TestCase):
+  """Tests the _GitCall class (via git_command.git)."""
+
+  def test_version_tuple(self):
+    """Check git.version_tuple() handling."""
+    ver = git_command.git.version_tuple()
+    self.assertIsNotNone(ver)
+
+    # We don't dive too deep into the values here to avoid having to update
+    # whenever git versions change.  We do check relative to this min version
+    # as this is what `repo` itself requires via MIN_GIT_VERSION.
+    MIN_GIT_VERSION = (1, 7, 2)
+    self.assertTrue(isinstance(ver.major, int))
+    self.assertTrue(isinstance(ver.minor, int))
+    self.assertTrue(isinstance(ver.micro, int))
+
+    self.assertGreater(ver.major, MIN_GIT_VERSION[0] - 1)
+    self.assertGreaterEqual(ver.micro, 0)
+    self.assertGreaterEqual(ver.major, 0)
+
+    self.assertGreaterEqual(ver, MIN_GIT_VERSION)
+    self.assertLess(ver, (9999, 9999, 9999))
+
+    self.assertNotEqual('', ver.full)
+
+
+class UserAgentUnitTest(unittest.TestCase):
+  """Tests the UserAgent function."""
+
+  def test_smoke_os(self):
+    """Make sure UA OS setting returns something useful."""
+    os_name = git_command.user_agent.os
+    # We can't dive too deep because of OS/tool differences, but we can check
+    # the general form.
+    m = re.match(r'^[^ ]+$', os_name)
+    self.assertIsNotNone(m)
+
+  def test_smoke_repo(self):
+    """Make sure repo UA returns something useful."""
+    ua = git_command.user_agent.repo
+    # We can't dive too deep because of OS/tool differences, but we can check
+    # the general form.
+    m = re.match(r'^git-repo/[^ ]+ ([^ ]+) git/[^ ]+ Python/[0-9.]+', ua)
+    self.assertIsNotNone(m)
+
+  def test_smoke_git(self):
+    """Make sure git UA returns something useful."""
+    ua = git_command.user_agent.git
+    # We can't dive too deep because of OS/tool differences, but we can check
+    # the general form.
+    m = re.match(r'^git/[^ ]+ ([^ ]+) git-repo/[^ ]+', ua)
+    self.assertIsNotNone(m)

tests/test_git_config.py

@@ -1,3 +1,23 @@
+# -*- coding:utf-8 -*-
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Unittests for the git_config.py module."""
+
+from __future__ import print_function
+
 import os
 import unittest
 

tests/test_project.py

@@ -0,0 +1,136 @@
+# -*- coding:utf-8 -*-
+#
+# Copyright (C) 2019 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Unittests for the project.py module."""
+
+from __future__ import print_function
+
+import contextlib
+import os
+import shutil
+import subprocess
+import tempfile
+import unittest
+
+import git_config
+import project
+
+
+@contextlib.contextmanager
+def TempGitTree():
+  """Create a new empty git checkout for testing."""
+  # TODO(vapier): Convert this to tempfile.TemporaryDirectory once we drop
+  # Python 2 support entirely.
+  try:
+    tempdir = tempfile.mkdtemp(prefix='repo-tests')
+    subprocess.check_call(['git', 'init'], cwd=tempdir)
+    yield tempdir
+  finally:
+    shutil.rmtree(tempdir)
+
+
+class RepoHookShebang(unittest.TestCase):
+  """Check shebang parsing in RepoHook."""
+
+  def test_no_shebang(self):
+    """Lines w/out shebangs should be rejected."""
+    DATA = (
+        '',
+        '# -*- coding:utf-8 -*-\n',
+        '#\n# foo\n',
+        '# Bad shebang in script\n#!/foo\n'
+    )
+    for data in DATA:
+      self.assertIsNone(project.RepoHook._ExtractInterpFromShebang(data))
+
+  def test_direct_interp(self):
+    """Lines whose shebang points directly to the interpreter."""
+    DATA = (
+        ('#!/foo', '/foo'),
+        ('#! /foo', '/foo'),
+        ('#!/bin/foo ', '/bin/foo'),
+        ('#! /usr/foo ', '/usr/foo'),
+        ('#! /usr/foo -args', '/usr/foo'),
+    )
+    for shebang, interp in DATA:
+      self.assertEqual(project.RepoHook._ExtractInterpFromShebang(shebang),
+                       interp)
+
+  def test_env_interp(self):
+    """Lines whose shebang launches through `env`."""
+    DATA = (
+        ('#!/usr/bin/env foo', 'foo'),
+        ('#!/bin/env foo', 'foo'),
+        ('#! /bin/env /bin/foo ', '/bin/foo'),
+    )
+    for shebang, interp in DATA:
+      self.assertEqual(project.RepoHook._ExtractInterpFromShebang(shebang),
+                       interp)
+
+
+class FakeProject(object):
+  """A fake for Project for basic functionality."""
+
+  def __init__(self, worktree):
+    self.worktree = worktree
+    self.gitdir = os.path.join(worktree, '.git')
+    self.name = 'fakeproject'
+    self.work_git = project.Project._GitGetByExec(
+        self, bare=False, gitdir=self.gitdir)
+    self.bare_git = project.Project._GitGetByExec(
+        self, bare=True, gitdir=self.gitdir)
+    self.config = git_config.GitConfig.ForRepository(gitdir=self.gitdir)
+
+
+class ReviewableBranchTests(unittest.TestCase):
+  """Check ReviewableBranch behavior."""
+
+  def test_smoke(self):
+    """A quick run through everything."""
+    with TempGitTree() as tempdir:
+      fakeproj = FakeProject(tempdir)
+
+      # Generate some commits.
+      with open(os.path.join(tempdir, 'readme'), 'w') as fp:
+        fp.write('txt')
+      fakeproj.work_git.add('readme')
+      fakeproj.work_git.commit('-mAdd file')
+      fakeproj.work_git.checkout('-b', 'work')
+      fakeproj.work_git.rm('-f', 'readme')
+      fakeproj.work_git.commit('-mDel file')
+
+      # Start off with the normal details.
+      rb = project.ReviewableBranch(
+          fakeproj, fakeproj.config.GetBranch('work'), 'master')
+      self.assertEqual('work', rb.name)
+      self.assertEqual(1, len(rb.commits))
+      self.assertIn('Del file', rb.commits[0])
+      d = rb.unabbrev_commits
+      self.assertEqual(1, len(d))
+      short, long = next(iter(d.items()))
+      self.assertTrue(long.startswith(short))
+      self.assertTrue(rb.base_exists)
+      # Hard to assert anything useful about this.
+      self.assertTrue(rb.date)
+
+      # Now delete the tracking branch!
+      fakeproj.work_git.branch('-D', 'master')
+      rb = project.ReviewableBranch(
+          fakeproj, fakeproj.config.GetBranch('work'), 'master')
+      self.assertEqual(0, len(rb.commits))
+      self.assertFalse(rb.base_exists)
+      # Hard to assert anything useful about this.
+      self.assertTrue(rb.date)

tests/test_wrapper.py

@@ -1,3 +1,4 @@
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2015 The Android Open Source Project
 #
@@ -13,6 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+"""Unittests for the wrapper.py module."""
+
+from __future__ import print_function
+
 import os
 import unittest
 

wrapper.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+# -*- coding:utf-8 -*-
 #
 # Copyright (C) 2014 The Android Open Source Project
 #
@@ -15,7 +15,12 @@
 # limitations under the License.
 
 from __future__ import print_function
-import imp
+try:
+  from importlib.machinery import SourceFileLoader
+  _loader = lambda *args: SourceFileLoader(*args).load_module()
+except ImportError:
+  import imp
+  _loader = lambda *args: imp.load_source(*args)
 import os
 
 
@@ -26,5 +31,5 @@ _wrapper_module = None
 def Wrapper():
   global _wrapper_module
   if not _wrapper_module:
-    _wrapper_module = imp.load_source('wrapper', WrapperPath())
+    _wrapper_module = _loader('wrapper', WrapperPath())
   return _wrapper_module