Merge "Revert "GITC: Always update the gitc manifest from the repo manifest""

This reverts commit 250303b437.

Change-Id: I1fd8af20f802553151aacb953c913f3305ca6057

.pylintrc

@@ -61,9 +61,6 @@ disable=R0903,R0912,R0913,R0914,R0915,W0141,C0111,C0103,W0603,W0703,R0911,C0301,
 # (visual studio) and html
 output-format=text
 
-# Include message's id in output
-include-ids=yes
-
 # Put messages in a separate file for each module / package specified on the
 # command line instead of printing them on stdout. Reports (if any) will be
 # written in a file name "pylint_global.[txt|html]".

color.py

@@ -36,15 +36,17 @@ ATTRS = {None     :-1,
          'blink': 5,
          'reverse': 7}
 
-RESET = "\033[m"  # pylint: disable=W1401
+RESET = "\033[m"
-                  # backslash is not anomalous
+
 
 def is_color(s):
   return s in COLORS
 
+
 def is_attr(s):
   return s in ATTRS
 
+
 def _Color(fg=None, bg=None, attr=None):
   fg = COLORS[fg]
   bg = COLORS[bg]
@@ -52,7 +54,7 @@ def _Color(fg = None, bg = None, attr = None):
 
   if attr >= 0 or fg >= 0 or bg >= 0:
     need_sep = False
-    code = "\033["  #pylint: disable=W1401
+    code = "\033["
 
     if attr >= 0:
       code += chr(ord('0') + attr)
@@ -71,7 +73,6 @@ def _Color(fg = None, bg = None, attr = None):
     if bg >= 0:
       if need_sep:
         code += ';'
-      need_sep = True
 
       if bg < 8:
         code += '4%c' % (ord('0') + bg)
@@ -82,6 +83,27 @@ def _Color(fg = None, bg = None, attr = None):
     code = ''
   return code
 
+DEFAULT = None
+
+
+def SetDefaultColoring(state):
+  """Set coloring behavior to |state|.
+
+  This is useful for overriding config options via the command line.
+  """
+  if state is None:
+    # Leave it alone -- return quick!
+    return
+
+  global DEFAULT
+  state = state.lower()
+  if state in ('auto',):
+    DEFAULT = state
+  elif state in ('always', 'yes', 'true', True):
+    DEFAULT = 'always'
+  elif state in ('never', 'no', 'false', False):
+    DEFAULT = 'never'
+
 
 class Coloring(object):
   def __init__(self, config, section_type):
@@ -89,6 +111,8 @@ class Coloring(object):
     self._config = config
     self._out = sys.stdout
 
+    on = DEFAULT
+    if on is None:
       on = self._config.GetString(self._section)
       if on is None:
         on = self._config.GetString('color.ui')
@@ -122,6 +146,7 @@ class Coloring(object):
   def printer(self, opt=None, fg=None, bg=None, attr=None):
     s = self
     c = self.colorer(opt, fg, bg, attr)
+
     def f(fmt, *args):
       s._out.write(c(fmt, *args))
     return f
@@ -129,6 +154,7 @@ class Coloring(object):
   def nofmt_printer(self, opt=None, fg=None, bg=None, attr=None):
     s = self
     c = self.nofmt_colorer(opt, fg, bg, attr)
+
     def f(fmt):
       s._out.write(c(fmt))
     return f
@@ -136,11 +162,13 @@ class Coloring(object):
   def colorer(self, opt=None, fg=None, bg=None, attr=None):
     if self._on:
       c = self._parse(opt, fg, bg, attr)
+
       def f(fmt, *args):
         output = fmt % args
         return ''.join([c, output, RESET])
       return f
     else:
+
       def f(fmt, *args):
         return fmt % args
       return f
@@ -148,6 +176,7 @@ class Coloring(object):
   def nofmt_colorer(self, opt=None, fg=None, bg=None, attr=None):
     if self._on:
       c = self._parse(opt, fg, bg, attr)
+
       def f(fmt):
         return ''.join([c, fmt, RESET])
       return f

command.py

@@ -106,13 +106,13 @@ class Command(object):
   def _UpdatePathToProjectMap(self, project):
     self._by_path[project.worktree] = project
 
-  def _GetProjectByPath(self, path):
+  def _GetProjectByPath(self, manifest, path):
     project = None
     if os.path.exists(path):
       oldpath = None
       while path \
         and path != oldpath \
-        and path != self.manifest.topdir:
+        and path != manifest.topdir:
         try:
           project = self._by_path[path]
           break
@@ -126,14 +126,18 @@ class Command(object):
         pass
     return project
 
-  def GetProjects(self, args, missing_ok=False, submodules_ok=False):
+  def GetProjects(self, args, manifest=None, groups='', missing_ok=False,
+                  submodules_ok=False):
     """A list of projects that match the arguments.
     """
-    all_projects_list = self.manifest.projects
+    if not manifest:
+      manifest = self.manifest
+    all_projects_list = manifest.projects
     result = []
 
-    mp = self.manifest.manifestProject
+    mp = manifest.manifestProject
 
+    if not groups:
         groups = mp.config.GetString('manifest.groups')
     if not groups:
       groups = 'default,platform-' + platform.system().lower()
@@ -154,11 +158,11 @@ class Command(object):
       self._ResetPathToProjectMap(all_projects_list)
 
       for arg in args:
-        projects = self.manifest.GetProjectsWithName(arg)
+        projects = manifest.GetProjectsWithName(arg)
 
         if not projects:
           path = os.path.abspath(arg).replace('\\', '/')
-          project = self._GetProjectByPath(path)
+          project = self._GetProjectByPath(manifest, path)
 
           # If it's not a derived project, update path->project mapping and
           # search again, as arg might actually point to a derived subproject.
@@ -169,7 +173,7 @@ class Command(object):
               self._UpdatePathToProjectMap(subproject)
               search_again = True
             if search_again:
-              project = self._GetProjectByPath(path) or project
+              project = self._GetProjectByPath(manifest, path) or project
 
           if project:
             projects = [project]
@@ -226,3 +230,8 @@ class MirrorSafeCommand(object):
   """Command permits itself to run within a mirror,
      and does not require a working directory.
   """
+
+class RequiresGitcCommand(object):
+  """Command that requires GITC to be available, but does
+     not require the local client to be a GITC client.
+  """

docs/manifest-format.txt

@@ -26,6 +26,7 @@ following DTD:
                         manifest-server?,
                         remove-project*,
                         project*,
+                        extend-project*,
                         repo-hooks?)>
 
     <!ELEMENT notice (#PCDATA)>
@@ -35,6 +36,7 @@ following DTD:
     <!ATTLIST remote alias        CDATA #IMPLIED>
     <!ATTLIST remote fetch        CDATA #REQUIRED>
     <!ATTLIST remote review       CDATA #IMPLIED>
+    <!ATTLIST remote revision     CDATA #IMPLIED>
 
     <!ELEMENT default (EMPTY)>
     <!ATTLIST default remote      IDREF #IMPLIED>
@@ -66,6 +68,11 @@ following DTD:
     <!ATTLIST annotation value CDATA #REQUIRED>
     <!ATTLIST annotation keep  CDATA "true">
 
+    <!ELEMENT extend-project>
+    <!ATTLIST extend-project name CDATA #REQUIRED>
+    <!ATTLIST extend-project path CDATA #IMPLIED>
+    <!ATTLIST extend-project groups CDATA #IMPLIED>
+
     <!ELEMENT remove-project (EMPTY)>
     <!ATTLIST remove-project name  CDATA #REQUIRED>
 
@@ -112,6 +119,10 @@ Attribute `review`: Hostname of the Gerrit server where reviews
 are uploaded to by `repo upload`.  This attribute is optional;
 if not specified then `repo upload` will not function.
 
+Attribute `revision`: Name of a Git branch (e.g. `master` or
+`refs/heads/master`). Remotes with their own revision will override
+the default revision.
+
 Element default
 ---------------
 
@@ -132,14 +143,14 @@ Project elements not setting their own `dest-branch` will inherit
 this value. If this value is not set, projects will use `revision`
 by default instead.
 
-Attribute `sync_j`: Number of parallel jobs to use when synching.
+Attribute `sync-j`: Number of parallel jobs to use when synching.
 
-Attribute `sync_c`: Set to true to only sync the given Git
+Attribute `sync-c`: Set to true to only sync the given Git
 branch (specified in the `revision` attribute) rather than the
-whole ref space.  Project elements lacking a sync_c element of
+whole ref space.  Project elements lacking a sync-c element of
 their own will use this value.
 
-Attribute `sync_s`: Set to true to also sync sub-projects.
+Attribute `sync-s`: Set to true to also sync sub-projects.
 
 
 Element manifest-server
@@ -208,7 +219,8 @@ to track for this project.  Names can be relative to refs/heads
 (e.g. just "master") or absolute (e.g. "refs/heads/master").
 Tags and/or explicit SHA-1s should work in theory, but have not
 been extensively tested.  If not supplied the revision given by
-the default element is used.
+the remote element is used if applicable, else the default
+element is used.
 
 Attribute `dest-branch`: Name of a Git branch (e.g. `master`).
 When using `repo upload`, changes will be submitted for code
@@ -226,13 +238,13 @@ group "notdefault", it will not be automatically downloaded by repo.
 If the project has a parent element, the `name` and `path` here
 are the prefixed ones.
 
-Attribute `sync_c`: Set to true to only sync the given Git
+Attribute `sync-c`: Set to true to only sync the given Git
 branch (specified in the `revision` attribute) rather than the
 whole ref space.
 
-Attribute `sync_s`: Set to true to also sync sub-projects.
+Attribute `sync-s`: Set to true to also sync sub-projects.
 
-Attribute `upstream`: Name of the Git branch in which a sha1
+Attribute `upstream`: Name of the Git ref in which a sha1
 can be found.  Used when syncing a revision locked manifest in
 -c mode to avoid having to sync the entire ref space.
 
@@ -246,6 +258,22 @@ rather than the `name` attribute.  This attribute only applies to the
 local mirrors syncing, it will be ignored when syncing the projects in a
 client working directory.
 
+Element extend-project
+----------------------
+
+Modify the attributes of the named project.
+
+This element is mostly useful in a local manifest file, to modify the
+attributes of an existing project without completely replacing the
+existing project definition.  This makes the local manifest more robust
+against changes to the original manifest.
+
+Attribute `path`: If specified, limit the change to projects checked out
+at the specified path, rather than all projects with the given name.
+
+Attribute `groups`: List of additional groups to which this project
+belongs.  Same syntax as the corresponding element of `project`.
+
 Element annotation
 ------------------
 

error.py

@@ -80,7 +80,7 @@ class NoSuchProjectError(Exception):
     self.name = name
 
   def __str__(self):
-    if self.Name is None:
+    if self.name is None:
       return 'in current directory'
     return self.name
 
@@ -93,7 +93,7 @@ class InvalidProjectGroupsError(Exception):
     self.name = name
 
   def __str__(self):
-    if self.Name is None:
+    if self.name is None:
       return 'in current directory'
     return self.name
 

git_command.py

@@ -14,7 +14,9 @@
 # limitations under the License.
 
 from __future__ import print_function
+import fcntl
 import os
+import select
 import sys
 import subprocess
 import tempfile
@@ -76,17 +78,30 @@ def terminate_ssh_clients():
 
 _git_version = None
 
+class _sfd(object):
+  """select file descriptor class"""
+  def __init__(self, fd, dest, std_name):
+    assert std_name in ('stdout', 'stderr')
+    self.fd = fd
+    self.dest = dest
+    self.std_name = std_name
+  def fileno(self):
+    return self.fd.fileno()
+
 class _GitCall(object):
   def version(self):
     p = GitCommand(None, ['--version'], capture_stdout=True)
     if p.Wait() == 0:
+      if hasattr(p.stdout, 'decode'):
+        return p.stdout.decode('utf-8')
+      else:
         return p.stdout
     return None
 
   def version_tuple(self):
     global _git_version
     if _git_version is None:
-      ver_str = git.version().decode('utf-8')
+      ver_str = git.version()
       _git_version = Wrapper().ParseGitVersion(ver_str)
       if _git_version is None:
         print('fatal: "%s" unsupported' % ver_str, file=sys.stderr)
@@ -139,6 +154,9 @@ class GitCommand(object):
       if key in env:
         del env[key]
 
+    # If we are not capturing std* then need to print it.
+    self.tee = {'stdout': not capture_stdout, 'stderr': not capture_stderr}
+
     if disable_editor:
       _setenv(env, 'GIT_EDITOR', ':')
     if ssh_proxy:
@@ -162,22 +180,21 @@ class GitCommand(object):
       if gitdir:
         _setenv(env, GIT_DIR, gitdir)
       cwd = None
-    command.extend(cmdv)
+    command.append(cmdv[0])
+    # Need to use the --progress flag for fetch/clone so output will be
+    # displayed as by default git only does progress output if stderr is a TTY.
+    if sys.stderr.isatty() and cmdv[0] in ('fetch', 'clone'):
+      if '--progress' not in cmdv and '--quiet' not in cmdv:
+        command.append('--progress')
+    command.extend(cmdv[1:])
 
     if provide_stdin:
       stdin = subprocess.PIPE
     else:
       stdin = None
 
-    if capture_stdout:
     stdout = subprocess.PIPE
-    else:
-      stdout = None
-
-    if capture_stderr:
     stderr = subprocess.PIPE
-    else:
-      stderr = None
 
     if IsTrace():
       global LAST_CWD
@@ -226,8 +243,36 @@ class GitCommand(object):
   def Wait(self):
     try:
       p = self.process
-      (self.stdout, self.stderr) = p.communicate()
+      rc = self._CaptureOutput()
-      rc = p.returncode
     finally:
       _remove_ssh_client(p)
     return rc
+
+  def _CaptureOutput(self):
+    p = self.process
+    s_in = [_sfd(p.stdout, sys.stdout, 'stdout'),
+            _sfd(p.stderr, sys.stderr, 'stderr')]
+    self.stdout = ''
+    self.stderr = ''
+
+    for s in s_in:
+      flags = fcntl.fcntl(s.fd, fcntl.F_GETFL)
+      fcntl.fcntl(s.fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
+
+    while s_in:
+      in_ready, _, _ = select.select(s_in, [], [])
+      for s in in_ready:
+        buf = s.fd.read(4096)
+        if not buf:
+          s_in.remove(s)
+          continue
+        if not hasattr(buf, 'encode'):
+          buf = buf.decode()
+        if s.std_name == 'stdout':
+          self.stdout += buf
+        else:
+          self.stderr += buf
+        if self.tee[s.std_name]:
+          s.dest.write(buf)
+          s.dest.flush()
+    return p.wait()

git_config.py

@@ -15,8 +15,10 @@
 
 from __future__ import print_function
 
+import contextlib
+import errno
+import json
 import os
-import pickle
 import re
 import subprocess
 import sys
@@ -80,7 +82,7 @@ class GitConfig(object):
     return cls(configfile = os.path.join(gitdir, 'config'),
                defaults = defaults)
 
-  def __init__(self, configfile, defaults=None, pickleFile=None):
+  def __init__(self, configfile, defaults=None, jsonFile=None):
     self.file = configfile
     self.defaults = defaults
     self._cache_dict = None
@@ -88,12 +90,11 @@ class GitConfig(object):
     self._remotes = {}
     self._branches = {}
 
-    if pickleFile is None:
+    self._json = jsonFile
-      self._pickle = os.path.join(
+    if self._json is None:
+      self._json = os.path.join(
         os.path.dirname(self.file),
-        '.repopickle_' + os.path.basename(self.file))
+        '.repo_' + os.path.basename(self.file) + '.json')
-    else:
-      self._pickle = pickleFile
 
   def Has(self, name, include_defaults = True):
     """Return true if this configuration file has the key.
@@ -217,7 +218,7 @@ class GitConfig(object):
     """Resolve any url.*.insteadof references.
     """
     for new_url in self.GetSubSections('url'):
-      old_url = self.GetString('url.%s.insteadof' % new_url)
+      for old_url in self.GetString('url.%s.insteadof' % new_url, True):
         if old_url is not None and url.startswith(old_url):
           return new_url + url[len(old_url):]
     return url
@@ -248,50 +249,41 @@ class GitConfig(object):
     return self._cache_dict
 
   def _Read(self):
-    d = self._ReadPickle()
+    d = self._ReadJson()
     if d is None:
       d = self._ReadGit()
-      self._SavePickle(d)
+      self._SaveJson(d)
     return d
 
-  def _ReadPickle(self):
+  def _ReadJson(self):
     try:
-      if os.path.getmtime(self._pickle) \
+      if os.path.getmtime(self._json) \
       <= os.path.getmtime(self.file):
-        os.remove(self._pickle)
+        os.remove(self._json)
         return None
     except OSError:
       return None
     try:
-      Trace(': unpickle %s', self.file)
+      Trace(': parsing %s', self.file)
-      fd = open(self._pickle, 'rb')
+      fd = open(self._json)
       try:
-        return pickle.load(fd)
+        return json.load(fd)
       finally:
         fd.close()
-    except EOFError:
+    except (IOError, ValueError):
-      os.remove(self._pickle)
+      os.remove(self._json)
-      return None
-    except IOError:
-      os.remove(self._pickle)
-      return None
-    except pickle.PickleError:
-      os.remove(self._pickle)
       return None
 
-  def _SavePickle(self, cache):
+  def _SaveJson(self, cache):
     try:
-      fd = open(self._pickle, 'wb')
+      fd = open(self._json, 'w')
       try:
-        pickle.dump(cache, fd, pickle.HIGHEST_PROTOCOL)
+        json.dump(cache, fd, indent=2)
       finally:
         fd.close()
-    except IOError:
+    except (IOError, TypeError):
-      if os.path.exists(self._pickle):
+      if os.path.exists(self._json):
-        os.remove(self._pickle)
+        os.remove(self._json)
-    except pickle.PickleError:
-      if os.path.exists(self._pickle):
-        os.remove(self._pickle)
 
   def _ReadGit(self):
     """
@@ -512,6 +504,43 @@ def GetSchemeFromUrl(url):
     return m.group(1)
   return None
 
+@contextlib.contextmanager
+def GetUrlCookieFile(url, quiet):
+  if url.startswith('persistent-'):
+    try:
+      p = subprocess.Popen(
+          ['git-remote-persistent-https', '-print_config', url],
+          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+          stderr=subprocess.PIPE)
+      try:
+        cookieprefix = 'http.cookiefile='
+        proxyprefix = 'http.proxy='
+        cookiefile = None
+        proxy = None
+        for line in p.stdout:
+          line = line.strip()
+          if line.startswith(cookieprefix):
+            cookiefile = line[len(cookieprefix):]
+          if line.startswith(proxyprefix):
+            proxy = line[len(proxyprefix):]
+        # Leave subprocess open, as cookie file may be transient.
+        if cookiefile or proxy:
+          yield cookiefile, proxy
+          return
+      finally:
+        p.stdin.close()
+        if p.wait():
+          err_msg = p.stderr.read()
+          if ' -print_config' in err_msg:
+            pass  # Persistent proxy doesn't support -print_config.
+          elif not quiet:
+            print(err_msg, file=sys.stderr)
+    except OSError as e:
+      if e.errno == errno.ENOENT:
+        pass  # No persistent proxy.
+      raise
+  yield GitConfig.ForUser().GetString('http.cookiefile'), None
+
 def _preconnect(url):
   m = URI_ALL.match(url)
   if m:
@@ -576,6 +605,8 @@ class Remote(object):
         return None
 
       u = self.review
+      if u.startswith('persistent-'):
+        u = u[len('persistent-'):]
       if u.split(':')[0] not in ('http', 'https', 'sso'):
         u = 'http://%s' % u
       if u.endswith('/Gerrit'):
@@ -627,9 +658,7 @@ class Remote(object):
   def ToLocal(self, rev):
     """Convert a remote revision string to something we have locally.
     """
-    if IsId(rev):
+    if self.name == '.' or IsId(rev):
-      return rev
-    if rev.startswith(R_TAGS):
       return rev
 
     if not rev.startswith('refs/'):
@@ -638,6 +667,10 @@ class Remote(object):
     for spec in self.fetch:
       if spec.SourceMatches(rev):
         return spec.MapSource(rev)
+
+    if not rev.startswith(R_HEADS):
+      return rev
+
     raise GitError('remote %s does not have %s' % (self.name, rev))
 
   def WritesTo(self, ref):
@@ -707,7 +740,7 @@ class Branch(object):
       self._Set('merge', self.merge)
 
     else:
-      fd = open(self._config.file, 'ab')
+      fd = open(self._config.file, 'a')
       try:
         fd.write('[branch "%s"]\n' % self.name)
         if self.remote:

gitc_utils.py

@@ -0,0 +1,101 @@
+#
+# Copyright (C) 2015 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+import os
+import sys
+import time
+
+import git_command
+import git_config
+import wrapper
+
+
+GITC_FS_ROOT_DIR = '/gitc/manifest-rw/'
+NUM_BATCH_RETRIEVE_REVISIONID = 300
+
+def get_gitc_manifest_dir():
+  return wrapper.Wrapper().get_gitc_manifest_dir()
+
+def parse_clientdir(gitc_fs_path):
+  """Parse a path in the GITC FS and return its client name.
+
+  @param gitc_fs_path: A subdirectory path within the GITC_FS_ROOT_DIR.
+
+  @returns: The GITC client name
+  """
+  if (gitc_fs_path == GITC_FS_ROOT_DIR or
+      not gitc_fs_path.startswith(GITC_FS_ROOT_DIR)):
+    return None
+  return gitc_fs_path.split(GITC_FS_ROOT_DIR)[1].split('/')[0]
+
+def _set_project_revisions(projects):
+  """Sets the revisionExpr for a list of projects.
+
+  Because of the limit of open file descriptors allowed, length of projects
+  should not be overly large. Recommend calling this function multiple times
+  with each call not exceeding NUM_BATCH_RETRIEVE_REVISIONID projects.
+
+  @param projects: List of project objects to set the revionExpr for.
+  """
+  # Retrieve the commit id for each project based off of it's current
+  # revisionExpr and it is not already a commit id.
+  project_gitcmds = [(
+      project, git_command.GitCommand(None,
+                                      ['ls-remote',
+                                       project.remote.url,
+                                       project.revisionExpr],
+                                      capture_stdout=True, cwd='/tmp'))
+      for project in projects if not git_config.IsId(project.revisionExpr)]
+  for proj, gitcmd in project_gitcmds:
+    if gitcmd.Wait():
+      print('FATAL: Failed to retrieve revisionExpr for %s' % proj)
+      sys.exit(1)
+    proj.revisionExpr = gitcmd.stdout.split('\t')[0]
+
+def generate_gitc_manifest(client_dir, manifest, projects=None):
+  """Generate a manifest for shafsd to use for this GITC client.
+
+  @param client_dir: GITC client directory to install the .manifest file in.
+  @param manifest: XmlManifest object representing the repo manifest.
+  @param projects: List of projects we want to update, this must be a sublist
+                   of manifest.projects to work properly. If not provided,
+                   manifest.projects is used.
+  """
+  print('Generating GITC Manifest by fetching revision SHAs for each '
+        'project.')
+  if projects is None:
+    projects = manifest.projects
+  index = 0
+  while index < len(projects):
+    _set_project_revisions(
+        projects[index:(index+NUM_BATCH_RETRIEVE_REVISIONID)])
+    index += NUM_BATCH_RETRIEVE_REVISIONID
+  # Save the manifest.
+  save_manifest(manifest, client_dir=client_dir)
+
+def save_manifest(manifest, client_dir=None):
+  """Save the manifest file in the client_dir.
+
+  @param client_dir: Client directory to save the manifest in.
+  @param manifest: Manifest object to save.
+  """
+  if not client_dir:
+    client_dir = manifest.gitc_client_dir
+  with open(os.path.join(client_dir, '.manifest'), 'w') as f:
+    manifest.Save(f)
+  # TODO(sbasi/jorg): Come up with a solution to remove the sleep below.
+  # Give the GITC filesystem time to register the manifest changes.
+  time.sleep(3)

hooks/commit-msg

@@ -1,5 +1,4 @@
 #!/bin/sh
-# From Gerrit Code Review 2.6
 #
 # Part of Gerrit Code Review (http://code.google.com/p/gerrit/)
 #
@@ -27,7 +26,7 @@ MSG="$1"
 #
 add_ChangeId() {
 	clean_message=`sed -e '
-		/^diff --git a\/.*/{
+		/^diff --git .*/{
 			s///
 			q
 		}
@@ -39,6 +38,11 @@ add_ChangeId() {
 		return
 	fi
 
+	if test "false" = "`git config --bool --get gerrit.createChangeId`"
+	then
+		return
+	fi
+
 	# Does Change-Id: already exist? if so, exit (no change).
 	if grep -i '^Change-Id:' "$MSG" >/dev/null
 	then
@@ -77,7 +81,7 @@ add_ChangeId() {
 	# Skip the line starting with the diff command and everything after it,
 	# up to the end of the file, assuming it is only patch data.
 	# If more than one line before the diff was empty, strip all but one.
-	/^diff --git a/ {
+	/^diff --git / {
 		blankLines = 0
 		while (getline) { }
 		next

main.py

@@ -36,20 +36,24 @@ try:
 except ImportError:
   kerberos = None
 
+from color import SetDefaultColoring
 from trace import SetTrace
 from git_command import git, GitCommand
 from git_config import init_ssh, close_ssh
 from command import InteractiveCommand
 from command import MirrorSafeCommand
+from command import RequiresGitcCommand
 from subcmds.version import Version
 from editor import Editor
 from error import DownloadError
+from error import InvalidProjectGroupsError
 from error import ManifestInvalidRevisionError
 from error import ManifestParseError
 from error import NoManifestException
 from error import NoSuchProjectError
 from error import RepoChangedException
-from manifest_xml import XmlManifest
+import gitc_utils
+from manifest_xml import GitcManifest, XmlManifest
 from pager import RunPager
 from wrapper import WrapperPath, Wrapper
 
@@ -69,6 +73,9 @@ global_options.add_option('-p', '--paginate',
 global_options.add_option('--no-pager',
                           dest='no_pager', action='store_true',
                           help='disable the pager')
+global_options.add_option('--color',
+                          choices=('auto', 'always', 'never'), default=None,
+                          help='control color usage: auto, always, never')
 global_options.add_option('--trace',
                           dest='trace', action='store_true',
                           help='trace git command execution')
@@ -113,6 +120,8 @@ class _Repo(object):
         print('fatal: invalid usage of --version', file=sys.stderr)
         return 1
 
+    SetDefaultColoring(gopts.color)
+
     try:
       cmd = self.commands[name]
     except KeyError:
@@ -122,6 +131,12 @@ class _Repo(object):
 
     cmd.repodir = self.repodir
     cmd.manifest = XmlManifest(cmd.repodir)
+    cmd.gitc_manifest = None
+    gitc_client_name = gitc_utils.parse_clientdir(os.getcwd())
+    if gitc_client_name:
+      cmd.gitc_manifest = GitcManifest(cmd.repodir, gitc_client_name)
+      cmd.manifest.isGitcClient = True
+
     Editor.globalConfig = cmd.manifest.globalConfig
 
     if not isinstance(cmd, MirrorSafeCommand) and cmd.manifest.IsMirror:
@@ -129,6 +144,11 @@ class _Repo(object):
             file=sys.stderr)
       return 1
 
+    if isinstance(cmd, RequiresGitcCommand) and not gitc_utils.get_gitc_manifest_dir():
+      print("fatal: '%s' requires GITC to be available" % name,
+            file=sys.stderr)
+      return 1
+
     try:
       copts, cargs = cmd.OptionParser.parse_args(argv)
       copts = cmd.ReadEnvironmentOptions(copts)
@@ -167,6 +187,12 @@ class _Repo(object):
       else:
         print('error: no project in current directory', file=sys.stderr)
       result = 1
+    except InvalidProjectGroupsError as e:
+      if e.name:
+        print('error: project group must be enabled for project %s' % e.name, file=sys.stderr)
+      else:
+        print('error: project group must be enabled for the project in the current directory', file=sys.stderr)
+      result = 1
     finally:
       elapsed = time.time() - start
       hours, remainder = divmod(elapsed, 3600)

manifest_xml.py

@@ -29,6 +29,7 @@ else:
   urllib = imp.new_module('urllib')
   urllib.parse = urlparse
 
+import gitc_utils
 from git_config import GitConfig
 from git_refs import R_HEADS, HEAD
 from project import RemoteSpec, Project, MetaProject
@@ -38,8 +39,9 @@ MANIFEST_FILE_NAME = 'manifest.xml'
 LOCAL_MANIFEST_NAME = 'local_manifest.xml'
 LOCAL_MANIFESTS_DIR_NAME = 'local_manifests'
 
-urllib.parse.uses_relative.extend(['ssh', 'git'])
+# urljoin gets confused if the scheme is not known.
-urllib.parse.uses_netloc.extend(['ssh', 'git'])
+urllib.parse.uses_relative.extend(['ssh', 'git', 'persistent-https', 'rpc'])
+urllib.parse.uses_netloc.extend(['ssh', 'git', 'persistent-https', 'rpc'])
 
 class _Default(object):
   """Project defaults within the manifest."""
@@ -63,12 +65,14 @@ class _XmlRemote(object):
                alias=None,
                fetch=None,
                manifestUrl=None,
-               review=None):
+               review=None,
+               revision=None):
     self.name = name
     self.fetchUrl = fetch
     self.manifestUrl = manifestUrl
     self.remoteAlias = alias
     self.reviewUrl = review
+    self.revision = revision
     self.resolvedFetchUrl = self._resolveFetchUrl()
 
   def __eq__(self, other):
@@ -83,17 +87,14 @@ class _XmlRemote(object):
     # urljoin will gets confused over quite a few things.  The ones we care
     # about here are:
     # * no scheme in the base url, like <hostname:port>
-    # * persistent-https://
+    # We handle no scheme by replacing it with an obscure protocol, gopher
-    # We handle this by replacing these with obscure protocols
+    # and then replacing it with the original when we are done.
-    # and then replacing them with the original when we are done.
+
-    # gopher -> <none>
-    # wais -> persistent-https
     if manifestUrl.find(':') != manifestUrl.find('/') - 1:
-      manifestUrl = 'gopher://' + manifestUrl
+      url = urllib.parse.urljoin('gopher://' + manifestUrl, url)
-    manifestUrl = re.sub(r'^persistent-https://', 'wais://', manifestUrl)
-    url = urllib.parse.urljoin(manifestUrl, url)
       url = re.sub(r'^gopher://', '', url)
-    url = re.sub(r'^wais://', 'persistent-https://', url)
+    else:
+      url = urllib.parse.urljoin(manifestUrl, url)
     return url
 
   def ToRemoteSpec(self, projectName):
@@ -112,6 +113,7 @@ class XmlManifest(object):
     self.manifestFile = os.path.join(self.repodir, MANIFEST_FILE_NAME)
     self.globalConfig = GitConfig.ForUser()
     self.localManifestWarning = False
+    self.isGitcClient = False
 
     self.repoProject = MetaProject(self, 'repo',
       gitdir   = os.path.join(repodir, 'repo/.git'),
@@ -159,6 +161,11 @@ class XmlManifest(object):
       e.setAttribute('alias', r.remoteAlias)
     if r.reviewUrl is not None:
       e.setAttribute('review', r.reviewUrl)
+    if r.revision is not None:
+      e.setAttribute('revision', r.revision)
+
+  def _ParseGroups(self, groups):
+    return [x for x in re.split(r'[,\s]+', groups) if x]
 
   def Save(self, fd, peg_rev=False, peg_rev_upstream=True):
     """Write the current manifest out to the given file descriptor.
@@ -167,7 +174,7 @@ class XmlManifest(object):
 
     groups = mp.config.GetString('manifest.groups')
     if groups:
-      groups = [x for x in re.split(r'[,\s]+', groups) if x]
+      groups = self._ParseGroups(groups)
 
     doc = xml.dom.minidom.Document()
     root = doc.createElement('manifest')
@@ -197,6 +204,9 @@ class XmlManifest(object):
     if d.revisionExpr:
       have_default = True
       e.setAttribute('revision', d.revisionExpr)
+    if d.destBranchExpr:
+      have_default = True
+      e.setAttribute('dest-branch', d.destBranchExpr)
     if d.sync_j > 1:
       have_default = True
       e.setAttribute('sync-j', '%d' % d.sync_j)
@@ -240,20 +250,30 @@ class XmlManifest(object):
       if d.remote:
         remoteName = d.remote.remoteAlias or d.remote.name
       if not d.remote or p.remote.name != remoteName:
-        e.setAttribute('remote', p.remote.name)
+        remoteName = p.remote.name
+        e.setAttribute('remote', remoteName)
       if peg_rev:
         if self.IsMirror:
           value = p.bare_git.rev_parse(p.revisionExpr + '^0')
         else:
           value = p.work_git.rev_parse(HEAD + '^0')
         e.setAttribute('revision', value)
-        if peg_rev_upstream and value != p.revisionExpr:
+        if peg_rev_upstream:
+          if p.upstream:
+            e.setAttribute('upstream', p.upstream)
+          elif value != p.revisionExpr:
             # Only save the origin if the origin is not a sha1, and the default
-          # isn't our value, and the if the default doesn't already have that
+            # isn't our value
-          # covered.
             e.setAttribute('upstream', p.revisionExpr)
-      elif not d.revisionExpr or p.revisionExpr != d.revisionExpr:
+      else:
+        revision = self.remotes[remoteName].revision or d.revisionExpr
+        if not revision or revision != p.revisionExpr:
           e.setAttribute('revision', p.revisionExpr)
+        if p.upstream and p.upstream != p.revisionExpr:
+          e.setAttribute('upstream', p.upstream)
+
+      if p.dest_branch and p.dest_branch != d.destBranchExpr:
+        e.setAttribute('dest-branch', p.dest_branch)
 
       for c in p.copyfiles:
         ce = doc.createElement('copyfile')
@@ -285,6 +305,11 @@ class XmlManifest(object):
       if p.sync_s:
         e.setAttribute('sync-s', 'true')
 
+      if p.clone_depth:
+        e.setAttribute('clone-depth', str(p.clone_depth))
+
+      self._output_manifest_project_extras(p, e)
+
       if p.subprojects:
         subprojects = set(subp.name for subp in p.subprojects)
         output_projects(p, e, list(sorted(subprojects)))
@@ -302,6 +327,10 @@ class XmlManifest(object):
 
     doc.writexml(fd, '', '  ', '\n', 'UTF-8')
 
+  def _output_manifest_project_extras(self, p, e):
+    """Manifests can modify e if they support extra project attributes."""
+    pass
+
   @property
   def paths(self):
     self._Load()
@@ -310,7 +339,7 @@ class XmlManifest(object):
   @property
   def projects(self):
     self._Load()
-    return self._paths.values()
+    return list(self._paths.values())
 
   @property
   def remotes(self):
@@ -498,6 +527,23 @@ class XmlManifest(object):
       if node.nodeName == 'project':
         project = self._ParseProject(node)
         recursively_add_projects(project)
+      if node.nodeName == 'extend-project':
+        name = self._reqatt(node, 'name')
+
+        if name not in self._projects:
+          raise ManifestParseError('extend-project element specifies non-existent '
+                                   'project: %s' % name)
+
+        path = node.getAttribute('path')
+        groups = node.getAttribute('groups')
+        if groups:
+          groups = self._ParseGroups(groups)
+
+        for p in self._projects[name]:
+          if path and p.relpath != path:
+            continue
+          if groups:
+            p.groups.extend(groups)
       if node.nodeName == 'repo-hooks':
         # Get the name of the project and the (space-separated) list of enabled.
         repo_hooks_project = self._reqatt(node, 'in-project')
@@ -592,8 +638,11 @@ class XmlManifest(object):
     review = node.getAttribute('review')
     if review == '':
       review = None
+    revision = node.getAttribute('revision')
+    if revision == '':
+      revision = None
     manifestUrl = self.manifestProject.config.GetString('remote.origin.url')
-    return _XmlRemote(name, alias, fetch, manifestUrl, review)
+    return _XmlRemote(name, alias, fetch, manifestUrl, review, revision)
 
   def _ParseDefault(self, node):
     """
@@ -671,7 +720,7 @@ class XmlManifest(object):
   def _UnjoinName(self, parent_name, name):
     return os.path.relpath(name, parent_name)
 
-  def _ParseProject(self, node, parent = None):
+  def _ParseProject(self, node, parent = None, **extra_proj_attrs):
     """
     reads a <project> element from the manifest file
     """
@@ -686,7 +735,7 @@ class XmlManifest(object):
       raise ManifestParseError("no remote for project %s within %s" %
             (name, self.manifestFile))
 
-    revisionExpr = node.getAttribute('revision')
+    revisionExpr = node.getAttribute('revision') or remote.revision
     if not revisionExpr:
       revisionExpr = self._default.revisionExpr
     if not revisionExpr:
@@ -735,7 +784,7 @@ class XmlManifest(object):
     groups = ''
     if node.hasAttribute('groups'):
       groups = node.getAttribute('groups')
-    groups = [x for x in re.split(r'[,\s]+', groups) if x]
+    groups = self._ParseGroups(groups)
 
     if parent is None:
       relpath, worktree, gitdir, objdir = self.GetProjectPaths(name, path)
@@ -766,7 +815,8 @@ class XmlManifest(object):
                       clone_depth = clone_depth,
                       upstream = upstream,
                       parent = parent,
-                      dest_branch = dest_branch)
+                      dest_branch = dest_branch,
+                      **extra_proj_attrs)
 
     for n in node.childNodes:
       if n.nodeName == 'copyfile':
@@ -872,10 +922,8 @@ class XmlManifest(object):
     fromProjects = self.paths
     toProjects = manifest.paths
 
-    fromKeys = fromProjects.keys()
+    fromKeys = sorted(fromProjects.keys())
-    fromKeys.sort()
+    toKeys = sorted(toProjects.keys())
-    toKeys = toProjects.keys()
-    toKeys.sort()
 
     diff = {'added': [], 'removed': [], 'changed': [], 'unreachable': []}
 
@@ -899,3 +947,26 @@ class XmlManifest(object):
       diff['added'].append(toProjects[proj])
 
     return diff
+
+
+class GitcManifest(XmlManifest):
+
+  def __init__(self, repodir, gitc_client_name):
+    """Initialize the GitcManifest object."""
+    super(GitcManifest, self).__init__(repodir)
+    self.isGitcClient = True
+    self.gitc_client_name = gitc_client_name
+    self.gitc_client_dir = os.path.join(gitc_utils.get_gitc_manifest_dir(),
+                                        gitc_client_name)
+    self.manifestFile = os.path.join(self.gitc_client_dir, '.manifest')
+
+  def _ParseProject(self, node, parent = None):
+    """Override _ParseProject and add support for GITC specific attributes."""
+    return super(GitcManifest, self)._ParseProject(
+        node, parent=parent, old_revision=node.getAttribute('old-revision'))
+
+  def _output_manifest_project_extras(self, p, e):
+    """Output GITC Specific Project attributes"""
+    if p.old_revision:
+        e.setAttribute('old-revision', str(p.old_revision))
+

project.py

@@ -13,9 +13,9 @@
 # limitations under the License.
 
 from __future__ import print_function
-import traceback
 import errno
 import filecmp
+import glob
 import os
 import random
 import re
@@ -26,11 +26,12 @@ import sys
 import tarfile
 import tempfile
 import time
+import traceback
 
 from color import Coloring
 from git_command import GitCommand, git_require
-from git_config import GitConfig, IsId, GetSchemeFromUrl, ID_RE
+from git_config import GitConfig, IsId, GetSchemeFromUrl, GetUrlCookieFile, ID_RE
-from error import GitError, HookError, UploadError
+from error import GitError, HookError, UploadError, DownloadError
 from error import ManifestInvalidRevisionError
 from error import NoManifestException
 from trace import IsTrace, Trace
@@ -46,7 +47,7 @@ if not is_python3():
 def _lwrite(path, content):
   lock = '%s.lock' % path
 
-  fd = open(lock, 'wb')
+  fd = open(lock, 'w')
   try:
     fd.write(content)
   finally:
@@ -62,6 +63,10 @@ def _error(fmt, *args):
   msg = fmt % args
   print('error: %s' % msg, file=sys.stderr)
 
+def _warn(fmt, *args):
+  msg = fmt % args
+  print('warn: %s' % msg, file=sys.stderr)
+
 def not_rev(r):
   return '^' + r
 
@@ -197,13 +202,13 @@ class DiffColoring(Coloring):
     Coloring.__init__(self, config, 'diff')
     self.project = self.printer('header', attr='bold')
 
-class _Annotation:
+class _Annotation(object):
   def __init__(self, name, value, keep):
     self.name = name
     self.value = value
     self.keep = keep
 
-class _CopyFile:
+class _CopyFile(object):
   def __init__(self, src, dest, abssrc, absdest):
     self.src = src
     self.dest = dest
@@ -231,38 +236,72 @@ class _CopyFile:
       except IOError:
         _error('Cannot copy file %s to %s', src, dest)
 
-class _LinkFile:
+class _LinkFile(object):
-  def __init__(self, src, dest, abssrc, absdest):
+  def __init__(self, git_worktree, src, dest, relsrc, absdest):
+    self.git_worktree = git_worktree
     self.src = src
     self.dest = dest
-    self.abs_src = abssrc
+    self.src_rel_to_dest = relsrc
     self.abs_dest = absdest
 
-  def _Link(self):
+  def __linkIt(self, relSrc, absDest):
-    src = self.abs_src
-    dest = self.abs_dest
     # link file if it does not exist or is out of date
-    if not os.path.islink(dest) or os.readlink(dest) != src:
+    if not os.path.islink(absDest) or (os.readlink(absDest) != relSrc):
       try:
         # remove existing file first, since it might be read-only
-        if os.path.exists(dest):
+        if os.path.exists(absDest):
-          os.remove(dest)
+          os.remove(absDest)
         else:
-          dest_dir = os.path.dirname(dest)
+          dest_dir = os.path.dirname(absDest)
           if not os.path.isdir(dest_dir):
             os.makedirs(dest_dir)
-        os.symlink(src, dest)
+        os.symlink(relSrc, absDest)
       except IOError:
-        _error('Cannot link file %s to %s', src, dest)
+        _error('Cannot link file %s to %s', relSrc, absDest)
+
+  def _Link(self):
+    """Link the self.rel_src_to_dest and self.abs_dest. Handles wild cards
+    on the src linking all of the files in the source in to the destination
+    directory.
+    """
+    # We use the absSrc to handle the situation where the current directory
+    # is not the root of the repo
+    absSrc = os.path.join(self.git_worktree, self.src)
+    if os.path.exists(absSrc):
+      # Entity exists so just a simple one to one link operation
+      self.__linkIt(self.src_rel_to_dest, self.abs_dest)
+    else:
+      # Entity doesn't exist assume there is a wild card
+      absDestDir = self.abs_dest
+      if os.path.exists(absDestDir) and not os.path.isdir(absDestDir):
+        _error('Link error: src with wildcard, %s must be a directory',
+            absDestDir)
+      else:
+        absSrcFiles = glob.glob(absSrc)
+        for absSrcFile in absSrcFiles:
+          # Create a releative path from source dir to destination dir
+          absSrcDir = os.path.dirname(absSrcFile)
+          relSrcDir = os.path.relpath(absSrcDir, absDestDir)
+
+          # Get the source file name
+          srcFile = os.path.basename(absSrcFile)
+
+          # Now form the final full paths to srcFile. They will be
+          # absolute for the desintaiton and relative for the srouce.
+          absDest = os.path.join(absDestDir, srcFile)
+          relSrc = os.path.join(relSrcDir, srcFile)
+          self.__linkIt(relSrc, absDest)
 
 class RemoteSpec(object):
   def __init__(self,
                name,
                url=None,
-               review = None):
+               review=None,
+               revision=None):
     self.name = name
     self.url = url
     self.review = review
+    self.revision = revision
 
 class RepoHook(object):
   """A RepoHook contains information about a script to run as a hook.
@@ -438,7 +477,8 @@ class RepoHook(object):
       # and  convert to a HookError w/ just the failing traceback.
       context = {}
       try:
-        execfile(self._script_fullpath, context)
+        exec(compile(open(self._script_fullpath).read(),
+                     self._script_fullpath, 'exec'), context)
       except Exception:
         raise HookError('%s\nFailed to import %s hook; see traceback above.' % (
                         traceback.format_exc(), self._hook_type))
@@ -507,6 +547,12 @@ class RepoHook(object):
 
 
 class Project(object):
+  # These objects can be shared between several working trees.
+  shareable_files = ['description', 'info']
+  shareable_dirs = ['hooks', 'objects', 'rr-cache', 'svn']
+  # These objects can only be used by a single working tree.
+  working_tree_files = ['config', 'packed-refs', 'shallow']
+  working_tree_dirs = ['logs', 'refs']
   def __init__(self,
                manifest,
                name,
@@ -525,7 +571,9 @@ class Project(object):
                upstream=None,
                parent=None,
                is_derived=False,
-               dest_branch = None):
+               dest_branch=None,
+               optimized_fetch=False,
+               old_revision=None):
     """Init a Project object.
 
     Args:
@@ -547,6 +595,9 @@ class Project(object):
       is_derived: False if the project was explicitly defined in the manifest;
                   True if the project is a discovered submodule.
       dest_branch: The branch to which to push changes for review by default.
+      optimized_fetch: If True, when a project is set to a sha1 revision, only
+                       fetch from the remote if the sha1 is not present locally.
+      old_revision: saved git commit id for open GITC projects.
     """
     self.manifest = manifest
     self.name = name
@@ -575,6 +626,7 @@ class Project(object):
     self.upstream = upstream
     self.parent = parent
     self.is_derived = is_derived
+    self.optimized_fetch = optimized_fetch
     self.subprojects = []
 
     self.snapshots = {}
@@ -593,6 +645,7 @@ class Project(object):
     self.bare_ref = GitRefs(gitdir)
     self.bare_objdir = self._GitGetByExec(self, bare=True, gitdir=objdir)
     self.dest_branch = dest_branch
+    self.old_revision = old_revision
 
     # This will be filled in if a project is later identified to be the
     # project containing repo hooks.
@@ -604,7 +657,7 @@ class Project(object):
 
   @property
   def Exists(self):
-    return os.path.isdir(self.gitdir)
+    return os.path.isdir(self.gitdir) and os.path.isdir(self.objdir)
 
   @property
   def CurrentBranch(self):
@@ -733,26 +786,48 @@ class Project(object):
     return matched
 
 ## Status Display ##
+  def UncommitedFiles(self, get_all=True):
+    """Returns a list of strings, uncommitted files in the git tree.
 
-  def HasChanges(self):
+    Args:
-    """Returns true if there are uncommitted changes.
+      get_all: a boolean, if True - get information about all different
+               uncommitted files. If False - return as soon as any kind of
+               uncommitted files is detected.
     """
+    details = []
     self.work_git.update_index('-q',
                                '--unmerged',
                                '--ignore-missing',
                                '--refresh')
     if self.IsRebaseInProgress():
-      return True
+      details.append("rebase in progress")
+      if not get_all:
+        return details
 
-    if self.work_git.DiffZ('diff-index', '--cached', HEAD):
+    changes = self.work_git.DiffZ('diff-index', '--cached', HEAD).keys()
-      return True
+    if changes:
+      details.extend(changes)
+      if not get_all:
+        return details
 
-    if self.work_git.DiffZ('diff-files'):
+    changes = self.work_git.DiffZ('diff-files').keys()
-      return True
+    if changes:
+      details.extend(changes)
+      if not get_all:
+        return details
 
-    if self.work_git.LsOthers():
+    changes = self.work_git.LsOthers()
-      return True
+    if changes:
+      details.extend(changes)
 
+    return details
+
+  def HasChanges(self):
+    """Returns true if there are uncommitted changes.
+    """
+    if self.UncommitedFiles(get_all=False):
+      return True
+    else:
       return False
 
   def PrintWorkTreeStatus(self, output_redir=None):
@@ -1024,24 +1099,24 @@ class Project(object):
         tar.extractall(path=path)
         return True
     except (IOError, tarfile.TarError) as e:
-      print("error: Cannot extract archive %s: "
+      _error("Cannot extract archive %s: %s", tarpath, str(e))
-            "%s" % (tarpath, str(e)), file=sys.stderr)
     return False
 
   def Sync_NetworkHalf(self,
       quiet=False,
       is_new=None,
       current_branch_only=False,
+      force_sync=False,
       clone_bundle=True,
       no_tags=False,
-      archive=False):
+      archive=False,
+      optimized_fetch=False):
     """Perform only the network IO portion of the sync process.
        Local working directory/branch state is not affected.
     """
     if archive and not isinstance(self, MetaProject):
       if self.remote.url.startswith(('http://', 'https://')):
-        print("error: %s: Cannot fetch archives from http/https "
+        _error("%s: Cannot fetch archives from http/https remotes.", self.name)
-              "remotes." % self.name, file=sys.stderr)
         return False
 
       name = self.relpath.replace('\\', '/')
@@ -1052,7 +1127,7 @@ class Project(object):
       try:
         self._FetchArchive(tarpath, cwd=topdir)
       except GitError as e:
-        print('error: %s' % str(e), file=sys.stderr)
+        _error('%s', e)
         return False
 
       # From now on, we only need absolute tarpath
@@ -1063,15 +1138,13 @@ class Project(object):
       try:
         os.remove(tarpath)
       except OSError as e:
-        print("warn: Cannot remove archive %s: "
+        _warn("Cannot remove archive %s: %s", tarpath, str(e))
-              "%s" % (tarpath, str(e)), file=sys.stderr)
       self._CopyAndLinkFiles()
       return True
-
     if is_new is None:
       is_new = not self.Exists
     if is_new:
-      self._InitGitDir()
+      self._InitGitDir(force_sync=force_sync)
     else:
       self._UpdateHooks()
     self._InitRemote()
@@ -1103,16 +1176,12 @@ class Project(object):
       elif self.manifest.default.sync_c:
         current_branch_only = True
 
-    is_sha1 = False
+    need_to_fetch = not (optimized_fetch and \
-    if ID_RE.match(self.revisionExpr) is not None:
+      (ID_RE.match(self.revisionExpr) and self._CheckForSha1()))
-      is_sha1 = True
+    if (need_to_fetch
-    if is_sha1 and self._CheckForSha1():
+        and not self._RemoteFetch(initial=is_new, quiet=quiet, alt_dir=alt_dir,
-      # Don't need to fetch since we already have this revision
-      return True
-
-    if not self._RemoteFetch(initial=is_new, quiet=quiet, alt_dir=alt_dir,
                                   current_branch_only=current_branch_only,
-                             no_tags=no_tags):
+                                  no_tags=no_tags)):
       return False
 
     if self.worktree:
@@ -1129,6 +1198,8 @@ class Project(object):
     self._InitHooks()
 
   def _CopyAndLinkFiles(self):
+    if self.manifest.isGitcClient:
+      return
     for copyfile in self.copyfiles:
       copyfile._Copy()
     for linkfile in self.linkfiles:
@@ -1168,11 +1239,11 @@ class Project(object):
         'revision %s in %s not found' % (self.revisionExpr,
                                          self.name))
 
-  def Sync_LocalHalf(self, syncbuf):
+  def Sync_LocalHalf(self, syncbuf, force_sync=False):
     """Perform only the local IO portion of the sync process.
        Network access is not required.
     """
-    self._InitWorkTree()
+    self._InitWorkTree(force_sync=force_sync)
     all_refs = self.bare_ref.all
     self.CleanPublishedCache(all_refs)
     revid = self.GetRevisionId(all_refs)
@@ -1204,6 +1275,8 @@ class Project(object):
         # Except if the head needs to be detached
         #
         if not syncbuf.detach_head:
+          # The copy/linkfile config may have changed.
+          self._CopyAndLinkFiles()
           return
       else:
         lost = self._revlist(not_rev(revid), HEAD)
@@ -1221,6 +1294,8 @@ class Project(object):
     if head == revid:
       # No changes; don't do anything further.
       #
+      # The copy/linkfile config may have changed.
+      self._CopyAndLinkFiles()
       return
 
     branch = self.GetBranch(branch)
@@ -1305,6 +1380,8 @@ class Project(object):
     if not ID_RE.match(self.revisionExpr):
       # in case of manifest sync the revisionExpr might be a SHA1
       branch.merge = self.revisionExpr
+      if not branch.merge.startswith('refs/'):
+        branch.merge = R_HEADS + branch.merge
     branch.Save()
 
     if cnt_mine > 0 and self.rebase:
@@ -1330,9 +1407,10 @@ class Project(object):
 
   def AddLinkFile(self, src, dest, absdest):
     # dest should already be an absolute path, but src is project relative
-    # make src an absolute path
+    # make src relative path to dest
-    abssrc = os.path.join(self.worktree, src)
+    absdestdir = os.path.dirname(absdest)
-    self.linkfiles.append(_LinkFile(src, dest, abssrc, absdest))
+    relsrc = os.path.relpath(os.path.join(self.worktree, src), absdestdir)
+    self.linkfiles.append(_LinkFile(self.worktree, src, dest, relsrc, absdest))
 
   def AddAnnotation(self, name, value, keep):
     self.annotations.append(_Annotation(name, value, keep))
@@ -1356,15 +1434,17 @@ class Project(object):
 
 ## Branch Management ##
 
-  def StartBranch(self, name):
+  def StartBranch(self, name, branch_merge=''):
     """Create a new branch off the manifest's revision.
     """
+    if not branch_merge:
+      branch_merge = self.revisionExpr
     head = self.work_git.GetHead()
     if head == (R_HEADS + name):
       return True
 
     all_refs = self.bare_ref.all
-    if (R_HEADS + name) in all_refs:
+    if R_HEADS + name in all_refs:
       return GitCommand(self,
                         ['checkout', name, '--'],
                         capture_stdout=True,
@@ -1372,7 +1452,9 @@ class Project(object):
 
     branch = self.GetBranch(name)
     branch.remote = self.GetRemote(self.remote.name)
-    branch.merge = self.revisionExpr
+    branch.merge = branch_merge
+    if not branch.merge.startswith('refs/') and not ID_RE.match(branch_merge):
+      branch.merge = R_HEADS + branch_merge
     revid = self.GetRevisionId(all_refs)
 
     if head.startswith(R_HEADS):
@@ -1380,7 +1462,6 @@ class Project(object):
         head = all_refs[head]
       except KeyError:
         head = None
-
     if revid and head and revid == head:
       ref = os.path.join(self.gitdir, R_HEADS + name)
       try:
@@ -1528,7 +1609,7 @@ class Project(object):
 
     kept = []
     for branch in kill:
-      if (R_HEADS + branch) in left:
+      if R_HEADS + branch in left:
         branch = self.GetBranch(branch)
         base = branch.LocalMerge
         if not base:
@@ -1662,7 +1743,8 @@ class Project(object):
 
       remote = RemoteSpec(self.remote.name,
                           url=url,
-                          review = self.remote.review)
+                          review=self.remote.review,
+                          revision=self.remote.revision)
       subproject = Project(manifest=self.manifest,
                            name=name,
                            remote=remote,
@@ -1707,6 +1789,7 @@ class Project(object):
     if command.Wait() != 0:
       raise GitError('git archive %s: %s' % (self.name, command.stderr))
 
+
   def _RemoteFetch(self, name=None,
                    current_branch_only=False,
                    initial=False,
@@ -1726,22 +1809,33 @@ class Project(object):
         depth = self.clone_depth
       else:
         depth = self.manifest.manifestProject.config.GetString('repo.depth')
+      # The repo project should never be synced with partial depth
+      if self.relpath == '.repo/repo':
+        depth = None
 
     if depth:
       current_branch_only = True
 
-    if current_branch_only:
     if ID_RE.match(self.revisionExpr) is not None:
       is_sha1 = True
-      elif self.revisionExpr.startswith(R_TAGS):
+
+    if current_branch_only:
+      if self.revisionExpr.startswith(R_TAGS):
         # this is a tag and its sha1 value should never change
         tag_name = self.revisionExpr[len(R_TAGS):]
 
       if is_sha1 or tag_name is not None:
         if self._CheckForSha1():
           return True
-      if is_sha1 and (not self.upstream or ID_RE.match(self.upstream)):
+      if is_sha1 and not depth:
-        current_branch_only = False
+        # When syncing a specific commit and --depth is not set:
+        # * if upstream is explicitly specified and is not a sha1, fetch only
+        #   upstream as users expect only upstream to be fetch.
+        #   Note: The commit might not be in upstream in which case the sync
+        #   will fail.
+        # * otherwise, fetch all branches to make sure we end up with the
+        #   specific commit.
+        current_branch_only = self.upstream and not ID_RE.match(self.upstream)
 
     if not name:
       name = self.remote.name
@@ -1791,10 +1885,15 @@ class Project(object):
 
     cmd = ['fetch']
 
-    # The --depth option only affects the initial fetch; after that we'll do
+    if depth:
-    # full fetches of changes.
-    if depth and initial:
       cmd.append('--depth=%s' % depth)
+    else:
+      # If this repo has shallow objects, then we don't know which refs have
+      # shallow objects or not. Tell git to unshallow all fetched refs.  Don't
+      # do this with projects that don't have shallow objects, since it is less
+      # efficient.
+      if os.path.exists(os.path.join(self.gitdir, 'shallow')):
+        cmd.append('--depth=2147483647')
 
     if quiet:
       cmd.append('--quiet')
@@ -1809,39 +1908,57 @@ class Project(object):
     else:
       cmd.append('--tags')
 
+    spec = []
     if not current_branch_only:
       # Fetch whole repo
-      cmd.append(str((u'+refs/heads/*:') + remote.ToLocal('refs/heads/*')))
+      spec.append(str((u'+refs/heads/*:') + remote.ToLocal('refs/heads/*')))
     elif tag_name is not None:
-      cmd.append('tag')
+      spec.append('tag')
-      cmd.append(tag_name)
+      spec.append(tag_name)
-    else:
+
+    if not self.manifest.IsMirror:
       branch = self.revisionExpr
+      if is_sha1 and depth and git_require((1, 8, 3)):
+        # Shallow checkout of a specific commit, fetch from that commit and not
+        # the heads only as the commit might be deeper in the history.
+        spec.append(branch)
+      else:
         if is_sha1:
           branch = self.upstream
-      if branch.startswith(R_HEADS):
+        if branch is not None and branch.strip():
-        branch = branch[len(R_HEADS):]
+          if not branch.startswith('refs/'):
-      cmd.append(str((u'+refs/heads/%s:' % branch) + remote.ToLocal('refs/heads/%s' % branch)))
+            branch = R_HEADS + branch
+          spec.append(str((u'+%s:' % branch) + remote.ToLocal(branch)))
+    cmd.extend(spec)
 
     ok = False
     for _i in range(2):
-      ret = GitCommand(self, cmd, bare=True, ssh_proxy=ssh_proxy).Wait()
+      gitcmd = GitCommand(self, cmd, bare=True, ssh_proxy=ssh_proxy)
+      ret = gitcmd.Wait()
       if ret == 0:
         ok = True
         break
+      # If needed, run the 'git remote prune' the first time through the loop
+      elif (not _i and
+            "error:" in gitcmd.stderr and
+            "git remote prune" in gitcmd.stderr):
+        prunecmd = GitCommand(self, ['remote', 'prune', name], bare=True,
+                              ssh_proxy=ssh_proxy)
+        ret = prunecmd.Wait()
+        if ret:
+          break
+        continue
       elif current_branch_only and is_sha1 and ret == 128:
         # Exit code 128 means "couldn't find the ref you asked for"; if we're in sha1
         # mode, we just tried sync'ing from the upstream field; it doesn't exist, thus
         # abort the optimization attempt and do a full sync.
         break
+      elif ret < 0:
+        # Git died with a signal, exit immediately
+        break
       time.sleep(random.randint(30, 45))
 
     if initial:
-      # Ensure that some refs exist.  Otherwise, we probably aren't looking
-      # at a real git repository and may have a bad url.
-      if not self.bare_ref.all:
-        ok = False
-
       if alt_dir:
         if old_packed != '':
           _lwrite(packed_refs, old_packed)
@@ -1854,8 +1971,15 @@ class Project(object):
       # got what we wanted, else trigger a second run of all
       # refs.
       if not self._CheckForSha1():
+        if not depth:
+          # Avoid infinite recursion when depth is True (since depth implies
+          # current_branch_only)
           return self._RemoteFetch(name=name, current_branch_only=False,
                                    initial=False, quiet=quiet, alt_dir=alt_dir)
+        if self.clone_depth:
+          self.clone_depth = None
+          return self._RemoteFetch(name=name, current_branch_only=current_branch_only,
+                                   initial=False, quiet=quiet, alt_dir=alt_dir)
 
     return ok
 
@@ -1916,9 +2040,9 @@ class Project(object):
         os.remove(tmpPath)
     if 'http_proxy' in os.environ and 'darwin' == sys.platform:
       cmd += ['--proxy', os.environ['http_proxy']]
-    cookiefile = self._GetBundleCookieFile(srcUrl)
+    with GetUrlCookieFile(srcUrl, quiet) as (cookiefile, proxy):
       if cookiefile:
-      cmd += ['--cookie', cookiefile]
+        cmd += ['--cookie', cookiefile, '--cookie-jar', cookiefile]
       if srcUrl.startswith('persistent-'):
         srcUrl = srcUrl[len('persistent-'):]
       cmd += [srcUrl]
@@ -1943,7 +2067,7 @@ class Project(object):
         return False
 
     if os.path.exists(tmpPath):
-      if curlret == 0 and self._IsValidBundle(tmpPath):
+      if curlret == 0 and self._IsValidBundle(tmpPath, quiet):
         os.rename(tmpPath, dstPath)
         return True
       else:
@@ -1952,46 +2076,18 @@ class Project(object):
     else:
       return False
 
-  def _IsValidBundle(self, path):
+  def _IsValidBundle(self, path, quiet):
     try:
       with open(path) as f:
         if f.read(16) == '# v2 git bundle\n':
           return True
         else:
+          if not quiet:
             print("Invalid clone.bundle file; ignoring.", file=sys.stderr)
           return False
     except OSError:
       return False
 
-  def _GetBundleCookieFile(self, url):
-    if url.startswith('persistent-'):
-      try:
-        p = subprocess.Popen(
-            ['git-remote-persistent-https', '-print_config', url],
-            stdin=subprocess.PIPE, stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE)
-        p.stdin.close()  # Tell subprocess it's ok to close.
-        prefix = 'http.cookiefile='
-        cookiefile = None
-        for line in p.stdout:
-          line = line.strip()
-          if line.startswith(prefix):
-            cookiefile = line[len(prefix):]
-            break
-        if p.wait():
-          err_msg = p.stderr.read()
-          if ' -print_config' in err_msg:
-            pass  # Persistent proxy doesn't support -print_config.
-          else:
-            print(err_msg, file=sys.stderr)
-        if cookiefile:
-          return cookiefile
-      except OSError as e:
-        if e.errno == errno.ENOENT:
-          pass  # No persistent proxy.
-        raise
-    return GitConfig.ForUser().GetString('http.cookiefile')
-
   def _Checkout(self, rev, quiet=False):
     cmd = ['checkout']
     if quiet:
@@ -2002,7 +2098,7 @@ class Project(object):
       if self._allrefs:
         raise GitError('%s checkout %s ' % (self.name, rev))
 
-  def _CherryPick(self, rev, quiet=False):
+  def _CherryPick(self, rev):
     cmd = ['cherry-pick']
     cmd.append(rev)
     cmd.append('--')
@@ -2010,7 +2106,7 @@ class Project(object):
       if self._allrefs:
         raise GitError('%s cherry-pick %s ' % (self.name, rev))
 
-  def _Revert(self, rev, quiet=False):
+  def _Revert(self, rev):
     cmd = ['revert']
     cmd.append('--no-edit')
     cmd.append(rev)
@@ -2042,20 +2138,39 @@ class Project(object):
     if GitCommand(self, cmd).Wait() != 0:
       raise GitError('%s merge %s ' % (self.name, head))
 
-  def _InitGitDir(self, mirror_git=None):
+  def _InitGitDir(self, mirror_git=None, force_sync=False):
-    if not os.path.exists(self.gitdir):
+    init_git_dir = not os.path.exists(self.gitdir)
-
+    init_obj_dir = not os.path.exists(self.objdir)
+    try:
       # Initialize the bare repository, which contains all of the objects.
-      if not os.path.exists(self.objdir):
+      if init_obj_dir:
         os.makedirs(self.objdir)
         self.bare_objdir.init()
 
       # If we have a separate directory to hold refs, initialize it as well.
       if self.objdir != self.gitdir:
+        if init_git_dir:
           os.makedirs(self.gitdir)
+
+        if init_obj_dir or init_git_dir:
           self._ReferenceGitDir(self.objdir, self.gitdir, share_refs=False,
                                 copy_all=True)
+        try:
+          self._CheckDirReference(self.objdir, self.gitdir, share_refs=False)
+        except GitError as e:
+          if force_sync:
+            print("Retrying clone after deleting %s" % self.gitdir, file=sys.stderr)
+            try:
+              shutil.rmtree(os.path.realpath(self.gitdir))
+              if self.worktree and os.path.exists(
+                  os.path.realpath(self.worktree)):
+                shutil.rmtree(os.path.realpath(self.worktree))
+              return self._InitGitDir(mirror_git=mirror_git, force_sync=False)
+            except:
+              raise e
+          raise e
 
+      if init_git_dir:
         mp = self.manifest.manifestProject
         ref_dir = mp.config.GetString('repo.reference') or ''
 
@@ -2088,18 +2203,15 @@ class Project(object):
           self.config.SetString('core.bare', 'true')
         else:
           self.config.SetString('core.bare', None)
+    except Exception:
+      if init_obj_dir and os.path.exists(self.objdir):
+        shutil.rmtree(self.objdir)
+      if init_git_dir and os.path.exists(self.gitdir):
+        shutil.rmtree(self.gitdir)
+      raise
 
   def _UpdateHooks(self):
     if os.path.exists(self.gitdir):
-      # Always recreate hooks since they can have been changed
-      # since the latest update.
-      hooks = self._gitdir_path('hooks')
-      try:
-        to_rm = os.listdir(hooks)
-      except OSError:
-        to_rm = []
-      for old_hook in to_rm:
-        os.remove(os.path.join(hooks, old_hook))
       self._InitHooks()
 
   def _InitHooks(self):
@@ -2125,7 +2237,7 @@ class Project(object):
         if filecmp.cmp(stock_hook, dst, shallow=False):
           os.remove(dst)
         else:
-          _error("%s: Not replacing %s hook", self.relpath, name)
+          _warn("%s: Not replacing locally modified %s hook", self.relpath, name)
           continue
       try:
         os.symlink(os.path.relpath(stock_hook, os.path.dirname(dst)), dst)
@@ -2170,6 +2282,25 @@ class Project(object):
         msg = 'manifest set to %s' % self.revisionExpr
         self.bare_git.symbolic_ref('-m', msg, ref, dst)
 
+  def _CheckDirReference(self, srcdir, destdir, share_refs):
+    symlink_files = self.shareable_files
+    symlink_dirs = self.shareable_dirs
+    if share_refs:
+      symlink_files += self.working_tree_files
+      symlink_dirs += self.working_tree_dirs
+    to_symlink = symlink_files + symlink_dirs
+    for name in set(to_symlink):
+      dst = os.path.realpath(os.path.join(destdir, name))
+      if os.path.lexists(dst):
+        src = os.path.realpath(os.path.join(srcdir, name))
+        # Fail if the links are pointing to the wrong place
+        if src != dst:
+          raise GitError('--force-sync not enabled; cannot overwrite a local '
+                         'work tree. If you\'re comfortable with the '
+                         'possibility of losing the work tree\'s git metadata,'
+                         ' use `repo sync --force-sync {0}` to '
+                         'proceed.'.format(self.relpath))
+
   def _ReferenceGitDir(self, gitdir, dotgit, share_refs, copy_all):
     """Update |dotgit| to reference |gitdir|, using symlinks where possible.
 
@@ -2181,31 +2312,38 @@ class Project(object):
       copy_all: If true, copy all remaining files from |gitdir| -> |dotgit|.
           This saves you the effort of initializing |dotgit| yourself.
     """
-    # These objects can be shared between several working trees.
+    symlink_files = self.shareable_files
-    symlink_files = ['description', 'info']
+    symlink_dirs = self.shareable_dirs
-    symlink_dirs = ['hooks', 'objects', 'rr-cache', 'svn']
     if share_refs:
-      # These objects can only be used by a single working tree.
+      symlink_files += self.working_tree_files
-      symlink_files += ['config', 'packed-refs', 'shallow']
+      symlink_dirs += self.working_tree_dirs
-      symlink_dirs += ['logs', 'refs']
     to_symlink = symlink_files + symlink_dirs
 
     to_copy = []
     if copy_all:
       to_copy = os.listdir(gitdir)
 
+    dotgit = os.path.realpath(dotgit)
     for name in set(to_copy).union(to_symlink):
       try:
         src = os.path.realpath(os.path.join(gitdir, name))
-        dst = os.path.realpath(os.path.join(dotgit, name))
+        dst = os.path.join(dotgit, name)
 
-        if os.path.lexists(dst) and not os.path.islink(dst):
+        if os.path.lexists(dst):
-          raise GitError('cannot overwrite a local work tree')
+          continue
 
         # If the source dir doesn't exist, create an empty dir.
         if name in symlink_dirs and not os.path.lexists(src):
           os.makedirs(src)
 
+        # If the source file doesn't exist, ensure the destination
+        # file doesn't either.
+        if name in symlink_files and not os.path.lexists(src):
+          try:
+            os.remove(dst)
+          except OSError:
+            pass
+
         if name in to_symlink:
           os.symlink(os.path.relpath(src, os.path.dirname(dst)), dst)
         elif copy_all and not os.path.islink(dst):
@@ -2215,17 +2353,31 @@ class Project(object):
             shutil.copy(src, dst)
       except OSError as e:
         if e.errno == errno.EPERM:
-          raise GitError('filesystem must support symlinks')
+          raise DownloadError('filesystem must support symlinks')
         else:
           raise
 
-  def _InitWorkTree(self):
+  def _InitWorkTree(self, force_sync=False):
     dotgit = os.path.join(self.worktree, '.git')
-    if not os.path.exists(dotgit):
+    init_dotgit = not os.path.exists(dotgit)
+    try:
+      if init_dotgit:
         os.makedirs(dotgit)
         self._ReferenceGitDir(self.gitdir, dotgit, share_refs=True,
                               copy_all=False)
 
+      try:
+        self._CheckDirReference(self.gitdir, dotgit, share_refs=True)
+      except GitError as e:
+        if force_sync:
+          try:
+            shutil.rmtree(dotgit)
+            return self._InitWorkTree(force_sync=False)
+          except:
+            raise e
+        raise e
+
+      if init_dotgit:
         _lwrite(os.path.join(dotgit, HEAD), '%s\n' % self.GetRevisionId())
 
         cmd = ['read-tree', '--reset', '-u']
@@ -2235,6 +2387,10 @@ class Project(object):
           raise GitError("cannot initialize work tree")
 
         self._CopyAndLinkFiles()
+    except Exception:
+      if init_dotgit:
+        shutil.rmtree(dotgit)
+      raise
 
   def _gitdir_path(self, path):
     return os.path.realpath(os.path.join(self.gitdir, path))
@@ -2326,8 +2482,8 @@ class Project(object):
           out = iter(out[:-1].split('\0'))  # pylint: disable=W1401
           while out:
             try:
-              info = out.next()
+              info = next(out)
-              path = out.next()
+              path = next(out)
             except StopIteration:
               break
 
@@ -2353,7 +2509,7 @@ class Project(object):
             info = _Info(path, *info)
             if info.status in ('R', 'C'):
               info.src_path = info.path
-              info.path = out.next()
+              info.path = next(out)
             r[info.path] = info
         return r
       finally:
@@ -2629,7 +2785,7 @@ class MetaProject(Project):
           self.revisionExpr = base
           self.revisionId = None
 
-  def MetaBranchSwitch(self, target):
+  def MetaBranchSwitch(self):
     """ Prepare MetaProject for manifest branch switch
     """
 

repo

@@ -108,6 +108,7 @@ S_repo = 'repo'                 # special repo repository
 S_manifests = 'manifests'       # special manifest repository
 REPO_MAIN = S_repo + '/main.py' # main script
 MIN_PYTHON_VERSION = (2, 6)     # minimum supported python version
+GITC_CONFIG_FILE = '/gitc/.config'
 
 
 import errno
@@ -139,10 +140,6 @@ def _print(*objects, **kwargs):
 
 # Python version check
 ver = sys.version_info
-if ver[0] == 3:
-  _print('warning: Python 3 support is currently experimental. YMMV.\n'
-         'Please use Python 2.6 - 2.7 instead.',
-         file=sys.stderr)
 if (ver[0], ver[1]) < MIN_PYTHON_VERSION:
   _print('error: Python version %s unsupported.\n'
          'Please use Python 2.6 - 2.7 instead.'
@@ -216,14 +213,41 @@ group.add_option('--config-name',
                  dest='config_name', action="store_true", default=False,
                  help='Always prompt for name/e-mail')
 
+def _GitcInitOptions(init_optparse):
+  init_optparse.set_usage("repo gitc-init -u url -c client [options]")
+  g = init_optparse.add_option_group('GITC options')
+  g.add_option('-f', '--manifest-file',
+               dest='manifest_file',
+               help='Optional manifest file to use for this GITC client.')
+  g.add_option('-c', '--gitc-client',
+               dest='gitc_client',
+               help='The name for the new gitc_client instance.')
+
+_gitc_manifest_dir = None
+def get_gitc_manifest_dir():
+  global _gitc_manifest_dir
+  if _gitc_manifest_dir is None:
+    _gitc_manifest_dir = ''
+    try:
+      with open(GITC_CONFIG_FILE, 'r') as gitc_config:
+        for line in gitc_config:
+          match = re.match('gitc_dir=(?P<gitc_manifest_dir>.*)', line)
+          if match:
+            _gitc_manifest_dir = match.group('gitc_manifest_dir')
+    except IOError:
+      pass
+  return _gitc_manifest_dir
+
 class CloneFailure(Exception):
   """Indicate the remote clone of repo itself failed.
   """
 
 
-def _Init(args):
+def _Init(args, gitc_init=False):
   """Installs repo by cloning it over the network.
   """
+  if gitc_init:
+    _GitcInitOptions(init_optparse)
   opt, args = init_optparse.parse_args(args)
   if args:
     init_optparse.print_usage()
@@ -246,6 +270,23 @@ def _Init(args):
     raise CloneFailure()
 
   try:
+    if gitc_init:
+      gitc_manifest_dir = get_gitc_manifest_dir()
+      if not gitc_manifest_dir:
+        _print('fatal: GITC filesystem is not available. Exiting...',
+               file=sys.stderr)
+        sys.exit(1)
+      if not opt.gitc_client:
+        _print('fatal: GITC client (-c) is required.', file=sys.stderr)
+        sys.exit(1)
+      client_dir = os.path.join(gitc_manifest_dir, opt.gitc_client)
+      if not os.path.exists(client_dir):
+        os.makedirs(client_dir)
+      os.chdir(client_dir)
+      if os.path.exists(repodir):
+        # This GITC Client has already initialized repo so continue.
+        return
+
     os.mkdir(repodir)
   except OSError as e:
     if e.errno != errno.EEXIST:
@@ -466,7 +507,7 @@ def _DownloadBundle(url, local, quiet):
     try:
       r = urllib.request.urlopen(url)
     except urllib.error.HTTPError as e:
-      if e.code in [403, 404]:
+      if e.code in [401, 403, 404]:
         return False
       _print('fatal: Cannot get %s' % url, file=sys.stderr)
       _print('fatal: HTTP error %s' % e.code, file=sys.stderr)
@@ -644,6 +685,10 @@ def _ParseArguments(args):
 
 
 def _Usage():
+  gitc_usage = ""
+  if get_gitc_manifest_dir():
+    gitc_usage = "  gitc-init Initialize a GITC Client.\n"
+
   _print(
 """usage: repo COMMAND [ARGS]
 
@@ -652,7 +697,8 @@ repo is not yet installed.  Use "repo init" to install it here.
 The most commonly used repo commands are:
 
   init      Install repo in the current working directory
-  help      Display detailed help on a command
+""" + gitc_usage +
+"""  help      Display detailed help on a command
 
 For access to the full online help, install repo ("repo init").
 """, file=sys.stderr)
@@ -664,6 +710,10 @@ def _Help(args):
     if args[0] == 'init':
       init_optparse.print_help()
       sys.exit(0)
+    elif args[0] == 'gitc-init':
+      _GitcInitOptions(init_optparse)
+      init_optparse.print_help()
+      sys.exit(0)
     else:
       _print("error: '%s' is not a bootstrap command.\n"
              '        For access to online help, install repo ("repo init").'
@@ -729,6 +779,12 @@ def main(orig_args):
   wrapper_path = os.path.abspath(__file__)
   my_main, my_git = _RunSelf(wrapper_path)
 
+  cwd = os.getcwd()
+  if get_gitc_manifest_dir() and cwd.startswith(get_gitc_manifest_dir()):
+    _print('error: repo cannot be used in the GITC local manifest directory.'
+           '\nIf you want to work on this GITC client please rerun this '
+           'command from the corresponding client under /gitc/', file=sys.stderr)
+    sys.exit(1)
   if not repo_main:
     if opt.help:
       _Usage()
@@ -736,13 +792,13 @@ def main(orig_args):
       _Help(args)
     if not cmd:
       _NotInstalled()
-    if cmd == 'init':
+    if cmd == 'init' or cmd == 'gitc-init':
       if my_git:
         _SetDefaultsTo(my_git)
       try:
-        _Init(args)
+        _Init(args, gitc_init=(cmd == 'gitc-init'))
       except CloneFailure:
-        shutil.rmtree(repodir, ignore_errors=True)
+        shutil.rmtree(os.path.join(repodir, S_repo), ignore_errors=True)
         sys.exit(1)
       repo_main, rel_repo_dir = _FindRepo()
     else:
@@ -768,4 +824,8 @@ def main(orig_args):
 
 
 if __name__ == '__main__':
+  if ver[0] == 3:
+    _print('warning: Python 3 support is currently experimental. YMMV.\n'
+           'Please use Python 2.6 - 2.7 instead.',
+           file=sys.stderr)
   main(sys.argv[1:])

subcmds/branches.py

@@ -46,6 +46,10 @@ class BranchInfo(object):
   def IsCurrent(self):
     return self.current > 0
 
+  @property
+  def IsSplitCurrent(self):
+    return self.current != 0 and self.current != len(self.projects)
+
   @property
   def IsPublished(self):
     return self.published > 0
@@ -139,10 +143,14 @@ is shown, then the branch appears in all projects.
       if in_cnt < project_cnt:
         fmt = out.write
         paths = []
-        if in_cnt < project_cnt - in_cnt:
+        non_cur_paths = []
+        if i.IsSplitCurrent or (in_cnt < project_cnt - in_cnt):
           in_type = 'in'
           for b in i.projects:
+            if not i.IsSplitCurrent or b.current:
               paths.append(b.project.relpath)
+            else:
+              non_cur_paths.append(b.project.relpath)
         else:
           fmt = out.notinproject
           in_type = 'not in'
@@ -154,13 +162,19 @@ is shown, then the branch appears in all projects.
               paths.append(p.relpath)
 
         s = ' %s %s' % (in_type, ', '.join(paths))
-        if width + 7 + len(s) < 80:
+        if not i.IsSplitCurrent and (width + 7 + len(s) < 80):
+          fmt = out.current if i.IsCurrent else fmt
           fmt(s)
         else:
           fmt(' %s:' % in_type)
+          fmt = out.current if i.IsCurrent else out.write
           for p in paths:
             out.nl()
             fmt(width*' ' + '          %s' % p)
+          fmt = out.write
+          for p in non_cur_paths:
+            out.nl()
+            fmt(width*' ' + '          %s' % p)
       else:
         out.write(' in all projects')
       out.nl()

subcmds/cherry_pick.py

@@ -76,6 +76,7 @@ change id will be added.
                      capture_stdout = True,
                      capture_stderr = True)
       p.stdin.write(new_msg)
+      p.stdin.close()
       if p.Wait() != 0:
         print("error: Failed to update commit message", file=sys.stderr)
         sys.exit(1)

subcmds/download.py

@@ -93,6 +93,7 @@ makes it available in your project's local working directory.
         except GitError:
           print('[%s] Could not complete the cherry-pick of %s' \
                 % (project.name, dl.commit), file=sys.stderr)
+          sys.exit(1)
 
       elif opt.revert:
         project._Revert(dl.commit)

subcmds/forall.py

@@ -14,10 +14,13 @@
 # limitations under the License.
 
 from __future__ import print_function
+import errno
 import fcntl
+import multiprocessing
 import re
 import os
 import select
+import signal
 import sys
 import subprocess
 
@@ -31,6 +34,7 @@ _CAN_COLOR = [
   'log',
 ]
 
+
 class ForallColoring(Coloring):
   def __init__(self, config):
     Coloring.__init__(self, config, 'forall')
@@ -116,6 +120,9 @@ without iterating through the remaining projects.
     p.add_option('-r', '--regex',
                  dest='regex', action='store_true',
                  help="Execute the command only on projects matching regex or wildcard expression")
+    p.add_option('-g', '--groups',
+                 dest='groups',
+                 help="Execute the command only on projects matching the specified groups")
     p.add_option('-c', '--command',
                  help='Command (and arguments) to execute',
                  dest='command',
@@ -132,9 +139,35 @@ without iterating through the remaining projects.
     g.add_option('-v', '--verbose',
                  dest='verbose', action='store_true',
                  help='Show command error messages')
+    g.add_option('-j', '--jobs',
+                 dest='jobs', action='store', type='int', default=1,
+                 help='number of commands to execute simultaneously')
 
   def WantPager(self, opt):
-    return opt.project_header
+    return opt.project_header and opt.jobs == 1
+
+  def _SerializeProject(self, project):
+    """ Serialize a project._GitGetByExec instance.
+
+    project._GitGetByExec is not pickle-able. Instead of trying to pass it
+    around between processes, make a dict ourselves containing only the
+    attributes that we need.
+
+    """
+    if not self.manifest.IsMirror:
+      lrev = project.GetRevisionId()
+    else:
+      lrev = None
+    return {
+      'name': project.name,
+      'relpath': project.relpath,
+      'remote_name': project.remote.name,
+      'lrev': lrev,
+      'rrev': project.revisionExpr,
+      'annotations': dict((a.name, a.value) for a in project.annotations),
+      'gitdir': project.gitdir,
+      'worktree': project.worktree,
+    }
 
   def Execute(self, opt, args):
     if not opt.command:
@@ -173,46 +206,116 @@ without iterating through the remaining projects.
       # pylint: enable=W0631
 
     mirror = self.manifest.IsMirror
-    out = ForallColoring(self.manifest.manifestProject.config)
-    out.redirect(sys.stdout)
-
     rc = 0
-    first = True
+
+    smart_sync_manifest_name = "smart_sync_override.xml"
+    smart_sync_manifest_path = os.path.join(
+      self.manifest.manifestProject.worktree, smart_sync_manifest_name)
+
+    if os.path.isfile(smart_sync_manifest_path):
+      self.manifest.Override(smart_sync_manifest_path)
 
     if not opt.regex:
-      projects = self.GetProjects(args)
+      projects = self.GetProjects(args, groups=opt.groups)
     else:
       projects = self.FindProjects(args)
 
     os.environ['REPO_COUNT'] = str(len(projects))
 
-    for (cnt, project) in enumerate(projects):
+    pool = multiprocessing.Pool(opt.jobs, InitWorker)
+    try:
+      config = self.manifest.manifestProject.config
+      results_it = pool.imap(
+         DoWorkWrapper,
+         self.ProjectArgs(projects, mirror, opt, cmd, shell, config))
+      pool.close()
+      for r in results_it:
+        rc = rc or r
+        if r != 0 and opt.abort_on_errors:
+          raise Exception('Aborting due to previous error')
+    except (KeyboardInterrupt, WorkerKeyboardInterrupt):
+      # Catch KeyboardInterrupt raised inside and outside of workers
+      print('Interrupted - terminating the pool')
+      pool.terminate()
+      rc = rc or errno.EINTR
+    except Exception as e:
+      # Catch any other exceptions raised
+      print('Got an error, terminating the pool: %r' % e,
+            file=sys.stderr)
+      pool.terminate()
+      rc = rc or getattr(e, 'errno', 1)
+    finally:
+      pool.join()
+    if rc != 0:
+      sys.exit(rc)
+
+  def ProjectArgs(self, projects, mirror, opt, cmd, shell, config):
+    for cnt, p in enumerate(projects):
+      try:
+        project = self._SerializeProject(p)
+      except Exception as e:
+        print('Project list error: %r' % e,
+              file=sys.stderr)
+        return
+      except KeyboardInterrupt:
+        print('Project list interrupted',
+              file=sys.stderr)
+        return
+      yield [mirror, opt, cmd, shell, cnt, config, project]
+
+class WorkerKeyboardInterrupt(Exception):
+  """ Keyboard interrupt exception for worker processes. """
+  pass
+
+
+def InitWorker():
+  signal.signal(signal.SIGINT, signal.SIG_IGN)
+
+def DoWorkWrapper(args):
+  """ A wrapper around the DoWork() method.
+
+  Catch the KeyboardInterrupt exceptions here and re-raise them as a different,
+  ``Exception``-based exception to stop it flooding the console with stacktraces
+  and making the parent hang indefinitely.
+
+  """
+  project = args.pop()
+  try:
+    return DoWork(project, *args)
+  except KeyboardInterrupt:
+    print('%s: Worker interrupted' % project['name'])
+    raise WorkerKeyboardInterrupt()
+
+
+def DoWork(project, mirror, opt, cmd, shell, cnt, config):
   env = os.environ.copy()
   def setenv(name, val):
     if val is None:
       val = ''
-        env[name] = val.encode()
+    if hasattr(val, 'encode'):
+      val = val.encode()
+    env[name] = val
 
-      setenv('REPO_PROJECT', project.name)
+  setenv('REPO_PROJECT', project['name'])
-      setenv('REPO_PATH', project.relpath)
+  setenv('REPO_PATH', project['relpath'])
-      setenv('REPO_REMOTE', project.remote.name)
+  setenv('REPO_REMOTE', project['remote_name'])
-      setenv('REPO_LREV', project.GetRevisionId())
+  setenv('REPO_LREV', project['lrev'])
-      setenv('REPO_RREV', project.revisionExpr)
+  setenv('REPO_RREV', project['rrev'])
   setenv('REPO_I', str(cnt + 1))
-      for a in project.annotations:
+  for name in project['annotations']:
-        setenv("REPO__%s" % (a.name), a.value)
+    setenv("REPO__%s" % (name), project['annotations'][name])
 
   if mirror:
-        setenv('GIT_DIR', project.gitdir)
+    setenv('GIT_DIR', project['gitdir'])
-        cwd = project.gitdir
+    cwd = project['gitdir']
   else:
-        cwd = project.worktree
+    cwd = project['worktree']
 
   if not os.path.exists(cwd):
     if (opt.project_header and opt.verbose) \
     or not opt.project_header:
-          print('skipping %s/' % project.relpath, file=sys.stderr)
+      print('skipping %s/' % project['relpath'], file=sys.stderr)
-        continue
+    return
 
   if opt.project_header:
     stdin = subprocess.PIPE
@@ -232,6 +335,8 @@ without iterating through the remaining projects.
                        stderr=stderr)
 
   if opt.project_header:
+    out = ForallColoring(config)
+    out.redirect(sys.stdout)
     class sfd(object):
       def __init__(self, fd, dest):
         self.fd = fd
@@ -264,16 +369,14 @@ without iterating through the remaining projects.
             errbuf += buf
             continue
 
-            if empty:
+        if empty and out:
-              if first:
+          if not cnt == 0:
-                first = False
-              else:
             out.nl()
 
           if mirror:
-                project_header_path = project.name
+            project_header_path = project['name']
           else:
-                project_header_path = project.relpath
+            project_header_path = project['relpath']
           out.project('project %s/', project_header_path)
           out.nl()
           out.flush()
@@ -287,12 +390,4 @@ without iterating through the remaining projects.
         s.dest.flush()
 
   r = p.wait()
-      if r != 0:
+  return r
-        if r != rc:
-          rc = r
-        if opt.abort_on_errors:
-          print("error: %s: Aborting due to previous error" % project.relpath,
-                file=sys.stderr)
-          sys.exit(r)
-    if rc != 0:
-      sys.exit(rc)

subcmds/gitc_init.py

@@ -0,0 +1,82 @@
+#
+# Copyright (C) 2015 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+import os
+import sys
+
+import gitc_utils
+from command import RequiresGitcCommand
+from subcmds import init
+
+
+class GitcInit(init.Init, RequiresGitcCommand):
+  common = True
+  helpSummary = "Initialize a GITC Client."
+  helpUsage = """
+%prog [options] [client name]
+"""
+  helpDescription = """
+The '%prog' command is ran to initialize a new GITC client for use
+with the GITC file system.
+
+This command will setup the client directory, initialize repo, just
+like repo init does, and then downloads the manifest collection
+and installs in in the .repo/directory of the GITC client.
+
+Once this is done, a GITC manifest is generated by pulling the HEAD
+SHA for each project and generates the properly formatted XML file
+and installs it as .manifest in the GITC client directory.
+
+The -c argument is required to specify the GITC client name.
+
+The optional -f argument can be used to specify the manifest file to
+use for this GITC client.
+"""
+
+  def _Options(self, p):
+    super(GitcInit, self)._Options(p)
+    g = p.add_option_group('GITC options')
+    g.add_option('-f', '--manifest-file',
+                 dest='manifest_file',
+                 help='Optional manifest file to use for this GITC client.')
+    g.add_option('-c', '--gitc-client',
+                 dest='gitc_client',
+                 help='The name for the new gitc_client instance.')
+
+  def Execute(self, opt, args):
+    if not opt.gitc_client:
+      print('fatal: gitc client (-c) is required', file=sys.stderr)
+      sys.exit(1)
+    self.client_dir = os.path.join(gitc_utils.get_gitc_manifest_dir(),
+                                   opt.gitc_client)
+    if not os.path.exists(gitc_utils.get_gitc_manifest_dir()):
+      os.makedirs(gitc_utils.get_gitc_manifest_dir())
+    if not os.path.exists(self.client_dir):
+      os.mkdir(self.client_dir)
+    super(GitcInit, self).Execute(opt, args)
+
+    for name, remote in self.manifest.remotes.iteritems():
+      remote.fetchUrl = remote.resolvedFetchUrl
+
+    if opt.manifest_file:
+      if not os.path.exists(opt.manifest_file):
+        print('fatal: Specified manifest file %s does not exist.' %
+              opt.manifest_file)
+        sys.exit(1)
+      self.manifest.Override(opt.manifest_file)
+    gitc_utils.generate_gitc_manifest(self.client_dir, self.manifest)
+    print('Please run `cd %s` to view your GITC client.' %
+          os.path.join(gitc_utils.GITC_FS_ROOT_DIR, opt.gitc_client))

subcmds/help.py

@@ -19,7 +19,8 @@ import sys
 from formatter import AbstractFormatter, DumbWriter
 
 from color import Coloring
-from command import PagedCommand, MirrorSafeCommand
+from command import PagedCommand, MirrorSafeCommand, RequiresGitcCommand
+import gitc_utils
 
 class Help(PagedCommand, MirrorSafeCommand):
   common = False
@@ -54,9 +55,17 @@ Displays detailed usage information about a command.
   def _PrintCommonCommands(self):
     print('usage: repo COMMAND [ARGS]')
     print('The most commonly used repo commands are:')
+
+    def gitc_supported(cmd):
+      if not isinstance(cmd, RequiresGitcCommand):
+        return True
+      if gitc_utils.get_gitc_manifest_dir():
+        return True
+      return False
+
     commandNames = list(sorted([name
                     for name, command in self.commands.items()
-                    if command.common]))
+                    if command.common and gitc_supported(command)]))
 
     maxlen = 0
     for name in commandNames:

subcmds/info.py

@@ -59,6 +59,7 @@ class Info(PagedCommand):
                       or 'all,-notdefault')
 
     self.heading("Manifest branch: ")
+    if self.manifest.default.revisionExpr:
         self.headtext(self.manifest.default.revisionExpr)
     self.out.nl()
     self.heading("Manifest merge branch: ")

subcmds/init.py

@@ -27,7 +27,7 @@ else:
   import imp
   import urlparse
   urllib = imp.new_module('urllib')
-  urllib.parse = urlparse.urlparse
+  urllib.parse = urlparse
 
 from color import Coloring
 from command import InteractiveCommand, MirrorSafeCommand
@@ -153,7 +153,7 @@ to update the working directory files.
       # server where this git is located, so let's save that here.
       mirrored_manifest_git = None
       if opt.reference:
-        manifest_git_path = urllib.parse(opt.manifest_url).path[1:]
+        manifest_git_path = urllib.parse.urlparse(opt.manifest_url).path[1:]
         mirrored_manifest_git = os.path.join(opt.reference, manifest_git_path)
         if not mirrored_manifest_git.endswith(".git"):
           mirrored_manifest_git += ".git"
@@ -233,7 +233,7 @@ to update the working directory files.
       sys.exit(1)
 
     if opt.manifest_branch:
-      m.MetaBranchSwitch(opt.manifest_branch)
+      m.MetaBranchSwitch()
 
     syncbuf = SyncBuffer(m.config)
     m.Sync_LocalHalf(syncbuf)

subcmds/list.py

@@ -35,6 +35,9 @@ This is similar to running: repo forall -c 'echo "$REPO_PATH : $REPO_PROJECT"'.
     p.add_option('-r', '--regex',
                  dest='regex', action='store_true',
                  help="Filter the project list based on regex or wildcard matching of strings")
+    p.add_option('-g', '--groups',
+                 dest='groups',
+                 help="Filter the project list based on the groups the project is in")
     p.add_option('-f', '--fullpath',
                  dest='fullpath', action='store_true',
                  help="Display the full work tree path instead of the relative path")
@@ -62,7 +65,7 @@ This is similar to running: repo forall -c 'echo "$REPO_PATH : $REPO_PROJECT"'.
       sys.exit(1)
 
     if not opt.regex:
-      projects = self.GetProjects(args)
+      projects = self.GetProjects(args, groups=opt.groups)
     else:
       projects = self.FindProjects(args)
 

subcmds/start.py

@@ -14,11 +14,15 @@
 # limitations under the License.
 
 from __future__ import print_function
+import os
 import sys
+
 from command import Command
 from git_config import IsId
 from git_command import git
+import gitc_utils
 from progress import Progress
+from project import SyncBuffer
 
 class Start(Command):
   common = True
@@ -53,16 +57,50 @@ revision specified in the manifest.
         print("error: at least one project must be specified", file=sys.stderr)
         sys.exit(1)
 
-    all_projects = self.GetProjects(projects)
+    proj_name_to_gitc_proj_dict = {}
+    if self.gitc_manifest:
+      all_projects = self.GetProjects(projects, manifest=self.gitc_manifest,
+                                      missing_ok=True)
+      for project in all_projects:
+        if project.old_revision:
+          project.already_synced = True
+        else:
+          project.already_synced = False
+          project.old_revision = project.revisionExpr
+        proj_name_to_gitc_proj_dict[project.name] = project
+        project.revisionExpr = None
+      # Save the GITC manifest.
+      gitc_utils.save_manifest(self.gitc_manifest)
 
+    all_projects = self.GetProjects(projects,
+                                    missing_ok=bool(self.gitc_manifest))
     pm = Progress('Starting %s' % nb, len(all_projects))
     for project in all_projects:
       pm.update()
+      if self.gitc_manifest:
+        gitc_project = proj_name_to_gitc_proj_dict[project.name]
+        # Sync projects that have already been opened.
+        if not gitc_project.already_synced:
+          proj_localdir = os.path.join(self.gitc_manifest.gitc_client_dir,
+                                       project.relpath)
+          project.worktree = proj_localdir
+          if not os.path.exists(proj_localdir):
+            os.makedirs(proj_localdir)
+          project.Sync_NetworkHalf()
+          sync_buf = SyncBuffer(self.manifest.manifestProject.config)
+          project.Sync_LocalHalf(sync_buf)
+          project.revisionExpr = gitc_project.old_revision
+
       # If the current revision is a specific SHA1 then we can't push back
-      # to it so substitute the manifest default revision instead.
+      # to it; so substitute with dest_branch if defined, or with manifest
+      # default revision instead.
+      branch_merge = ''
       if IsId(project.revisionExpr):
-        project.revisionExpr = self.manifest.default.revisionExpr
+        if project.dest_branch:
-      if not project.StartBranch(nb):
+          branch_merge = project.dest_branch
+        else:
+          branch_merge = self.manifest.default.revisionExpr
+      if not project.StartBranch(nb, branch_merge=branch_merge):
         err.append(project)
     pm.end()
 

subcmds/status.py

@@ -22,15 +22,8 @@ except ImportError:
 
 import glob
 
-from pyversion import is_python3
-if is_python3():
-  import io
-else:
-  import StringIO as io
-
 import itertools
 import os
-import sys
 
 from color import Coloring
 
@@ -97,7 +90,7 @@ the following meanings:
                  dest='orphans', action='store_true',
                  help="include objects in working directory outside of repo projects")
 
-  def _StatusHelper(self, project, clean_counter, sem, output):
+  def _StatusHelper(self, project, clean_counter, sem):
     """Obtains the status for a specific project.
 
     Obtains the status for a project, redirecting the output to
@@ -111,9 +104,9 @@ the following meanings:
       output: Where to output the status.
     """
     try:
-      state = project.PrintWorkTreeStatus(output)
+      state = project.PrintWorkTreeStatus()
       if state == 'CLEAN':
-        clean_counter.next()
+        next(clean_counter)
     finally:
       sem.release()
 
@@ -122,16 +115,16 @@ the following meanings:
     status_header = ' --\t'
     for item in dirs:
       if not os.path.isdir(item):
-        outstring.write(''.join([status_header, item]))
+        outstring.append(''.join([status_header, item]))
         continue
       if item in proj_dirs:
         continue
       if item in proj_dirs_parents:
-        self._FindOrphans(glob.glob('%s/.*' % item) + \
+        self._FindOrphans(glob.glob('%s/.*' % item) +
-            glob.glob('%s/*' % item), \
+            glob.glob('%s/*' % item),
             proj_dirs, proj_dirs_parents, outstring)
         continue
-      outstring.write(''.join([status_header, item, '/']))
+      outstring.append(''.join([status_header, item, '/']))
 
   def Execute(self, opt, args):
     all_projects = self.GetProjects(args)
@@ -141,30 +134,21 @@ the following meanings:
       for project in all_projects:
         state = project.PrintWorkTreeStatus()
         if state == 'CLEAN':
-          counter.next()
+          next(counter)
     else:
       sem = _threading.Semaphore(opt.jobs)
-      threads_and_output = []
+      threads = []
       for project in all_projects:
         sem.acquire()
 
-        class BufList(io.StringIO):
-          def dump(self, ostream):
-            for entry in self.buflist:
-              ostream.write(entry)
-
-        output = BufList()
-
         t = _threading.Thread(target=self._StatusHelper,
-                              args=(project, counter, sem, output))
+                              args=(project, counter, sem))
-        threads_and_output.append((t, output))
+        threads.append(t)
         t.daemon = True
         t.start()
-      for (t, output) in threads_and_output:
+      for t in threads:
         t.join()
-        output.dump(sys.stdout)
+    if len(all_projects) == next(counter):
-        output.close()
-    if len(all_projects) == counter.next():
       print('nothing to commit (working directory clean)')
 
     if opt.orphans:
@@ -188,23 +172,21 @@ the following meanings:
       try:
         os.chdir(self.manifest.topdir)
 
-        outstring = io.StringIO()
+        outstring = []
-        self._FindOrphans(glob.glob('.*') + \
+        self._FindOrphans(glob.glob('.*') +
-            glob.glob('*'), \
+            glob.glob('*'),
             proj_dirs, proj_dirs_parents, outstring)
 
-        if outstring.buflist:
+        if outstring:
           output = StatusColoring(self.manifest.globalConfig)
           output.project('Objects not within a project (orphans)')
           output.nl()
-          for entry in outstring.buflist:
+          for entry in outstring:
             output.untracked(entry)
             output.nl()
         else:
           print('No orphan files or directories')
 
-        outstring.close()
-
       finally:
         # Restore CWD.
         os.chdir(orig_path)

subcmds/sync.py

@@ -14,27 +14,35 @@
 # limitations under the License.
 
 from __future__ import print_function
+import json
 import netrc
 from optparse import SUPPRESS_HELP
 import os
-import pickle
 import re
 import shutil
 import socket
 import subprocess
 import sys
+import tempfile
 import time
 
 from pyversion import is_python3
 if is_python3():
+  import http.cookiejar as cookielib
+  import urllib.error
   import urllib.parse
+  import urllib.request
   import xmlrpc.client
 else:
+  import cookielib
   import imp
+  import urllib2
   import urlparse
   import xmlrpclib
   urllib = imp.new_module('urllib')
+  urllib.error = urllib2
   urllib.parse = urlparse
+  urllib.request = urllib2
   xmlrpc = imp.new_module('xmlrpc')
   xmlrpc.client = xmlrpclib
 
@@ -57,7 +65,9 @@ except ImportError:
   multiprocessing = None
 
 from git_command import GIT, git_require
+from git_config import GetUrlCookieFile
 from git_refs import R_HEADS, HEAD
+import gitc_utils
 from project import Project
 from project import RemoteSpec
 from command import Command, MirrorSafeCommand
@@ -119,6 +129,11 @@ credentials.
 The -f/--force-broken option can be used to proceed with syncing
 other projects if a project sync fails.
 
+The --force-sync option can be used to overwrite existing git
+directories if they have previously been linked to a different
+object direcotry. WARNING: This may cause data to be lost since
+refs may be removed when overwriting.
+
 The --no-clone-bundle option disables any attempt to use
 $URL/clone.bundle to bootstrap a new Git repository from a
 resumeable bundle file on a content delivery network. This
@@ -128,6 +143,13 @@ HTTP client or proxy configuration, but the Git binary works.
 The --fetch-submodules option enables fetching Git submodules
 of a project from server.
 
+The -c/--current-branch option can be used to only fetch objects that
+are on the branch specified by a project's revision.
+
+The --optimized-fetch option can be used to only fetch projects that
+are fixed to a sha1 revision if the sha1 revision does not already
+exist locally.
+
 SSH Connections
 ---------------
 
@@ -167,6 +189,11 @@ later is required to fix a server side protocol bug.
     p.add_option('-f', '--force-broken',
                  dest='force_broken', action='store_true',
                  help="continue sync even if a project fails to sync")
+    p.add_option('--force-sync',
+                 dest='force_sync', action='store_true',
+                 help="overwrite an existing git directory if it needs to "
+                      "point to a different object directory. WARNING: this "
+                      "may cause loss of data")
     p.add_option('-l', '--local-only',
                  dest='local_only', action='store_true',
                  help="only update working tree, don't fetch")
@@ -203,6 +230,9 @@ later is required to fix a server side protocol bug.
     p.add_option('--no-tags',
                  dest='no_tags', action='store_true',
                  help="don't fetch tags")
+    p.add_option('--optimized-fetch',
+                 dest='optimized_fetch', action='store_true',
+                 help='only fetch projects fixed to sha1 if revision does not exist locally')
     if show_smart:
       p.add_option('-s', '--smart-sync',
                    dest='smart_sync', action='store_true',
@@ -271,8 +301,10 @@ later is required to fix a server side protocol bug.
         success = project.Sync_NetworkHalf(
           quiet=opt.quiet,
           current_branch_only=opt.current_branch_only,
+          force_sync=opt.force_sync,
           clone_bundle=not opt.no_clone_bundle,
-          no_tags=opt.no_tags, archive=self.manifest.IsArchive)
+          no_tags=opt.no_tags, archive=self.manifest.IsArchive,
+          optimized_fetch=opt.optimized_fetch)
         self._fetch_times.Set(project, time.time() - start)
 
         # Lock around all the rest of the code, since printing, updating a set
@@ -292,7 +324,9 @@ later is required to fix a server side protocol bug.
         pm.update()
       except _FetchError:
         err_event.set()
-      except:
+      except Exception as e:
+        print('error: Cannot fetch %s (%s: %s)' \
+            % (project.name, type(e).__name__, str(e)), file=sys.stderr)
         err_event.set()
         raise
     finally:
@@ -506,6 +540,9 @@ later is required to fix a server side protocol bug.
       self.manifest.Override(opt.manifest_name)
 
     manifest_name = opt.manifest_name
+    smart_sync_manifest_name = "smart_sync_override.xml"
+    smart_sync_manifest_path = os.path.join(
+      self.manifest.manifestProject.worktree, smart_sync_manifest_name)
 
     if opt.smart_sync or opt.smart_tag:
       if not self.manifest.manifest_server:
@@ -527,17 +564,16 @@ later is required to fix a server side protocol bug.
           try:
             info = netrc.netrc()
           except IOError:
-            print('.netrc file does not exist or could not be opened',
+            # .netrc file does not exist or could not be opened
-                  file=sys.stderr)
+            pass
           else:
             try:
               parse_result = urllib.parse.urlparse(manifest_server)
               if parse_result.hostname:
-                username, _account, password = \
+                auth = info.authenticators(parse_result.hostname)
-                  info.authenticators(parse_result.hostname)
+                if auth:
-            except TypeError:
+                  username, _account, password = auth
-              # TypeError is raised when the given hostname is not present
+                else:
-              # in the .netrc file.
                   print('No credentials found for %s in .netrc'
                         % parse_result.hostname, file=sys.stderr)
             except netrc.NetrcParseError as e:
@@ -548,8 +584,12 @@ later is required to fix a server side protocol bug.
                                                     (username, password),
                                                     1)
 
+      transport = PersistentTransport(manifest_server)
+      if manifest_server.startswith('persistent-'):
+        manifest_server = manifest_server[len('persistent-'):]
+
       try:
-        server = xmlrpc.client.Server(manifest_server)
+        server = xmlrpc.client.Server(manifest_server, transport=transport)
         if opt.smart_sync:
           p = self.manifest.manifestProject
           b = p.GetBranch(p.CurrentBranch)
@@ -558,7 +598,10 @@ later is required to fix a server side protocol bug.
             branch = branch[len(R_HEADS):]
 
           env = os.environ.copy()
-          if 'TARGET_PRODUCT' in env and 'TARGET_BUILD_VARIANT' in env:
+          if 'SYNC_TARGET' in env:
+            target = env['SYNC_TARGET']
+            [success, manifest_str] = server.GetApprovedManifest(branch, target)
+          elif 'TARGET_PRODUCT' in env and 'TARGET_BUILD_VARIANT' in env:
             target = '%s-%s' % (env['TARGET_PRODUCT'],
                                 env['TARGET_BUILD_VARIANT'])
             [success, manifest_str] = server.GetApprovedManifest(branch, target)
@@ -569,17 +612,16 @@ later is required to fix a server side protocol bug.
           [success, manifest_str] = server.GetManifest(opt.smart_tag)
 
         if success:
-          manifest_name = "smart_sync_override.xml"
+          manifest_name = smart_sync_manifest_name
-          manifest_path = os.path.join(self.manifest.manifestProject.worktree,
-                                       manifest_name)
           try:
-            f = open(manifest_path, 'w')
+            f = open(smart_sync_manifest_path, 'w')
             try:
               f.write(manifest_str)
             finally:
               f.close()
-          except IOError:
+          except IOError as e:
-            print('error: cannot write manifest to %s' % manifest_path,
+            print('error: cannot write manifest to %s:\n%s'
+                  % (smart_sync_manifest_path, e),
                   file=sys.stderr)
             sys.exit(1)
           self._ReloadManifest(manifest_name)
@@ -596,6 +638,13 @@ later is required to fix a server side protocol bug.
               % (self.manifest.manifest_server, e.errcode, e.errmsg),
               file=sys.stderr)
         sys.exit(1)
+    else:  # Not smart sync or smart tag mode
+      if os.path.isfile(smart_sync_manifest_path):
+        try:
+          os.remove(smart_sync_manifest_path)
+        except OSError as e:
+          print('error: failed to remove existing smart sync override manifest: %s' %
+                e, file=sys.stderr)
 
     rp = self.manifest.repoProject
     rp.PreSync()
@@ -609,7 +658,8 @@ later is required to fix a server side protocol bug.
     if not opt.local_only:
       mp.Sync_NetworkHalf(quiet=opt.quiet,
                           current_branch_only=opt.current_branch_only,
-                          no_tags=opt.no_tags)
+                          no_tags=opt.no_tags,
+                          optimized_fetch=opt.optimized_fetch)
 
     if mp.HasChanges:
       syncbuf = SyncBuffer(mp.config)
@@ -619,6 +669,35 @@ later is required to fix a server side protocol bug.
       self._ReloadManifest(manifest_name)
       if opt.jobs is None:
         self.jobs = self.manifest.default.sync_j
+
+    # TODO (sbasi) - Add support for manifest changes, aka projects
+    # have been added or deleted from the manifest.
+    if self.gitc_manifest:
+      gitc_manifest_projects = self.GetProjects(args,
+                                                manifest=self.gitc_manifest,
+                                                missing_ok=True)
+      gitc_projects = []
+      opened_projects = []
+      for project in gitc_manifest_projects:
+        if not project.old_revision:
+          gitc_projects.append(project)
+        else:
+          opened_projects.append(project)
+
+      if gitc_projects and not opt.local_only:
+        print('Updating GITC client: %s' % self.gitc_manifest.gitc_client_name)
+        gitc_utils.generate_gitc_manifest(self.gitc_manifest.gitc_client_dir,
+                                          self.gitc_manifest,
+                                          gitc_projects)
+        print('GITC client successfully synced.')
+
+      # The opened projects need to be synced as normal, therefore we
+      # generate a new args list to represent the opened projects.
+      args = []
+      for proj in opened_projects:
+        args.append(os.path.relpath(proj.worktree, os.getcwd()))
+      if not args:
+        return
     all_projects = self.GetProjects(args,
                                     missing_ok=True,
                                     submodules_ok=opt.fetch_submodules)
@@ -672,7 +751,7 @@ later is required to fix a server side protocol bug.
     for project in all_projects:
       pm.update()
       if project.worktree:
-        project.Sync_LocalHalf(syncbuf)
+        project.Sync_LocalHalf(syncbuf, force_sync=opt.force_sync)
     pm.end()
     print(file=sys.stderr)
     if not syncbuf.Finish():
@@ -760,7 +839,7 @@ class _FetchTimes(object):
   _ALPHA = 0.5
 
   def __init__(self, manifest):
-    self._path = os.path.join(manifest.repodir, '.repopickle_fetchtimes')
+    self._path = os.path.join(manifest.repodir, '.repo_fetchtimes.json')
     self._times = None
     self._seen = set()
 
@@ -779,22 +858,17 @@ class _FetchTimes(object):
   def _Load(self):
     if self._times is None:
       try:
-        f = open(self._path, 'rb')
+        f = open(self._path)
-      except IOError:
-        self._times = {}
-        return self._times
         try:
-        try:
+          self._times = json.load(f)
-          self._times = pickle.load(f)
+        finally:
-        except IOError:
+          f.close()
+      except (IOError, ValueError):
         try:
           os.remove(self._path)
         except OSError:
           pass
         self._times = {}
-      finally:
-        f.close()
-    return self._times
 
   def Save(self):
     if self._times is None:
@@ -808,13 +882,106 @@ class _FetchTimes(object):
       del self._times[name]
 
     try:
-      f = open(self._path, 'wb')
+      f = open(self._path, 'w')
       try:
-        pickle.dump(self._times, f)
+        json.dump(self._times, f, indent=2)
-      except (IOError, OSError, pickle.PickleError):
+      finally:
+        f.close()
+    except (IOError, TypeError):
       try:
         os.remove(self._path)
       except OSError:
         pass
+
+# This is a replacement for xmlrpc.client.Transport using urllib2
+# and supporting persistent-http[s]. It cannot change hosts from
+# request to request like the normal transport, the real url
+# is passed during initialization.
+class PersistentTransport(xmlrpc.client.Transport):
+  def __init__(self, orig_host):
+    self.orig_host = orig_host
+
+  def request(self, host, handler, request_body, verbose=False):
+    with GetUrlCookieFile(self.orig_host, not verbose) as (cookiefile, proxy):
+      # Python doesn't understand cookies with the #HttpOnly_ prefix
+      # Since we're only using them for HTTP, copy the file temporarily,
+      # stripping those prefixes away.
+      if cookiefile:
+        tmpcookiefile = tempfile.NamedTemporaryFile()
+        try:
+          with open(cookiefile) as f:
+            for line in f:
+              if line.startswith("#HttpOnly_"):
+                line = line[len("#HttpOnly_"):]
+              tmpcookiefile.write(line)
+          tmpcookiefile.flush()
+
+          cookiejar = cookielib.MozillaCookieJar(tmpcookiefile.name)
+          cookiejar.load()
         finally:
-      f.close()
+          tmpcookiefile.close()
+      else:
+        cookiejar = cookielib.CookieJar()
+
+      proxyhandler = urllib.request.ProxyHandler
+      if proxy:
+        proxyhandler = urllib.request.ProxyHandler({
+            "http": proxy,
+            "https": proxy })
+
+      opener = urllib.request.build_opener(
+          urllib.request.HTTPCookieProcessor(cookiejar),
+          proxyhandler)
+
+      url = urllib.parse.urljoin(self.orig_host, handler)
+      parse_results = urllib.parse.urlparse(url)
+
+      scheme = parse_results.scheme
+      if scheme == 'persistent-http':
+        scheme = 'http'
+      if scheme == 'persistent-https':
+        # If we're proxying through persistent-https, use http. The
+        # proxy itself will do the https.
+        if proxy:
+          scheme = 'http'
+        else:
+          scheme = 'https'
+
+      # Parse out any authentication information using the base class
+      host, extra_headers, _ = self.get_host_info(parse_results.netloc)
+
+      url = urllib.parse.urlunparse((
+          scheme,
+          host,
+          parse_results.path,
+          parse_results.params,
+          parse_results.query,
+          parse_results.fragment))
+
+      request = urllib.request.Request(url, request_body)
+      if extra_headers is not None:
+        for (name, header) in extra_headers:
+          request.add_header(name, header)
+      request.add_header('Content-Type', 'text/xml')
+      try:
+        response = opener.open(request)
+      except urllib.error.HTTPError as e:
+        if e.code == 501:
+          # We may have been redirected through a login process
+          # but our POST turned into a GET. Retry.
+          response = opener.open(request)
+        else:
+          raise
+
+      p, u = xmlrpc.client.getparser()
+      while 1:
+        data = response.read(1024)
+        if not data:
+          break
+        p.feed(data)
+      p.close()
+      return u.close()
+
+  def close(self):
+    pass
+

subcmds/upload.py

@@ -25,9 +25,11 @@ from git_command import GitCommand
 from project import RepoHook
 
 from pyversion import is_python3
-if not is_python3():
 # pylint:disable=W0622
+if not is_python3():
   input = raw_input
+else:
+  unicode = str
 # pylint:enable=W0622
 
 UNUSUAL_COMMIT_THRESHOLD = 5
@@ -337,13 +339,17 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
         self._AppendAutoList(branch, people)
 
         # Check if there are local changes that may have been forgotten
-        if branch.project.HasChanges():
+        changes = branch.project.UncommitedFiles()
+        if changes:
           key = 'review.%s.autoupload' % branch.project.remote.review
           answer = branch.project.config.GetBoolean(key)
 
           # if they want to auto upload, let's not ask because it could be automated
           if answer is None:
-            sys.stdout.write('Uncommitted changes in ' + branch.project.name + ' (did you forget to amend?). Continue uploading? (y/N) ')
+            sys.stdout.write('Uncommitted changes in ' + branch.project.name)
+            sys.stdout.write(' (did you forget to amend?):\n')
+            sys.stdout.write('\n'.join(changes) + '\n')
+            sys.stdout.write('Continue uploading? (y/N) ')
             a = sys.stdin.readline().strip().lower()
             if a not in ('y', 'yes', 't', 'true', 'on'):
               print("skipping upload", file=sys.stderr)