project: Set config option to skip lfs process filter

During sync, repo runs `git read-tree --reset -u -v HEAD` which causes
git-lfs's smudge filter to run, which fails because git-lfs does not
work with bare repositories.

This was fixed in I091ff37998131e2e6bbc59aa37ee352fe12d7fcd to
automatically disable this smudge filter. However, later versions of
Git (2.11.0) introduced a new filter protocol [1], to avoid spawning
a new command for each filtered file. This was implemented in Git-LFS
1.5.0 [2].

This patch fixes the issue by setting the git lfs process filter, in
addition to the smudge filter. For any projects that have LFS objects,
`git lfs pull` must still be executed manually afterwards.

[1] edcc85814c
[2] https://github.com/git-lfs/git-lfs/pull/1617

Bug: https://crbug.com/gerrit/10911
Change-Id: I277fc68fdefc91514a2412b3887e3be9106cab48

docs/manifest-format.md

@@ -66,7 +66,7 @@ following DTD:
   <!ATTLIST project groups      CDATA #IMPLIED>
   <!ATTLIST project sync-c      CDATA #IMPLIED>
   <!ATTLIST project sync-s      CDATA #IMPLIED>
-  <!ATTLIST default sync-tags   CDATA #IMPLIED>
+  <!ATTLIST project sync-tags   CDATA #IMPLIED>
   <!ATTLIST project upstream CDATA #IMPLIED>
   <!ATTLIST project clone-depth CDATA #IMPLIED>
   <!ATTLIST project force-path CDATA #IMPLIED>

docs/python-support.md

@@ -0,0 +1,32 @@
+# Supported Python Versions
+
+With Python 2.7 officially going EOL on [01 Jan 2020](https://pythonclock.org/),
+we need a support plan for the repo project itself.
+Inevitably, there will be a long tail of users who still want to use Python 2 on
+their old LTS/corp systems and have little power to change the system.
+
+## Summary
+
+* Python 3.6 (released Dec 2016) is required by default starting with repo-1.14.
+* Older versions of Python (e.g. v2.7) may use the legacy feature-frozen branch
+  based on repo-1.13.
+
+## Overview
+
+We provide a branch for Python 2 users that is feature-frozen.
+Bugfixes may be added on a best-effort basis or from the community, but largely
+no new features will be added, nor is support guaranteed.
+
+Users can select this during `repo init` time via the [repo launcher].
+Otherwise the default branches (e.g. stable & master) will be used which will
+require Python 3.
+
+This means the [repo launcher] needs to support both Python 2 & Python 3, but
+since it doesn't import any other repo code, this shouldn't be too problematic.
+
+The master branch will require Python 3.6 at a minimum.
+If the system has an older version of Python 3, then users will have to select
+the legacy Python 2 branch instead.
+
+
+[repo launcher]: ../repo

event_log.py

@@ -51,7 +51,6 @@ class EventLog(object):
   def __init__(self):
     """Initializes the event log."""
     self._log = []
-    self._next_id = _EventIdGenerator()
     self._parent = None
 
   def Add(self, name, task_name, start, finish=None, success=None,
@@ -71,7 +70,7 @@ class EventLog(object):
       A dictionary of the event added to the log.
     """
     event = {
-        'id': (kind, next(self._next_id)),
+        'id': (kind, _NextEventId()),
         'name': name,
         'task_name': task_name,
         'start_time': start,
@@ -162,16 +161,16 @@ class EventLog(object):
         f.write('\n')
 
 
-def _EventIdGenerator():
+# An integer id that is unique across this invocation of the program.
-  """Returns multi-process safe iterator that generates locally unique id.
+_EVENT_ID = multiprocessing.Value('i', 1)
 
-  Yields:
+def _NextEventId():
+  """Helper function for grabbing the next unique id.
+
+  Returns:
     A unique, to this invocation of the program, integer id.
   """
-  eid = multiprocessing.Value('i', 1)
+  with _EVENT_ID.get_lock():
-
+    val = _EVENT_ID.value
-  while True:
+    _EVENT_ID.value += 1
-    with eid.get_lock():
+  return val
-      val = eid.value
-      eid.value += 1
-    yield val

project.py

@@ -1172,10 +1172,11 @@ class Project(object):
 
     ref_spec = '%s:refs/%s/%s' % (R_HEADS + branch.name, upload_type,
                                   dest_branch)
+    opts = []
     if auto_topic:
-      ref_spec = ref_spec + '/' + branch.name
+      opts += ['topic=' + branch.name]
 
-    opts = ['r=%s' % p for p in people[0]]
+    opts += ['r=%s' % p for p in people[0]]
     opts += ['cc=%s' % p for p in people[1]]
     if notify:
       opts += ['notify=' + notify]
@@ -1306,7 +1307,7 @@ class Project(object):
         not self._RemoteFetch(initial=is_new, quiet=quiet, alt_dir=alt_dir,
                               current_branch_only=current_branch_only,
                               no_tags=no_tags, prune=prune, depth=depth,
-                              submodules=submodules)):
+                              submodules=submodules, force_sync=force_sync)):
       return False
 
     mp = self.manifest.manifestProject
@@ -1807,8 +1808,8 @@ class Project(object):
         submodules.append((sub_rev, sub_path, sub_url))
       return submodules
 
-    re_path = re.compile(r'^submodule\.([^.]+)\.path=(.*)$')
+    re_path = re.compile(r'^submodule\.(.+)\.path=(.*)$')
-    re_url = re.compile(r'^submodule\.([^.]+)\.url=(.*)$')
+    re_url = re.compile(r'^submodule\.(.+)\.url=(.*)$')
 
     def parse_gitmodules(gitdir, rev):
       cmd = ['cat-file', 'blob', '%s:.gitmodules' % rev]
@@ -1955,7 +1956,8 @@ class Project(object):
                    no_tags=False,
                    prune=False,
                    depth=None,
-                   submodules=False):
+                   submodules=False,
+                   force_sync=False):
 
     is_sha1 = False
     tag_name = None
@@ -1979,6 +1981,7 @@ class Project(object):
 
       if is_sha1 or tag_name is not None:
         if self._CheckForImmutableRevision():
+          if not quiet:
             print('Skipped fetching project %s (already have persistent ref)'
                   % self.name)
           return True
@@ -2068,6 +2071,9 @@ class Project(object):
     else:
       cmd.append('--tags')
 
+    if force_sync:
+      cmd.append('--force')
+
     if prune:
       cmd.append('--prune')
 
@@ -2393,6 +2399,7 @@ class Project(object):
           if m.Has(key, include_defaults=False):
             self.config.SetString(key, m.GetString(key))
         self.config.SetString('filter.lfs.smudge', 'git-lfs smudge --skip -- %f')
+        self.config.SetString('filter.lfs.process', 'git-lfs filter-process --skip')
         if self.manifest.IsMirror:
           self.config.SetString('core.bare', 'true')
         else:
@@ -2584,7 +2591,7 @@ class Project(object):
         cmd.append('-v')
         cmd.append(HEAD)
         if GitCommand(self, cmd).Wait() != 0:
-          raise GitError("cannot initialize work tree")
+          raise GitError("cannot initialize work tree for " + self.name)
 
         if submodules:
           self._SyncSubmodules(quiet=True)
@@ -2684,6 +2691,7 @@ class Project(object):
     def DiffZ(self, name, *args):
       cmd = [name]
       cmd.append('-z')
+      cmd.append('--ignore-submodules')
       cmd.extend(args)
       p = GitCommand(self._project,
                      cmd,

subcmds/info.py

@@ -45,7 +45,7 @@ class Info(PagedCommand):
   def Execute(self, opt, args):
     self.out = _Coloring(self.manifest.globalConfig)
     self.heading = self.out.printer('heading', attr = 'bold')
-    self.headtext = self.out.printer('headtext', fg = 'yellow')
+    self.headtext = self.out.nofmt_printer('headtext', fg = 'yellow')
     self.redtext = self.out.printer('redtext', fg = 'red')
     self.sha = self.out.printer("sha", fg = 'yellow')
     self.text = self.out.nofmt_printer('text')

subcmds/init.py

@@ -197,6 +197,8 @@ to update the working directory files.
       else:
         m.PreSync()
 
+    self._ConfigureDepth(opt)
+
     if opt.manifest_url:
       r = m.GetRemote(m.remote.name)
       r.url = opt.manifest_url
@@ -429,6 +431,4 @@ to update the working directory files.
         self._ConfigureUser()
       self._ConfigureColor()
 
-    self._ConfigureDepth(opt)
-
     self._DisplayResult()

subcmds/sync.py

@@ -136,6 +136,11 @@ directories if they have previously been linked to a different
 object direcotry. WARNING: This may cause data to be lost since
 refs may be removed when overwriting.
 
+The --force-remove-dirty option can be used to remove previously used
+projects with uncommitted changes. WARNING: This may cause data to be
+lost since uncommitted changes may be removed with projects that no longer
+exist in the manifest.
+
 The --no-clone-bundle option disables any attempt to use
 $URL/clone.bundle to bootstrap a new Git repository from a
 resumeable bundle file on a content delivery network. This
@@ -197,6 +202,11 @@ later is required to fix a server side protocol bug.
                  help="overwrite an existing git directory if it needs to "
                       "point to a different object directory. WARNING: this "
                       "may cause loss of data")
+    p.add_option('--force-remove-dirty',
+                 dest='force_remove_dirty', action='store_true',
+                 help="force remove projects with uncommitted modifications if "
+                      "projects no longer exist in the manifest. "
+                      "WARNING: this may cause loss of data")
     p.add_option('-l', '--local-only',
                  dest='local_only', action='store_true',
                  help="only update working tree, don't fetch")
@@ -525,7 +535,7 @@ later is required to fix a server side protocol bug.
 
     return 0
 
-  def UpdateProjectList(self):
+  def UpdateProjectList(self, opt):
     new_project_paths = []
     for project in self.GetProjects(None, missing_ok=True):
       if project.relpath:
@@ -540,7 +550,8 @@ later is required to fix a server side protocol bug.
         old_project_paths = fd.read().split('\n')
       finally:
         fd.close()
-      for path in old_project_paths:
+      # In reversed order, so subfolders are deleted before parent folder.
+      for path in sorted(old_project_paths, reverse=True):
         if not path:
           continue
         if path not in new_project_paths:
@@ -559,7 +570,11 @@ later is required to fix a server side protocol bug.
                            revisionId = None,
                            groups = None)
 
-            if project.IsDirty():
+            if project.IsDirty() and opt.force_remove_dirty:
+              print('WARNING: Removing dirty project "%s": uncommitted changes '
+                    'erased' % project.relpath, file=sys.stderr)
+              self._DeleteProject(project.worktree)
+            elif project.IsDirty():
               print('error: Cannot remove project "%s": uncommitted changes '
                     'are present' % project.relpath, file=sys.stderr)
               print('       commit changes, then run sync again',
@@ -827,7 +842,7 @@ later is required to fix a server side protocol bug.
       # bail out now, we have no working tree
       return
 
-    if self.UpdateProjectList():
+    if self.UpdateProjectList(opt):
       sys.exit(1)
 
     syncbuf = SyncBuffer(mp.config,