Add manifest groups

Allows specifying a list of groups with a -g argument to repo init.
The groups act on a group= attribute specified on projects in the
manifest.
All projects are implicitly labelled with "default" unless they are
explicitly labelled "-default".
Prefixing a group with "-" removes matching projects from the list
of projects to sync.
If any non-inverted manifest groups are specified, the default label
is ignored.

Change-Id: I3a0dd7a93a8a1756205de1d03eee8c00906af0e5
Reviewed-on: https://gerrit-review.googlesource.com/34570
Reviewed-by: Shawn Pearce <sop@google.com>
Tested-by: Shawn Pearce <sop@google.com>

command.py

@@ -15,9 +15,11 @@
 
 import os
 import optparse
+import re
 import sys
 
 from error import NoSuchProjectError
+from error import InvalidProjectGroupsError
 
 class Command(object):
   """Base class for any command line action in repo.
@@ -63,9 +65,16 @@ class Command(object):
     all = self.manifest.projects
     result = []
 
+    mp = self.manifest.manifestProject
+
+    groups = mp.config.GetString('manifest.groups')
+    if groups:
+      groups = re.split('[,\s]+', groups)
+
     if not args:
       for project in all.values():
-        if missing_ok or project.Exists:
+        if ((missing_ok or project.Exists) and
+            project.MatchesGroups(groups)):
           result.append(project)
     else:
       by_path = None
@@ -102,6 +111,8 @@ class Command(object):
           raise NoSuchProjectError(arg)
         if not missing_ok and not project.Exists:
           raise NoSuchProjectError(arg)
+        if not project.MatchesGroups(groups):
+          raise InvalidProjectGroupsError(arg)
 
         result.append(project)
 

docs/manifest-format.txt

@@ -48,6 +48,7 @@ following DTD:
     <!ATTLIST project path     CDATA #IMPLIED>
     <!ATTLIST project remote   IDREF #IMPLIED>
     <!ATTLIST project revision CDATA #IMPLIED>
+    <!ATTLIST project groups   CDATA #IMPLIED>
   
     <!ELEMENT remove-project (EMPTY)>
     <!ATTLIST remove-project name  CDATA #REQUIRED>
@@ -158,6 +159,10 @@ Tags and/or explicit SHA-1s should work in theory, but have not
 been extensively tested.  If not supplied the revision given by
 the default element is used.
 
+Attribute `groups`: List of groups to which this project belongs,
+whitespace or comma separated.  All projects are part of the group
+"default" unless "-default" is specified in the list of groups.
+
 Element remove-project
 ----------------------
 

error.py

@@ -77,6 +77,18 @@ class NoSuchProjectError(Exception):
       return 'in current directory'
     return self.name
 
+
+class InvalidProjectGroupsError(Exception):
+  """A specified project is not suitable for the specified groups
+  """
+  def __init__(self, name=None):
+    self.name = name
+
+  def __str__(self):
+    if self.Name is None:
+      return 'in current directory'
+    return self.name
+
 class RepoChangedException(Exception):
   """Thrown if 'repo sync' results in repo updating its internal
      repo or manifest repositories.  In this special case we must

manifest_xml.py

@@ -119,6 +119,12 @@ class XmlManifest(object):
   def Save(self, fd, peg_rev=False):
     """Write the current manifest out to the given file descriptor.
     """
+    mp = self.manifestProject
+
+    groups = mp.config.GetString('manifest.groups')
+    if groups:
+      groups = re.split('[,\s]+', groups)
+
     doc = xml.dom.minidom.Document()
     root = doc.createElement('manifest')
     doc.appendChild(root)
@@ -167,6 +173,10 @@ class XmlManifest(object):
 
     for p in sort_projects:
       p = self.projects[p]
+
+      if not p.MatchesGroups(groups):
+        continue
+
       e = doc.createElement('project')
       root.appendChild(e)
       e.setAttribute('name', p.name)
@@ -190,6 +200,9 @@ class XmlManifest(object):
         ce.setAttribute('dest', c.dest)
         e.appendChild(ce)
 
+      if p.groups:
+        e.setAttribute('groups', ','.join(p.groups))
+
     if self._repo_hooks_project:
       root.appendChild(doc.createTextNode(''))
       e = doc.createElement('repo-hooks')
@@ -504,6 +517,12 @@ class XmlManifest(object):
     else:
       rebase = rebase.lower() in ("yes", "true", "1")
 
+    groups = node.getAttribute('groups')
+    if groups:
+      groups = re.split('[,\s]+', groups)
+    else:
+      groups = None
+
     if self.IsMirror:
       relpath = None
       worktree = None
@@ -520,7 +539,8 @@ class XmlManifest(object):
                       relpath = path,
                       revisionExpr = revisionExpr,
                       revisionId = None,
-                      rebase = rebase)
+                      rebase = rebase,
+                      groups = groups)
 
     for n in node.childNodes:
       if n.nodeName == 'copyfile':

project.py

@@ -504,7 +504,8 @@ class Project(object):
                relpath,
                revisionExpr,
                revisionId,
-               rebase = True):
+               rebase = True,
+               groups = None):
     self.manifest = manifest
     self.name = name
     self.remote = remote
@@ -524,6 +525,7 @@ class Project(object):
       self.revisionId = revisionId
 
     self.rebase = rebase
+    self.groups = groups
 
     self.snapshots = {}
     self.copyfiles = []
@@ -645,6 +647,45 @@ class Project(object):
 
     return heads
 
+  def MatchesGroups(self, manifest_groups):
+    """Returns true if the manifest groups specified at init should cause
+       this project to be synced.
+       Prefixing a manifest group with "-" inverts the meaning of a group.
+       All projects are implicitly labelled with "default" unless they are
+       explicitly labelled "-default".
+       If any non-inverted manifest groups are specified, the default label
+       is ignored.
+       Specifying only inverted groups implies "default".
+    """
+    project_groups = self.groups
+    if not manifest_groups:
+      return not project_groups or not "-default" in project_groups
+
+    if not project_groups:
+      project_groups = ["default"]
+    elif not ("default" in project_groups or "-default" in project_groups):
+      project_groups.append("default")
+
+    plus_groups = [x for x in manifest_groups if not x.startswith("-")]
+    minus_groups = [x[1:] for x in manifest_groups if x.startswith("-")]
+
+    if not plus_groups:
+      plus_groups.append("default")
+
+    for group in minus_groups:
+      if group in project_groups:
+        # project was excluded by -group
+        return False
+
+    for group in plus_groups:
+      if group in project_groups:
+        # project was included by group
+        return True
+
+    # groups were specified that did not include this project
+    if plus_groups:
+      return False
+    return True
 
 ## Status Display ##
 
@@ -749,7 +790,7 @@ class Project(object):
 
     return 'DIRTY'
 
-  def PrintWorkTreeDiff(self):
+  def PrintWorkTreeDiff(self, absolute_paths=False):
     """Prints the status of the repository to stdout.
     """
     out = DiffColoring(self.config)
@@ -757,6 +798,9 @@ class Project(object):
     if out.is_on:
       cmd.append('--color')
     cmd.append(HEAD)
+    if absolute_paths:
+      cmd.append('--src-prefix=a/%s/' % self.relpath)
+      cmd.append('--dst-prefix=b/%s/' % self.relpath)
     cmd.append('--')
     p = GitCommand(self,
                    cmd,
@@ -1102,7 +1146,9 @@ class Project(object):
                    len(local_changes) - cnt_mine)
 
     branch.remote = self.GetRemote(self.remote.name)
-    branch.merge = self.revisionExpr
+    if not ID_RE.match(self.revisionExpr):
+      # in case of manifest sync the revisionExpr might be a SHA1
+      branch.merge = self.revisionExpr
     branch.Save()
 
     if cnt_mine > 0 and self.rebase:
@@ -1514,7 +1560,7 @@ class Project(object):
             except:
               return None
 
-          if e.code == 404:
+          if e.code in (401, 403, 404):
             keep = False
             return False
           elif _content_type() == 'text/plain':
@@ -2086,7 +2132,8 @@ class MetaProject(Project):
                      remote = RemoteSpec('origin'),
                      relpath = '.repo/%s' % name,
                      revisionExpr = 'refs/heads/master',
-                     revisionId = None)
+                     revisionId = None,
+                     groups = None)
 
   def PreSync(self):
     if self.Exists:

repo

@@ -28,7 +28,7 @@ if __name__ == '__main__':
 del magic
 
 # increment this whenever we make important changes to this script
-VERSION = (1, 14)
+VERSION = (1, 15)
 
 # increment this if the MAINTAINER_KEYS block is modified
 KEYRING_VERSION = (1,0)
@@ -125,6 +125,10 @@ group.add_option('--reference',
 group.add_option('--depth', type='int', default=None,
                  dest='depth',
                  help='create a shallow clone with given depth; see git clone')
+group.add_option('-g', '--groups',
+                 dest='groups', default="",
+                 help='restrict manifest projects to ones with a specified group',
+                 metavar='GROUP')
 
 
 # Tool

subcmds/diff.py

@@ -20,8 +20,21 @@ class Diff(PagedCommand):
   helpSummary = "Show changes between commit and working tree"
   helpUsage = """
 %prog [<project>...]
+
+The -u option causes '%prog' to generate diff output with file paths
+relative to the repository root, so the output can be applied
+to the Unix 'patch' command.
 """
 
+  def _Options(self, p):
+    def cmd(option, opt_str, value, parser):
+      setattr(parser.values, option.dest, list(parser.rargs))
+      while parser.rargs:
+        del parser.rargs[0]
+    p.add_option('-u', '--absolute',
+                 dest='absolute', action='store_true',
+                 help='Paths are relative to the repository root')
+
   def Execute(self, opt, args):
     for project in self.GetProjects(args):
-      project.PrintWorkTreeDiff()
+      project.PrintWorkTreeDiff(opt.absolute)

subcmds/init.py

@@ -86,6 +86,10 @@ to update the working directory files.
     g.add_option('--depth', type='int', default=None,
                  dest='depth',
                  help='create a shallow clone with given depth; see git clone')
+    g.add_option('-g', '--groups',
+                 dest='groups', default="",
+                 help='restrict manifest projects to ones with a specified group',
+                 metavar='GROUP')
 
     # Tool
     g = p.add_option_group('repo Version options')
@@ -135,6 +139,8 @@ to update the working directory files.
       r.ResetFetch()
       r.Save()
 
+    m.config.SetString('manifest.groups', opt.groups)
+
     if opt.reference:
       m.config.SetString('repo.reference', opt.reference)
 

subcmds/sync.py

@@ -277,7 +277,7 @@ later is required to fix a server side protocol bug.
 
   def UpdateProjectList(self):
     new_project_paths = []
-    for project in self.manifest.projects.values():
+    for project in self.GetProjects(None, missing_ok=True):
       if project.relpath:
         new_project_paths.append(project.relpath)
     file_name = 'project.list'
@@ -306,7 +306,8 @@ later is required to fix a server side protocol bug.
                              worktree = os.path.join(self.manifest.topdir, path),
                              relpath = path,
                              revisionExpr = 'HEAD',
-                             revisionId = None)
+                             revisionId = None,
+                             groups = None)
 
               if project.IsDirty():
                 print >>sys.stderr, 'error: Cannot remove project "%s": \

subcmds/upload.py

@@ -123,6 +123,9 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
     p.add_option('--br',
                  type='string',  action='store', dest='branch',
                  help='Branch to upload.')
+    p.add_option('--cbr', '--current-branch',
+                 dest='current_branch', action='store_true',
+                 help='Upload current git branch.')
 
     # Options relating to upload hook.  Note that verify and no-verify are NOT
     # opposites of each other, which is why they store to different locations.
@@ -351,7 +354,11 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
       branch = opt.branch
 
     for project in project_list:
-      avail = project.GetUploadableBranches(branch)
+      if opt.current_branch:
+        cbr = project.CurrentBranch
+        avail = [project.GetUploadableBranch(cbr)] if cbr else None
+      else:
+        avail = project.GetUploadableBranches(branch)
       if avail:
         pending.append((project, avail))