rebase: Pass through more options

Passing through --whitespace=fix to rebase can be useful
to clean up a branch prior to uploading it for review.

Change-Id: Id85f1912e5e11ff9602e3b342c2fd7441abe67d7
Signed-off-by: Shawn O. Pearce <sop@google.com>

docs/manifest-format.txt

@@ -22,6 +22,7 @@ following DTD:
   <!DOCTYPE manifest [
     <!ELEMENT manifest (remote*,
                         default?,
+                        manifest-server?,
                         remove-project*,
                         project*)>
   
@@ -34,6 +35,9 @@ following DTD:
     <!ATTLIST default remote   IDREF #IMPLIED>
     <!ATTLIST default revision CDATA #IMPLIED>
 
+    <!ELEMENT manifest-server (EMPTY)>
+    <!ATTLIST url              CDATA #REQUIRED>
+  
     <!ELEMENT project (EMPTY)>
     <!ATTLIST project name     CDATA #REQUIRED>
     <!ATTLIST project path     CDATA #IMPLIED>
@@ -89,6 +93,27 @@ Attribute `revision`: Name of a Git branch (e.g. `master` or
 revision attribute will use this revision.
 
 
+Element manifest-server
+-----------------------
+
+At most one manifest-server may be specified. The url attribute
+is used to specify the URL of a manifest server, which is an
+XML RPC service that will return a manifest in which each project
+is pegged to a known good revision for the current branch and
+target.
+
+The manifest server should implement:
+
+  GetApprovedManifest(branch, target)
+
+The target to use is defined by environment variables TARGET_PRODUCT
+and TARGET_BUILD_VARIANT. These variables are used to create a string
+of the form $TARGET_PRODUCT-$TARGET_BUILD_VARIANT, e.g. passion-userdebug.
+If one of those variables or both are not present, the program will call
+GetApprovedManifest without the target paramater and the manifest server
+should choose a reasonable default target.
+
+
 Element project
 ---------------
 

git_command.py

@@ -17,6 +17,7 @@ import os
 import sys
 import subprocess
 import tempfile
+from signal import SIGTERM
 from error import GitError
 from trace import REPO_TRACE, IsTrace, Trace
 
@@ -29,8 +30,9 @@ LAST_CWD = None
 
 _ssh_proxy_path = None
 _ssh_sock_path = None
+_ssh_clients = []
 
-def _ssh_sock(create=True):
+def ssh_sock(create=True):
   global _ssh_sock_path
   if _ssh_sock_path is None:
     if not create:
@@ -51,6 +53,24 @@ def _ssh_proxy():
       'git_ssh')
   return _ssh_proxy_path
 
+def _add_ssh_client(p):
+  _ssh_clients.append(p)
+
+def _remove_ssh_client(p):
+  try:
+    _ssh_clients.remove(p)
+  except ValueError:
+    pass
+
+def terminate_ssh_clients():
+  global _ssh_clients
+  for p in _ssh_clients:
+    try:
+      os.kill(p.pid, SIGTERM)
+      p.wait()
+    except OSError:
+      pass
+  _ssh_clients = []
 
 class _GitCall(object):
   def version(self):
@@ -119,7 +139,7 @@ class GitCommand(object):
     if disable_editor:
       env['GIT_EDITOR'] = ':'
     if ssh_proxy:
-      env['REPO_SSH_SOCK'] = _ssh_sock()
+      env['REPO_SSH_SOCK'] = ssh_sock()
       env['GIT_SSH'] = _ssh_proxy()
 
     if project:
@@ -188,6 +208,9 @@ class GitCommand(object):
     except Exception, e:
       raise GitError('%s: %s' % (command[1], e))
 
+    if ssh_proxy:
+      _add_ssh_client(p)
+
     self.process = p
     self.stdin = p.stdin
 
@@ -210,4 +233,8 @@ class GitCommand(object):
     else:
       p.stderr = None
 
-    return self.process.wait()
+    try:
+      rc = p.wait()
+    finally:
+      _remove_ssh_client(p)
+    return rc

git_config.py

@@ -23,7 +23,10 @@ from signal import SIGTERM
 from urllib2 import urlopen, HTTPError
 from error import GitError, UploadError
 from trace import Trace
-from git_command import GitCommand, _ssh_sock
+
+from git_command import GitCommand
+from git_command import ssh_sock
+from git_command import terminate_ssh_clients
 
 R_HEADS = 'refs/heads/'
 R_TAGS  = 'refs/tags/'
@@ -356,10 +359,14 @@ class RefSpec(object):
 _ssh_cache = {}
 _ssh_master = True
 
-def _open_ssh(host, port):
+def _open_ssh(host, port=None):
   global _ssh_master
 
-  key = '%s:%s' % (host, port)
+  if port is not None:
+    key = '%s:%s' % (host, port)
+  else:
+    key = host
+
   if key in _ssh_cache:
     return True
 
@@ -371,11 +378,14 @@ def _open_ssh(host, port):
     return False
 
   command = ['ssh',
-             '-o','ControlPath %s' % _ssh_sock(),
-             '-p',str(port),
+             '-o','ControlPath %s' % ssh_sock(),
              '-M',
              '-N',
              host]
+
+  if port is not None:
+    command[3:3] = ['-p',str(port)]
+
   try:
     Trace(': %s', ' '.join(command))
     p = subprocess.Popen(command)
@@ -391,6 +401,8 @@ def _open_ssh(host, port):
   return True
 
 def close_ssh():
+  terminate_ssh_clients()
+
   for key,p in _ssh_cache.iteritems():
     try:
       os.kill(p.pid, SIGTERM)
@@ -399,7 +411,7 @@ def close_ssh():
       pass
   _ssh_cache.clear()
 
-  d = _ssh_sock(create=False)
+  d = ssh_sock(create=False)
   if d:
     try:
       os.rmdir(os.path.dirname(d))
@@ -417,7 +429,7 @@ def _preconnect(url):
     if ':' in host:
       host, port = host.split(':')
     else:
-      port = 22
+      port = None
     if scheme in ('ssh', 'git+ssh', 'ssh+git'):
       return _open_ssh(host, port)
     return False
@@ -425,7 +437,7 @@ def _preconnect(url):
   m = URI_SCP.match(url)
   if m:
     host = m.group(1)
-    return _open_ssh(host, 22)
+    return _open_ssh(host)
 
   return False
 
@@ -519,8 +531,11 @@ class Remote(object):
   def SshReviewUrl(self, userEmail):
     if self.ReviewProtocol != 'ssh':
       return None
+    username = self._config.GetString('review.%s.username' % self.review)
+    if username is None:
+      username = userEmail.split("@")[0]
     return 'ssh://%s@%s:%s/%s' % (
-      userEmail.split("@")[0],
+      username,
       self._review_host,
       self._review_port,
       self.projectname)

hooks/commit-msg

@@ -0,0 +1,101 @@
+#!/bin/sh
+# From Gerrit Code Review 2.1.2-rc2-33-g7e30c72
+#
+# Part of Gerrit Code Review (http://code.google.com/p/gerrit/)
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+CHANGE_ID_AFTER="Bug|Issue"
+MSG="$1"
+
+# Check for, and add if missing, a unique Change-Id
+#
+add_ChangeId() {
+	clean_message=$(sed -e '
+		/^diff --git a\/.*/{
+			s///
+			q
+		}
+		/^Signed-off-by:/d
+		/^#/d
+	' "$MSG" | git stripspace)
+	if test -z "$clean_message"
+	then
+		return
+	fi
+
+	if grep -i '^Change-Id:' "$MSG" >/dev/null
+	then
+		return
+	fi
+
+	id=$(_gen_ChangeId)
+	perl -e '
+		$MSG = shift;
+		$id = shift;
+		$CHANGE_ID_AFTER = shift;
+
+		undef $/;
+		open(I, $MSG); $_ = <I>; close I;
+		s|^diff --git a/.*||ms;
+		s|^#.*$||mg;
+		exit unless $_;
+
+		@message = split /\n/;
+		$haveFooter = 0;
+		$startFooter = @message;
+		for($line = @message - 1; $line >= 0; $line--) {
+			$_ = $message[$line];
+
+			($haveFooter++, next) if /^[a-zA-Z0-9-]+:/;
+			next if /^[ []/;
+			$startFooter = $line if ($haveFooter && /^\r?$/);
+			last;
+		}
+
+		@footer = @message[$startFooter+1..@message];
+		@message = @message[0..$startFooter];
+		push(@footer, "") unless @footer;
+
+		for ($line = 0; $line < @footer; $line++) {
+			$_ = $footer[$line];
+			next if /^($CHANGE_ID_AFTER):/i;
+			last;
+		}
+		splice(@footer, $line, 0, "Change-Id: I$id");
+
+		$_ = join("\n", @message, @footer);
+		open(O, ">$MSG"); print O; close O;
+	' "$MSG" "$id" "$CHANGE_ID_AFTER"
+}
+_gen_ChangeIdInput() {
+	echo "tree $(git write-tree)"
+	if parent=$(git rev-parse HEAD^0 2>/dev/null)
+	then
+		echo "parent $parent"
+	fi
+	echo "author $(git var GIT_AUTHOR_IDENT)"
+	echo "committer $(git var GIT_COMMITTER_IDENT)"
+	echo
+	printf '%s' "$clean_message"
+}
+_gen_ChangeId() {
+	_gen_ChangeIdInput |
+	git hash-object -t commit --stdin
+}
+
+
+add_ChangeId

manifest_xml.py

@@ -65,8 +65,8 @@ class XmlManifest(object):
 
     self._Unload()
 
-  def Link(self, name):
-    """Update the repo metadata to use a different manifest.
+  def Override(self, name):
+    """Use a different manifest, just for the current instantiation.
     """
     path = os.path.join(self.manifestProject.worktree, name)
     if not os.path.isfile(path):
@@ -80,6 +80,11 @@ class XmlManifest(object):
     finally:
       self.manifestFile = old
 
+  def Link(self, name):
+    """Update the repo metadata to use a different manifest.
+    """
+    self.Override(name)
+
     try:
       if os.path.exists(self.manifestFile):
         os.remove(self.manifestFile)
@@ -123,6 +128,12 @@ class XmlManifest(object):
       root.appendChild(e)
       root.appendChild(doc.createTextNode(''))
 
+    if self._manifest_server:
+      e = doc.createElement('manifest-server')
+      e.setAttribute('url', self._manifest_server)
+      root.appendChild(e)
+      root.appendChild(doc.createTextNode(''))
+
     sort_projects = list(self.projects.keys())
     sort_projects.sort()
 
@@ -168,6 +179,11 @@ class XmlManifest(object):
     self._Load()
     return self._default
 
+  @property
+  def manifest_server(self):
+    self._Load()
+    return self._manifest_server
+
   @property
   def IsMirror(self):
     return self.manifestProject.config.GetBoolean('repo.mirror')
@@ -178,6 +194,7 @@ class XmlManifest(object):
     self._remotes = {}
     self._default = None
     self.branch = None
+    self._manifest_server = None
 
   def _Load(self):
     if not self._loaded:
@@ -246,6 +263,15 @@ class XmlManifest(object):
     if self._default is None:
       self._default = _Default()
 
+    for node in config.childNodes:
+      if node.nodeName == 'manifest-server':
+        url = self._reqatt(node, 'url')
+        if self._manifest_server is not None:
+            raise ManifestParseError, \
+                'duplicate manifest-server in %s' % \
+                (self.manifestFile)
+        self._manifest_server = url
+
     for node in config.childNodes:
       if node.nodeName == 'project':
         project = self._ParseProject(node)

progress.py

@@ -13,10 +13,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
 import sys
 from time import time
 from trace import IsTrace
 
+_NOT_TTY = not os.isatty(2)
+
 class Progress(object):
   def __init__(self, title, total=0):
     self._title = title
@@ -29,7 +32,7 @@ class Progress(object):
   def update(self, inc=1):
     self._done += inc
 
-    if IsTrace():
+    if _NOT_TTY or IsTrace():
       return
 
     if not self._show:
@@ -56,7 +59,7 @@ class Progress(object):
         sys.stderr.flush()
 
   def end(self):
-    if IsTrace() or not self._show:
+    if _NOT_TTY or IsTrace() or not self._show:
       return
 
     if self._total <= 0:

project.py

@@ -149,10 +149,11 @@ class ReviewableBranch(object):
       R_HEADS + self.name,
       '--')
 
-  def UploadForReview(self, people):
+  def UploadForReview(self, people, auto_topic=False):
     self.project.UploadForReview(self.name,
                                  self.replace_changes,
-                                 people)
+                                 people,
+                                 auto_topic=auto_topic)
 
   def GetPublishedRefs(self):
     refs = {}
@@ -203,6 +204,10 @@ class _CopyFile:
         # remove existing file first, since it might be read-only
         if os.path.exists(dest):
           os.remove(dest)
+        else:
+          dir = os.path.dirname(dest)
+          if not os.path.isdir(dir):
+            os.makedirs(dir)
         shutil.copy(src, dest)
         # make the file read-only
         mode = os.stat(dest)[stat.ST_MODE]
@@ -364,6 +369,27 @@ class Project(object):
 
 ## Status Display ##
 
+  def HasChanges(self):
+    """Returns true if there are uncommitted changes.
+    """
+    self.work_git.update_index('-q',
+                               '--unmerged',
+                               '--ignore-missing',
+                               '--refresh')
+    if self.IsRebaseInProgress():
+      return True
+
+    if self.work_git.DiffZ('diff-index', '--cached', HEAD):
+      return True
+
+    if self.work_git.DiffZ('diff-files'):
+      return True
+
+    if self.work_git.LsOthers():
+      return True
+
+    return False
+
   def PrintWorkTreeStatus(self):
     """Prints the status of the repository to stdout.
     """
@@ -530,7 +556,10 @@ class Project(object):
         return rb
     return None
 
-  def UploadForReview(self, branch=None, replace_changes=None, people=([],[])):
+  def UploadForReview(self, branch=None,
+                      replace_changes=None,
+                      people=([],[]),
+                      auto_topic=False):
     """Uploads the named branch for code review.
     """
     if branch is None:
@@ -562,10 +591,15 @@ class Project(object):
       for e in people[1]:
         rp.append('--cc=%s' % sq(e))
 
+      ref_spec = '%s:refs/for/%s' % (R_HEADS + branch.name, dest_branch)
+      if auto_topic:
+        ref_spec = ref_spec + '/' + branch.name
+
       cmd = ['push']
       cmd.append('--receive-pack=%s' % " ".join(rp))
       cmd.append(branch.remote.SshReviewUrl(self.UserEmail))
-      cmd.append('%s:refs/for/%s' % (R_HEADS + branch.name, dest_branch))
+      cmd.append(ref_spec)
+
       if replace_changes:
         for change_id,commit_id in replace_changes.iteritems():
           cmd.append('%s:refs/changes/%s/new' % (commit_id, change_id))
@@ -597,6 +631,18 @@ class Project(object):
     if not self._RemoteFetch():
       return False
 
+    #Check that the requested ref was found after fetch
+    #
+    try:
+      self.GetRevisionId()
+    except ManifestInvalidRevisionError:
+      # if the ref is a tag. We can try fetching
+      # the tag manually as a last resort
+      #
+      rev = self.revisionExpr
+      if rev.startswith(R_TAGS):
+        self._RemoteFetch(None, rev[len(R_TAGS):])
+
     if self.worktree:
       self._InitMRef()
     else:
@@ -706,10 +752,9 @@ class Project(object):
           # commits are not yet merged upstream.  We do not want
           # to rewrite the published commits so we punt.
           #
-          syncbuf.info(self,
-                       "branch %s is published but is now %d commits behind",
-                       branch.name,
-                       len(upstream_gain))
+          syncbuf.fail(self,
+                       "branch %s is published (but not merged) and is now %d commits behind"
+                       % (branch.name, len(upstream_gain)))
         return
       elif pub == head:
         # All published commits are merged, and thus we are a
@@ -979,7 +1024,7 @@ class Project(object):
 
 ## Direct Git Commands ##
 
-  def _RemoteFetch(self, name=None):
+  def _RemoteFetch(self, name=None, tag=None):
     if not name:
       name = self.remote.name
 
@@ -991,6 +1036,9 @@ class Project(object):
     if not self.worktree:
       cmd.append('--update-head-ok')
     cmd.append(name)
+    if tag is not None:
+      cmd.append('tag')
+      cmd.append(tag)
     return GitCommand(self,
                       cmd,
                       bare = True,
@@ -1056,13 +1104,27 @@ class Project(object):
     if not os.path.exists(hooks):
       os.makedirs(hooks)
     for stock_hook in repo_hooks():
-      dst = os.path.join(hooks, os.path.basename(stock_hook))
+      name = os.path.basename(stock_hook)
+
+      if name in ('commit-msg') and not self.remote.review:
+        # Don't install a Gerrit Code Review hook if this
+        # project does not appear to use it for reviews.
+        #
+        continue
+
+      dst = os.path.join(hooks, name)
+      if os.path.islink(dst):
+        continue
+      if os.path.exists(dst):
+        if filecmp.cmp(stock_hook, dst, shallow=False):
+          os.remove(dst)
+        else:
+          _error("%s: Not replacing %s hook", self.relpath, name)
+          continue
       try:
         os.symlink(relpath(stock_hook, dst), dst)
       except OSError, e:
-        if e.errno == errno.EEXIST:
-          pass
-        elif e.errno == errno.EPERM:
+        if e.errno == errno.EPERM:
           raise GitError('filesystem must support symlinks')
         else:
           raise

subcmds/branches.py

@@ -136,7 +136,7 @@ is shown, then the branch appears in all projects.
       hdr('%c%c %-*s' % (current, published, width, name))
       out.write(' |')
 
-      if in_cnt < project_cnt and (in_cnt == 1):
+      if in_cnt < project_cnt:
         fmt = out.write
         paths = []
         if in_cnt < project_cnt - in_cnt: 
@@ -150,15 +150,17 @@ is shown, then the branch appears in all projects.
           for b in i.projects:
             have.add(b.project)
           for p in projects:
-            paths.append(p.relpath)
+            if not p in have:
+              paths.append(p.relpath)
 
         s = ' %s %s' % (type, ', '.join(paths))
         if width + 7 + len(s) < 80:
           fmt(s)
         else:
-          out.nl()
-          fmt('    %s:' % type)
+          fmt(' %s:' % type)
           for p in paths:
             out.nl()
-            fmt('      %s' % p)
+            fmt(width*' ' + '          %s' % p)
+      else:
+        out.write(' in all projects')
       out.nl()

subcmds/grep.py

@@ -204,7 +204,7 @@ contain a line that matches both expressions:
           else:
             out.project('--- project %s ---' % project.relpath)
             out.nl()
-            out.write(p.stderr)
+            out.write("%s", p.stderr)
             out.nl()
         continue
       have_match = True
@@ -217,17 +217,17 @@ contain a line that matches both expressions:
       if have_rev and full_name:
         for line in r:
           rev, line = line.split(':', 1)
-          out.write(rev)
+          out.write("%s", rev)
           out.write(':')
           out.project(project.relpath)
           out.write('/')
-          out.write(line)
+          out.write("%s", line)
           out.nl()
       elif full_name:
         for line in r:
           out.project(project.relpath)
           out.write('/')
-          out.write(line)
+          out.write("%s", line)
           out.nl()
       else:
         for line in r:

subcmds/init.py

@@ -167,8 +167,9 @@ to update the working directory files.
 
       print ''
       print 'Your identity is: %s <%s>' % (name, email)
-      sys.stdout.write('is this correct [yes/no]? ')
-      if 'yes' == sys.stdin.readline().strip():
+      sys.stdout.write('is this correct [y/n]? ')
+      a = sys.stdin.readline().strip()
+      if a in ('yes', 'y', 't', 'true'):
         break
 
     if name != mp.UserName:

subcmds/rebase.py

@@ -0,0 +1,107 @@
+#
+# Copyright (C) 2010 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+from command import Command
+from git_command import GitCommand
+from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M
+from error import GitError
+
+class Rebase(Command):
+  common = True
+  helpSummary = "Rebase local branches on upstream branch"
+  helpUsage = """
+%prog {[<project>...] | -i <project>...}
+"""
+  helpDescription = """
+'%prog' uses git rebase to move local changes in the current topic branch to
+the HEAD of the upstream history, useful when you have made commits in a topic
+branch but need to incorporate new upstream changes "underneath" them.
+"""
+
+  def _Options(self, p):
+    p.add_option('-i', '--interactive',
+                dest="interactive", action="store_true",
+                help="interactive rebase (single project only)")
+
+    p.add_option('-f', '--force-rebase',
+                 dest='force_rebase', action='store_true',
+                 help='Pass --force-rebase to git rebase')
+    p.add_option('--no-ff',
+                 dest='no_ff', action='store_true',
+                 help='Pass --no-ff to git rebase')
+    p.add_option('-q', '--quiet',
+                 dest='quiet', action='store_true',
+                 help='Pass --quiet to git rebase')
+    p.add_option('--autosquash',
+                 dest='autosquash', action='store_true',
+                 help='Pass --autosquash to git rebase')
+    p.add_option('--whitespace',
+                 dest='whitespace', action='store', metavar='WS',
+                 help='Pass --whitespace to git rebase')
+
+  def Execute(self, opt, args):
+    all = self.GetProjects(args)
+    one_project = len(all) == 1
+
+    if opt.interactive and not one_project:
+      print >>sys.stderr, 'error: interactive rebase not supported with multiple projects'
+      return -1
+
+    for project in all:
+      cb = project.CurrentBranch
+      if not cb:
+        if one_project:
+          print >>sys.stderr, "error: project %s has a detatched HEAD" % project.relpath
+          return -1
+        # ignore branches with detatched HEADs
+        continue
+
+      upbranch = project.GetBranch(cb)
+      if not upbranch.LocalMerge:
+        if one_project:
+          print >>sys.stderr, "error: project %s does not track any remote branches" % project.relpath
+          return -1
+        # ignore branches without remotes
+        continue
+
+      args = ["rebase"]
+
+      if opt.whitespace:
+        args.append('--whitespace=%s' % opt.whitespace)
+
+      if opt.quiet:
+        args.append('--quiet')
+
+      if opt.force_rebase:
+        args.append('--force-rebase')
+
+      if opt.no_ff:
+        args.append('--no-ff')
+
+      if opt.autosquash:
+        args.append('--autosquash')
+
+      if opt.interactive:
+        args.append("-i")
+
+      args.append(upbranch.LocalMerge)
+
+      print >>sys.stderr, '# %s: rebasing %s -> %s' % \
+        (project.relpath, cb, upbranch.LocalMerge)
+
+      if GitCommand(project, args).Wait() != 0:
+        return -1

subcmds/smartsync.py

@@ -0,0 +1,33 @@
+#
+# Copyright (C) 2010 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from sync import Sync
+
+class Smartsync(Sync):
+  common = True
+  helpSummary = "Update working tree to the latest known good revision"
+  helpUsage = """
+%prog [<project>...]
+"""
+  helpDescription = """
+The '%prog' command is a shortcut for sync -s.
+"""
+
+  def _Options(self, p):
+    Sync._Options(self, p, show_smart=False)
+
+  def Execute(self, opt, args):
+    opt.smart_sync = True
+    Sync.Execute(self, opt, args)

subcmds/sync.py

@@ -17,11 +17,19 @@ from optparse import SUPPRESS_HELP
 import os
 import re
 import shutil
+import socket
 import subprocess
 import sys
 import time
+import xmlrpclib
+
+try:
+  import threading as _threading
+except ImportError:
+  import dummy_threading as _threading
 
 from git_command import GIT
+from git_refs import R_HEADS
 from project import HEAD
 from project import Project
 from project import RemoteSpec
@@ -32,6 +40,7 @@ from project import SyncBuffer
 from progress import Progress
 
 class Sync(Command, MirrorSafeCommand):
+  jobs = 1
   common = True
   helpSummary = "Update working tree to the latest revision"
   helpUsage = """
@@ -57,6 +66,10 @@ back to the manifest revision.  This option is especially helpful
 if the project is currently on a topic branch, but the manifest
 revision is temporarily needed.
 
+The -s/--smart-sync option can be used to sync to a known good
+build as specified by the manifest-server element in the current
+manifest.
+
 SSH Connections
 ---------------
 
@@ -87,7 +100,7 @@ later is required to fix a server side protocol bug.
 
 """
 
-  def _Options(self, p):
+  def _Options(self, p, show_smart=True):
     p.add_option('-l','--local-only',
                  dest='local_only', action='store_true',
                  help="only update working tree, don't fetch")
@@ -97,6 +110,13 @@ later is required to fix a server side protocol bug.
     p.add_option('-d','--detach',
                  dest='detach_head', action='store_true',
                  help='detach projects back to manifest revision')
+    p.add_option('-j','--jobs',
+                 dest='jobs', action='store', type='int',
+                 help="number of projects to fetch simultaneously")
+    if show_smart:
+      p.add_option('-s', '--smart-sync',
+                   dest='smart_sync', action='store_true',
+                   help='smart sync using manifest from a known good build')
 
     g = p.add_option_group('repo Version options')
     g.add_option('--no-repo-verify',
@@ -106,16 +126,44 @@ later is required to fix a server side protocol bug.
                  dest='repo_upgraded', action='store_true',
                  help=SUPPRESS_HELP)
 
+  def _FetchHelper(self, project, lock, fetched, pm, sem):
+      if not project.Sync_NetworkHalf():
+        print >>sys.stderr, 'error: Cannot fetch %s' % project.name
+        sem.release()
+        sys.exit(1)
+
+      lock.acquire()
+      fetched.add(project.gitdir)
+      pm.update()
+      lock.release()
+      sem.release()
+
   def _Fetch(self, projects):
     fetched = set()
     pm = Progress('Fetching projects', len(projects))
-    for project in projects:
-      pm.update()
-      if project.Sync_NetworkHalf():
-        fetched.add(project.gitdir)
-      else:
-        print >>sys.stderr, 'error: Cannot fetch %s' % project.name
-        sys.exit(1)
+
+    if self.jobs == 1:
+      for project in projects:
+        pm.update()
+        if project.Sync_NetworkHalf():
+          fetched.add(project.gitdir)
+        else:
+          print >>sys.stderr, 'error: Cannot fetch %s' % project.name
+          sys.exit(1)
+    else:
+      threads = set()
+      lock = _threading.Lock()
+      sem = _threading.Semaphore(self.jobs)
+      for project in projects:
+        sem.acquire()
+        t = _threading.Thread(target = self._FetchHelper,
+                             args = (project, lock, fetched, pm, sem))
+        threads.add(t)
+        t.start()
+
+      for t in threads:
+        t.join()
+
     pm.end()
     return fetched
 
@@ -138,32 +186,36 @@ later is required to fix a server side protocol bug.
         if not path:
           continue
         if path not in new_project_paths:
-          project = Project(
-                         manifest = self.manifest,
-                         name = path,
-                         remote = RemoteSpec('origin'),
-                         gitdir = os.path.join(self.manifest.topdir,
-                                               path, '.git'),
-                         worktree = os.path.join(self.manifest.topdir, path),
-                         relpath = path,
-                         revisionExpr = 'HEAD',
-                         revisionId = None)
-          if project.IsDirty():
-            print >>sys.stderr, 'error: Cannot remove project "%s": \
+          """If the path has already been deleted, we don't need to do it
+          """
+          if os.path.exists(self.manifest.topdir + '/' + path):
+              project = Project(
+                             manifest = self.manifest,
+                             name = path,
+                             remote = RemoteSpec('origin'),
+                             gitdir = os.path.join(self.manifest.topdir,
+                                                   path, '.git'),
+                             worktree = os.path.join(self.manifest.topdir, path),
+                             relpath = path,
+                             revisionExpr = 'HEAD',
+                             revisionId = None)
+
+              if project.IsDirty():
+                print >>sys.stderr, 'error: Cannot remove project "%s": \
 uncommitted changes are present' % project.relpath
-            print >>sys.stderr, '       commit changes, then run sync again'
-            return -1
-          else:
-            print >>sys.stderr, 'Deleting obsolete path %s' % project.worktree
-            shutil.rmtree(project.worktree)
-            # Try deleting parent subdirs if they are empty
-            dir = os.path.dirname(project.worktree)
-            while dir != self.manifest.topdir:
-              try:
-                os.rmdir(dir)
-              except OSError:
-                break
-              dir = os.path.dirname(dir)
+                print >>sys.stderr, '       commit changes, then run sync again'
+                return -1
+              else:
+                print >>sys.stderr, 'Deleting obsolete path %s' % project.worktree
+                shutil.rmtree(project.worktree)
+                # Try deleting parent subdirs if they are empty
+                dir = os.path.dirname(project.worktree)
+                while dir != self.manifest.topdir:
+                  try:
+                    os.rmdir(dir)
+                  except OSError:
+                    break
+                  dir = os.path.dirname(dir)
 
     new_project_paths.sort()
     fd = open(file_path, 'w')
@@ -175,6 +227,8 @@ uncommitted changes are present' % project.relpath
     return 0
 
   def Execute(self, opt, args):
+    if opt.jobs:
+      self.jobs = opt.jobs
     if opt.network_only and opt.detach_head:
       print >>sys.stderr, 'error: cannot combine -n and -d'
       sys.exit(1)
@@ -182,6 +236,51 @@ uncommitted changes are present' % project.relpath
       print >>sys.stderr, 'error: cannot combine -n and -l'
       sys.exit(1)
 
+    if opt.smart_sync:
+      if not self.manifest.manifest_server:
+        print >>sys.stderr, \
+            'error: cannot smart sync: no manifest server defined in manifest'
+        sys.exit(1)
+      try:
+        server = xmlrpclib.Server(self.manifest.manifest_server)
+        p = self.manifest.manifestProject
+        b = p.GetBranch(p.CurrentBranch)
+        branch = b.merge
+        if branch.startswith(R_HEADS):
+          branch = branch[len(R_HEADS):]
+
+        env = dict(os.environ)
+        if (env.has_key('TARGET_PRODUCT') and
+            env.has_key('TARGET_BUILD_VARIANT')):
+          target = '%s-%s' % (env['TARGET_PRODUCT'],
+                              env['TARGET_BUILD_VARIANT'])
+          [success, manifest_str] = server.GetApprovedManifest(branch, target)
+        else:
+          [success, manifest_str] = server.GetApprovedManifest(branch)
+
+        if success:
+          manifest_name = "smart_sync_override.xml"
+          manifest_path = os.path.join(self.manifest.manifestProject.worktree,
+                                       manifest_name)
+          try:
+            f = open(manifest_path, 'w')
+            try:
+              f.write(manifest_str)
+            finally:
+              f.close()
+          except IOError:
+            print >>sys.stderr, 'error: cannot write manifest to %s' % \
+                manifest_path
+            sys.exit(1)
+          self.manifest.Override(manifest_name)
+        else:
+          print >>sys.stderr, 'error: %s' % manifest_str
+          sys.exit(1)
+      except socket.error:
+        print >>sys.stderr, 'error: cannot connect to manifest server %s' % (
+            self.manifest.manifest_server)
+        sys.exit(1)
+
     rp = self.manifest.repoProject
     rp.PreSync()
 

subcmds/upload.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import copy
 import re
 import sys
 
@@ -20,6 +21,17 @@ from command import InteractiveCommand
 from editor import Editor
 from error import UploadError
 
+UNUSUAL_COMMIT_THRESHOLD = 5
+
+def _ConfirmManyUploads(multiple_branches=False):
+  if multiple_branches:
+    print "ATTENTION: One or more branches has an unusually high number of commits."
+  else:
+    print "ATTENTION: You are uploading an unusually high number of commits."
+  print "YOU PROBABLY DO NOT MEAN TO DO THIS. (Did you rebase across branches?)"
+  answer = raw_input("If you are sure you intend to do this, type 'yes': ").strip()
+  return answer == "yes"
+
 def _die(fmt, *args):
   msg = fmt % args
   print >>sys.stderr, 'error: %s' % msg
@@ -72,6 +84,19 @@ to "true" then repo will assume you always answer "y" at the prompt,
 and will not prompt you further.  If it is set to "false" then repo
 will assume you always answer "n", and will abort.
 
+review.URL.autocopy:
+
+To automatically copy a user or mailing list to all uploaded reviews,
+you can set a per-project or global Git option to do so. Specifically,
+review.URL.autocopy can be set to a comma separated list of reviewers
+who you always want copied on all uploads with a non-empty --re
+argument.
+
+review.URL.username:
+
+Override the username used to connect to Gerrit Code Review.
+By default the local part of the email address is used.
+
 The URL must match the review URL listed in the manifest XML file,
 or in the .git/config within the project.  For example:
 
@@ -81,6 +106,7 @@ or in the .git/config within the project.  For example:
 
   [review "http://review.example.com/"]
     autoupload = true
+    autocopy = johndoe@company.com,my-team-alias@company.com
 
 References
 ----------
@@ -90,6 +116,9 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
 """
 
   def _Options(self, p):
+    p.add_option('-t',
+                 dest='auto_topic', action='store_true',
+                 help='Send local branch name to Gerrit Code Review')
     p.add_option('--replace',
                  dest='replace', action='store_true',
                  help='Upload replacement patchesets from this branch')
@@ -100,7 +129,7 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
                  type='string',  action='append', dest='cc',
                  help='Also send email to these email addresses.')
 
-  def _SingleBranch(self, branch, people):
+  def _SingleBranch(self, opt, branch, people):
     project = branch.project
     name = branch.name
     remote = project.GetBranch(name).remote
@@ -129,11 +158,15 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
       answer = answer in ('y', 'Y', 'yes', '1', 'true', 't')
 
     if answer:
-      self._UploadAndReport([branch], people)
+      if len(branch.commits) > UNUSUAL_COMMIT_THRESHOLD:
+        answer = _ConfirmManyUploads()
+
+    if answer:
+      self._UploadAndReport(opt, [branch], people)
     else:
       _die("upload aborted by user")
 
-  def _MultipleBranches(self, pending, people):
+  def _MultipleBranches(self, opt, pending, people):
     projects = {}
     branches = {}
 
@@ -192,7 +225,30 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
         todo.append(branch)
     if not todo:
       _die("nothing uncommented for upload")
-    self._UploadAndReport(todo, people)
+
+    many_commits = False
+    for branch in todo:
+      if len(branch.commits) > UNUSUAL_COMMIT_THRESHOLD:
+        many_commits = True
+        break
+    if many_commits:
+      if not _ConfirmManyUploads(multiple_branches=True):
+        _die("upload aborted by user")
+
+    self._UploadAndReport(opt, todo, people)
+
+  def _AppendAutoCcList(self, branch, people):
+    """
+    Appends the list of users in the CC list in the git project's config if a
+    non-empty reviewer list was found.
+    """
+
+    name = branch.name
+    project = branch.project
+    key = 'review.%s.autocopy' % project.GetBranch(name).remote.review
+    raw_list = project.config.GetString(key)
+    if not raw_list is None and len(people[0]) > 0:
+      people[1].extend([entry.strip() for entry in raw_list.split(',')])
 
   def _FindGerritChange(self, branch):
     last_pub = branch.project.WasPublished(branch.name)
@@ -258,14 +314,36 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
       print >>sys.stderr, "       use 'repo upload' without --replace"
       sys.exit(1)
 
-    branch.replace_changes = to_replace
-    self._UploadAndReport([branch], people)
+    if len(branch.commits) > UNUSUAL_COMMIT_THRESHOLD:
+      if not _ConfirmManyUploads(multiple_branches=True):
+        _die("upload aborted by user")
 
-  def _UploadAndReport(self, todo, people):
+    branch.replace_changes = to_replace
+    self._UploadAndReport(opt, [branch], people)
+
+  def _UploadAndReport(self, opt, todo, original_people):
     have_errors = False
     for branch in todo:
       try:
-        branch.UploadForReview(people)
+        people = copy.deepcopy(original_people)
+        self._AppendAutoCcList(branch, people)
+
+        # Check if there are local changes that may have been forgotten
+        if branch.project.HasChanges():
+            key = 'review.%s.autoupload' % branch.project.remote.review
+            answer = branch.project.config.GetBoolean(key)
+
+            # if they want to auto upload, let's not ask because it could be automated
+            if answer is None:
+                sys.stdout.write('Uncommitted changes in ' + branch.project.name + ' (did you forget to amend?). Continue uploading? (y/n) ')
+                a = sys.stdin.readline().strip().lower()
+                if a not in ('y', 'yes', 't', 'true', 'on'):
+                    print >>sys.stderr, "skipping upload"
+                    branch.uploaded = False
+                    branch.error = 'User aborted'
+                    continue
+
+        branch.UploadForReview(people, auto_topic=opt.auto_topic)
         branch.uploaded = True
       except UploadError, e:
         branch.error = e
@@ -321,6 +399,6 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
     if not pending:
       print >>sys.stdout, "no branches ready for upload"
     elif len(pending) == 1 and len(pending[0][1]) == 1:
-      self._SingleBranch(pending[0][1][0], people)
+      self._SingleBranch(opt, pending[0][1][0], people)
     else:
-      self._MultipleBranches(pending, people)
+      self._MultipleBranches(opt, pending, people)