sync: Safely skip already deleted projects

Do not error if a project is missing on the filesystem, is deleted
from manifest.xml, but still exists in project.list.

Change-Id: I1d13e435473c83091e27e4df571504ef493282dd

docs/manifest-format.txt

@@ -22,6 +22,7 @@ following DTD:
   <!DOCTYPE manifest [
     <!ELEMENT manifest (remote*,
                         default?,
+                        manifest-server?,
                         remove-project*,
                         project*)>
   
@@ -33,6 +34,9 @@ following DTD:
     <!ELEMENT default (EMPTY)>
     <!ATTLIST default remote   IDREF #IMPLIED>
     <!ATTLIST default revision CDATA #IMPLIED>
+
+    <!ELEMENT manifest-server (EMPTY)>
+    <!ATTLIST url              CDATA #REQUIRED>
   
     <!ELEMENT project (EMPTY)>
     <!ATTLIST project name     CDATA #REQUIRED>
@@ -89,6 +93,27 @@ Attribute `revision`: Name of a Git branch (e.g. `master` or
 revision attribute will use this revision.
 
 
+Element manifest-server
+-----------------------
+
+At most one manifest-server may be specified. The url attribute
+is used to specify the URL of a manifest server, which is an
+XML RPC service that will return a manifest in which each project
+is pegged to a known good revision for the current branch and
+target.
+
+The manifest server should implement:
+
+  GetApprovedManifest(branch, target)
+
+The target to use is defined by environment variables TARGET_PRODUCT
+and TARGET_BUILD_VARIANT. These variables are used to create a string
+of the form $TARGET_PRODUCT-$TARGET_BUILD_VARIANT, e.g. passion-userdebug.
+If one of those variables or both are not present, the program will call
+GetApprovedManifest without the target paramater and the manifest server
+should choose a reasonable default target.
+
+
 Element project
 ---------------
 

git_config.py

@@ -442,8 +442,30 @@ class Remote(object):
                      self._Get('fetch', all=True))
     self._review_protocol = None
 
+  def _InsteadOf(self):
+    globCfg = GitConfig.ForUser()
+    urlList = globCfg.GetSubSections('url')
+    longest = ""
+    longestUrl = ""
+
+    for url in urlList:
+      key = "url." + url + ".insteadOf"
+      insteadOfList = globCfg.GetString(key, all=True)
+
+      for insteadOf in insteadOfList:
+        if self.url.startswith(insteadOf) \
+        and len(insteadOf) > len(longest):
+          longest = insteadOf
+          longestUrl = url
+
+    if len(longest) == 0:
+      return self.url
+
+    return self.url.replace(longest, longestUrl, 1)
+
   def PreConnectFetch(self):
-    return _preconnect(self.url)
+    connectionUrl = self._InsteadOf()
+    return _preconnect(connectionUrl)
 
   @property
   def ReviewProtocol(self):

hooks/commit-msg

@@ -0,0 +1,101 @@
+#!/bin/sh
+# From Gerrit Code Review 2.1.2-rc2-33-g7e30c72
+#
+# Part of Gerrit Code Review (http://code.google.com/p/gerrit/)
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+CHANGE_ID_AFTER="Bug|Issue"
+MSG="$1"
+
+# Check for, and add if missing, a unique Change-Id
+#
+add_ChangeId() {
+	clean_message=$(sed -e '
+		/^diff --git a\/.*/{
+			s///
+			q
+		}
+		/^Signed-off-by:/d
+		/^#/d
+	' "$MSG" | git stripspace)
+	if test -z "$clean_message"
+	then
+		return
+	fi
+
+	if grep -i '^Change-Id:' "$MSG" >/dev/null
+	then
+		return
+	fi
+
+	id=$(_gen_ChangeId)
+	perl -e '
+		$MSG = shift;
+		$id = shift;
+		$CHANGE_ID_AFTER = shift;
+
+		undef $/;
+		open(I, $MSG); $_ = <I>; close I;
+		s|^diff --git a/.*||ms;
+		s|^#.*$||mg;
+		exit unless $_;
+
+		@message = split /\n/;
+		$haveFooter = 0;
+		$startFooter = @message;
+		for($line = @message - 1; $line >= 0; $line--) {
+			$_ = $message[$line];
+
+			($haveFooter++, next) if /^[a-zA-Z0-9-]+:/;
+			next if /^[ []/;
+			$startFooter = $line if ($haveFooter && /^\r?$/);
+			last;
+		}
+
+		@footer = @message[$startFooter+1..@message];
+		@message = @message[0..$startFooter];
+		push(@footer, "") unless @footer;
+
+		for ($line = 0; $line < @footer; $line++) {
+			$_ = $footer[$line];
+			next if /^($CHANGE_ID_AFTER):/i;
+			last;
+		}
+		splice(@footer, $line, 0, "Change-Id: I$id");
+
+		$_ = join("\n", @message, @footer);
+		open(O, ">$MSG"); print O; close O;
+	' "$MSG" "$id" "$CHANGE_ID_AFTER"
+}
+_gen_ChangeIdInput() {
+	echo "tree $(git write-tree)"
+	if parent=$(git rev-parse HEAD^0 2>/dev/null)
+	then
+		echo "parent $parent"
+	fi
+	echo "author $(git var GIT_AUTHOR_IDENT)"
+	echo "committer $(git var GIT_COMMITTER_IDENT)"
+	echo
+	printf '%s' "$clean_message"
+}
+_gen_ChangeId() {
+	_gen_ChangeIdInput |
+	git hash-object -t commit --stdin
+}
+
+
+add_ChangeId

manifest_xml.py

@@ -65,8 +65,8 @@ class XmlManifest(object):
 
     self._Unload()
 
-  def Link(self, name):
-    """Update the repo metadata to use a different manifest.
+  def Override(self, name):
+    """Use a different manifest, just for the current instantiation.
     """
     path = os.path.join(self.manifestProject.worktree, name)
     if not os.path.isfile(path):
@@ -80,6 +80,11 @@ class XmlManifest(object):
     finally:
       self.manifestFile = old
 
+  def Link(self, name):
+    """Update the repo metadata to use a different manifest.
+    """
+    self.Override(name)
+
     try:
       if os.path.exists(self.manifestFile):
         os.remove(self.manifestFile)
@@ -123,6 +128,12 @@ class XmlManifest(object):
       root.appendChild(e)
       root.appendChild(doc.createTextNode(''))
 
+    if self._manifest_server:
+      e = doc.createElement('manifest-server')
+      e.setAttribute('url', self._manifest_server)
+      root.appendChild(e)
+      root.appendChild(doc.createTextNode(''))
+
     sort_projects = list(self.projects.keys())
     sort_projects.sort()
 
@@ -168,6 +179,11 @@ class XmlManifest(object):
     self._Load()
     return self._default
 
+  @property
+  def manifest_server(self):
+    self._Load()
+    return self._manifest_server
+
   @property
   def IsMirror(self):
     return self.manifestProject.config.GetBoolean('repo.mirror')
@@ -178,6 +194,7 @@ class XmlManifest(object):
     self._remotes = {}
     self._default = None
     self.branch = None
+    self._manifest_server = None
 
   def _Load(self):
     if not self._loaded:
@@ -246,6 +263,15 @@ class XmlManifest(object):
     if self._default is None:
       self._default = _Default()
 
+    for node in config.childNodes:
+      if node.nodeName == 'manifest-server':
+        url = self._reqatt(node, 'url')
+        if self._manifest_server is not None:
+            raise ManifestParseError, \
+                'duplicate manifest-server in %s' % \
+                (self.manifestFile)
+        self._manifest_server = url
+
     for node in config.childNodes:
       if node.nodeName == 'project':
         project = self._ParseProject(node)
@@ -315,7 +341,7 @@ class XmlManifest(object):
   def _ParseProject(self, node):
     """
     reads a <project> element from the manifest file
-    """ 
+    """
     name = self._reqatt(node, 'name')
 
     remote = self._get_remote(node)

project.py

@@ -706,10 +706,9 @@ class Project(object):
           # commits are not yet merged upstream.  We do not want
           # to rewrite the published commits so we punt.
           #
-          syncbuf.info(self,
-                       "branch %s is published but is now %d commits behind",
-                       branch.name,
-                       len(upstream_gain))
+          syncbuf.fail(self,
+                       "branch %s is published (but not merged) and is now %d commits behind"
+                       % (branch.name, len(upstream_gain)))
         return
       elif pub == head:
         # All published commits are merged, and thus we are a
@@ -728,7 +727,7 @@ class Project(object):
     last_mine = None
     cnt_mine = 0
     for commit in local_changes:
-      commit_id, committer_email = commit.split(' ', 2)
+      commit_id, committer_email = commit.split(' ', 1)
       if committer_email == self.UserEmail:
         last_mine = commit_id
         cnt_mine += 1
@@ -1056,13 +1055,27 @@ class Project(object):
     if not os.path.exists(hooks):
       os.makedirs(hooks)
     for stock_hook in repo_hooks():
-      dst = os.path.join(hooks, os.path.basename(stock_hook))
+      name = os.path.basename(stock_hook)
+
+      if name in ('commit-msg') and not self.remote.review:
+        # Don't install a Gerrit Code Review hook if this
+        # project does not appear to use it for reviews.
+        #
+        continue
+
+      dst = os.path.join(hooks, name)
+      if os.path.islink(dst):
+        continue
+      if os.path.exists(dst):
+        if filecmp.cmp(stock_hook, dst, shallow=False):
+          os.remove(dst)
+        else:
+          _error("%s: Not replacing %s hook", self.relpath, name)
+          continue
       try:
         os.symlink(relpath(stock_hook, dst), dst)
       except OSError, e:
-        if e.errno == errno.EEXIST:
-          pass
-        elif e.errno == errno.EPERM:
+        if e.errno == errno.EPERM:
           raise GitError('filesystem must support symlinks')
         else:
           raise
@@ -1120,7 +1133,10 @@ class Project(object):
         try:
           src = os.path.join(self.gitdir, name)
           dst = os.path.join(dotgit, name)
-          os.symlink(relpath(src, dst), dst)
+          if os.path.islink(dst) or not os.path.exists(dst):
+            os.symlink(relpath(src, dst), dst)
+          else:
+            raise GitError('cannot overwrite a local work tree')
         except OSError, e:
           if e.errno == errno.EPERM:
             raise GitError('filesystem must support symlinks')

subcmds/init.py

@@ -167,8 +167,9 @@ to update the working directory files.
 
       print ''
       print 'Your identity is: %s <%s>' % (name, email)
-      sys.stdout.write('is this correct [yes/no]? ')
-      if 'yes' == sys.stdin.readline().strip():
+      sys.stdout.write('is this correct [y/n]? ')
+      a = sys.stdin.readline().strip()
+      if a in ('yes', 'y', 't', 'true'):
         break
 
     if name != mp.UserName:

subcmds/sync.py

@@ -17,9 +17,11 @@ from optparse import SUPPRESS_HELP
 import os
 import re
 import shutil
+import socket
 import subprocess
 import sys
 import time
+import xmlrpclib
 
 from git_command import GIT
 from project import HEAD
@@ -57,6 +59,10 @@ back to the manifest revision.  This option is especially helpful
 if the project is currently on a topic branch, but the manifest
 revision is temporarily needed.
 
+The -s/--smart-sync option can be used to sync to a known good
+build as specified by the manifest-server element in the current
+manifest.
+
 SSH Connections
 ---------------
 
@@ -97,6 +103,9 @@ later is required to fix a server side protocol bug.
     p.add_option('-d','--detach',
                  dest='detach_head', action='store_true',
                  help='detach projects back to manifest revision')
+    p.add_option('-s', '--smart-sync',
+                 dest='smart_sync', action='store_true',
+                 help='smart sync using manifest from a known good build')
 
     g = p.add_option_group('repo Version options')
     g.add_option('--no-repo-verify',
@@ -111,7 +120,6 @@ later is required to fix a server side protocol bug.
     pm = Progress('Fetching projects', len(projects))
     for project in projects:
       pm.update()
-
       if project.Sync_NetworkHalf():
         fetched.add(project.gitdir)
       else:
@@ -139,32 +147,36 @@ later is required to fix a server side protocol bug.
         if not path:
           continue
         if path not in new_project_paths:
-          project = Project(
-                         manifest = self.manifest,
-                         name = path,
-                         remote = RemoteSpec('origin'),
-                         gitdir = os.path.join(self.manifest.topdir,
-                                               path, '.git'),
-                         worktree = os.path.join(self.manifest.topdir, path),
-                         relpath = path,
-                         revisionExpr = 'HEAD',
-                         revisionId = None)
-          if project.IsDirty():
-            print >>sys.stderr, 'error: Cannot remove project "%s": \
+          """If the path has already been deleted, we don't need to do it
+          """
+          if os.path.exists(self.manifest.topdir + '/' + path):
+              project = Project(
+                             manifest = self.manifest,
+                             name = path,
+                             remote = RemoteSpec('origin'),
+                             gitdir = os.path.join(self.manifest.topdir,
+                                                   path, '.git'),
+                             worktree = os.path.join(self.manifest.topdir, path),
+                             relpath = path,
+                             revisionExpr = 'HEAD',
+                             revisionId = None)
+
+              if project.IsDirty():
+                print >>sys.stderr, 'error: Cannot remove project "%s": \
 uncommitted changes are present' % project.relpath
-            print >>sys.stderr, '       commit changes, then run sync again'
-            return -1
-          else:
-            print >>sys.stderr, 'Deleting obsolete path %s' % project.worktree
-            shutil.rmtree(project.worktree)
-            # Try deleting parent subdirs if they are empty
-            dir = os.path.dirname(project.worktree)
-            while dir != self.manifest.topdir:
-              try:
-                os.rmdir(dir)
-              except OSError:
-                break
-              dir = os.path.dirname(dir)
+                print >>sys.stderr, '       commit changes, then run sync again'
+                return -1
+              else:
+                print >>sys.stderr, 'Deleting obsolete path %s' % project.worktree
+                shutil.rmtree(project.worktree)
+                # Try deleting parent subdirs if they are empty
+                dir = os.path.dirname(project.worktree)
+                while dir != self.manifest.topdir:
+                  try:
+                    os.rmdir(dir)
+                  except OSError:
+                    break
+                  dir = os.path.dirname(dir)
 
     new_project_paths.sort()
     fd = open(file_path, 'w')
@@ -183,6 +195,49 @@ uncommitted changes are present' % project.relpath
       print >>sys.stderr, 'error: cannot combine -n and -l'
       sys.exit(1)
 
+    if opt.smart_sync:
+      if not self.manifest.manifest_server:
+        print >>sys.stderr, \
+            'error: cannot smart sync: no manifest server defined in manifest'
+        sys.exit(1)
+      try:
+        server = xmlrpclib.Server(self.manifest.manifest_server)
+        p = self.manifest.manifestProject
+        b = p.GetBranch(p.CurrentBranch)
+        branch = b.merge
+
+        env = dict(os.environ)
+        if (env.has_key('TARGET_PRODUCT') and
+            env.has_key('TARGET_BUILD_VARIANT')):
+          target = '%s-%s' % (env['TARGET_PRODUCT'],
+                              env['TARGET_BUILD_VARIANT'])
+          [success, manifest_str] = server.GetApprovedManifest(branch, target)
+        else:
+          [success, manifest_str] = server.GetApprovedManifest(branch)
+
+        if success:
+          manifest_name = "smart_sync_override.xml"
+          manifest_path = os.path.join(self.manifest.manifestProject.worktree,
+                                       manifest_name)
+          try:
+            f = open(manifest_path, 'w')
+            try:
+              f.write(manifest_str)
+              self.manifest.Override(manifest_name)
+            finally:
+              f.close()
+          except IOError:
+            print >>sys.stderr, 'error: cannot write manifest to %s' % \
+                manifest_path
+            sys.exit(1)
+        else:
+          print >>sys.stderr, 'error: %s' % manifest_str
+          sys.exit(1)
+      except socket.error:
+        print >>sys.stderr, 'error: cannot connect to manifest server %s' % (
+            self.manifest.manifest_server)
+        sys.exit(1)
+
     rp = self.manifest.repoProject
     rp.PreSync()
 
@@ -192,6 +247,15 @@ uncommitted changes are present' % project.relpath
     if opt.repo_upgraded:
       _PostRepoUpgrade(self.manifest)
 
+    if not opt.local_only:
+      mp.Sync_NetworkHalf()
+
+    if mp.HasChanges:
+      syncbuf = SyncBuffer(mp.config)
+      mp.Sync_LocalHalf(syncbuf)
+      if not syncbuf.Finish():
+        sys.exit(1)
+      self.manifest._Unload()
     all = self.GetProjects(args, missing_ok=True)
 
     if not opt.local_only:
@@ -199,7 +263,6 @@ uncommitted changes are present' % project.relpath
       now = time.time()
       if (24 * 60 * 60) <= (now - rp.LastFetch):
         to_fetch.append(rp)
-      to_fetch.append(mp)
       to_fetch.extend(all)
 
       fetched = self._Fetch(to_fetch)
@@ -208,12 +271,6 @@ uncommitted changes are present' % project.relpath
         # bail out now; the rest touches the working tree
         return
 
-      if mp.HasChanges:
-        syncbuf = SyncBuffer(mp.config)
-        mp.Sync_LocalHalf(syncbuf)
-        if not syncbuf.Finish():
-          sys.exit(1)
-
         self.manifest._Unload()
         all = self.GetProjects(args, missing_ok=True)
         missing = []
@@ -241,7 +298,6 @@ uncommitted changes are present' % project.relpath
     if not syncbuf.Finish():
       sys.exit(1)
 
-
 def _PostRepoUpgrade(manifest):
   for project in manifest.projects.values():
     if project.Exists: