Fix percent done on resumed /clone.bundle

The Content-Length when resuming is the number of bytes that
remain in the file. To compute the total size as expected by
the progress meter, we must add the bytes already stored.

While we are in this method fix uses of % operator to ensure
a tuple is always supplied.

Change-Id: Ic899231b5bc0ab43b3ddb1d29845f6390e820115

docs/manifest-format.txt

@@ -32,6 +32,7 @@ following DTD:
   
     <!ELEMENT remote (EMPTY)>
     <!ATTLIST remote name         ID    #REQUIRED>
+    <!ATTLIST remote alias        CDATA #IMPLIED>
     <!ATTLIST remote fetch        CDATA #REQUIRED>
     <!ATTLIST remote review       CDATA #IMPLIED>
   
@@ -89,6 +90,12 @@ name specified here is used as the remote name in each project's
 .git/config, and is therefore automatically available to commands
 like `git fetch`, `git remote`, `git pull` and `git push`.
 
+Attribute `alias`: The alias, if specified, is used to override
+`name` to be set as the remote name in each project's .git/config.
+Its value can be duplicated while attribute `name` has to be unique
+in the manifest file. This helps each project to be able to have
+same remote name which actually points to different remote url.
+
 Attribute `fetch`: The Git URL prefix for all projects which use
 this remote.  Each project's name is appended to this prefix to
 form the actual URL used to clone the project.
@@ -171,7 +178,11 @@ the default element is used.
 
 Attribute `groups`: List of groups to which this project belongs,
 whitespace or comma separated.  All projects belong to the group
-"default".
+"default", and each project automatically belongs to a group of
+it's name:`name` and path:`path`.  E.g. for
+<project name="monkeys" path="barrel-of"/>, that project
+definition is implicitly in the following manifest groups:
+default, name:monkeys, and path:barrel-of.
 
 Element annotation
 ------------------

git_command.py

@@ -89,7 +89,7 @@ class _GitCall(object):
       if ver_str.startswith('git version '):
         _git_version = tuple(
           map(lambda x: int(x),
-            ver_str[len('git version '):].strip().split('.')[0:3]
+            ver_str[len('git version '):].strip().split('-')[0].split('.')[0:3]
           ))
       else:
         print >>sys.stderr, 'fatal: "%s" unsupported' % ver_str
@@ -147,6 +147,12 @@ class GitCommand(object):
     if ssh_proxy:
       _setenv(env, 'REPO_SSH_SOCK', ssh_sock())
       _setenv(env, 'GIT_SSH', _ssh_proxy())
+    if 'http_proxy' in env and 'darwin' == sys.platform:
+      s = "'http.proxy=%s'" % (env['http_proxy'],)
+      p = env.get('GIT_CONFIG_PARAMETERS')
+      if p is not None:
+        s = p + ' ' + s
+      _setenv(env, 'GIT_CONFIG_PARAMETERS', s)
 
     if project:
       if not cwd:
@@ -155,8 +161,6 @@ class GitCommand(object):
         gitdir = project.gitdir
 
     command = [GIT]
-    if 'http_proxy' in env and 'darwin' == sys.platform:
-      command.extend(['-c', 'http.proxy=' + env['http_proxy']])
     if bare:
       if gitdir:
         _setenv(env, GIT_DIR, gitdir)

manifest_xml.py

@@ -41,12 +41,14 @@ class _Default(object):
 class _XmlRemote(object):
   def __init__(self,
                name,
+               alias=None,
                fetch=None,
                manifestUrl=None,
                review=None):
     self.name = name
     self.fetchUrl = fetch
     self.manifestUrl = manifestUrl
+    self.remoteAlias = alias
     self.reviewUrl = review
     self.resolvedFetchUrl = self._resolveFetchUrl()
 
@@ -62,7 +64,10 @@ class _XmlRemote(object):
 
   def ToRemoteSpec(self, projectName):
     url = self.resolvedFetchUrl.rstrip('/') + '/' + projectName
-    return RemoteSpec(self.name, url, self.reviewUrl)
+    remoteName = self.name
+    if self.remoteAlias:
+      remoteName = self.remoteAlias
+    return RemoteSpec(remoteName, url, self.reviewUrl)
 
 class XmlManifest(object):
   """manages the repo configuration file"""
@@ -283,11 +288,12 @@ class XmlManifest(object):
       self.branch = b
 
       nodes = []
-      nodes.append(self._ParseManifestXml(self.manifestFile))
+      nodes.append(self._ParseManifestXml(self.manifestFile,
+                                          self.manifestProject.worktree))
 
       local = os.path.join(self.repodir, LOCAL_MANIFEST_NAME)
       if os.path.exists(local):
-        nodes.append(self._ParseManifestXml(local))
+        nodes.append(self._ParseManifestXml(local, self.repodir))
 
       self._ParseManifest(nodes)
 
@@ -297,7 +303,7 @@ class XmlManifest(object):
 
       self._loaded = True
 
-  def _ParseManifestXml(self, path):
+  def _ParseManifestXml(self, path, include_root):
     root = xml.dom.minidom.parse(path)
     if not root or not root.childNodes:
       raise ManifestParseError("no root node in %s" % (path,))
@@ -310,13 +316,13 @@ class XmlManifest(object):
     for node in config.childNodes:
         if node.nodeName == 'include':
             name = self._reqatt(node, 'name')
-            fp = os.path.join(os.path.dirname(path), name)
+            fp = os.path.join(include_root, name)
             if not os.path.isfile(fp):
                 raise ManifestParseError, \
                     "include %s doesn't exist or isn't a file" % \
                     (name,)
             try:
-                nodes.extend(self._ParseManifestXml(fp))
+                nodes.extend(self._ParseManifestXml(fp, include_root))
             # should isolate this to the exact exception, but that's
             # tricky.  actual parsing implementation may vary.
             except (KeyboardInterrupt, RuntimeError, SystemExit):
@@ -450,12 +456,15 @@ class XmlManifest(object):
     reads a <remote> element from the manifest file
     """
     name = self._reqatt(node, 'name')
+    alias = node.getAttribute('alias')
+    if alias == '':
+      alias = None
     fetch = self._reqatt(node, 'fetch')
     review = node.getAttribute('review')
     if review == '':
       review = None
     manifestUrl = self.manifestProject.config.GetString('remote.origin.url')
-    return _XmlRemote(name, fetch, manifestUrl, review)
+    return _XmlRemote(name, alias, fetch, manifestUrl, review)
 
   def _ParseDefault(self, node):
     """
@@ -565,8 +574,9 @@ class XmlManifest(object):
     if node.hasAttribute('groups'):
       groups = node.getAttribute('groups')
     groups = [x for x in re.split('[,\s]+', groups) if x]
-    if 'default' not in groups:
-      groups.append('default')
+
+    default_groups = ['default', 'name:%s' % name, 'path:%s' % path]
+    groups.extend(set(default_groups).difference(groups))
 
     if self.IsMirror:
       relpath = None

project.py

@@ -176,10 +176,11 @@ class ReviewableBranch(object):
       R_HEADS + self.name,
       '--')
 
-  def UploadForReview(self, people, auto_topic=False):
+  def UploadForReview(self, people, auto_topic=False, draft=False):
     self.project.UploadForReview(self.name,
                                  people,
-                                 auto_topic=auto_topic)
+                                 auto_topic=auto_topic,
+                                 draft=draft)
 
   def GetPublishedRefs(self):
     refs = {}
@@ -881,7 +882,8 @@ class Project(object):
 
   def UploadForReview(self, branch=None,
                       people=([],[]),
-                      auto_topic=False):
+                      auto_topic=False,
+                      draft=False):
     """Uploads the named branch for code review.
     """
     if branch is None:
@@ -920,7 +922,13 @@ class Project(object):
 
     if dest_branch.startswith(R_HEADS):
       dest_branch = dest_branch[len(R_HEADS):]
-    ref_spec = '%s:refs/for/%s' % (R_HEADS + branch.name, dest_branch)
+
+    upload_type = 'for'
+    if draft:
+      upload_type = 'drafts'
+
+    ref_spec = '%s:refs/%s/%s' % (R_HEADS + branch.name, upload_type,
+                                  dest_branch)
     if auto_topic:
       ref_spec = ref_spec + '/' + branch.name
     cmd.append(ref_spec)
@@ -1044,12 +1052,15 @@ class Project(object):
 
       if head == revid:
         # No changes; don't do anything further.
+        # Except if the head needs to be detached
         #
-        return
+        if not syncbuf.detach_head:
+          return
+      else:
+        lost = self._revlist(not_rev(revid), HEAD)
+        if lost:
+          syncbuf.info(self, "discarding %d commits", len(lost))
 
-      lost = self._revlist(not_rev(revid), HEAD)
-      if lost:
-        syncbuf.info(self, "discarding %d commits", len(lost))
       try:
         self._Checkout(revid, quiet=True)
       except GitError, e:
@@ -1553,7 +1564,7 @@ class Project(object):
       try:
         req = urllib2.Request(srcUrl)
         if pos > 0:
-          req.add_header('Range', 'bytes=%d-' % pos)
+          req.add_header('Range', 'bytes=%d-' % (pos,))
 
         try:
           r = urllib2.urlopen(req)
@@ -1572,7 +1583,7 @@ class Project(object):
               msg = e.read()
               if len(msg) > 0 and msg[-1] == '\n':
                 msg = msg[0:-1]
-              msg = ' (%s)' % msg
+              msg = ' (%s)' % (msg,)
             except:
               msg = ''
           else:
@@ -1590,7 +1601,7 @@ class Project(object):
 
       p = None
       try:
-        size = r.headers.get('content-length', 0)
+        size = pos + r.headers.get('content-length', 0)
         unit = 1 << 10
 
         if size and not quiet:
@@ -1600,7 +1611,7 @@ class Project(object):
           else:
             desc = 'KB'
           p = Progress(
-            'Downloading %s' % self.relpath,
+            'Downloading %s' % (self.relpath,),
             int(size) / unit,
             units=desc)
           if pos > 0:
@@ -2167,6 +2178,22 @@ class MetaProject(Project):
           self.revisionExpr = base
           self.revisionId = None
 
+  def MetaBranchSwitch(self, target):
+    """ Prepare MetaProject for manifest branch switch
+    """
+
+    # detach and delete manifest branch, allowing a new
+    # branch to take over
+    syncbuf = SyncBuffer(self.config, detach_head = True)
+    self.Sync_LocalHalf(syncbuf)
+    syncbuf.Finish()
+
+    return GitCommand(self,
+                        ['update-ref', '-d', 'refs/heads/default'],
+                        capture_stdout = True,
+                        capture_stderr = True).Wait() == 0
+
+
   @property
   def LastFetch(self):
     try:

subcmds/init.py

@@ -187,6 +187,9 @@ to update the working directory files.
         shutil.rmtree(m.gitdir)
       sys.exit(1)
 
+    if opt.manifest_branch:
+      m.MetaBranchSwitch(opt.manifest_branch)
+
     syncbuf = SyncBuffer(m.config)
     m.Sync_LocalHalf(syncbuf)
     syncbuf.Finish()

subcmds/overview.py

@@ -0,0 +1,80 @@
+#
+# Copyright (C) 2012 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from color import Coloring
+from command import PagedCommand
+
+
+class Overview(PagedCommand):
+  common = True
+  helpSummary = "Display overview of unmerged project branches"
+  helpUsage = """
+%prog [--current-branch] [<project>...]
+"""
+  helpDescription = """
+The '%prog' command is used to display an overview of the projects branches,
+and list any local commits that have not yet been merged into the project.
+
+The -b/--current-branch option can be used to restrict the output to only
+branches currently checked out in each project.  By default, all branches
+are displayed.
+"""
+
+  def _Options(self, p):
+    p.add_option('-b', '--current-branch',
+                 dest="current_branch", action="store_true",
+                 help="Consider only checked out branches")
+
+  def Execute(self, opt, args):
+    all = []
+    for project in self.GetProjects(args):
+      br = [project.GetUploadableBranch(x)
+            for x in project.GetBranches().keys()]
+      br = [x for x in br if x]
+      if opt.current_branch:
+        br = [x for x in br if x.name == project.CurrentBranch]
+      all.extend(br)
+
+    if not all:
+      return
+
+    class Report(Coloring):
+      def __init__(self, config):
+        Coloring.__init__(self, config, 'status')
+        self.project = self.printer('header', attr='bold')
+
+    out = Report(all[0].project.config)
+    out.project('Projects Overview')
+    out.nl()
+
+    project = None
+
+    for branch in all:
+      if project != branch.project:
+        project = branch.project
+        out.nl()
+        out.project('project %s/' % project.relpath)
+        out.nl()
+
+      commits = branch.commits
+      date = branch.date
+      print '%s %-33s (%2d commit%s, %s)' % (
+            branch.name == project.CurrentBranch and '*' or ' ',
+            branch.name,
+            len(commits),
+            len(commits) != 1 and 's' or ' ',
+            date)
+      for commit in commits:
+        print '%-35s   - %s' % ('', commit)

subcmds/rebase.py

@@ -52,6 +52,9 @@ branch but need to incorporate new upstream changes "underneath" them.
     p.add_option('--whitespace',
                  dest='whitespace', action='store', metavar='WS',
                  help='Pass --whitespace to git rebase')
+    p.add_option('--auto-stash',
+                 dest='auto_stash', action='store_true',
+                 help='Stash local modifications before starting')
 
   def Execute(self, opt, args):
     all = self.GetProjects(args)
@@ -103,5 +106,23 @@ branch but need to incorporate new upstream changes "underneath" them.
       print >>sys.stderr, '# %s: rebasing %s -> %s' % \
         (project.relpath, cb, upbranch.LocalMerge)
 
+      needs_stash = False
+      if opt.auto_stash:
+        stash_args = ["update-index", "--refresh", "-q"]
+
+        if GitCommand(project, stash_args).Wait() != 0:
+          needs_stash = True
+          # Dirty index, requires stash...
+          stash_args = ["stash"]
+
+          if GitCommand(project, stash_args).Wait() != 0:
+            return -1
+
       if GitCommand(project, args).Wait() != 0:
         return -1
+
+      if needs_stash:
+        stash_args.append('pop')
+        stash_args.append('--quiet')
+        if GitCommand(project, stash_args).Wait() != 0:
+          return -1

subcmds/start.py

@@ -15,6 +15,7 @@
 
 import sys
 from command import Command
+from git_config import IsId
 from git_command import git
 from progress import Progress
 
@@ -56,6 +57,10 @@ revision specified in the manifest.
     pm = Progress('Starting %s' % nb, len(all))
     for project in all:
       pm.update()
+      # If the current revision is a specific SHA1 then we can't push back
+      # to it so substitute the manifest default revision instead.
+      if IsId(project.revisionExpr):
+        project.revisionExpr = self.manifest.default.revisionExpr
       if not project.StartBranch(nb):
         err.append(project)
     pm.end()

subcmds/upload.py

@@ -134,6 +134,9 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
     p.add_option('--cbr', '--current-branch',
                  dest='current_branch', action='store_true',
                  help='Upload current git branch.')
+    p.add_option('-d', '--draft',
+                 action='store_true', dest='draft', default=False,
+                 help='If specified, upload as a draft.')
 
     # Options relating to upload hook.  Note that verify and no-verify are NOT
     # opposites of each other, which is why they store to different locations.
@@ -324,7 +327,7 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
            key = 'review.%s.uploadtopic' % branch.project.remote.review
            opt.auto_topic = branch.project.config.GetBoolean(key)
 
-        branch.UploadForReview(people, auto_topic=opt.auto_topic)
+        branch.UploadForReview(people, auto_topic=opt.auto_topic, draft=opt.draft)
         branch.uploaded = True
       except UploadError, e:
         branch.error = e