Merge "Don't exit with error on HTTP 403 when downloading clone bundle"

If the server returns HTTP 403 (forbidden) when attempting to
download clone bundle files, ignore it and continue, rather than
exiting with a fatal error.

Change-Id: Icf78cba0332b51b0e7b622f7c7924369b551b6f6

docs/manifest-format.txt

@@ -55,6 +55,7 @@ following DTD:
     <!ATTLIST project groups   CDATA #IMPLIED>
     <!ATTLIST project sync-c   CDATA #IMPLIED>
     <!ATTLIST project sync-s   CDATA #IMPLIED>
+    <!ATTLIST project upstream CDATA #IMPLIED>
 
     <!ELEMENT annotation (EMPTY)>
     <!ATTLIST annotation name  CDATA #REQUIRED>
@@ -122,6 +123,15 @@ Attribute `revision`: Name of a Git branch (e.g. `master` or
 `refs/heads/master`).  Project elements lacking their own
 revision attribute will use this revision.
 
+Attribute `sync_j`: Number of parallel jobs to use when synching.
+
+Attribute `sync_c`: Set to true to only sync the given Git
+branch (specified in the `revision` attribute) rather than the
+whole ref space.  Project elements lacking a sync_c element of
+their own will use this value.
+
+Attribute `sync_s`: Set to true to also sync sub-projects.
+
 
 Element manifest-server
 -----------------------
@@ -202,6 +212,16 @@ group "notdefault", it will not be automatically downloaded by repo.
 If the project has a parent element, the `name` and `path` here
 are the prefixed ones.
 
+Attribute `sync_c`: Set to true to only sync the given Git
+branch (specified in the `revision` attribute) rather than the
+whole ref space.
+
+Attribute `sync_s`: Set to true to also sync sub-projects.
+
+Attribute `upstream`: Name of the Git branch in which a sha1
+can be found.  Used when syncing a revision locked manifest in
+-c mode to avoid having to sync the entire ref space.
+
 Element annotation
 ------------------
 

manifest_xml.py

@@ -126,11 +126,11 @@ class XmlManifest(object):
     self.Override(name)
 
     try:
-      if os.path.exists(self.manifestFile):
+      if os.path.lexists(self.manifestFile):
         os.remove(self.manifestFile)
       os.symlink('manifests/%s' % name, self.manifestFile)
-    except OSError:
-      raise ManifestParseError('cannot link manifest %s' % name)
+    except OSError as e:
+      raise ManifestParseError('cannot link manifest %s: %s' % (name, str(e)))
 
   def _RemoteToXml(self, r, doc, root):
     e = doc.createElement('remote')
@@ -146,8 +146,7 @@ class XmlManifest(object):
     mp = self.manifestProject
 
     groups = mp.config.GetString('manifest.groups')
-    if not groups:
-      groups = 'all'
+    if groups:
       groups = [x for x in re.split(r'[,\s]+', groups) if x]
 
     doc = xml.dom.minidom.Document()
@@ -346,13 +345,20 @@ class XmlManifest(object):
         for local_file in sorted(os.listdir(local_dir)):
           if local_file.endswith('.xml'):
             try:
-              nodes.append(self._ParseManifestXml(local_file, self.repodir))
+              local = os.path.join(local_dir, local_file)
+              nodes.append(self._ParseManifestXml(local, self.repodir))
             except ManifestParseError as e:
               print('%s' % str(e), file=sys.stderr)
       except OSError:
         pass
 
+      try:
         self._ParseManifest(nodes)
+      except ManifestParseError as e:
+        # There was a problem parsing, unload ourselves in case they catch
+        # this error and try again later, we will show the correct error
+        self._Unload()
+        raise e
 
       if self.IsMirror:
         self._AddMetaProjectMirror(self.repoProject)

project.py

@@ -1746,6 +1746,9 @@ class Project(object):
         os.remove(tmpPath)
     if 'http_proxy' in os.environ and 'darwin' == sys.platform:
       cmd += ['--proxy', os.environ['http_proxy']]
+    cookiefile = GitConfig.ForUser().GetString('http.cookiefile')
+    if cookiefile:
+      cmd += ['--cookie', cookiefile]
     cmd += [srcUrl]
 
     if IsTrace():

repo

@@ -428,7 +428,7 @@ def _DownloadBundle(url, local, quiet):
     try:
       r = urllib.request.urlopen(url)
     except urllib.error.HTTPError as e:
-      if e.code == 404:
+      if e.code in [403, 404]:
         return False
       print('fatal: Cannot get %s' % url, file=sys.stderr)
       print('fatal: HTTP error %s' % e.code, file=sys.stderr)

subcmds/info.py

@@ -111,7 +111,7 @@ class Info(PagedCommand):
     if not self.opt.local:
       project.Sync_NetworkHalf(quiet=True, current_branch_only=True)
 
-    logTarget = R_M + self.manifest.default.revisionExpr
+    logTarget = R_M + self.manifest.manifestProject.config.GetBranch("default").merge
 
     bareTmp = project.bare_git._bare
     project.bare_git._bare = False

subcmds/init.py

@@ -180,8 +180,10 @@ to update the working directory files.
       if is_new:
         m.config.SetString('repo.mirror', 'true')
       else:
-        print('fatal: --mirror not supported on existing client',
-              file=sys.stderr)
+        print('fatal: --mirror is only supported when initializing a new '
+              'workspace.', file=sys.stderr)
+        print('Either delete the .repo folder in this workspace, or initialize '
+              'in another location.', file=sys.stderr)
         sys.exit(1)
 
     if not m.Sync_NetworkHalf(is_new=is_new):

subcmds/status.py

@@ -20,10 +20,14 @@ try:
 except ImportError:
   import dummy_threading as _threading
 
+import glob
 import itertools
+import os
 import sys
 import StringIO
 
+from color import Coloring
+
 class Status(PagedCommand):
   common = True
   helpSummary = "Show the working tree status"
@@ -39,6 +43,13 @@ is a difference between these three states.
 The -j/--jobs option can be used to run multiple status queries
 in parallel.
 
+The -o/--orphans option can be used to show objects that are in
+the working directory, but not associated with a repo project.
+This includes unmanaged top-level files and directories, but also
+includes deeper items.  For example, if dir/subdir/proj1 and
+dir/subdir/proj2 are repo projects, dir/subdir/proj3 will be shown
+if it is not known to repo.
+
 Status Display
 --------------
 
@@ -76,6 +87,9 @@ the following meanings:
     p.add_option('-j', '--jobs',
                  dest='jobs', action='store', type='int', default=2,
                  help="number of projects to check simultaneously")
+    p.add_option('-o', '--orphans',
+                 dest='orphans', action='store_true',
+                 help="include objects in working directory outside of repo projects")
 
   def _StatusHelper(self, project, clean_counter, sem, output):
     """Obtains the status for a specific project.
@@ -97,6 +111,22 @@ the following meanings:
     finally:
       sem.release()
 
+  def _FindOrphans(self, dirs, proj_dirs, proj_dirs_parents, outstring):
+    """find 'dirs' that are present in 'proj_dirs_parents' but not in 'proj_dirs'"""
+    status_header = ' --\t'
+    for item in dirs:
+      if not os.path.isdir(item):
+        outstring.write(''.join([status_header, item]))
+        continue
+      if item in proj_dirs:
+        continue
+      if item in proj_dirs_parents:
+        self._FindOrphans(glob.glob('%s/.*' % item) + \
+            glob.glob('%s/*' % item), \
+            proj_dirs, proj_dirs_parents, outstring)
+        continue
+      outstring.write(''.join([status_header, item, '/']))
+
   def Execute(self, opt, args):
     all_projects = self.GetProjects(args)
     counter = itertools.count()
@@ -130,3 +160,45 @@ the following meanings:
         output.close()
     if len(all_projects) == counter.next():
       print('nothing to commit (working directory clean)')
+
+    if opt.orphans:
+      proj_dirs = set()
+      proj_dirs_parents = set()
+      for project in self.GetProjects(None, missing_ok=True):
+        proj_dirs.add(project.relpath)
+        (head, _tail) = os.path.split(project.relpath)
+        while head != "":
+          proj_dirs_parents.add(head)
+          (head, _tail) = os.path.split(head)
+      proj_dirs.add('.repo')
+
+      class StatusColoring(Coloring):
+        def __init__(self, config):
+          Coloring.__init__(self, config, 'status')
+          self.project = self.printer('header', attr = 'bold')
+          self.untracked = self.printer('untracked', fg = 'red')
+
+      orig_path = os.getcwd()
+      try:
+        os.chdir(self.manifest.topdir)
+
+        outstring = StringIO.StringIO()
+        self._FindOrphans(glob.glob('.*') + \
+            glob.glob('*'), \
+            proj_dirs, proj_dirs_parents, outstring)
+
+        if outstring.buflist:
+          output = StatusColoring(self.manifest.globalConfig)
+          output.project('Objects not within a project (orphans)')
+          output.nl()
+          for entry in outstring.buflist:
+            output.untracked(entry)
+            output.nl()
+        else:
+          print('No orphan files or directories')
+
+        outstring.close()
+
+      finally:
+        # Restore CWD.
+        os.chdir(orig_path)