Add 'repo init --mirror' to download a complete forrest

The mirror option downloads a complete forrest (as described by the
manifest) and creates a replica of the remote repositories rather
than a client working directory.  This permits other clients to
sync off the mirror site.

A mirror can be positioned in a "DMZ", where the mirror executes
"repo sync" to obtain changes from the external upstream and
clients inside the protected zone operate off the mirror only,
and therefore do not require direct git:// access to the external
upstream repositories.

Signed-off-by: Shawn O. Pearce <sop@google.com>

git_config.py

@@ -285,12 +285,14 @@ class Remote(object):
         return True
     return False
 
-  def ResetFetch(self):
+  def ResetFetch(self, mirror=False):
     """Set the fetch refspec to its default value.
     """
-    self.fetch = [RefSpec(True,
+    if mirror:
-                          'refs/heads/*',
+      dst = 'refs/heads/*'
-                          'refs/remotes/%s/*' % self.name)]
+    else:
+      dst = 'refs/remotes/%s/*' % self.name
+    self.fetch = [RefSpec(True, 'refs/heads/*', dst)]
 
   def Save(self):
     """Save this remote to the configuration.

manifest.py

@@ -88,6 +88,10 @@ class Manifest(object):
     self._Load()
     return self._default
 
+  @property
+  def IsMirror(self):
+    return self.manifestProject.config.GetBoolean('repo.mirror')
+
   def _Unload(self):
     self._loaded = False
     self._projects = {}
@@ -114,6 +118,10 @@ class Manifest(object):
         finally:
           self.manifestFile = real
 
+      if self.IsMirror:
+        self._AddMetaProjectMirror(self.repoProject)
+        self._AddMetaProjectMirror(self.manifestProject)
+
       self._loaded = True
 
   def _ParseManifest(self, is_root_file):
@@ -157,6 +165,40 @@ class Manifest(object):
                 (project.name, self.manifestFile)
         self._projects[project.name] = project
 
+  def _AddMetaProjectMirror(self, m):
+    name = None
+    m_url = m.GetRemote(m.remote.name).url
+    if m_url.endswith('/.git'):
+      raise ManifestParseError, 'refusing to mirror %s' % m_url
+
+    if self._default and self._default.remote:
+      url = self._default.remote.fetchUrl
+      if not url.endswith('/'):
+        url += '/'
+      if m_url.startswith(url):
+        remote = self._default.remote
+        name = m_url[len(url):]
+
+    if name is None:
+      s = m_url.rindex('/') + 1
+      remote = Remote('origin', fetch = m_url[:s])
+      name = m_url[s:]
+
+    if name.endswith('.git'):
+      name = name[:-4]
+
+    if name not in self._projects:
+      m.PreSync()
+      gitdir = os.path.join(self.topdir, '%s.git' % name)
+      project = Project(manifest = self,
+                        name = name,
+                        remote = remote,
+                        gitdir = gitdir,
+                        worktree = None,
+                        relpath = None,
+                        revision = m.revision)
+      self._projects[project.name] = project
+
   def _ParseRemote(self, node):
     """
     reads a <remote> element from the manifest file
@@ -214,8 +256,13 @@ class Manifest(object):
             "project %s path cannot be absolute in %s" % \
             (name, self.manifestFile)
 
-    worktree = os.path.join(self.topdir, path)
+    if self.IsMirror:
-    gitdir = os.path.join(self.repodir, 'projects/%s.git' % path)
+      relpath = None
+      worktree = None
+      gitdir = os.path.join(self.topdir, '%s.git' % name)
+    else:
+      worktree = os.path.join(self.topdir, path)
+      gitdir = os.path.join(self.repodir, 'projects/%s.git' % path)
 
     project = Project(manifest = self,
                       name = name,
@@ -242,8 +289,10 @@ class Manifest(object):
   def _ParseCopyFile(self, project, node):
     src = self._reqatt(node, 'src')
     dest = self._reqatt(node, 'dest')
-    # src is project relative, and dest is relative to the top of the tree
+    if not self.IsMirror:
-    project.AddCopyFile(src, os.path.join(self.topdir, dest))
+      # src is project relative;
+      # dest is relative to the top of the tree
+      project.AddCopyFile(src, os.path.join(self.topdir, dest))
 
   def _get_remote(self, node):
     name = node.getAttribute('remote')

project.py

@@ -211,7 +211,10 @@ class Project(object):
                     gitdir = self.gitdir,
                     defaults =  self.manifest.globalConfig)
 
-    self.work_git = self._GitGetByExec(self, bare=False)
+    if self.worktree:
+      self.work_git = self._GitGetByExec(self, bare=False)
+    else:
+      self.work_git = None
     self.bare_git = self._GitGetByExec(self, bare=True)
 
   @property
@@ -489,14 +492,23 @@ class Project(object):
       print >>sys.stderr
       print >>sys.stderr, 'Initializing project %s ...' % self.name
       self._InitGitDir()
+
     self._InitRemote()
     for r in self.extraRemotes.values():
       if not self._RemoteFetch(r.name):
         return False
     if not self._RemoteFetch():
       return False
-    self._RepairAndroidImportErrors()
+
-    self._InitMRef()
+    if self.worktree:
+      self._RepairAndroidImportErrors()
+      self._InitMRef()
+    else:
+      self._InitMirrorHead()
+      try:
+        os.remove(os.path.join(self.gitdir, 'FETCH_HEAD'))
+      except OSError:
+        pass
     return True
 
   def PostRepoUpgrade(self):
@@ -792,9 +804,11 @@ class Project(object):
   def _RemoteFetch(self, name=None):
     if not name:
       name = self.remote.name
-    return GitCommand(self,
+    cmd = ['fetch']
-                      ['fetch', name],
+    if not self.worktree:
-                      bare = True).Wait() == 0
+      cmd.append('--update-head-ok')
+    cmd.append(name)
+    return GitCommand(self, cmd, bare = True).Wait() == 0
 
   def _Checkout(self, rev, quiet=False):
     cmd = ['checkout']
@@ -874,7 +888,10 @@ class Project(object):
       remote.url = url
       remote.review = self.remote.reviewUrl
 
-      remote.ResetFetch()
+      if self.worktree:
+        remote.ResetFetch(mirror=False)
+      else:
+        remote.ResetFetch(mirror=True)
       remote.Save()
 
     for r in self.extraRemotes.values():
@@ -897,6 +914,11 @@ class Project(object):
         dst = remote.ToLocal(self.revision)
         self.bare_git.symbolic_ref('-m', msg, ref, dst)
 
+  def _InitMirrorHead(self):
+    dst = self.GetRemote(self.remote.name).ToLocal(self.revision)
+    msg = 'manifest set to %s' % self.revision
+    self.bare_git.SetHead(dst, message=msg)
+
   def _InitWorkTree(self):
     dotgit = os.path.join(self.worktree, '.git')
     if not os.path.exists(dotgit):

repo

@@ -28,7 +28,7 @@ if __name__ == '__main__':
 del magic
 
 # increment this whenever we make important changes to this script
-VERSION = (1, 6)
+VERSION = (1, 7)
 
 # increment this if the MAINTAINER_KEYS block is modified
 KEYRING_VERSION = (1,0)
@@ -115,6 +115,9 @@ group.add_option('-b', '--manifest-branch',
 group.add_option('-m', '--manifest-name',
                  dest='manifest_name',
                  help='initial manifest file', metavar='NAME.xml')
+group.add_option('--mirror',
+                 dest='mirror', action='store_true',
+                 help='mirror the forrest')
 
 # Tool
 group = init_optparse.add_option_group('Version options')

subcmds/init.py

@@ -57,6 +57,10 @@ default.xml will be used.
     g.add_option('-m', '--manifest-name',
                  dest='manifest_name', default='default.xml',
                  help='initial manifest file', metavar='NAME.xml')
+    g.add_option('--mirror',
+                 dest='mirror', action='store_true',
+                 help='mirror the forrest')
+
 
     # Tool
     g = p.add_option_group('Version options')
@@ -112,6 +116,9 @@ default.xml will be used.
       r.ResetFetch()
       r.Save()
 
+    if opt.mirror:
+      m.config.SetString('repo.mirror', 'true')
+
     m.Sync_NetworkHalf()
     m.Sync_LocalHalf()
     m.StartBranch('default')
@@ -185,9 +192,14 @@ default.xml will be used.
     self._SyncManifest(opt)
     self._LinkManifest(opt.manifest_name)
 
-    if os.isatty(0) and os.isatty(1):
+    if os.isatty(0) and os.isatty(1) and not opt.mirror:
       self._ConfigureUser()
       self._ConfigureColor()
 
+    if opt.mirror:
+      type = 'mirror '
+    else:
+      type = ''
+
     print ''
-    print 'repo initialized in %s' % self.manifest.topdir
+    print 'repo %sinitialized in %s' % (type, self.manifest.topdir)

subcmds/sync.py

@@ -102,8 +102,9 @@ the manifest.
       self._Fetch(*missing)
 
     for project in all:
-      if not project.Sync_LocalHalf():
+      if project.worktree:
-        sys.exit(1)
+        if not project.Sync_LocalHalf():
+          sys.exit(1)
 
 
 def _VerifyTag(project):