From 8d20116038ff78b22069dd4e993b5819775f03d1 Mon Sep 17 00:00:00 2001 From: David James Date: Fri, 11 Oct 2013 17:03:19 -0700 Subject: [PATCH] repo: Support multiple branches for the same project. It is often useful to be able to include the same project more than once, but with different branches and placed in different paths in the workspace. Add this feature. This CL adds the concept of an object directory. The object directory stores objects that can be shared amongst several working trees. For newly synced repositories, we set up the git repo now to share its objects with an object repo. Each worktree for a given repo shares objects, but has an independent set of references and branches. This ensures that repo only has to update the objects once; however the references for each worktree are updated separately. Storing the references separately is needed to ensure that commits to a branch on one worktree will not change the HEAD commits of the others. One nice side effect of sharing objects between different worktrees is that you can easily cherry-pick changes between the two worktrees without needing to fetch them. Bug: Issue 141 Change-Id: I5e2f4e1a7abb56f9d3f310fa6fd0c17019330ecd --- command.py | 28 ++++++----- manifest_xml.py | 64 +++++++++++++++++-------- project.py | 118 ++++++++++++++++++++++++++++++++-------------- subcmds/rebase.py | 3 ++ subcmds/sync.py | 54 ++++++++++++++++----- subcmds/upload.py | 4 +- 6 files changed, 191 insertions(+), 80 deletions(-) diff --git a/command.py b/command.py index 287f4d30..207ef46b 100644 --- a/command.py +++ b/command.py @@ -129,7 +129,7 @@ class Command(object): def GetProjects(self, args, missing_ok=False, submodules_ok=False): """A list of projects that match the arguments. """ - all_projects = self.manifest.projects + all_projects_list = self.manifest.projects result = [] mp = self.manifest.manifestProject @@ -140,7 +140,6 @@ class Command(object): groups = [x for x in re.split(r'[,\s]+', groups) if x] if not args: - all_projects_list = list(all_projects.values()) derived_projects = {} for project in all_projects_list: if submodules_ok or project.sync_s: @@ -152,12 +151,12 @@ class Command(object): project.MatchesGroups(groups)): result.append(project) else: - self._ResetPathToProjectMap(all_projects.values()) + self._ResetPathToProjectMap(all_projects_list) for arg in args: - project = all_projects.get(arg) + projects = self.manifest.GetProjectsWithName(arg) - if not project: + if not projects: path = os.path.abspath(arg).replace('\\', '/') project = self._GetProjectByPath(path) @@ -172,14 +171,19 @@ class Command(object): if search_again: project = self._GetProjectByPath(path) or project - if not project: - raise NoSuchProjectError(arg) - if not missing_ok and not project.Exists: - raise NoSuchProjectError(arg) - if not project.MatchesGroups(groups): - raise InvalidProjectGroupsError(arg) + if project: + projects = [project] - result.append(project) + if not projects: + raise NoSuchProjectError(arg) + + for project in projects: + if not missing_ok and not project.Exists: + raise NoSuchProjectError(arg) + if not project.MatchesGroups(groups): + raise InvalidProjectGroupsError(arg) + + result.extend(projects) def _getpath(x): return x.relpath diff --git a/manifest_xml.py b/manifest_xml.py index bdbb1d40..647e89f9 100644 --- a/manifest_xml.py +++ b/manifest_xml.py @@ -209,8 +209,9 @@ class XmlManifest(object): root.appendChild(doc.createTextNode('')) def output_projects(parent, parent_node, projects): - for p in projects: - output_project(parent, parent_node, self.projects[p]) + for project_name in projects: + for project in self._projects[project_name]: + output_project(parent, parent_node, project) def output_project(parent, parent_node, p): if not p.MatchesGroups(groups): @@ -269,13 +270,11 @@ class XmlManifest(object): e.setAttribute('sync-s', 'true') if p.subprojects: - sort_projects = list(sorted([subp.name for subp in p.subprojects])) - output_projects(p, e, sort_projects) + subprojects = set(subp.name for subp in p.subprojects) + output_projects(p, e, list(sorted(subprojects))) - sort_projects = list(sorted([key for key, value in self.projects.items() - if not value.parent])) - sort_projects.sort() - output_projects(None, root, sort_projects) + projects = set(p.name for p in self._paths.values() if not p.parent) + output_projects(None, root, list(sorted(projects))) if self._repo_hooks_project: root.appendChild(doc.createTextNode('')) @@ -287,10 +286,15 @@ class XmlManifest(object): doc.writexml(fd, '', ' ', '\n', 'UTF-8') + @property + def paths(self): + self._Load() + return self._paths + @property def projects(self): self._Load() - return self._projects + return self._paths.values() @property def remotes(self): @@ -324,6 +328,7 @@ class XmlManifest(object): def _Unload(self): self._loaded = False self._projects = {} + self._paths = {} self._remotes = {} self._default = None self._repo_hooks_project = None @@ -453,11 +458,17 @@ class XmlManifest(object): self._manifest_server = url def recursively_add_projects(project): - if self._projects.get(project.name): + projects = self._projects.setdefault(project.name, []) + if project.relpath is None: raise ManifestParseError( - 'duplicate project %s in %s' % + 'missing path for %s in %s' % (project.name, self.manifestFile)) - self._projects[project.name] = project + if project.relpath in self._paths: + raise ManifestParseError( + 'duplicate path %s in %s' % + (project.relpath, self.manifestFile)) + self._paths[project.relpath] = project + projects.append(project) for subproject in project.subprojects: recursively_add_projects(subproject) @@ -478,12 +489,18 @@ class XmlManifest(object): # Store a reference to the Project. try: - self._repo_hooks_project = self._projects[repo_hooks_project] + repo_hooks_projects = self._projects[repo_hooks_project] except KeyError: raise ManifestParseError( 'project %s not found for repo-hooks' % (repo_hooks_project)) + if len(repo_hooks_projects) != 1: + raise ManifestParseError( + 'internal error parsing repo-hooks in %s' % + (self.manifestFile)) + self._repo_hooks_project = repo_hooks_projects[0] + # Store the enabled hooks in the Project object. self._repo_hooks_project.enabled_repo_hooks = enabled_repo_hooks if node.nodeName == 'remove-project': @@ -530,11 +547,12 @@ class XmlManifest(object): name = name, remote = remote.ToRemoteSpec(name), gitdir = gitdir, + objdir = gitdir, worktree = None, relpath = None, revisionExpr = m.revisionExpr, revisionId = None) - self._projects[project.name] = project + self._projects[project.name] = [project] def _ParseRemote(self, node): """ @@ -694,9 +712,10 @@ class XmlManifest(object): groups = [x for x in re.split(r'[,\s]+', groups) if x] if parent is None: - relpath, worktree, gitdir = self.GetProjectPaths(name, path) + relpath, worktree, gitdir, objdir = self.GetProjectPaths(name, path) else: - relpath, worktree, gitdir = self.GetSubprojectPaths(parent, path) + relpath, worktree, gitdir, objdir = \ + self.GetSubprojectPaths(parent, name, path) default_groups = ['all', 'name:%s' % name, 'path:%s' % relpath] groups.extend(set(default_groups).difference(groups)) @@ -709,6 +728,7 @@ class XmlManifest(object): name = name, remote = remote.ToRemoteSpec(name), gitdir = gitdir, + objdir = objdir, worktree = worktree, relpath = relpath, revisionExpr = revisionExpr, @@ -737,10 +757,15 @@ class XmlManifest(object): if self.IsMirror: worktree = None gitdir = os.path.join(self.topdir, '%s.git' % name) + objdir = gitdir else: worktree = os.path.join(self.topdir, path).replace('\\', '/') gitdir = os.path.join(self.repodir, 'projects', '%s.git' % path) - return relpath, worktree, gitdir + objdir = os.path.join(self.repodir, 'project-objects', '%s.git' % name) + return relpath, worktree, gitdir, objdir + + def GetProjectsWithName(self, name): + return self._projects.get(name, []) def GetSubprojectName(self, parent, submodule_path): return os.path.join(parent.name, submodule_path) @@ -751,14 +776,15 @@ class XmlManifest(object): def _UnjoinRelpath(self, parent_relpath, relpath): return os.path.relpath(relpath, parent_relpath) - def GetSubprojectPaths(self, parent, path): + def GetSubprojectPaths(self, parent, name, path): relpath = self._JoinRelpath(parent.relpath, path) gitdir = os.path.join(parent.gitdir, 'subprojects', '%s.git' % path) + objdir = os.path.join(parent.gitdir, 'subproject-objects', '%s.git' % name) if self.IsMirror: worktree = None else: worktree = os.path.join(parent.worktree, path).replace('\\', '/') - return relpath, worktree, gitdir + return relpath, worktree, gitdir, objdir def _ParseCopyFile(self, project, node): src = self._reqatt(node, 'src') diff --git a/project.py b/project.py index dec21ab1..f9f1f75d 100644 --- a/project.py +++ b/project.py @@ -487,6 +487,7 @@ class Project(object): name, remote, gitdir, + objdir, worktree, relpath, revisionExpr, @@ -507,6 +508,7 @@ class Project(object): name: The `name` attribute of manifest.xml's project element. remote: RemoteSpec object specifying its remote's properties. gitdir: Absolute path of git directory. + objdir: Absolute path of directory to store git objects. worktree: Absolute path of git working tree. relpath: Relative path of git working tree to repo's top directory. revisionExpr: The `revision` attribute of manifest.xml's project element. @@ -525,6 +527,7 @@ class Project(object): self.name = name self.remote = remote self.gitdir = gitdir.replace('\\', '/') + self.objdir = objdir.replace('\\', '/') if worktree: self.worktree = worktree.replace('\\', '/') else: @@ -557,11 +560,12 @@ class Project(object): defaults = self.manifest.globalConfig) if self.worktree: - self.work_git = self._GitGetByExec(self, bare=False) + self.work_git = self._GitGetByExec(self, bare=False, gitdir=gitdir) else: self.work_git = None - self.bare_git = self._GitGetByExec(self, bare=True) + self.bare_git = self._GitGetByExec(self, bare=True, gitdir=gitdir) self.bare_ref = GitRefs(gitdir) + self.bare_objdir = self._GitGetByExec(self, bare=True, gitdir=objdir) self.dest_branch = dest_branch # This will be filled in if a project is later identified to be the @@ -1069,6 +1073,7 @@ class Project(object): """Perform only the local IO portion of the sync process. Network access is not required. """ + self._InitWorkTree() all_refs = self.bare_ref.all self.CleanPublishedCache(all_refs) revid = self.GetRevisionId(all_refs) @@ -1077,7 +1082,6 @@ class Project(object): self._FastForward(revid) self._CopyFiles() - self._InitWorkTree() head = self.work_git.GetHead() if head.startswith(R_HEADS): branch = head[len(R_HEADS):] @@ -1544,11 +1548,13 @@ class Project(object): return result for rev, path, url in self._GetSubmodules(): name = self.manifest.GetSubprojectName(self, path) - project = self.manifest.projects.get(name) + relpath, worktree, gitdir, objdir = \ + self.manifest.GetSubprojectPaths(self, name, path) + project = self.manifest.paths.get(relpath) if project: result.extend(project.GetDerivedSubprojects()) continue - relpath, worktree, gitdir = self.manifest.GetSubprojectPaths(self, path) + remote = RemoteSpec(self.remote.name, url = url, review = self.remote.review) @@ -1556,6 +1562,7 @@ class Project(object): name = name, remote = remote, gitdir = gitdir, + objdir = objdir, worktree = worktree, relpath = relpath, revisionExpr = self.revisionExpr, @@ -1905,8 +1912,17 @@ class Project(object): def _InitGitDir(self, mirror_git=None): if not os.path.exists(self.gitdir): - os.makedirs(self.gitdir) - self.bare_git.init() + + # Initialize the bare repository, which contains all of the objects. + if not os.path.exists(self.objdir): + os.makedirs(self.objdir) + self.bare_objdir.init() + + # If we have a separate directory to hold refs, initialize it as well. + if self.objdir != self.gitdir: + os.makedirs(self.gitdir) + self._ReferenceGitDir(self.objdir, self.gitdir, share_refs=False, + copy_all=True) mp = self.manifest.manifestProject ref_dir = mp.config.GetString('repo.reference') or '' @@ -2022,33 +2038,61 @@ class Project(object): msg = 'manifest set to %s' % self.revisionExpr self.bare_git.symbolic_ref('-m', msg, ref, dst) + def _ReferenceGitDir(self, gitdir, dotgit, share_refs, copy_all): + """Update |dotgit| to reference |gitdir|, using symlinks where possible. + + Args: + gitdir: The bare git repository. Must already be initialized. + dotgit: The repository you would like to initialize. + share_refs: If true, |dotgit| will store its refs under |gitdir|. + Only one work tree can store refs under a given |gitdir|. + copy_all: If true, copy all remaining files from |gitdir| -> |dotgit|. + This saves you the effort of initializing |dotgit| yourself. + """ + # These objects can be shared between several working trees. + symlink_files = ['description', 'info'] + symlink_dirs = ['hooks', 'objects', 'rr-cache', 'svn'] + if share_refs: + # These objects can only be used by a single working tree. + symlink_files += ['config', 'packed-refs'] + symlink_dirs += ['logs', 'refs'] + to_symlink = symlink_files + symlink_dirs + + to_copy = [] + if copy_all: + to_copy = os.listdir(gitdir) + + for name in set(to_copy).union(to_symlink): + try: + src = os.path.realpath(os.path.join(gitdir, name)) + dst = os.path.realpath(os.path.join(dotgit, name)) + + if os.path.lexists(dst) and not os.path.islink(dst): + raise GitError('cannot overwrite a local work tree') + + # If the source dir doesn't exist, create an empty dir. + if name in symlink_dirs and not os.path.lexists(src): + os.makedirs(src) + + if name in to_symlink: + os.symlink(os.path.relpath(src, os.path.dirname(dst)), dst) + elif copy_all and not os.path.islink(dst): + if os.path.isdir(src): + shutil.copytree(src, dst) + elif os.path.isfile(src): + shutil.copy(src, dst) + except OSError as e: + if e.errno == errno.EPERM: + raise GitError('filesystem must support symlinks') + else: + raise + def _InitWorkTree(self): dotgit = os.path.join(self.worktree, '.git') if not os.path.exists(dotgit): os.makedirs(dotgit) - - for name in ['config', - 'description', - 'hooks', - 'info', - 'logs', - 'objects', - 'packed-refs', - 'refs', - 'rr-cache', - 'svn']: - try: - src = os.path.join(self.gitdir, name) - dst = os.path.join(dotgit, name) - if os.path.islink(dst) or not os.path.exists(dst): - os.symlink(os.path.relpath(src, os.path.dirname(dst)), dst) - else: - raise GitError('cannot overwrite a local work tree') - except OSError as e: - if e.errno == errno.EPERM: - raise GitError('filesystem must support symlinks') - else: - raise + self._ReferenceGitDir(self.gitdir, dotgit, share_refs=True, + copy_all=False) _lwrite(os.path.join(dotgit, HEAD), '%s\n' % self.GetRevisionId()) @@ -2058,14 +2102,10 @@ class Project(object): if GitCommand(self, cmd).Wait() != 0: raise GitError("cannot initialize work tree") - rr_cache = os.path.join(self.gitdir, 'rr-cache') - if not os.path.exists(rr_cache): - os.makedirs(rr_cache) - self._CopyFiles() def _gitdir_path(self, path): - return os.path.join(self.gitdir, path) + return os.path.realpath(os.path.join(self.gitdir, path)) def _revlist(self, *args, **kw): a = [] @@ -2078,9 +2118,10 @@ class Project(object): return self.bare_ref.all class _GitGetByExec(object): - def __init__(self, project, bare): + def __init__(self, project, bare, gitdir): self._project = project self._bare = bare + self._gitdir = gitdir def LsOthers(self): p = GitCommand(self._project, @@ -2089,6 +2130,7 @@ class Project(object): '--others', '--exclude-standard'], bare = False, + gitdir=self._gitdir, capture_stdout = True, capture_stderr = True) if p.Wait() == 0: @@ -2104,6 +2146,7 @@ class Project(object): cmd.extend(args) p = GitCommand(self._project, cmd, + gitdir=self._gitdir, bare = False, capture_stdout = True, capture_stderr = True) @@ -2213,6 +2256,7 @@ class Project(object): p = GitCommand(self._project, cmdv, bare = self._bare, + gitdir=self._gitdir, capture_stdout = True, capture_stderr = True) r = [] @@ -2265,6 +2309,7 @@ class Project(object): p = GitCommand(self._project, cmdv, bare = self._bare, + gitdir=self._gitdir, capture_stdout = True, capture_stderr = True) if p.Wait() != 0: @@ -2398,6 +2443,7 @@ class MetaProject(Project): manifest = manifest, name = name, gitdir = gitdir, + objdir = gitdir, worktree = worktree, remote = RemoteSpec('origin'), relpath = '.repo/%s' % name, diff --git a/subcmds/rebase.py b/subcmds/rebase.py index b9a7774d..1bdc1f0b 100644 --- a/subcmds/rebase.py +++ b/subcmds/rebase.py @@ -62,6 +62,9 @@ branch but need to incorporate new upstream changes "underneath" them. if opt.interactive and not one_project: print('error: interactive rebase not supported with multiple projects', file=sys.stderr) + if len(args) == 1: + print('note: project %s is mapped to more than one path' % (args[0],), + file=sys.stderr) return -1 for project in all_projects: diff --git a/subcmds/sync.py b/subcmds/sync.py index e9d52b7b..d1a06412 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py @@ -219,9 +219,25 @@ later is required to fix a server side protocol bug. dest='repo_upgraded', action='store_true', help=SUPPRESS_HELP) - def _FetchHelper(self, opt, project, lock, fetched, pm, sem, err_event): + def _FetchProjectList(self, opt, projects, *args): """Main function of the fetch threads when jobs are > 1. + Delegates most of the work to _FetchHelper. + + Args: + opt: Program options returned from optparse. See _Options(). + projects: Projects to fetch. + *args: Remaining arguments to pass to _FetchHelper. See the + _FetchHelper docstring for details. + """ + for project in projects: + success = self._FetchHelper(opt, project, *args) + if not success and not opt.force_broken: + break + + def _FetchHelper(self, opt, project, lock, fetched, pm, sem, err_event): + """Fetch git objects for a single project. + Args: opt: Program options returned from optparse. See _Options(). project: Project object for the project to fetch. @@ -235,6 +251,9 @@ later is required to fix a server side protocol bug. can be started up. err_event: We'll set this event in the case of an error (after printing out info about the error). + + Returns: + Whether the fetch was successful. """ # We'll set to true once we've locked the lock. did_lock = False @@ -281,6 +300,8 @@ later is required to fix a server side protocol bug. lock.release() sem.release() + return success + def _Fetch(self, projects, opt): fetched = set() pm = Progress('Fetching projects', len(projects)) @@ -303,20 +324,24 @@ later is required to fix a server side protocol bug. else: sys.exit(1) else: + objdir_project_map = dict() + for project in projects: + objdir_project_map.setdefault(project.objdir, []).append(project) + threads = set() lock = _threading.Lock() sem = _threading.Semaphore(self.jobs) err_event = _threading.Event() - for project in projects: + for project_list in objdir_project_map.values(): # Check for any errors before starting any new threads. # ...we'll let existing threads finish, though. if err_event.isSet(): break sem.acquire() - t = _threading.Thread(target = self._FetchHelper, + t = _threading.Thread(target = self._FetchProjectList, args = (opt, - project, + project_list, lock, fetched, pm, @@ -342,6 +367,10 @@ later is required to fix a server side protocol bug. return fetched def _GCProjects(self, projects): + gitdirs = {} + for project in projects: + gitdirs[project.gitdir] = project.bare_git + has_dash_c = git_require((1, 7, 2)) if multiprocessing and has_dash_c: cpu_count = multiprocessing.cpu_count() @@ -350,8 +379,8 @@ later is required to fix a server side protocol bug. jobs = min(self.jobs, cpu_count) if jobs < 2: - for project in projects: - project.bare_git.gc('--auto') + for bare_git in gitdirs.values(): + bare_git.gc('--auto') return config = {'pack.threads': cpu_count / jobs if cpu_count > jobs else 1} @@ -360,10 +389,10 @@ later is required to fix a server side protocol bug. sem = _threading.Semaphore(jobs) err_event = _threading.Event() - def GC(project): + def GC(bare_git): try: try: - project.bare_git.gc('--auto', config=config) + bare_git.gc('--auto', config=config) except GitError: err_event.set() except: @@ -372,11 +401,11 @@ later is required to fix a server side protocol bug. finally: sem.release() - for project in projects: + for bare_git in gitdirs.values(): if err_event.isSet(): break sem.acquire() - t = _threading.Thread(target=GC, args=(project,)) + t = _threading.Thread(target=GC, args=(bare_git,)) t.daemon = True threads.add(t) t.start() @@ -416,12 +445,13 @@ later is required to fix a server side protocol bug. if path not in new_project_paths: # If the path has already been deleted, we don't need to do it if os.path.exists(self.manifest.topdir + '/' + path): + gitdir = os.path.join(self.manifest.topdir, path, '.git') project = Project( manifest = self.manifest, name = path, remote = RemoteSpec('origin'), - gitdir = os.path.join(self.manifest.topdir, - path, '.git'), + gitdir = gitdir, + objdir = gitdir, worktree = os.path.join(self.manifest.topdir, path), relpath = path, revisionExpr = 'HEAD', diff --git a/subcmds/upload.py b/subcmds/upload.py index 8d801e08..9ad55d79 100644 --- a/subcmds/upload.py +++ b/subcmds/upload.py @@ -431,8 +431,10 @@ Gerrit Code Review: http://code.google.com/p/gerrit/ hook = RepoHook('pre-upload', self.manifest.repo_hooks_project, self.manifest.topdir, abort_if_user_denies=True) pending_proj_names = [project.name for (project, avail) in pending] + pending_worktrees = [project.worktree for (project, avail) in pending] try: - hook.Run(opt.allow_all_hooks, project_list=pending_proj_names) + hook.Run(opt.allow_all_hooks, project_list=pending_proj_names, + worktree_list=pending_worktrees) except HookError as e: print("ERROR: %s" % str(e), file=sys.stderr) return