diff --git a/command.py b/command.py index 287f4d30..207ef46b 100644 --- a/command.py +++ b/command.py @@ -129,7 +129,7 @@ class Command(object): def GetProjects(self, args, missing_ok=False, submodules_ok=False): """A list of projects that match the arguments. """ - all_projects = self.manifest.projects + all_projects_list = self.manifest.projects result = [] mp = self.manifest.manifestProject @@ -140,7 +140,6 @@ class Command(object): groups = [x for x in re.split(r'[,\s]+', groups) if x] if not args: - all_projects_list = list(all_projects.values()) derived_projects = {} for project in all_projects_list: if submodules_ok or project.sync_s: @@ -152,12 +151,12 @@ class Command(object): project.MatchesGroups(groups)): result.append(project) else: - self._ResetPathToProjectMap(all_projects.values()) + self._ResetPathToProjectMap(all_projects_list) for arg in args: - project = all_projects.get(arg) + projects = self.manifest.GetProjectsWithName(arg) - if not project: + if not projects: path = os.path.abspath(arg).replace('\\', '/') project = self._GetProjectByPath(path) @@ -172,14 +171,19 @@ class Command(object): if search_again: project = self._GetProjectByPath(path) or project - if not project: - raise NoSuchProjectError(arg) - if not missing_ok and not project.Exists: - raise NoSuchProjectError(arg) - if not project.MatchesGroups(groups): - raise InvalidProjectGroupsError(arg) + if project: + projects = [project] - result.append(project) + if not projects: + raise NoSuchProjectError(arg) + + for project in projects: + if not missing_ok and not project.Exists: + raise NoSuchProjectError(arg) + if not project.MatchesGroups(groups): + raise InvalidProjectGroupsError(arg) + + result.extend(projects) def _getpath(x): return x.relpath diff --git a/manifest_xml.py b/manifest_xml.py index 785976bc..c5f3bcc9 100644 --- a/manifest_xml.py +++ b/manifest_xml.py @@ -215,8 +215,9 @@ class XmlManifest(object): root.appendChild(doc.createTextNode('')) def output_projects(parent, parent_node, projects): - for p in projects: - output_project(parent, parent_node, self.projects[p]) + for project_name in projects: + for project in self._projects[project_name]: + output_project(parent, parent_node, project) def output_project(parent, parent_node, p): if not p.MatchesGroups(groups): @@ -277,13 +278,11 @@ class XmlManifest(object): e.setAttribute('sync-s', 'true') if p.subprojects: - sort_projects = list(sorted([subp.name for subp in p.subprojects])) - output_projects(p, e, sort_projects) + subprojects = set(subp.name for subp in p.subprojects) + output_projects(p, e, list(sorted(subprojects))) - sort_projects = list(sorted([key for key, value in self.projects.items() - if not value.parent])) - sort_projects.sort() - output_projects(None, root, sort_projects) + projects = set(p.name for p in self._paths.values() if not p.parent) + output_projects(None, root, list(sorted(projects))) if self._repo_hooks_project: root.appendChild(doc.createTextNode('')) @@ -295,10 +294,15 @@ class XmlManifest(object): doc.writexml(fd, '', ' ', '\n', 'UTF-8') + @property + def paths(self): + self._Load() + return self._paths + @property def projects(self): self._Load() - return self._projects + return self._paths.values() @property def remotes(self): @@ -336,6 +340,7 @@ class XmlManifest(object): def _Unload(self): self._loaded = False self._projects = {} + self._paths = {} self._remotes = {} self._default = None self._repo_hooks_project = None @@ -467,11 +472,17 @@ class XmlManifest(object): self._manifest_server = url def recursively_add_projects(project): - if self._projects.get(project.name): + projects = self._projects.setdefault(project.name, []) + if project.relpath is None: raise ManifestParseError( - 'duplicate project %s in %s' % + 'missing path for %s in %s' % (project.name, self.manifestFile)) - self._projects[project.name] = project + if project.relpath in self._paths: + raise ManifestParseError( + 'duplicate path %s in %s' % + (project.relpath, self.manifestFile)) + self._paths[project.relpath] = project + projects.append(project) for subproject in project.subprojects: recursively_add_projects(subproject) @@ -492,12 +503,18 @@ class XmlManifest(object): # Store a reference to the Project. try: - self._repo_hooks_project = self._projects[repo_hooks_project] + repo_hooks_projects = self._projects[repo_hooks_project] except KeyError: raise ManifestParseError( 'project %s not found for repo-hooks' % (repo_hooks_project)) + if len(repo_hooks_projects) != 1: + raise ManifestParseError( + 'internal error parsing repo-hooks in %s' % + (self.manifestFile)) + self._repo_hooks_project = repo_hooks_projects[0] + # Store the enabled hooks in the Project object. self._repo_hooks_project.enabled_repo_hooks = enabled_repo_hooks if node.nodeName == 'remove-project': @@ -544,11 +561,12 @@ class XmlManifest(object): name = name, remote = remote.ToRemoteSpec(name), gitdir = gitdir, + objdir = gitdir, worktree = None, relpath = None, revisionExpr = m.revisionExpr, revisionId = None) - self._projects[project.name] = project + self._projects[project.name] = [project] def _ParseRemote(self, node): """ @@ -708,9 +726,10 @@ class XmlManifest(object): groups = [x for x in re.split(r'[,\s]+', groups) if x] if parent is None: - relpath, worktree, gitdir = self.GetProjectPaths(name, path) + relpath, worktree, gitdir, objdir = self.GetProjectPaths(name, path) else: - relpath, worktree, gitdir = self.GetSubprojectPaths(parent, path) + relpath, worktree, gitdir, objdir = \ + self.GetSubprojectPaths(parent, name, path) default_groups = ['all', 'name:%s' % name, 'path:%s' % relpath] groups.extend(set(default_groups).difference(groups)) @@ -723,6 +742,7 @@ class XmlManifest(object): name = name, remote = remote.ToRemoteSpec(name), gitdir = gitdir, + objdir = objdir, worktree = worktree, relpath = relpath, revisionExpr = revisionExpr, @@ -751,10 +771,15 @@ class XmlManifest(object): if self.IsMirror: worktree = None gitdir = os.path.join(self.topdir, '%s.git' % name) + objdir = gitdir else: worktree = os.path.join(self.topdir, path).replace('\\', '/') gitdir = os.path.join(self.repodir, 'projects', '%s.git' % path) - return relpath, worktree, gitdir + objdir = os.path.join(self.repodir, 'project-objects', '%s.git' % name) + return relpath, worktree, gitdir, objdir + + def GetProjectsWithName(self, name): + return self._projects.get(name, []) def GetSubprojectName(self, parent, submodule_path): return os.path.join(parent.name, submodule_path) @@ -765,14 +790,15 @@ class XmlManifest(object): def _UnjoinRelpath(self, parent_relpath, relpath): return os.path.relpath(relpath, parent_relpath) - def GetSubprojectPaths(self, parent, path): + def GetSubprojectPaths(self, parent, name, path): relpath = self._JoinRelpath(parent.relpath, path) gitdir = os.path.join(parent.gitdir, 'subprojects', '%s.git' % path) + objdir = os.path.join(parent.gitdir, 'subproject-objects', '%s.git' % name) if self.IsMirror: worktree = None else: worktree = os.path.join(parent.worktree, path).replace('\\', '/') - return relpath, worktree, gitdir + return relpath, worktree, gitdir, objdir def _ParseCopyFile(self, project, node): src = self._reqatt(node, 'src') diff --git a/project.py b/project.py index eb8bad3e..46f3b8f7 100644 --- a/project.py +++ b/project.py @@ -488,6 +488,7 @@ class Project(object): name, remote, gitdir, + objdir, worktree, relpath, revisionExpr, @@ -508,6 +509,7 @@ class Project(object): name: The `name` attribute of manifest.xml's project element. remote: RemoteSpec object specifying its remote's properties. gitdir: Absolute path of git directory. + objdir: Absolute path of directory to store git objects. worktree: Absolute path of git working tree. relpath: Relative path of git working tree to repo's top directory. revisionExpr: The `revision` attribute of manifest.xml's project element. @@ -526,6 +528,7 @@ class Project(object): self.name = name self.remote = remote self.gitdir = gitdir.replace('\\', '/') + self.objdir = objdir.replace('\\', '/') if worktree: self.worktree = worktree.replace('\\', '/') else: @@ -558,11 +561,12 @@ class Project(object): defaults = self.manifest.globalConfig) if self.worktree: - self.work_git = self._GitGetByExec(self, bare=False) + self.work_git = self._GitGetByExec(self, bare=False, gitdir=gitdir) else: self.work_git = None - self.bare_git = self._GitGetByExec(self, bare=True) + self.bare_git = self._GitGetByExec(self, bare=True, gitdir=gitdir) self.bare_ref = GitRefs(gitdir) + self.bare_objdir = self._GitGetByExec(self, bare=True, gitdir=objdir) self.dest_branch = dest_branch # This will be filled in if a project is later identified to be the @@ -1117,6 +1121,7 @@ class Project(object): """Perform only the local IO portion of the sync process. Network access is not required. """ + self._InitWorkTree() all_refs = self.bare_ref.all self.CleanPublishedCache(all_refs) revid = self.GetRevisionId(all_refs) @@ -1125,7 +1130,6 @@ class Project(object): self._FastForward(revid) self._CopyFiles() - self._InitWorkTree() head = self.work_git.GetHead() if head.startswith(R_HEADS): branch = head[len(R_HEADS):] @@ -1592,11 +1596,13 @@ class Project(object): return result for rev, path, url in self._GetSubmodules(): name = self.manifest.GetSubprojectName(self, path) - project = self.manifest.projects.get(name) + relpath, worktree, gitdir, objdir = \ + self.manifest.GetSubprojectPaths(self, name, path) + project = self.manifest.paths.get(relpath) if project: result.extend(project.GetDerivedSubprojects()) continue - relpath, worktree, gitdir = self.manifest.GetSubprojectPaths(self, path) + remote = RemoteSpec(self.remote.name, url = url, review = self.remote.review) @@ -1604,6 +1610,7 @@ class Project(object): name = name, remote = remote, gitdir = gitdir, + objdir = objdir, worktree = worktree, relpath = relpath, revisionExpr = self.revisionExpr, @@ -1966,8 +1973,17 @@ class Project(object): def _InitGitDir(self, mirror_git=None): if not os.path.exists(self.gitdir): - os.makedirs(self.gitdir) - self.bare_git.init() + + # Initialize the bare repository, which contains all of the objects. + if not os.path.exists(self.objdir): + os.makedirs(self.objdir) + self.bare_objdir.init() + + # If we have a separate directory to hold refs, initialize it as well. + if self.objdir != self.gitdir: + os.makedirs(self.gitdir) + self._ReferenceGitDir(self.objdir, self.gitdir, share_refs=False, + copy_all=True) mp = self.manifest.manifestProject ref_dir = mp.config.GetString('repo.reference') or '' @@ -2083,33 +2099,61 @@ class Project(object): msg = 'manifest set to %s' % self.revisionExpr self.bare_git.symbolic_ref('-m', msg, ref, dst) + def _ReferenceGitDir(self, gitdir, dotgit, share_refs, copy_all): + """Update |dotgit| to reference |gitdir|, using symlinks where possible. + + Args: + gitdir: The bare git repository. Must already be initialized. + dotgit: The repository you would like to initialize. + share_refs: If true, |dotgit| will store its refs under |gitdir|. + Only one work tree can store refs under a given |gitdir|. + copy_all: If true, copy all remaining files from |gitdir| -> |dotgit|. + This saves you the effort of initializing |dotgit| yourself. + """ + # These objects can be shared between several working trees. + symlink_files = ['description', 'info'] + symlink_dirs = ['hooks', 'objects', 'rr-cache', 'svn'] + if share_refs: + # These objects can only be used by a single working tree. + symlink_files += ['config', 'packed-refs'] + symlink_dirs += ['logs', 'refs'] + to_symlink = symlink_files + symlink_dirs + + to_copy = [] + if copy_all: + to_copy = os.listdir(gitdir) + + for name in set(to_copy).union(to_symlink): + try: + src = os.path.realpath(os.path.join(gitdir, name)) + dst = os.path.realpath(os.path.join(dotgit, name)) + + if os.path.lexists(dst) and not os.path.islink(dst): + raise GitError('cannot overwrite a local work tree') + + # If the source dir doesn't exist, create an empty dir. + if name in symlink_dirs and not os.path.lexists(src): + os.makedirs(src) + + if name in to_symlink: + os.symlink(os.path.relpath(src, os.path.dirname(dst)), dst) + elif copy_all and not os.path.islink(dst): + if os.path.isdir(src): + shutil.copytree(src, dst) + elif os.path.isfile(src): + shutil.copy(src, dst) + except OSError as e: + if e.errno == errno.EPERM: + raise GitError('filesystem must support symlinks') + else: + raise + def _InitWorkTree(self): dotgit = os.path.join(self.worktree, '.git') if not os.path.exists(dotgit): os.makedirs(dotgit) - - for name in ['config', - 'description', - 'hooks', - 'info', - 'logs', - 'objects', - 'packed-refs', - 'refs', - 'rr-cache', - 'svn']: - try: - src = os.path.join(self.gitdir, name) - dst = os.path.join(dotgit, name) - if os.path.islink(dst) or not os.path.exists(dst): - os.symlink(os.path.relpath(src, os.path.dirname(dst)), dst) - else: - raise GitError('cannot overwrite a local work tree') - except OSError as e: - if e.errno == errno.EPERM: - raise GitError('filesystem must support symlinks') - else: - raise + self._ReferenceGitDir(self.gitdir, dotgit, share_refs=True, + copy_all=False) _lwrite(os.path.join(dotgit, HEAD), '%s\n' % self.GetRevisionId()) @@ -2119,14 +2163,10 @@ class Project(object): if GitCommand(self, cmd).Wait() != 0: raise GitError("cannot initialize work tree") - rr_cache = os.path.join(self.gitdir, 'rr-cache') - if not os.path.exists(rr_cache): - os.makedirs(rr_cache) - self._CopyFiles() def _gitdir_path(self, path): - return os.path.join(self.gitdir, path) + return os.path.realpath(os.path.join(self.gitdir, path)) def _revlist(self, *args, **kw): a = [] @@ -2139,9 +2179,10 @@ class Project(object): return self.bare_ref.all class _GitGetByExec(object): - def __init__(self, project, bare): + def __init__(self, project, bare, gitdir): self._project = project self._bare = bare + self._gitdir = gitdir def LsOthers(self): p = GitCommand(self._project, @@ -2150,6 +2191,7 @@ class Project(object): '--others', '--exclude-standard'], bare = False, + gitdir=self._gitdir, capture_stdout = True, capture_stderr = True) if p.Wait() == 0: @@ -2165,6 +2207,7 @@ class Project(object): cmd.extend(args) p = GitCommand(self._project, cmd, + gitdir=self._gitdir, bare = False, capture_stdout = True, capture_stderr = True) @@ -2274,6 +2317,7 @@ class Project(object): p = GitCommand(self._project, cmdv, bare = self._bare, + gitdir=self._gitdir, capture_stdout = True, capture_stderr = True) r = [] @@ -2326,6 +2370,7 @@ class Project(object): p = GitCommand(self._project, cmdv, bare = self._bare, + gitdir=self._gitdir, capture_stdout = True, capture_stderr = True) if p.Wait() != 0: @@ -2459,6 +2504,7 @@ class MetaProject(Project): manifest = manifest, name = name, gitdir = gitdir, + objdir = gitdir, worktree = worktree, remote = RemoteSpec('origin'), relpath = '.repo/%s' % name, diff --git a/subcmds/rebase.py b/subcmds/rebase.py index b9a7774d..1bdc1f0b 100644 --- a/subcmds/rebase.py +++ b/subcmds/rebase.py @@ -62,6 +62,9 @@ branch but need to incorporate new upstream changes "underneath" them. if opt.interactive and not one_project: print('error: interactive rebase not supported with multiple projects', file=sys.stderr) + if len(args) == 1: + print('note: project %s is mapped to more than one path' % (args[0],), + file=sys.stderr) return -1 for project in all_projects: diff --git a/subcmds/sync.py b/subcmds/sync.py index 0279ff60..5e7385db 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py @@ -219,9 +219,25 @@ later is required to fix a server side protocol bug. dest='repo_upgraded', action='store_true', help=SUPPRESS_HELP) - def _FetchHelper(self, opt, project, lock, fetched, pm, sem, err_event): + def _FetchProjectList(self, opt, projects, *args): """Main function of the fetch threads when jobs are > 1. + Delegates most of the work to _FetchHelper. + + Args: + opt: Program options returned from optparse. See _Options(). + projects: Projects to fetch. + *args: Remaining arguments to pass to _FetchHelper. See the + _FetchHelper docstring for details. + """ + for project in projects: + success = self._FetchHelper(opt, project, *args) + if not success and not opt.force_broken: + break + + def _FetchHelper(self, opt, project, lock, fetched, pm, sem, err_event): + """Fetch git objects for a single project. + Args: opt: Program options returned from optparse. See _Options(). project: Project object for the project to fetch. @@ -235,6 +251,9 @@ later is required to fix a server side protocol bug. can be started up. err_event: We'll set this event in the case of an error (after printing out info about the error). + + Returns: + Whether the fetch was successful. """ # We'll set to true once we've locked the lock. did_lock = False @@ -281,6 +300,8 @@ later is required to fix a server side protocol bug. lock.release() sem.release() + return success + def _Fetch(self, projects, opt): fetched = set() pm = Progress('Fetching projects', len(projects)) @@ -304,20 +325,24 @@ later is required to fix a server side protocol bug. else: sys.exit(1) else: + objdir_project_map = dict() + for project in projects: + objdir_project_map.setdefault(project.objdir, []).append(project) + threads = set() lock = _threading.Lock() sem = _threading.Semaphore(self.jobs) err_event = _threading.Event() - for project in projects: + for project_list in objdir_project_map.values(): # Check for any errors before starting any new threads. # ...we'll let existing threads finish, though. if err_event.isSet(): break sem.acquire() - t = _threading.Thread(target = self._FetchHelper, + t = _threading.Thread(target = self._FetchProjectList, args = (opt, - project, + project_list, lock, fetched, pm, @@ -345,6 +370,10 @@ later is required to fix a server side protocol bug. return fetched def _GCProjects(self, projects): + gitdirs = {} + for project in projects: + gitdirs[project.gitdir] = project.bare_git + has_dash_c = git_require((1, 7, 2)) if multiprocessing and has_dash_c: cpu_count = multiprocessing.cpu_count() @@ -353,8 +382,8 @@ later is required to fix a server side protocol bug. jobs = min(self.jobs, cpu_count) if jobs < 2: - for project in projects: - project.bare_git.gc('--auto') + for bare_git in gitdirs.values(): + bare_git.gc('--auto') return config = {'pack.threads': cpu_count / jobs if cpu_count > jobs else 1} @@ -363,10 +392,10 @@ later is required to fix a server side protocol bug. sem = _threading.Semaphore(jobs) err_event = _threading.Event() - def GC(project): + def GC(bare_git): try: try: - project.bare_git.gc('--auto', config=config) + bare_git.gc('--auto', config=config) except GitError: err_event.set() except: @@ -375,11 +404,11 @@ later is required to fix a server side protocol bug. finally: sem.release() - for project in projects: + for bare_git in gitdirs.values(): if err_event.isSet(): break sem.acquire() - t = _threading.Thread(target=GC, args=(project,)) + t = _threading.Thread(target=GC, args=(bare_git,)) t.daemon = True threads.add(t) t.start() @@ -419,12 +448,13 @@ later is required to fix a server side protocol bug. if path not in new_project_paths: # If the path has already been deleted, we don't need to do it if os.path.exists(self.manifest.topdir + '/' + path): + gitdir = os.path.join(self.manifest.topdir, path, '.git') project = Project( manifest = self.manifest, name = path, remote = RemoteSpec('origin'), - gitdir = os.path.join(self.manifest.topdir, - path, '.git'), + gitdir = gitdir, + objdir = gitdir, worktree = os.path.join(self.manifest.topdir, path), relpath = path, revisionExpr = 'HEAD', diff --git a/subcmds/upload.py b/subcmds/upload.py index 7f7585ae..56212408 100644 --- a/subcmds/upload.py +++ b/subcmds/upload.py @@ -432,8 +432,10 @@ Gerrit Code Review: http://code.google.com/p/gerrit/ hook = RepoHook('pre-upload', self.manifest.repo_hooks_project, self.manifest.topdir, abort_if_user_denies=True) pending_proj_names = [project.name for (project, avail) in pending] + pending_worktrees = [project.worktree for (project, avail) in pending] try: - hook.Run(opt.allow_all_hooks, project_list=pending_proj_names) + hook.Run(opt.allow_all_hooks, project_list=pending_proj_names, + worktree_list=pending_worktrees) except HookError as e: print("ERROR: %s" % str(e), file=sys.stderr) return