android/git-repo
1
Fork 0
mirror of https://gerrit.googlesource.com/git-repo synced 2024-12-21 07:16:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

repo: Support multiple branches for the same project.

Browse Source 
It is often useful to be able to include the same project more than
once, but with different branches and placed in different paths in the
workspace. Add this feature.

This CL adds the concept of an object directory. The object directory
stores objects that can be shared amongst several working trees. For
newly synced repositories, we set up the git repo now to share its
objects with an object repo.

Each worktree for a given repo shares objects, but has an independent
set of references and branches. This ensures that repo only has to
update the objects once; however the references for each worktree are
updated separately. Storing the references separately is needed to
ensure that commits to a branch on one worktree will not change the
HEAD commits of the others.

One nice side effect of sharing objects between different worktrees is
that you can easily cherry-pick changes between the two worktrees
without needing to fetch them.

Bug: Issue 141
Change-Id: I5e2f4e1a7abb56f9d3f310fa6fd0c17019330ecd
This commit is contained in:
David James 2013-10-11 17:03:19 -07:00
parent b25ea555c3
commit 8d20116038
6 changed files with 191 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
command.py
View File

@ -129,7 +129,7 @@ class Command(object):
def GetProjects(self, args, missing_ok=False, submodules_ok=False): def GetProjects(self, args, missing_ok=False, submodules_ok=False):
"""A list of projects that match the arguments. """A list of projects that match the arguments.
""" """
all_projects = self.manifest.projects all_projects_list = self.manifest.projects
result = [] result = []
mp = self.manifest.manifestProject mp = self.manifest.manifestProject
@ -140,7 +140,6 @@ class Command(object):
groups = [x for x in re.split(r'[,\s]+', groups) if x] groups = [x for x in re.split(r'[,\s]+', groups) if x]
if not args: if not args:
all_projects_list = list(all_projects.values())
derived_projects = {} derived_projects = {}
for project in all_projects_list: for project in all_projects_list:
if submodules_ok or project.sync_s: if submodules_ok or project.sync_s:
@ -152,12 +151,12 @@ class Command(object):
project.MatchesGroups(groups)): project.MatchesGroups(groups)):
result.append(project) result.append(project)
else: else:
self._ResetPathToProjectMap(all_projects.values()) self._ResetPathToProjectMap(all_projects_list)
for arg in args: for arg in args:
project = all_projects.get(arg) projects = self.manifest.GetProjectsWithName(arg)
if not project: if not projects:
path = os.path.abspath(arg).replace('\\', '/') path = os.path.abspath(arg).replace('\\', '/')
project = self._GetProjectByPath(path) project = self._GetProjectByPath(path)
@ -172,14 +171,19 @@ class Command(object):
if search_again: if search_again:
project = self._GetProjectByPath(path) or project project = self._GetProjectByPath(path) or project
if not project: if project:
projects = [project]
if not projects:
raise NoSuchProjectError(arg) raise NoSuchProjectError(arg)
for project in projects:
if not missing_ok and not project.Exists: if not missing_ok and not project.Exists:
raise NoSuchProjectError(arg) raise NoSuchProjectError(arg)
if not project.MatchesGroups(groups): if not project.MatchesGroups(groups):
raise InvalidProjectGroupsError(arg) raise InvalidProjectGroupsError(arg)
result.append(project) result.extend(projects)
def _getpath(x): def _getpath(x):
return x.relpath return x.relpath

64
manifest_xml.py
View File

@ -209,8 +209,9 @@ class XmlManifest(object):
root.appendChild(doc.createTextNode('')) root.appendChild(doc.createTextNode(''))
def output_projects(parent, parent_node, projects): def output_projects(parent, parent_node, projects):
for p in projects: for project_name in projects:
output_project(parent, parent_node, self.projects[p]) for project in self._projects[project_name]:
output_project(parent, parent_node, project)
def output_project(parent, parent_node, p): def output_project(parent, parent_node, p):
if not p.MatchesGroups(groups): if not p.MatchesGroups(groups):
@ -269,13 +270,11 @@ class XmlManifest(object):
e.setAttribute('sync-s', 'true') e.setAttribute('sync-s', 'true')
if p.subprojects: if p.subprojects:
sort_projects = list(sorted([subp.name for subp in p.subprojects])) subprojects = set(subp.name for subp in p.subprojects)
output_projects(p, e, sort_projects) output_projects(p, e, list(sorted(subprojects)))
sort_projects = list(sorted([key for key, value in self.projects.items() projects = set(p.name for p in self._paths.values() if not p.parent)
if not value.parent])) output_projects(None, root, list(sorted(projects)))
sort_projects.sort()
output_projects(None, root, sort_projects)
if self._repo_hooks_project: if self._repo_hooks_project:
root.appendChild(doc.createTextNode('')) root.appendChild(doc.createTextNode(''))
@ -287,10 +286,15 @@ class XmlManifest(object):
doc.writexml(fd, '', ' ', '\n', 'UTF-8') doc.writexml(fd, '', ' ', '\n', 'UTF-8')
@property
def paths(self):
self._Load()
return self._paths
@property @property
def projects(self): def projects(self):
self._Load() self._Load()
return self._projects return self._paths.values()
@property @property
def remotes(self): def remotes(self):
@ -324,6 +328,7 @@ class XmlManifest(object):
def _Unload(self): def _Unload(self):
self._loaded = False self._loaded = False
self._projects = {} self._projects = {}
self._paths = {}
self._remotes = {} self._remotes = {}
self._default = None self._default = None
self._repo_hooks_project = None self._repo_hooks_project = None
@ -453,11 +458,17 @@ class XmlManifest(object):
self._manifest_server = url self._manifest_server = url
def recursively_add_projects(project): def recursively_add_projects(project):
if self._projects.get(project.name): projects = self._projects.setdefault(project.name, [])
if project.relpath is None:
raise ManifestParseError( raise ManifestParseError(
'duplicate project %s in %s' % 'missing path for %s in %s' %
(project.name, self.manifestFile)) (project.name, self.manifestFile))
self._projects[project.name] = project if project.relpath in self._paths:
raise ManifestParseError(
'duplicate path %s in %s' %
(project.relpath, self.manifestFile))
self._paths[project.relpath] = project
projects.append(project)
for subproject in project.subprojects: for subproject in project.subprojects:
recursively_add_projects(subproject) recursively_add_projects(subproject)
@ -478,12 +489,18 @@ class XmlManifest(object):
# Store a reference to the Project. # Store a reference to the Project.
try: try:
self._repo_hooks_project = self._projects[repo_hooks_project] repo_hooks_projects = self._projects[repo_hooks_project]
except KeyError: except KeyError:
raise ManifestParseError( raise ManifestParseError(
'project %s not found for repo-hooks' % 'project %s not found for repo-hooks' %
(repo_hooks_project)) (repo_hooks_project))
if len(repo_hooks_projects) != 1:
raise ManifestParseError(
'internal error parsing repo-hooks in %s' %
(self.manifestFile))
self._repo_hooks_project = repo_hooks_projects[0]
# Store the enabled hooks in the Project object. # Store the enabled hooks in the Project object.
self._repo_hooks_project.enabled_repo_hooks = enabled_repo_hooks self._repo_hooks_project.enabled_repo_hooks = enabled_repo_hooks
if node.nodeName == 'remove-project': if node.nodeName == 'remove-project':
@ -530,11 +547,12 @@ class XmlManifest(object):
name = name, name = name,
remote = remote.ToRemoteSpec(name), remote = remote.ToRemoteSpec(name),
gitdir = gitdir, gitdir = gitdir,
objdir = gitdir,
worktree = None, worktree = None,
relpath = None, relpath = None,
revisionExpr = m.revisionExpr, revisionExpr = m.revisionExpr,
revisionId = None) revisionId = None)
self._projects[project.name] = project self._projects[project.name] = [project]
def _ParseRemote(self, node): def _ParseRemote(self, node):
""" """
@ -694,9 +712,10 @@ class XmlManifest(object):
groups = [x for x in re.split(r'[,\s]+', groups) if x] groups = [x for x in re.split(r'[,\s]+', groups) if x]
if parent is None: if parent is None:
relpath, worktree, gitdir = self.GetProjectPaths(name, path) relpath, worktree, gitdir, objdir = self.GetProjectPaths(name, path)
else: else:
relpath, worktree, gitdir = self.GetSubprojectPaths(parent, path) relpath, worktree, gitdir, objdir = \
self.GetSubprojectPaths(parent, name, path)
default_groups = ['all', 'name:%s' % name, 'path:%s' % relpath] default_groups = ['all', 'name:%s' % name, 'path:%s' % relpath]
groups.extend(set(default_groups).difference(groups)) groups.extend(set(default_groups).difference(groups))
@ -709,6 +728,7 @@ class XmlManifest(object):
name = name, name = name,
remote = remote.ToRemoteSpec(name), remote = remote.ToRemoteSpec(name),
gitdir = gitdir, gitdir = gitdir,
objdir = objdir,
worktree = worktree, worktree = worktree,
relpath = relpath, relpath = relpath,
revisionExpr = revisionExpr, revisionExpr = revisionExpr,
@ -737,10 +757,15 @@ class XmlManifest(object):
if self.IsMirror: if self.IsMirror:
worktree = None worktree = None
gitdir = os.path.join(self.topdir, '%s.git' % name) gitdir = os.path.join(self.topdir, '%s.git' % name)
objdir = gitdir
else: else:
worktree = os.path.join(self.topdir, path).replace('\\', '/') worktree = os.path.join(self.topdir, path).replace('\\', '/')
gitdir = os.path.join(self.repodir, 'projects', '%s.git' % path) gitdir = os.path.join(self.repodir, 'projects', '%s.git' % path)
return relpath, worktree, gitdir objdir = os.path.join(self.repodir, 'project-objects', '%s.git' % name)
return relpath, worktree, gitdir, objdir
def GetProjectsWithName(self, name):
return self._projects.get(name, [])
def GetSubprojectName(self, parent, submodule_path): def GetSubprojectName(self, parent, submodule_path):
return os.path.join(parent.name, submodule_path) return os.path.join(parent.name, submodule_path)
@ -751,14 +776,15 @@ class XmlManifest(object):
def _UnjoinRelpath(self, parent_relpath, relpath): def _UnjoinRelpath(self, parent_relpath, relpath):
return os.path.relpath(relpath, parent_relpath) return os.path.relpath(relpath, parent_relpath)
def GetSubprojectPaths(self, parent, path): def GetSubprojectPaths(self, parent, name, path):
relpath = self._JoinRelpath(parent.relpath, path) relpath = self._JoinRelpath(parent.relpath, path)
gitdir = os.path.join(parent.gitdir, 'subprojects', '%s.git' % path) gitdir = os.path.join(parent.gitdir, 'subprojects', '%s.git' % path)
objdir = os.path.join(parent.gitdir, 'subproject-objects', '%s.git' % name)
if self.IsMirror: if self.IsMirror:
worktree = None worktree = None
else: else:
worktree = os.path.join(parent.worktree, path).replace('\\', '/') worktree = os.path.join(parent.worktree, path).replace('\\', '/')
return relpath, worktree, gitdir return relpath, worktree, gitdir, objdir
def _ParseCopyFile(self, project, node): def _ParseCopyFile(self, project, node):
src = self._reqatt(node, 'src') src = self._reqatt(node, 'src')

108
project.py
View File

@ -487,6 +487,7 @@ class Project(object):
name, name,
remote, remote,
gitdir, gitdir,
objdir,
worktree, worktree,
relpath, relpath,
revisionExpr, revisionExpr,
@ -507,6 +508,7 @@ class Project(object):
name: The `name` attribute of manifest.xml's project element. name: The `name` attribute of manifest.xml's project element.
remote: RemoteSpec object specifying its remote's properties. remote: RemoteSpec object specifying its remote's properties.
gitdir: Absolute path of git directory. gitdir: Absolute path of git directory.
objdir: Absolute path of directory to store git objects.
worktree: Absolute path of git working tree. worktree: Absolute path of git working tree.
relpath: Relative path of git working tree to repo's top directory. relpath: Relative path of git working tree to repo's top directory.
revisionExpr: The `revision` attribute of manifest.xml's project element. revisionExpr: The `revision` attribute of manifest.xml's project element.
@ -525,6 +527,7 @@ class Project(object):
self.name = name self.name = name
self.remote = remote self.remote = remote
self.gitdir = gitdir.replace('\\', '/') self.gitdir = gitdir.replace('\\', '/')
self.objdir = objdir.replace('\\', '/')
if worktree: if worktree:
self.worktree = worktree.replace('\\', '/') self.worktree = worktree.replace('\\', '/')
else: else:
@ -557,11 +560,12 @@ class Project(object):
defaults = self.manifest.globalConfig) defaults = self.manifest.globalConfig)
if self.worktree: if self.worktree:
self.work_git = self._GitGetByExec(self, bare=False) self.work_git = self._GitGetByExec(self, bare=False, gitdir=gitdir)
else: else:
self.work_git = None self.work_git = None
self.bare_git = self._GitGetByExec(self, bare=True) self.bare_git = self._GitGetByExec(self, bare=True, gitdir=gitdir)
self.bare_ref = GitRefs(gitdir) self.bare_ref = GitRefs(gitdir)
self.bare_objdir = self._GitGetByExec(self, bare=True, gitdir=objdir)
self.dest_branch = dest_branch self.dest_branch = dest_branch
# This will be filled in if a project is later identified to be the # This will be filled in if a project is later identified to be the
@ -1069,6 +1073,7 @@ class Project(object):
"""Perform only the local IO portion of the sync process. """Perform only the local IO portion of the sync process.
Network access is not required. Network access is not required.
""" """
self._InitWorkTree()
all_refs = self.bare_ref.all all_refs = self.bare_ref.all
self.CleanPublishedCache(all_refs) self.CleanPublishedCache(all_refs)
revid = self.GetRevisionId(all_refs) revid = self.GetRevisionId(all_refs)
@ -1077,7 +1082,6 @@ class Project(object):
self._FastForward(revid) self._FastForward(revid)
self._CopyFiles() self._CopyFiles()
self._InitWorkTree()
head = self.work_git.GetHead() head = self.work_git.GetHead()
if head.startswith(R_HEADS): if head.startswith(R_HEADS):
branch = head[len(R_HEADS):] branch = head[len(R_HEADS):]
@ -1544,11 +1548,13 @@ class Project(object):
return result return result
for rev, path, url in self._GetSubmodules(): for rev, path, url in self._GetSubmodules():
name = self.manifest.GetSubprojectName(self, path) name = self.manifest.GetSubprojectName(self, path)
project = self.manifest.projects.get(name) relpath, worktree, gitdir, objdir = \
self.manifest.GetSubprojectPaths(self, name, path)
project = self.manifest.paths.get(relpath)
if project: if project:
result.extend(project.GetDerivedSubprojects()) result.extend(project.GetDerivedSubprojects())
continue continue
relpath, worktree, gitdir = self.manifest.GetSubprojectPaths(self, path)
remote = RemoteSpec(self.remote.name, remote = RemoteSpec(self.remote.name,
url = url, url = url,
review = self.remote.review) review = self.remote.review)
@ -1556,6 +1562,7 @@ class Project(object):
name = name, name = name,
remote = remote, remote = remote,
gitdir = gitdir, gitdir = gitdir,
objdir = objdir,
worktree = worktree, worktree = worktree,
relpath = relpath, relpath = relpath,
revisionExpr = self.revisionExpr, revisionExpr = self.revisionExpr,
@ -1905,8 +1912,17 @@ class Project(object):
def _InitGitDir(self, mirror_git=None): def _InitGitDir(self, mirror_git=None):
if not os.path.exists(self.gitdir): if not os.path.exists(self.gitdir):
# Initialize the bare repository, which contains all of the objects.
if not os.path.exists(self.objdir):
os.makedirs(self.objdir)
self.bare_objdir.init()
# If we have a separate directory to hold refs, initialize it as well.
if self.objdir != self.gitdir:
os.makedirs(self.gitdir) os.makedirs(self.gitdir)
self.bare_git.init() self._ReferenceGitDir(self.objdir, self.gitdir, share_refs=False,
copy_all=True)
mp = self.manifest.manifestProject mp = self.manifest.manifestProject
ref_dir = mp.config.GetString('repo.reference') or '' ref_dir = mp.config.GetString('repo.reference') or ''
@ -2022,34 +2038,62 @@ class Project(object):
msg = 'manifest set to %s' % self.revisionExpr msg = 'manifest set to %s' % self.revisionExpr
self.bare_git.symbolic_ref('-m', msg, ref, dst) self.bare_git.symbolic_ref('-m', msg, ref, dst)
def _InitWorkTree(self): def _ReferenceGitDir(self, gitdir, dotgit, share_refs, copy_all):
dotgit = os.path.join(self.worktree, '.git') """Update |dotgit| to reference |gitdir|, using symlinks where possible.
if not os.path.exists(dotgit):
os.makedirs(dotgit)
for name in ['config', Args:
'description', gitdir: The bare git repository. Must already be initialized.
'hooks', dotgit: The repository you would like to initialize.
'info', share_refs: If true, |dotgit| will store its refs under |gitdir|.
'logs', Only one work tree can store refs under a given |gitdir|.
'objects', copy_all: If true, copy all remaining files from |gitdir| -> |dotgit|.
'packed-refs', This saves you the effort of initializing |dotgit| yourself.
'refs', """
'rr-cache', # These objects can be shared between several working trees.
'svn']: symlink_files = ['description', 'info']
symlink_dirs = ['hooks', 'objects', 'rr-cache', 'svn']
if share_refs:
# These objects can only be used by a single working tree.
symlink_files += ['config', 'packed-refs']
symlink_dirs += ['logs', 'refs']
to_symlink = symlink_files + symlink_dirs
to_copy = []
if copy_all:
to_copy = os.listdir(gitdir)
for name in set(to_copy).union(to_symlink):
try: try:
src = os.path.join(self.gitdir, name) src = os.path.realpath(os.path.join(gitdir, name))
dst = os.path.join(dotgit, name) dst = os.path.realpath(os.path.join(dotgit, name))
if os.path.islink(dst) or not os.path.exists(dst):
os.symlink(os.path.relpath(src, os.path.dirname(dst)), dst) if os.path.lexists(dst) and not os.path.islink(dst):
else:
raise GitError('cannot overwrite a local work tree') raise GitError('cannot overwrite a local work tree')
# If the source dir doesn't exist, create an empty dir.
if name in symlink_dirs and not os.path.lexists(src):
os.makedirs(src)
if name in to_symlink:
os.symlink(os.path.relpath(src, os.path.dirname(dst)), dst)
elif copy_all and not os.path.islink(dst):
if os.path.isdir(src):
shutil.copytree(src, dst)
elif os.path.isfile(src):
shutil.copy(src, dst)
except OSError as e: except OSError as e:
if e.errno == errno.EPERM: if e.errno == errno.EPERM:
raise GitError('filesystem must support symlinks') raise GitError('filesystem must support symlinks')
else: else:
raise raise
def _InitWorkTree(self):
dotgit = os.path.join(self.worktree, '.git')
if not os.path.exists(dotgit):
os.makedirs(dotgit)
self._ReferenceGitDir(self.gitdir, dotgit, share_refs=True,
copy_all=False)
_lwrite(os.path.join(dotgit, HEAD), '%s\n' % self.GetRevisionId()) _lwrite(os.path.join(dotgit, HEAD), '%s\n' % self.GetRevisionId())
cmd = ['read-tree', '--reset', '-u'] cmd = ['read-tree', '--reset', '-u']
@ -2058,14 +2102,10 @@ class Project(object):
if GitCommand(self, cmd).Wait() != 0: if GitCommand(self, cmd).Wait() != 0:
raise GitError("cannot initialize work tree") raise GitError("cannot initialize work tree")
rr_cache = os.path.join(self.gitdir, 'rr-cache')
if not os.path.exists(rr_cache):
os.makedirs(rr_cache)
self._CopyFiles() self._CopyFiles()
def _gitdir_path(self, path): def _gitdir_path(self, path):
return os.path.join(self.gitdir, path) return os.path.realpath(os.path.join(self.gitdir, path))
def _revlist(self, *args, **kw): def _revlist(self, *args, **kw):
a = [] a = []
@ -2078,9 +2118,10 @@ class Project(object):
return self.bare_ref.all return self.bare_ref.all
class _GitGetByExec(object): class _GitGetByExec(object):
def __init__(self, project, bare): def __init__(self, project, bare, gitdir):
self._project = project self._project = project
self._bare = bare self._bare = bare
self._gitdir = gitdir
def LsOthers(self): def LsOthers(self):
p = GitCommand(self._project, p = GitCommand(self._project,
@ -2089,6 +2130,7 @@ class Project(object):
'--others', '--others',
'--exclude-standard'], '--exclude-standard'],
bare = False, bare = False,
gitdir=self._gitdir,
capture_stdout = True, capture_stdout = True,
capture_stderr = True) capture_stderr = True)
if p.Wait() == 0: if p.Wait() == 0:
@ -2104,6 +2146,7 @@ class Project(object):
cmd.extend(args) cmd.extend(args)
p = GitCommand(self._project, p = GitCommand(self._project,
cmd, cmd,
gitdir=self._gitdir,
bare = False, bare = False,
capture_stdout = True, capture_stdout = True,
capture_stderr = True) capture_stderr = True)
@ -2213,6 +2256,7 @@ class Project(object):
p = GitCommand(self._project, p = GitCommand(self._project,
cmdv, cmdv,
bare = self._bare, bare = self._bare,
gitdir=self._gitdir,
capture_stdout = True, capture_stdout = True,
capture_stderr = True) capture_stderr = True)
r = [] r = []
@ -2265,6 +2309,7 @@ class Project(object):
p = GitCommand(self._project, p = GitCommand(self._project,
cmdv, cmdv,
bare = self._bare, bare = self._bare,
gitdir=self._gitdir,
capture_stdout = True, capture_stdout = True,
capture_stderr = True) capture_stderr = True)
if p.Wait() != 0: if p.Wait() != 0:
@ -2398,6 +2443,7 @@ class MetaProject(Project):
manifest = manifest, manifest = manifest,
name = name, name = name,
gitdir = gitdir, gitdir = gitdir,
objdir = gitdir,
worktree = worktree, worktree = worktree,
remote = RemoteSpec('origin'), remote = RemoteSpec('origin'),
relpath = '.repo/%s' % name, relpath = '.repo/%s' % name,

3
subcmds/rebase.py
View File

@ -62,6 +62,9 @@ branch but need to incorporate new upstream changes "underneath" them.
if opt.interactive and not one_project: if opt.interactive and not one_project:
print('error: interactive rebase not supported with multiple projects', print('error: interactive rebase not supported with multiple projects',
file=sys.stderr) file=sys.stderr)
if len(args) == 1:
print('note: project %s is mapped to more than one path' % (args[0],),
file=sys.stderr)
return -1 return -1
for project in all_projects: for project in all_projects:

54
subcmds/sync.py
View File

@ -219,9 +219,25 @@ later is required to fix a server side protocol bug.
dest='repo_upgraded', action='store_true', dest='repo_upgraded', action='store_true',
help=SUPPRESS_HELP) help=SUPPRESS_HELP)
def _FetchHelper(self, opt, project, lock, fetched, pm, sem, err_event): def _FetchProjectList(self, opt, projects, *args):
"""Main function of the fetch threads when jobs are > 1. """Main function of the fetch threads when jobs are > 1.
Delegates most of the work to _FetchHelper.
Args:
opt: Program options returned from optparse. See _Options().
projects: Projects to fetch.
*args: Remaining arguments to pass to _FetchHelper. See the
_FetchHelper docstring for details.
"""
for project in projects:
success = self._FetchHelper(opt, project, *args)
if not success and not opt.force_broken:
break
def _FetchHelper(self, opt, project, lock, fetched, pm, sem, err_event):
"""Fetch git objects for a single project.
Args: Args:
opt: Program options returned from optparse. See _Options(). opt: Program options returned from optparse. See _Options().
project: Project object for the project to fetch. project: Project object for the project to fetch.
@ -235,6 +251,9 @@ later is required to fix a server side protocol bug.
can be started up. can be started up.
err_event: We'll set this event in the case of an error (after printing err_event: We'll set this event in the case of an error (after printing
out info about the error). out info about the error).
Returns:
Whether the fetch was successful.
""" """
# We'll set to true once we've locked the lock. # We'll set to true once we've locked the lock.
did_lock = False did_lock = False
@ -281,6 +300,8 @@ later is required to fix a server side protocol bug.
lock.release() lock.release()
sem.release() sem.release()
return success
def _Fetch(self, projects, opt): def _Fetch(self, projects, opt):
fetched = set() fetched = set()
pm = Progress('Fetching projects', len(projects)) pm = Progress('Fetching projects', len(projects))
@ -303,20 +324,24 @@ later is required to fix a server side protocol bug.
else: else:
sys.exit(1) sys.exit(1)
else: else:
objdir_project_map = dict()
for project in projects:
objdir_project_map.setdefault(project.objdir, []).append(project)
threads = set() threads = set()
lock = _threading.Lock() lock = _threading.Lock()
sem = _threading.Semaphore(self.jobs) sem = _threading.Semaphore(self.jobs)
err_event = _threading.Event() err_event = _threading.Event()
for project in projects: for project_list in objdir_project_map.values():
# Check for any errors before starting any new threads. # Check for any errors before starting any new threads.
# ...we'll let existing threads finish, though. # ...we'll let existing threads finish, though.
if err_event.isSet(): if err_event.isSet():
break break
sem.acquire() sem.acquire()
t = _threading.Thread(target = self._FetchHelper, t = _threading.Thread(target = self._FetchProjectList,
args = (opt, args = (opt,
project, project_list,
lock, lock,
fetched, fetched,
pm, pm,
@ -342,6 +367,10 @@ later is required to fix a server side protocol bug.
return fetched return fetched
def _GCProjects(self, projects): def _GCProjects(self, projects):
gitdirs = {}
for project in projects:
gitdirs[project.gitdir] = project.bare_git
has_dash_c = git_require((1, 7, 2)) has_dash_c = git_require((1, 7, 2))
if multiprocessing and has_dash_c: if multiprocessing and has_dash_c:
cpu_count = multiprocessing.cpu_count() cpu_count = multiprocessing.cpu_count()
@ -350,8 +379,8 @@ later is required to fix a server side protocol bug.
jobs = min(self.jobs, cpu_count) jobs = min(self.jobs, cpu_count)
if jobs < 2: if jobs < 2:
for project in projects: for bare_git in gitdirs.values():
project.bare_git.gc('--auto') bare_git.gc('--auto')
return return
config = {'pack.threads': cpu_count / jobs if cpu_count > jobs else 1} config = {'pack.threads': cpu_count / jobs if cpu_count > jobs else 1}
@ -360,10 +389,10 @@ later is required to fix a server side protocol bug.
sem = _threading.Semaphore(jobs) sem = _threading.Semaphore(jobs)
err_event = _threading.Event() err_event = _threading.Event()
def GC(project): def GC(bare_git):
try: try:
try: try:
project.bare_git.gc('--auto', config=config) bare_git.gc('--auto', config=config)
except GitError: except GitError:
err_event.set() err_event.set()
except: except:
@ -372,11 +401,11 @@ later is required to fix a server side protocol bug.
finally: finally:
sem.release() sem.release()
for project in projects: for bare_git in gitdirs.values():
if err_event.isSet(): if err_event.isSet():
break break
sem.acquire() sem.acquire()
t = _threading.Thread(target=GC, args=(project,)) t = _threading.Thread(target=GC, args=(bare_git,))
t.daemon = True t.daemon = True
threads.add(t) threads.add(t)
t.start() t.start()
@ -416,12 +445,13 @@ later is required to fix a server side protocol bug.
if path not in new_project_paths: if path not in new_project_paths:
# If the path has already been deleted, we don't need to do it # If the path has already been deleted, we don't need to do it
if os.path.exists(self.manifest.topdir + '/' + path): if os.path.exists(self.manifest.topdir + '/' + path):
gitdir = os.path.join(self.manifest.topdir, path, '.git')
project = Project( project = Project(
manifest = self.manifest, manifest = self.manifest,
name = path, name = path,
remote = RemoteSpec('origin'), remote = RemoteSpec('origin'),
gitdir = os.path.join(self.manifest.topdir, gitdir = gitdir,
path, '.git'), objdir = gitdir,
worktree = os.path.join(self.manifest.topdir, path), worktree = os.path.join(self.manifest.topdir, path),
relpath = path, relpath = path,
revisionExpr = 'HEAD', revisionExpr = 'HEAD',

4
subcmds/upload.py
View File

@ -431,8 +431,10 @@ Gerrit Code Review: http://code.google.com/p/gerrit/
hook = RepoHook('pre-upload', self.manifest.repo_hooks_project, hook = RepoHook('pre-upload', self.manifest.repo_hooks_project,
self.manifest.topdir, abort_if_user_denies=True) self.manifest.topdir, abort_if_user_denies=True)
pending_proj_names = [project.name for (project, avail) in pending] pending_proj_names = [project.name for (project, avail) in pending]
pending_worktrees = [project.worktree for (project, avail) in pending]
try: try:
hook.Run(opt.allow_all_hooks, project_list=pending_proj_names) hook.Run(opt.allow_all_hooks, project_list=pending_proj_names,
worktree_list=pending_worktrees)
except HookError as e: except HookError as e:
print("ERROR: %s" % str(e), file=sys.stderr) print("ERROR: %s" % str(e), file=sys.stderr)
return return