android/git-repo
1
Fork 0
mirror of https://gerrit.googlesource.com/git-repo synced 2024-12-21 07:16:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge "repo: Support multiple branches for the same project."

Browse Source
This commit is contained in:
Conley Owens 2014-01-10 01:20:12 +00:00 committed by Gerrit Code Review
commit e695338e21
6 changed files with 191 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
command.py
View File

@ -129,7 +129,7 @@ class Command(object):
def GetProjects(self, args, missing_ok=False, submodules_ok=False):
"""A list of projects that match the arguments.
"""
all_projects = self.manifest.projects
all_projects_list = self.manifest.projects
result = []
mp = self.manifest.manifestProject
@ -140,7 +140,6 @@ class Command(object):
groups = [x for x in re.split(r'[,\s]+', groups) if x]
if not args:
all_projects_list = list(all_projects.values())
derived_projects = {}
for project in all_projects_list:
if submodules_ok or project.sync_s:
@ -152,12 +151,12 @@ class Command(object):
project.MatchesGroups(groups)):
result.append(project)
else:
self._ResetPathToProjectMap(all_projects.values())
self._ResetPathToProjectMap(all_projects_list)
for arg in args:
project = all_projects.get(arg)
projects = self.manifest.GetProjectsWithName(arg)
if not project:
if not projects:
path = os.path.abspath(arg).replace('\\', '/')
project = self._GetProjectByPath(path)
@ -172,14 +171,19 @@ class Command(object):
if search_again:
project = self._GetProjectByPath(path) or project
if not project:
if project:
projects = [project]
if not projects:
raise NoSuchProjectError(arg)
for project in projects:
if not missing_ok and not project.Exists:
raise NoSuchProjectError(arg)
if not project.MatchesGroups(groups):
raise InvalidProjectGroupsError(arg)
result.append(project)
result.extend(projects)
def _getpath(x):
return x.relpath

64
manifest_xml.py
View File

@ -215,8 +215,9 @@ class XmlManifest(object):
root.appendChild(doc.createTextNode(''))
def output_projects(parent, parent_node, projects):
for p in projects:
output_project(parent, parent_node, self.projects[p])
for project_name in projects:
for project in self._projects[project_name]:
output_project(parent, parent_node, project)
def output_project(parent, parent_node, p):
if not p.MatchesGroups(groups):
@ -277,13 +278,11 @@ class XmlManifest(object):
e.setAttribute('sync-s', 'true')
if p.subprojects:
sort_projects = list(sorted([subp.name for subp in p.subprojects]))
output_projects(p, e, sort_projects)
subprojects = set(subp.name for subp in p.subprojects)
output_projects(p, e, list(sorted(subprojects)))
sort_projects = list(sorted([key for key, value in self.projects.items()
if not value.parent]))
sort_projects.sort()
output_projects(None, root, sort_projects)
projects = set(p.name for p in self._paths.values() if not p.parent)
output_projects(None, root, list(sorted(projects)))
if self._repo_hooks_project:
root.appendChild(doc.createTextNode(''))
@ -295,10 +294,15 @@ class XmlManifest(object):
doc.writexml(fd, '', ' ', '\n', 'UTF-8')
@property
def paths(self):
self._Load()
return self._paths
@property
def projects(self):
self._Load()
return self._projects
return self._paths.values()
@property
def remotes(self):
@ -336,6 +340,7 @@ class XmlManifest(object):
def _Unload(self):
self._loaded = False
self._projects = {}
self._paths = {}
self._remotes = {}
self._default = None
self._repo_hooks_project = None
@ -467,11 +472,17 @@ class XmlManifest(object):
self._manifest_server = url
def recursively_add_projects(project):
if self._projects.get(project.name):
projects = self._projects.setdefault(project.name, [])
if project.relpath is None:
raise ManifestParseError(
'duplicate project %s in %s' %
'missing path for %s in %s' %
(project.name, self.manifestFile))
self._projects[project.name] = project
if project.relpath in self._paths:
raise ManifestParseError(
'duplicate path %s in %s' %
(project.relpath, self.manifestFile))
self._paths[project.relpath] = project
projects.append(project)
for subproject in project.subprojects:
recursively_add_projects(subproject)
@ -492,12 +503,18 @@ class XmlManifest(object):
# Store a reference to the Project.
try:
self._repo_hooks_project = self._projects[repo_hooks_project]
repo_hooks_projects = self._projects[repo_hooks_project]
except KeyError:
raise ManifestParseError(
'project %s not found for repo-hooks' %
(repo_hooks_project))
if len(repo_hooks_projects) != 1:
raise ManifestParseError(
'internal error parsing repo-hooks in %s' %
(self.manifestFile))
self._repo_hooks_project = repo_hooks_projects[0]
# Store the enabled hooks in the Project object.
self._repo_hooks_project.enabled_repo_hooks = enabled_repo_hooks
if node.nodeName == 'remove-project':
@ -544,11 +561,12 @@ class XmlManifest(object):
name = name,
remote = remote.ToRemoteSpec(name),
gitdir = gitdir,
objdir = gitdir,
worktree = None,
relpath = None,
revisionExpr = m.revisionExpr,
revisionId = None)
self._projects[project.name] = project
self._projects[project.name] = [project]
def _ParseRemote(self, node):
"""
@ -708,9 +726,10 @@ class XmlManifest(object):
groups = [x for x in re.split(r'[,\s]+', groups) if x]
if parent is None:
relpath, worktree, gitdir = self.GetProjectPaths(name, path)
relpath, worktree, gitdir, objdir = self.GetProjectPaths(name, path)
else:
relpath, worktree, gitdir = self.GetSubprojectPaths(parent, path)
relpath, worktree, gitdir, objdir = \
self.GetSubprojectPaths(parent, name, path)
default_groups = ['all', 'name:%s' % name, 'path:%s' % relpath]
groups.extend(set(default_groups).difference(groups))
@ -723,6 +742,7 @@ class XmlManifest(object):
name = name,
remote = remote.ToRemoteSpec(name),
gitdir = gitdir,
objdir = objdir,
worktree = worktree,
relpath = relpath,
revisionExpr = revisionExpr,
@ -751,10 +771,15 @@ class XmlManifest(object):
if self.IsMirror:
worktree = None
gitdir = os.path.join(self.topdir, '%s.git' % name)
objdir = gitdir
else:
worktree = os.path.join(self.topdir, path).replace('\\', '/')
gitdir = os.path.join(self.repodir, 'projects', '%s.git' % path)
return relpath, worktree, gitdir
objdir = os.path.join(self.repodir, 'project-objects', '%s.git' % name)
return relpath, worktree, gitdir, objdir
def GetProjectsWithName(self, name):
return self._projects.get(name, [])
def GetSubprojectName(self, parent, submodule_path):
return os.path.join(parent.name, submodule_path)
@ -765,14 +790,15 @@ class XmlManifest(object):
def _UnjoinRelpath(self, parent_relpath, relpath):
return os.path.relpath(relpath, parent_relpath)
def GetSubprojectPaths(self, parent, path):
def GetSubprojectPaths(self, parent, name, path):
relpath = self._JoinRelpath(parent.relpath, path)
gitdir = os.path.join(parent.gitdir, 'subprojects', '%s.git' % path)
objdir = os.path.join(parent.gitdir, 'subproject-objects', '%s.git' % name)
if self.IsMirror:
worktree = None
else:
worktree = os.path.join(parent.worktree, path).replace('\\', '/')
return relpath, worktree, gitdir
return relpath, worktree, gitdir, objdir
def _ParseCopyFile(self, project, node):
src = self._reqatt(node, 'src')

108
project.py
View File

@ -488,6 +488,7 @@ class Project(object):
name,
remote,
gitdir,
objdir,
worktree,
relpath,
revisionExpr,
@ -508,6 +509,7 @@ class Project(object):
name: The `name` attribute of manifest.xml's project element.
remote: RemoteSpec object specifying its remote's properties.
gitdir: Absolute path of git directory.
objdir: Absolute path of directory to store git objects.
worktree: Absolute path of git working tree.
relpath: Relative path of git working tree to repo's top directory.
revisionExpr: The `revision` attribute of manifest.xml's project element.
@ -526,6 +528,7 @@ class Project(object):
self.name = name
self.remote = remote
self.gitdir = gitdir.replace('\\', '/')
self.objdir = objdir.replace('\\', '/')
if worktree:
self.worktree = worktree.replace('\\', '/')
else:
@ -558,11 +561,12 @@ class Project(object):
defaults = self.manifest.globalConfig)
if self.worktree:
self.work_git = self._GitGetByExec(self, bare=False)
self.work_git = self._GitGetByExec(self, bare=False, gitdir=gitdir)
else:
self.work_git = None
self.bare_git = self._GitGetByExec(self, bare=True)
self.bare_git = self._GitGetByExec(self, bare=True, gitdir=gitdir)
self.bare_ref = GitRefs(gitdir)
self.bare_objdir = self._GitGetByExec(self, bare=True, gitdir=objdir)
self.dest_branch = dest_branch
# This will be filled in if a project is later identified to be the
@ -1117,6 +1121,7 @@ class Project(object):
"""Perform only the local IO portion of the sync process.
Network access is not required.
"""
self._InitWorkTree()
all_refs = self.bare_ref.all
self.CleanPublishedCache(all_refs)
revid = self.GetRevisionId(all_refs)
@ -1125,7 +1130,6 @@ class Project(object):
self._FastForward(revid)
self._CopyFiles()
self._InitWorkTree()
head = self.work_git.GetHead()
if head.startswith(R_HEADS):
branch = head[len(R_HEADS):]
@ -1592,11 +1596,13 @@ class Project(object):
return result
for rev, path, url in self._GetSubmodules():
name = self.manifest.GetSubprojectName(self, path)
project = self.manifest.projects.get(name)
relpath, worktree, gitdir, objdir = \
self.manifest.GetSubprojectPaths(self, name, path)
project = self.manifest.paths.get(relpath)
if project:
result.extend(project.GetDerivedSubprojects())
continue
relpath, worktree, gitdir = self.manifest.GetSubprojectPaths(self, path)
remote = RemoteSpec(self.remote.name,
url = url,
review = self.remote.review)
@ -1604,6 +1610,7 @@ class Project(object):
name = name,
remote = remote,
gitdir = gitdir,
objdir = objdir,
worktree = worktree,
relpath = relpath,
revisionExpr = self.revisionExpr,
@ -1966,8 +1973,17 @@ class Project(object):
def _InitGitDir(self, mirror_git=None):
if not os.path.exists(self.gitdir):
# Initialize the bare repository, which contains all of the objects.
if not os.path.exists(self.objdir):
os.makedirs(self.objdir)
self.bare_objdir.init()
# If we have a separate directory to hold refs, initialize it as well.
if self.objdir != self.gitdir:
os.makedirs(self.gitdir)
self.bare_git.init()
self._ReferenceGitDir(self.objdir, self.gitdir, share_refs=False,
copy_all=True)
mp = self.manifest.manifestProject
ref_dir = mp.config.GetString('repo.reference') or ''
@ -2083,34 +2099,62 @@ class Project(object):
msg = 'manifest set to %s' % self.revisionExpr
self.bare_git.symbolic_ref('-m', msg, ref, dst)
def _InitWorkTree(self):
dotgit = os.path.join(self.worktree, '.git')
if not os.path.exists(dotgit):
os.makedirs(dotgit)
def _ReferenceGitDir(self, gitdir, dotgit, share_refs, copy_all):
"""Update |dotgit| to reference |gitdir|, using symlinks where possible.
for name in ['config',
'description',
'hooks',
'info',
'logs',
'objects',
'packed-refs',
'refs',
'rr-cache',
'svn']:
Args:
gitdir: The bare git repository. Must already be initialized.
dotgit: The repository you would like to initialize.
share_refs: If true, |dotgit| will store its refs under |gitdir|.
Only one work tree can store refs under a given |gitdir|.
copy_all: If true, copy all remaining files from |gitdir| -> |dotgit|.
This saves you the effort of initializing |dotgit| yourself.
"""
# These objects can be shared between several working trees.
symlink_files = ['description', 'info']
symlink_dirs = ['hooks', 'objects', 'rr-cache', 'svn']
if share_refs:
# These objects can only be used by a single working tree.
symlink_files += ['config', 'packed-refs']
symlink_dirs += ['logs', 'refs']
to_symlink = symlink_files + symlink_dirs
to_copy = []
if copy_all:
to_copy = os.listdir(gitdir)
for name in set(to_copy).union(to_symlink):
try:
src = os.path.join(self.gitdir, name)
dst = os.path.join(dotgit, name)
if os.path.islink(dst) or not os.path.exists(dst):
os.symlink(os.path.relpath(src, os.path.dirname(dst)), dst)
else:
src = os.path.realpath(os.path.join(gitdir, name))
dst = os.path.realpath(os.path.join(dotgit, name))
if os.path.lexists(dst) and not os.path.islink(dst):
raise GitError('cannot overwrite a local work tree')
# If the source dir doesn't exist, create an empty dir.
if name in symlink_dirs and not os.path.lexists(src):
os.makedirs(src)
if name in to_symlink:
os.symlink(os.path.relpath(src, os.path.dirname(dst)), dst)
elif copy_all and not os.path.islink(dst):
if os.path.isdir(src):
shutil.copytree(src, dst)
elif os.path.isfile(src):
shutil.copy(src, dst)
except OSError as e:
if e.errno == errno.EPERM:
raise GitError('filesystem must support symlinks')
else:
raise
def _InitWorkTree(self):
dotgit = os.path.join(self.worktree, '.git')
if not os.path.exists(dotgit):
os.makedirs(dotgit)
self._ReferenceGitDir(self.gitdir, dotgit, share_refs=True,
copy_all=False)
_lwrite(os.path.join(dotgit, HEAD), '%s\n' % self.GetRevisionId())
cmd = ['read-tree', '--reset', '-u']
@ -2119,14 +2163,10 @@ class Project(object):
if GitCommand(self, cmd).Wait() != 0:
raise GitError("cannot initialize work tree")
rr_cache = os.path.join(self.gitdir, 'rr-cache')
if not os.path.exists(rr_cache):
os.makedirs(rr_cache)
self._CopyFiles()
def _gitdir_path(self, path):
return os.path.join(self.gitdir, path)
return os.path.realpath(os.path.join(self.gitdir, path))
def _revlist(self, *args, **kw):
a = []
@ -2139,9 +2179,10 @@ class Project(object):
return self.bare_ref.all
class _GitGetByExec(object):
def __init__(self, project, bare):
def __init__(self, project, bare, gitdir):
self._project = project
self._bare = bare
self._gitdir = gitdir
def LsOthers(self):
p = GitCommand(self._project,
@ -2150,6 +2191,7 @@ class Project(object):
'--others',
'--exclude-standard'],
bare = False,
gitdir=self._gitdir,
capture_stdout = True,
capture_stderr = True)
if p.Wait() == 0:
@ -2165,6 +2207,7 @@ class Project(object):
cmd.extend(args)
p = GitCommand(self._project,
cmd,
gitdir=self._gitdir,
bare = False,
capture_stdout = True,
capture_stderr = True)
@ -2274,6 +2317,7 @@ class Project(object):
p = GitCommand(self._project,
cmdv,
bare = self._bare,
gitdir=self._gitdir,
capture_stdout = True,
capture_stderr = True)
r = []
@ -2326,6 +2370,7 @@ class Project(object):
p = GitCommand(self._project,
cmdv,
bare = self._bare,
gitdir=self._gitdir,
capture_stdout = True,
capture_stderr = True)
if p.Wait() != 0:
@ -2459,6 +2504,7 @@ class MetaProject(Project):
manifest = manifest,
name = name,
gitdir = gitdir,
objdir = gitdir,
worktree = worktree,
remote = RemoteSpec('origin'),
relpath = '.repo/%s' % name,

3
subcmds/rebase.py
View File

@ -62,6 +62,9 @@ branch but need to incorporate new upstream changes "underneath" them.
if opt.interactive and not one_project:
print('error: interactive rebase not supported with multiple projects',
file=sys.stderr)
if len(args) == 1:
print('note: project %s is mapped to more than one path' % (args[0],),
file=sys.stderr)
return -1
for project in all_projects:

54
subcmds/sync.py
View File

@ -219,9 +219,25 @@ later is required to fix a server side protocol bug.
dest='repo_upgraded', action='store_true',
help=SUPPRESS_HELP)
def _FetchHelper(self, opt, project, lock, fetched, pm, sem, err_event):
def _FetchProjectList(self, opt, projects, *args):
"""Main function of the fetch threads when jobs are > 1.
Delegates most of the work to _FetchHelper.
Args:
opt: Program options returned from optparse. See _Options().
projects: Projects to fetch.
*args: Remaining arguments to pass to _FetchHelper. See the
_FetchHelper docstring for details.
"""
for project in projects:
success = self._FetchHelper(opt, project, *args)
if not success and not opt.force_broken:
break
def _FetchHelper(self, opt, project, lock, fetched, pm, sem, err_event):
"""Fetch git objects for a single project.
Args:
opt: Program options returned from optparse. See _Options().
project: Project object for the project to fetch.
@ -235,6 +251,9 @@ later is required to fix a server side protocol bug.
can be started up.
err_event: We'll set this event in the case of an error (after printing
out info about the error).
Returns:
Whether the fetch was successful.
"""
# We'll set to true once we've locked the lock.
did_lock = False
@ -281,6 +300,8 @@ later is required to fix a server side protocol bug.
lock.release()
sem.release()
return success
def _Fetch(self, projects, opt):
fetched = set()
pm = Progress('Fetching projects', len(projects))
@ -304,20 +325,24 @@ later is required to fix a server side protocol bug.
else:
sys.exit(1)
else:
objdir_project_map = dict()
for project in projects:
objdir_project_map.setdefault(project.objdir, []).append(project)
threads = set()
lock = _threading.Lock()
sem = _threading.Semaphore(self.jobs)
err_event = _threading.Event()
for project in projects:
for project_list in objdir_project_map.values():
# Check for any errors before starting any new threads.
# ...we'll let existing threads finish, though.
if err_event.isSet():
break
sem.acquire()
t = _threading.Thread(target = self._FetchHelper,
t = _threading.Thread(target = self._FetchProjectList,
args = (opt,
project,
project_list,
lock,
fetched,
pm,
@ -345,6 +370,10 @@ later is required to fix a server side protocol bug.
return fetched
def _GCProjects(self, projects):
gitdirs = {}
for project in projects:
gitdirs[project.gitdir] = project.bare_git
has_dash_c = git_require((1, 7, 2))
if multiprocessing and has_dash_c:
cpu_count = multiprocessing.cpu_count()
@ -353,8 +382,8 @@ later is required to fix a server side protocol bug.
jobs = min(self.jobs, cpu_count)
if jobs < 2:
for project in projects:
project.bare_git.gc('--auto')
for bare_git in gitdirs.values():
bare_git.gc('--auto')
return
config = {'pack.threads': cpu_count / jobs if cpu_count > jobs else 1}
@ -363,10 +392,10 @@ later is required to fix a server side protocol bug.
sem = _threading.Semaphore(jobs)
err_event = _threading.Event()
def GC(project):
def GC(bare_git):
try:
try:
project.bare_git.gc('--auto', config=config)
bare_git.gc('--auto', config=config)
except GitError:
err_event.set()
except:
@ -375,11 +404,11 @@ later is required to fix a server side protocol bug.
finally:
sem.release()
for project in projects:
for bare_git in gitdirs.values():
if err_event.isSet():
break
sem.acquire()
t = _threading.Thread(target=GC, args=(project,))
t = _threading.Thread(target=GC, args=(bare_git,))
t.daemon = True
threads.add(t)
t.start()
@ -419,12 +448,13 @@ later is required to fix a server side protocol bug.
if path not in new_project_paths:
# If the path has already been deleted, we don't need to do it
if os.path.exists(self.manifest.topdir + '/' + path):
gitdir = os.path.join(self.manifest.topdir, path, '.git')
project = Project(
manifest = self.manifest,
name = path,
remote = RemoteSpec('origin'),
gitdir = os.path.join(self.manifest.topdir,
path, '.git'),
gitdir = gitdir,
objdir = gitdir,
worktree = os.path.join(self.manifest.topdir, path),
relpath = path,
revisionExpr = 'HEAD',

4
subcmds/upload.py
View File

@ -432,8 +432,10 @@ Gerrit Code Review: http://code.google.com/p/gerrit/
hook = RepoHook('pre-upload', self.manifest.repo_hooks_project,
self.manifest.topdir, abort_if_user_denies=True)
pending_proj_names = [project.name for (project, avail) in pending]
pending_worktrees = [project.worktree for (project, avail) in pending]
try:
hook.Run(opt.allow_all_hooks, project_list=pending_proj_names)
hook.Run(opt.allow_all_hooks, project_list=pending_proj_names,
worktree_list=pending_worktrees)
except HookError as e:
print("ERROR: %s" % str(e), file=sys.stderr)
return