diff --git a/command.py b/command.py index 38cacd3b..78dab96d 100644 --- a/command.py +++ b/command.py @@ -106,13 +106,13 @@ class Command(object): def _UpdatePathToProjectMap(self, project): self._by_path[project.worktree] = project - def _GetProjectByPath(self, path): + def _GetProjectByPath(self, manifest, path): project = None if os.path.exists(path): oldpath = None while path \ and path != oldpath \ - and path != self.manifest.topdir: + and path != manifest.topdir: try: project = self._by_path[path] break @@ -126,13 +126,16 @@ class Command(object): pass return project - def GetProjects(self, args, groups='', missing_ok=False, submodules_ok=False): + def GetProjects(self, args, manifest=None, groups='', missing_ok=False, + submodules_ok=False): """A list of projects that match the arguments. """ - all_projects_list = self.manifest.projects + if not manifest: + manifest = self.manifest + all_projects_list = manifest.projects result = [] - mp = self.manifest.manifestProject + mp = manifest.manifestProject if not groups: groups = mp.config.GetString('manifest.groups') @@ -155,11 +158,11 @@ class Command(object): self._ResetPathToProjectMap(all_projects_list) for arg in args: - projects = self.manifest.GetProjectsWithName(arg) + projects = manifest.GetProjectsWithName(arg) if not projects: path = os.path.abspath(arg).replace('\\', '/') - project = self._GetProjectByPath(path) + project = self._GetProjectByPath(manifest, path) # If it's not a derived project, update path->project mapping and # search again, as arg might actually point to a derived subproject. @@ -170,7 +173,7 @@ class Command(object): self._UpdatePathToProjectMap(subproject) search_again = True if search_again: - project = self._GetProjectByPath(path) or project + project = self._GetProjectByPath(manifest, path) or project if project: projects = [project] diff --git a/gitc_utils.py b/gitc_utils.py index ef028b08..4d8d5366 100644 --- a/gitc_utils.py +++ b/gitc_utils.py @@ -16,6 +16,7 @@ from __future__ import print_function import os import sys +import time import git_command import git_config @@ -26,6 +27,18 @@ GITC_MANIFEST_DIR = '/usr/local/google/gitc/' GITC_FS_ROOT_DIR = '/gitc/manifest-rw/' NUM_BATCH_RETRIEVE_REVISIONID = 300 +def parse_clientdir(gitc_fs_path): + """Parse a path in the GITC FS and return its client name. + + @param gitc_fs_path: A subdirectory path within the GITC_FS_ROOT_DIR. + + @returns: The GITC client name + """ + if (gitc_fs_path == GITC_FS_ROOT_DIR or + not gitc_fs_path.startswith(GITC_FS_ROOT_DIR)): + return None + return gitc_fs_path.split(GITC_FS_ROOT_DIR)[1].split('/')[0] + def _set_project_revisions(projects): """Sets the revisionExpr for a list of projects. @@ -50,19 +63,37 @@ def _set_project_revisions(projects): sys.exit(1) proj.revisionExpr = gitcmd.stdout.split('\t')[0] -def generate_gitc_manifest(client_dir, manifest): +def generate_gitc_manifest(client_dir, manifest, projects=None): """Generate a manifest for shafsd to use for this GITC client. @param client_dir: GITC client directory to install the .manifest file in. @param manifest: XmlManifest object representing the repo manifest. + @param projects: List of projects we want to update, this must be a sublist + of manifest.projects to work properly. If not provided, + manifest.projects is used. """ print('Generating GITC Manifest by fetching revision SHAs for each ' 'project.') + if projects is None: + projects = manifest.projects index = 0 - while index < len(manifest.projects): + while index < len(projects): _set_project_revisions( - manifest.projects[index:(index+NUM_BATCH_RETRIEVE_REVISIONID)]) + projects[index:(index+NUM_BATCH_RETRIEVE_REVISIONID)]) index += NUM_BATCH_RETRIEVE_REVISIONID # Save the manifest. + save_manifest(manifest, client_dir=client_dir) + +def save_manifest(manifest, client_dir=None): + """Save the manifest file in the client_dir. + + @param client_dir: Client directory to save the manifest in. + @param manifest: Manifest object to save. + """ + if not client_dir: + client_dir = manifest.gitc_client_dir with open(os.path.join(client_dir, '.manifest'), 'w') as f: manifest.Save(f) + # TODO(sbasi/jorg): Come up with a solution to remove the sleep below. + # Give the GITC filesystem time to register the manifest changes. + time.sleep(3) \ No newline at end of file diff --git a/main.py b/main.py index 6736abc9..adfaffb0 100755 --- a/main.py +++ b/main.py @@ -51,7 +51,8 @@ from error import ManifestParseError from error import NoManifestException from error import NoSuchProjectError from error import RepoChangedException -from manifest_xml import XmlManifest +import gitc_utils +from manifest_xml import GitcManifest, XmlManifest from pager import RunPager from wrapper import WrapperPath, Wrapper @@ -129,6 +130,12 @@ class _Repo(object): cmd.repodir = self.repodir cmd.manifest = XmlManifest(cmd.repodir) + cmd.gitc_manifest = None + gitc_client_name = gitc_utils.parse_clientdir(os.getcwd()) + if gitc_client_name: + cmd.gitc_manifest = GitcManifest(cmd.repodir, gitc_client_name) + cmd.manifest.isGitcClient = True + Editor.globalConfig = cmd.manifest.globalConfig if not isinstance(cmd, MirrorSafeCommand) and cmd.manifest.IsMirror: diff --git a/manifest_xml.py b/manifest_xml.py index 6dc01a47..b33ec627 100644 --- a/manifest_xml.py +++ b/manifest_xml.py @@ -29,6 +29,7 @@ else: urllib = imp.new_module('urllib') urllib.parse = urlparse +import gitc_utils from git_config import GitConfig from git_refs import R_HEADS, HEAD from project import RemoteSpec, Project, MetaProject @@ -112,6 +113,7 @@ class XmlManifest(object): self.manifestFile = os.path.join(self.repodir, MANIFEST_FILE_NAME) self.globalConfig = GitConfig.ForUser() self.localManifestWarning = False + self.isGitcClient = False self.repoProject = MetaProject(self, 'repo', gitdir = os.path.join(repodir, 'repo/.git'), @@ -306,6 +308,8 @@ class XmlManifest(object): if p.clone_depth: e.setAttribute('clone-depth', str(p.clone_depth)) + self._output_manifest_project_extras(p, e) + if p.subprojects: subprojects = set(subp.name for subp in p.subprojects) output_projects(p, e, list(sorted(subprojects))) @@ -323,6 +327,10 @@ class XmlManifest(object): doc.writexml(fd, '', ' ', '\n', 'UTF-8') + def _output_manifest_project_extras(self, p, e): + """Manifests can modify e if they support extra project attributes.""" + pass + @property def paths(self): self._Load() @@ -712,7 +720,7 @@ class XmlManifest(object): def _UnjoinName(self, parent_name, name): return os.path.relpath(name, parent_name) - def _ParseProject(self, node, parent = None): + def _ParseProject(self, node, parent = None, **extra_proj_attrs): """ reads a element from the manifest file """ @@ -807,7 +815,8 @@ class XmlManifest(object): clone_depth = clone_depth, upstream = upstream, parent = parent, - dest_branch = dest_branch) + dest_branch = dest_branch, + **extra_proj_attrs) for n in node.childNodes: if n.nodeName == 'copyfile': @@ -938,3 +947,26 @@ class XmlManifest(object): diff['added'].append(toProjects[proj]) return diff + + +class GitcManifest(XmlManifest): + + def __init__(self, repodir, gitc_client_name): + """Initialize the GitcManifest object.""" + super(GitcManifest, self).__init__(repodir) + self.isGitcClient = True + self.gitc_client_name = gitc_client_name + self.gitc_client_dir = os.path.join(gitc_utils.GITC_MANIFEST_DIR, + gitc_client_name) + self.manifestFile = os.path.join(self.gitc_client_dir, '.manifest') + + def _ParseProject(self, node, parent = None): + """Override _ParseProject and add support for GITC specific attributes.""" + return super(GitcManifest, self)._ParseProject( + node, parent=parent, old_revision=node.getAttribute('old-revision')) + + def _output_manifest_project_extras(self, p, e): + """Output GITC Specific Project attributes""" + if p.old_revision: + e.setAttribute('old-revision', str(p.old_revision)) + diff --git a/project.py b/project.py index a8d012d2..f964b2fc 100644 --- a/project.py +++ b/project.py @@ -572,7 +572,8 @@ class Project(object): parent=None, is_derived=False, dest_branch=None, - optimized_fetch=False): + optimized_fetch=False, + old_revision=None): """Init a Project object. Args: @@ -596,6 +597,7 @@ class Project(object): dest_branch: The branch to which to push changes for review by default. optimized_fetch: If True, when a project is set to a sha1 revision, only fetch from the remote if the sha1 is not present locally. + old_revision: saved git commit id for open GITC projects. """ self.manifest = manifest self.name = name @@ -643,6 +645,7 @@ class Project(object): self.bare_ref = GitRefs(gitdir) self.bare_objdir = self._GitGetByExec(self, bare=True, gitdir=objdir) self.dest_branch = dest_branch + self.old_revision = old_revision # This will be filled in if a project is later identified to be the # project containing repo hooks. @@ -1195,6 +1198,8 @@ class Project(object): self._InitHooks() def _CopyAndLinkFiles(self): + if self.manifest.isGitcClient: + return for copyfile in self.copyfiles: copyfile._Copy() for linkfile in self.linkfiles: @@ -1425,9 +1430,11 @@ class Project(object): ## Branch Management ## - def StartBranch(self, name): + def StartBranch(self, name, branch_merge=''): """Create a new branch off the manifest's revision. """ + if not branch_merge: + branch_merge = self.revisionExpr head = self.work_git.GetHead() if head == (R_HEADS + name): return True @@ -1441,9 +1448,9 @@ class Project(object): branch = self.GetBranch(name) branch.remote = self.GetRemote(self.remote.name) - branch.merge = self.revisionExpr - if not branch.merge.startswith('refs/') and not ID_RE.match(self.revisionExpr): - branch.merge = R_HEADS + self.revisionExpr + branch.merge = branch_merge + if not branch.merge.startswith('refs/') and not ID_RE.match(branch_merge): + branch.merge = R_HEADS + branch_merge revid = self.GetRevisionId(all_refs) if head.startswith(R_HEADS): @@ -1451,7 +1458,6 @@ class Project(object): head = all_refs[head] except KeyError: head = None - if revid and head and revid == head: ref = os.path.join(self.gitdir, R_HEADS + name) try: diff --git a/subcmds/start.py b/subcmds/start.py index 60ad41e0..188fd7c6 100644 --- a/subcmds/start.py +++ b/subcmds/start.py @@ -14,11 +14,15 @@ # limitations under the License. from __future__ import print_function +import os import sys + from command import Command from git_config import IsId from git_command import git +import gitc_utils from progress import Progress +from project import SyncBuffer class Start(Command): common = True @@ -53,20 +57,50 @@ revision specified in the manifest. print("error: at least one project must be specified", file=sys.stderr) sys.exit(1) - all_projects = self.GetProjects(projects) + proj_name_to_gitc_proj_dict = {} + if self.gitc_manifest: + all_projects = self.GetProjects(projects, manifest=self.gitc_manifest, + missing_ok=True) + for project in all_projects: + if project.old_revision: + project.already_synced = True + else: + project.already_synced = False + project.old_revision = project.revisionExpr + proj_name_to_gitc_proj_dict[project.name] = project + project.revisionExpr = None + # Save the GITC manifest. + gitc_utils.save_manifest(self.gitc_manifest) + all_projects = self.GetProjects(projects, + missing_ok=bool(self.gitc_manifest)) pm = Progress('Starting %s' % nb, len(all_projects)) for project in all_projects: pm.update() + if self.gitc_manifest: + gitc_project = proj_name_to_gitc_proj_dict[project.name] + # Sync projects that have already been opened. + if not gitc_project.already_synced: + proj_localdir = os.path.join(self.gitc_manifest.gitc_client_dir, + project.relpath) + project.worktree = proj_localdir + if not os.path.exists(proj_localdir): + os.makedirs(proj_localdir) + project.Sync_NetworkHalf() + sync_buf = SyncBuffer(self.manifest.manifestProject.config) + project.Sync_LocalHalf(sync_buf) + project.revisionExpr = gitc_project.old_revision + # If the current revision is a specific SHA1 then we can't push back # to it; so substitute with dest_branch if defined, or with manifest # default revision instead. + branch_merge = '' if IsId(project.revisionExpr): if project.dest_branch: - project.revisionExpr = project.dest_branch + branch_merge = project.dest_branch else: - project.revisionExpr = self.manifest.default.revisionExpr - if not project.StartBranch(nb): + branch_merge = self.manifest.default.revisionExpr + if not project.StartBranch(nb, branch_merge=branch_merge): err.append(project) pm.end() diff --git a/subcmds/sync.py b/subcmds/sync.py index ad0ecdf4..934aaa80 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py @@ -549,15 +549,6 @@ later is required to fix a server side protocol bug. cwd.split(gitc_utils.GITC_MANIFEST_DIR)[1])) sys.exit(1) - self._gitc_sync = False - if cwd.startswith(gitc_utils.GITC_FS_ROOT_DIR): - self._gitc_sync = True - self._client_name = cwd.split(gitc_utils.GITC_FS_ROOT_DIR)[1].split( - '/')[0] - self._client_dir = os.path.join(gitc_utils.GITC_MANIFEST_DIR, - self._client_name) - print('Updating GITC client: %s' % self._client_name) - if opt.manifest_name: self.manifest.Override(opt.manifest_name) @@ -677,12 +668,6 @@ later is required to fix a server side protocol bug. if opt.repo_upgraded: _PostRepoUpgrade(self.manifest, quiet=opt.quiet) - if self._gitc_sync: - gitc_utils.generate_gitc_manifest(self._client_dir, self.manifest) - print('GITC client successfully synced.') - return - - if not opt.local_only: mp.Sync_NetworkHalf(quiet=opt.quiet, current_branch_only=opt.current_branch_only, @@ -697,6 +682,35 @@ later is required to fix a server side protocol bug. self._ReloadManifest(manifest_name) if opt.jobs is None: self.jobs = self.manifest.default.sync_j + + # TODO (sbasi) - Add support for manifest changes, aka projects + # have been added or deleted from the manifest. + if self.gitc_manifest: + gitc_manifest_projects = self.GetProjects(args, + manifest=self.gitc_manifest, + missing_ok=True) + gitc_projects = [] + opened_projects = [] + for project in gitc_manifest_projects: + if not project.old_revision: + gitc_projects.append(project) + else: + opened_projects.append(project) + + if gitc_projects and not opt.local_only: + print('Updating GITC client: %s' % self.gitc_manifest.gitc_client_name) + gitc_utils.generate_gitc_manifest(self.gitc_manifest.gitc_client_dir, + self.gitc_manifest, + gitc_projects) + print('GITC client successfully synced.') + + # The opened projects need to be synced as normal, therefore we + # generate a new args list to represent the opened projects. + args = [] + for proj in opened_projects: + args.append(os.path.relpath(proj.worktree, cwd)) + if not args: + return all_projects = self.GetProjects(args, missing_ok=True, submodules_ok=opt.fetch_submodules)