From b9a1b73425773dc97843f92aeee9c57c9a08c0f7 Mon Sep 17 00:00:00 2001 From: Simran Basi Date: Thu, 20 Aug 2015 12:19:28 -0700 Subject: [PATCH] GITC: Add repo start support. Add repo start support for GITC checkouts. If the user is in the GITC FS view, they can now run repo start to check out the sources and create a new working branch. When "repo start" is called on a GITC project, the revision tag is set to an empty string and saved in a new tag: old-revision. This tells the GITC filesystem to display the local copy of the sources when being viewed. The local copy is created by pulling the project sources and the new branch is created based off the original project revision. Updated main.py to setup each command's gitc_manifest when appropriate. Updated repo sync's logic to sync opened projects and updating the GITC manifest file for the rest. Change-Id: I7e4809d1c4fc43c69b26f2f1bebe45aab0cae628 --- command.py | 19 +++++++++++-------- gitc_utils.py | 37 ++++++++++++++++++++++++++++++++++--- main.py | 9 ++++++++- manifest_xml.py | 36 ++++++++++++++++++++++++++++++++++-- project.py | 18 ++++++++++++------ subcmds/start.py | 42 ++++++++++++++++++++++++++++++++++++++---- subcmds/sync.py | 44 +++++++++++++++++++++++++++++--------------- 7 files changed, 166 insertions(+), 39 deletions(-) diff --git a/command.py b/command.py index 38cacd3b..78dab96d 100644 --- a/command.py +++ b/command.py @@ -106,13 +106,13 @@ class Command(object): def _UpdatePathToProjectMap(self, project): self._by_path[project.worktree] = project - def _GetProjectByPath(self, path): + def _GetProjectByPath(self, manifest, path): project = None if os.path.exists(path): oldpath = None while path \ and path != oldpath \ - and path != self.manifest.topdir: + and path != manifest.topdir: try: project = self._by_path[path] break @@ -126,13 +126,16 @@ class Command(object): pass return project - def GetProjects(self, args, groups='', missing_ok=False, submodules_ok=False): + def GetProjects(self, args, manifest=None, groups='', missing_ok=False, + submodules_ok=False): """A list of projects that match the arguments. """ - all_projects_list = self.manifest.projects + if not manifest: + manifest = self.manifest + all_projects_list = manifest.projects result = [] - mp = self.manifest.manifestProject + mp = manifest.manifestProject if not groups: groups = mp.config.GetString('manifest.groups') @@ -155,11 +158,11 @@ class Command(object): self._ResetPathToProjectMap(all_projects_list) for arg in args: - projects = self.manifest.GetProjectsWithName(arg) + projects = manifest.GetProjectsWithName(arg) if not projects: path = os.path.abspath(arg).replace('\\', '/') - project = self._GetProjectByPath(path) + project = self._GetProjectByPath(manifest, path) # If it's not a derived project, update path->project mapping and # search again, as arg might actually point to a derived subproject. @@ -170,7 +173,7 @@ class Command(object): self._UpdatePathToProjectMap(subproject) search_again = True if search_again: - project = self._GetProjectByPath(path) or project + project = self._GetProjectByPath(manifest, path) or project if project: projects = [project] diff --git a/gitc_utils.py b/gitc_utils.py index ef028b08..4d8d5366 100644 --- a/gitc_utils.py +++ b/gitc_utils.py @@ -16,6 +16,7 @@ from __future__ import print_function import os import sys +import time import git_command import git_config @@ -26,6 +27,18 @@ GITC_MANIFEST_DIR = '/usr/local/google/gitc/' GITC_FS_ROOT_DIR = '/gitc/manifest-rw/' NUM_BATCH_RETRIEVE_REVISIONID = 300 +def parse_clientdir(gitc_fs_path): + """Parse a path in the GITC FS and return its client name. + + @param gitc_fs_path: A subdirectory path within the GITC_FS_ROOT_DIR. + + @returns: The GITC client name + """ + if (gitc_fs_path == GITC_FS_ROOT_DIR or + not gitc_fs_path.startswith(GITC_FS_ROOT_DIR)): + return None + return gitc_fs_path.split(GITC_FS_ROOT_DIR)[1].split('/')[0] + def _set_project_revisions(projects): """Sets the revisionExpr for a list of projects. @@ -50,19 +63,37 @@ def _set_project_revisions(projects): sys.exit(1) proj.revisionExpr = gitcmd.stdout.split('\t')[0] -def generate_gitc_manifest(client_dir, manifest): +def generate_gitc_manifest(client_dir, manifest, projects=None): """Generate a manifest for shafsd to use for this GITC client. @param client_dir: GITC client directory to install the .manifest file in. @param manifest: XmlManifest object representing the repo manifest. + @param projects: List of projects we want to update, this must be a sublist + of manifest.projects to work properly. If not provided, + manifest.projects is used. """ print('Generating GITC Manifest by fetching revision SHAs for each ' 'project.') + if projects is None: + projects = manifest.projects index = 0 - while index < len(manifest.projects): + while index < len(projects): _set_project_revisions( - manifest.projects[index:(index+NUM_BATCH_RETRIEVE_REVISIONID)]) + projects[index:(index+NUM_BATCH_RETRIEVE_REVISIONID)]) index += NUM_BATCH_RETRIEVE_REVISIONID # Save the manifest. + save_manifest(manifest, client_dir=client_dir) + +def save_manifest(manifest, client_dir=None): + """Save the manifest file in the client_dir. + + @param client_dir: Client directory to save the manifest in. + @param manifest: Manifest object to save. + """ + if not client_dir: + client_dir = manifest.gitc_client_dir with open(os.path.join(client_dir, '.manifest'), 'w') as f: manifest.Save(f) + # TODO(sbasi/jorg): Come up with a solution to remove the sleep below. + # Give the GITC filesystem time to register the manifest changes. + time.sleep(3) \ No newline at end of file diff --git a/main.py b/main.py index 6736abc9..adfaffb0 100755 --- a/main.py +++ b/main.py @@ -51,7 +51,8 @@ from error import ManifestParseError from error import NoManifestException from error import NoSuchProjectError from error import RepoChangedException -from manifest_xml import XmlManifest +import gitc_utils +from manifest_xml import GitcManifest, XmlManifest from pager import RunPager from wrapper import WrapperPath, Wrapper @@ -129,6 +130,12 @@ class _Repo(object): cmd.repodir = self.repodir cmd.manifest = XmlManifest(cmd.repodir) + cmd.gitc_manifest = None + gitc_client_name = gitc_utils.parse_clientdir(os.getcwd()) + if gitc_client_name: + cmd.gitc_manifest = GitcManifest(cmd.repodir, gitc_client_name) + cmd.manifest.isGitcClient = True + Editor.globalConfig = cmd.manifest.globalConfig if not isinstance(cmd, MirrorSafeCommand) and cmd.manifest.IsMirror: diff --git a/manifest_xml.py b/manifest_xml.py index 6dc01a47..b33ec627 100644 --- a/manifest_xml.py +++ b/manifest_xml.py @@ -29,6 +29,7 @@ else: urllib = imp.new_module('urllib') urllib.parse = urlparse +import gitc_utils from git_config import GitConfig from git_refs import R_HEADS, HEAD from project import RemoteSpec, Project, MetaProject @@ -112,6 +113,7 @@ class XmlManifest(object): self.manifestFile = os.path.join(self.repodir, MANIFEST_FILE_NAME) self.globalConfig = GitConfig.ForUser() self.localManifestWarning = False + self.isGitcClient = False self.repoProject = MetaProject(self, 'repo', gitdir = os.path.join(repodir, 'repo/.git'), @@ -306,6 +308,8 @@ class XmlManifest(object): if p.clone_depth: e.setAttribute('clone-depth', str(p.clone_depth)) + self._output_manifest_project_extras(p, e) + if p.subprojects: subprojects = set(subp.name for subp in p.subprojects) output_projects(p, e, list(sorted(subprojects))) @@ -323,6 +327,10 @@ class XmlManifest(object): doc.writexml(fd, '', ' ', '\n', 'UTF-8') + def _output_manifest_project_extras(self, p, e): + """Manifests can modify e if they support extra project attributes.""" + pass + @property def paths(self): self._Load() @@ -712,7 +720,7 @@ class XmlManifest(object): def _UnjoinName(self, parent_name, name): return os.path.relpath(name, parent_name) - def _ParseProject(self, node, parent = None): + def _ParseProject(self, node, parent = None, **extra_proj_attrs): """ reads a element from the manifest file """ @@ -807,7 +815,8 @@ class XmlManifest(object): clone_depth = clone_depth, upstream = upstream, parent = parent, - dest_branch = dest_branch) + dest_branch = dest_branch, + **extra_proj_attrs) for n in node.childNodes: if n.nodeName == 'copyfile': @@ -938,3 +947,26 @@ class XmlManifest(object): diff['added'].append(toProjects[proj]) return diff + + +class GitcManifest(XmlManifest): + + def __init__(self, repodir, gitc_client_name): + """Initialize the GitcManifest object.""" + super(GitcManifest, self).__init__(repodir) + self.isGitcClient = True + self.gitc_client_name = gitc_client_name + self.gitc_client_dir = os.path.join(gitc_utils.GITC_MANIFEST_DIR, + gitc_client_name) + self.manifestFile = os.path.join(self.gitc_client_dir, '.manifest') + + def _ParseProject(self, node, parent = None): + """Override _ParseProject and add support for GITC specific attributes.""" + return super(GitcManifest, self)._ParseProject( + node, parent=parent, old_revision=node.getAttribute('old-revision')) + + def _output_manifest_project_extras(self, p, e): + """Output GITC Specific Project attributes""" + if p.old_revision: + e.setAttribute('old-revision', str(p.old_revision)) + diff --git a/project.py b/project.py index a8d012d2..f964b2fc 100644 --- a/project.py +++ b/project.py @@ -572,7 +572,8 @@ class Project(object): parent=None, is_derived=False, dest_branch=None, - optimized_fetch=False): + optimized_fetch=False, + old_revision=None): """Init a Project object. Args: @@ -596,6 +597,7 @@ class Project(object): dest_branch: The branch to which to push changes for review by default. optimized_fetch: If True, when a project is set to a sha1 revision, only fetch from the remote if the sha1 is not present locally. + old_revision: saved git commit id for open GITC projects. """ self.manifest = manifest self.name = name @@ -643,6 +645,7 @@ class Project(object): self.bare_ref = GitRefs(gitdir) self.bare_objdir = self._GitGetByExec(self, bare=True, gitdir=objdir) self.dest_branch = dest_branch + self.old_revision = old_revision # This will be filled in if a project is later identified to be the # project containing repo hooks. @@ -1195,6 +1198,8 @@ class Project(object): self._InitHooks() def _CopyAndLinkFiles(self): + if self.manifest.isGitcClient: + return for copyfile in self.copyfiles: copyfile._Copy() for linkfile in self.linkfiles: @@ -1425,9 +1430,11 @@ class Project(object): ## Branch Management ## - def StartBranch(self, name): + def StartBranch(self, name, branch_merge=''): """Create a new branch off the manifest's revision. """ + if not branch_merge: + branch_merge = self.revisionExpr head = self.work_git.GetHead() if head == (R_HEADS + name): return True @@ -1441,9 +1448,9 @@ class Project(object): branch = self.GetBranch(name) branch.remote = self.GetRemote(self.remote.name) - branch.merge = self.revisionExpr - if not branch.merge.startswith('refs/') and not ID_RE.match(self.revisionExpr): - branch.merge = R_HEADS + self.revisionExpr + branch.merge = branch_merge + if not branch.merge.startswith('refs/') and not ID_RE.match(branch_merge): + branch.merge = R_HEADS + branch_merge revid = self.GetRevisionId(all_refs) if head.startswith(R_HEADS): @@ -1451,7 +1458,6 @@ class Project(object): head = all_refs[head] except KeyError: head = None - if revid and head and revid == head: ref = os.path.join(self.gitdir, R_HEADS + name) try: diff --git a/subcmds/start.py b/subcmds/start.py index 60ad41e0..188fd7c6 100644 --- a/subcmds/start.py +++ b/subcmds/start.py @@ -14,11 +14,15 @@ # limitations under the License. from __future__ import print_function +import os import sys + from command import Command from git_config import IsId from git_command import git +import gitc_utils from progress import Progress +from project import SyncBuffer class Start(Command): common = True @@ -53,20 +57,50 @@ revision specified in the manifest. print("error: at least one project must be specified", file=sys.stderr) sys.exit(1) - all_projects = self.GetProjects(projects) + proj_name_to_gitc_proj_dict = {} + if self.gitc_manifest: + all_projects = self.GetProjects(projects, manifest=self.gitc_manifest, + missing_ok=True) + for project in all_projects: + if project.old_revision: + project.already_synced = True + else: + project.already_synced = False + project.old_revision = project.revisionExpr + proj_name_to_gitc_proj_dict[project.name] = project + project.revisionExpr = None + # Save the GITC manifest. + gitc_utils.save_manifest(self.gitc_manifest) + all_projects = self.GetProjects(projects, + missing_ok=bool(self.gitc_manifest)) pm = Progress('Starting %s' % nb, len(all_projects)) for project in all_projects: pm.update() + if self.gitc_manifest: + gitc_project = proj_name_to_gitc_proj_dict[project.name] + # Sync projects that have already been opened. + if not gitc_project.already_synced: + proj_localdir = os.path.join(self.gitc_manifest.gitc_client_dir, + project.relpath) + project.worktree = proj_localdir + if not os.path.exists(proj_localdir): + os.makedirs(proj_localdir) + project.Sync_NetworkHalf() + sync_buf = SyncBuffer(self.manifest.manifestProject.config) + project.Sync_LocalHalf(sync_buf) + project.revisionExpr = gitc_project.old_revision + # If the current revision is a specific SHA1 then we can't push back # to it; so substitute with dest_branch if defined, or with manifest # default revision instead. + branch_merge = '' if IsId(project.revisionExpr): if project.dest_branch: - project.revisionExpr = project.dest_branch + branch_merge = project.dest_branch else: - project.revisionExpr = self.manifest.default.revisionExpr - if not project.StartBranch(nb): + branch_merge = self.manifest.default.revisionExpr + if not project.StartBranch(nb, branch_merge=branch_merge): err.append(project) pm.end() diff --git a/subcmds/sync.py b/subcmds/sync.py index ad0ecdf4..934aaa80 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py @@ -549,15 +549,6 @@ later is required to fix a server side protocol bug. cwd.split(gitc_utils.GITC_MANIFEST_DIR)[1])) sys.exit(1) - self._gitc_sync = False - if cwd.startswith(gitc_utils.GITC_FS_ROOT_DIR): - self._gitc_sync = True - self._client_name = cwd.split(gitc_utils.GITC_FS_ROOT_DIR)[1].split( - '/')[0] - self._client_dir = os.path.join(gitc_utils.GITC_MANIFEST_DIR, - self._client_name) - print('Updating GITC client: %s' % self._client_name) - if opt.manifest_name: self.manifest.Override(opt.manifest_name) @@ -677,12 +668,6 @@ later is required to fix a server side protocol bug. if opt.repo_upgraded: _PostRepoUpgrade(self.manifest, quiet=opt.quiet) - if self._gitc_sync: - gitc_utils.generate_gitc_manifest(self._client_dir, self.manifest) - print('GITC client successfully synced.') - return - - if not opt.local_only: mp.Sync_NetworkHalf(quiet=opt.quiet, current_branch_only=opt.current_branch_only, @@ -697,6 +682,35 @@ later is required to fix a server side protocol bug. self._ReloadManifest(manifest_name) if opt.jobs is None: self.jobs = self.manifest.default.sync_j + + # TODO (sbasi) - Add support for manifest changes, aka projects + # have been added or deleted from the manifest. + if self.gitc_manifest: + gitc_manifest_projects = self.GetProjects(args, + manifest=self.gitc_manifest, + missing_ok=True) + gitc_projects = [] + opened_projects = [] + for project in gitc_manifest_projects: + if not project.old_revision: + gitc_projects.append(project) + else: + opened_projects.append(project) + + if gitc_projects and not opt.local_only: + print('Updating GITC client: %s' % self.gitc_manifest.gitc_client_name) + gitc_utils.generate_gitc_manifest(self.gitc_manifest.gitc_client_dir, + self.gitc_manifest, + gitc_projects) + print('GITC client successfully synced.') + + # The opened projects need to be synced as normal, therefore we + # generate a new args list to represent the opened projects. + args = [] + for proj in opened_projects: + args.append(os.path.relpath(proj.worktree, cwd)) + if not args: + return all_projects = self.GetProjects(args, missing_ok=True, submodules_ok=opt.fetch_submodules)