diff --git a/gitc_utils.py b/gitc_utils.py new file mode 100644 index 00000000..bf79bd28 --- /dev/null +++ b/gitc_utils.py @@ -0,0 +1,69 @@ +# +# Copyright (C) 2015 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function +import os +import shutil + +import git_command +import git_config + + +# TODO (sbasi) - Remove this constant and fetch manifest dir from /gitc/.config +GITC_MANIFEST_DIR = '/usr/local/google/gitc/' +GITC_FS_ROOT_DIR = '/gitc/manifest-rw/' +NUM_BATCH_RETRIEVE_REVISIONID = 300 + +def _set_project_revisions(projects): + """Sets the revisionExpr for a list of projects. + + Because of the limit of open file descriptors allowed, length of projects + should not be overly large. Recommend calling this function multiple times + with each call not exceeding NUM_BATCH_RETRIEVE_REVISIONID projects. + + @param projects: List of project objects to set the revionExpr for. + """ + # Retrieve the commit id for each project based off of it's current + # revisionExpr and it is not already a commit id. + project_gitcmds = [( + project, git_command.GitCommand(None, + ['ls-remote', + project.remote.url, + project.revisionExpr], + capture_stdout=True, cwd='/tmp')) + for project in projects if not git_config.IsId(project.revisionExpr)] + for proj, gitcmd in project_gitcmds: + if gitcmd.Wait(): + print('FATAL: Failed to retrieve revisionExpr for %s' % project) + sys.exit(1) + proj.revisionExpr = gitcmd.stdout.split('\t')[0] + +def generate_gitc_manifest(client_dir, manifest): + """Generate a manifest for shafsd to use for this GITC client. + + @param client_dir: GITC client directory to install the .manifest file in. + @param manifest: XmlManifest object representing the repo manifest. + """ + print('Generating GITC Manifest by fetching revision SHAs for each ' + 'project.') + project_gitcmd_dict = {} + index = 0 + while index < len(manifest.projects): + _set_project_revisions( + manifest.projects[index:(index+NUM_BATCH_RETRIEVE_REVISIONID)]) + index += NUM_BATCH_RETRIEVE_REVISIONID + # Save the manifest. + with open(os.path.join(client_dir, '.manifest'), 'w') as f: + manifest.Save(f) diff --git a/subcmds/gitc_init.py b/subcmds/gitc_init.py index 9b9cefda..03d8cc30 100644 --- a/subcmds/gitc_init.py +++ b/subcmds/gitc_init.py @@ -18,15 +18,10 @@ import os import shutil import sys -import git_command +import gitc_utils from subcmds import init -GITC_MANIFEST_DIR = '/usr/local/google/gitc' -GITC_FS_ROOT_DIR = '/gitc/sha/rw' -NUM_BATCH_RETRIEVE_REVISIONID = 300 - - class GitcInit(init.Init): common = True helpSummary = "Initialize a GITC Client." @@ -65,59 +60,21 @@ use for this GITC client. if not opt.gitc_client: print('fatal: gitc client (-c) is required', file=sys.stderr) sys.exit(1) - self.client_dir = os.path.join(GITC_MANIFEST_DIR, opt.gitc_client) - if not os.path.exists(GITC_MANIFEST_DIR): - os.makedirs(GITC_MANIFEST_DIR) + self.client_dir = os.path.join(gitc_utils.GITC_MANIFEST_DIR, + opt.gitc_client) + if not os.path.exists(gitc_utils.GITC_MANIFEST_DIR): + os.makedirs(gitc_utils.GITC_MANIFEST_DIR) if not os.path.exists(self.client_dir): os.mkdir(self.client_dir) super(GitcInit, self).Execute(opt, args) + # Make the destination manifest file a symlink to repo's so both repo and + # GITC refer to the same manifest. if opt.manifest_file: if not os.path.exists(opt.manifest_file): print('fatal: Specified manifest file %s does not exist.' % opt.manifest_file) sys.exit(1) - shutil.copyfile(opt.manifest_file, - os.path.join(self.client_dir, '.manifest')) - else: - self._GenerateGITCManifest() + self.manifest.Override(opt.manifest_file) + gitc_utils.generate_gitc_manifest(self.client_dir, self.manifest) print('Please run `cd %s` to view your GITC client.' % - os.path.join(GITC_FS_ROOT_DIR, opt.gitc_client)) - - def _SetProjectRevisions(self, projects, branch): - """Sets the revisionExpr for a list of projects. - - Because of the limit of open file descriptors allowed, length of projects - should not be overly large. Recommend calling this function multiple times - with each call not exceeding NUM_BATCH_RETRIEVE_REVISIONID projects. - - @param projects: List of project objects to set the revionExpr for. - @param branch: The remote branch to retrieve the SHA from. If branch is - None, 'HEAD' is used. - """ - project_gitcmds = [( - project, git_command.GitCommand(None, - ['ls-remote', - project.remote.url, - branch], capture_stdout=True)) - for project in projects] - for proj, gitcmd in project_gitcmds: - if gitcmd.Wait(): - print('FATAL: Failed to retrieve revisionID for %s' % project) - sys.exit(1) - proj.revisionExpr = gitcmd.stdout.split('\t')[0] - - def _GenerateGITCManifest(self): - """Generate a manifest for shafsd to use for this GITC client.""" - print('Generating GITC Manifest by fetching revision SHAs for each ' - 'project.') - manifest = self.manifest - project_gitcmd_dict = {} - index = 0 - while index < len(manifest.projects): - self._SetProjectRevisions( - manifest.projects[index:(index+NUM_BATCH_RETRIEVE_REVISIONID)], - manifest.default.revisionExpr) - index += NUM_BATCH_RETRIEVE_REVISIONID - # Save the manifest. - with open(os.path.join(self.client_dir, '.manifest'), 'w') as f: - manifest.Save(f) + os.path.join(gitc_utils.GITC_FS_ROOT_DIR, opt.gitc_client)) \ No newline at end of file diff --git a/subcmds/sync.py b/subcmds/sync.py index 43d450be..652a0c0d 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py @@ -58,6 +58,7 @@ except ImportError: from git_command import GIT, git_require from git_refs import R_HEADS, HEAD +import gitc_utils from project import Project from project import RemoteSpec from command import Command, MirrorSafeCommand @@ -184,6 +185,9 @@ later is required to fix a server side protocol bug. help="overwrite an existing git directory if it needs to " "point to a different object directory. WARNING: this " "may cause loss of data") + p.add_option('--force-gitc', + dest='force_gitc', action='store_true', + help="actually sync sources in the gitc client directory.") p.add_option('-l', '--local-only', dest='local_only', action='store_true', help="only update working tree, don't fetch") @@ -526,6 +530,25 @@ later is required to fix a server side protocol bug. print('error: both -u and -p must be given', file=sys.stderr) sys.exit(1) + cwd = os.getcwd() + if cwd.startswith(gitc_utils.GITC_MANIFEST_DIR) and not opt.force_gitc: + print('WARNING this will pull all the sources like a normal repo sync.\n' + '\nIf you want to update your GITC Client View please rerun this ' + 'command in \n%s%s.\nOr if you actually want to pull the sources, ' + 'rerun with --force-gitc.' % + (gitc_utils.GITC_FS_ROOT_DIR, + cwd.split(gitc_utils.GITC_MANIFEST_DIR)[1])) + sys.exit(1) + + self._gitc_sync = False + if cwd.startswith(gitc_utils.GITC_FS_ROOT_DIR): + self._gitc_sync = True + self._client_name = cwd.split(gitc_utils.GITC_FS_ROOT_DIR)[1].split( + '/')[0] + self._client_dir = os.path.join(gitc_utils.GITC_MANIFEST_DIR, + self._client_name) + print('Updating GITC client: %s' % self._client_name) + if opt.manifest_name: self.manifest.Override(opt.manifest_name) @@ -642,6 +665,12 @@ later is required to fix a server side protocol bug. if opt.repo_upgraded: _PostRepoUpgrade(self.manifest, quiet=opt.quiet) + if self._gitc_sync: + gitc_utils.generate_gitc_manifest(self._client_dir, self.manifest) + print('GITC client successfully synced.') + return + + if not opt.local_only: mp.Sync_NetworkHalf(quiet=opt.quiet, current_branch_only=opt.current_branch_only,