android/git-repo
1
Fork 0
mirror of https://gerrit.googlesource.com/git-repo synced 2024-12-21 07:16:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

sync: refactor main fetch loop

Browse Source 
This is a large chunk of code that is largely isolated.  Move it into
a class method to make it easier to manage & reason about, and in a
follow up CL, easier to scope.

Bug: https://crbug.com/gerrit/12389
Change-Id: I0c69d95a9e03478d347b761580b2343bffa012d5
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/305484
Tested-by: Mike Frysinger <vapier@google.com>
Reviewed-by: Chris Mcdonald <cjmcdonald@google.com>
This commit is contained in:
Mike Frysinger 2021-05-05 20:03:26 -04:00
parent 5291eafa41
commit b44294395f
1 changed files with 59 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

101
subcmds/sync.py
View File

@ -439,6 +439,63 @@ later is required to fix a server side protocol bug.
return (ret, fetched)
def _FetchMain(self, opt, args, all_projects, err_event, manifest_name,
load_local_manifests):
"""The main network fetch loop.
Args:
opt: Program options returned from optparse. See _Options().
args: Command line args used to filter out projects.
all_projects: List of all projects that should be checked out.
err_event: Whether an error was hit while processing.
manifest_name: Manifest file to be reloaded.
load_local_manifests: Whether to load local manifests.
"""
rp = self.manifest.repoProject
to_fetch = []
now = time.time()
if _ONE_DAY_S <= (now - rp.LastFetch):
to_fetch.append(rp)
to_fetch.extend(all_projects)
to_fetch.sort(key=self._fetch_times.Get, reverse=True)
success, fetched = self._Fetch(to_fetch, opt, err_event)
if not success:
err_event.set()
_PostRepoFetch(rp, opt.repo_verify)
if opt.network_only:
# bail out now; the rest touches the working tree
if err_event.is_set():
print('\nerror: Exited sync due to fetch errors.\n', file=sys.stderr)
sys.exit(1)
return
# Iteratively fetch missing and/or nested unregistered submodules
previously_missing_set = set()
while True:
self._ReloadManifest(manifest_name, load_local_manifests)
all_projects = self.GetProjects(args,
missing_ok=True,
submodules_ok=opt.fetch_submodules)
missing = []
for project in all_projects:
if project.gitdir not in fetched:
missing.append(project)
if not missing:
break
# Stop us from non-stopped fetching actually-missing repos: If set of
# missing repos has not been changed from last fetch, we break.
missing_set = set(p.name for p in missing)
if previously_missing_set == missing_set:
break
previously_missing_set = missing_set
success, new_fetched = self._Fetch(missing, opt, err_event)
if not success:
err_event.set()
fetched.update(new_fetched)
def _CheckoutOne(self, detach_head, force_sync, project):
"""Checkout work tree for one project
@ -921,48 +978,8 @@ later is required to fix a server side protocol bug.
self._fetch_times = _FetchTimes(self.manifest)
if not opt.local_only:
to_fetch = []
now = time.time()
if _ONE_DAY_S <= (now - rp.LastFetch):
to_fetch.append(rp)
to_fetch.extend(all_projects)
to_fetch.sort(key=self._fetch_times.Get, reverse=True)
success, fetched = self._Fetch(to_fetch, opt, err_event)
if not success:
err_event.set()
_PostRepoFetch(rp, opt.repo_verify)
if opt.network_only:
# bail out now; the rest touches the working tree
if err_event.is_set():
print('\nerror: Exited sync due to fetch errors.\n', file=sys.stderr)
sys.exit(1)
return
# Iteratively fetch missing and/or nested unregistered submodules
previously_missing_set = set()
while True:
self._ReloadManifest(manifest_name, load_local_manifests)
all_projects = self.GetProjects(args,
missing_ok=True,
submodules_ok=opt.fetch_submodules)
missing = []
for project in all_projects:
if project.gitdir not in fetched:
missing.append(project)
if not missing:
break
# Stop us from non-stopped fetching actually-missing repos: If set of
# missing repos has not been changed from last fetch, we break.
missing_set = set(p.name for p in missing)
if previously_missing_set == missing_set:
break
previously_missing_set = missing_set
success, new_fetched = self._Fetch(missing, opt, err_event)
if not success:
err_event.set()
fetched.update(new_fetched)
self._FetchMain(opt, args, all_projects, err_event, manifest_name,
load_local_manifests)
# If we saw an error, exit with code 1 so that other scripts can check.
if err_event.is_set():