From c325dc35f621fe24d0460bb14547cdb51e00c00b Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Mon, 3 Oct 2011 08:30:24 -0700 Subject: [PATCH] sync: Fetch after applying bundle and retry after errors After a $GIT_URL/clone.bundle has been applied to the new local repository, perform an incremental fetch using `git fetch` to ensure the local repository is up-to-date. This allows the hosting server to offer stale /clone.bundle files to bootstrap a new client. If a single git fetch fails, it may succeed again after a short delay. Transient failures are typical in environments where the remote Git server happens to have limits on how many requests it can serve at once (the anonymous git daemon, or an HTTP server). Wait a randomized delay between 30 and 45 seconds and retry the failed project once. This delay gives the site time to recover from a transient traffic spike, and the randomization makes it less likely that a spike occurs again from all of the same clients. Change-Id: I97fb0fcb33630fb78ac1a21d1a4a3e2268ab60c0 Signed-off-by: Shawn O. Pearce --- project.py | 134 +++++++++++++++++++++++++++++------------------------ 1 file changed, 73 insertions(+), 61 deletions(-) diff --git a/project.py b/project.py index 5adfe82e..43f4713c 100644 --- a/project.py +++ b/project.py @@ -16,10 +16,12 @@ import traceback import errno import filecmp import os +import random import re import shutil import stat import sys +import time import urllib2 from color import Coloring @@ -894,9 +896,25 @@ class Project(object): is_new = not self.Exists if is_new: self._InitGitDir() - self._InitRemote() - if not self._RemoteFetch(initial=is_new, quiet=quiet): + + if is_new: + alt = os.path.join(self.gitdir, 'objects/info/alternates') + try: + fd = open(alt, 'rb') + try: + alt_dir = fd.readline().rstrip() + finally: + fd.close() + except IOError: + alt_dir = None + else: + alt_dir = None + + if alt_dir is None and self._ApplyCloneBundle(initial=is_new, quiet=quiet): + is_new = False + + if not self._RemoteFetch(initial=is_new, quiet=quiet, alt_dir=alt_dir): return False #Check that the requested ref was found after fetch @@ -1307,7 +1325,8 @@ class Project(object): def _RemoteFetch(self, name=None, tag=None, initial=False, - quiet=False): + quiet=False, + alt_dir=None): if not name: name = self.remote.name @@ -1316,29 +1335,9 @@ class Project(object): if remote.PreConnectFetch(): ssh_proxy = True - bundle_dst = os.path.join(self.gitdir, 'clone.bundle') - bundle_tmp = os.path.join(self.gitdir, 'clone.bundle.tmp') - use_bundle = False - if os.path.exists(bundle_dst) or os.path.exists(bundle_tmp): - use_bundle = True - if initial: - alt = os.path.join(self.gitdir, 'objects/info/alternates') - try: - fd = open(alt, 'rb') - try: - ref_dir = fd.readline() - if ref_dir and ref_dir.endswith('\n'): - ref_dir = ref_dir[:-1] - finally: - fd.close() - except IOError, e: - ref_dir = None - - if ref_dir and 'objects' == os.path.basename(ref_dir): - if use_bundle: - use_bundle = False - ref_dir = os.path.dirname(ref_dir) + if alt_dir and 'objects' == os.path.basename(alt_dir): + ref_dir = os.path.dirname(alt_dir) packed_refs = os.path.join(self.gitdir, 'packed-refs') remote = self.GetRemote(name) @@ -1374,10 +1373,8 @@ class Project(object): old_packed += line _lwrite(packed_refs, tmp_packed) - else: - ref_dir = None - use_bundle = True + alt_dir = None cmd = ['fetch'] @@ -1386,59 +1383,74 @@ class Project(object): depth = self.manifest.manifestProject.config.GetString('repo.depth') if depth and initial: cmd.append('--depth=%s' % depth) - use_bundle = False if quiet: cmd.append('--quiet') if not self.worktree: cmd.append('--update-head-ok') + cmd.append(name) + if tag is not None: + cmd.append('tag') + cmd.append(tag) - if use_bundle and not os.path.exists(bundle_dst): - bundle_url = remote.url + '/clone.bundle' - bundle_url = GitConfig.ForUser().UrlInsteadOf(bundle_url) - if GetSchemeFromUrl(bundle_url) in ('http', 'https'): - use_bundle = self._FetchBundle( - bundle_url, - bundle_tmp, - bundle_dst, - quiet=quiet) - else: - use_bundle = False - - if use_bundle: - if not quiet: - cmd.append('--quiet') - cmd.append(bundle_dst) - for f in remote.fetch: - cmd.append(str(f)) - cmd.append('refs/tags/*:refs/tags/*') - else: - cmd.append(name) - if tag is not None: - cmd.append('tag') - cmd.append(tag) - - ok = GitCommand(self, - cmd, - bare = True, - ssh_proxy = ssh_proxy).Wait() == 0 + ok = False + for i in range(2): + if GitCommand(self, cmd, bare=True, ssh_proxy=ssh_proxy).Wait() == 0: + ok = True + break + time.sleep(random.randint(30, 45)) if initial: - if ref_dir: + if alt_dir: if old_packed != '': _lwrite(packed_refs, old_packed) else: os.remove(packed_refs) self.bare_git.pack_refs('--all', '--prune') + return ok + def _ApplyCloneBundle(self, initial=False, quiet=False): + if initial and self.manifest.manifestProject.config.GetString('repo.depth'): + return False + + remote = self.GetRemote(self.remote.name) + bundle_url = remote.url + '/clone.bundle' + bundle_url = GitConfig.ForUser().UrlInsteadOf(bundle_url) + if GetSchemeFromUrl(bundle_url) not in ('http', 'https'): + return False + + bundle_dst = os.path.join(self.gitdir, 'clone.bundle') + bundle_tmp = os.path.join(self.gitdir, 'clone.bundle.tmp') + + exist_dst = os.path.exists(bundle_dst) + exist_tmp = os.path.exists(bundle_tmp) + + if not initial and not exist_dst and not exist_tmp: + return False + + if not exist_dst: + exist_dst = self._FetchBundle(bundle_url, bundle_tmp, bundle_dst, quiet) + if not exist_dst: + return False + + cmd = ['fetch'] + if quiet: + cmd.append('--quiet') + if not self.worktree: + cmd.append('--update-head-ok') + cmd.append(bundle_dst) + for f in remote.fetch: + cmd.append(str(f)) + cmd.append('refs/tags/*:refs/tags/*') + + ok = GitCommand(self, cmd, bare=True).Wait() == 0 if os.path.exists(bundle_dst): os.remove(bundle_dst) if os.path.exists(bundle_tmp): os.remove(bundle_tmp) - return ok - def _FetchBundle(self, srcUrl, tmpPath, dstPath, quiet=False): + def _FetchBundle(self, srcUrl, tmpPath, dstPath, quiet): keep = True done = False dest = open(tmpPath, 'a+b')