Merge "sync: Order projects according to last fetch time"

This commit is contained in:
Shawn Pearce 2012-10-25 17:33:36 -07:00 committed by Gerrit Code Review
commit e48d34659e

View File

@ -16,6 +16,7 @@
import netrc import netrc
from optparse import SUPPRESS_HELP from optparse import SUPPRESS_HELP
import os import os
import pickle
import re import re
import shutil import shutil
import socket import socket
@ -47,6 +48,8 @@ from error import RepoChangedException, GitError
from project import SyncBuffer from project import SyncBuffer
from progress import Progress from progress import Progress
_ONE_DAY_S = 24 * 60 * 60
class _FetchError(Exception): class _FetchError(Exception):
"""Internal error thrown in _FetchHelper() when we don't want stack trace.""" """Internal error thrown in _FetchHelper() when we don't want stack trace."""
pass pass
@ -212,10 +215,12 @@ later is required to fix a server side protocol bug.
# - We always make sure we unlock the lock if we locked it. # - We always make sure we unlock the lock if we locked it.
try: try:
try: try:
start = time.time()
success = project.Sync_NetworkHalf( success = project.Sync_NetworkHalf(
quiet=opt.quiet, quiet=opt.quiet,
current_branch_only=opt.current_branch_only, current_branch_only=opt.current_branch_only,
clone_bundle=not opt.no_clone_bundle) clone_bundle=not opt.no_clone_bundle)
self._fetch_times.Set(project, time.time() - start)
# Lock around all the rest of the code, since printing, updating a set # Lock around all the rest of the code, since printing, updating a set
# and Progress.update() are not thread safe. # and Progress.update() are not thread safe.
@ -293,6 +298,7 @@ later is required to fix a server side protocol bug.
sys.exit(1) sys.exit(1)
pm.end() pm.end()
self._fetch_times.Save()
for project in projects: for project in projects:
project.bare_git.gc('--auto') project.bare_git.gc('--auto')
return fetched return fetched
@ -496,12 +502,15 @@ uncommitted changes are present' % project.relpath
self.jobs = self.manifest.default.sync_j self.jobs = self.manifest.default.sync_j
all_projects = self.GetProjects(args, missing_ok=True) all_projects = self.GetProjects(args, missing_ok=True)
self._fetch_times = _FetchTimes(self.manifest)
if not opt.local_only: if not opt.local_only:
to_fetch = [] to_fetch = []
now = time.time() now = time.time()
if (24 * 60 * 60) <= (now - rp.LastFetch): if _ONE_DAY_S <= (now - rp.LastFetch):
to_fetch.append(rp) to_fetch.append(rp)
to_fetch.extend(all_projects) to_fetch.extend(all_projects)
to_fetch.sort(key=self._fetch_times.Get, reverse=True)
self._fetch_times.Clear()
fetched = self._Fetch(to_fetch, opt) fetched = self._Fetch(to_fetch, opt)
_PostRepoFetch(rp, opt.no_repo_verify) _PostRepoFetch(rp, opt.no_repo_verify)
@ -621,3 +630,53 @@ warning: Cannot automatically authenticate repo."""
print >>sys.stderr print >>sys.stderr
return False return False
return True return True
class _FetchTimes(object):
def __init__(self, manifest):
self._path = os.path.join(manifest.repodir, '.repopickle_fetchtimes')
self._times = None
def Clear(self):
self._times = {}
def Get(self, project):
self._Load()
return self._times.get(project.name, _ONE_DAY_S)
def Set(self, project, t):
self._times[project.name] = t
def _Load(self):
if self._times is None:
try:
f = open(self._path)
except IOError:
self._times = {}
return self._times
try:
try:
self._times = pickle.load(f)
except:
try:
os.remove(self._path)
except OSError:
pass
self._times = {}
finally:
f.close()
return self._times
def Save(self):
if self._times is None:
return
try:
f = open(self._path, 'wb')
try:
pickle.dump(self._times, f)
except (IOError, OSError, pickle.PickleError):
try:
os.remove(self._path)
except OSError:
pass
finally:
f.close()