Mercurial > hg-git
changeset 1123:70c9dfdb0bc1
merge with 0.8.11 release
| author | Kevin Bullock <kbullock@ringworld.org> |
|---|---|
| date | Sat, 24 Feb 2018 17:30:57 -0600 |
| parents | e01f59918160 (diff) d38cade1e09e (current diff) |
| children | 666ab5997802 |
| files | hggit/__init__.py |
| diffstat | 23 files changed, 1169 insertions(+), 427 deletions(-) [+] |
line wrap: on
line diff
--- a/hggit/__init__.py Sat Feb 24 17:22:27 2018 -0600 +++ b/hggit/__init__.py Sat Feb 24 17:30:57 2018 -0600 @@ -112,6 +112,7 @@ def localpath(self): return self.p + def _isgitdir(path): """True if the given file path is a git repo.""" if os.path.exists(os.path.join(path, '.hg')): @@ -129,6 +130,7 @@ return False + def _local(path): p = urlcls(path).localpath() if _isgitdir(p): @@ -138,8 +140,10 @@ return gitrepo return _oldlocal(path) + hg.schemes['file'] = _local + # we need to wrap this so that git-like ssh paths are not prepended with a # local filesystem path. ugh. def _url(orig, path, **kwargs): @@ -157,6 +161,7 @@ return orig(gituri, **kwargs) return orig(path, **kwargs) + extensions.wrapfunction(hgutil, 'url', _url) @@ -172,10 +177,12 @@ return httpgitscheme + hg.schemes['https'] = _httpgitwrapper(hg.schemes['https']) hg.schemes['http'] = _httpgitwrapper(hg.schemes['http']) +hgdefaultdest = hg.defaultdest -hgdefaultdest = hg.defaultdest + def defaultdest(source): for scheme in util.gitschemes: if source.startswith('%s://' % scheme) and source.endswith('.git'): @@ -185,23 +192,30 @@ return hgdefaultdest(source[:-4]) return hgdefaultdest(source) + + hg.defaultdest = defaultdest + def getversion(): """return version with dependencies for hg --version -v""" import dulwich dulver = '.'.join(str(i) for i in dulwich.__version__) return __version__ + (" (dulwich %s)" % dulver) + # defend against tracebacks if we specify -r in 'hg pull' def safebranchrevs(orig, lrepo, repo, branches, revs): revs, co = orig(lrepo, repo, branches, revs) if hgutil.safehasattr(lrepo, 'changelog') and co not in lrepo.changelog: co = None return revs, co + + if getattr(hg, 'addbranchrevs', False): extensions.wrapfunction(hg, 'addbranchrevs', safebranchrevs) + def extsetup(ui): templatekw.keywords.update({'gitnode': gitnodekw}) revset.symbols.update({ @@ -213,6 +227,7 @@ lambda *args: open(os.path.join(helpdir, 'git.rst')).read()) insort(help.helptable, entry) + def reposetup(ui, repo): if not isinstance(repo, gitrepo.gitrepo): @@ -231,21 +246,25 @@ klass = hgrepo.generate_repo_subclass(repo.__class__) repo.__class__ = klass + if hgutil.safehasattr(manifest, '_lazymanifest'): # Mercurial >= 3.4 extensions.wrapfunction(manifest.manifestdict, 'diff', overlay.wrapmanifestdictdiff) + @command('gimport') def gimport(ui, repo, remote_name=None): '''import commits from Git to Mercurial''' repo.githandler.import_commits(remote_name) + @command('gexport') def gexport(ui, repo): '''export commits from Mercurial to Git''' repo.githandler.export_commits() + @command('gclear') def gclear(ui, repo): '''clear out the Git cached data @@ -258,20 +277,22 @@ repo.ui.status(_("clearing out the git cache data\n")) repo.githandler.clear() + @command('gverify', [('r', 'rev', '', _('revision to verify'), _('REV'))], _('[-r REV]')) def gverify(ui, repo, **opts): '''verify that a Mercurial rev matches the corresponding Git rev - Given a Mercurial revision that has a corresponding Git revision in the map, - this attempts to answer whether that revision has the same contents as the - corresponding Git revision. + Given a Mercurial revision that has a corresponding Git revision in the + map, this attempts to answer whether that revision has the same contents as + the corresponding Git revision. ''' ctx = scmutil.revsingle(repo, opts.get('rev'), '.') return verify.verify(ui, repo, ctx) + @command('git-cleanup') def git_cleanup(ui, repo): '''clean up Git commit map after history editing''' @@ -289,6 +310,7 @@ wlock.release() ui.status(_('git commit map cleaned\n')) + def findcommonoutgoing(orig, repo, other, *args, **kwargs): if isinstance(other, gitrepo.gitrepo): heads = repo.githandler.get_refs(other.path)[0] @@ -305,8 +327,11 @@ kw['commoninc'] = commoninc return orig(repo, other, **kw) return orig(repo, other, *args, **kwargs) + + extensions.wrapfunction(discovery, 'findcommonoutgoing', findcommonoutgoing) + def getremotechanges(orig, ui, repo, other, *args, **opts): if isinstance(other, gitrepo.gitrepo): if args: @@ -319,22 +344,31 @@ cleanup = None return r, c, cleanup return orig(ui, repo, other, *args, **opts) + + extensions.wrapfunction(bundlerepo, 'getremotechanges', getremotechanges) + def peer(orig, uiorrepo, *args, **opts): newpeer = orig(uiorrepo, *args, **opts) if isinstance(newpeer, gitrepo.gitrepo): if isinstance(uiorrepo, localrepo.localrepository): newpeer.localrepo = uiorrepo return newpeer + + extensions.wrapfunction(hg, 'peer', peer) + def isvalidlocalpath(orig, self, path): return orig(self, path) or _isgitdir(path) + + if (hgutil.safehasattr(hgui, 'path') and hgutil.safehasattr(hgui.path, '_isvalidlocalpath')): extensions.wrapfunction(hgui.path, '_isvalidlocalpath', isvalidlocalpath) + @util.transform_notgit def exchangepull(orig, repo, remote, heads=None, force=False, bookmarks=(), **kwargs): @@ -366,10 +400,13 @@ wlock.release() else: return orig(repo, remote, heads, force, bookmarks=bookmarks, **kwargs) + + if not hgutil.safehasattr(localrepo.localrepository, 'pull'): # Mercurial >= 3.2 extensions.wrapfunction(exchange, 'pull', exchangepull) + # TODO figure out something useful to do with the newbranch param @util.transform_notgit def exchangepush(orig, repo, remote, force=False, revs=None, newbranch=False, @@ -386,10 +423,13 @@ else: return orig(repo, remote, force, revs, newbranch, bookmarks=bookmarks, **kwargs) + + if not hgutil.safehasattr(localrepo.localrepository, 'push'): # Mercurial >= 3.2 extensions.wrapfunction(exchange, 'push', exchangepush) + def revset_fromgit(repo, subset, x): '''``fromgit()`` Select changesets that originate from Git. @@ -400,6 +440,7 @@ return baseset(r for r in subset if git.map_git_get(hex(node(r))) is not None) + def revset_gitnode(repo, subset, x): '''``gitnode(hash)`` Select the changeset that originates in the given Git revision. The hash @@ -422,8 +463,10 @@ return result raise LookupError(rev, git.map_file, _('ambiguous identifier')) + def gitnodekw(**args): - """:gitnode: String. The Git changeset identification hash, as a 40 hexadecimal digit string.""" + """:gitnode: String. The Git changeset identification hash, as a 40 hexadecimal +digit string.""" node = args['ctx'] repo = args['repo'] gitnode = repo.githandler.map_git_get(node.hex())
--- a/hggit/_ssh.py Sat Feb 24 17:22:27 2018 -0600 +++ b/hggit/_ssh.py Sat Feb 24 17:30:57 2018 -0600 @@ -5,6 +5,7 @@ util, ) + class SSHVendor(object): """Parent class for ui-linked Vendor classes."""
--- a/hggit/compat.py Sat Feb 24 17:22:27 2018 -0600 +++ b/hggit/compat.py Sat Feb 24 17:30:57 2018 -0600 @@ -37,6 +37,7 @@ from mercurial import scmutil hgvfs = scmutil.vfs + def gitvfs(repo): """return a vfs suitable to read git related data""" # Mercurial >= 3.3: repo.shared() @@ -45,6 +46,7 @@ else: return repo.vfs + def passwordmgr(ui): try: realm = hgutil.urlreq.httppasswordmgrwithdefaultrealm() @@ -139,6 +141,7 @@ hasconfigitems = False + def registerconfigs(configitem): global hasconfigitems hasconfigitems = True @@ -146,6 +149,7 @@ for item, default in items.iteritems(): configitem(section, item, default=default) + def config(ui, subtype, section, item): if subtype == 'string': subtype = ''
--- a/hggit/git2hg.py Sat Feb 24 17:22:27 2018 -0600 +++ b/hggit/git2hg.py Sat Feb 24 17:30:57 2018 -0600 @@ -3,12 +3,16 @@ import urllib from dulwich.objects import Commit, Tag + def find_incoming(git_object_store, git_map, refs): '''find what commits need to be imported - git_object_store is a dulwich object store. - git_map is a map with keys being Git commits that have already been imported - refs is a map of refs to SHAs that we're interested in.''' + git_object_store: is a dulwich object store. + git_map: is a map with keys being Git commits that have already been + imported + refs: is a map of refs to SHAs that we're interested in. + + ''' done = set() commit_cache = {} @@ -16,7 +20,7 @@ # sort by commit date def commitdate(sha): obj = git_object_store[sha] - return obj.commit_time-obj.commit_timezone + return obj.commit_time - obj.commit_timezone # get a list of all the head shas def get_heads(refs): @@ -73,25 +77,30 @@ return GitIncomingResult(commits, commit_cache) + class GitIncomingResult(object): '''struct to store result from find_incoming''' def __init__(self, commits, commit_cache): self.commits = commits self.commit_cache = commit_cache + def extract_hg_metadata(message, git_extra): split = message.split("\n--HG--\n", 1) # Renames are explicitly stored in Mercurial but inferred in Git. For # commits that originated in Git we'd like to optionally infer rename # information to store in Mercurial, but for commits that originated in # Mercurial we'd like to disable this. How do we tell whether the commit - # originated in Mercurial or in Git? We rely on the presence of extra hg-git - # fields in the Git commit. - # - Commits exported by hg-git versions past 0.7.0 always store at least one - # hg-git field. + # originated in Mercurial or in Git? We rely on the presence of extra + # hg-git fields in the Git commit. + # + # - Commits exported by hg-git versions past 0.7.0 always store at least + # one hg-git field. + # # - For commits exported by hg-git versions before 0.7.0, this becomes a - # heuristic: if the commit has any extra hg fields, it definitely originated - # in Mercurial. If the commit doesn't, we aren't really sure. + # heuristic: if the commit has any extra hg fields, it definitely + # originated in Mercurial. If the commit doesn't, we aren't really sure. + # # If we think the commit originated in Mercurial, we set renames to a # dict. If we don't, we set renames to None. Callers can then determine # whether to infer rename information.
--- a/hggit/git_handler.py Sat Feb 24 17:22:27 2018 -0600 +++ b/hggit/git_handler.py Sat Feb 24 17:30:57 2018 -0600 @@ -24,7 +24,6 @@ encoding, phases, util as hgutil, - url, ) import _ssh @@ -57,6 +56,7 @@ CALLBACK_BUFFER = '' + class GitProgress(object): """convert git server progress strings into mercurial progress""" def __init__(self, ui): @@ -97,6 +97,7 @@ if msg: self.ui.note(msg + '\n') + class GitHandler(object): map_file = 'git-mapfile' remote_refs_file = 'git-remote-refs' @@ -240,7 +241,7 @@ paths = self.paths # if paths are set, we should still check 'default' if not paths: - paths = [('default', None),] + paths = (('default', None),) # we avoid using dulwich's refs method because it is incredibly slow; # on a repo with a few hundred branches and a few thousand tags, @@ -477,7 +478,7 @@ nodes = (clnode(n) for n in repo) to_export = (repo[node] for node in nodes if not hex(node) in - self._map_hg) + self._map_hg) todo_total = len(repo) - len(self._map_hg) topic = 'find commits to export' @@ -520,7 +521,7 @@ self.repo, pctx, self.git.object_store, gitcommit) mapsavefreq = compat.config(self.ui, 'int', 'hggit', - 'mapsavefrequency') + 'mapsavefrequency') for i, ctx in enumerate(export): self.ui.progress('exporting', i, total=total) self.export_hg_commit(ctx.node(), exporter) @@ -575,7 +576,7 @@ else: timezone = -int(timezone) commit.commit_timezone = timezone - except: # extra is essentially user-supplied, we must be careful + except: # extra is essentially user-supplied, we must be careful self.set_commiter_from_author(commit) else: self.set_commiter_from_author(commit) @@ -596,6 +597,8 @@ if 'encoding' in extra: commit.encoding = extra['encoding'] + if 'gpgsig' in extra: + commit.gpgsig = extra['gpgsig'] for obj, nodeid in exporter.update_changeset(ctx): if obj.id not in self.git.object_store: @@ -680,7 +683,7 @@ if a: name = self.get_valid_git_username_email(a.group(1)) email = self.get_valid_git_username_email(a.group(2)) - if a.group(3) != None and len(a.group(3)) != 0: + if a.group(3) is not None and len(a.group(3)) != 0: name += ' ext:(' + urllib.quote(a.group(3)) + ')' author = '%s <%s>' \ % (self.get_valid_git_username_email(name), @@ -810,7 +813,7 @@ self.ui.status(_("no changes found\n")) mapsavefreq = compat.config(self.ui, 'int', 'hggit', - 'mapsavefrequency') + 'mapsavefrequency') for i, csha in enumerate(commits): self.ui.progress('importing', i, total=total, unit='commits') commit = commit_cache[csha] @@ -1028,6 +1031,8 @@ if commit.encoding: extra['encoding'] = commit.encoding + if commit.gpgsig: + extra['gpgsig'] = commit.gpgsig if octopus: extra['hg-git'] = 'octopus-done' @@ -1206,10 +1211,10 @@ localclient, path = self.get_transport_and_path(remote_name) # The dulwich default walk only checks refs/heads/. We also want to - # consider remotes when doing discovery, so we build our own list. We + # consider remotes when doing discovery, so we build our own list. We # can't just do 'refs/' here because the tag class doesn't have a - # parents function for walking, and older versions of dulwich don't like - # that. + # parents function for walking, and older versions of dulwich don't + # like that. haveheads = self.git.refs.as_dict('refs/remotes/').values() haveheads.extend(self.git.refs.as_dict('refs/heads/').values()) graphwalker = self.git.get_graph_walker(heads=haveheads) @@ -1248,9 +1253,8 @@ if head: symrefs["HEAD"] = head # use FetchPackResult as upstream now does - ret = compat.FetchPackResult(ret, - symrefs, - client.default_user_agent_string()) + agent = client.default_user_agent_string() + ret = compat.FetchPackResult(ret, symrefs, agent) return ret except (HangupException, GitProtocolError), e: raise hgutil.Abort(_("git remote error: ") + str(e)) @@ -1695,12 +1699,13 @@ >>> mockrepo.sharedpath = '' >>> mockrepo.path = '' >>> g = GitHandler(mockrepo, ui()) - >>> client, url = g.get_transport_and_path('http://fqdn.com/test.git') + >>> tp = g.get_transport_and_path + >>> client, url = tp('http://fqdn.com/test.git') >>> print isinstance(client, HttpGitClient) True >>> print url http://fqdn.com/test.git - >>> client, url = g.get_transport_and_path('git@fqdn.com:user/repo.git') + >>> client, url = tp('git@fqdn.com:user/repo.git') >>> print isinstance(client, SSHGitClient) True >>> print url
--- a/hggit/gitdirstate.py Sat Feb 24 17:22:27 2018 -0600 +++ b/hggit/gitdirstate.py Sat Feb 24 17:30:57 2018 -0600 @@ -25,6 +25,7 @@ pathutil = scmutil from mercurial.i18n import _ + def gignorepats(orig, lines, root=None): '''parse lines (iterable) of .gitignore text, returning a tuple of (patterns, parse errors). These patterns should be given to compile() @@ -70,6 +71,7 @@ return patterns, warnings + def gignore(root, files, warn, extrapatterns=None): allpats = [] pats = [] @@ -103,6 +105,7 @@ raise util.Abort('%s: %s' % ('extra patterns', inst[0])) return ignorefunc + class gitdirstate(dirstate.dirstate): @dirstate.rootcache('.hgignore') def _ignore(self): @@ -128,7 +131,7 @@ self._ui.warn("%s: %s\n" % (fn, warning)) patterns.extend(pats) return gignore(self._root, files, self._ui.warn, - extrapatterns=patterns) + extrapatterns=patterns) def _finddotgitignores(self): """A copy of dirstate.walk. This is called from the new _ignore method,
--- a/hggit/gitrepo.py Sat Feb 24 17:22:27 2018 -0600 +++ b/hggit/gitrepo.py Sat Feb 24 17:30:57 2018 -0600 @@ -9,6 +9,7 @@ except ImportError: from mercurial.peer import peerrepository + class gitrepo(peerrepository): def __init__(self, ui, path, create): if create: # pragma: no cover @@ -93,8 +94,10 @@ def unbundle(self): raise NotImplementedError + instance = gitrepo + def islocal(path): if isgitsshuri(path): return True
--- a/hggit/hg2git.py Sat Feb 24 17:22:27 2018 -0600 +++ b/hggit/hg2git.py Sat Feb 24 17:30:57 2018 -0600 @@ -13,6 +13,7 @@ import compat import util + def parse_subrepos(ctx): sub = util.OrderedDict() if '.hgsub' in ctx: @@ -34,12 +35,14 @@ ... print s >>> u = fakeui() >>> audit_git_path(u, 'foo/git~100/wat') - warning: path 'foo/git~100/wat' contains a potentially dangerous path component. + ... # doctest: +ELLIPSIS + warning: path 'foo/git~100/wat' contains a potentially dangerous ... It may not be legal to check out in Git. It may also be rejected by some git server configurations. <BLANKLINE> >>> audit_git_path(u, u'foo/.gi\u200ct'.encode('utf-8')) - warning: path 'foo/.gi\xe2\x80\x8ct' contains a potentially dangerous path component. + ... # doctest: +ELLIPSIS + warning: path 'foo/.gi\xe2\x80\x8ct' contains a potentially dangerous ... It may not be legal to check out in Git. It may also be rejected by some git server configurations. <BLANKLINE>
--- a/hggit/hgrepo.py Sat Feb 24 17:22:27 2018 -0600 +++ b/hggit/hgrepo.py Sat Feb 24 17:30:57 2018 -0600 @@ -8,6 +8,7 @@ from gitrepo import gitrepo import util + def generate_repo_subclass(baseclass): class hgrepo(baseclass): if hgutil.safehasattr(localrepo.localrepository, 'pull'):
--- a/hggit/overlay.py Sat Feb 24 17:22:27 2018 -0600 +++ b/hggit/overlay.py Sat Feb 24 17:30:57 2018 -0600 @@ -14,11 +14,13 @@ ) from mercurial.node import bin, hex, nullid + def _maybehex(n): if len(n) == 20: return hex(n) return n + class overlaymanifest(object): def __init__(self, repo, sha): self.repo = repo @@ -101,8 +103,9 @@ return self._map.get(path, default) def diff(self, m2, match=None, clean=False): - # Older mercurial clients used diff(m2, clean=False). If a caller failed - # to specify clean as a keyword arg, it might get passed as match here. + # Older mercurial clients used diff(m2, clean=False). If a caller + # failed to specify clean as a keyword arg, it might get passed as + # match here. if isinstance(match, bool): clean = match match = None @@ -147,6 +150,7 @@ def __delitem__(self, path): del self._map[path] + def wrapmanifestdictdiff(orig, self, m2, match=None, clean=False): '''avoid calling into lazymanifest code if m2 is an overlaymanifest''' # Older mercurial clients used diff(m2, clean=False). If a caller failed @@ -156,10 +160,10 @@ match = None kwargs = { - 'clean' : clean + 'clean': clean } - # Older versions of mercurial don't support the match arg, so only add it if - # it exists. + # Older versions of mercurial don't support the match arg, so only add it + # if it exists. if match is not None: kwargs['match'] = match if isinstance(m2, overlaymanifest): @@ -172,6 +176,7 @@ else: return orig(self, m2, **kwargs) + class overlayfilectx(object): def __init__(self, repo, path, fileid=None): self._repo = repo @@ -204,6 +209,7 @@ def isbinary(self): return util.binary(self.data()) + class overlaychangectx(context.changectx): def __init__(self, repo, sha): # Can't store this in self._repo because the base class uses that field @@ -287,6 +293,7 @@ return (self.commit.tree, self.user(), self.date(), self.files(), self.description(), self.extra()) + class overlayrevlog(object): def __init__(self, repo, base): self.repo = repo @@ -345,6 +352,7 @@ def __len__(self): return len(self.repo.handler.repo) + len(self.repo.revmap) + class overlayoldmanifestlog(overlayrevlog): def read(self, sha): if sha == nullid: @@ -354,9 +362,11 @@ def __getitem__(self, sha): return overlaymanifestctx(self.repo, sha) + class overlaymanifestrevlog(overlayrevlog): pass + class overlaymanifestctx(object): def __init__(self, repo, node): self._repo = repo @@ -365,6 +375,7 @@ def read(self): return overlaymanifest(self._repo, self._node) + try: class overlaymanifestlog(manifest.manifestlog): def __init__(self, repo): @@ -385,6 +396,7 @@ # manifestlog did not exist prior to 4.0 pass + class overlaychangelog(overlayrevlog): def read(self, sha): if isinstance(sha, int): @@ -407,6 +419,7 @@ extra=values[5], ) + class overlayrepo(object): def __init__(self, handler, commits, refs): self.handler = handler @@ -442,7 +455,7 @@ def _constructmanifest(self): return overlaymanifestrevlog(self, - self.handler.repo._constructmanifest()) + self.handler.repo._constructmanifest()) def __getitem__(self, n): if n not in self.revmap:
--- a/hggit/util.py Sat Feb 24 17:22:27 2018 -0600 +++ b/hggit/util.py Sat Feb 24 17:30:57 2018 -0600 @@ -11,7 +11,6 @@ from dulwich import errors from mercurial.i18n import _ from mercurial import ( - encoding, error, lock as lockmod, util as hgutil, @@ -19,6 +18,7 @@ gitschemes = ('git', 'git+ssh', 'git+http', 'git+https') + def parse_hgsub(lines): """Fills OrderedDict with hgsub file content passed as list of lines""" rv = OrderedDict() @@ -30,10 +30,12 @@ rv[name.strip()] = value.strip() return rv + def serialize_hgsub(data): """Produces a string from OrderedDict hgsub content""" return ''.join(['%s = %s\n' % (n, v) for n, v in data.iteritems()]) + def parse_hgsubstate(lines): """Fills OrderedDict with hgsubtate file content passed as list of lines""" rv = OrderedDict() @@ -45,10 +47,12 @@ rv[name.strip()] = value.strip() return rv + def serialize_hgsubstate(data): """Produces a string from OrderedDict hgsubstate content""" return ''.join(['%s %s\n' % (data[n], n) for n in sorted(data)]) + def transform_notgit(f): '''use as a decorator around functions that call into dulwich''' def inner(*args, **kwargs): @@ -58,6 +62,7 @@ raise hgutil.Abort('not a git repository') return inner + def isgitsshuri(uri): """Method that returns True if a uri looks like git-style uri @@ -98,6 +103,7 @@ return True return False + def updatebookmarks(repo, changes, name='git_handler'): """abstract writing bookmarks for backwards compatibility""" bms = repo._bookmarks @@ -124,6 +130,7 @@ finally: lockmod.release(tr, lock, wlock) + def checksafessh(host): """check if a hostname is a potentially unsafe ssh exploit (SEC)
--- a/hggit/verify.py Sat Feb 24 17:22:27 2018 -0600 +++ b/hggit/verify.py Sat Feb 24 17:30:57 2018 -0600 @@ -14,12 +14,13 @@ from dulwich import diff_tree from dulwich.objects import Commit, S_IFGITLINK + def verify(ui, repo, hgctx): '''verify that a Mercurial rev matches the corresponding Git rev - Given a Mercurial revision that has a corresponding Git revision in the map, - this attempts to answer whether that revision has the same contents as the - corresponding Git revision. + Given a Mercurial revision that has a corresponding Git revision in the + map, this attempts to answer whether that revision has the same contents as + the corresponding Git revision. ''' handler = repo.githandler
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/setup.cfg Sat Feb 24 17:30:57 2018 -0600 @@ -0,0 +1,4 @@ +[flake8] +# E,W will ignore all pep8 +ignore=E129 +exclude=./tests,./contrib/
--- a/setup.py Sat Feb 24 17:22:27 2018 -0600 +++ b/setup.py Sat Feb 24 17:30:57 2018 -0600 @@ -1,16 +1,16 @@ +from os.path import dirname, join + try: from setuptools import setup except: from distutils.core import setup try: - from collections import OrderedDict extra_req = [] except ImportError: extra_req = ['ordereddict>=1.1'] -from os.path import dirname, join def get_version(relpath): root = dirname(__file__) for line in open(join(root, relpath), 'rb'): @@ -35,7 +35,7 @@ keywords='hg git mercurial', license='GPLv2', packages=['hggit'], - package_data={ 'hggit': ['help/git.rst'] }, + package_data={'hggit': ['help/git.rst']}, include_package_data=True, install_requires=['dulwich>=0.9.7'] + extra_req, )
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/helpers-testrepo.sh Sat Feb 24 17:30:57 2018 -0600 @@ -0,0 +1,53 @@ +# In most cases, the mercurial repository can be read by the bundled hg, but +# that isn't always true because third-party extensions may change the store +# format, for example. In which case, the system hg installation is used. +# +# We want to use the hg version being tested when interacting with the test +# repository, and the system hg when interacting with the mercurial source code +# repository. +# +# The mercurial source repository was typically orignally cloned with the +# system mercurial installation, and may require extensions or settings from +# the system installation. +syshg () { + ( + syshgenv + exec hg "$@" + ) +} + +# Revert the environment so that running "hg" runs the system hg +# rather than the test hg installation. +syshgenv () { + . "$HGTEST_RESTOREENV" + HGPLAIN=1 + export HGPLAIN +} + +# The test-repo is a live hg repository which may have evolution markers +# created, e.g. when a ~/.hgrc enabled evolution. +# +# Tests may be run using a custom HGRCPATH, which do not enable evolution +# markers by default. +# +# If test-repo includes evolution markers, and we do not enable evolution +# markers, hg will occasionally complain when it notices them, which disrupts +# tests resulting in sporadic failures. +# +# Since we aren't performing any write operations on the test-repo, there's +# no harm in telling hg that we support evolution markers, which is what the +# following lines for the hgrc file do: +cat >> "$HGRCPATH" << EOF +[experimental] +evolution = createmarkers +EOF + +# Use the system hg command if the bundled hg can't read the repository with +# no warning nor error. +if [ -n "`hg id -R "$TESTDIR/.." 2>&1 >/dev/null`" ]; then + alias testrepohg=syshg + alias testrepohgenv=syshgenv +else + alias testrepohg=hg + alias testrepohgenv=: +fi
--- a/tests/hghave Sat Feb 24 17:22:27 2018 -0600 +++ b/tests/hghave Sat Feb 24 17:30:57 2018 -0600 @@ -3,25 +3,29 @@ if all features are there, non-zero otherwise. If a feature name is prefixed with "no-", the absence of feature is tested. """ + +from __future__ import absolute_import, print_function + +import hghave import optparse +import os import sys -import hghave checks = hghave.checks def list_features(): - for name, feature in checks.iteritems(): + for name, feature in sorted(checks.items()): desc = feature[1] - print name + ':', desc + print(name + ':', desc) def test_features(): failed = 0 - for name, feature in checks.iteritems(): + for name, feature in checks.items(): check, _ = feature try: check() - except Exception, e: - print "feature %s failed: %s" % (name, e) + except Exception as e: + print("feature %s failed: %s" % (name, e)) failed += 1 return failed @@ -30,11 +34,31 @@ help="test available features") parser.add_option("--list-features", action="store_true", help="list available features") -parser.add_option("-q", "--quiet", action="store_true", - help="check features silently") + +def _loadaddon(): + if 'TESTDIR' in os.environ: + # loading from '.' isn't needed, because `hghave` should be + # running at TESTTMP in this case + path = os.environ['TESTDIR'] + else: + path = '.' + + if not os.path.exists(os.path.join(path, 'hghaveaddon.py')): + return + + sys.path.insert(0, path) + try: + import hghaveaddon + assert hghaveaddon # silence pyflakes + except BaseException as inst: + sys.stderr.write('failed to import hghaveaddon.py from %r: %s\n' + % (path, inst)) + sys.exit(2) + sys.path.pop(0) if __name__ == '__main__': options, args = parser.parse_args() + _loadaddon() if options.list_features: list_features() sys.exit(0) @@ -42,36 +66,4 @@ if options.test_features: sys.exit(test_features()) - quiet = options.quiet - - failures = 0 - - def error(msg): - global failures - if not quiet: - sys.stderr.write(msg + '\n') - failures += 1 - - for feature in args: - negate = feature.startswith('no-') - if negate: - feature = feature[3:] - - if feature not in checks: - error('skipped: unknown feature: ' + feature) - continue - - check, desc = checks[feature] - try: - available = check() - except Exception, e: - error('hghave check failed: ' + feature) - continue - - if not negate and not available: - error('skipped: missing feature: ' + desc) - elif negate and available: - error('skipped: system supports %s' % desc) - - if failures != 0: - sys.exit(1) + hghave.require(args)
--- a/tests/hghave.py Sat Feb 24 17:22:27 2018 -0600 +++ b/tests/hghave.py Sat Feb 24 17:30:57 2018 -0600 @@ -1,53 +1,163 @@ -import os, stat, socket +from __future__ import absolute_import + +import errno +import os import re +import socket +import stat +import subprocess import sys import tempfile tempprefix = 'hg-hghave-' +checks = { + "true": (lambda: True, "yak shaving"), + "false": (lambda: False, "nail clipper"), +} + +def check(name, desc): + """Registers a check function for a feature.""" + def decorator(func): + checks[name] = (func, desc) + return func + return decorator + +def checkvers(name, desc, vers): + """Registers a check function for each of a series of versions. + + vers can be a list or an iterator""" + def decorator(func): + def funcv(v): + def f(): + return func(v) + return f + for v in vers: + v = str(v) + f = funcv(v) + checks['%s%s' % (name, v.replace('.', ''))] = (f, desc % v) + return func + return decorator + +def checkfeatures(features): + result = { + 'error': [], + 'missing': [], + 'skipped': [], + } + + for feature in features: + negate = feature.startswith('no-') + if negate: + feature = feature[3:] + + if feature not in checks: + result['missing'].append(feature) + continue + + check, desc = checks[feature] + try: + available = check() + except Exception: + result['error'].append('hghave check failed: %s' % feature) + continue + + if not negate and not available: + result['skipped'].append('missing feature: %s' % desc) + elif negate and available: + result['skipped'].append('system supports %s' % desc) + + return result + +def require(features): + """Require that features are available, exiting if not.""" + result = checkfeatures(features) + + for missing in result['missing']: + sys.stderr.write('skipped: unknown feature: %s\n' % missing) + for msg in result['skipped']: + sys.stderr.write('skipped: %s\n' % msg) + for msg in result['error']: + sys.stderr.write('%s\n' % msg) + + if result['missing']: + sys.exit(2) + + if result['skipped'] or result['error']: + sys.exit(1) + def matchoutput(cmd, regexp, ignorestatus=False): - """Return True if cmd executes successfully and its output + """Return the match object if cmd executes successfully and its output is matched by the supplied regular expression. """ r = re.compile(regexp) - fh = os.popen(cmd) - s = fh.read() try: - ret = fh.close() - except IOError: - # Happen in Windows test environment - ret = 1 - return (ignorestatus or ret is None) and r.search(s) + p = subprocess.Popen( + cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + except OSError as e: + if e.errno != errno.ENOENT: + raise + ret = -1 + ret = p.wait() + s = p.stdout.read() + return (ignorestatus or not ret) and r.search(s) +@check("baz", "GNU Arch baz client") def has_baz(): - return matchoutput('baz --version 2>&1', r'baz Bazaar version') + return matchoutput('baz --version 2>&1', br'baz Bazaar version') +@check("bzr", "Canonical's Bazaar client") def has_bzr(): try: import bzrlib + import bzrlib.bzrdir + import bzrlib.errors + import bzrlib.revision + import bzrlib.revisionspec + bzrlib.revisionspec.RevisionSpec return bzrlib.__doc__ is not None - except ImportError: + except (AttributeError, ImportError): return False -def has_bzr114(): +@checkvers("bzr", "Canonical's Bazaar client >= %s", (1.14,)) +def has_bzr_range(v): + major, minor = v.split('.')[0:2] try: import bzrlib return (bzrlib.__doc__ is not None - and bzrlib.version_info[:2] >= (1, 14)) + and bzrlib.version_info[:2] >= (int(major), int(minor))) except ImportError: return False +@check("chg", "running with chg") +def has_chg(): + return 'CHGHG' in os.environ + +@check("cvs", "cvs client/server") def has_cvs(): - re = r'Concurrent Versions System.*?server' + re = br'Concurrent Versions System.*?server' + return matchoutput('cvs --version 2>&1', re) and not has_msys() + +@check("cvs112", "cvs client/server 1.12.* (not cvsnt)") +def has_cvs112(): + re = br'Concurrent Versions System \(CVS\) 1.12.*?server' return matchoutput('cvs --version 2>&1', re) and not has_msys() +@check("cvsnt", "cvsnt client/server") +def has_cvsnt(): + re = br'Concurrent Versions System \(CVSNT\) (\d+).(\d+).*\(client/server\)' + return matchoutput('cvsnt --version 2>&1', re) + +@check("darcs", "darcs client") def has_darcs(): - return matchoutput('darcs --version', r'2\.[2-9]', True) + return matchoutput('darcs --version', br'\b2\.([2-9]|\d{2})', True) +@check("mtn", "monotone client (>= 1.0)") def has_mtn(): - return matchoutput('mtn --version', r'monotone', True) and not matchoutput( - 'mtn --version', r'monotone 0\.', True) + return matchoutput('mtn --version', br'monotone', True) and not matchoutput( + 'mtn --version', br'monotone 0\.', True) +@check("eol-in-paths", "end-of-lines in paths") def has_eol_in_paths(): try: fd, path = tempfile.mkstemp(dir='.', prefix=tempprefix, suffix='\n\r') @@ -57,16 +167,17 @@ except (IOError, OSError): return False +@check("execbit", "executable bit") def has_executablebit(): try: EXECFLAGS = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH fh, fn = tempfile.mkstemp(dir='.', prefix=tempprefix) try: os.close(fh) - m = os.stat(fn).st_mode & 0777 + m = os.stat(fn).st_mode & 0o777 new_file_has_exec = m & EXECFLAGS os.chmod(fn, m ^ EXECFLAGS) - exec_flags_cannot_flip = ((os.stat(fn).st_mode & 0777) == m) + exec_flags_cannot_flip = ((os.stat(fn).st_mode & 0o777) == m) finally: os.unlink(fn) except (IOError, OSError): @@ -74,6 +185,7 @@ return False return not (new_file_has_exec or exec_flags_cannot_flip) +@check("icasefs", "case insensitive file system") def has_icasefs(): # Stolen from mercurial.util fd, path = tempfile.mkstemp(dir='.', prefix=tempprefix) @@ -92,21 +204,7 @@ finally: os.remove(path) -def has_inotify(): - try: - import hgext.inotify.linux.watcher - except ImportError: - return False - name = tempfile.mktemp(dir='.', prefix=tempprefix) - sock = socket.socket(socket.AF_UNIX) - try: - sock.bind(name) - except socket.error, err: - return False - sock.close() - os.unlink(name) - return True - +@check("fifo", "named pipes") def has_fifo(): if getattr(os, "mkfifo", None) is None: return False @@ -118,6 +216,11 @@ except OSError: return False +@check("killdaemons", 'killdaemons.py support') +def has_killdaemons(): + return True + +@check("cacheable", "cacheable filesystem") def has_cacheable_fs(): from mercurial import util @@ -128,42 +231,101 @@ finally: os.remove(path) +@check("lsprof", "python lsprof module") def has_lsprof(): try: import _lsprof + _lsprof.Profiler # silence unused import warning return True except ImportError: return False -def has_gettext(): - return matchoutput('msgfmt --version', 'GNU gettext-tools') +def gethgversion(): + m = matchoutput('hg --version --quiet 2>&1', br'(\d+)\.(\d+)') + if not m: + return (0, 0) + return (int(m.group(1)), int(m.group(2))) + +@checkvers("hg", "Mercurial >= %s", + list([(1.0 * x) / 10 for x in range(9, 99)])) +def has_hg_range(v): + major, minor = v.split('.')[0:2] + return gethgversion() >= (int(major), int(minor)) + +@check("hg08", "Mercurial >= 0.8") +def has_hg08(): + if checks["hg09"][0](): + return True + return matchoutput('hg help annotate 2>&1', '--date') + +@check("hg07", "Mercurial >= 0.7") +def has_hg07(): + if checks["hg08"][0](): + return True + return matchoutput('hg --version --quiet 2>&1', 'Mercurial Distributed SCM') + +@check("hg06", "Mercurial >= 0.6") +def has_hg06(): + if checks["hg07"][0](): + return True + return matchoutput('hg --version --quiet 2>&1', 'Mercurial version') +@check("gettext", "GNU Gettext (msgfmt)") +def has_gettext(): + return matchoutput('msgfmt --version', br'GNU gettext-tools') + +@check("git", "git command line client") def has_git(): - return matchoutput('git --version 2>&1', r'^git version') + return matchoutput('git --version 2>&1', br'^git version') + +def getgitversion(): + m = matchoutput('git --version 2>&1', br'git version (\d+)\.(\d+)') + if not m: + return (0, 0) + return (int(m.group(1)), int(m.group(2))) +# https://github.com/git-lfs/lfs-test-server <https://github.com/git-lfs/lfs-test-server> +@check("lfs-test-server", "git-lfs test server") +def has_lfsserver(): + exe = 'lfs-test-server' + if has_windows(): + exe = 'lfs-test-server.exe' + return any( + os.access(os.path.join(path, exe), os.X_OK) + for path in os.environ["PATH"].split(os.pathsep) + ) + +@checkvers("git", "git client (with ext::sh support) version >= %s", (1.9,)) +def has_git_range(v): + major, minor = v.split('.')[0:2] + return getgitversion() >= (int(major), int(minor)) + +@check("docutils", "Docutils text processing library") def has_docutils(): try: - from docutils.core import publish_cmdline + import docutils.core + docutils.core.publish_cmdline # silence unused import return True except ImportError: return False def getsvnversion(): - m = matchoutput('svn --version 2>&1', r'^svn,\s+version\s+(\d+)\.(\d+)') + m = matchoutput('svn --version --quiet 2>&1', br'^(\d+)\.(\d+)') if not m: return (0, 0) return (int(m.group(1)), int(m.group(2))) -def has_svn15(): - return getsvnversion() >= (1, 5) +@checkvers("svn", "subversion client and admin tools >= %s", (1.3, 1.5)) +def has_svn_range(v): + major, minor = v.split('.')[0:2] + return getsvnversion() >= (int(major), int(minor)) -def has_svn13(): - return getsvnversion() >= (1, 3) - +@check("svn", "subversion client and admin tools") def has_svn(): - return matchoutput('svn --version 2>&1', r'^svn, version') and \ - matchoutput('svnadmin --version 2>&1', r'^svnadmin, version') + return matchoutput('svn --version 2>&1', br'^svn, version') and \ + matchoutput('svnadmin --version 2>&1', br'^svnadmin, version') +@check("svn-bindings", "subversion python bindings") def has_svn_bindings(): try: import svn.core @@ -174,10 +336,12 @@ except ImportError: return False +@check("p4", "Perforce server and client") def has_p4(): - return (matchoutput('p4 -V', r'Rev\. P4/') and - matchoutput('p4d -V', r'Rev\. P4D/')) + return (matchoutput('p4 -V', br'Rev\. P4/') and + matchoutput('p4d -V', br'Rev\. P4D/')) +@check("symlink", "symbolic links") def has_symlink(): if getattr(os, "symlink", None) is None: return False @@ -189,120 +353,358 @@ except (OSError, AttributeError): return False +@check("hardlink", "hardlinks") def has_hardlink(): from mercurial import util fh, fn = tempfile.mkstemp(dir='.', prefix=tempprefix) os.close(fh) name = tempfile.mktemp(dir='.', prefix=tempprefix) try: - try: - util.oslink(fn, name) - os.unlink(name) - return True - except OSError: - return False + util.oslink(fn, name) + os.unlink(name) + return True + except OSError: + return False finally: os.unlink(fn) -def has_tla(): - return matchoutput('tla --version 2>&1', r'The GNU Arch Revision') +@check("hardlink-whitelisted", "hardlinks on whitelisted filesystems") +def has_hardlink_whitelisted(): + from mercurial import util + try: + fstype = util.getfstype('.') + except OSError: + return False + return fstype in util._hardlinkfswhitelist +@check("rmcwd", "can remove current working directory") +def has_rmcwd(): + ocwd = os.getcwd() + temp = tempfile.mkdtemp(dir='.', prefix=tempprefix) + try: + os.chdir(temp) + # On Linux, 'rmdir .' isn't allowed, but the other names are okay. + # On Solaris and Windows, the cwd can't be removed by any names. + os.rmdir(os.getcwd()) + return True + except OSError: + return False + finally: + os.chdir(ocwd) + # clean up temp dir on platforms where cwd can't be removed + try: + os.rmdir(temp) + except OSError: + pass + +@check("tla", "GNU Arch tla client") +def has_tla(): + return matchoutput('tla --version 2>&1', br'The GNU Arch Revision') + +@check("gpg", "gpg client") def has_gpg(): - return matchoutput('gpg --version 2>&1', r'GnuPG') + return matchoutput('gpg --version 2>&1', br'GnuPG') + +@check("gpg2", "gpg client v2") +def has_gpg2(): + return matchoutput('gpg --version 2>&1', br'GnuPG[^0-9]+2\.') +@check("gpg21", "gpg client v2.1+") +def has_gpg21(): + return matchoutput('gpg --version 2>&1', br'GnuPG[^0-9]+2\.(?!0)') + +@check("unix-permissions", "unix-style permissions") def has_unix_permissions(): d = tempfile.mkdtemp(dir='.', prefix=tempprefix) try: fname = os.path.join(d, 'foo') - for umask in (077, 007, 022): + for umask in (0o77, 0o07, 0o22): os.umask(umask) f = open(fname, 'w') f.close() mode = os.stat(fname).st_mode os.unlink(fname) - if mode & 0777 != ~umask & 0666: + if mode & 0o777 != ~umask & 0o666: return False return True finally: os.rmdir(d) +@check("unix-socket", "AF_UNIX socket family") +def has_unix_socket(): + return getattr(socket, 'AF_UNIX', None) is not None + +@check("root", "root permissions") +def has_root(): + return getattr(os, 'geteuid', None) and os.geteuid() == 0 + +@check("pyflakes", "Pyflakes python linter") def has_pyflakes(): return matchoutput("sh -c \"echo 'import re' 2>&1 | pyflakes\"", - r"<stdin>:1: 're' imported but unused", + br"<stdin>:1: 're' imported but unused", + True) + +@check("pylint", "Pylint python linter") +def has_pylint(): + return matchoutput("pylint --help", + br"Usage: pylint", True) +@check("clang-format", "clang-format C code formatter") +def has_clang_format(): + return matchoutput("clang-format --help", + br"^OVERVIEW: A tool to format C/C\+\+[^ ]+ code.") + +@check("jshint", "JSHint static code analysis tool") +def has_jshint(): + return matchoutput("jshint --version 2>&1", br"jshint v") + +@check("pygments", "Pygments source highlighting library") def has_pygments(): try: import pygments + pygments.highlight # silence unused import warning return True except ImportError: return False +@check("outer-repo", "outer repo") def has_outer_repo(): # failing for other reasons than 'no repo' imply that there is a repo return not matchoutput('hg root 2>&1', - r'abort: no repository found', True) + br'abort: no repository found', True) +@check("ssl", "ssl module available") def has_ssl(): try: import ssl - import OpenSSL - OpenSSL.SSL.Context + ssl.CERT_NONE return True except ImportError: return False +@check("sslcontext", "python >= 2.7.9 ssl") +def has_sslcontext(): + try: + import ssl + ssl.SSLContext + return True + except (ImportError, AttributeError): + return False + +@check("defaultcacerts", "can verify SSL certs by system's CA certs store") +def has_defaultcacerts(): + from mercurial import sslutil, ui as uimod + ui = uimod.ui.load() + return sslutil._defaultcacerts(ui) or sslutil._canloaddefaultcerts + +@check("defaultcacertsloaded", "detected presence of loaded system CA certs") +def has_defaultcacertsloaded(): + import ssl + from mercurial import sslutil, ui as uimod + + if not has_defaultcacerts(): + return False + if not has_sslcontext(): + return False + + ui = uimod.ui.load() + cafile = sslutil._defaultcacerts(ui) + ctx = ssl.create_default_context() + if cafile: + ctx.load_verify_locations(cafile=cafile) + else: + ctx.load_default_certs() + + return len(ctx.get_ca_certs()) > 0 + +@check("tls1.2", "TLS 1.2 protocol support") +def has_tls1_2(): + from mercurial import sslutil + return 'tls1.2' in sslutil.supportedprotocols + +@check("windows", "Windows") def has_windows(): return os.name == 'nt' +@check("system-sh", "system() uses sh") def has_system_sh(): return os.name != 'nt' +@check("serve", "platform and python can manage 'hg serve -d'") def has_serve(): - return os.name != 'nt' # gross approximation + return True + +@check("test-repo", "running tests from repository") +def has_test_repo(): + t = os.environ["TESTDIR"] + return os.path.isdir(os.path.join(t, "..", ".hg")) +@check("tic", "terminfo compiler and curses module") def has_tic(): - return matchoutput('test -x "`which tic`"', '') + try: + import curses + curses.COLOR_BLUE + return matchoutput('test -x "`which tic`"', br'') + except ImportError: + return False +@check("msys", "Windows with MSYS") def has_msys(): return os.getenv('MSYSTEM') -checks = { - "true": (lambda: True, "yak shaving"), - "false": (lambda: False, "nail clipper"), - "baz": (has_baz, "GNU Arch baz client"), - "bzr": (has_bzr, "Canonical's Bazaar client"), - "bzr114": (has_bzr114, "Canonical's Bazaar client >= 1.14"), - "cacheable": (has_cacheable_fs, "cacheable filesystem"), - "cvs": (has_cvs, "cvs client/server"), - "darcs": (has_darcs, "darcs client"), - "docutils": (has_docutils, "Docutils text processing library"), - "eol-in-paths": (has_eol_in_paths, "end-of-lines in paths"), - "execbit": (has_executablebit, "executable bit"), - "fifo": (has_fifo, "named pipes"), - "gettext": (has_gettext, "GNU Gettext (msgfmt)"), - "git": (has_git, "git command line client"), - "gpg": (has_gpg, "gpg client"), - "hardlink": (has_hardlink, "hardlinks"), - "icasefs": (has_icasefs, "case insensitive file system"), - "inotify": (has_inotify, "inotify extension support"), - "lsprof": (has_lsprof, "python lsprof module"), - "mtn": (has_mtn, "monotone client (>= 1.0)"), - "outer-repo": (has_outer_repo, "outer repo"), - "p4": (has_p4, "Perforce server and client"), - "pyflakes": (has_pyflakes, "Pyflakes python linter"), - "pygments": (has_pygments, "Pygments source highlighting library"), - "serve": (has_serve, "platform and python can manage 'hg serve -d'"), - "ssl": (has_ssl, "python >= 2.6 ssl module and python OpenSSL"), - "svn": (has_svn, "subversion client and admin tools"), - "svn13": (has_svn13, "subversion client and admin tools >= 1.3"), - "svn15": (has_svn15, "subversion client and admin tools >= 1.5"), - "svn-bindings": (has_svn_bindings, "subversion python bindings"), - "symlink": (has_symlink, "symbolic links"), - "system-sh": (has_system_sh, "system() uses sh"), - "tic": (has_tic, "terminfo compiler"), - "tla": (has_tla, "GNU Arch tla client"), - "unix-permissions": (has_unix_permissions, "unix-style permissions"), - "windows": (has_windows, "Windows"), - "msys": (has_msys, "Windows with MSYS"), -} +@check("aix", "AIX") +def has_aix(): + return sys.platform.startswith("aix") + +@check("osx", "OS X") +def has_osx(): + return sys.platform == 'darwin' + +@check("osxpackaging", "OS X packaging tools") +def has_osxpackaging(): + try: + return (matchoutput('pkgbuild', br'Usage: pkgbuild ', ignorestatus=1) + and matchoutput( + 'productbuild', br'Usage: productbuild ', + ignorestatus=1) + and matchoutput('lsbom', br'Usage: lsbom', ignorestatus=1) + and matchoutput( + 'xar --help', br'Usage: xar', ignorestatus=1)) + except ImportError: + return False + +@check('linuxormacos', 'Linux or MacOS') +def has_linuxormacos(): + # This isn't a perfect test for MacOS. But it is sufficient for our needs. + return sys.platform.startswith(('linux', 'darwin')) + +@check("docker", "docker support") +def has_docker(): + pat = br'A self-sufficient runtime for' + if matchoutput('docker --help', pat): + if 'linux' not in sys.platform: + # TODO: in theory we should be able to test docker-based + # package creation on non-linux using boot2docker, but in + # practice that requires extra coordination to make sure + # $TESTTEMP is going to be visible at the same path to the + # boot2docker VM. If we figure out how to verify that, we + # can use the following instead of just saying False: + # return 'DOCKER_HOST' in os.environ + return False + + return True + return False + +@check("debhelper", "debian packaging tools") +def has_debhelper(): + # Some versions of dpkg say `dpkg', some say 'dpkg' (` vs ' on the first + # quote), so just accept anything in that spot. + dpkg = matchoutput('dpkg --version', + br"Debian .dpkg' package management program") + dh = matchoutput('dh --help', + br'dh is a part of debhelper.', ignorestatus=True) + dh_py2 = matchoutput('dh_python2 --help', + br'other supported Python versions') + # debuild comes from the 'devscripts' package, though you might want + # the 'build-debs' package instead, which has a dependency on devscripts. + debuild = matchoutput('debuild --help', + br'to run debian/rules with given parameter') + return dpkg and dh and dh_py2 and debuild + +@check("debdeps", + "debian build dependencies (run dpkg-checkbuilddeps in contrib/)") +def has_debdeps(): + # just check exit status (ignoring output) + path = '%s/../contrib/debian/control' % os.environ['TESTDIR'] + return matchoutput('dpkg-checkbuilddeps %s' % path, br'') + +@check("demandimport", "demandimport enabled") +def has_demandimport(): + # chg disables demandimport intentionally for performance wins. + return ((not has_chg()) and os.environ.get('HGDEMANDIMPORT') != 'disable') + +@check("py3k", "running with Python 3.x") +def has_py3k(): + return 3 == sys.version_info[0] + +@check("py3exe", "a Python 3.x interpreter is available") +def has_python3exe(): + return 'PYTHON3' in os.environ + +@check("py3pygments", "Pygments available on Python 3.x") +def has_py3pygments(): + if has_py3k(): + return has_pygments() + elif has_python3exe(): + # just check exit status (ignoring output) + py3 = os.environ['PYTHON3'] + return matchoutput('%s -c "import pygments"' % py3, br'') + return False + +@check("pure", "running with pure Python code") +def has_pure(): + return any([ + os.environ.get("HGMODULEPOLICY") == "py", + os.environ.get("HGTEST_RUN_TESTS_PURE") == "--pure", + ]) + +@check("slow", "allow slow tests (use --allow-slow-tests)") +def has_slow(): + return os.environ.get('HGTEST_SLOW') == 'slow' + +@check("hypothesis", "Hypothesis automated test generation") +def has_hypothesis(): + try: + import hypothesis + hypothesis.given + return True + except ImportError: + return False + +@check("unziplinks", "unzip(1) understands and extracts symlinks") +def unzip_understands_symlinks(): + return matchoutput('unzip --help', br'Info-ZIP') + +@check("zstd", "zstd Python module available") +def has_zstd(): + try: + import mercurial.zstd + mercurial.zstd.__version__ + return True + except ImportError: + return False + +@check("devfull", "/dev/full special file") +def has_dev_full(): + return os.path.exists('/dev/full') + +@check("virtualenv", "Python virtualenv support") +def has_virtualenv(): + try: + import virtualenv + virtualenv.ACTIVATE_SH + return True + except ImportError: + return False + +@check("fsmonitor", "running tests with fsmonitor") +def has_fsmonitor(): + return 'HGFSMONITOR_TESTS' in os.environ + +@check("fuzzywuzzy", "Fuzzy string matching library") +def has_fuzzywuzzy(): + try: + import fuzzywuzzy + fuzzywuzzy.__version__ + return True + except ImportError: + return False + +@check("clang-libfuzzer", "clang new enough to include libfuzzer") +def has_clang_libfuzzer(): + mat = matchoutput('clang --version', 'clang version (\d)') + if mat: + # libfuzzer is new in clang 6 + return int(mat.group(1)) > 5 + return False
--- a/tests/run-tests.py Sat Feb 24 17:22:27 2018 -0600 +++ b/tests/run-tests.py Sat Feb 24 17:30:57 2018 -0600 @@ -45,11 +45,12 @@ from __future__ import absolute_import, print_function +import argparse +import collections import difflib import distutils.version as version import errno import json -import optparse import os import random import re @@ -119,6 +120,7 @@ } class TestRunnerLexer(lexer.RegexLexer): + testpattern = r'[\w-]+\.(t|py)( \(case [\w-]+\))?' tokens = { 'root': [ (r'^Skipped', token.Generic.Skipped, 'skipped'), @@ -126,11 +128,11 @@ (r'^ERROR: ', token.Generic.Failed, 'failed'), ], 'skipped': [ - (r'[\w-]+\.(t|py)', token.Generic.SName), + (testpattern, token.Generic.SName), (r':.*', token.Generic.Skipped), ], 'failed': [ - (r'[\w-]+\.(t|py)', token.Generic.FName), + (testpattern, token.Generic.FName), (r'(:| ).*', token.Generic.Failed), ] } @@ -296,122 +298,132 @@ def getparser(): """Obtain the OptionParser used by the CLI.""" - parser = optparse.OptionParser("%prog [options] [tests]") - - # keep these sorted - parser.add_option("--blacklist", action="append", + parser = argparse.ArgumentParser(usage='%(prog)s [options] [tests]') + + selection = parser.add_argument_group('Test Selection') + selection.add_argument('--allow-slow-tests', action='store_true', + help='allow extremely slow tests') + selection.add_argument("--blacklist", action="append", help="skip tests listed in the specified blacklist file") - parser.add_option("--whitelist", action="append", + selection.add_argument("--changed", + help="run tests that are changed in parent rev or working directory") + selection.add_argument("-k", "--keywords", + help="run tests matching keywords") + selection.add_argument("-r", "--retest", action="store_true", + help = "retest failed tests") + selection.add_argument("--test-list", action="append", + help="read tests to run from the specified file") + selection.add_argument("--whitelist", action="append", help="always run tests listed in the specified whitelist file") - parser.add_option("--test-list", action="append", - help="read tests to run from the specified file") - parser.add_option("--changed", type="string", - help="run tests that are changed in parent rev or working directory") - parser.add_option("-C", "--annotate", action="store_true", - help="output files annotated with coverage") - parser.add_option("-c", "--cover", action="store_true", - help="print a test coverage report") - parser.add_option("--color", choices=["always", "auto", "never"], - default=os.environ.get('HGRUNTESTSCOLOR', 'auto'), - help="colorisation: always|auto|never (default: auto)") - parser.add_option("-d", "--debug", action="store_true", + selection.add_argument('tests', metavar='TESTS', nargs='*', + help='Tests to run') + + harness = parser.add_argument_group('Test Harness Behavior') + harness.add_argument('--bisect-repo', + metavar='bisect_repo', + help=("Path of a repo to bisect. Use together with " + "--known-good-rev")) + harness.add_argument("-d", "--debug", action="store_true", help="debug mode: write output of test scripts to console" " rather than capturing and diffing it (disables timeout)") - parser.add_option("-f", "--first", action="store_true", + harness.add_argument("-f", "--first", action="store_true", help="exit on the first test failure") - parser.add_option("-H", "--htmlcov", action="store_true", - help="create an HTML report of the coverage of the files") - parser.add_option("-i", "--interactive", action="store_true", + harness.add_argument("-i", "--interactive", action="store_true", help="prompt to accept changed output") - parser.add_option("-j", "--jobs", type="int", + harness.add_argument("-j", "--jobs", type=int, help="number of jobs to run in parallel" " (default: $%s or %d)" % defaults['jobs']) - parser.add_option("--keep-tmpdir", action="store_true", + harness.add_argument("--keep-tmpdir", action="store_true", help="keep temporary directory after running tests") - parser.add_option("-k", "--keywords", - help="run tests matching keywords") - parser.add_option("--list-tests", action="store_true", + harness.add_argument('--known-good-rev', + metavar="known_good_rev", + help=("Automatically bisect any failures using this " + "revision as a known-good revision.")) + harness.add_argument("--list-tests", action="store_true", help="list tests instead of running them") - parser.add_option("-l", "--local", action="store_true", + harness.add_argument("--loop", action="store_true", + help="loop tests repeatedly") + harness.add_argument('--random', action="store_true", + help='run tests in random order') + harness.add_argument("-p", "--port", type=int, + help="port on which servers should listen" + " (default: $%s or %d)" % defaults['port']) + harness.add_argument('--profile-runner', action='store_true', + help='run statprof on run-tests') + harness.add_argument("-R", "--restart", action="store_true", + help="restart at last error") + harness.add_argument("--runs-per-test", type=int, dest="runs_per_test", + help="run each test N times (default=1)", default=1) + harness.add_argument("--shell", + help="shell to use (default: $%s or %s)" % defaults['shell']) + harness.add_argument('--showchannels', action='store_true', + help='show scheduling channels') + harness.add_argument("--slowtimeout", type=int, + help="kill errant slow tests after SLOWTIMEOUT seconds" + " (default: $%s or %d)" % defaults['slowtimeout']) + harness.add_argument("-t", "--timeout", type=int, + help="kill errant tests after TIMEOUT seconds" + " (default: $%s or %d)" % defaults['timeout']) + harness.add_argument("--tmpdir", + help="run tests in the given temporary directory" + " (implies --keep-tmpdir)") + harness.add_argument("-v", "--verbose", action="store_true", + help="output verbose messages") + + hgconf = parser.add_argument_group('Mercurial Configuration') + hgconf.add_argument("--chg", action="store_true", + help="install and use chg wrapper in place of hg") + hgconf.add_argument("--compiler", + help="compiler to build with") + hgconf.add_argument('--extra-config-opt', action="append", default=[], + help='set the given config opt in the test hgrc') + hgconf.add_argument("-l", "--local", action="store_true", help="shortcut for --with-hg=<testdir>/../hg, " "and --with-chg=<testdir>/../contrib/chg/chg if --chg is set") - parser.add_option("--loop", action="store_true", - help="loop tests repeatedly") - parser.add_option("--runs-per-test", type="int", dest="runs_per_test", - help="run each test N times (default=1)", default=1) - parser.add_option("-n", "--nodiff", action="store_true", - help="skip showing test changes") - parser.add_option("--outputdir", type="string", - help="directory to write error logs to (default=test directory)") - parser.add_option("-p", "--port", type="int", - help="port on which servers should listen" - " (default: $%s or %d)" % defaults['port']) - parser.add_option("--compiler", type="string", - help="compiler to build with") - parser.add_option("--pure", action="store_true", + hgconf.add_argument("--ipv6", action="store_true", + help="prefer IPv6 to IPv4 for network related tests") + hgconf.add_argument("--pure", action="store_true", help="use pure Python code instead of C extensions") - parser.add_option("-R", "--restart", action="store_true", - help="restart at last error") - parser.add_option("-r", "--retest", action="store_true", - help="retest failed tests") - parser.add_option("-S", "--noskips", action="store_true", - help="don't report skip tests verbosely") - parser.add_option("--shell", type="string", - help="shell to use (default: $%s or %s)" % defaults['shell']) - parser.add_option("-t", "--timeout", type="int", - help="kill errant tests after TIMEOUT seconds" - " (default: $%s or %d)" % defaults['timeout']) - parser.add_option("--slowtimeout", type="int", - help="kill errant slow tests after SLOWTIMEOUT seconds" - " (default: $%s or %d)" % defaults['slowtimeout']) - parser.add_option("--time", action="store_true", - help="time how long each test takes") - parser.add_option("--json", action="store_true", - help="store test result data in 'report.json' file") - parser.add_option("--tmpdir", type="string", - help="run tests in the given temporary directory" - " (implies --keep-tmpdir)") - parser.add_option("-v", "--verbose", action="store_true", - help="output verbose messages") - parser.add_option("--xunit", type="string", - help="record xunit results at specified path") - parser.add_option("--view", type="string", - help="external diff viewer") - parser.add_option("--with-hg", type="string", + hgconf.add_argument("-3", "--py3k-warnings", action="store_true", + help="enable Py3k warnings on Python 2.7+") + hgconf.add_argument("--with-chg", metavar="CHG", + help="use specified chg wrapper in place of hg") + hgconf.add_argument("--with-hg", metavar="HG", help="test using specified hg script rather than a " "temporary installation") - parser.add_option("--chg", action="store_true", - help="install and use chg wrapper in place of hg") - parser.add_option("--with-chg", metavar="CHG", - help="use specified chg wrapper in place of hg") - parser.add_option("--ipv6", action="store_true", - help="prefer IPv6 to IPv4 for network related tests") - parser.add_option("-3", "--py3k-warnings", action="store_true", - help="enable Py3k warnings on Python 2.7+") # This option should be deleted once test-check-py3-compat.t and other # Python 3 tests run with Python 3. - parser.add_option("--with-python3", metavar="PYTHON3", - help="Python 3 interpreter (if running under Python 2)" - " (TEMPORARY)") - parser.add_option('--extra-config-opt', action="append", - help='set the given config opt in the test hgrc') - parser.add_option('--random', action="store_true", - help='run tests in random order') - parser.add_option('--profile-runner', action='store_true', - help='run statprof on run-tests') - parser.add_option('--allow-slow-tests', action='store_true', - help='allow extremely slow tests') - parser.add_option('--showchannels', action='store_true', - help='show scheduling channels') - parser.add_option('--known-good-rev', type="string", - metavar="known_good_rev", - help=("Automatically bisect any failures using this " - "revision as a known-good revision.")) - parser.add_option('--bisect-repo', type="string", - metavar='bisect_repo', - help=("Path of a repo to bisect. Use together with " - "--known-good-rev")) + hgconf.add_argument("--with-python3", metavar="PYTHON3", + help="Python 3 interpreter (if running under Python 2)" + " (TEMPORARY)") + + reporting = parser.add_argument_group('Results Reporting') + reporting.add_argument("-C", "--annotate", action="store_true", + help="output files annotated with coverage") + reporting.add_argument("--color", choices=["always", "auto", "never"], + default=os.environ.get('HGRUNTESTSCOLOR', 'auto'), + help="colorisation: always|auto|never (default: auto)") + reporting.add_argument("-c", "--cover", action="store_true", + help="print a test coverage report") + reporting.add_argument('--exceptions', action='store_true', + help='log all exceptions and generate an exception report') + reporting.add_argument("-H", "--htmlcov", action="store_true", + help="create an HTML report of the coverage of the files") + reporting.add_argument("--json", action="store_true", + help="store test result data in 'report.json' file") + reporting.add_argument("--outputdir", + help="directory to write error logs to (default=test directory)") + reporting.add_argument("-n", "--nodiff", action="store_true", + help="skip showing test changes") + reporting.add_argument("-S", "--noskips", action="store_true", + help="don't report skip tests verbosely") + reporting.add_argument("--time", action="store_true", + help="time how long each test takes") + reporting.add_argument("--view", + help="external diff viewer") + reporting.add_argument("--xunit", + help="record xunit results at specified path") for option, (envvar, default) in defaults.items(): defaults[option] = type(default)(os.environ.get(envvar, default)) @@ -421,7 +433,7 @@ def parseargs(args, parser): """Parse arguments with our OptionParser and validate results.""" - (options, args) = parser.parse_args(args) + options = parser.parse_args(args) # jython is always pure if 'java' in sys.platform or '__pypy__' in sys.modules: @@ -550,7 +562,7 @@ if options.showchannels: options.nodiff = True - return (options, args) + return options def rename(src, dst): """Like os.rename(), trade atomicity and opened files friendliness @@ -659,6 +671,7 @@ def __init__(self, path, outputdir, tmpdir, keeptmpdir=False, debug=False, + first=False, timeout=None, startport=None, extraconfigopts=None, py3kwarnings=False, shell=None, hgcommand=None, @@ -711,6 +724,7 @@ self._threadtmp = tmpdir self._keeptmpdir = keeptmpdir self._debug = debug + self._first = first self._timeout = timeout self._slowtimeout = slowtimeout self._startport = startport @@ -890,15 +904,18 @@ # Diff generation may rely on written .err file. if (ret != 0 or out != self._refout) and not self._skipped \ and not self._debug: - f = open(self.errpath, 'wb') - for line in out: - f.write(line) - f.close() + with open(self.errpath, 'wb') as f: + for line in out: + f.write(line) # The result object handles diff calculation for us. - if self._result.addOutputMismatch(self, ret, out, self._refout): - # change was accepted, skip failing - return + with firstlock: + if self._result.addOutputMismatch(self, ret, out, self._refout): + # change was accepted, skip failing + return + if self._first: + global firsterror + firsterror = True if ret: msg = 'output changed and ' + describe(ret) @@ -930,10 +947,9 @@ if (self._ret != 0 or self._out != self._refout) and not self._skipped \ and not self._debug and self._out: - f = open(self.errpath, 'wb') - for line in self._out: - f.write(line) - f.close() + with open(self.errpath, 'wb') as f: + for line in self._out: + f.write(line) vlog("# Ret was:", self._ret, '(%s)' % self.name) @@ -961,13 +977,20 @@ self._portmap(0), self._portmap(1), self._portmap(2), - (br'(?m)^(saved backup bundle to .*\.hg)( \(glob\))?$', - br'\1 (glob)'), (br'([^0-9])%s' % re.escape(self._localip()), br'\1$LOCALIP'), (br'\bHG_TXNID=TXN:[a-f0-9]{40}\b', br'HG_TXNID=TXN:$ID$'), ] r.append((self._escapepath(self._testtmp), b'$TESTTMP')) + replacementfile = os.path.join(self._testdir, b'common-pattern.py') + + if os.path.exists(replacementfile): + data = {} + with open(replacementfile, mode='rb') as source: + # the intermediate 'compile' step help with debugging + code = compile(source.read(), replacementfile, 'exec') + exec(code, data) + r.extend(data.get('substitutions', ())) return r def _escapepath(self, p): @@ -1021,7 +1044,7 @@ offset = '' if i == 0 else '%s' % i env["HGPORT%s" % offset] = '%s' % (self._startport + i) env = os.environ.copy() - env['PYTHONUSERBASE'] = sysconfig.get_config_var('userbase') + env['PYTHONUSERBASE'] = sysconfig.get_config_var('userbase') or '' env['HGEMITWARNINGS'] = '1' env['TESTTMP'] = self._testtmp env['HOME'] = self._testtmp @@ -1069,29 +1092,31 @@ def _createhgrc(self, path): """Create an hgrc file for this test.""" - hgrc = open(path, 'wb') - hgrc.write(b'[ui]\n') - hgrc.write(b'slash = True\n') - hgrc.write(b'interactive = False\n') - hgrc.write(b'mergemarkers = detailed\n') - hgrc.write(b'promptecho = True\n') - hgrc.write(b'[defaults]\n') - hgrc.write(b'[devel]\n') - hgrc.write(b'all-warnings = true\n') - hgrc.write(b'default-date = 0 0\n') - hgrc.write(b'[largefiles]\n') - hgrc.write(b'usercache = %s\n' % - (os.path.join(self._testtmp, b'.cache/largefiles'))) - hgrc.write(b'[web]\n') - hgrc.write(b'address = localhost\n') - hgrc.write(b'ipv6 = %s\n' % str(self._useipv6).encode('ascii')) - - for opt in self._extraconfigopts: - section, key = opt.split('.', 1) - assert '=' in key, ('extra config opt %s must ' - 'have an = for assignment' % opt) - hgrc.write(b'[%s]\n%s\n' % (section, key)) - hgrc.close() + with open(path, 'wb') as hgrc: + hgrc.write(b'[ui]\n') + hgrc.write(b'slash = True\n') + hgrc.write(b'interactive = False\n') + hgrc.write(b'mergemarkers = detailed\n') + hgrc.write(b'promptecho = True\n') + hgrc.write(b'[defaults]\n') + hgrc.write(b'[devel]\n') + hgrc.write(b'all-warnings = true\n') + hgrc.write(b'default-date = 0 0\n') + hgrc.write(b'[largefiles]\n') + hgrc.write(b'usercache = %s\n' % + (os.path.join(self._testtmp, b'.cache/largefiles'))) + hgrc.write(b'[lfs]\n') + hgrc.write(b'usercache = %s\n' % + (os.path.join(self._testtmp, b'.cache/lfs'))) + hgrc.write(b'[web]\n') + hgrc.write(b'address = localhost\n') + hgrc.write(b'ipv6 = %s\n' % str(self._useipv6).encode('ascii')) + + for opt in self._extraconfigopts: + section, key = opt.encode('utf-8').split(b'.', 1) + assert b'=' in key, ('extra config opt %s must ' + 'have an = for assignment' % opt) + hgrc.write(b'[%s]\n%s\n' % (section, key)) def fail(self, msg): # unittest differentiates between errored and failed. @@ -1197,9 +1222,7 @@ def __init__(self, path, *args, **kwds): # accept an extra "case" parameter - case = None - if 'case' in kwds: - case = kwds.pop('case') + case = kwds.pop('case', None) self._case = case self._allcases = parsettestcases(path) super(TTest, self).__init__(path, *args, **kwds) @@ -1213,9 +1236,8 @@ return os.path.join(self._testdir, self.bname) def _run(self, env): - f = open(self.path, 'rb') - lines = f.readlines() - f.close() + with open(self.path, 'rb') as f: + lines = f.readlines() # .t file is both reference output and the test input, keep reference # output updated with the the test input. This avoids some race @@ -1227,10 +1249,9 @@ # Write out the generated script. fname = b'%s.sh' % self._testtmp - f = open(fname, 'wb') - for l in script: - f.write(l) - f.close() + with open(fname, 'wb') as f: + for l in script: + f.write(l) cmd = b'%s "%s"' % (self._shell, fname) vlog("# Running", cmd) @@ -1320,6 +1341,13 @@ script.append(b'alias hg="%s"\n' % self._hgcommand) if os.getenv('MSYSTEM'): script.append(b'alias pwd="pwd -W"\n') + if self._case: + if isinstance(self._case, str): + quoted = shellquote(self._case) + else: + quoted = shellquote(self._case.decode('utf8')).encode('utf8') + script.append(b'TESTCASE=%s\n' % quoted) + script.append(b'export TESTCASE\n') n = 0 for n, l in enumerate(lines): @@ -1430,10 +1458,7 @@ r = self.linematch(el, lout) if isinstance(r, str): - if r == '+glob': - lout = el[:-1] + ' (glob)\n' - r = '' # Warn only this line. - elif r == '-glob': + if r == '-glob': lout = ''.join(el.rsplit(' (glob)', 1)) r = '' # Warn only this line. elif r == "retry": @@ -1517,6 +1542,7 @@ @staticmethod def rematch(el, l): try: + el = b'(?:' + el + b')' # use \Z to ensure that the regex matches to the end of the string if os.name == 'nt': return re.match(el + br'\r?\n\Z', l) @@ -1588,8 +1614,10 @@ if l.endswith(b" (glob)\n"): l = l[:-8] + b"\n" return TTest.globmatch(el[:-8], l) or retry - if os.altsep and l.replace(b'\\', b'/') == el: - return b'+glob' + if os.altsep: + _l = l.replace(b'\\', b'/') + if el == _l or os.name == 'nt' and el[:-1] + b'\r\n' == _l: + return True return retry @staticmethod @@ -1620,6 +1648,8 @@ return TTest.ESCAPESUB(TTest._escapef, s) iolock = threading.RLock() +firstlock = threading.RLock() +firsterror = False class TestResult(unittest._TextTestResult): """Holds results when executing via unittest.""" @@ -1705,7 +1735,7 @@ def addOutputMismatch(self, test, ret, got, expected): """Record a mismatch in test output for a particular test.""" - if self.shouldStop: + if self.shouldStop or firsterror: # don't print, some other test case already failed and # printed, we're just stale and probably failed due to our # temp dir getting cleaned up. @@ -1865,9 +1895,8 @@ continue if self._keywords: - f = open(test.path, 'rb') - t = f.read().lower() + test.bname.lower() - f.close() + with open(test.path, 'rb') as f: + t = f.read().lower() + test.bname.lower() ignored = False for k in self._keywords.lower().split(): if k not in t: @@ -2096,6 +2125,18 @@ os.environ['PYTHONHASHSEED']) if self._runner.options.time: self.printtimes(result.times) + + if self._runner.options.exceptions: + exceptions = aggregateexceptions( + os.path.join(self._runner._outputdir, b'exceptions')) + total = sum(exceptions.values()) + + self.stream.writeln('Exceptions Report:') + self.stream.writeln('%d total from %d frames' % + (total, len(exceptions))) + for (frame, line, exc), count in exceptions.most_common(): + self.stream.writeln('%d\t%s: %s' % (count, frame, exc)) + self.stream.flush() return result @@ -2243,6 +2284,50 @@ separators=(',', ': ')) outf.writelines(("testreport =", jsonout)) +def sorttests(testdescs, shuffle=False): + """Do an in-place sort of tests.""" + if shuffle: + random.shuffle(testdescs) + return + + # keywords for slow tests + slow = {b'svn': 10, + b'cvs': 10, + b'hghave': 10, + b'largefiles-update': 10, + b'run-tests': 10, + b'corruption': 10, + b'race': 10, + b'i18n': 10, + b'check': 100, + b'gendoc': 100, + b'contrib-perf': 200, + } + perf = {} + + def sortkey(f): + # run largest tests first, as they tend to take the longest + f = f['path'] + try: + return perf[f] + except KeyError: + try: + val = -os.stat(f).st_size + except OSError as e: + if e.errno != errno.ENOENT: + raise + perf[f] = -1e9 # file does not exist, tell early + return -1e9 + for kw, mul in slow.items(): + if kw in f: + val *= mul + if f.endswith(b'.py'): + val /= 10.0 + perf[f] = val / 1000.0 + return perf[f] + + testdescs.sort(key=sortkey) + class TestRunner(object): """Holds context for executing tests. @@ -2287,18 +2372,16 @@ oldmask = os.umask(0o22) try: parser = parser or getparser() - options, args = parseargs(args, parser) - # positional arguments are paths to test files to run, so - # we make sure they're all bytestrings - args = [_bytespath(a) for a in args] + options = parseargs(args, parser) + tests = [_bytespath(a) for a in options.tests] if options.test_list is not None: for listfile in options.test_list: with open(listfile, 'rb') as f: - args.extend(t for t in f.read().splitlines() if t) + tests.extend(t for t in f.read().splitlines() if t) self.options = options self._checktools() - testdescs = self.findtests(args) + testdescs = self.findtests(tests) if options.profile_runner: import statprof statprof.start() @@ -2312,51 +2395,22 @@ os.umask(oldmask) def _run(self, testdescs): - if self.options.random: - random.shuffle(testdescs) - else: - # keywords for slow tests - slow = {b'svn': 10, - b'cvs': 10, - b'hghave': 10, - b'largefiles-update': 10, - b'run-tests': 10, - b'corruption': 10, - b'race': 10, - b'i18n': 10, - b'check': 100, - b'gendoc': 100, - b'contrib-perf': 200, - } - perf = {} - def sortkey(f): - # run largest tests first, as they tend to take the longest - f = f['path'] - try: - return perf[f] - except KeyError: - try: - val = -os.stat(f).st_size - except OSError as e: - if e.errno != errno.ENOENT: - raise - perf[f] = -1e9 # file does not exist, tell early - return -1e9 - for kw, mul in slow.items(): - if kw in f: - val *= mul - if f.endswith(b'.py'): - val /= 10.0 - perf[f] = val / 1000.0 - return perf[f] - testdescs.sort(key=sortkey) + sorttests(testdescs, shuffle=self.options.random) self._testdir = osenvironb[b'TESTDIR'] = getattr( os, 'getcwdb', os.getcwd)() + # assume all tests in same folder for now + if testdescs: + pathname = os.path.dirname(testdescs[0]['path']) + if pathname: + osenvironb[b'TESTDIR'] = os.path.join(osenvironb[b'TESTDIR'], + pathname) if self.options.outputdir: self._outputdir = canonpath(_bytespath(self.options.outputdir)) else: self._outputdir = self._testdir + if testdescs and pathname: + self._outputdir = os.path.join(self._outputdir, pathname) if 'PYTHONHASHSEED' not in os.environ: # use a random python hash seed all the time @@ -2373,11 +2427,6 @@ print("error: temp dir %r already exists" % tmpdir) return 1 - # Automatically removing tmpdir sounds convenient, but could - # really annoy anyone in the habit of using "--tmpdir=/tmp" - # or "--tmpdir=$HOME". - #vlog("# Removing temp dir", tmpdir) - #shutil.rmtree(tmpdir) os.makedirs(tmpdir) else: d = None @@ -2399,12 +2448,27 @@ self._tmpbindir = os.path.join(self._hgtmp, b'install', b'bin') os.makedirs(self._tmpbindir) - # This looks redundant with how Python initializes sys.path from - # the location of the script being executed. Needed because the - # "hg" specified by --with-hg is not the only Python script - # executed in the test suite that needs to import 'mercurial' - # ... which means it's not really redundant at all. - self._pythondir = self._bindir + normbin = os.path.normpath(os.path.abspath(whg)) + normbin = normbin.replace(os.sep.encode('ascii'), b'/') + + # Other Python scripts in the test harness need to + # `import mercurial`. If `hg` is a Python script, we assume + # the Mercurial modules are relative to its path and tell the tests + # to load Python modules from its directory. + with open(whg, 'rb') as fh: + initial = fh.read(1024) + + if re.match(b'#!.*python', initial): + self._pythondir = self._bindir + # If it looks like our in-repo Rust binary, use the source root. + # This is a bit hacky. But rhg is still not supported outside the + # source directory. So until it is, do the simple thing. + elif re.search(b'/rust/target/[^/]+/hg', normbin): + self._pythondir = os.path.dirname(self._testdir) + # Fall back to the legacy behavior. + else: + self._pythondir = self._bindir + else: self._installdir = os.path.join(self._hgtmp, b"install") self._bindir = os.path.join(self._installdir, b"bin") @@ -2476,6 +2540,23 @@ self._coveragefile = os.path.join(self._testdir, b'.coverage') + if self.options.exceptions: + exceptionsdir = os.path.join(self._outputdir, b'exceptions') + try: + os.makedirs(exceptionsdir) + except OSError as e: + if e.errno != errno.EEXIST: + raise + + # Remove all existing exception reports. + for f in os.listdir(exceptionsdir): + os.unlink(os.path.join(exceptionsdir, f)) + + osenvironb[b'HGEXCEPTIONSDIR'] = exceptionsdir + logexceptions = os.path.join(self._testdir, b'logexceptions.py') + self.options.extra_config_opt.append( + 'extensions.logexceptions=%s' % logexceptions.decode('utf-8')) + vlog("# Using TESTDIR", self._testdir) vlog("# Using RUNTESTDIR", osenvironb[b'RUNTESTDIR']) vlog("# Using HGTMP", self._hgtmp) @@ -2504,6 +2585,16 @@ else: args = os.listdir(b'.') + expanded_args = [] + for arg in args: + if os.path.isdir(arg): + if not arg.endswith(b'/'): + arg += b'/' + expanded_args.extend([arg + a for a in os.listdir(arg)]) + else: + expanded_args.append(arg) + args = expanded_args + tests = [] for t in args: if not (os.path.basename(t).startswith(b'test-') @@ -2637,6 +2728,7 @@ t = testcls(refpath, self._outputdir, tmpdir, keeptmpdir=self.options.keep_tmpdir, debug=self.options.debug, + first=self.options.first, timeout=self.options.timeout, startport=self._getport(count), extraconfigopts=self.options.extra_config_opt, @@ -2758,13 +2850,12 @@ if e.errno != errno.ENOENT: raise else: - f = open(installerrs, 'rb') - for line in f: - if PYTHON3: - sys.stdout.buffer.write(line) - else: - sys.stdout.write(line) - f.close() + with open(installerrs, 'rb') as f: + for line in f: + if PYTHON3: + sys.stdout.buffer.write(line) + else: + sys.stdout.write(line) sys.exit(1) os.chdir(self._testdir) @@ -2772,28 +2863,24 @@ if self.options.py3k_warnings and not self.options.anycoverage: vlog("# Updating hg command to enable Py3k Warnings switch") - f = open(os.path.join(self._bindir, 'hg'), 'rb') - lines = [line.rstrip() for line in f] - lines[0] += ' -3' - f.close() - f = open(os.path.join(self._bindir, 'hg'), 'wb') - for line in lines: - f.write(line + '\n') - f.close() + with open(os.path.join(self._bindir, 'hg'), 'rb') as f: + lines = [line.rstrip() for line in f] + lines[0] += ' -3' + with open(os.path.join(self._bindir, 'hg'), 'wb') as f: + for line in lines: + f.write(line + '\n') hgbat = os.path.join(self._bindir, b'hg.bat') if os.path.isfile(hgbat): # hg.bat expects to be put in bin/scripts while run-tests.py # installation layout put it in bin/ directly. Fix it - f = open(hgbat, 'rb') - data = f.read() - f.close() + with open(hgbat, 'rb') as f: + data = f.read() if b'"%~dp0..\python" "%~dp0hg" %*' in data: data = data.replace(b'"%~dp0..\python" "%~dp0hg" %*', b'"%~dp0python" "%~dp0hg" %*') - f = open(hgbat, 'wb') - f.write(data) - f.close() + with open(hgbat, 'wb') as f: + f.write(data) else: print('WARNING: cannot fix hg.bat reference to python.exe') @@ -2918,6 +3005,24 @@ print("WARNING: Did not find prerequisite tool: %s " % p.decode("utf-8")) +def aggregateexceptions(path): + exceptions = collections.Counter() + + for f in os.listdir(path): + with open(os.path.join(path, f), 'rb') as fh: + data = fh.read().split(b'\0') + if len(data) != 4: + continue + + exc, mainframe, hgframe, hgline = data + exc = exc.decode('utf-8') + mainframe = mainframe.decode('utf-8') + hgframe = hgframe.decode('utf-8') + hgline = hgline.decode('utf-8') + exceptions[(hgframe, hgline, exc)] += 1 + + return exceptions + if __name__ == '__main__': runner = TestRunner()
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/test-check-pyflakes.t Sat Feb 24 17:30:57 2018 -0600 @@ -0,0 +1,19 @@ +#require test-repo pyflakes hg10 + + $ . "$TESTDIR/helpers-testrepo.sh" + +run pyflakes on all tracked files ending in .py or without a file ending +(skipping binary file random-seed) + + $ cat > test.py <<EOF + > print(undefinedname) + > EOF + $ pyflakes test.py 2>/dev/null + test.py:1: undefined name 'undefinedname' + [1] + $ cd "`dirname "$TESTDIR"`" + + $ testrepohg locate 'set:**.py or grep("^#!.*python")' \ + > -X tests/ \ + > 2>/dev/null \ + > | xargs pyflakes 2>/dev/null
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/test-git-gpg.t Sat Feb 24 17:30:57 2018 -0600 @@ -0,0 +1,74 @@ +#require gpg + +Load commonly used test logic + $ . "$TESTDIR/testutil" + + $ export GNUPGHOME="$(mktemp -d)" + $ cp -R "$TESTDIR"/gpg/* "$GNUPGHOME" + +Start gpg-agent, which is required by GnuPG v2 + +#if gpg21 + $ gpg-connect-agent -q --subst /serverpid '/echo ${get serverpid}' /bye \ + > >> $DAEMON_PIDS +#endif + +and migrate secret keys + +#if gpg2 + $ gpg --no-permission-warning --no-secmem-warning --list-secret-keys \ + > > /dev/null 2>&1 +#endif + + $ alias gpg='gpg --no-permission-warning --no-secmem-warning --no-auto-check-trustdb' + +Set up two identical git repos. + + $ mkdir gitrepo + $ cd gitrepo + $ git init + Initialized empty Git repository in $TESTTMP/gitrepo/.git/ + $ touch a + $ git add a + $ git commit -m "initial commit" + [master (root-commit) *] initial commit (glob) + 1 file changed, 0 insertions(+), 0 deletions(-) + create mode 100644 a + $ cd .. + $ git clone gitrepo gitrepo2 + Cloning into 'gitrepo2'... + done. + +Add a signed commit to the first clone. + + $ cd gitrepo + $ git checkout -b signed + Switched to a new branch 'signed' + $ touch b + $ git add b + $ git commit -m "message" -Shgtest + [signed *] message (glob) + 1 file changed, 0 insertions(+), 0 deletions(-) + create mode 100644 b + $ cd .. + +Hg clone it + + $ hg clone gitrepo hgrepo + importing git objects into hg + updating to branch default + 2 files updated, 0 files merged, 0 files removed, 0 files unresolved + + $ cd hgrepo + $ hg push ../gitrepo2 -B signed + pushing to ../gitrepo2 + searching for changes + adding objects + added 1 commits with 1 trees and 0 blobs + $ cd .. + +Verify the commit + + $ cd gitrepo2 + $ git show --show-signature signed | grep "Good signature from" + gpg: Good signature from "hgtest" [ultimate]