# HG changeset patch
# User Tony Tung <ttung@chanzuckerberg.com <mailto:ttung@chanzuckerberg.com>>
# Date 1517963668 28800
# Node ID cca56bbea1434e94ce1ac16144c85124ead636c5
# Parent  bb255eaf3d14c598dc26b9f17999abc7bf327173
tests: copy hg test infra from hg repo @5cfdf6137af8

diff -r bb255eaf3d14 -r cca56bbea143 tests/hghave
--- a/tests/hghave	Tue Feb 06 12:12:49 2018 -0600
+++ b/tests/hghave	Tue Feb 06 16:34:28 2018 -0800
@@ -3,25 +3,29 @@
 if all features are there, non-zero otherwise. If a feature name is
 prefixed with "no-", the absence of feature is tested.
 """
+
+from __future__ import absolute_import, print_function
+
+import hghave
 import optparse
+import os
 import sys
-import hghave
 
 checks = hghave.checks
 
 def list_features():
-    for name, feature in checks.iteritems():
+    for name, feature in sorted(checks.items()):
         desc = feature[1]
-        print name + ':', desc
+        print(name + ':', desc)
 
 def test_features():
     failed = 0
-    for name, feature in checks.iteritems():
+    for name, feature in checks.items():
         check, _ = feature
         try:
             check()
-        except Exception, e:
-            print "feature %s failed:  %s" % (name, e)
+        except Exception as e:
+            print("feature %s failed:  %s" % (name, e))
             failed += 1
     return failed
 
@@ -30,11 +34,31 @@
                   help="test available features")
 parser.add_option("--list-features", action="store_true",
                   help="list available features")
-parser.add_option("-q", "--quiet", action="store_true",
-                  help="check features silently")
+
+def _loadaddon():
+    if 'TESTDIR' in os.environ:
+        # loading from '.' isn't needed, because `hghave` should be
+        # running at TESTTMP in this case
+        path = os.environ['TESTDIR']
+    else:
+        path = '.'
+
+    if not os.path.exists(os.path.join(path, 'hghaveaddon.py')):
+        return
+
+    sys.path.insert(0, path)
+    try:
+        import hghaveaddon
+        assert hghaveaddon  # silence pyflakes
+    except BaseException as inst:
+        sys.stderr.write('failed to import hghaveaddon.py from %r: %s\n'
+                         % (path, inst))
+        sys.exit(2)
+    sys.path.pop(0)
 
 if __name__ == '__main__':
     options, args = parser.parse_args()
+    _loadaddon()
     if options.list_features:
         list_features()
         sys.exit(0)
@@ -42,36 +66,4 @@
     if options.test_features:
         sys.exit(test_features())
 
-    quiet = options.quiet
-
-    failures = 0
-
-    def error(msg):
-        global failures
-        if not quiet:
-            sys.stderr.write(msg + '\n')
-        failures += 1
-
-    for feature in args:
-        negate = feature.startswith('no-')
-        if negate:
-            feature = feature[3:]
-
-        if feature not in checks:
-            error('skipped: unknown feature: ' + feature)
-            continue
-
-        check, desc = checks[feature]
-        try:
-            available = check()
-        except Exception, e:
-            error('hghave check failed: ' + feature)
-            continue
-
-        if not negate and not available:
-            error('skipped: missing feature: ' + desc)
-        elif negate and available:
-            error('skipped: system supports %s' % desc)
-
-    if failures != 0:
-        sys.exit(1)
+    hghave.require(args)
diff -r bb255eaf3d14 -r cca56bbea143 tests/hghave.py
--- a/tests/hghave.py	Tue Feb 06 12:12:49 2018 -0600
+++ b/tests/hghave.py	Tue Feb 06 16:34:28 2018 -0800
@@ -1,53 +1,163 @@
-import os, stat, socket
+from __future__ import absolute_import
+
+import errno
+import os
 import re
+import socket
+import stat
+import subprocess
 import sys
 import tempfile
 
 tempprefix = 'hg-hghave-'
 
+checks = {
+    "true": (lambda: True, "yak shaving"),
+    "false": (lambda: False, "nail clipper"),
+}
+
+def check(name, desc):
+    """Registers a check function for a feature."""
+    def decorator(func):
+        checks[name] = (func, desc)
+        return func
+    return decorator
+
+def checkvers(name, desc, vers):
+    """Registers a check function for each of a series of versions.
+
+    vers can be a list or an iterator"""
+    def decorator(func):
+        def funcv(v):
+            def f():
+                return func(v)
+            return f
+        for v in vers:
+            v = str(v)
+            f = funcv(v)
+            checks['%s%s' % (name, v.replace('.', ''))] = (f, desc % v)
+        return func
+    return decorator
+
+def checkfeatures(features):
+    result = {
+        'error': [],
+        'missing': [],
+        'skipped': [],
+    }
+
+    for feature in features:
+        negate = feature.startswith('no-')
+        if negate:
+            feature = feature[3:]
+
+        if feature not in checks:
+            result['missing'].append(feature)
+            continue
+
+        check, desc = checks[feature]
+        try:
+            available = check()
+        except Exception:
+            result['error'].append('hghave check failed: %s' % feature)
+            continue
+
+        if not negate and not available:
+            result['skipped'].append('missing feature: %s' % desc)
+        elif negate and available:
+            result['skipped'].append('system supports %s' % desc)
+
+    return result
+
+def require(features):
+    """Require that features are available, exiting if not."""
+    result = checkfeatures(features)
+
+    for missing in result['missing']:
+        sys.stderr.write('skipped: unknown feature: %s\n' % missing)
+    for msg in result['skipped']:
+        sys.stderr.write('skipped: %s\n' % msg)
+    for msg in result['error']:
+        sys.stderr.write('%s\n' % msg)
+
+    if result['missing']:
+        sys.exit(2)
+
+    if result['skipped'] or result['error']:
+        sys.exit(1)
+
 def matchoutput(cmd, regexp, ignorestatus=False):
-    """Return True if cmd executes successfully and its output
+    """Return the match object if cmd executes successfully and its output
     is matched by the supplied regular expression.
     """
     r = re.compile(regexp)
-    fh = os.popen(cmd)
-    s = fh.read()
     try:
-        ret = fh.close()
-    except IOError:
-        # Happen in Windows test environment
-        ret = 1
-    return (ignorestatus or ret is None) and r.search(s)
+        p = subprocess.Popen(
+            cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    except OSError as e:
+        if e.errno != errno.ENOENT:
+            raise
+        ret = -1
+    ret = p.wait()
+    s = p.stdout.read()
+    return (ignorestatus or not ret) and r.search(s)
 
+@check("baz", "GNU Arch baz client")
 def has_baz():
-    return matchoutput('baz --version 2>&1', r'baz Bazaar version')
+    return matchoutput('baz --version 2>&1', br'baz Bazaar version')
 
+@check("bzr", "Canonical's Bazaar client")
 def has_bzr():
     try:
         import bzrlib
+        import bzrlib.bzrdir
+        import bzrlib.errors
+        import bzrlib.revision
+        import bzrlib.revisionspec
+        bzrlib.revisionspec.RevisionSpec
         return bzrlib.__doc__ is not None
-    except ImportError:
+    except (AttributeError, ImportError):
         return False
 
-def has_bzr114():
+@checkvers("bzr", "Canonical's Bazaar client >= %s", (1.14,))
+def has_bzr_range(v):
+    major, minor = v.split('.')[0:2]
     try:
         import bzrlib
         return (bzrlib.__doc__ is not None
-                and bzrlib.version_info[:2] >= (1, 14))
+                and bzrlib.version_info[:2] >= (int(major), int(minor)))
     except ImportError:
         return False
 
+@check("chg", "running with chg")
+def has_chg():
+    return 'CHGHG' in os.environ
+
+@check("cvs", "cvs client/server")
 def has_cvs():
-    re = r'Concurrent Versions System.*?server'
+    re = br'Concurrent Versions System.*?server'
+    return matchoutput('cvs --version 2>&1', re) and not has_msys()
+
+@check("cvs112", "cvs client/server 1.12.* (not cvsnt)")
+def has_cvs112():
+    re = br'Concurrent Versions System \(CVS\) 1.12.*?server'
     return matchoutput('cvs --version 2>&1', re) and not has_msys()
 
+@check("cvsnt", "cvsnt client/server")
+def has_cvsnt():
+    re = br'Concurrent Versions System \(CVSNT\) (\d+).(\d+).*\(client/server\)'
+    return matchoutput('cvsnt --version 2>&1', re)
+
+@check("darcs", "darcs client")
 def has_darcs():
-    return matchoutput('darcs --version', r'2\.[2-9]', True)
+    return matchoutput('darcs --version', br'\b2\.([2-9]|\d{2})', True)
 
+@check("mtn", "monotone client (>= 1.0)")
 def has_mtn():
-    return matchoutput('mtn --version', r'monotone', True) and not matchoutput(
-        'mtn --version', r'monotone 0\.', True)
+    return matchoutput('mtn --version', br'monotone', True) and not matchoutput(
+        'mtn --version', br'monotone 0\.', True)
 
+@check("eol-in-paths", "end-of-lines in paths")
 def has_eol_in_paths():
     try:
         fd, path = tempfile.mkstemp(dir='.', prefix=tempprefix, suffix='\n\r')
@@ -57,16 +167,17 @@
     except (IOError, OSError):
         return False
 
+@check("execbit", "executable bit")
 def has_executablebit():
     try:
         EXECFLAGS = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
         fh, fn = tempfile.mkstemp(dir='.', prefix=tempprefix)
         try:
             os.close(fh)
-            m = os.stat(fn).st_mode & 0777
+            m = os.stat(fn).st_mode & 0o777
             new_file_has_exec = m & EXECFLAGS
             os.chmod(fn, m ^ EXECFLAGS)
-            exec_flags_cannot_flip = ((os.stat(fn).st_mode & 0777) == m)
+            exec_flags_cannot_flip = ((os.stat(fn).st_mode & 0o777) == m)
         finally:
             os.unlink(fn)
     except (IOError, OSError):
@@ -74,6 +185,7 @@
         return False
     return not (new_file_has_exec or exec_flags_cannot_flip)
 
+@check("icasefs", "case insensitive file system")
 def has_icasefs():
     # Stolen from mercurial.util
     fd, path = tempfile.mkstemp(dir='.', prefix=tempprefix)
@@ -92,21 +204,7 @@
     finally:
         os.remove(path)
 
-def has_inotify():
-    try:
-        import hgext.inotify.linux.watcher
-    except ImportError:
-        return False
-    name = tempfile.mktemp(dir='.', prefix=tempprefix)
-    sock = socket.socket(socket.AF_UNIX)
-    try:
-        sock.bind(name)
-    except socket.error, err:
-        return False
-    sock.close()
-    os.unlink(name)
-    return True
-
+@check("fifo", "named pipes")
 def has_fifo():
     if getattr(os, "mkfifo", None) is None:
         return False
@@ -118,6 +216,11 @@
     except OSError:
         return False
 
+@check("killdaemons", 'killdaemons.py support')
+def has_killdaemons():
+    return True
+
+@check("cacheable", "cacheable filesystem")
 def has_cacheable_fs():
     from mercurial import util
 
@@ -128,42 +231,101 @@
     finally:
         os.remove(path)
 
+@check("lsprof", "python lsprof module")
 def has_lsprof():
     try:
         import _lsprof
+        _lsprof.Profiler # silence unused import warning
         return True
     except ImportError:
         return False
 
-def has_gettext():
-    return matchoutput('msgfmt --version', 'GNU gettext-tools')
+def gethgversion():
+    m = matchoutput('hg --version --quiet 2>&1', br'(\d+)\.(\d+)')
+    if not m:
+        return (0, 0)
+    return (int(m.group(1)), int(m.group(2)))
+
+@checkvers("hg", "Mercurial >= %s",
+            list([(1.0 * x) / 10 for x in range(9, 99)]))
+def has_hg_range(v):
+    major, minor = v.split('.')[0:2]
+    return gethgversion() >= (int(major), int(minor))
+
+@check("hg08", "Mercurial >= 0.8")
+def has_hg08():
+    if checks["hg09"][0]():
+        return True
+    return matchoutput('hg help annotate 2>&1', '--date')
+
+@check("hg07", "Mercurial >= 0.7")
+def has_hg07():
+    if checks["hg08"][0]():
+        return True
+    return matchoutput('hg --version --quiet 2>&1', 'Mercurial Distributed SCM')
+
+@check("hg06", "Mercurial >= 0.6")
+def has_hg06():
+    if checks["hg07"][0]():
+        return True
+    return matchoutput('hg --version --quiet 2>&1', 'Mercurial version')
 
+@check("gettext", "GNU Gettext (msgfmt)")
+def has_gettext():
+    return matchoutput('msgfmt --version', br'GNU gettext-tools')
+
+@check("git", "git command line client")
 def has_git():
-    return matchoutput('git --version 2>&1', r'^git version')
+    return matchoutput('git --version 2>&1', br'^git version')
+
+def getgitversion():
+    m = matchoutput('git --version 2>&1', br'git version (\d+)\.(\d+)')
+    if not m:
+        return (0, 0)
+    return (int(m.group(1)), int(m.group(2)))
 
+# https://github.com/git-lfs/lfs-test-server <https://github.com/git-lfs/lfs-test-server>
+@check("lfs-test-server", "git-lfs test server")
+def has_lfsserver():
+    exe = 'lfs-test-server'
+    if has_windows():
+        exe = 'lfs-test-server.exe'
+    return any(
+        os.access(os.path.join(path, exe), os.X_OK)
+        for path in os.environ["PATH"].split(os.pathsep)
+    )
+
+@checkvers("git", "git client (with ext::sh support) version >= %s", (1.9,))
+def has_git_range(v):
+    major, minor = v.split('.')[0:2]
+    return getgitversion() >= (int(major), int(minor))
+
+@check("docutils", "Docutils text processing library")
 def has_docutils():
     try:
-        from docutils.core import publish_cmdline
+        import docutils.core
+        docutils.core.publish_cmdline # silence unused import
         return True
     except ImportError:
         return False
 
 def getsvnversion():
-    m = matchoutput('svn --version 2>&1', r'^svn,\s+version\s+(\d+)\.(\d+)')
+    m = matchoutput('svn --version --quiet 2>&1', br'^(\d+)\.(\d+)')
     if not m:
         return (0, 0)
     return (int(m.group(1)), int(m.group(2)))
 
-def has_svn15():
-    return getsvnversion() >= (1, 5)
+@checkvers("svn", "subversion client and admin tools >= %s", (1.3, 1.5))
+def has_svn_range(v):
+    major, minor = v.split('.')[0:2]
+    return getsvnversion() >= (int(major), int(minor))
 
-def has_svn13():
-    return getsvnversion() >= (1, 3)
-
+@check("svn", "subversion client and admin tools")
 def has_svn():
-    return matchoutput('svn --version 2>&1', r'^svn, version') and \
-        matchoutput('svnadmin --version 2>&1', r'^svnadmin, version')
+    return matchoutput('svn --version 2>&1', br'^svn, version') and \
+        matchoutput('svnadmin --version 2>&1', br'^svnadmin, version')
 
+@check("svn-bindings", "subversion python bindings")
 def has_svn_bindings():
     try:
         import svn.core
@@ -174,10 +336,12 @@
     except ImportError:
         return False
 
+@check("p4", "Perforce server and client")
 def has_p4():
-    return (matchoutput('p4 -V', r'Rev\. P4/') and
-            matchoutput('p4d -V', r'Rev\. P4D/'))
+    return (matchoutput('p4 -V', br'Rev\. P4/') and
+            matchoutput('p4d -V', br'Rev\. P4D/'))
 
+@check("symlink", "symbolic links")
 def has_symlink():
     if getattr(os, "symlink", None) is None:
         return False
@@ -189,120 +353,358 @@
     except (OSError, AttributeError):
         return False
 
+@check("hardlink", "hardlinks")
 def has_hardlink():
     from mercurial import util
     fh, fn = tempfile.mkstemp(dir='.', prefix=tempprefix)
     os.close(fh)
     name = tempfile.mktemp(dir='.', prefix=tempprefix)
     try:
-        try:
-            util.oslink(fn, name)
-            os.unlink(name)
-            return True
-        except OSError:
-            return False
+        util.oslink(fn, name)
+        os.unlink(name)
+        return True
+    except OSError:
+        return False
     finally:
         os.unlink(fn)
 
-def has_tla():
-    return matchoutput('tla --version 2>&1', r'The GNU Arch Revision')
+@check("hardlink-whitelisted", "hardlinks on whitelisted filesystems")
+def has_hardlink_whitelisted():
+    from mercurial import util
+    try:
+        fstype = util.getfstype('.')
+    except OSError:
+        return False
+    return fstype in util._hardlinkfswhitelist
 
+@check("rmcwd", "can remove current working directory")
+def has_rmcwd():
+    ocwd = os.getcwd()
+    temp = tempfile.mkdtemp(dir='.', prefix=tempprefix)
+    try:
+        os.chdir(temp)
+        # On Linux, 'rmdir .' isn't allowed, but the other names are okay.
+        # On Solaris and Windows, the cwd can't be removed by any names.
+        os.rmdir(os.getcwd())
+        return True
+    except OSError:
+        return False
+    finally:
+        os.chdir(ocwd)
+        # clean up temp dir on platforms where cwd can't be removed
+        try:
+            os.rmdir(temp)
+        except OSError:
+            pass
+
+@check("tla", "GNU Arch tla client")
+def has_tla():
+    return matchoutput('tla --version 2>&1', br'The GNU Arch Revision')
+
+@check("gpg", "gpg client")
 def has_gpg():
-    return matchoutput('gpg --version 2>&1', r'GnuPG')
+    return matchoutput('gpg --version 2>&1', br'GnuPG')
+
+@check("gpg2", "gpg client v2")
+def has_gpg2():
+    return matchoutput('gpg --version 2>&1', br'GnuPG[^0-9]+2\.')
 
+@check("gpg21", "gpg client v2.1+")
+def has_gpg21():
+    return matchoutput('gpg --version 2>&1', br'GnuPG[^0-9]+2\.(?!0)')
+
+@check("unix-permissions", "unix-style permissions")
 def has_unix_permissions():
     d = tempfile.mkdtemp(dir='.', prefix=tempprefix)
     try:
         fname = os.path.join(d, 'foo')
-        for umask in (077, 007, 022):
+        for umask in (0o77, 0o07, 0o22):
             os.umask(umask)
             f = open(fname, 'w')
             f.close()
             mode = os.stat(fname).st_mode
             os.unlink(fname)
-            if mode & 0777 != ~umask & 0666:
+            if mode & 0o777 != ~umask & 0o666:
                 return False
         return True
     finally:
         os.rmdir(d)
 
+@check("unix-socket", "AF_UNIX socket family")
+def has_unix_socket():
+    return getattr(socket, 'AF_UNIX', None) is not None
+
+@check("root", "root permissions")
+def has_root():
+    return getattr(os, 'geteuid', None) and os.geteuid() == 0
+
+@check("pyflakes", "Pyflakes python linter")
 def has_pyflakes():
     return matchoutput("sh -c \"echo 'import re' 2>&1 | pyflakes\"",
-                       r"<stdin>:1: 're' imported but unused",
+                       br"<stdin>:1: 're' imported but unused",
+                       True)
+
+@check("pylint", "Pylint python linter")
+def has_pylint():
+    return matchoutput("pylint --help",
+                       br"Usage:  pylint",
                        True)
 
+@check("clang-format", "clang-format C code formatter")
+def has_clang_format():
+    return matchoutput("clang-format --help",
+                       br"^OVERVIEW: A tool to format C/C\+\+[^ ]+ code.")
+
+@check("jshint", "JSHint static code analysis tool")
+def has_jshint():
+    return matchoutput("jshint --version 2>&1", br"jshint v")
+
+@check("pygments", "Pygments source highlighting library")
 def has_pygments():
     try:
         import pygments
+        pygments.highlight # silence unused import warning
         return True
     except ImportError:
         return False
 
+@check("outer-repo", "outer repo")
 def has_outer_repo():
     # failing for other reasons than 'no repo' imply that there is a repo
     return not matchoutput('hg root 2>&1',
-                           r'abort: no repository found', True)
+                           br'abort: no repository found', True)
 
+@check("ssl", "ssl module available")
 def has_ssl():
     try:
         import ssl
-        import OpenSSL
-        OpenSSL.SSL.Context
+        ssl.CERT_NONE
         return True
     except ImportError:
         return False
 
+@check("sslcontext", "python >= 2.7.9 ssl")
+def has_sslcontext():
+    try:
+        import ssl
+        ssl.SSLContext
+        return True
+    except (ImportError, AttributeError):
+        return False
+
+@check("defaultcacerts", "can verify SSL certs by system's CA certs store")
+def has_defaultcacerts():
+    from mercurial import sslutil, ui as uimod
+    ui = uimod.ui.load()
+    return sslutil._defaultcacerts(ui) or sslutil._canloaddefaultcerts
+
+@check("defaultcacertsloaded", "detected presence of loaded system CA certs")
+def has_defaultcacertsloaded():
+    import ssl
+    from mercurial import sslutil, ui as uimod
+
+    if not has_defaultcacerts():
+        return False
+    if not has_sslcontext():
+        return False
+
+    ui = uimod.ui.load()
+    cafile = sslutil._defaultcacerts(ui)
+    ctx = ssl.create_default_context()
+    if cafile:
+        ctx.load_verify_locations(cafile=cafile)
+    else:
+        ctx.load_default_certs()
+
+    return len(ctx.get_ca_certs()) > 0
+
+@check("tls1.2", "TLS 1.2 protocol support")
+def has_tls1_2():
+    from mercurial import sslutil
+    return 'tls1.2' in sslutil.supportedprotocols
+
+@check("windows", "Windows")
 def has_windows():
     return os.name == 'nt'
 
+@check("system-sh", "system() uses sh")
 def has_system_sh():
     return os.name != 'nt'
 
+@check("serve", "platform and python can manage 'hg serve -d'")
 def has_serve():
-    return os.name != 'nt' # gross approximation
+    return True
+
+@check("test-repo", "running tests from repository")
+def has_test_repo():
+    t = os.environ["TESTDIR"]
+    return os.path.isdir(os.path.join(t, "..", ".hg"))
 
+@check("tic", "terminfo compiler and curses module")
 def has_tic():
-    return matchoutput('test -x "`which tic`"', '')
+    try:
+        import curses
+        curses.COLOR_BLUE
+        return matchoutput('test -x "`which tic`"', br'')
+    except ImportError:
+        return False
 
+@check("msys", "Windows with MSYS")
 def has_msys():
     return os.getenv('MSYSTEM')
 
-checks = {
-    "true": (lambda: True, "yak shaving"),
-    "false": (lambda: False, "nail clipper"),
-    "baz": (has_baz, "GNU Arch baz client"),
-    "bzr": (has_bzr, "Canonical's Bazaar client"),
-    "bzr114": (has_bzr114, "Canonical's Bazaar client >= 1.14"),
-    "cacheable": (has_cacheable_fs, "cacheable filesystem"),
-    "cvs": (has_cvs, "cvs client/server"),
-    "darcs": (has_darcs, "darcs client"),
-    "docutils": (has_docutils, "Docutils text processing library"),
-    "eol-in-paths": (has_eol_in_paths, "end-of-lines in paths"),
-    "execbit": (has_executablebit, "executable bit"),
-    "fifo": (has_fifo, "named pipes"),
-    "gettext": (has_gettext, "GNU Gettext (msgfmt)"),
-    "git": (has_git, "git command line client"),
-    "gpg": (has_gpg, "gpg client"),
-    "hardlink": (has_hardlink, "hardlinks"),
-    "icasefs": (has_icasefs, "case insensitive file system"),
-    "inotify": (has_inotify, "inotify extension support"),
-    "lsprof": (has_lsprof, "python lsprof module"),
-    "mtn": (has_mtn, "monotone client (>= 1.0)"),
-    "outer-repo": (has_outer_repo, "outer repo"),
-    "p4": (has_p4, "Perforce server and client"),
-    "pyflakes": (has_pyflakes, "Pyflakes python linter"),
-    "pygments": (has_pygments, "Pygments source highlighting library"),
-    "serve": (has_serve, "platform and python can manage 'hg serve -d'"),
-    "ssl": (has_ssl, "python >= 2.6 ssl module and python OpenSSL"),
-    "svn": (has_svn, "subversion client and admin tools"),
-    "svn13": (has_svn13, "subversion client and admin tools >= 1.3"),
-    "svn15": (has_svn15, "subversion client and admin tools >= 1.5"),
-    "svn-bindings": (has_svn_bindings, "subversion python bindings"),
-    "symlink": (has_symlink, "symbolic links"),
-    "system-sh": (has_system_sh, "system() uses sh"),
-    "tic": (has_tic, "terminfo compiler"),
-    "tla": (has_tla, "GNU Arch tla client"),
-    "unix-permissions": (has_unix_permissions, "unix-style permissions"),
-    "windows": (has_windows, "Windows"),
-    "msys": (has_msys, "Windows with MSYS"),
-}
+@check("aix", "AIX")
+def has_aix():
+    return sys.platform.startswith("aix")
+
+@check("osx", "OS X")
+def has_osx():
+    return sys.platform == 'darwin'
+
+@check("osxpackaging", "OS X packaging tools")
+def has_osxpackaging():
+    try:
+        return (matchoutput('pkgbuild', br'Usage: pkgbuild ', ignorestatus=1)
+                and matchoutput(
+                    'productbuild', br'Usage: productbuild ',
+                    ignorestatus=1)
+                and matchoutput('lsbom', br'Usage: lsbom', ignorestatus=1)
+                and matchoutput(
+                    'xar --help', br'Usage: xar', ignorestatus=1))
+    except ImportError:
+        return False
+
+@check('linuxormacos', 'Linux or MacOS')
+def has_linuxormacos():
+    # This isn't a perfect test for MacOS. But it is sufficient for our needs.
+    return sys.platform.startswith(('linux', 'darwin'))
+
+@check("docker", "docker support")
+def has_docker():
+    pat = br'A self-sufficient runtime for'
+    if matchoutput('docker --help', pat):
+        if 'linux' not in sys.platform:
+            # TODO: in theory we should be able to test docker-based
+            # package creation on non-linux using boot2docker, but in
+            # practice that requires extra coordination to make sure
+            # $TESTTEMP is going to be visible at the same path to the
+            # boot2docker VM. If we figure out how to verify that, we
+            # can use the following instead of just saying False:
+            # return 'DOCKER_HOST' in os.environ
+            return False
+
+        return True
+    return False
+
+@check("debhelper", "debian packaging tools")
+def has_debhelper():
+    # Some versions of dpkg say `dpkg', some say 'dpkg' (` vs ' on the first
+    # quote), so just accept anything in that spot.
+    dpkg = matchoutput('dpkg --version',
+                       br"Debian .dpkg' package management program")
+    dh = matchoutput('dh --help',
+                     br'dh is a part of debhelper.', ignorestatus=True)
+    dh_py2 = matchoutput('dh_python2 --help',
+                         br'other supported Python versions')
+    # debuild comes from the 'devscripts' package, though you might want
+    # the 'build-debs' package instead, which has a dependency on devscripts.
+    debuild = matchoutput('debuild --help',
+                          br'to run debian/rules with given parameter')
+    return dpkg and dh and dh_py2 and debuild
+
+@check("debdeps",
+       "debian build dependencies (run dpkg-checkbuilddeps in contrib/)")
+def has_debdeps():
+    # just check exit status (ignoring output)
+    path = '%s/../contrib/debian/control' % os.environ['TESTDIR']
+    return matchoutput('dpkg-checkbuilddeps %s' % path, br'')
+
+@check("demandimport", "demandimport enabled")
+def has_demandimport():
+    # chg disables demandimport intentionally for performance wins.
+    return ((not has_chg()) and os.environ.get('HGDEMANDIMPORT') != 'disable')
+
+@check("py3k", "running with Python 3.x")
+def has_py3k():
+    return 3 == sys.version_info[0]
+
+@check("py3exe", "a Python 3.x interpreter is available")
+def has_python3exe():
+    return 'PYTHON3' in os.environ
+
+@check("py3pygments", "Pygments available on Python 3.x")
+def has_py3pygments():
+    if has_py3k():
+        return has_pygments()
+    elif has_python3exe():
+        # just check exit status (ignoring output)
+        py3 = os.environ['PYTHON3']
+        return matchoutput('%s -c "import pygments"' % py3, br'')
+    return False
+
+@check("pure", "running with pure Python code")
+def has_pure():
+    return any([
+        os.environ.get("HGMODULEPOLICY") == "py",
+        os.environ.get("HGTEST_RUN_TESTS_PURE") == "--pure",
+    ])
+
+@check("slow", "allow slow tests (use --allow-slow-tests)")
+def has_slow():
+    return os.environ.get('HGTEST_SLOW') == 'slow'
+
+@check("hypothesis", "Hypothesis automated test generation")
+def has_hypothesis():
+    try:
+        import hypothesis
+        hypothesis.given
+        return True
+    except ImportError:
+        return False
+
+@check("unziplinks", "unzip(1) understands and extracts symlinks")
+def unzip_understands_symlinks():
+    return matchoutput('unzip --help', br'Info-ZIP')
+
+@check("zstd", "zstd Python module available")
+def has_zstd():
+    try:
+        import mercurial.zstd
+        mercurial.zstd.__version__
+        return True
+    except ImportError:
+        return False
+
+@check("devfull", "/dev/full special file")
+def has_dev_full():
+    return os.path.exists('/dev/full')
+
+@check("virtualenv", "Python virtualenv support")
+def has_virtualenv():
+    try:
+        import virtualenv
+        virtualenv.ACTIVATE_SH
+        return True
+    except ImportError:
+        return False
+
+@check("fsmonitor", "running tests with fsmonitor")
+def has_fsmonitor():
+    return 'HGFSMONITOR_TESTS' in os.environ
+
+@check("fuzzywuzzy", "Fuzzy string matching library")
+def has_fuzzywuzzy():
+    try:
+        import fuzzywuzzy
+        fuzzywuzzy.__version__
+        return True
+    except ImportError:
+        return False
+
+@check("clang-libfuzzer", "clang new enough to include libfuzzer")
+def has_clang_libfuzzer():
+    mat = matchoutput('clang --version', 'clang version (\d)')
+    if mat:
+        # libfuzzer is new in clang 6
+        return int(mat.group(1)) > 5
+    return False
diff -r bb255eaf3d14 -r cca56bbea143 tests/run-tests.py
--- a/tests/run-tests.py	Tue Feb 06 12:12:49 2018 -0600
+++ b/tests/run-tests.py	Tue Feb 06 16:34:28 2018 -0800
@@ -45,11 +45,12 @@
 
 from __future__ import absolute_import, print_function
 
+import argparse
+import collections
 import difflib
 import distutils.version as version
 import errno
 import json
-import optparse
 import os
 import random
 import re
@@ -119,6 +120,7 @@
         }
 
     class TestRunnerLexer(lexer.RegexLexer):
+        testpattern = r'[\w-]+\.(t|py)( \(case [\w-]+\))?'
         tokens = {
             'root': [
                 (r'^Skipped', token.Generic.Skipped, 'skipped'),
@@ -126,11 +128,11 @@
                 (r'^ERROR: ', token.Generic.Failed, 'failed'),
             ],
             'skipped': [
-                (r'[\w-]+\.(t|py)', token.Generic.SName),
+                (testpattern, token.Generic.SName),
                 (r':.*', token.Generic.Skipped),
             ],
             'failed': [
-                (r'[\w-]+\.(t|py)', token.Generic.FName),
+                (testpattern, token.Generic.FName),
                 (r'(:| ).*', token.Generic.Failed),
             ]
         }
@@ -296,122 +298,132 @@
 
 def getparser():
     """Obtain the OptionParser used by the CLI."""
-    parser = optparse.OptionParser("%prog [options] [tests]")
-
-    # keep these sorted
-    parser.add_option("--blacklist", action="append",
+    parser = argparse.ArgumentParser(usage='%(prog)s [options] [tests]')
+
+    selection = parser.add_argument_group('Test Selection')
+    selection.add_argument('--allow-slow-tests', action='store_true',
+        help='allow extremely slow tests')
+    selection.add_argument("--blacklist", action="append",
         help="skip tests listed in the specified blacklist file")
-    parser.add_option("--whitelist", action="append",
+    selection.add_argument("--changed",
+        help="run tests that are changed in parent rev or working directory")
+    selection.add_argument("-k", "--keywords",
+        help="run tests matching keywords")
+    selection.add_argument("-r", "--retest", action="store_true",
+        help = "retest failed tests")
+    selection.add_argument("--test-list", action="append",
+        help="read tests to run from the specified file")
+    selection.add_argument("--whitelist", action="append",
         help="always run tests listed in the specified whitelist file")
-    parser.add_option("--test-list", action="append",
-                      help="read tests to run from the specified file")
-    parser.add_option("--changed", type="string",
-        help="run tests that are changed in parent rev or working directory")
-    parser.add_option("-C", "--annotate", action="store_true",
-        help="output files annotated with coverage")
-    parser.add_option("-c", "--cover", action="store_true",
-        help="print a test coverage report")
-    parser.add_option("--color", choices=["always", "auto", "never"],
-                      default=os.environ.get('HGRUNTESTSCOLOR', 'auto'),
-                      help="colorisation: always|auto|never (default: auto)")
-    parser.add_option("-d", "--debug", action="store_true",
+    selection.add_argument('tests', metavar='TESTS', nargs='*',
+                        help='Tests to run')
+
+    harness = parser.add_argument_group('Test Harness Behavior')
+    harness.add_argument('--bisect-repo',
+                        metavar='bisect_repo',
+                        help=("Path of a repo to bisect. Use together with "
+                              "--known-good-rev"))
+    harness.add_argument("-d", "--debug", action="store_true",
         help="debug mode: write output of test scripts to console"
              " rather than capturing and diffing it (disables timeout)")
-    parser.add_option("-f", "--first", action="store_true",
+    harness.add_argument("-f", "--first", action="store_true",
         help="exit on the first test failure")
-    parser.add_option("-H", "--htmlcov", action="store_true",
-        help="create an HTML report of the coverage of the files")
-    parser.add_option("-i", "--interactive", action="store_true",
+    harness.add_argument("-i", "--interactive", action="store_true",
         help="prompt to accept changed output")
-    parser.add_option("-j", "--jobs", type="int",
+    harness.add_argument("-j", "--jobs", type=int,
         help="number of jobs to run in parallel"
              " (default: $%s or %d)" % defaults['jobs'])
-    parser.add_option("--keep-tmpdir", action="store_true",
+    harness.add_argument("--keep-tmpdir", action="store_true",
         help="keep temporary directory after running tests")
-    parser.add_option("-k", "--keywords",
-        help="run tests matching keywords")
-    parser.add_option("--list-tests", action="store_true",
+    harness.add_argument('--known-good-rev',
+                        metavar="known_good_rev",
+                        help=("Automatically bisect any failures using this "
+                              "revision as a known-good revision."))
+    harness.add_argument("--list-tests", action="store_true",
         help="list tests instead of running them")
-    parser.add_option("-l", "--local", action="store_true",
+    harness.add_argument("--loop", action="store_true",
+        help="loop tests repeatedly")
+    harness.add_argument('--random', action="store_true",
+        help='run tests in random order')
+    harness.add_argument("-p", "--port", type=int,
+        help="port on which servers should listen"
+             " (default: $%s or %d)" % defaults['port'])
+    harness.add_argument('--profile-runner', action='store_true',
+                        help='run statprof on run-tests')
+    harness.add_argument("-R", "--restart", action="store_true",
+        help="restart at last error")
+    harness.add_argument("--runs-per-test", type=int, dest="runs_per_test",
+        help="run each test N times (default=1)", default=1)
+    harness.add_argument("--shell",
+        help="shell to use (default: $%s or %s)" % defaults['shell'])
+    harness.add_argument('--showchannels', action='store_true',
+                        help='show scheduling channels')
+    harness.add_argument("--slowtimeout", type=int,
+        help="kill errant slow tests after SLOWTIMEOUT seconds"
+             " (default: $%s or %d)" % defaults['slowtimeout'])
+    harness.add_argument("-t", "--timeout", type=int,
+        help="kill errant tests after TIMEOUT seconds"
+             " (default: $%s or %d)" % defaults['timeout'])
+    harness.add_argument("--tmpdir",
+        help="run tests in the given temporary directory"
+             " (implies --keep-tmpdir)")
+    harness.add_argument("-v", "--verbose", action="store_true",
+        help="output verbose messages")
+
+    hgconf = parser.add_argument_group('Mercurial Configuration')
+    hgconf.add_argument("--chg", action="store_true",
+        help="install and use chg wrapper in place of hg")
+    hgconf.add_argument("--compiler",
+        help="compiler to build with")
+    hgconf.add_argument('--extra-config-opt', action="append", default=[],
+        help='set the given config opt in the test hgrc')
+    hgconf.add_argument("-l", "--local", action="store_true",
         help="shortcut for --with-hg=<testdir>/../hg, "
              "and --with-chg=<testdir>/../contrib/chg/chg if --chg is set")
-    parser.add_option("--loop", action="store_true",
-        help="loop tests repeatedly")
-    parser.add_option("--runs-per-test", type="int", dest="runs_per_test",
-        help="run each test N times (default=1)", default=1)
-    parser.add_option("-n", "--nodiff", action="store_true",
-        help="skip showing test changes")
-    parser.add_option("--outputdir", type="string",
-        help="directory to write error logs to (default=test directory)")
-    parser.add_option("-p", "--port", type="int",
-        help="port on which servers should listen"
-             " (default: $%s or %d)" % defaults['port'])
-    parser.add_option("--compiler", type="string",
-        help="compiler to build with")
-    parser.add_option("--pure", action="store_true",
+    hgconf.add_argument("--ipv6", action="store_true",
+        help="prefer IPv6 to IPv4 for network related tests")
+    hgconf.add_argument("--pure", action="store_true",
         help="use pure Python code instead of C extensions")
-    parser.add_option("-R", "--restart", action="store_true",
-        help="restart at last error")
-    parser.add_option("-r", "--retest", action="store_true",
-        help="retest failed tests")
-    parser.add_option("-S", "--noskips", action="store_true",
-        help="don't report skip tests verbosely")
-    parser.add_option("--shell", type="string",
-        help="shell to use (default: $%s or %s)" % defaults['shell'])
-    parser.add_option("-t", "--timeout", type="int",
-        help="kill errant tests after TIMEOUT seconds"
-             " (default: $%s or %d)" % defaults['timeout'])
-    parser.add_option("--slowtimeout", type="int",
-        help="kill errant slow tests after SLOWTIMEOUT seconds"
-             " (default: $%s or %d)" % defaults['slowtimeout'])
-    parser.add_option("--time", action="store_true",
-        help="time how long each test takes")
-    parser.add_option("--json", action="store_true",
-                      help="store test result data in 'report.json' file")
-    parser.add_option("--tmpdir", type="string",
-        help="run tests in the given temporary directory"
-             " (implies --keep-tmpdir)")
-    parser.add_option("-v", "--verbose", action="store_true",
-        help="output verbose messages")
-    parser.add_option("--xunit", type="string",
-                      help="record xunit results at specified path")
-    parser.add_option("--view", type="string",
-        help="external diff viewer")
-    parser.add_option("--with-hg", type="string",
+    hgconf.add_argument("-3", "--py3k-warnings", action="store_true",
+        help="enable Py3k warnings on Python 2.7+")
+    hgconf.add_argument("--with-chg", metavar="CHG",
+        help="use specified chg wrapper in place of hg")
+    hgconf.add_argument("--with-hg",
         metavar="HG",
         help="test using specified hg script rather than a "
              "temporary installation")
-    parser.add_option("--chg", action="store_true",
-                      help="install and use chg wrapper in place of hg")
-    parser.add_option("--with-chg", metavar="CHG",
-                      help="use specified chg wrapper in place of hg")
-    parser.add_option("--ipv6", action="store_true",
-                      help="prefer IPv6 to IPv4 for network related tests")
-    parser.add_option("-3", "--py3k-warnings", action="store_true",
-        help="enable Py3k warnings on Python 2.7+")
     # This option should be deleted once test-check-py3-compat.t and other
     # Python 3 tests run with Python 3.
-    parser.add_option("--with-python3", metavar="PYTHON3",
-                      help="Python 3 interpreter (if running under Python 2)"
-                           " (TEMPORARY)")
-    parser.add_option('--extra-config-opt', action="append",
-                      help='set the given config opt in the test hgrc')
-    parser.add_option('--random', action="store_true",
-                      help='run tests in random order')
-    parser.add_option('--profile-runner', action='store_true',
-                      help='run statprof on run-tests')
-    parser.add_option('--allow-slow-tests', action='store_true',
-                      help='allow extremely slow tests')
-    parser.add_option('--showchannels', action='store_true',
-                      help='show scheduling channels')
-    parser.add_option('--known-good-rev', type="string",
-                      metavar="known_good_rev",
-                      help=("Automatically bisect any failures using this "
-                            "revision as a known-good revision."))
-    parser.add_option('--bisect-repo', type="string",
-                      metavar='bisect_repo',
-                      help=("Path of a repo to bisect. Use together with "
-                            "--known-good-rev"))
+    hgconf.add_argument("--with-python3", metavar="PYTHON3",
+        help="Python 3 interpreter (if running under Python 2)"
+             " (TEMPORARY)")
+
+    reporting = parser.add_argument_group('Results Reporting')
+    reporting.add_argument("-C", "--annotate", action="store_true",
+        help="output files annotated with coverage")
+    reporting.add_argument("--color", choices=["always", "auto", "never"],
+        default=os.environ.get('HGRUNTESTSCOLOR', 'auto'),
+        help="colorisation: always|auto|never (default: auto)")
+    reporting.add_argument("-c", "--cover", action="store_true",
+        help="print a test coverage report")
+    reporting.add_argument('--exceptions', action='store_true',
+        help='log all exceptions and generate an exception report')
+    reporting.add_argument("-H", "--htmlcov", action="store_true",
+        help="create an HTML report of the coverage of the files")
+    reporting.add_argument("--json", action="store_true",
+        help="store test result data in 'report.json' file")
+    reporting.add_argument("--outputdir",
+        help="directory to write error logs to (default=test directory)")
+    reporting.add_argument("-n", "--nodiff", action="store_true",
+        help="skip showing test changes")
+    reporting.add_argument("-S", "--noskips", action="store_true",
+        help="don't report skip tests verbosely")
+    reporting.add_argument("--time", action="store_true",
+        help="time how long each test takes")
+    reporting.add_argument("--view",
+        help="external diff viewer")
+    reporting.add_argument("--xunit",
+        help="record xunit results at specified path")
 
     for option, (envvar, default) in defaults.items():
         defaults[option] = type(default)(os.environ.get(envvar, default))
@@ -421,7 +433,7 @@
 
 def parseargs(args, parser):
     """Parse arguments with our OptionParser and validate results."""
-    (options, args) = parser.parse_args(args)
+    options = parser.parse_args(args)
 
     # jython is always pure
     if 'java' in sys.platform or '__pypy__' in sys.modules:
@@ -550,7 +562,7 @@
     if options.showchannels:
         options.nodiff = True
 
-    return (options, args)
+    return options
 
 def rename(src, dst):
     """Like os.rename(), trade atomicity and opened files friendliness
@@ -659,6 +671,7 @@
 
     def __init__(self, path, outputdir, tmpdir, keeptmpdir=False,
                  debug=False,
+                 first=False,
                  timeout=None,
                  startport=None, extraconfigopts=None,
                  py3kwarnings=False, shell=None, hgcommand=None,
@@ -711,6 +724,7 @@
         self._threadtmp = tmpdir
         self._keeptmpdir = keeptmpdir
         self._debug = debug
+        self._first = first
         self._timeout = timeout
         self._slowtimeout = slowtimeout
         self._startport = startport
@@ -890,15 +904,18 @@
             # Diff generation may rely on written .err file.
             if (ret != 0 or out != self._refout) and not self._skipped \
                 and not self._debug:
-                f = open(self.errpath, 'wb')
-                for line in out:
-                    f.write(line)
-                f.close()
+                with open(self.errpath, 'wb') as f:
+                    for line in out:
+                        f.write(line)
 
             # The result object handles diff calculation for us.
-            if self._result.addOutputMismatch(self, ret, out, self._refout):
-                # change was accepted, skip failing
-                return
+            with firstlock:
+                if self._result.addOutputMismatch(self, ret, out, self._refout):
+                    # change was accepted, skip failing
+                    return
+                if self._first:
+                    global firsterror
+                    firsterror = True
 
             if ret:
                 msg = 'output changed and ' + describe(ret)
@@ -930,10 +947,9 @@
 
         if (self._ret != 0 or self._out != self._refout) and not self._skipped \
             and not self._debug and self._out:
-            f = open(self.errpath, 'wb')
-            for line in self._out:
-                f.write(line)
-            f.close()
+            with open(self.errpath, 'wb') as f:
+                for line in self._out:
+                    f.write(line)
 
         vlog("# Ret was:", self._ret, '(%s)' % self.name)
 
@@ -961,13 +977,20 @@
             self._portmap(0),
             self._portmap(1),
             self._portmap(2),
-            (br'(?m)^(saved backup bundle to .*\.hg)( \(glob\))?$',
-             br'\1 (glob)'),
             (br'([^0-9])%s' % re.escape(self._localip()), br'\1$LOCALIP'),
             (br'\bHG_TXNID=TXN:[a-f0-9]{40}\b', br'HG_TXNID=TXN:$ID$'),
             ]
         r.append((self._escapepath(self._testtmp), b'$TESTTMP'))
 
+        replacementfile = os.path.join(self._testdir, b'common-pattern.py')
+
+        if os.path.exists(replacementfile):
+            data = {}
+            with open(replacementfile, mode='rb') as source:
+                # the intermediate 'compile' step help with debugging
+                code = compile(source.read(), replacementfile, 'exec')
+                exec(code, data)
+                r.extend(data.get('substitutions', ()))
         return r
 
     def _escapepath(self, p):
@@ -1021,7 +1044,7 @@
             offset = '' if i == 0 else '%s' % i
             env["HGPORT%s" % offset] = '%s' % (self._startport + i)
         env = os.environ.copy()
-        env['PYTHONUSERBASE'] = sysconfig.get_config_var('userbase')
+        env['PYTHONUSERBASE'] = sysconfig.get_config_var('userbase') or ''
         env['HGEMITWARNINGS'] = '1'
         env['TESTTMP'] = self._testtmp
         env['HOME'] = self._testtmp
@@ -1069,29 +1092,31 @@
 
     def _createhgrc(self, path):
         """Create an hgrc file for this test."""
-        hgrc = open(path, 'wb')
-        hgrc.write(b'[ui]\n')
-        hgrc.write(b'slash = True\n')
-        hgrc.write(b'interactive = False\n')
-        hgrc.write(b'mergemarkers = detailed\n')
-        hgrc.write(b'promptecho = True\n')
-        hgrc.write(b'[defaults]\n')
-        hgrc.write(b'[devel]\n')
-        hgrc.write(b'all-warnings = true\n')
-        hgrc.write(b'default-date = 0 0\n')
-        hgrc.write(b'[largefiles]\n')
-        hgrc.write(b'usercache = %s\n' %
-                   (os.path.join(self._testtmp, b'.cache/largefiles')))
-        hgrc.write(b'[web]\n')
-        hgrc.write(b'address = localhost\n')
-        hgrc.write(b'ipv6 = %s\n' % str(self._useipv6).encode('ascii'))
-
-        for opt in self._extraconfigopts:
-            section, key = opt.split('.', 1)
-            assert '=' in key, ('extra config opt %s must '
-                                'have an = for assignment' % opt)
-            hgrc.write(b'[%s]\n%s\n' % (section, key))
-        hgrc.close()
+        with open(path, 'wb') as hgrc:
+            hgrc.write(b'[ui]\n')
+            hgrc.write(b'slash = True\n')
+            hgrc.write(b'interactive = False\n')
+            hgrc.write(b'mergemarkers = detailed\n')
+            hgrc.write(b'promptecho = True\n')
+            hgrc.write(b'[defaults]\n')
+            hgrc.write(b'[devel]\n')
+            hgrc.write(b'all-warnings = true\n')
+            hgrc.write(b'default-date = 0 0\n')
+            hgrc.write(b'[largefiles]\n')
+            hgrc.write(b'usercache = %s\n' %
+                       (os.path.join(self._testtmp, b'.cache/largefiles')))
+            hgrc.write(b'[lfs]\n')
+            hgrc.write(b'usercache = %s\n' %
+                       (os.path.join(self._testtmp, b'.cache/lfs')))
+            hgrc.write(b'[web]\n')
+            hgrc.write(b'address = localhost\n')
+            hgrc.write(b'ipv6 = %s\n' % str(self._useipv6).encode('ascii'))
+
+            for opt in self._extraconfigopts:
+                section, key = opt.encode('utf-8').split(b'.', 1)
+                assert b'=' in key, ('extra config opt %s must '
+                                     'have an = for assignment' % opt)
+                hgrc.write(b'[%s]\n%s\n' % (section, key))
 
     def fail(self, msg):
         # unittest differentiates between errored and failed.
@@ -1197,9 +1222,7 @@
 
     def __init__(self, path, *args, **kwds):
         # accept an extra "case" parameter
-        case = None
-        if 'case' in kwds:
-            case = kwds.pop('case')
+        case = kwds.pop('case', None)
         self._case = case
         self._allcases = parsettestcases(path)
         super(TTest, self).__init__(path, *args, **kwds)
@@ -1213,9 +1236,8 @@
         return os.path.join(self._testdir, self.bname)
 
     def _run(self, env):
-        f = open(self.path, 'rb')
-        lines = f.readlines()
-        f.close()
+        with open(self.path, 'rb') as f:
+            lines = f.readlines()
 
         # .t file is both reference output and the test input, keep reference
         # output updated with the the test input. This avoids some race
@@ -1227,10 +1249,9 @@
 
         # Write out the generated script.
         fname = b'%s.sh' % self._testtmp
-        f = open(fname, 'wb')
-        for l in script:
-            f.write(l)
-        f.close()
+        with open(fname, 'wb') as f:
+            for l in script:
+                f.write(l)
 
         cmd = b'%s "%s"' % (self._shell, fname)
         vlog("# Running", cmd)
@@ -1320,6 +1341,13 @@
             script.append(b'alias hg="%s"\n' % self._hgcommand)
         if os.getenv('MSYSTEM'):
             script.append(b'alias pwd="pwd -W"\n')
+        if self._case:
+            if isinstance(self._case, str):
+                quoted = shellquote(self._case)
+            else:
+                quoted = shellquote(self._case.decode('utf8')).encode('utf8')
+            script.append(b'TESTCASE=%s\n' % quoted)
+            script.append(b'export TESTCASE\n')
 
         n = 0
         for n, l in enumerate(lines):
@@ -1430,10 +1458,7 @@
 
                     r = self.linematch(el, lout)
                     if isinstance(r, str):
-                        if r == '+glob':
-                            lout = el[:-1] + ' (glob)\n'
-                            r = '' # Warn only this line.
-                        elif r == '-glob':
+                        if r == '-glob':
                             lout = ''.join(el.rsplit(' (glob)', 1))
                             r = '' # Warn only this line.
                         elif r == "retry":
@@ -1517,6 +1542,7 @@
     @staticmethod
     def rematch(el, l):
         try:
+            el = b'(?:' + el + b')'
             # use \Z to ensure that the regex matches to the end of the string
             if os.name == 'nt':
                 return re.match(el + br'\r?\n\Z', l)
@@ -1588,8 +1614,10 @@
                 if l.endswith(b" (glob)\n"):
                     l = l[:-8] + b"\n"
                 return TTest.globmatch(el[:-8], l) or retry
-            if os.altsep and l.replace(b'\\', b'/') == el:
-                return b'+glob'
+            if os.altsep:
+                _l = l.replace(b'\\', b'/')
+                if el == _l or os.name == 'nt' and el[:-1] + b'\r\n' == _l:
+                    return True
         return retry
 
     @staticmethod
@@ -1620,6 +1648,8 @@
         return TTest.ESCAPESUB(TTest._escapef, s)
 
 iolock = threading.RLock()
+firstlock = threading.RLock()
+firsterror = False
 
 class TestResult(unittest._TextTestResult):
     """Holds results when executing via unittest."""
@@ -1705,7 +1735,7 @@
 
     def addOutputMismatch(self, test, ret, got, expected):
         """Record a mismatch in test output for a particular test."""
-        if self.shouldStop:
+        if self.shouldStop or firsterror:
             # don't print, some other test case already failed and
             # printed, we're just stale and probably failed due to our
             # temp dir getting cleaned up.
@@ -1865,9 +1895,8 @@
                     continue
 
                 if self._keywords:
-                    f = open(test.path, 'rb')
-                    t = f.read().lower() + test.bname.lower()
-                    f.close()
+                    with open(test.path, 'rb') as f:
+                        t = f.read().lower() + test.bname.lower()
                     ignored = False
                     for k in self._keywords.lower().split():
                         if k not in t:
@@ -2096,6 +2125,18 @@
                     os.environ['PYTHONHASHSEED'])
             if self._runner.options.time:
                 self.printtimes(result.times)
+
+            if self._runner.options.exceptions:
+                exceptions = aggregateexceptions(
+                    os.path.join(self._runner._outputdir, b'exceptions'))
+                total = sum(exceptions.values())
+
+                self.stream.writeln('Exceptions Report:')
+                self.stream.writeln('%d total from %d frames' %
+                                    (total, len(exceptions)))
+                for (frame, line, exc), count in exceptions.most_common():
+                    self.stream.writeln('%d\t%s: %s' % (count, frame, exc))
+
             self.stream.flush()
 
         return result
@@ -2243,6 +2284,50 @@
                              separators=(',', ': '))
         outf.writelines(("testreport =", jsonout))
 
+def sorttests(testdescs, shuffle=False):
+    """Do an in-place sort of tests."""
+    if shuffle:
+        random.shuffle(testdescs)
+        return
+
+    # keywords for slow tests
+    slow = {b'svn': 10,
+            b'cvs': 10,
+            b'hghave': 10,
+            b'largefiles-update': 10,
+            b'run-tests': 10,
+            b'corruption': 10,
+            b'race': 10,
+            b'i18n': 10,
+            b'check': 100,
+            b'gendoc': 100,
+            b'contrib-perf': 200,
+            }
+    perf = {}
+
+    def sortkey(f):
+        # run largest tests first, as they tend to take the longest
+        f = f['path']
+        try:
+            return perf[f]
+        except KeyError:
+            try:
+                val = -os.stat(f).st_size
+            except OSError as e:
+                if e.errno != errno.ENOENT:
+                    raise
+                perf[f] = -1e9  # file does not exist, tell early
+                return -1e9
+            for kw, mul in slow.items():
+                if kw in f:
+                    val *= mul
+            if f.endswith(b'.py'):
+                val /= 10.0
+            perf[f] = val / 1000.0
+            return perf[f]
+
+    testdescs.sort(key=sortkey)
+
 class TestRunner(object):
     """Holds context for executing tests.
 
@@ -2287,18 +2372,16 @@
         oldmask = os.umask(0o22)
         try:
             parser = parser or getparser()
-            options, args = parseargs(args, parser)
-            # positional arguments are paths to test files to run, so
-            # we make sure they're all bytestrings
-            args = [_bytespath(a) for a in args]
+            options = parseargs(args, parser)
+            tests = [_bytespath(a) for a in options.tests]
             if options.test_list is not None:
                 for listfile in options.test_list:
                     with open(listfile, 'rb') as f:
-                        args.extend(t for t in f.read().splitlines() if t)
+                        tests.extend(t for t in f.read().splitlines() if t)
             self.options = options
 
             self._checktools()
-            testdescs = self.findtests(args)
+            testdescs = self.findtests(tests)
             if options.profile_runner:
                 import statprof
                 statprof.start()
@@ -2312,51 +2395,22 @@
             os.umask(oldmask)
 
     def _run(self, testdescs):
-        if self.options.random:
-            random.shuffle(testdescs)
-        else:
-            # keywords for slow tests
-            slow = {b'svn': 10,
-                    b'cvs': 10,
-                    b'hghave': 10,
-                    b'largefiles-update': 10,
-                    b'run-tests': 10,
-                    b'corruption': 10,
-                    b'race': 10,
-                    b'i18n': 10,
-                    b'check': 100,
-                    b'gendoc': 100,
-                    b'contrib-perf': 200,
-                   }
-            perf = {}
-            def sortkey(f):
-                # run largest tests first, as they tend to take the longest
-                f = f['path']
-                try:
-                    return perf[f]
-                except KeyError:
-                    try:
-                        val = -os.stat(f).st_size
-                    except OSError as e:
-                        if e.errno != errno.ENOENT:
-                            raise
-                        perf[f] = -1e9 # file does not exist, tell early
-                        return -1e9
-                    for kw, mul in slow.items():
-                        if kw in f:
-                            val *= mul
-                    if f.endswith(b'.py'):
-                        val /= 10.0
-                    perf[f] = val / 1000.0
-                    return perf[f]
-            testdescs.sort(key=sortkey)
+        sorttests(testdescs, shuffle=self.options.random)
 
         self._testdir = osenvironb[b'TESTDIR'] = getattr(
             os, 'getcwdb', os.getcwd)()
+        # assume all tests in same folder for now
+        if testdescs:
+            pathname = os.path.dirname(testdescs[0]['path'])
+            if pathname:
+                osenvironb[b'TESTDIR'] = os.path.join(osenvironb[b'TESTDIR'],
+                                                      pathname)
         if self.options.outputdir:
             self._outputdir = canonpath(_bytespath(self.options.outputdir))
         else:
             self._outputdir = self._testdir
+            if testdescs and pathname:
+                self._outputdir = os.path.join(self._outputdir, pathname)
 
         if 'PYTHONHASHSEED' not in os.environ:
             # use a random python hash seed all the time
@@ -2373,11 +2427,6 @@
                 print("error: temp dir %r already exists" % tmpdir)
                 return 1
 
-                # Automatically removing tmpdir sounds convenient, but could
-                # really annoy anyone in the habit of using "--tmpdir=/tmp"
-                # or "--tmpdir=$HOME".
-                #vlog("# Removing temp dir", tmpdir)
-                #shutil.rmtree(tmpdir)
             os.makedirs(tmpdir)
         else:
             d = None
@@ -2399,12 +2448,27 @@
             self._tmpbindir = os.path.join(self._hgtmp, b'install', b'bin')
             os.makedirs(self._tmpbindir)
 
-            # This looks redundant with how Python initializes sys.path from
-            # the location of the script being executed.  Needed because the
-            # "hg" specified by --with-hg is not the only Python script
-            # executed in the test suite that needs to import 'mercurial'
-            # ... which means it's not really redundant at all.
-            self._pythondir = self._bindir
+            normbin = os.path.normpath(os.path.abspath(whg))
+            normbin = normbin.replace(os.sep.encode('ascii'), b'/')
+
+            # Other Python scripts in the test harness need to
+            # `import mercurial`. If `hg` is a Python script, we assume
+            # the Mercurial modules are relative to its path and tell the tests
+            # to load Python modules from its directory.
+            with open(whg, 'rb') as fh:
+                initial = fh.read(1024)
+
+            if re.match(b'#!.*python', initial):
+                self._pythondir = self._bindir
+            # If it looks like our in-repo Rust binary, use the source root.
+            # This is a bit hacky. But rhg is still not supported outside the
+            # source directory. So until it is, do the simple thing.
+            elif re.search(b'/rust/target/[^/]+/hg', normbin):
+                self._pythondir = os.path.dirname(self._testdir)
+            # Fall back to the legacy behavior.
+            else:
+                self._pythondir = self._bindir
+
         else:
             self._installdir = os.path.join(self._hgtmp, b"install")
             self._bindir = os.path.join(self._installdir, b"bin")
@@ -2476,6 +2540,23 @@
 
         self._coveragefile = os.path.join(self._testdir, b'.coverage')
 
+        if self.options.exceptions:
+            exceptionsdir = os.path.join(self._outputdir, b'exceptions')
+            try:
+                os.makedirs(exceptionsdir)
+            except OSError as e:
+                if e.errno != errno.EEXIST:
+                    raise
+
+            # Remove all existing exception reports.
+            for f in os.listdir(exceptionsdir):
+                os.unlink(os.path.join(exceptionsdir, f))
+
+            osenvironb[b'HGEXCEPTIONSDIR'] = exceptionsdir
+            logexceptions = os.path.join(self._testdir, b'logexceptions.py')
+            self.options.extra_config_opt.append(
+                'extensions.logexceptions=%s' % logexceptions.decode('utf-8'))
+
         vlog("# Using TESTDIR", self._testdir)
         vlog("# Using RUNTESTDIR", osenvironb[b'RUNTESTDIR'])
         vlog("# Using HGTMP", self._hgtmp)
@@ -2504,6 +2585,16 @@
             else:
                 args = os.listdir(b'.')
 
+        expanded_args = []
+        for arg in args:
+            if os.path.isdir(arg):
+                if not arg.endswith(b'/'):
+                    arg += b'/'
+                expanded_args.extend([arg + a for a in os.listdir(arg)])
+            else:
+                expanded_args.append(arg)
+        args = expanded_args
+
         tests = []
         for t in args:
             if not (os.path.basename(t).startswith(b'test-')
@@ -2637,6 +2728,7 @@
         t = testcls(refpath, self._outputdir, tmpdir,
                     keeptmpdir=self.options.keep_tmpdir,
                     debug=self.options.debug,
+                    first=self.options.first,
                     timeout=self.options.timeout,
                     startport=self._getport(count),
                     extraconfigopts=self.options.extra_config_opt,
@@ -2758,13 +2850,12 @@
                     if e.errno != errno.ENOENT:
                         raise
         else:
-            f = open(installerrs, 'rb')
-            for line in f:
-                if PYTHON3:
-                    sys.stdout.buffer.write(line)
-                else:
-                    sys.stdout.write(line)
-            f.close()
+            with open(installerrs, 'rb') as f:
+                for line in f:
+                    if PYTHON3:
+                        sys.stdout.buffer.write(line)
+                    else:
+                        sys.stdout.write(line)
             sys.exit(1)
         os.chdir(self._testdir)
 
@@ -2772,28 +2863,24 @@
 
         if self.options.py3k_warnings and not self.options.anycoverage:
             vlog("# Updating hg command to enable Py3k Warnings switch")
-            f = open(os.path.join(self._bindir, 'hg'), 'rb')
-            lines = [line.rstrip() for line in f]
-            lines[0] += ' -3'
-            f.close()
-            f = open(os.path.join(self._bindir, 'hg'), 'wb')
-            for line in lines:
-                f.write(line + '\n')
-            f.close()
+            with open(os.path.join(self._bindir, 'hg'), 'rb') as f:
+                lines = [line.rstrip() for line in f]
+                lines[0] += ' -3'
+            with open(os.path.join(self._bindir, 'hg'), 'wb') as f:
+                for line in lines:
+                    f.write(line + '\n')
 
         hgbat = os.path.join(self._bindir, b'hg.bat')
         if os.path.isfile(hgbat):
             # hg.bat expects to be put in bin/scripts while run-tests.py
             # installation layout put it in bin/ directly. Fix it
-            f = open(hgbat, 'rb')
-            data = f.read()
-            f.close()
+            with open(hgbat, 'rb') as f:
+                data = f.read()
             if b'"%~dp0..\python" "%~dp0hg" %*' in data:
                 data = data.replace(b'"%~dp0..\python" "%~dp0hg" %*',
                                     b'"%~dp0python" "%~dp0hg" %*')
-                f = open(hgbat, 'wb')
-                f.write(data)
-                f.close()
+                with open(hgbat, 'wb') as f:
+                    f.write(data)
             else:
                 print('WARNING: cannot fix hg.bat reference to python.exe')
 
@@ -2918,6 +3005,24 @@
                 print("WARNING: Did not find prerequisite tool: %s " %
                       p.decode("utf-8"))
 
+def aggregateexceptions(path):
+    exceptions = collections.Counter()
+
+    for f in os.listdir(path):
+        with open(os.path.join(path, f), 'rb') as fh:
+            data = fh.read().split(b'\0')
+            if len(data) != 4:
+                continue
+
+            exc, mainframe, hgframe, hgline = data
+            exc = exc.decode('utf-8')
+            mainframe = mainframe.decode('utf-8')
+            hgframe = hgframe.decode('utf-8')
+            hgline = hgline.decode('utf-8')
+            exceptions[(hgframe, hgline, exc)] += 1
+
+    return exceptions
+
 if __name__ == '__main__':
     runner = TestRunner()