Mercurial > gub
changeset 3455:f50975335ad8
Support testing user-checked-out trees again; 3rd option in gub-tester help.
Oops, actually remove old test-gub.
| author | Jan Nieuwenhuizen <janneke@gnu.org> |
|---|---|
| date | Wed, 09 May 2007 17:02:27 +0200 |
| parents | 07510997bd2f |
| children | 43681bb0c189 |
| files | TODO bin/gub-tester bin/test-gub gub/repository.py |
| diffstat | 4 files changed, 90 insertions(+), 458 deletions(-) [+] |
line wrap: on
line diff
--- a/TODO Wed May 09 15:40:54 2007 +0200 +++ b/TODO Wed May 09 17:02:27 2007 +0200 @@ -1,7 +1,5 @@ * Add diff option: 561096638b3718af86d61570ac63bafa5c4a99f8 -* Make gub-tester work with plain, user-checked-out repositories again? - * Softcode stages. * move Scons, Bjam recipes to gub lib.
--- a/bin/gub-tester Wed May 09 15:40:54 2007 +0200 +++ b/bin/gub-tester Wed May 09 17:02:27 2007 +0200 @@ -77,26 +77,19 @@ address = '%s@localhost' % os.getlogin () except OSError: address = 'root@localhost' - + p = optparse.OptionParser (usage='gub-tester [options] command command ... ') - + p.address = [address] + examples = '''Examples: gub-tester --repository=downloads/lilypond.git \\ gub --branch=lilypond:master lilypond gub-tester --url=http://bazaar.launchpad.net/~yaffut/yaffut/yaffut.bzr \\ - 'make all test' - -# FIXME: does not work any more -bzr branch http://bazaar.launchpad.net/~yaffut/yaffut/yaffut.bzr -cd yaffut.bzr && test-gub 'make all test' + 'make all check' -# For gub-tester to work with user-checkouts again, for now use a workaround -# like -# mkdir foo.bzr && cd foo.bzr -# bzr branch URL HEAD -# cd HEAD && mkdir log -# gub-tester --repository $(cd .. && pwd) +bzr branch http://bazaar.launchpad.net/~yaffut/yaffut/yaffut.bzr +cd yaffut.bzr && gub-tester 'make all check' ''' @@ -119,6 +112,12 @@ action='store_true', help='do not run any commands') + p.add_option ('--append-diff', + dest='append_diff', + default=False, + action='store_true', + help='append diff since last successful run') + p.add_option ('--bcc', action='append', dest='bcc_address', @@ -237,7 +236,7 @@ return ('SUCCESS', ['dryrun']) stat = os.system (cmd) - + base_tag = 'success-%(canonicalize)s-' % locals () result = 'unknown' attachments = [] @@ -246,8 +245,20 @@ result = 'FAIL' attachments = ['error for\n\n\t%s\n\n\n%s' % (target, '\n'.join (body[-0:]))] + if options.append_diff: + attachments += [repo.get_diff_from_tag (base_tag)] + else: + if 0: + # FIXME: find in archives, what did this do? + # format of last_patch? + # canonicalize_target vs canonicalize_string? + print last_patch + tag = base_tag + canonicalize_string (last_patch['date']) + repo.tag (tag) + log_file.log ('tagging with %s' % tag) + if options.tag_repo: + repo.push (tag, options.tag_repo) - else: result = 'SUCCESS' attachments = ['success for\n\n\t%s\n\n%s' % (target, @@ -261,6 +272,7 @@ def send_message (options, msg): if not options.address: log_file.info ('No recipients for result mail') + # FIXME: what about env[EMAIL]? return COMMASPACE = ', '
--- a/bin/test-gub Wed May 09 15:40:54 2007 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,394 +0,0 @@ -#!/usr/bin/python - -def argv0_relocation (): - import os, sys - bindir = os.path.dirname (sys.argv[0]) - prefix = os.path.dirname (bindir) - sys.path.insert (0, prefix) - -argv0_relocation () - -import sys -import re -import os -import smtplib -import email.MIMEText -import email.Message -import email.MIMEMultipart -import optparse -import time -import dbhash - - -from gub import repository - - -################################################################ -# utils. - -def system (c): - print c - if os.system (c): - raise Exception ('barf') - - -def read_tail (file, amount=10240): - f = open (file) - f.seek (0, 2) - length = f.tell() - f.seek (- min (length, amount), 1) - return f.read () - -def canonicalize_string (target): - canonicalize = re.sub ('[ \t\n]', '_', target) - canonicalize = re.sub ('[^a-zA-Z0-9-]+', '_', canonicalize) - return canonicalize - -class LogFile: - def __init__ (self, name): - if name: - self.file = open (name, 'a') - else: - self.file = sys.stdout - - self.prefix = 'gub-tester [%d]: ' % os.getpid () - - def log (self, msg): - self.file.write ('%s%s\n' % (self.prefix, msg)) - self.file.flush () - - def __del__ (self): - self.log (' *** finished') - -log_file = None - -################################################################ -# - -def result_message (parts, subject='') : - """Concatenate PARTS to a Message object.""" - - if not parts: - parts.append ('(empty)') - - parts = [email.MIMEText.MIMEText (p) for p in parts if p] - - msg = parts[0] - if len (parts) > 1: - msg = email.MIMEMultipart.MIMEMultipart () - for p in parts: - msg.attach (p) - - msg['Subject'] = subject - msg.epilogue = '' - - return msg - -def opt_parser (): - if os.environ.has_key ('EMAIL'): - address = os.environ['EMAIL'] - else: - try: - address = '%s@localhost' % os.getlogin () - except OSError: - address = 'root@localhost' - - p = optparse.OptionParser (usage='gub-tester [options] command command ... ') - - examples = '''Examples: -gub-tester --repository=downloads/lilypond.git \\ - gub --branch=lilypond:master lilypond - -gub-tester --url=http://bazaar.launchpad.net/~yaffut/yaffut/yaffut.bzr \\ - 'make all test' - -# FIXME: does not work any more -bzr branch http://bazaar.launchpad.net/~yaffut/yaffut/yaffut.bzr -cd yaffut.bzr && test-gub 'make all test' -''' - - def format_examples (self): - return examples - if p.__dict__.has_key ('epilog'): - p.formatter.format_epilog = format_examples - p.epilog = examples - else: - p.formatter.format_description = format_examples - p.description = examples - p.add_option ('-t', '--to', - action='append', - dest='address', - default=[], - help='where to send error report') - p.add_option ('--dry-run', - dest='dry_run', - default=False, - action='store_true', - help='do not run any commands') - - p.add_option ('--bcc', - action='append', - dest='bcc_address', - default=[], - help='BCC for error report') - - p.add_option ('-f', '--from', - action='store', - dest='sender', - default=address, - help='whom to list as sender') - - p.add_option ('--branch', - action='store', - dest='branch', - default='HEAD', - help='which branch to fetch [GIT repositories]') - - p.add_option ('--url', - action='store', - dest='url', - default=None, - help='where to fetch sources') - - p.add_option ('--update', - action='store', - dest='update', - default=False, - help='checkout or update sources') - - p.add_option ('--revision', - action='store', - dest='revision', - default=None, - help='what revision to fetch') - - p.add_option ('--repository', - action='store', - dest='repository', - default='.', - help='where to download/cache repository') - - p.add_option ('--tag-repo', - action='store', - dest='tag_repo', - default='', - help='where to push success tags.') - - p.add_option ('--quiet', - action='store_true', - dest='be_quiet', - default=False, - help='only send mail when there was an error.') - - p.add_option ('--dependent', - action='store_true', - dest='is_dependent', - default=False, - help='test targets depend on each other') - - p.add_option ('--posthook', - action='append', - dest='posthooks', - default=[], - help='commands to execute after successful tests.') - - p.add_option ('--test-self', - action='store_true', - dest='test_self', - default=False, - help='run a cursory self test.') - - p.add_option ('-s', '--smtp', - action='store', - dest='smtp', - default='localhost', - help='SMTP server to use.') - - p.add_option ('--result-directory', - action='store', - dest='result_dir', - help='Where to store databases test results', - default='log') - - return p - - -def get_db (options, name): - name = options.result_dir + '/%s.db' % name - - db_file = os.path.join (options.result_dir, name) - db = dbhash.open (db_file, 'c') - return db - - -def test_target (repo, options, target, last_patch): - canonicalize = canonicalize_string (target) - release_hash = repo.get_checksum () - - done_db = get_db (options, canonicalize) - if done_db.has_key (release_hash): - log_file.log ('release %(release_hash)s has already been checked' - % locals ()) - return None - - logfile = 'test-%(canonicalize)s.log' % locals () - logfile = os.path.join (options.result_dir, logfile) - - cmd = 'nice time %(target)s > %(logfile)s 2>&1' % locals () - - log_file.log (cmd) - - if options.dry_run: - return ('SUCCESS', ['dryrun']) - - stat = os.system (cmd) - - result = 'unknown' - attachments = [] - - body = read_tail (logfile, 10240).split ('\n') - if stat: - result = 'FAIL' - attachments = ['error for\n\n\t%s\n\n\n%s' % (target, - '\n'.join (body[-0:]))] - - else: - result = 'SUCCESS' - attachments = ['success for\n\n\t%s\n\n%s' - % (target, - '\n'.join (body[-10:]))] - - log_file.log ('%s: %s' % (target, result)) - - done_db[release_hash] = time.ctime () - return (result, attachments) - -def send_message (options, msg): - if not options.address: - log_file.log ('No recipients for result mail') - return - - COMMASPACE = ', ' - msg['From'] = options.sender - msg['To'] = COMMASPACE.join (options.address) - if options.bcc_address: - msg['BCC'] = COMMASPACE.join (options.bcc_address) - - msg['X-Autogenerated'] = 'lilypond' - connection = smtplib.SMTP (options.smtp) - connection.sendmail (options.sender, options.address, msg.as_string ()) - - -def send_result_by_mail (options, parts, subject='Autotester result'): - msg = result_message (parts, subject) - send_message (options, msg) - -def print_results (options, parts, subject='Autotester result'): - print '\n===\n\nSUBJECT: ', subject - print '\n---\n'.join (parts) - print 'END RESULT\n===' - -def real_main (options, args, handle_result): - global log_file - - log = 'log/test-gubb.log' - if options.dry_run: - log = '' - - log_file = LogFile (log) - - log_file.log (' *** %s' % time.ctime ()) - log_file.log (' *** Starting tests:\n %s' % '\n '.join (args)) - - repo = repository.get_repository_proxy (options.repository, - options.url, - options.revision, - options.branch) - - if (not repo.is_downloaded () or options.update or options.revision): - repo.download () - repo.update_workdir ('.') - - log_file.log ('Repository %s' % str (repo)) - - last_patch = repo.get_revision_description () - release_hash = repo.get_checksum () - - release_id = ''' - -Last patch of this release: - -%(last_patch)s\n - -Checksum of revision: %(release_hash)s - -''' % locals () - - - summary_body = '\n\n' - results = {} - failures = 0 - for a in args: - result_tup = test_target (repo, options, a, last_patch) - if not result_tup: - continue - (result, atts) = result_tup - results[a] = result_tup - success = result.startswith ('SUCCESS') - if not (options.be_quiet and success): - handle_result (options, atts, subject='Autotester: %s %s' - % (result, a)) - - summary_body += '%s\n %s\n' % (a, result) - - if not success: - failures += 1 - if options.is_dependent: - break - - if (results - and len (args) > 1 - and (failures > 0 or not options.be_quiet)): - - handle_result (options, - [summary_body, release_id], - subject='Autotester: summary') - - if failures == 0 and results: - for p in options.posthooks: - os.system (p) - -def test_self (options, args): - self_test_dir = 'test-gub-test.darcs' - system ('rm -rf %s ' % self_test_dir) - system ('mkdir %s ' % self_test_dir) - os.chdir (self_test_dir) - system ('mkdir log') - system (r"echo -e '#!/bin/sh\ntrue\n' > foo.sh") - system ('darcs init') - system ('echo author > _darcs/prefs/author') - system ('darcs add foo.sh') - system ('darcs record -am "add bla"') - options.repository = os.getcwd () - - real_main (options, ['false', 'true', 'sh foo.sh'], print_results) - - system (r"echo -e '#!/bin/sh\nfalse\n' > foo.sh") - system ('darcs record -am "change bla"') - real_main (options, ['sh foo.sh'], print_results) - -def main (): - (options, args) = opt_parser ().parse_args () - - if not os.path.isdir (options.result_dir): - os.makedirs (options.result_dir) - - options.result_dir = os.path.abspath (options.result_dir) - - if options.test_self: - test_self (options, args) - else: - real_main (options, args, send_result_by_mail) - -if __name__ == '__main__': - main ()
--- a/gub/repository.py Wed May 09 15:40:54 2007 +0200 +++ b/gub/repository.py Wed May 09 17:02:27 2007 +0200 @@ -34,8 +34,16 @@ def __init__ (self, dir, vcs, source): self.vcs = vcs self.dir = os.path.normpath (dir) + self.vcs + if not dir or dir == '.': - self.dir = os.path.join (os.getcwd (), self.vcs) + dir = os.getcwd () + if os.path.isdir (os.path.join (dir, self.vcs)): + # Support user-checkouts: If we're already checked-out + # HERE, use that as repository + self.dir = dir + else: + # Otherwise, check fresh repository out under .VCS + self.dir = os.path.join (os.getcwd (), self.vcs) self .source = source self.oslog = None @@ -81,6 +89,16 @@ """A human-readable revision number. It need not be unique over revisions.""" return '0' + def read_last_patch (self): + """Return a dict with info about the last patch""" + assert 0 + return {} + + def get_diff_from_tag (self, name): + """Return diff wrt to last tag that starts with NAME """ + assert 0 + return 'baseclass method called' + class Version: def __init__ (self, version): self.dir = None @@ -101,6 +119,9 @@ def version (self): return self._version + def set_oslog (self, oslog): + pass + class Darcs (Repository): def __init__ (self, dir, source=''): Repository.__init__ (self, dir, '.darcs', source) @@ -245,12 +266,17 @@ class Git (Repository): def __init__ (self, dir, source='', branch='', revision=''): Repository.__init__ (self, dir, '.git', source) + user_repo_dir = os.path.join (self.dir, self.vcs) + if os.path.isdir (user_repo_dir): + self.dir = user_repo_dir self.checksums = {} self.local_branch = '' self.remote_branch = branch self.revision = revision - self.repo_url_suffix = re.sub ('.*://', '', source) + self.repo_url_suffix = None + if source: + self.repo_url_suffix = re.sub ('.*://', '', source) if self.repo_url_suffix: # FIXME: logic copied foo times @@ -284,7 +310,6 @@ b = self.local_branch if not b: b = self.revision - return '#<GitRepository %s#%s>' % (self.dir, b) def get_revision_description (self): @@ -294,52 +319,42 @@ committish = self.git_pipe ('log --max-count=1 --pretty=oneline %(local_branch)s' % self.__dict__).split (' ')[0] m = re.search ('^tree ([0-9a-f]+)', - self.git_pipe ('cat-file commit %(committish)s' % locals ())) - + self.git_pipe ('cat-file commit %(committish)s' + % locals ())) treeish = m.group (1) - for f in self.git_pipe ('ls-tree -r %(treeish)s' % - locals ()).split ('\n'): + for f in self.git_pipe ('ls-tree -r %(treeish)s' + % locals ()).split ('\n'): (info, name) = f.split ('\t') (mode, type, fileish) = info.split (' ') - if name == file_name: return self.git_pipe ('cat-file blob %(fileish)s ' % locals ()) raise RepositoryException ('file not found') def get_branches (self): - branch_lines = self.read_pipe (self.git_command () + ' branch -l ').split ('\n') - + branch_lines = self.read_pipe (self.git_command () + + ' branch -l ').split ('\n') branches = [b[2:] for b in branch_lines] return [b for b in branches if b] def git_command (self, dir, repo_dir): if repo_dir: repo_dir = '--git-dir %s' % repo_dir - c = 'git %(repo_dir)s' % locals () if dir: c = 'cd %s && %s' % (dir, c) - return c - def git (self, cmd, dir='', ignore_errors=False, - repo_dir=''): - + def git (self, cmd, dir='', ignore_errors=False, repo_dir=''): if repo_dir == '' and dir == '': repo_dir = self.dir - gc = self.git_command (dir, repo_dir) cmd = '%(gc)s %(cmd)s' % locals () - self.system (cmd, ignore_errors=ignore_errors) - def git_pipe (self, cmd, ignore_errors=False, - dir='', repo_dir=''): - + def git_pipe (self, cmd, ignore_errors=False, dir='', repo_dir=''): if repo_dir == '' and dir == '': repo_dir = self.dir - gc = self.git_command (dir, repo_dir) return self.read_pipe ('%(gc)s %(cmd)s' % locals ()) @@ -380,6 +395,8 @@ refs = '%s:%s' % (self.remote_branch, self.branch) + # FIXME: if source == None (for user checkouts), how to ask + # git what parent url is? `git info' does not work self.git ('fetch --update-head-ok %(source)s %(refs)s ' % locals ()) self.checksums = {} @@ -607,6 +624,9 @@ % locals ()) def _checkout_dir (self): + # Support user-check-outs + if os.path.isdir (os.path.join (self.dir, self.vcs)): + return self.dir revision = self.revision dir = self.dir branch = self.branch @@ -676,6 +696,8 @@ def _update (self, revision): rev_opt = '-r %(revision)s ' % locals () source = self.source + if not source: + source = '' self.bzr_system ('pull %(rev_opt)s %(source)s' % locals ()) def bzr_pipe (self, cmd): @@ -689,27 +711,6 @@ def get_revision_description (self): return self.bzr_pipe ('log --verbose -r-1') -# FIXME: repository detection AND repositories only work if they are -# checked-out in a dir named .../name.REPOSITORY Eg, for GIT, this -# means that only the first arbitrary in .git in -# `downloads/lilypond.git/.git' is `detected'. Repositories passed to -# test-gub must have the .REPOSITORY stripped, --repository=. does not -# work. - -# This is not trivial to fix, as the DIR passed to Repository () is -# not an existing directory, it gets `.REPOSITORY' appended in the -# constructors. - -# Also, different revisions get checked-out in different directories: -#, eg: foo.svn/trunk-7111, foo.svn/trunk-HEAD, etc. - -# For gub-tester to work with user-checkouts again, for now use a workaround -# like -# mkdir foo.bzr && cd foo.bzr -# bzr branch URL HEAD -# cd HEAD && mkdir log -# gub-tester --repository $(cd .. && pwd) - def get_appended_vcs_name (name): return re.search (r"(.*)\.(bzr|git|cvs|svn|darcs|.tar(.gz|.bz2))", name) @@ -720,9 +721,19 @@ # and use that as cache def get_vcs_type_from_checkout_directory_name (dir): m = get_appended_vcs_name (dir) - dir = m.group (1) - type = m.group (2) - return dir, type + if m: + dir = m.group (1) + type = m.group (2) + return dir, type + return dir, None + +def get_vcs_type_of_dir (dir): + # FIXME: get these from all repositories... + for i in ('.bzr', '.git', 'CVS', '.svn', '_darcs'): + if os.path.isdir (os.path.join (dir, i)): + return i.replace ('.', '').replace ('_', '') + #Hmm + return 'tar.gz' def get_vcs_type_from_url (url): m = get_prepended_vcs_name (url) @@ -750,6 +761,9 @@ url, type = get_vcs_type_from_url (url) if not type: dir, type = get_vcs_type_from_checkout_directory_name (dir) + if not type: + # FIXME: todo: teach repositories that they might be + type = get_vcs_type_of_dir (dir) if type == 'bzr': return Bazaar (dir, source=url, revision=revision) @@ -761,8 +775,10 @@ return Git (dir, source=url, branch=branch, revision=revision) elif type == 'svn': return Subversion (dir, source=url, branch=branch) - elif type.startswith ('.tar.'): + elif type and type.startswith ('.tar.'): return TarBall (dir, url=url, branch=branch) - raise UnknownVcSystem ('Cannot determine vcs type: url=%(url)s, dir=%(dir)' + class UnknownVcSystem (Exception): + pass + raise UnknownVcSystem ('Cannot determine vcs type: url=%(url)s, dir=%(dir)s' % locals ())