Mercurial > gub
changeset 3451:baff8104a529
Add checkout option to gub tester. Fix bashisms. Update repositories,
refactor some initialisations to Repository. Rename test-gub to gub-tester.
author | Jan Nieuwenhuizen <janneke@gnu.org> |
---|---|
date | Wed, 09 May 2007 13:08:07 +0200 |
parents | d226617e202e |
children | bbb361d4d1d9 |
files | README TODO bin/cygwin-packager bin/gpkg bin/gub bin/gub-tester bin/installer-builder bin/test-gub gub/repository.py test-lily/cron-builder.py |
diffstat | 10 files changed, 667 insertions(+), 136 deletions(-) [+] |
line wrap: on
line diff
--- a/README Mon May 07 16:13:12 2007 +0200 +++ b/README Wed May 09 13:08:07 2007 +0200 @@ -94,5 +94,6 @@ bin/gpkg - package manager bin/gub - build packages bin/installer-builder - build (platform dependent) installers - bin/test-gub - build tester; integrates with Darcs/CVS + bin/gub-tester - build tester; integrates with + Bazaar,CVS,Darcs,Git,SVN and gub bin/cygwin-packager - roll cygwin package
--- a/TODO Mon May 07 16:13:12 2007 +0200 +++ b/TODO Wed May 09 13:08:07 2007 +0200 @@ -1,13 +1,4 @@ -* Make test-gub work with plain, user-checked-out repositories again - (and/or remove lilypond hardcoding from cron-builder, move checkout - command from cron-builder to separate app (or into test-gub)). - Add option to send diffs again. Use case - - bzr branch http://foo/bar.bzr && test-gub bar - - [or alternatively, something like - - gub --stage=download http://foo/bar.bzr && test-gub bar] +* Make gub-tester work with plain, user-checked-out repositories again? * Softcode stages. @@ -31,10 +22,13 @@ LOWER PRIORITY -- replace os. calls with os.context wrapper ones +- replace os. calls with os.context wrapper ones (make a real + dry-run to quick-test all .py scripts?) - better name for gubb.py +- name for gub-tester (test-repo, repo-builder?) + - Split gub/*py into packager, builder, platform - Explode inheritance, and use membership in GUP
--- a/bin/cygwin-packager Mon May 07 16:13:12 2007 +0200 +++ b/bin/cygwin-packager Wed May 09 13:08:07 2007 +0200 @@ -22,10 +22,16 @@ """ +def argv0_relocation (): + import os, sys + bindir = os.path.dirname (sys.argv[0]) + prefix = os.path.dirname (bindir) + sys.path.insert (0, prefix) + +argv0_relocation () + import optparse import os -import sys -sys.path.insert (0, '.') # from gub import context from gub import gup @@ -68,7 +74,7 @@ self.create () def build_number (self, package): - import versiondb + from gub import versiondb version_file = '%(uploads)s/%(name)s.versions' % package db = versiondb.VersionDataBase (version_file) version = '%(full_version)s' % package @@ -178,7 +184,7 @@ return readme def strip_dir (self, dir): - import misc + from gub import misc misc.map_command_dir (self.expand (dir), self.expand ('%(strip_command)s'), self.no_binary_strip, @@ -320,13 +326,14 @@ (options, args) = p.parse_args () if len (args) != 1: p.print_help () + import sys sys.exit (2) return (options, args) def main (): (options, commands) = parse_command_line () options.platform = 'cygwin' - settings = gub.settings.Settins (options) + settings = gub.settings.Settings (options) # We want to be able to specify a build number for cygwin packages # that are not distributed on lp.org settings.build = options.build
--- a/bin/gpkg Mon May 07 16:13:12 2007 +0200 +++ b/bin/gpkg Wed May 09 13:08:07 2007 +0200 @@ -23,11 +23,18 @@ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. """ +def argv0_relocation (): + import os, sys + bindir = os.path.dirname (sys.argv[0]) + prefix = os.path.dirname (bindir) + sys.path.insert (0, prefix) + +argv0_relocation () + import optparse import re import string import sys -sys.path.insert (0, '.') # from gub import gup from gub import oslog
--- a/bin/gub Mon May 07 16:13:12 2007 +0200 +++ b/bin/gub Wed May 09 13:08:07 2007 +0200 @@ -20,8 +20,15 @@ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. """ +def argv0_relocation (): + import os, sys + bindir = os.path.dirname (sys.argv[0]) + prefix = os.path.dirname (bindir) + sys.path.insert (0, prefix) + +argv0_relocation () + import sys -sys.path.insert (0, '.') # from gub import cross ## fixme: double use of gub name.
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/bin/gub-tester Wed May 09 13:08:07 2007 +0200 @@ -0,0 +1,403 @@ +#!/usr/bin/python + +def argv0_relocation (): + import os, sys + bindir = os.path.dirname (sys.argv[0]) + prefix = os.path.dirname (bindir) + sys.path.insert (0, prefix) + +argv0_relocation () + +import sys +import re +import os +import smtplib +import email.MIMEText +import email.Message +import email.MIMEMultipart +import optparse +import time +import dbhash + + +from gub import repository + + +################################################################ +# utils. + +def system (c): + print c + if os.system (c): + raise Exception ('barf') + + +def read_tail (file, amount=10240): + f = open (file) + f.seek (0, 2) + length = f.tell() + f.seek (- min (length, amount), 1) + return f.read () + +def canonicalize_string (target): + canonicalize = re.sub ('[ \t\n]', '_', target) + canonicalize = re.sub ('[^a-zA-Z0-9-]+', '_', canonicalize) + return canonicalize + +class LogFile: + def __init__ (self, name): + if name: + self.file = open (name, 'a') + else: + self.file = sys.stdout + + self.prefix = 'gub-tester [%d]: ' % os.getpid () + + def log (self, msg): + self.file.write ('%s%s\n' % (self.prefix, msg)) + self.file.flush () + + def __del__ (self): + self.log (' *** finished') + +log_file = None + +################################################################ +# + +def result_message (parts, subject='') : + """Concatenate PARTS to a Message object.""" + + if not parts: + parts.append ('(empty)') + + parts = [email.MIMEText.MIMEText (p) for p in parts if p] + + msg = parts[0] + if len (parts) > 1: + msg = email.MIMEMultipart.MIMEMultipart () + for p in parts: + msg.attach (p) + + msg['Subject'] = subject + msg.epilogue = '' + + return msg + +def opt_parser (): + if os.environ.has_key ('EMAIL'): + address = os.environ['EMAIL'] + else: + try: + address = '%s@localhost' % os.getlogin () + except OSError: + address = 'root@localhost' + + p = optparse.OptionParser (usage='gub-tester [options] command command ... ') + + examples = '''Examples: +gub-tester --repository=downloads/lilypond.git \\ + gub --branch=lilypond:master lilypond + +gub-tester --url=http://bazaar.launchpad.net/~yaffut/yaffut/yaffut.bzr \\ + 'make all test' + +# FIXME: does not work any more +bzr branch http://bazaar.launchpad.net/~yaffut/yaffut/yaffut.bzr +cd yaffut.bzr && test-gub 'make all test' + +# For gub-tester to work with user-checkouts again, for now use a workaround +# like +# mkdir foo.bzr && cd foo.bzr +# bzr branch URL HEAD +# cd HEAD && mkdir log +# gub-tester --repository $(cd .. && pwd) + +''' + + def format_examples (self): + return examples + if p.__dict__.has_key ('epilog'): + p.formatter.format_epilog = format_examples + p.epilog = examples + else: + p.formatter.format_description = format_examples + p.description = examples + p.add_option ('-t', '--to', + action='append', + dest='address', + default=[], + help='where to send error report') + p.add_option ('--dry-run', + dest='dry_run', + default=False, + action='store_true', + help='do not run any commands') + + p.add_option ('--bcc', + action='append', + dest='bcc_address', + default=[], + help='BCC for error report') + + p.add_option ('-f', '--from', + action='store', + dest='sender', + default=address, + help='whom to list as sender') + + p.add_option ('--branch', + action='store', + dest='branch', + default='HEAD', + help='which branch to fetch [GIT repositories]') + + p.add_option ('--url', + action='store', + dest='url', + default=None, + help='where to fetch sources') + + p.add_option ('--update', + action='store', + dest='update', + default=False, + help='checkout or update sources') + + p.add_option ('--revision', + action='store', + dest='revision', + default=None, + help='what revision to fetch') + + p.add_option ('--repository', + action='store', + dest='repository', + default='.', + help='where to download/cache repository') + + p.add_option ('--tag-repo', + action='store', + dest='tag_repo', + default='', + help='where to push success tags.') + + p.add_option ('--quiet', + action='store_true', + dest='be_quiet', + default=False, + help='only send mail when there was an error.') + + p.add_option ('--dependent', + action='store_true', + dest='is_dependent', + default=False, + help='test targets depend on each other') + + p.add_option ('--posthook', + action='append', + dest='posthooks', + default=[], + help='commands to execute after successful tests.') + + p.add_option ('--test-self', + action='store_true', + dest='test_self', + default=False, + help='run a cursory self test.') + + p.add_option ('-s', '--smtp', + action='store', + dest='smtp', + default='localhost', + help='SMTP server to use.') + + p.add_option ('--result-directory', + action='store', + dest='result_dir', + help='Where to store databases test results', + default='log') + + return p + + +def get_db (options, name): + name = options.result_dir + '/%s.db' % name + + db_file = os.path.join (options.result_dir, name) + db = dbhash.open (db_file, 'c') + return db + + +def test_target (repo, options, target, last_patch): + canonicalize = canonicalize_string (target) + release_hash = repo.get_checksum () + + done_db = get_db (options, canonicalize) + if done_db.has_key (release_hash): + log_file.log ('release %(release_hash)s has already been checked' + % locals ()) + return None + + logfile = 'test-%(canonicalize)s.log' % locals () + logfile = os.path.join (options.result_dir, logfile) + + cmd = 'nice time %(target)s > %(logfile)s 2>&1' % locals () + + log_file.log (cmd) + + if options.dry_run: + return ('SUCCESS', ['dryrun']) + + stat = os.system (cmd) + + result = 'unknown' + attachments = [] + + body = read_tail (logfile, 10240).split ('\n') + if stat: + result = 'FAIL' + attachments = ['error for\n\n\t%s\n\n\n%s' % (target, + '\n'.join (body[-0:]))] + + else: + result = 'SUCCESS' + attachments = ['success for\n\n\t%s\n\n%s' + % (target, + '\n'.join (body[-10:]))] + + log_file.log ('%s: %s' % (target, result)) + + done_db[release_hash] = time.ctime () + return (result, attachments) + +def send_message (options, msg): + if not options.address: + log_file.log ('No recipients for result mail') + return + + COMMASPACE = ', ' + msg['From'] = options.sender + msg['To'] = COMMASPACE.join (options.address) + if options.bcc_address: + msg['BCC'] = COMMASPACE.join (options.bcc_address) + + msg['X-Autogenerated'] = 'lilypond' + connection = smtplib.SMTP (options.smtp) + connection.sendmail (options.sender, options.address, msg.as_string ()) + + +def send_result_by_mail (options, parts, subject='Autotester result'): + msg = result_message (parts, subject) + send_message (options, msg) + +def print_results (options, parts, subject='Autotester result'): + print '\n===\n\nSUBJECT: ', subject + print '\n---\n'.join (parts) + print 'END RESULT\n===' + +def real_main (options, args, handle_result): + global log_file + + log = 'log/test-gubb.log' + if options.dry_run: + log = '' + + log_file = LogFile (log) + + log_file.log (' *** %s' % time.ctime ()) + log_file.log (' *** Starting tests:\n %s' % '\n '.join (args)) + + repo = repository.get_repository_proxy (options.repository, + options.url, + options.revision, + options.branch) + + if (not repo.is_downloaded () or options.update or options.revision): + repo.download () + repo.update_workdir ('.') + + log_file.log ('Repository %s' % str (repo)) + + last_patch = repo.get_revision_description () + release_hash = repo.get_checksum () + + release_id = ''' + +Last patch of this release: + +%(last_patch)s\n + +Checksum of revision: %(release_hash)s + +''' % locals () + + + summary_body = '\n\n' + results = {} + failures = 0 + for a in args: + result_tup = test_target (repo, options, a, last_patch) + if not result_tup: + continue + (result, atts) = result_tup + results[a] = result_tup + success = result.startswith ('SUCCESS') + if not (options.be_quiet and success): + handle_result (options, atts, subject='Autotester: %s %s' + % (result, a)) + + summary_body += '%s\n %s\n' % (a, result) + + if not success: + failures += 1 + if options.is_dependent: + break + + if (results + and len (args) > 1 + and (failures > 0 or not options.be_quiet)): + + handle_result (options, + [summary_body, release_id], + subject='Autotester: summary') + + if failures == 0 and results: + for p in options.posthooks: + os.system (p) + +def test_self (options, args): + self_test_dir = 'test-gub-test.darcs' + system ('rm -rf %s ' % self_test_dir) + system ('mkdir %s ' % self_test_dir) + os.chdir (self_test_dir) + system ('mkdir log') + system ('''echo '#!/bin/sh +true' > foo.sh''') + system ('darcs init') + system ('echo author > _darcs/prefs/author') + system ('darcs add foo.sh') + system ('darcs record -am "add bla"') + options.repository = os.getcwd () + real_main (options, ['false', 'true', 'sh foo.sh'], print_results) + + system ('''echo '#!/bin/sh +true' > foo.sh''') + system ('darcs record -am "change bla"') + real_main (options, ['sh foo.sh'], print_results) + +def main (): + (options, args) = opt_parser ().parse_args () + + if not os.path.isdir (options.result_dir): + os.makedirs (options.result_dir) + + options.result_dir = os.path.abspath (options.result_dir) + + if options.test_self: + test_self (options, args) + else: + real_main (options, args, send_result_by_mail) + +if __name__ == '__main__': + main ()
--- a/bin/installer-builder Mon May 07 16:13:12 2007 +0200 +++ b/bin/installer-builder Wed May 09 13:08:07 2007 +0200 @@ -20,12 +20,18 @@ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. """ +def argv0_relocation (): + import os, sys + bindir = os.path.dirname (sys.argv[0]) + prefix = os.path.dirname (bindir) + sys.path.insert (0, prefix) + +argv0_relocation () import optparse import os import sys import pickle -sys.path.insert (0, '.') # from gub import gup from gub import installer
--- a/bin/test-gub Mon May 07 16:13:12 2007 +0200 +++ b/bin/test-gub Wed May 09 13:08:07 2007 +0200 @@ -1,5 +1,13 @@ #!/usr/bin/python +def argv0_relocation (): + import os, sys + bindir = os.path.dirname (sys.argv[0]) + prefix = os.path.dirname (bindir) + sys.path.insert (0, prefix) + +argv0_relocation () + import sys import re import os @@ -11,7 +19,7 @@ import time import dbhash -sys.path.insert (0, '.') + from gub import repository @@ -43,7 +51,7 @@ else: self.file = sys.stdout - self.prefix = 'test-gub[%d]: ' % os.getpid () + self.prefix = 'gub-tester [%d]: ' % os.getpid () def log (self, msg): self.file.write ('%s%s\n' % (self.prefix, msg)) @@ -85,7 +93,28 @@ except OSError: address = 'root@localhost' - p = optparse.OptionParser (usage="test-gub [options] command command ... ") + p = optparse.OptionParser (usage='gub-tester [options] command command ... ') + + examples = '''Examples: +gub-tester --repository=downloads/lilypond.git \\ + gub --branch=lilypond:master lilypond + +gub-tester --url=http://bazaar.launchpad.net/~yaffut/yaffut/yaffut.bzr \\ + 'make all test' + +# FIXME: does not work any more +bzr branch http://bazaar.launchpad.net/~yaffut/yaffut/yaffut.bzr +cd yaffut.bzr && test-gub 'make all test' +''' + + def format_examples (self): + return examples + if p.__dict__.has_key ('epilog'): + p.formatter.format_epilog = format_examples + p.epilog = examples + else: + p.formatter.format_description = format_examples + p.description = examples p.add_option ('-t', '--to', action='append', dest='address', @@ -94,8 +123,8 @@ p.add_option ('--dry-run', dest='dry_run', default=False, - action="store_true", - help="don't run any commands") + action='store_true', + help='do not run any commands') p.add_option ('--bcc', action='append', @@ -110,16 +139,34 @@ help='whom to list as sender') p.add_option ('--branch', - action="store", - dest="branch", - default="HEAD", - help="which branch to test") + action='store', + dest='branch', + default='HEAD', + help='which branch to fetch [GIT repositories]') + + p.add_option ('--url', + action='store', + dest='url', + default=None, + help='where to fetch sources') + + p.add_option ('--update', + action='store', + dest='update', + default=False, + help='checkout or update sources') + + p.add_option ('--revision', + action='store', + dest='revision', + default=None, + help='what revision to fetch') p.add_option ('--repository', - action="store", - dest="repository", - default=".", - help="repository directory") + action='store', + dest='repository', + default='.', + help='where to download/cache repository') p.add_option ('--tag-repo', action='store', @@ -134,10 +181,10 @@ help='only send mail when there was an error.') p.add_option ('--dependent', - action="store_true", - dest="is_dependent", + action='store_true', + dest='is_dependent', default=False, - help="test targets depend on each other") + help='test targets depend on each other') p.add_option ('--posthook', action='append', @@ -158,10 +205,10 @@ help='SMTP server to use.') p.add_option ('--result-directory', - action="store", + action='store', dest='result_dir', - help="Where to store databases test results", - default="log") + help='Where to store databases test results', + default='log') return p @@ -180,13 +227,14 @@ done_db = get_db (options, canonicalize) if done_db.has_key (release_hash): - log_file.log ('release %(release_hash)s has already been checked' % locals ()) + log_file.log ('release %(release_hash)s has already been checked' + % locals ()) return None logfile = 'test-%(canonicalize)s.log' % locals () logfile = os.path.join (options.result_dir, logfile) - cmd = "nice time %(target)s > %(logfile)s 2>&1" % locals () + cmd = 'nice time %(target)s > %(logfile)s 2>&1' % locals () log_file.log (cmd) @@ -205,7 +253,7 @@ '\n'.join (body[-0:]))] else: - result = "SUCCESS" + result = 'SUCCESS' attachments = ['success for\n\n\t%s\n\n%s' % (target, '\n'.join (body[-10:]))] @@ -231,11 +279,11 @@ connection.sendmail (options.sender, options.address, msg.as_string ()) -def send_result_by_mail (options, parts, subject="Autotester result"): +def send_result_by_mail (options, parts, subject='Autotester result'): msg = result_message (parts, subject) send_message (options, msg) -def print_results (options, parts, subject="Autotester result"): +def print_results (options, parts, subject='Autotester result'): print '\n===\n\nSUBJECT: ', subject print '\n---\n'.join (parts) print 'END RESULT\n===' @@ -252,8 +300,16 @@ log_file.log (' *** %s' % time.ctime ()) log_file.log (' *** Starting tests:\n %s' % '\n '.join (args)) - repo = repository.get_repository_proxy (options.repository, options.branch) - log_file.log ("Repository %s" % str (repo)) + repo = repository.get_repository_proxy (options.repository, + options.url, + options.revision, + options.branch) + + if (not repo.is_downloaded () or options.update or options.revision): + repo.download () + repo.update_workdir ('.') + + log_file.log ('Repository %s' % str (repo)) last_patch = repo.get_revision_description () release_hash = repo.get_checksum () @@ -276,18 +332,15 @@ result_tup = test_target (repo, options, a, last_patch) if not result_tup: continue - (result, atts) = result_tup - results[a] = result_tup - success = result.startswith ('SUCCESS') if not (options.be_quiet and success): - handle_result (options, atts, subject="Autotester: %s %s" % (result, a)) + handle_result (options, atts, subject='Autotester: %s %s' + % (result, a)) summary_body += '%s\n %s\n' % (a, result) - if not success: failures += 1 if options.is_dependent: @@ -298,7 +351,8 @@ and (failures > 0 or not options.be_quiet)): handle_result (options, - [summary_body, release_id], subject="Autotester: summary") + [summary_body, release_id], + subject='Autotester: summary') if failures == 0 and results: for p in options.posthooks: @@ -335,9 +389,6 @@ test_self (options, args) else: real_main (options, args, send_result_by_mail) - - if __name__ == '__main__': -# test () main ()
--- a/gub/repository.py Mon May 07 16:13:12 2007 +0200 +++ b/gub/repository.py Wed May 09 13:08:07 2007 +0200 @@ -1,4 +1,3 @@ - """ Copyright (c) 2005--2007 Jan Nieuwenhuizen <janneke@gnu.org> @@ -32,12 +31,18 @@ ## Rename to Source/source.py? class Repository: - def __init__ (self): - # Fallback, this will go through oslog + def __init__ (self, dir, vcs, source): + self.vcs = vcs + self.dir = os.path.normpath (dir) + self.vcs + if not dir or dir == '.': + self.dir = os.path.join (os.getcwd (), self.vcs) + self .source = source + + # Fallbacks, this will go through oslog self.system = misc.system self.read_pipe = misc.read_pipe self.download_url = misc.download_url - + def download (self): pass @@ -52,7 +57,10 @@ def is_tracking (self): "Whether download will fetch newer versions if available" - + return False + + def is_downloaded (self): + "Whether repository is available" return False def update_workdir (self, destdir): @@ -67,6 +75,7 @@ class Version: def __init__ (self, version): + self.dir = None self._version = version def download (self): @@ -86,12 +95,9 @@ class Darcs (Repository): def __init__ (self, dir, source=''): - Repository.__init__ (self) - self.dir = dir + '.darcs' - self.source = source + Repository.__init__ (self, dir, '.darcs', source) def darcs_pipe (self, cmd): - dir = self.dir return self.read_pipe ('cd %(dir)s && darcs %(cmd)s' % locals ()) @@ -102,13 +108,15 @@ def get_revision_description (self): return self.darcs_pipe ('changes --last=1') + def is_downloaded (self): + return os.path.isdir (self.dir + '/_darcs') + def download (self): - dir = self.dir source = self.source - - if os.path.exists (dir + '/_darcs'): + if not self.is_downloaded (): self.darcs ('pull -a %(source)s' % locals ()) else: + dir = self.dir self.system ('darcs get %(source)s %(dir)s' % locals ()) def is_tracking (self): @@ -148,14 +156,19 @@ class TarBall (Repository): - def __init__ (self, dir, url, version, strip_components=1): - Repository.__init__ (self) - if not os.path.isdir (dir): - self.system ('mkdir -p %s' % dir) + # TODO: s/url/source + def __init__ (self, dir, url, version=None, strip_components=1): + Repository.__init__ (self, dir, '.tar', url) self.dir = dir - self.url = url + if not os.path.isdir (self.dir): + self.system ('mkdir -p %s' % self.dir) + self._version = version + if not version: + x, v = misc.split_ball (url) + self._version = '.'.join (v) + self.branch = None self.strip_components = strip_components @@ -163,16 +176,16 @@ return False def _file_name (self): - return re.search ('.*/([^/]+)$', self.url).group (1) + return re.search ('.*/([^/]+)$', self.source).group (1) - def _is_downloaded (self): + def is_downloaded (self): name = os.path.join (self.dir, self._file_name ()) return os.path.exists (name) def download (self): - if self._is_downloaded (): + if self.is_downloaded (): return - self.download_url (self.url, self.dir) + self.download_url (self.source, self.dir) def get_checksum (self): from gub import misc @@ -219,14 +232,11 @@ pass class Git (Repository): - def __init__ (self, git_dir, source='', branch='', revision=''): - Repository.__init__ (self) - - self.repo_dir = os.path.normpath (git_dir) + '.git' + def __init__ (self, dir, source='', branch='', revision=''): + Repository.__init__ (self, dir, '.git', source) self.checksums = {} self.local_branch = '' self.remote_branch = branch - self.source = source self.revision = revision self.repo_url_suffix = re.sub ('.*://', '', source) @@ -264,7 +274,7 @@ if not b: b = self.revision - return '#<GitRepository %s#%s>' % (self.repo_dir, b) + return '#<GitRepository %s#%s>' % (self.dir, b) def get_revision_description (self): return self.git_pipe ('log --max-count=1 %s' % self.local_branch) @@ -306,7 +316,7 @@ repo_dir=''): if repo_dir == '' and dir == '': - repo_dir = self.repo_dir + repo_dir = self.dir gc = self.git_command (dir, repo_dir) cmd = '%(gc)s %(cmd)s' % locals () @@ -317,17 +327,20 @@ dir='', repo_dir=''): if repo_dir == '' and dir == '': - repo_dir = self.repo_dir + repo_dir = self.dir gc = self.git_command (dir, repo_dir) return self.read_pipe ('%(gc)s %(cmd)s' % locals ()) + def is_downloaded (self): + return os.path.isdir (self.dir) + def download (self): - repo = self.repo_dir + repo = self.dir source = self.source revision = self.revision - if not os.path.isdir (self.repo_dir): + if not self.is_downloaded (): self.git ('--git-dir %(repo)s clone --bare -n %(source)s %(repo)s' % locals ()) for (root, dirs, files) in os.walk ('%(repo)s/refs/heads/' % locals ()): @@ -367,7 +380,7 @@ if self.checksums.has_key (branch): return self.checksums[branch] - repo_dir = self.repo_dir + repo_dir = self.dir if os.path.isdir (repo_dir): ## can't use describe: fails in absence of tags. cs = self.git_pipe ('rev-list --max-count=1 %(branch)s' % locals ()) @@ -384,7 +397,7 @@ def update_workdir (self, destdir): - repo_dir = self.repo_dir + repo_dir = self.dir branch = self.local_branch revision = self.revision @@ -402,24 +415,22 @@ open ('%(destdir)s/.git/refs/heads/%(branch)s' % locals (), 'w').write (revision) self.git ('checkout %(branch)s' % locals (), dir=destdir) -class CVS(Repository): +class CVS (Repository): cvs_entries_line = re.compile ("^/([^/]*)/([^/]*)/([^/]*)/([^/]*)/") #tag_dateformat = '%Y/%m/%d %H:%M:%S' - def __init__ (self, dir, - source='', module='', tag='HEAD'): - Repository.__init__ (self) - self.repo_dir = os.path.normpath (dir) + '.cvs' + def __init__ (self, dir, source='', module='', tag='HEAD'): + Repository.__init__ (self, dir, '.cvs', source) self.module = module self.checksums = {} self.source = source self.tag = tag self.branch = tag # for vc_version_suffix - if not os.path.isdir (self.repo_dir): - self.system ('mkdir -p %s' % self.repo_dir) + if not os.path.isdir (self.dir): + self.system ('mkdir -p %s' % self.dir) def _checkout_dir (self): - return '%s/%s' % (self.repo_dir, self.tag) + return '%s/%s' % (self.dir, self.tag) def is_tracking (self): return True ##FIXME @@ -433,7 +444,7 @@ changelog_rev = '' - for (name, version, date, dontknow) in self.cvs_entries (self.repo_dir + '/CVS'): + for (name, version, date, dontknow) in self.cvs_entries (self.dir + '/CVS'): if name == 'ChangeLog': changelog_rev = version break @@ -448,7 +459,7 @@ if self.checksums.has_key (self.tag): return self.checksums[self.tag] - file = '%s/%s/.vc-checksum' % (self.repo_dir, self.tag) + file = '%s/%s/.vc-checksum' % (self.dir, self.tag) if os.path.exists (file): cs = open (file).read () @@ -485,6 +496,10 @@ ## TODO: can we get deletes from vc? self.system ('rsync -av --delete --exclude CVS %(dir)s/ %(destdir)s' % locals ()) + def is_downloaded (self): + dir = self._checkout_dir () + return os.path.isdir (dir + '/CVS') + def download (self): suffix = self.tag rev_opt = '-r ' + self.tag @@ -493,10 +508,10 @@ lock_dir = locker.Locker (dir + '.lock') module = self.module cmd = '' - if os.path.isdir (dir + '/CVS'): + if not self.is_downloaded (): cmd += 'cd %(dir)s && cvs -q up -dCAP %(rev_opt)s' % locals() else: - repo_dir = self.repo_dir + repo_dir = self.dir cmd += 'cd %(repo_dir)s/ && cvs -d %(source)s -q co -d %(suffix)s %(rev_opt)s %(module)s''' % locals () self.system (cmd) @@ -528,28 +543,23 @@ ds = self.cvs_dirs (dir) es = [] for d in ds: - ## strip CVS/ basedir = os.path.split (d)[0] for e in self.cvs_entries (d): file_name = os.path.join (basedir, e[0]) - file_name = file_name.replace (self.repo_dir + '/', '') + file_name = file_name.replace (self.dir + '/', '') es.append ((file_name,) + e[1:]) - - return es def all_files (self, branch): - entries = self.all_cvs_entries (self.repo_dir + '/' + branch) + entries = self.all_cvs_entries (self.dir + '/' + branch) return [e[0] for e in entries] # FIXME: why are cvs, darcs, git so complicated? class SimpleRepo (Repository): - def __init__ (self, dir, repository, source, branch, revision='HEAD'): - Repository.__init__ (self) - self.repository = repository - self.dir = os.path.normpath (dir) + self.repository + def __init__ (self, dir, vcs, source, branch, revision='HEAD'): + Repository.__init__ (self, dir, vcs, source) self.source = source self.revision = revision self.branch = branch @@ -558,21 +568,24 @@ def is_tracking (self): ## FIXME, probably wrong. - return self.revision == 'HEAD' + return (not self.revision or self.revision == 'HEAD') def update_workdir (self, destdir): dir = self._checkout_dir () self._copy_working_dir (dir, destdir) - def download (self): + def is_downloaded (self): dir = self._checkout_dir () - if not os.path.isdir (dir + '/' + self.repository): + return os.path.isdir (os.path.join (dir, self.vcs)) + + def download (self): + if not self.is_downloaded (): self._checkout () if self._current_revision () != self.revision: self._update (self.revision) def _copy_working_dir (self, dir, copy): - repository = self.repository + repository = self.vcs self.system ('rsync -av --exclude %(repository)s %(dir)s/ %(copy)s' % locals ()) @@ -595,6 +608,8 @@ class Subversion (SimpleRepo): def __init__ (self, dir, source, branch, module, revision='HEAD'): + if not revision: + revision = 'HEAD' SimpleRepo.__init__ (self, dir, '.svn', source, branch, revision) self.module = module @@ -624,6 +639,8 @@ class Bazaar (SimpleRepo): def __init__ (self, dir, source, revision='HEAD'): # FIXME: multi-branch repos not supported for now + if not revision: + revision = '0' SimpleRepo.__init__ (self, dir, '.bzr', source, '', revision) def _current_revision (self): @@ -632,15 +649,17 @@ return revno[:-1] def _checkout (self): + dir = self.dir source = self.source revision = self.revision rev_opt = '-r %(revision)s ' % locals () - self.system ('''branch %(rev_opt)s %(source)s %(revision)s''' - % locals ()) + self.system ('''cd %(dir)s && bzr branch %(rev_opt)s %(source)s %(revision)s''' + % locals ()) def _update (self, revision): rev_opt = '-r %(revision)s ' % locals () - self.bzr_system ('pull %(rev_opt)s' % locals ()) + source = self.source + self.bzr_system ('pull %(rev_opt)s %(source)s' % locals ()) def bzr_pipe (self, cmd): dir = self._checkout_dir () @@ -667,31 +686,66 @@ # Also, different revisions get checked-out in different directories: #, eg: foo.svn/trunk-7111, foo.svn/trunk-HEAD, etc. -# For test-gub to work outside gub again, for now use a workaround +# For gub-tester to work with user-checkouts again, for now use a workaround # like # mkdir foo.bzr && cd foo.bzr # bzr branch URL HEAD # cd HEAD && mkdir log -# test-gub --repository $(cd .. && pwd) -def get_repository_proxy (dir, branch): - m = re.search (r"(.*)\.(bzr|git|cvs|svn|darcs)", dir) +# gub-tester --repository $(cd .. && pwd) + +def get_appended_vcs_name (name): + return re.search (r"(.*)\.(bzr|git|cvs|svn|darcs|.tar(.gz|.bz2))", name) - print 'dir:', dir - +def get_prepended_vcs_name (name): + return re.search (r"(bzr|git|cvs|svn|darcs):", name) + +# FIXME: removeme, allow for user to checkout sources in any directory +# and use that as cache +def get_vcs_type_from_checkout_directory_name (dir): + m = get_appended_vcs_name (dir) dir = m.group (1) type = m.group (2) + return dir, type + +def get_vcs_type_from_url (url): + m = get_prepended_vcs_name (url) + if m: + type = m.group (1) + url = m.group (2) + return url, type + p = url.find ('//:') + if p > 0: + protocol = url[:p] + type = {'bzr+ssh': 'bzr', + 'svn+ssh': 'svn', + }.get (protocol, None) + if type: + return url, type + m = get_appended_vcs_name (url) + if m: + type = m.group (2) + return url, type + return url, None + +def get_repository_proxy (dir, url, revision, branch): + type = None + if url: + url, type = get_vcs_type_from_url (url) + if not type: + dir, type = get_vcs_type_from_checkout_directory_name (dir) if type == 'bzr': - return Bazaar (dir, source='unknown') + return Bazaar (dir, source=url, revision=revision) elif type == 'cvs': - return CVS (dir, branch=branch) + return CVS (dir, source=url, tag=branch) elif type == 'darcs': - return Darcs (dir) + return Darcs (dir, source=url) elif type == 'git': - return Git (dir, branch=branch) + return Git (dir, source=url, branch=branch, revision=revision) elif type == 'svn': - return Subversion (dir, branch=branch) - else: - raise UnknownVcSystem('repo format unknown: ' + dir) - - return Repository('', '') + return Subversion (dir, source=url, branch=branch) + elif type.startswith ('.tar.'): + return TarBall (dir, url=url, branch=branch) + + raise UnknownVcSystem ('Cannot determine vcs type: url=%(url)s, dir=%(dir)' + % locals ())
--- a/test-lily/cron-builder.py Mon May 07 16:13:12 2007 +0200 +++ b/test-lily/cron-builder.py Wed May 09 13:08:07 2007 +0200 @@ -68,7 +68,7 @@ action="store_true", dest="build_docs", default=None, - help="build docs. Implies --dependent for test-gub") + help="build docs. Implies --dependent for gub-tester") p.add_option ('--package', action="store_true", @@ -92,7 +92,7 @@ action='store', dest='test_options', default="", - help='what to pass to test-gub') + help='what to pass to gub-tester') p.add_option ('--unversioned', action="store_true", @@ -149,7 +149,8 @@ make_cmd = 'make %s ' % opts.make_options python_cmd = sys.executable + ' ' - ## can't have these in test-gub, since these + # FIXME: use gub-tester's download facility + ## can't have these in gub-tester, since these ## will always usually result in "release already tested" for a in args: system (python_cmd + 'bin/gub --branch %s:%s -p %s --stage=download lilypond' @@ -158,7 +159,7 @@ test_cmds = [] if opts.build_package: - test_cmds += [python_cmd + 'bin/gub --branch %s:%s -lp %s build lilypond ' + test_cmds += [python_cmd + 'bin/gub --branch %s:%s -lp %s lilypond ' % (opts.branch, opts.local_branch, p) for p in args] if opts.build_installer: @@ -176,7 +177,7 @@ if opts.build_tarball: test_cmds += [make_cmd + " dist-check"] - system (python_cmd + 'bin/test-gub %s %s ' + system (python_cmd + 'bin/gub-tester %s %s ' % (opts.test_options, ' '.join (["'%s'" % c for c in test_cmds]))) if __name__ == '__main__':