# HG changeset patch # User Han-Wen Nienhuys # Date 1178765505 10800 # Node ID a726a656f0b88836f30137af241d513cd8e9dfca # Parent bec99ae5bac5c608e5930c0a5ec1efe621b5a829# Parent 43681bb0c18984de07ddc7983df2a4c9bed042c0 Merge branch 'gub' of ssh+git://hanwen@git.sv.gnu.org/srv/git/lilypond Conflicts: bin/cygwin-packager diff -r bec99ae5bac5 -r a726a656f0b8 README --- a/README Wed May 09 23:46:52 2007 -0300 +++ b/README Wed May 09 23:51:45 2007 -0300 @@ -94,5 +94,6 @@ bin/gpkg - package manager bin/gub - build packages bin/installer-builder - build (platform dependent) installers - bin/test-gub - build tester; integrates with Darcs/CVS + bin/gub-tester - build tester; integrates with + Bazaar,CVS,Darcs,Git,SVN and gub bin/cygwin-packager - roll cygwin package diff -r bec99ae5bac5 -r a726a656f0b8 TODO --- a/TODO Wed May 09 23:46:52 2007 -0300 +++ b/TODO Wed May 09 23:51:45 2007 -0300 @@ -1,13 +1,9 @@ -* Make test-gub work with plain, user-checked-out repositories again - (and/or remove lilypond hardcoding from cron-builder, move checkout - command from cron-builder to separate app (or into test-gub)). - Add option to send diffs again. Use case +* Add diff option: 561096638b3718af86d61570ac63bafa5c4a99f8 - bzr branch http://foo/bar.bzr && test-gub bar - - [or alternatively, something like - - gub --stage=download http://foo/bar.bzr && test-gub bar] +* Make downloaded repositories easier, something like + downloads/lilypond.git/.git # lilypond.git usable with git commands + downloads/ghostscript-trunk-HEAD.svn/.svn + downloads/ghostscript-trunk-7882.svn/.svn * Softcode stages. @@ -28,13 +24,26 @@ * Bootstrap whole toolchain from source on more platforms (freebsd)? +* Fix configure/make command env var settings (grep FIXME) + + @subst_method + def CFLAGS(self): + return 'bla' + +or + + def init(self): + self.CFLAGS='bla' LOWER PRIORITY -- replace os. calls with os.context wrapper ones +- replace os. calls with os.context wrapper ones (make a real + dry-run to quick-test all .py scripts?) - better name for gubb.py +- name for gub-tester (test-repo, repo-builder?) + - Split gub/*py into packager, builder, platform - Explode inheritance, and use membership in GUP diff -r bec99ae5bac5 -r a726a656f0b8 bin/cygwin-packager --- a/bin/cygwin-packager Wed May 09 23:46:52 2007 -0300 +++ b/bin/cygwin-packager Wed May 09 23:51:45 2007 -0300 @@ -22,17 +22,16 @@ """ +def argv0_relocation (): + import os, sys + bindir = os.path.dirname (sys.argv[0]) + prefix = os.path.dirname (bindir) + sys.path.insert (0, prefix) + +argv0_relocation () + import optparse import os -import sys - - -sys.path.insert (0, 'lib') ## for versiondb -sys.path.insert (0, '.') - - - - # from gub import context from gub import gup @@ -75,7 +74,7 @@ self.create () def build_number (self, package): - import versiondb + from gub import versiondb version_file = '%(uploads)s/%(name)s.versions' % package db = versiondb.VersionDataBase (version_file) version = '%(full_version)s' % package @@ -185,7 +184,7 @@ return readme def strip_dir (self, dir): - import misc + from gub import misc misc.map_command_dir (self.expand (dir), self.expand ('%(strip_command)s'), self.no_binary_strip, @@ -327,6 +326,7 @@ (options, args) = p.parse_args () if len (args) != 1: p.print_help () + import sys sys.exit (2) return (options, args) diff -r bec99ae5bac5 -r a726a656f0b8 bin/gpkg --- a/bin/gpkg Wed May 09 23:46:52 2007 -0300 +++ b/bin/gpkg Wed May 09 23:51:45 2007 -0300 @@ -23,11 +23,18 @@ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. """ +def argv0_relocation (): + import os, sys + bindir = os.path.dirname (sys.argv[0]) + prefix = os.path.dirname (bindir) + sys.path.insert (0, prefix) + +argv0_relocation () + import optparse import re import string import sys -sys.path.insert (0, '.') # from gub import gup from gub import oslog diff -r bec99ae5bac5 -r a726a656f0b8 bin/gub --- a/bin/gub Wed May 09 23:46:52 2007 -0300 +++ b/bin/gub Wed May 09 23:51:45 2007 -0300 @@ -20,8 +20,15 @@ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. """ +def argv0_relocation (): + import os, sys + bindir = os.path.dirname (sys.argv[0]) + prefix = os.path.dirname (bindir) + sys.path.insert (0, prefix) + +argv0_relocation () + import sys -sys.path.insert (0, '.') # from gub import cross ## fixme: double use of gub name. diff -r bec99ae5bac5 -r a726a656f0b8 bin/gub-tester --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/bin/gub-tester Wed May 09 23:51:45 2007 -0300 @@ -0,0 +1,402 @@ +#!/usr/bin/python + +def argv0_relocation (): + import os, sys + bindir = os.path.dirname (sys.argv[0]) + prefix = os.path.dirname (bindir) + sys.path.insert (0, prefix) + +argv0_relocation () + +import sys +import re +import os +import smtplib +import email.MIMEText +import email.Message +import email.MIMEMultipart +import optparse +import time +import dbhash + + +from gub import repository +from gub import oslog + + +################################################################ +# utils. + +def system (c): + print c + if os.system (c): + raise Exception ('barf') + + +def read_tail (file, amount=10240): + f = open (file) + f.seek (0, 2) + length = f.tell() + f.seek (- min (length, amount), 1) + return f.read () + +def canonicalize_string (target): + canonicalize = re.sub ('[ \t\n]', '_', target) + canonicalize = re.sub ('[^a-zA-Z0-9-]+', '_', canonicalize) + return canonicalize + +log_file = None + +################################################################ +# + +def result_message (parts, subject='') : + """Concatenate PARTS to a Message object.""" + + if not parts: + parts.append ('(empty)') + + parts = [email.MIMEText.MIMEText (p) for p in parts if p] + + msg = parts[0] + if len (parts) > 1: + msg = email.MIMEMultipart.MIMEMultipart () + for p in parts: + msg.attach (p) + + msg['Subject'] = subject + msg.epilogue = '' + + return msg + +def opt_parser (): + if os.environ.has_key ('EMAIL'): + address = os.environ['EMAIL'] + else: + try: + address = '%s@localhost' % os.getlogin () + except OSError: + address = 'root@localhost' + + p = optparse.OptionParser (usage='gub-tester [options] command command ... ') + p.address = [address] + + examples = '''Examples: +gub-tester --repository=downloads/lilypond.git \\ + gub --branch=lilypond:master lilypond + +gub-tester --url=http://bazaar.launchpad.net/~yaffut/yaffut/yaffut.bzr \\ + 'make all check' + +bzr branch http://bazaar.launchpad.net/~yaffut/yaffut/yaffut.bzr +cd yaffut.bzr && gub-tester 'make all check' + +''' + + def format_examples (self): + return examples + if p.__dict__.has_key ('epilog'): + p.formatter.format_epilog = format_examples + p.epilog = examples + else: + p.formatter.format_description = format_examples + p.description = examples + p.add_option ('-t', '--to', + action='append', + dest='address', + default=[], + help='where to send error report') + p.add_option ('--dry-run', + dest='dry_run', + default=False, + action='store_true', + help='do not run any commands') + + p.add_option ('--append-diff', + dest='append_diff', + default=False, + action='store_true', + help='append diff since last successful run') + + p.add_option ('--bcc', + action='append', + dest='bcc_address', + default=[], + help='BCC for error report') + + p.add_option ('-f', '--from', + action='store', + dest='sender', + default=address, + help='whom to list as sender') + + p.add_option ('--branch', + action='store', + dest='branch', + default='HEAD', + help='which branch to fetch [GIT repositories]') + + p.add_option ('--url', + action='store', + dest='url', + default=None, + help='where to fetch sources') + + p.add_option ('--update', + action='store_true', + dest='update', + default=False, + help='checkout or update sources') + + p.add_option ('--revision', + action='store', + dest='revision', + default=None, + help='what revision to fetch') + + p.add_option ('--repository', + action='store', + dest='repository', + default='.', + help='where to download/cache repository') + + p.add_option ('--tag-repo', + action='store', + dest='tag_repo', + default='', + help='where to push success tags.') + + p.add_option ('--quiet', + action='store_true', + dest='be_quiet', + default=False, + help='only send mail when there was an error.') + + p.add_option ('--dependent', + action='store_true', + dest='is_dependent', + default=False, + help='test targets depend on each other') + + p.add_option ('--posthook', + action='append', + dest='posthooks', + default=[], + help='commands to execute after successful tests.') + + p.add_option ('--test-self', + action='store_true', + dest='test_self', + default=False, + help='run a cursory self test.') + + p.add_option ('-s', '--smtp', + action='store', + dest='smtp', + default='localhost', + help='SMTP server to use.') + + p.add_option ('--result-directory', + action='store', + dest='result_dir', + help='Where to store databases test results', + default='log') + + p.add_option ('-v', '--verbose', action='count', dest='verbose', default=0) + + return p + + +def get_db (options, name): + name = options.result_dir + '/%s.db' % name + + db_file = os.path.join (options.result_dir, name) + db = dbhash.open (db_file, 'c') + return db + + +def test_target (repo, options, target, last_patch): + canonicalize = canonicalize_string (target) + release_hash = repo.get_checksum () + + done_db = get_db (options, canonicalize) + if done_db.has_key (release_hash): + log_file.info ('release %(release_hash)s has already been checked' + % locals ()) + return None + + logfile = 'test-%(canonicalize)s.log' % locals () + logfile = os.path.join (options.result_dir, logfile) + + cmd = 'nice time sh -c "(%(target)s)" > %(logfile)s 2>&1' % locals () + + log_file.command (cmd) + + if options.dry_run: + return ('SUCCESS', ['dryrun']) + + stat = os.system (cmd) + base_tag = 'success-%(canonicalize)s-' % locals () + result = 'unknown' + attachments = [] + + body = read_tail (logfile, 10240).split ('\n') + if stat: + result = 'FAIL' + attachments = ['error for\n\n\t%s\n\n\n%s' % (target, + '\n'.join (body[-0:]))] + if options.append_diff: + attachments += [repo.get_diff_from_tag (base_tag)] + else: + if 0: + # FIXME: find in archives, what did this do? + # format of last_patch? + # canonicalize_target vs canonicalize_string? + print last_patch + tag = base_tag + canonicalize_string (last_patch['date']) + repo.tag (tag) + log_file.log ('tagging with %s' % tag) + if options.tag_repo: + repo.push (tag, options.tag_repo) + + result = 'SUCCESS' + attachments = ['success for\n\n\t%s\n\n%s' + % (target, + '\n'.join (body[-10:]))] + + log_file.info ('%s: %s' % (target, result)) + + done_db[release_hash] = time.ctime () + return (result, attachments) + +def send_message (options, msg): + if not options.address: + log_file.info ('No recipients for result mail') + # FIXME: what about env[EMAIL]? + return + + COMMASPACE = ', ' + msg['From'] = options.sender + msg['To'] = COMMASPACE.join (options.address) + if options.bcc_address: + msg['BCC'] = COMMASPACE.join (options.bcc_address) + + msg['X-Autogenerated'] = 'lilypond' + connection = smtplib.SMTP (options.smtp) + connection.sendmail (options.sender, options.address, msg.as_string ()) + + +def send_result_by_mail (options, parts, subject='Autotester result'): + msg = result_message (parts, subject) + send_message (options, msg) + +def print_results (options, parts, subject='Autotester result'): + print '\n===\n\nSUBJECT: ', subject + print '\n---\n'.join (parts) + print 'END RESULT\n===' + +def real_main (options, args, handle_result): + global log_file + + log = 'log/gub-tester.log' + if options.dry_run: + log = '/dev/stdout' + + log_file = oslog.Os_commands (log, options.verbose) + + log_file.info (' *** %s' % time.ctime ()) + log_file.info (' *** Starting tests:\n %s' % '\n '.join (args)) + + repo = repository.get_repository_proxy (options.repository, + options.url, + options.revision, + options.branch) + + if (not repo.is_downloaded () or options.update or options.revision): + repo.download () + repo.update_workdir ('.') + + log_file.info ('Repository %s' % str (repo)) + + last_patch = repo.get_revision_description () + release_hash = repo.get_checksum () + + release_id = ''' + +Last patch of this release: + +%(last_patch)s\n + +Checksum of revision: %(release_hash)s + +''' % locals () + + + summary_body = '\n\n' + results = {} + failures = 0 + for a in args: + result_tup = test_target (repo, options, a, last_patch) + if not result_tup: + continue + (result, atts) = result_tup + results[a] = result_tup + success = result.startswith ('SUCCESS') + if not (options.be_quiet and success): + handle_result (options, atts, subject='Autotester: %s %s' + % (result, a)) + + summary_body += '%s\n %s\n' % (a, result) + + if not success: + failures += 1 + if options.is_dependent: + break + + if (results + and len (args) > 1 + and (failures > 0 or not options.be_quiet)): + + handle_result (options, + [summary_body, release_id], + subject='Autotester: summary') + + if failures == 0 and results: + for p in options.posthooks: + os.system (p) + +def test_self (options, args): + self_test_dir = 'test-gub-test.darcs' + system ('rm -rf %s ' % self_test_dir) + system ('mkdir %s ' % self_test_dir) + os.chdir (self_test_dir) + system ('mkdir log') + system ('''echo '#!/bin/sh +true' > foo.sh''') + system ('darcs init') + system ('echo author > _darcs/prefs/author') + system ('darcs add foo.sh') + system ('darcs record -am "add bla"') + options.repository = os.getcwd () + real_main (options, ['false', 'true', 'sh foo.sh'], print_results) + + system ('''echo '#!/bin/sh +true' > foo.sh''') + system ('darcs record -am "change bla"') + real_main (options, ['sh foo.sh'], print_results) + +def main (): + (options, args) = opt_parser ().parse_args () + + if not os.path.isdir (options.result_dir): + os.makedirs (options.result_dir) + + options.result_dir = os.path.abspath (options.result_dir) + + if options.test_self: + test_self (options, args) + else: + real_main (options, args, send_result_by_mail) + +if __name__ == '__main__': + main () diff -r bec99ae5bac5 -r a726a656f0b8 bin/installer-builder --- a/bin/installer-builder Wed May 09 23:46:52 2007 -0300 +++ b/bin/installer-builder Wed May 09 23:51:45 2007 -0300 @@ -20,12 +20,18 @@ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. """ +def argv0_relocation (): + import os, sys + bindir = os.path.dirname (sys.argv[0]) + prefix = os.path.dirname (bindir) + sys.path.insert (0, prefix) + +argv0_relocation () import optparse import os import sys import pickle -sys.path.insert (0, '.') # from gub import gup from gub import installer diff -r bec99ae5bac5 -r a726a656f0b8 bin/test-gub --- a/bin/test-gub Wed May 09 23:46:52 2007 -0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,343 +0,0 @@ -#!/usr/bin/python - -import sys -import re -import os -import smtplib -import email.MIMEText -import email.Message -import email.MIMEMultipart -import optparse -import time -import dbhash - -sys.path.insert (0, '.') -from gub import repository - - -################################################################ -# utils. - -def system (c): - print c - if os.system (c): - raise Exception ('barf') - - -def read_tail (file, amount=10240): - f = open (file) - f.seek (0, 2) - length = f.tell() - f.seek (- min (length, amount), 1) - return f.read () - -def canonicalize_string (target): - canonicalize = re.sub ('[ \t\n]', '_', target) - canonicalize = re.sub ('[^a-zA-Z0-9-]+', '_', canonicalize) - return canonicalize - -class LogFile: - def __init__ (self, name): - if name: - self.file = open (name, 'a') - else: - self.file = sys.stdout - - self.prefix = 'test-gub[%d]: ' % os.getpid () - - def log (self, msg): - self.file.write ('%s%s\n' % (self.prefix, msg)) - self.file.flush () - - def __del__ (self): - self.log (' *** finished') - -log_file = None - -################################################################ -# - -def result_message (parts, subject='') : - """Concatenate PARTS to a Message object.""" - - if not parts: - parts.append ('(empty)') - - parts = [email.MIMEText.MIMEText (p) for p in parts if p] - - msg = parts[0] - if len (parts) > 1: - msg = email.MIMEMultipart.MIMEMultipart () - for p in parts: - msg.attach (p) - - msg['Subject'] = subject - msg.epilogue = '' - - return msg - -def opt_parser (): - if os.environ.has_key ('EMAIL'): - address = os.environ['EMAIL'] - else: - try: - address = '%s@localhost' % os.getlogin () - except OSError: - address = 'root@localhost' - - p = optparse.OptionParser (usage="test-gub [options] command command ... ") - p.add_option ('-t', '--to', - action='append', - dest='address', - default=[], - help='where to send error report') - p.add_option ('--dry-run', - dest='dry_run', - default=False, - action="store_true", - help="don't run any commands") - - p.add_option ('--bcc', - action='append', - dest='bcc_address', - default=[], - help='BCC for error report') - - p.add_option ('-f', '--from', - action='store', - dest='sender', - default=address, - help='whom to list as sender') - - p.add_option ('--branch', - action="store", - dest="branch", - default="HEAD", - help="which branch to test") - - p.add_option ('--repository', - action="store", - dest="repository", - default=".", - help="repository directory") - - p.add_option ('--tag-repo', - action='store', - dest='tag_repo', - default='', - help='where to push success tags.') - - p.add_option ('--quiet', - action='store_true', - dest='be_quiet', - default=False, - help='only send mail when there was an error.') - - p.add_option ('--dependent', - action="store_true", - dest="is_dependent", - default=False, - help="test targets depend on each other") - - p.add_option ('--posthook', - action='append', - dest='posthooks', - default=[], - help='commands to execute after successful tests.') - - p.add_option ('--test-self', - action='store_true', - dest='test_self', - default=False, - help='run a cursory self test.') - - p.add_option ('-s', '--smtp', - action='store', - dest='smtp', - default='localhost', - help='SMTP server to use.') - - p.add_option ('--result-directory', - action="store", - dest='result_dir', - help="Where to store databases test results", - default="log") - - return p - - -def get_db (options, name): - name = options.result_dir + '/%s.db' % name - - db_file = os.path.join (options.result_dir, name) - db = dbhash.open (db_file, 'c') - return db - - -def test_target (repo, options, target, last_patch): - canonicalize = canonicalize_string (target) - release_hash = repo.get_checksum () - - done_db = get_db (options, canonicalize) - if done_db.has_key (release_hash): - log_file.log ('release %(release_hash)s has already been checked' % locals ()) - return None - - logfile = 'test-%(canonicalize)s.log' % locals () - logfile = os.path.join (options.result_dir, logfile) - - cmd = "nice time %(target)s > %(logfile)s 2>&1" % locals () - - log_file.log (cmd) - - if options.dry_run: - return ('SUCCESS', ['dryrun']) - - stat = os.system (cmd) - - result = 'unknown' - attachments = [] - - body = read_tail (logfile, 10240).split ('\n') - if stat: - result = 'FAIL' - attachments = ['error for\n\n\t%s\n\n\n%s' % (target, - '\n'.join (body[-0:]))] - - else: - result = "SUCCESS" - attachments = ['success for\n\n\t%s\n\n%s' - % (target, - '\n'.join (body[-10:]))] - - log_file.log ('%s: %s' % (target, result)) - - done_db[release_hash] = time.ctime () - return (result, attachments) - -def send_message (options, msg): - if not options.address: - log_file.log ('No recipients for result mail') - return - - COMMASPACE = ', ' - msg['From'] = options.sender - msg['To'] = COMMASPACE.join (options.address) - if options.bcc_address: - msg['BCC'] = COMMASPACE.join (options.bcc_address) - - msg['X-Autogenerated'] = 'lilypond' - connection = smtplib.SMTP (options.smtp) - connection.sendmail (options.sender, options.address, msg.as_string ()) - - -def send_result_by_mail (options, parts, subject="Autotester result"): - msg = result_message (parts, subject) - send_message (options, msg) - -def print_results (options, parts, subject="Autotester result"): - print '\n===\n\nSUBJECT: ', subject - print '\n---\n'.join (parts) - print 'END RESULT\n===' - -def real_main (options, args, handle_result): - global log_file - - log = 'log/test-gubb.log' - if options.dry_run: - log = '' - - log_file = LogFile (log) - - log_file.log (' *** %s' % time.ctime ()) - log_file.log (' *** Starting tests:\n %s' % '\n '.join (args)) - - repo = repository.get_repository_proxy (options.repository, options.branch) - log_file.log ("Repository %s" % str (repo)) - - last_patch = repo.get_revision_description () - release_hash = repo.get_checksum () - - release_id = ''' - -Last patch of this release: - -%(last_patch)s\n - -Checksum of revision: %(release_hash)s - -''' % locals () - - - summary_body = '\n\n' - results = {} - failures = 0 - for a in args: - result_tup = test_target (repo, options, a, last_patch) - if not result_tup: - continue - - (result, atts) = result_tup - - results[a] = result_tup - - success = result.startswith ('SUCCESS') - if not (options.be_quiet and success): - handle_result (options, atts, subject="Autotester: %s %s" % (result, a)) - - summary_body += '%s\n %s\n' % (a, result) - - - if not success: - failures += 1 - if options.is_dependent: - break - - if (results - and len (args) > 1 - and (failures > 0 or not options.be_quiet)): - - handle_result (options, - [summary_body, release_id], subject="Autotester: summary") - - if failures == 0 and results: - for p in options.posthooks: - os.system (p) - -def test_self (options, args): - self_test_dir = 'test-gub-test.darcs' - system ('rm -rf %s ' % self_test_dir) - system ('mkdir %s ' % self_test_dir) - os.chdir (self_test_dir) - system ('mkdir log') - system (r"echo -e '#!/bin/sh\ntrue\n' > foo.sh") - system ('darcs init') - system ('echo author > _darcs/prefs/author') - system ('darcs add foo.sh') - system ('darcs record -am "add bla"') - options.repository = os.getcwd () - - real_main (options, ['false', 'true', 'sh foo.sh'], print_results) - - system (r"echo -e '#!/bin/sh\nfalse\n' > foo.sh") - system ('darcs record -am "change bla"') - real_main (options, ['sh foo.sh'], print_results) - -def main (): - (options, args) = opt_parser ().parse_args () - - if not os.path.isdir (options.result_dir): - os.makedirs (options.result_dir) - - options.result_dir = os.path.abspath (options.result_dir) - - if options.test_self: - test_self (options, args) - else: - real_main (options, args, send_result_by_mail) - - - -if __name__ == '__main__': -# test () - main () diff -r bec99ae5bac5 -r a726a656f0b8 gub/gubb.py --- a/gub/gubb.py Wed May 09 23:46:52 2007 -0300 +++ b/gub/gubb.py Wed May 09 23:51:45 2007 -0300 @@ -620,9 +620,7 @@ # TODO: junk this, always set repo in __init__ def with_vc (self, repo): self.vc_repository = repo - self.vc_repository.system = self.os_interface.system - self.vc_repository.read_pipe = self.os_interface.read_pipe - self.vc_repository.download_url = self.os_interface.download_url + self.vc_repository.set_oslog (self.os_interface) return self def with_tarball (self, mirror='', version='', format='gz', strip_components=1, name=''): diff -r bec99ae5bac5 -r a726a656f0b8 gub/repository.py --- a/gub/repository.py Wed May 09 23:46:52 2007 -0300 +++ b/gub/repository.py Wed May 09 23:51:45 2007 -0300 @@ -1,4 +1,3 @@ - """ Copyright (c) 2005--2007 Jan Nieuwenhuizen @@ -32,12 +31,34 @@ ## Rename to Source/source.py? class Repository: - def __init__ (self): - # Fallback, this will go through oslog + def __init__ (self, dir, vcs, source): + self.vcs = vcs + self.dir = os.path.normpath (dir) + self.vcs + + if not dir or dir == '.': + dir = os.getcwd () + if os.path.isdir (os.path.join (dir, self.vcs)): + # Support user-checkouts: If we're already checked-out + # HERE, use that as repository + self.dir = dir + else: + # Otherwise, check fresh repository out under .VCS + self.dir = os.path.join (os.getcwd (), self.vcs) + self .source = source + + self.oslog = None + # Fallbacks, this will go through oslog self.system = misc.system self.read_pipe = misc.read_pipe self.download_url = misc.download_url - + + def set_oslog (self, oslog): + # Fallbacks, this will go through oslog + self.oslog = oslog + self.system = oslog.system + self.read_pipe = oslog.read_pipe + self.download_url = oslog.download_url + def download (self): pass @@ -52,7 +73,10 @@ def is_tracking (self): "Whether download will fetch newer versions if available" - + return False + + def is_downloaded (self): + "Whether repository is available" return False def update_workdir (self, destdir): @@ -65,8 +89,19 @@ """A human-readable revision number. It need not be unique over revisions.""" return '0' + def read_last_patch (self): + """Return a dict with info about the last patch""" + assert 0 + return {} + + def get_diff_from_tag (self, name): + """Return diff wrt to last tag that starts with NAME """ + assert 0 + return 'baseclass method called' + class Version: def __init__ (self, version): + self.dir = None self._version = version def download (self): @@ -84,14 +119,14 @@ def version (self): return self._version + def set_oslog (self, oslog): + pass + class Darcs (Repository): def __init__ (self, dir, source=''): - Repository.__init__ (self) - self.dir = dir + '.darcs' - self.source = source + Repository.__init__ (self, dir, '.darcs', source) def darcs_pipe (self, cmd): - dir = self.dir return self.read_pipe ('cd %(dir)s && darcs %(cmd)s' % locals ()) @@ -102,13 +137,15 @@ def get_revision_description (self): return self.darcs_pipe ('changes --last=1') + def is_downloaded (self): + return os.path.isdir (self.dir + '/_darcs') + def download (self): - dir = self.dir source = self.source - - if os.path.exists (dir + '/_darcs'): + if not self.is_downloaded (): self.darcs ('pull -a %(source)s' % locals ()) else: + dir = self.dir self.system ('darcs get %(source)s %(dir)s' % locals ()) def is_tracking (self): @@ -140,7 +177,10 @@ self.system ('mkdir -p %(destdir)s' % locals ()) dir = self.dir - self.system ('rsync --exclude _darcs -av %(dir)s/* %(destdir)s/' % locals()) + verbose = '' + if self.oslog and self.oslog.verbose >= self.oslog.commands: + verbose = 'v' + self.system ('rsync --exclude _darcs -a%(verbose)s %(dir)s/* %(destdir)s/' % locals()) def get_file_content (self, file): dir = self.dir @@ -148,14 +188,19 @@ class TarBall (Repository): - def __init__ (self, dir, url, version, strip_components=1): - Repository.__init__ (self) - if not os.path.isdir (dir): - self.system ('mkdir -p %s' % dir) + # TODO: s/url/source + def __init__ (self, dir, url, version=None, strip_components=1): + Repository.__init__ (self, dir, '.tar', url) self.dir = dir - self.url = url + if not os.path.isdir (self.dir): + self.system ('mkdir -p %s' % self.dir) + self._version = version + if not version: + x, v = misc.split_ball (url) + self._version = '.'.join (v) + self.branch = None self.strip_components = strip_components @@ -163,16 +208,16 @@ return False def _file_name (self): - return re.search ('.*/([^/]+)$', self.url).group (1) + return re.search ('.*/([^/]+)$', self.source).group (1) - def _is_downloaded (self): + def is_downloaded (self): name = os.path.join (self.dir, self._file_name ()) return os.path.exists (name) def download (self): - if self._is_downloaded (): + if self.is_downloaded (): return - self.download_url (self.url, self.dir) + self.download_url (self.source, self.dir) def get_checksum (self): from gub import misc @@ -219,17 +264,19 @@ pass class Git (Repository): - def __init__ (self, git_dir, source='', branch='', revision=''): - Repository.__init__ (self) - - self.repo_dir = os.path.normpath (git_dir) + '.git' + def __init__ (self, dir, source='', branch='', revision=''): + Repository.__init__ (self, dir, '.git', source) + user_repo_dir = os.path.join (self.dir, self.vcs) + if os.path.isdir (user_repo_dir): + self.dir = user_repo_dir self.checksums = {} self.local_branch = '' self.remote_branch = branch - self.source = source self.revision = revision - self.repo_url_suffix = re.sub ('.*://', '', source) + self.repo_url_suffix = None + if source: + self.repo_url_suffix = re.sub ('.*://', '', source) if self.repo_url_suffix: # FIXME: logic copied foo times @@ -263,8 +310,7 @@ b = self.local_branch if not b: b = self.revision - - return '#' % (self.repo_dir, b) + return '#' % (self.dir, b) def get_revision_description (self): return self.git_pipe ('log --max-count=1 %s' % self.local_branch) @@ -273,61 +319,54 @@ committish = self.git_pipe ('log --max-count=1 --pretty=oneline %(local_branch)s' % self.__dict__).split (' ')[0] m = re.search ('^tree ([0-9a-f]+)', - self.git_pipe ('cat-file commit %(committish)s' % locals ())) - + self.git_pipe ('cat-file commit %(committish)s' + % locals ())) treeish = m.group (1) - for f in self.git_pipe ('ls-tree -r %(treeish)s' % - locals ()).split ('\n'): + for f in self.git_pipe ('ls-tree -r %(treeish)s' + % locals ()).split ('\n'): (info, name) = f.split ('\t') (mode, type, fileish) = info.split (' ') - if name == file_name: return self.git_pipe ('cat-file blob %(fileish)s ' % locals ()) raise RepositoryException ('file not found') def get_branches (self): - branch_lines = self.read_pipe (self.git_command () + ' branch -l ').split ('\n') - + branch_lines = self.read_pipe (self.git_command () + + ' branch -l ').split ('\n') branches = [b[2:] for b in branch_lines] return [b for b in branches if b] def git_command (self, dir, repo_dir): if repo_dir: repo_dir = '--git-dir %s' % repo_dir - c = 'git %(repo_dir)s' % locals () if dir: c = 'cd %s && %s' % (dir, c) - return c - def git (self, cmd, dir='', ignore_errors=False, - repo_dir=''): - + def git (self, cmd, dir='', ignore_errors=False, repo_dir=''): if repo_dir == '' and dir == '': - repo_dir = self.repo_dir - + repo_dir = self.dir gc = self.git_command (dir, repo_dir) cmd = '%(gc)s %(cmd)s' % locals () - self.system (cmd, ignore_errors=ignore_errors) - def git_pipe (self, cmd, ignore_errors=False, - dir='', repo_dir=''): - + def git_pipe (self, cmd, ignore_errors=False, dir='', repo_dir=''): if repo_dir == '' and dir == '': - repo_dir = self.repo_dir - + repo_dir = self.dir gc = self.git_command (dir, repo_dir) return self.read_pipe ('%(gc)s %(cmd)s' % locals ()) + def is_downloaded (self): + return os.path.isdir (self.dir) + def download (self): - repo = self.repo_dir + repo = self.dir source = self.source revision = self.revision - if not os.path.isdir (self.repo_dir): + if not self.is_downloaded (): self.git ('--git-dir %(repo)s clone --bare -n %(source)s %(repo)s' % locals ()) for (root, dirs, files) in os.walk ('%(repo)s/refs/heads/' % locals ()): @@ -356,6 +395,8 @@ refs = '%s:%s' % (self.remote_branch, self.branch) + # FIXME: if source == None (for user checkouts), how to ask + # git what parent url is? `git info' does not work self.git ('fetch --update-head-ok %(source)s %(refs)s ' % locals ()) self.checksums = {} @@ -367,7 +408,7 @@ if self.checksums.has_key (branch): return self.checksums[branch] - repo_dir = self.repo_dir + repo_dir = self.dir if os.path.isdir (repo_dir): ## can't use describe: fails in absence of tags. cs = self.git_pipe ('rev-list --max-count=1 %(branch)s' % locals ()) @@ -384,7 +425,7 @@ def update_workdir (self, destdir): - repo_dir = self.repo_dir + repo_dir = self.dir branch = self.local_branch revision = self.revision @@ -402,24 +443,22 @@ open ('%(destdir)s/.git/refs/heads/%(branch)s' % locals (), 'w').write (revision) self.git ('checkout %(branch)s' % locals (), dir=destdir) -class CVS(Repository): +class CVS (Repository): cvs_entries_line = re.compile ("^/([^/]*)/([^/]*)/([^/]*)/([^/]*)/") #tag_dateformat = '%Y/%m/%d %H:%M:%S' - def __init__ (self, dir, - source='', module='', tag='HEAD'): - Repository.__init__ (self) - self.repo_dir = os.path.normpath (dir) + '.cvs' + def __init__ (self, dir, source='', module='', tag='HEAD'): + Repository.__init__ (self, dir, '.cvs', source) self.module = module self.checksums = {} self.source = source self.tag = tag self.branch = tag # for vc_version_suffix - if not os.path.isdir (self.repo_dir): - self.system ('mkdir -p %s' % self.repo_dir) + if not os.path.isdir (self.dir): + self.system ('mkdir -p %s' % self.dir) def _checkout_dir (self): - return '%s/%s' % (self.repo_dir, self.tag) + return '%s/%s' % (self.dir, self.tag) def is_tracking (self): return True ##FIXME @@ -433,7 +472,7 @@ changelog_rev = '' - for (name, version, date, dontknow) in self.cvs_entries (self.repo_dir + '/CVS'): + for (name, version, date, dontknow) in self.cvs_entries (self.dir + '/CVS'): if name == 'ChangeLog': changelog_rev = version break @@ -448,7 +487,7 @@ if self.checksums.has_key (self.tag): return self.checksums[self.tag] - file = '%s/%s/.vc-checksum' % (self.repo_dir, self.tag) + file = '%s/%s/.vc-checksum' % (self.dir, self.tag) if os.path.exists (file): cs = open (file).read () @@ -483,8 +522,15 @@ def update_workdir (self, destdir): dir = self._checkout_dir () ## TODO: can we get deletes from vc? - self.system ('rsync -av --delete --exclude CVS %(dir)s/ %(destdir)s' % locals ()) + verbose = '' + if self.oslog and self.oslog.verbose >= self.oslog.commands: + verbose = 'v' + self.system ('rsync -a%(verbose)s --delete --exclude CVS %(dir)s/ %(destdir)s' % locals ()) + def is_downloaded (self): + dir = self._checkout_dir () + return os.path.isdir (dir + '/CVS') + def download (self): suffix = self.tag rev_opt = '-r ' + self.tag @@ -493,10 +539,10 @@ lock_dir = locker.Locker (dir + '.lock') module = self.module cmd = '' - if os.path.isdir (dir + '/CVS'): + if not self.is_downloaded (): cmd += 'cd %(dir)s && cvs -q up -dCAP %(rev_opt)s' % locals() else: - repo_dir = self.repo_dir + repo_dir = self.dir cmd += 'cd %(repo_dir)s/ && cvs -d %(source)s -q co -d %(suffix)s %(rev_opt)s %(module)s''' % locals () self.system (cmd) @@ -528,28 +574,23 @@ ds = self.cvs_dirs (dir) es = [] for d in ds: - ## strip CVS/ basedir = os.path.split (d)[0] for e in self.cvs_entries (d): file_name = os.path.join (basedir, e[0]) - file_name = file_name.replace (self.repo_dir + '/', '') + file_name = file_name.replace (self.dir + '/', '') es.append ((file_name,) + e[1:]) - - return es def all_files (self, branch): - entries = self.all_cvs_entries (self.repo_dir + '/' + branch) + entries = self.all_cvs_entries (self.dir + '/' + branch) return [e[0] for e in entries] # FIXME: why are cvs, darcs, git so complicated? class SimpleRepo (Repository): - def __init__ (self, dir, repository, source, branch, revision='HEAD'): - Repository.__init__ (self) - self.repository = repository - self.dir = os.path.normpath (dir) + self.repository + def __init__ (self, dir, vcs, source, branch, revision='HEAD'): + Repository.__init__ (self, dir, vcs, source) self.source = source self.revision = revision self.branch = branch @@ -558,25 +599,34 @@ def is_tracking (self): ## FIXME, probably wrong. - return self.revision == 'HEAD' + return (not self.revision or self.revision == 'HEAD') def update_workdir (self, destdir): dir = self._checkout_dir () self._copy_working_dir (dir, destdir) - def download (self): + def is_downloaded (self): dir = self._checkout_dir () - if not os.path.isdir (dir + '/' + self.repository): + return os.path.isdir (os.path.join (dir, self.vcs)) + + def download (self): + if not self.is_downloaded (): self._checkout () if self._current_revision () != self.revision: self._update (self.revision) def _copy_working_dir (self, dir, copy): - repository = self.repository - self.system ('rsync -av --exclude %(repository)s %(dir)s/ %(copy)s' + repository = self.vcs + verbose = '' + if self.oslog and self.oslog.verbose >= self.oslog.commands: + verbose = 'v' + self.system ('rsync -a%(verbose)s --exclude %(repository)s %(dir)s/ %(copy)s' % locals ()) def _checkout_dir (self): + # Support user-check-outs + if os.path.isdir (os.path.join (self.dir, self.vcs)): + return self.dir revision = self.revision dir = self.dir branch = self.branch @@ -595,6 +645,8 @@ class Subversion (SimpleRepo): def __init__ (self, dir, source, branch, module, revision='HEAD'): + if not revision: + revision = 'HEAD' SimpleRepo.__init__ (self, dir, '.svn', source, branch, revision) self.module = module @@ -624,6 +676,8 @@ class Bazaar (SimpleRepo): def __init__ (self, dir, source, revision='HEAD'): # FIXME: multi-branch repos not supported for now + if not revision: + revision = '0' SimpleRepo.__init__ (self, dir, '.bzr', source, '', revision) def _current_revision (self): @@ -632,15 +686,19 @@ return revno[:-1] def _checkout (self): + dir = self.dir source = self.source revision = self.revision rev_opt = '-r %(revision)s ' % locals () - self.system ('''branch %(rev_opt)s %(source)s %(revision)s''' - % locals ()) + self.system ('''cd %(dir)s && bzr branch %(rev_opt)s %(source)s %(revision)s''' + % locals ()) def _update (self, revision): rev_opt = '-r %(revision)s ' % locals () - self.bzr_system ('pull %(rev_opt)s' % locals ()) + source = self.source + if not source: + source = '' + self.bzr_system ('pull %(rev_opt)s %(source)s' % locals ()) def bzr_pipe (self, cmd): dir = self._checkout_dir () @@ -653,45 +711,74 @@ def get_revision_description (self): return self.bzr_pipe ('log --verbose -r-1') -# FIXME: repository detection AND repositories only work if they are -# checked-out in a dir named .../name.REPOSITORY Eg, for GIT, this -# means that only the first arbitrary in .git in -# `downloads/lilypond.git/.git' is `detected'. Repositories passed to -# test-gub must have the .REPOSITORY stripped, --repository=. does not -# work. +def get_appended_vcs_name (name): + return re.search (r"(.*)\.(bzr|git|cvs|svn|darcs|.tar(.gz|.bz2))", name) + +def get_prepended_vcs_name (name): + return re.search (r"(bzr|git|cvs|svn|darcs):", name) -# This is not trivial to fix, as the DIR passed to Repository () is -# not an existing directory, it gets `.REPOSITORY' appended in the -# constructors. +# FIXME: removeme, allow for user to checkout sources in any directory +# and use that as cache +def get_vcs_type_from_checkout_directory_name (dir): + m = get_appended_vcs_name (dir) + if m: + dir = m.group (1) + type = m.group (2) + return dir, type + return dir, None -# Also, different revisions get checked-out in different directories: -#, eg: foo.svn/trunk-7111, foo.svn/trunk-HEAD, etc. +def get_vcs_type_of_dir (dir): + # FIXME: get these from all repositories... + for i in ('.bzr', '.git', 'CVS', '.svn', '_darcs'): + if os.path.isdir (os.path.join (dir, i)): + return i.replace ('.', '').replace ('_', '') + #Hmm + return 'tar.gz' -# For test-gub to work outside gub again, for now use a workaround -# like -# mkdir foo.bzr && cd foo.bzr -# bzr branch URL HEAD -# cd HEAD && mkdir log -# test-gub --repository $(cd .. && pwd) -def get_repository_proxy (dir, branch): - m = re.search (r"(.*)\.(bzr|git|cvs|svn|darcs)", dir) +def get_vcs_type_from_url (url): + m = get_prepended_vcs_name (url) + if m: + type = m.group (1) + url = m.group (2) + return url, type + p = url.find ('//:') + if p > 0: + protocol = url[:p] + type = {'bzr+ssh': 'bzr', + 'svn+ssh': 'svn', + }.get (protocol, None) + if type: + return url, type + m = get_appended_vcs_name (url) + if m: + type = m.group (2) + return url, type + return url, None - print 'dir:', dir - - dir = m.group (1) - type = m.group (2) +def get_repository_proxy (dir, url, revision, branch): + type = None + if url: + url, type = get_vcs_type_from_url (url) + if not type: + dir, type = get_vcs_type_from_checkout_directory_name (dir) + if not type: + # FIXME: todo: teach repositories that they might be + type = get_vcs_type_of_dir (dir) if type == 'bzr': - return Bazaar (dir, source='unknown') + return Bazaar (dir, source=url, revision=revision) elif type == 'cvs': - return CVS (dir, branch=branch) + return CVS (dir, source=url, tag=branch) elif type == 'darcs': - return Darcs (dir) + return Darcs (dir, source=url) elif type == 'git': - return Git (dir, branch=branch) + return Git (dir, source=url, branch=branch, revision=revision) elif type == 'svn': - return Subversion (dir, branch=branch) - else: - raise UnknownVcSystem('repo format unknown: ' + dir) - - return Repository('', '') + return Subversion (dir, source=url, branch=branch) + elif type and type.startswith ('.tar.'): + return TarBall (dir, url=url, branch=branch) + + class UnknownVcSystem (Exception): + pass + raise UnknownVcSystem ('Cannot determine vcs type: url=%(url)s, dir=%(dir)s' + % locals ()) diff -r bec99ae5bac5 -r a726a656f0b8 test-lily/cron-builder.py --- a/test-lily/cron-builder.py Wed May 09 23:46:52 2007 -0300 +++ b/test-lily/cron-builder.py Wed May 09 23:51:45 2007 -0300 @@ -68,7 +68,7 @@ action="store_true", dest="build_docs", default=None, - help="build docs. Implies --dependent for test-gub") + help="build docs. Implies --dependent for gub-tester") p.add_option ('--package', action="store_true", @@ -92,7 +92,7 @@ action='store', dest='test_options', default="", - help='what to pass to test-gub') + help='what to pass to gub-tester') p.add_option ('--unversioned', action="store_true", @@ -149,7 +149,8 @@ make_cmd = 'make %s ' % opts.make_options python_cmd = sys.executable + ' ' - ## can't have these in test-gub, since these + # FIXME: use gub-tester's download facility + ## can't have these in gub-tester, since these ## will always usually result in "release already tested" for a in args: system (python_cmd + 'bin/gub --branch %s:%s -p %s --stage=download lilypond' @@ -158,7 +159,7 @@ test_cmds = [] if opts.build_package: - test_cmds += [python_cmd + 'bin/gub --branch %s:%s -lp %s build lilypond ' + test_cmds += [python_cmd + 'bin/gub --branch %s:%s -lp %s lilypond ' % (opts.branch, opts.local_branch, p) for p in args] if opts.build_installer: @@ -176,7 +177,7 @@ if opts.build_tarball: test_cmds += [make_cmd + " dist-check"] - system (python_cmd + 'bin/test-gub %s %s ' + system (python_cmd + 'bin/gub-tester %s %s ' % (opts.test_options, ' '.join (["'%s'" % c for c in test_cmds]))) if __name__ == '__main__':