Mercurial > gub
changeset 3393:32de3bd3161c
Merge branch 'gub' of git+ssh://git.sv.gnu.org/srv/git/lilypond into gub
| author | Jan Nieuwenhuizen <janneke@gnu.org> |
|---|---|
| date | Wed, 02 May 2007 17:39:10 +0200 |
| parents | f8e3e3cb866c (current diff) 533b0a066744 (diff) |
| children | eb2fe94b5b57 |
| files | |
| diffstat | 11 files changed, 132 insertions(+), 106 deletions(-) [+] |
line wrap: on
line diff
--- a/bin/gub Wed May 02 17:38:56 2007 +0200 +++ b/bin/gub Wed May 02 17:39:10 2007 +0200 @@ -29,8 +29,8 @@ from gub import cross ## fixme: double use of gub name. from gub import gubb +import gub.settings from gub import misc -import gub.settings from gub import locker from gub import gup @@ -91,8 +91,7 @@ dest='native_distcc_hosts', default=[], help='Add another native distcc host') - p.add_option ('-V', '--verbose', action='store_true', - dest='verbose') + p.add_option ('-v', '--verbose', action='count', dest='verbose', default=0) p.add_option ('--lilypond-versions', action='store', default='uploads/lilypond.versions', @@ -175,7 +174,7 @@ if options.fresh: try: - spec_obj.os_interface.log_command ('Removing status file') + spec_obj.os_interface.action ('Removing status filex') os.unlink (spec_obj.get_stamp_file ()) except OSError: pass @@ -189,8 +188,8 @@ tainted = True continue - spec.os_interface.log_command (' *** Stage: %s (%s)\n' - % (stage, spec.name ())) + spec.os_interface.stage (' *** Stage: %s (%s)\n' + % (stage, spec.name ())) if stage == 'package' and tainted and not options.force_package: msg = spec.expand ('''Compile was continued from previous run. @@ -205,7 +204,7 @@ to skip this check. ''') - spec.os_interface.log_command (msg) + spec.os_interface.error (msg) raise 'abort' @@ -280,7 +279,7 @@ if (settings.options.stage or not is_installable or not checksum_ok): - settings.os_interface.log_command ('building package: %s\n' % spec_name) + settings.os_interface.stage ('building package: %s\n' % spec_name) run_one_builder (settings.options, spec) # FIXME, spec_install should be stage? @@ -339,17 +338,6 @@ for spec_name in deps: spec_build (settings, manager, specs, spec_name) -def get_settings (options): - # FIXME, move (all these get_setting*) to constructors - settings = gub.settings.get_settings (options.platform) - settings.set_branches (options.branches) - settings.build_source = options.build_source - settings.cpu_count = options.cpu_count - settings.set_distcc_hosts (options) - settings.lilypond_versions = options.lilypond_versions - settings.options = options ##ugh - return settings - def inspect (settings, files): (names, specs) = gup.get_source_packages (settings, files) pm = gup.get_target_manager (settings) @@ -374,8 +362,7 @@ # FIXME: why do we only use deps for download, not for build? deps = gup.topologically_sorted (files, {}, get_all_deps, None) - if settings.options.verbose: - print 'deps:' + `deps` + settings.os_interface.info ('deps:' + `deps` + '\n') PATH = os.environ['PATH'] os.environ['PATH'] = settings.expand ('%(local_prefix)s/bin:' + PATH) @@ -405,7 +392,7 @@ cli_parser.print_help () sys.exit (2) - settings = get_settings (options) + settings = gub.settings.Settings (options) if options.inspect_key: inspect (settings, files)
--- a/bin/installer-builder Wed May 02 17:38:56 2007 +0200 +++ b/bin/installer-builder Wed May 02 17:39:10 2007 +0200 @@ -163,8 +163,7 @@ installer.checksum = pickle.dumps (checksum_list) def strip_installer (obj): - obj.log_command (' ** Stage: %s (%s)\n' - % ('strip', obj.name)) + obj.command (' ** Stage: %s (%s)\n' % ('strip', obj.name)) obj.strip () def package_installer (installer):
--- a/gub/context.py Wed May 02 17:38:56 2007 +0200 +++ b/gub/context.py Wed May 02 17:39:10 2007 +0200 @@ -122,7 +122,7 @@ def __init__ (self, settings): Context.__init__ (self, settings) self.os_interface = settings.os_interface - self.verbose = settings.verbose () + self.verbose = settings.options.verbose def file_sub (self, re_pairs, name, to_name=None, env={}, must_succeed=False, use_re=True): substs = [] @@ -140,7 +140,7 @@ def log_command (self, str, env={}): str = self.expand (str, env) - self.os_interface.log_command (str) + self.os_interface.command (str) def read_pipe (self, cmd, env={}, ignore_errors=False): dict = self.get_substitution_dict (env)
--- a/gub/cross.py Wed May 02 17:38:56 2007 +0200 +++ b/gub/cross.py Wed May 02 17:39:10 2007 +0200 @@ -135,7 +135,8 @@ cross_module_checksums = {} cross_module_cache = {} -def get_cross_module (platform): +def get_cross_module (settings): + platform = settings.platform if cross_module_cache.has_key (platform): return cross_module_cache[platform] @@ -143,12 +144,12 @@ desc = ('.py', 'U', 1) base = re.sub ('[-0-9].*', '', platform) - for name in base, platform: + for name in platform, base: file_name = 'gub/%(name)s.py' % locals () if os.path.exists (file_name): break file = open (file_name) - print 'module-name: ' + file_name + settings.os_interface.info ('module-name: ' + file_name + '\n') import imp module = imp.load_module (base, file, file_name, desc) @@ -158,11 +159,11 @@ return module def get_cross_packages (settings): - mod = get_cross_module (settings.platform) + mod = get_cross_module (settings) return mod.get_cross_packages (settings) def get_build_dependencies (settings): - mod = get_cross_module (settings.platform) + mod = get_cross_module (settings) return mod.get_cross_build_dependencies (settings) def get_cross_checksum (platform):
--- a/gub/gubb.py Wed May 02 17:38:56 2007 +0200 +++ b/gub/gubb.py Wed May 02 17:39:10 2007 +0200 @@ -72,9 +72,9 @@ class BuildSpec (Os_context_wrapper): def __init__ (self, settings): - Os_context_wrapper.__init__(self, settings) + Os_context_wrapper.__init__ (self, settings) - self.verbose = settings.verbose () + self.verbose = settings.verbose self.settings = settings self.url = '' self.has_source = True @@ -400,7 +400,7 @@ if lst: new = self.expand ('%(system_root)s/usr/bin/libtool') if not os.path.exists (new): - self.log_command ("Cannot update libtool: no such file: %(new)s" % locals ()) + self.error ('Cannot update libtool: no such file: %(new)s' % locals ()) raise 'barf' for i in lst: self.system ('cp %(new)s %(i)s', locals ()) @@ -778,7 +778,7 @@ def get_class_from_spec_file (settings, file_name, name): import imp - print 'reading spec', file_name + settings.os_interface.info ('reading spec: ' + file_name + '\n') file = open (file_name) desc = ('.py', 'U', 1) module = imp.load_module (name, file, file_name, desc)
--- a/gub/gup.py Wed May 02 17:38:56 2007 +0200 +++ b/gub/gup.py Wed May 02 17:39:10 2007 +0200 @@ -75,8 +75,8 @@ return self._package_file_db.has_key (name) def install_tarball (self, ball, name): - self.os_interface.log_command ('installing package %(name)s from %(ball)s\n' - % locals ()) + self.os_interface.action ('installing package %(name)s from %(ball)s\n' + % locals ()) flag = tar_compression_flag (ball) root = self.root @@ -86,7 +86,7 @@ for f in lst: if (self._file_package_db.has_key (f) and not os.path.isdir (self.root + '/' + f)): - print 'already have file %s: %s' % (f, self._file_package_db[f]) + self.os_interface.error ('already have file %s: %s\n' % (f, self._file_package_db[f])) conflicts = True if conflicts and not self.is_distro: @@ -128,7 +128,7 @@ '%(root)s/%(file)s' % locals ()) def uninstall_package (self, name): - self.os_interface.log_command ('uninstalling package: %s\n' % name) + self.os_interface.action ('uninstalling package: %s\n' % name) lst = self.installed_files (name) @@ -193,11 +193,11 @@ nm = d['split_name'] if (self._packages.has_key (nm)): if self._packages[nm]['spec_checksum'] != d['spec_checksum']: - self.os_interface.log_command ('******** checksum of %s changed!\n\n' % nm) + self.os_interface.info ('******** checksum of %s changed!\n\n' % nm) ## UGH ; need to look at installed hdr. if self._packages[nm]['cross_checksum'] != d['cross_checksum']: - self.os_interface.log_command ('******** checksum of cross changed for %s\n' % nm) + self.os_interface.info ('******** checksum of cross changed for %s\n' % nm) return self._packages[nm] = d @@ -205,7 +205,7 @@ def register_package_header (self, package_hdr, branch_dict): if self.verbose: - self.os_interface.log_command ('reading package header: %s\n' + self.os_interface.info ('reading package header: %s\n' % `package_hdr`) str = open (package_hdr).read () @@ -228,7 +228,7 @@ ## FIXME ? if self._package_dict_db.has_key (name): if str != self._package_dict_db[name]: - self.os_interface.log_command ("package header changed for %s\n" % name) + self.os_interface.info ("package header changed for %s\n" % name) return @@ -294,10 +294,9 @@ def install_package (self, name): if self.is_installed (name): return - self.os_interface.log_command ('installing package: %s\n' - % name) + self.os_interface.action ('installing package: %s\n' % name) if self._package_file_db.has_key (name): - print 'already have package ', name + self.os_interface.error ('already have package: ' + name + '\n') raise Exception ('abort') d = self._packages[name] ball = '%(split_ball)s' % d
--- a/gub/misc.py Wed May 02 17:38:56 2007 +0200 +++ b/gub/misc.py Wed May 02 17:39:10 2007 +0200 @@ -172,6 +172,7 @@ def system (cmd, ignore_errors=False): + #URG, go through oslog print 'Executing command %s' % cmd stat = os.system (cmd) if stat and not ignore_errors:
--- a/gub/oslog.py Wed May 02 17:38:56 2007 +0200 +++ b/gub/oslog.py Wed May 02 17:39:10 2007 +0200 @@ -10,62 +10,103 @@ def now (): return time.asctime (time.localtime ()) +level = {'quiet': 0, + 'error': 0, + 'stage': 0, + 'info': 1, + 'command': 2, + 'action': 2, + 'output': 3, + 'debug': 4} + class Os_commands: + level = level + '''Encapsulate OS/File system commands for proper logging.''' - def __init__ (self, logfile): + def __init__ (self, logfile, verbose): + self.verbose = verbose self.log_file = open (logfile, 'a') self.log_file.write ('\n\n * Starting build: %s\n' % now ()) - ## TODO: ## capture complete output of CMD, by polling output, and copying to tty. - def system_one (self, cmd, env, ignore_errors): + def system_one (self, cmd, env, ignore_errors, verbose=None): '''Run CMD with environment vars ENV.''' - self.log_command ('invoking %s\n' % cmd) + if not verbose: + verbose = self.verbose - proc = subprocess.Popen (cmd, shell=True, env=env, - stderr=subprocess.STDOUT) + self.log ('invoking %s\n' % cmd, level['command'], verbose) + + log_file = open ('system.log', 'wa') - stat = proc.wait() + proc = subprocess.Popen (cmd, bufsize=1, shell=True, env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + close_fds=True) - if stat and not ignore_errors: + while proc.poll () is None: + line = proc.stdout.readline () + self.log (line, level['output'], verbose) + time.sleep (0.0001) + + if proc.returncode and not ignore_errors: m = 'Command barfed: %s\n' % cmd - self.log_command (m) - + self.error (m) raise SystemFailed (m) return 0 - def log_command (self, str): - '''Write STR in the build log.''' - - sys.stderr.write (str) + def log (self, str, threshold, verbose=None): + if not verbose: + verbose = self.verbose + if verbose >= threshold: + sys.stderr.write (str) if self.log_file: self.log_file.write (str) self.log_file.flush () - def system (self, cmd, env={}, ignore_errors=False, verbose=False): + # FIXME + def action (self, str): + self.log (str, level['action'], self.verbose) + + def stage (self, str): + self.log (str, level['stage'], self.verbose) + + def error (self, str): + self.log (str, level['error'], self.verbose) + + def info (self, str): + self.log (str, level['info'], self.verbose) + + def command (self, str): + self.log (str, level['command'], self.verbose) + + def debug (self, str): + self.log (str, level['debug'], self.verbose) + + def system (self, cmd, env={}, ignore_errors=False, verbose=None): '''Run os commands, and run multiple lines as multiple commands. ''' - + if not verbose: + verbose = self.verbose call_env = os.environ.copy () call_env.update (env) if verbose: - keys = env.keys() + keys = env.keys () keys.sort() for k in keys: - sys.stderr.write ('%s=%s\n' % (k, env[k])) + self.log ('%s=%s\n' % (k, env[k]), level['debug'], verbose) + self.log ('export %s\n' % ' '.join (keys), level['debug'], + verbose) - sys.stderr.write ('export %s\n' % ' '.join (keys)) for i in cmd.split ('\n'): if i: - self.system_one (i, call_env, ignore_errors) - + self.system_one (i, call_env, ignore_errors, verbose=verbose) return 0 def dump (self, str, name, mode='w'): @@ -73,7 +114,7 @@ if not os.path.exists (dir): self.system ('mkdir -p %s' % dir) - self.log_command ("Writing %s (%s)\n" % (name, mode)) + self.action ('Writing %s (%s)\n' % (name, mode)) f = open (name, mode) f.write (str) @@ -85,9 +126,9 @@ If TO_NAME is specified, the output is sent to there. ''' - self.log_command ('substituting in %s\n' % name) - self.log_command (''.join (map (lambda x: "'%s' -> '%s'\n" % x, - re_pairs))) + self.action ('substituting in %s\n' % name) + self.command (''.join (map (lambda x: "'%s' -> '%s'\n" % x, + re_pairs))) s = open (name).read () t = s @@ -116,7 +157,7 @@ def read_pipe (self, cmd, ignore_errors=False, silent=False): if not silent: - self.log_command ('Reading pipe: %s\n' % cmd) + self.action ('Reading pipe: %s\n' % cmd) pipe = os.popen (cmd, 'r') output = pipe.read () @@ -133,12 +174,10 @@ target = os.path.abspath (target) src = os.path.abspath (src) - self.log_command ("Shadowing %s to %s\n" % (src, target)) + self.action ('Shadowing %s to %s\n' % (src, target)) os.makedirs (target) - (root, dirs, files) = os.walk(src).next() + (root, dirs, files) = os.walk (src).next () for f in files: - os.symlink (os.path.join (root, f), - os.path.join (target, f)) + os.symlink (os.path.join (root, f), os.path.join (target, f)) for d in dirs: - self.shadow_tree (os.path.join (root, d), - os.path.join (target, d)) + self.shadow_tree (os.path.join (root, d), os.path.join (target, d))
--- a/gub/repository.py Wed May 02 17:38:56 2007 +0200 +++ b/gub/repository.py Wed May 02 17:39:10 2007 +0200 @@ -33,6 +33,7 @@ class Repository: def __init__ (self): + #URG, go through oslog self.system = misc.system self.read_pipe = misc.read_pipe
--- a/gub/settings.py Wed May 02 17:38:56 2007 +0200 +++ b/gub/settings.py Wed May 02 17:39:10 2007 +0200 @@ -29,17 +29,36 @@ distros = ('debian-arm', 'cygwin', 'debian', 'mipsel') class Settings (Context): - def __init__ (self, platform): + def __init__ (self, options): Context.__init__ (self) - self.platform = platform - self.os = re.sub ('[-0-9].*', '', platform) + self.platform = options.platform + + if self.platform not in platforms.keys (): + raise 'unknown platform', self.platform + + self.target_gcc_flags = '' + if self.platform == 'darwin-ppc': + self.target_gcc_flags = '-D__ppc__' + elif self.platform == 'mingw': + self.target_gcc_flags = '-mwindows -mms-bitfields' + + self.set_branches (options.branches) + self.build_source = options.build_source + self.cpu_count = options.cpu_count + self.set_distcc_hosts (options) + self.lilypond_versions = options.lilypond_versions + self.options = options ##ugh + + #urg + self.verbose = self.options.verbose + + self.os = re.sub ('[-0-9].*', '', self.platform) self.target_architecture = platforms[self.platform] self.cpu = self.target_architecture.split ('-')[0] self.build_source = False - self.is_distro = platform in distros + self.is_distro = self.platform in distros - self.target_gcc_flags = '' self.topdir = os.getcwd () self.logdir = self.topdir + '/log' self.downloads = self.topdir + '/downloads' @@ -101,7 +120,8 @@ os.mkdir ('log') self.os_interface = oslog.Os_commands ('log/build-%s.log' - % self.target_architecture) + % self.target_architecture, + self.options.verbose) self.create_dirs () self.build_architecture = self.os_interface.read_pipe ('gcc -dumpmachine', silent=True)[:-1] @@ -114,12 +134,6 @@ ## make sure we don't confuse build or target system. self.LD_LIBRARY_PATH = '%(system_root)s/' - def verbose (self): - try: - return self.options.verbose - except AttributeError: - return False - def create_dirs (self): for a in ( 'downloads', @@ -167,18 +181,3 @@ self.branch_dict[name] = br self.__dict__['%s_branch' % name]= br - -def get_settings (platform): - settings = Settings (platform) - - if platform not in platforms.keys (): - raise 'unknown platform', platform - - if platform == 'darwin-ppc': - settings.target_gcc_flags = '-D__ppc__' - elif platform == 'mingw': - settings.target_gcc_flags = '-mwindows -mms-bitfields' - - return settings - -
--- a/gub/targetpackage.py Wed May 02 17:38:56 2007 +0200 +++ b/gub/targetpackage.py Wed May 02 17:39:10 2007 +0200 @@ -163,7 +163,7 @@ """ package = gubb.get_build_spec (TargetBuildSpec, settings, url) - crossmod = cross.get_cross_module (settings.platform) + crossmod = cross.get_cross_module (settings) crossmod.change_target_package (package) return package