view gub/buildrunner.py @ 5457:f65d6e563f9f

/GUB: slew of bootstrap/root fixes.
author Jan Nieuwenhuizen <janneke@gnu.org>
date Thu, 13 Aug 2009 14:59:39 +0200
parents 4c5efe7c96e7
children df03d0971b5d
line wrap: on
line source

"""
    Copyright (c) 2005--2007
    Jan Nieuwenhuizen <janneke@gnu.org>
    Han-Wen Nienhuys <hanwen@xs4all.nl>

    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 2, or (at your option)
    any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program; if not, write to the Free Software
    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
"""

import datetime
import difflib
import pickle
import os
import sys
import time
#
from gub.syntax import printf
from gub import cross
from gub import build
from gub import misc
from gub import gup
from gub import loggedos
from gub import logging
from gub import runner
import gub.settings   # otherwise naming conflict with settings local vars.


def checksum_diff (a, b, fromfile='', tofile='',
                   fromfiledate='', tofiledate=''):
    return '\n'.join (difflib.unified_diff (a.split ('\n'),
                                            b.split ('\n'),
                                            fromfile,
                                            tofile,
                                            fromfiledate,
                                            tofiledate))

# FIXME s/spec/build/, but we also have two definitions of package/pkg
# here: sub packages and name of global package under build

#FIXME: split spec_* into SpecBuiler?
class BuildRunner:
    def __init__ (self, manager, settings, options, specs):
        info = logging.default_logger.harmless
        info.write ('MANAGER:' + settings.platform)
        self.managers = {settings.platform : manager }
        self.settings = settings
        self.options = options
        self.specs = specs

        # spec name -> string
        self.checksums = dict ()
        self.failed_checksums = dict ()

        # PATH = os.environ['PATH']
        # cross_prefix is also necessary for building cross packages, such as GCC
        # yes, so we set that in cross.AutoBuild.get_substitution_dict ()
        # we cannot do it here, as this will break all tools::* checksums
        # going from mingw::libtool to darwin-x86::libtool
        # os.environ['PATH'] = self.settings.expand ('%(cross_prefix)s/bin:' + PATH, locals ())
        self.add_packages_to_manager (self.specs)

    def manager (self, platform):
        if platform not in self.managers:
            info = logging.default_logger.harmless
            info.write ('MANAGER for platform' + platform)
            settings = gub.settings.Settings (platform)
            self.managers[platform] = gup.DependencyManager (settings.system_root)
        return self.managers[platform]
        
    def add_packages_to_manager (self, package_object_dict):
        ## Ugh, this sucks: we now have to have all packages
        ## registered at the same time.
        for spec in list (package_object_dict.values ()):
            for package in spec.get_packages ():
                self.manager (package.platform ()).register_package_dict (package.dict ())

    def calculate_checksums (self):
        logging.verbose ('calculating checksums\n')
        for spec in list (self.specs.values ()):
            name = spec.platform_name ()
            logger = logging.NullCommandLogger ()

            command_runner = runner.DeferredRunner (logger)
            spec.connect_command_runner (command_runner)
            spec.build (skip=['download'])
            spec.connect_command_runner (None)

            self.checksums[name] = command_runner.checksum ()
            reason = self.spec_checksums_fail_reason (spec)
            if reason:
                self.failed_checksums[name] = reason

    # FIXME: move to gup.py or to build.py?
    def spec_checksums_fail_reason (self, spec):
        # need to read package header to read checksum_file.  since
        # checksum is per buildspec, only need to inspect one package.
        pkg = spec.get_packages ()[0]    
        name = pkg.name ()
        pkg_dict = self.manager (pkg.platform ()).package_dict (name)

        checksum_file = pkg_dict['checksum_file']
        try:
            build_checksum_ondisk = open (pkg_dict['checksum_file']).read ()
            checksum_time = time.ctime (os.stat (checksum_file).st_mtime)
        except IOError:
            build_checksum_ondisk = '0000'
            checksum_time = '0000'

        # fixme: spec.build_checksum () should be method.
        reason = ''
        hdr = pkg.expand ('%(split_hdr)s')
        if spec.install_after_build:
            if spec.source_checksum () != pkg_dict['source_checksum']:
                reason = 'source %s -> %s (memory)' % (spec.source_checksum (), pkg_dict['source_checksum'])

        if reason == '' and self.checksums[spec.platform_name ()] != build_checksum_ondisk:
            failure = 'diff'
            spec_name = spec.name ()
            spec_platform = spec.platform ()
            diff = checksum_diff (build_checksum_ondisk,
                                  self.checksums[spec.platform_name ()],
                                  checksum_file,
                                  'THIS BUILD',
                                  checksum_time,
                                  time.ctime (time.time ()))
            message = '\n' + diff
            reason = '\n *** Checksum mismatch: %(failure)s (%(spec_name)s, %(spec_platform)s)%(message)s\n' % locals ()

        if spec.install_after_build:
            if not reason and not os.path.exists (hdr):
                reason = 'no such file: header: %(hdr)s' % locals ()
            elif not reason:
                hdr_dict = dict (pickle.load (open (hdr, 'rb')))
                if spec.source_checksum () != hdr_dict['source_checksum']:
                    reason = 'source %s -> %s (disk)' % (spec.source_checksum (), hdr_dict['source_checksum'])

        # we don't use cross package checksums, otherwise we have to
        # rebuild everything for every cross package change.
        return reason

    # FIXME: this should be in gpkg/gup.py otherwise it's impossible
    # to install packages in a conflict situation manually
    def spec_conflict_resolution (self, spec, pkg):
        pkg_name = pkg.name ()
        install_candidate = pkg
        subname = ''
        if spec.name () != pkg_name:
            subname = pkg_name.split ('-')[-1]
        manager = self.manager (spec.platform ())
        if subname in spec.get_conflict_dict ():
            for c in spec.get_conflict_dict ()[subname]:
                if manager.is_installed (c):
                    printf ('  %(c)s conflicts with %(pkg_name)s' % locals ())
                    conflict_source = manager.source_name (c)
                    # FIXME: implicit provides: foo-* provides foo-core,
                    # should implement explicit provides
                    if conflict_source + '-core' == pkg_name:
                        printf ('    non-core %(conflict_source)s already installed'
                               % locals ())
                        printf ('      skipping request to install %(pkg_name)s'
                               % locals ())
                        install_candidate = None
                        continue
                    printf ('    removing %(c)s' % locals ())
                    manager.uninstall_package (c)
        return install_candidate

    def pkg_install (self, spec, pkg):
        manager = self.manager (spec.platform ())
        if not manager.is_installed (pkg.name ()):
            install_candidate = self.spec_conflict_resolution (spec, pkg)
            if install_candidate:
                manager.unregister_package_dict (install_candidate.name ())
                manager.register_package_dict (install_candidate.dict ())
                manager.install_package (install_candidate.name ())

    def spec_install (self, spec):
        for pkg in spec.get_packages ():
            self.pkg_install (spec, pkg)

    def get_skip_stages (self):
        """Returns list of stages (strings) to be skipped.

        Uses command line options as input.
        """
        skip = []
        if self.options.offline:
            skip += ['download']
        if not self.options.src_package:
            skip += ['src_package']
        if self.options.keep_build:
            skip += ['clean']
        return skip
    
    def spec_is_installable (self, spec):
        return misc.forall (self.manager (p.platform ()).is_installable (p.name ())
                            for p in spec.get_packages ())

    def spec_all_installed (self, spec):
        all_installed = True
        for p in spec.get_packages ():
            all_installed = (all_installed
                             and self.manager (p.platform ()).is_installed (p.name ()))
        return all_installed

    def spec_build (self, spec_name):
        spec = self.specs[spec_name]
        if self.spec_all_installed (spec):
            return
        checksum_fail_reason = self.failed_checksums.get (spec_name, '')
        if ((not checksum_fail_reason or self.options.lax_checksums)
            and not spec.install_after_build):
            return
        logger = logging.default_logger
        if checksum_fail_reason:
            rebuild = 'must'
            if self.options.lax_checksums:
                rebuild = 'should'
            logger.write_log ('%(rebuild)s rebuild: %(spec_name)s\n' % locals (), 'verbose')
        else:
            logger.write_log ('checksum ok: %(spec_name)s\n' % locals (), 'verbose')

        if logging.get_numeric_loglevel ('verbose') > logger.threshold:
            logger.write_log ('\n'.join (checksum_fail_reason.split ('\n')[:10]), 'verbose')
        logger.write_log (checksum_fail_reason, 'output')

        if ((checksum_fail_reason and not self.options.lax_checksums)
            or not self.spec_is_installable (spec)):
            deferred_runner = runner.DeferredRunner (logger)
            spec.connect_command_runner (deferred_runner)
            spec.runner.stage ('building package: %s\n' % spec_name)
            skip = self.get_skip_stages () + spec.get_done ()
            skip = [x for x in skip if x != self.options.stage]
            
            spec.build (self.options, skip)
            spec.connect_command_runner (None)
            deferred_runner.execute_deferred_commands ()
            checksum_file = spec.expand ('%(checksum_file)s')
            if checksum_file:
                if 0: #len (self.checksums[spec_name].split ('\n')) < 5:
                    # Sanity check.  This can't be right.  Do not
                    # overwrite precious [possibly correct] checksum.
                    raise Exception ('BROKEN CHECKSUM:' + self.checksums[spec_name])
                open (checksum_file, 'w').write (self.checksums[spec_name])
            #spec.set_done ('')
            loggedos.system (logging.default_logger, spec.expand ('rm -f %(stamp_file)s'))
        # Ugh, pkg_install should be stage
        if spec.install_after_build and not self.spec_all_installed (spec):
            logger.write_log (spec.stage_message ('pkg_install'), 'stage')
            self.spec_install (spec)
        logging.default_logger.write_log ('\n', 'stage')

    def is_installed_spec (self, spec_name):
        spec = self.specs[spec_name]
        for pkg in spec.get_packages ():
            if self.manager (pkg.platform ()).is_installed (pkg.name ()):
                return True
        return False

    def is_outdated_spec (self, spec_name):
        spec = self.specs[spec_name]
        checksum_fail_reason = self.failed_checksums.get (spec_name, '')
        checksum_ok = '' == checksum_fail_reason
        for pkg in spec.get_packages ():
            if (not self.manager (pkg.platform ()).is_installable (pkg.name ())
                or not checksum_ok):
                return True
        return False

    def uninstall_spec (self, spec):
        for pkg in spec.get_packages ():
            if (self.manager (pkg.platform ()).is_installed (pkg.name ())):
                self.manager (pkg.platform ()).uninstall_package (pkg.name ())

    def outdated_names (self, deps):
        return [name for name in deps
                if (self.is_outdated_spec (name)
                    and not (self.options.lax_checksums
                             and self.spec_is_installable (self.specs[name])))]

    def uninstall_specs (self, lst):
        for name in lst:
            self.uninstall_spec (self.specs[name])

    def build_source_packages (self, names):
        deps = [d for d in names if d in self.specs]
        platform = self.settings.platform
        outdated = self.outdated_names (deps)
        # fail_str: keep ordering of names
        fail_str = (' '.join ([s for s in deps if s in outdated ])
                    .replace (misc.with_platform ('', platform), ''))
        if not fail_str:
            fail_str = '<nothing to be done>.'
        logging.default_logger.write_log ('must rebuild[%(platform)s]: %(fail_str)s\n' % locals (), 'stage')
        outdated_installed = [x for x in list (reversed (outdated))
                              if self.is_installed_spec (x)]
        if outdated_installed:
            platform = self.settings.platform
            outdated_str = (' '.join (outdated_installed)
                            .replace (misc.with_platform ('', platform), ''))
            logging.default_logger.write_log ('removing outdated[%(platform)s]: %(outdated_str)s\n' % locals (), 'stage')
            if not 'BOOTSTRAP' in os.environ.keys ():
                self.uninstall_specs (outdated_installed)
        global target
        for spec_name in deps:
            target = spec_name
            self.spec_build (spec_name)
        target = None

target = None

def main ():
    boe

if __name__ == '__main__':
    main ()