view gub/buildrunner.py @ 6512:ccc20ae889ca default tip guix

mingw::guile-2.0.7 builds.
author Jan Nieuwenhuizen <janneke@gnu.org>
date Thu, 24 Mar 2016 08:03:39 +0100
parents 72b46c22bf99
children
line wrap: on
line source

"""
    Copyright (c) 2005--2007
    Jan Nieuwenhuizen <janneke@gnu.org>
    Han-Wen Nienhuys <hanwen@xs4all.nl>

    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 2, or (at your option)
    any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program; if not, write to the Free Software
    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
"""

import datetime
import difflib
import pickle
import os
import sys
import time
#
from gub.syntax import printf
from gub import cross
from gub import build
from gub import misc
from gub import gup
from gub import loggedos
from gub import gub_log
from gub import runner
import gub.settings   # otherwise naming conflict with settings local vars.


def checksum_diff (a, b, fromfile='', tofile='',
                   fromfiledate='', tofiledate=''):
    return '\n'.join (difflib.unified_diff (a.split ('\n'),
                                            b.split ('\n'),
                                            fromfile,
                                            tofile,
                                            fromfiledate,
                                            tofiledate))

logger = gub_log.default_logger

# FIXME s/spec/build/, but we also have two definitions of package/pkg
# here: sub packages and name of global package under build

#FIXME: split spec_* into SpecBuiler?
class BuildRunner:
    def __init__ (self, manager, settings, options, specs):
        info = gub_log.default_logger.harmless
        info.write ('MANAGER: ' + settings.platform + '\n')
        self.managers = {settings.platform : manager }
        self.settings = settings
        self.options = options
        self.specs = specs

        # spec name -> string
        self.checksums = dict ()
        self.failed_checksums = dict ()

        # PATH = os.environ['PATH']
        # cross_prefix is also necessary for building cross packages, such as GCC
        # yes, so we set that in cross.AutoBuild.get_substitution_dict ()
        # we cannot do it here, as this will break all tools::* checksums
        # going from mingw::libtool to darwin-x86::libtool
        # os.environ['PATH'] = self.settings.expand ('%(cross_prefix)s/bin:' + PATH, locals ())
        self.add_packages_to_manager (self.specs)

    def manager (self, platform):
        if platform not in self.managers:
            info = gub_log.default_logger.harmless
            info.write ('MANAGER for platform: ' + platform + '\n')
            settings = gub.settings.Settings (platform)
            self.managers[platform] = gup.DependencyManager (settings.system_root)
        return self.managers[platform]
        
    def add_packages_to_manager (self, package_object_dict):
        ## Ugh, this sucks: we now have to have all packages
        ## registered at the same time.
        for spec in list (package_object_dict.values ()):
            for package in spec.get_packages ():
                self.manager (package.platform ()).register_package_dict (package.dict ())

    def calculate_checksums (self):
        gub_log.verbose ('calculating checksums\n')
        for spec in list (self.specs.values ()):
            name = spec.platform_name ()
            logger = gub_log.NullCommandLogger ()

            command_runner = runner.DeferredRunner (logger)
            spec.connect_command_runner (command_runner)
            spec.build (skip=['download'])
            spec.connect_command_runner (None)

            self.checksums[name] = command_runner.checksum ()
            reason = self.spec_checksums_fail_reason (spec)
            if reason:
                self.failed_checksums[name] = reason

    # FIXME: move to gup.py or to build.py?
    def spec_checksums_fail_reason (self, spec):
        # need to read package header to read checksum_file.  since
        # checksum is per buildspec, only need to inspect one package.
        pkg = spec.get_packages ()[0]    
        name = pkg.name ()
        pkg_dict = self.manager (pkg.platform ()).package_dict (name)

        checksum_file = pkg_dict['checksum_file']
        try:
            build_checksum_ondisk = open (pkg_dict['checksum_file']).read ()
            checksum_time = time.ctime (os.stat (checksum_file).st_mtime)
        except IOError:
            build_checksum_ondisk = '0000'
            checksum_time = '0000'

        # fixme: spec.build_checksum () should be method.
        reason = ''
        hdr = pkg.expand ('%(split_hdr)s')
        if spec.install_after_build:
            if spec.source_checksum () != pkg_dict['source_checksum']:
                reason = 'source %s -> %s (memory)' % (spec.source_checksum (), pkg_dict['source_checksum'])

        if reason == '' and self.checksums[spec.platform_name ()] != build_checksum_ondisk:
            failure = 'diff'
            spec_name = spec.name ()
            spec_platform = spec.platform ()
            diff = checksum_diff (build_checksum_ondisk,
                                  self.checksums[spec.platform_name ()],
                                  checksum_file,
                                  'THIS BUILD',
                                  checksum_time,
                                  time.ctime (time.time ()))
            message = '\n' + diff
            reason = '\n *** Checksum mismatch: %(failure)s (%(spec_name)s, %(spec_platform)s)%(message)s\n' % locals ()

        if spec.install_after_build:
            if not reason and not os.path.exists (hdr):
                reason = 'no such file: header: %(hdr)s' % locals ()
            elif not reason:
                hdr_dict = dict (pickle.load (open (hdr, 'rb')))
                if spec.source_checksum () != hdr_dict['source_checksum']:
                    reason = 'source %s -> %s (disk)' % (spec.source_checksum (), hdr_dict['source_checksum'])

        # we don't use cross package checksums, otherwise we have to
        # rebuild everything for every cross package change.
        return reason

    # FIXME: this should be in gpkg/gup.py otherwise it's impossible
    # to install packages in a conflict situation manually
    def spec_conflict_resolution (self, spec, pkg):
        pkg_name = pkg.name ()
        install_candidate = pkg
        subname = ''
        if spec.name () != pkg_name:
            subname = pkg_name.split ('-')[-1]
        manager = self.manager (spec.platform ())
        if subname in spec.get_conflict_dict ():
            for c in spec.get_conflict_dict ()[subname]:
                if manager.is_installed (c):
                    printf ('  %(c)s conflicts with %(pkg_name)s' % locals ())
                    conflict_source = manager.source_name (c)
                    # FIXME: implicit provides: foo-* provides foo-core,
                    # should implement explicit provides
                    if conflict_source + '-core' == pkg_name:
                        printf ('    non-core %(conflict_source)s already installed'
                               % locals ())
                        printf ('      skipping request to install %(pkg_name)s'
                               % locals ())
                        install_candidate = None
                        continue
                    printf ('    removing %(c)s' % locals ())
                    manager.uninstall_package (c)
        return install_candidate

    def pkg_install (self, spec, pkg):
        manager = self.manager (spec.platform ())
        if not manager.is_installed (pkg.name ()):
            install_candidate = self.spec_conflict_resolution (spec, pkg)
            if install_candidate:
                manager.unregister_package_dict (install_candidate.name ())
                manager.register_package_dict (install_candidate.dict ())
                manager.install_package (install_candidate.name ())

    def spec_install (self, spec):
        for pkg in spec.get_packages ():
            self.pkg_install (spec, pkg)

    def get_skip_stages (self):
        """Returns list of stages (strings) to be skipped.

        Uses command line options as input.
        """
        skip = []
        if self.options.offline:
            skip += ['download']
        if not self.options.build_source:
            skip += ['src_package']
        if self.options.keep_build:
            skip += ['clean']
        return skip
    
    def spec_is_installable (self, spec):
        return misc.forall (self.manager (p.platform ()).is_installable (p.name ())
                            for p in spec.get_packages ())

    def spec_all_installed (self, spec):
        all_installed = True
        for p in spec.get_packages ():
            all_installed = (all_installed
                             and self.manager (p.platform ()).is_installed (p.name ()))
        return all_installed

    def spec_build (self, spec_name):
        spec = self.specs[spec_name]
        if self.spec_all_installed (spec):
            return
        checksum_fail_reason = self.failed_checksums.get (spec_name, '')
        if ((not checksum_fail_reason or self.options.lax_checksums)
            and not spec.install_after_build):
            return
        global logger
        if self.options.log == 'build':
            # This is expecially broken with multi-platform builds...
            logger = gub_log.default_logger
        else:
            if self.options.log == 'platform':
                log = os.path.join (spec.settings.logdir, 'build.log')
            else:
                log = os.path.join (spec.settings.logdir,
                                    misc.strip_platform (spec_name)) + '.log'
            if os.path.isfile (log):
                misc.rename_append_time (log)
            logger = gub_log.CommandLogger (log, gub_log.default_logger.threshold)
        if checksum_fail_reason:
            rebuild = 'must'
            if self.options.lax_checksums:
                rebuild = 'should'
            logger.write_log ('%(rebuild)s rebuild: %(spec_name)s\n' % locals (), 'verbose')
        else:
            logger.write_log ('checksum ok: %(spec_name)s\n' % locals (), 'verbose')

        if gub_log.get_numeric_loglevel ('verbose') > logger.threshold:
            logger.write_log ('\n'.join (checksum_fail_reason.split ('\n')[:10]), 'verbose')
        logger.write_log (checksum_fail_reason, 'output')

        if ((checksum_fail_reason and not self.options.lax_checksums)
            or not self.spec_is_installable (spec)):
            deferred_runner = runner.DeferredRunner (logger)
            spec.connect_command_runner (deferred_runner)
            spec.runner.stage ('building package: %s\n' % spec_name)
            skip = self.get_skip_stages () ### + spec.get_done ()
            skip = [x for x in skip if x != self.options.stage]
            
            spec.build (self.options, skip)
            spec.connect_command_runner (None)
            deferred_runner.execute_deferred_commands ()
            checksum_file = spec.expand ('%(checksum_file)s')
            if checksum_file:
                if len (self.checksums[spec_name].split ('\n')) < 5:
                    # Sanity check.  This can't be right.  Do not
                    # overwrite precious [possibly correct] checksum.
                    raise Exception ('BROKEN CHECKSUM:' + self.checksums[spec_name])
                if os.path.isfile (checksum_file):
                    misc.rename_append_time (checksum_file)
                open (checksum_file, 'w').write (self.checksums[spec_name])
            loggedos.system (gub_log.default_logger, spec.expand ('rm -f %(stamp_file)s'))
        # Ugh, pkg_install should be stage
        if spec.install_after_build and not self.spec_all_installed (spec):
            logger.write_log (spec.stage_message ('pkg_install'), 'stage')
            self.spec_install (spec)
        gub_log.default_logger.write_log ('\n', 'stage')

    def is_installed_spec (self, spec_name):
        spec = self.specs[spec_name]
        for pkg in spec.get_packages ():
            if self.manager (pkg.platform ()).is_installed (pkg.name ()):
                return True
        return False

    def is_outdated_spec (self, spec_name):
        spec = self.specs[spec_name]
        checksum_fail_reason = self.failed_checksums.get (spec_name, '')
        checksum_ok = '' == checksum_fail_reason
        for pkg in spec.get_packages ():
            if (not self.manager (pkg.platform ()).is_installable (pkg.name ())
                or not checksum_ok):
                return True
        return False

    def uninstall_spec (self, spec):
        for pkg in spec.get_packages ():
            if (self.manager (pkg.platform ()).is_installed (pkg.name ())):
                self.manager (pkg.platform ()).uninstall_package (pkg.name ())

    def outdated_names (self, deps):
        return [name for name in deps
                if (self.is_outdated_spec (name)
                    and not (self.options.lax_checksums
                             and self.spec_is_installable (self.specs[name])))]

    def uninstall_specs (self, lst):
        for name in lst:
            self.uninstall_spec (self.specs[name])

    def build_source_packages (self, names):
        deps = [d for d in names if d in self.specs]
        platform = self.settings.platform
        outdated = self.outdated_names (deps)
        # fail_str: keep ordering of names
        fail_str = (' '.join ([s for s in deps if s in outdated ])
                    .replace (misc.with_platform ('', platform), ''))
        if not fail_str:
            fail_str = '<nothing to be done>.'
        gub_log.default_logger.write_log ('must rebuild[%(platform)s]: %(fail_str)s\n' % locals (), 'stage')
        if self.options.dry_run:
            sys.exit (0)
        outdated_installed = [x for x in list (reversed (outdated))
                              if self.is_installed_spec (x)]
        if outdated_installed:
            platform = self.settings.platform
            outdated_str = (' '.join (outdated_installed)
                            .replace (misc.with_platform ('', platform), ''))
            gub_log.default_logger.write_log ('removing outdated[%(platform)s]: %(outdated_str)s\n' % locals (), 'stage')
            self.uninstall_specs (outdated_installed)
        global target
        for spec_name in deps:
            target = spec_name
            self.spec_build (spec_name)
            logger = gub_log.default_logger
        target = None

target = None

def main ():
    boe

if __name__ == '__main__':
    main ()