Mercurial > gnulib
view pygnulib.py @ 39101:915fae496f3c
refactored generators; bugfix; tests Makefile.am
author | Dmitry Selyutin <ghostmansd@gmail.com> |
---|---|
date | Mon, 29 Jan 2018 19:07:42 +0300 |
parents | ba23c0ff778e |
children | 7d3f5166773a |
line wrap: on
line source
#!/usr/bin/python # encoding: UTF-8 import codecs import os import re import stat import sys import tempfile import traceback import subprocess as sp from pygnulib.error import CommandLineError from pygnulib.error import UnknownModuleError from pygnulib.config import BaseConfig from pygnulib.config import CachedConfig from pygnulib.generator import gnulib_cache from pygnulib.generator import gnulib_comp from pygnulib.generator import lib_makefile from pygnulib.generator import po_make_vars from pygnulib.generator import tests_makefile from pygnulib.module import DummyModule from pygnulib.module import Database from pygnulib.parser import CommandLine as CommandLineParser from pygnulib.tools import Executable from pygnulib.vfs import BaseVFS from pygnulib.vfs import GnulibGitVFS from pygnulib.vfs import backup as vfs_backup from pygnulib.vfs import compare as vfs_compare from pygnulib.vfs import copy as vfs_copy from pygnulib.vfs import exists as vfs_exists from pygnulib.vfs import hardlink as vfs_hardlink from pygnulib.vfs import lookup as vfs_lookup from pygnulib.vfs import move as vfs_move from pygnulib.vfs import iostream as vfs_iostream from pygnulib.vfs import symlink as vfs_symlink from pygnulib.vfs import unlink as vfs_unlink class GnulibExecutable(Executable): def __init__(self, name, encoding=None, shell_name=None, shell_path=None): path = None if shell_name is None: shell_name = shell_name.upper() if shell_path is None: shell_path = "{}PATH".format(shell_name) environ = dict(ENVIRON) environ.update(os.environ) if shell_name in environ: path = shell_name elif shell_path in environ: path = "{}{}".format(shell_path) super().__init__(name, path) AC_VERSION_PATTERN = re.compile(r"AC_PREREQ\(\[(.*?)\]\)", re.S | re.M) IGNORED_LICENSES = { "GPLed build tool", "public domain", "unlimited", "unmodifiable license text", } TRANSFER_MODES = { None: vfs_copy, "hardlink": vfs_hardlink, "symlink": vfs_symlink, } SUBSTITUTION = { "build-aux": "auxdir", "doc": "doc_base", "lib": "source_base", "m4": "m4_base", "tests": "tests_base", "tests=lib": "tests_base", "po": "po_base", "top": "", } TP_URL = "http://translationproject.org/latest/" TP_RSYNC_URI = "translationproject.org::tp/latest/" def list_hook(gnulib, *args, **kwargs): (_, _) = (args, kwargs) for module in sorted(gnulib.modules()): print(module.name, file=sys.stdout) return os.EX_OK def extract_hook(program, gnulib, mode, namespace, *args, **kwargs): (_, _) = (args, kwargs) mode = mode.replace("extract-", "") mode += "s" if mode == "maintainer" else "" for module in namespace["modules"]: try: module = gnulib.module(module) print(module[mode], file=sys.stdout) except UnknownModuleError as error: print("{0}:".format(program), "warning:", error, file=sys.stderr) return os.EX_OK def import_hook(script, gnulib, namespace, explicit, verbosity, options, *args, **kwargs): (_, _) = (args, kwargs) config = BaseConfig(**namespace) try: cache = CachedConfig(root=config.root) except FileNotFoundError: cache = BaseConfig(**config) cache.files = set() cache.ac_version = "2.59" for key in {"ac_version", "files"}: if key not in namespace: config[key] = cache[key] database = Database(gnulib.module, config) # Print some information about modules. print("Module list with included dependencies (indented):", file=sys.stdout) BOLD_ON = BOLD_OFF = "" mode = os.fstat(sys.stdout.fileno()).st_mode if not (stat.S_ISFIFO(mode) or stat.S_ISREG(mode)): BOLD_ON = "\033[1m" BOLD_OFF = "\033[0m" for module in database.final_modules: manual = module in database.explicit_modules prefix = " " if manual else " " bold_on = BOLD_ON if manual else "" bold_off = BOLD_OFF if manual else "" print("{0}{1}{2}{3}".format(prefix, bold_on, module.name, bold_off), file=sys.stdout) if verbosity >= 1: print("Main module list:", file=sys.stdout) for module in database.main_modules: if module is not DummyModule(): print(" {0}".format(module.name), file=sys.stdout) if database.main_modules: print("", file=sys.stdout) print("Tests-related module list:", file=sys.stdout) for module in database.test_modules: print(" {0}".format(module.name), file=sys.stdout) if database.test_modules: print("", file=sys.stdout) # Determine license incompatibilities, if any. incompatibilities = set() if config.licenses & {"LGPLv2", "LGPLv2+", "LGPLv3", "LGPLv3+"}: acceptable = IGNORED_LICENSES | config.licenses for (name, licenses) in ((module.name, module.licenses) for module in main): if not (acceptable & licenses): incompatibilities.add((name, licenses)) if incompatibilities: print("{0}: *** incompatible license on modules:".format(script), file=sys.stderr) for (name, licenses) in sorted(incompatibilities): print(" " * 16, "{0:50}{1}".format(name, " ".join(sorted(licenses))), file=sys.stderr) print("{0}: *** Stop.".format(script), file=sys.stderr) return os.EX_DATAERR # Show banner notice of every module. if verbosity >= -1: for module in database.main_modules: notice = module.notice if notice.strip(): print("Notice from module {0}:".format(module.name), file=sys.stdout) print("\n".join(" " + line for line in notice.splitlines()), file=sys.stdout) # Determine the final file list. files = (set(database.main_files) | set(database.test_files)) if verbosity >= 0: print("File list:", file=sys.stdout) for file in sorted(files): if file.startswith("tests=lib/"): name = file[len("tests=lib/"):] src = "lib/" + name dst = "tests/" + name print(" ", src, " -> ", dst, file=sys.stdout, sep="") else: print(" ", file, file=sys.stdout, sep="") table = {} overrides = [] project = BaseVFS(config.root, **table) overrides = {BaseVFS(override, **table) for override in config.overrides} for (name, key) in SUBSTITUTION.items(): project[name] = config[key] if key else "" for override in overrides: for (name, key) in SUBSTITUTION.items(): override[name] = config[key] if key else "" table = {} old_files = set(cache.files) if "m4/gnulib-tool.m4" in project: old_files |= set(["m4/gnulib-tool.m4"]) for (tbl_key, cfg_key) in SUBSTITUTION.items(): table[tbl_key] = cache[cfg_key] if cfg_key else "" new_files = frozenset(files | set(["m4/gnulib-tool.m4"])) dry_run = options["dry_run"] gnulib_copymode = config.copymode local_copymode = config.local_copymode def transfer_file(local, src_vfs, src_name, dst_vfs, dst_name): args = (src_vfs, src_name, dst_vfs, dst_name) if src_vfs is None: return vfs_copy(*args) action = TRANSFER_MODES[local_copymode if local else gnulib_copymode] try: return action(*args) except OSError as error: if error.errno == errno.EXDEV: return vfs_copy(*args) raise error def remove_file(project, file): action = ("Removing", "Remove")[dry_run] fmt = (action + " file {file} (backup in {file}~)") if not dry_run: try: vfs_unlink(project, file, backup=True) except FileNotFoundError: pass print(fmt.format(file=file), file=sys.stdout) def update_file(local, src_vfs, src_name, dst_vfs, dst_name, present): if not vfs_compare(src_vfs, src_name, dst_vfs, dst_name): action = (("Replacing", "Replace"), ("Updating", "Update"))[present][dry_run] message = ("(non-gnulib code backed up in {file}~) !!", "(backup in {file}~)")[present] fmt = (action + " file {file} " + message) if not dry_run: vfs_backup(dst_vfs, dst_name) transfer_file(local, src_vfs, src_name, dst_vfs, dst_name) print(fmt.format(file=dst_name), file=sys.stdout) def add_file(local, src_vfs, src_name, dst_vfs, dst_name, present): action = ("Copying", "Copy")[dry_run] fmt = (action + " file {file}") if not dry_run: transfer_file(local, src_vfs, src_name, dst_vfs, dst_name) print(fmt.format(file=dst_name), file=sys.stdout) # First the files that are in old_files, but not in new_files. # Then the files that are in new_files, but not in old_files. # Then the files that are in new_files and in old_files. removed_files = {file for file in old_files if file not in new_files} added_files = {file for file in new_files if file not in old_files} kept_files = (old_files & new_files) for file in sorted(removed_files): remove_file(project, file) for (present, files) in ((False, added_files), (True, kept_files)): for dst in sorted(files): match = tuple(override for override in overrides if dst in override) override = match[0] if match else gnulib (vfs, src) = vfs_lookup(dst, gnulib, override, patch="patch") action = update_file if vfs_exists(project, dst) else add_file action(bool(match), vfs, src, project, dst, present) mkedits = [] if config.makefile_name == "Makefile.am": dirname = os.path.dirname(config.source_base) basename = os.path.basename(config.source_base) dirname = "" if dirname == "." else (dirname + os.path.sep) mkedits.append((dirname, "SUBDIRS", basename)) if "po_base" in explicit: dirname = os.path.dirname(config.po_base) basename = os.path.basename(config.po_base) dirname = "" if dirname == "." else dirname mkedits.append((dirname, "SUBDIRS", basename)) if config.tests and config.makefile_name == "Makefile.am": dirname = os.path.dirname(config.tests_base) basename = os.path.basename(config.tests_base) dirname = "" if dirname == "." else (dirname + os.path.sep) mkedits.append((dirname, "SUBDIRS", basename)) mkedits.append(("", "ACLOCAL_AMFLAGS", "-I {}".format(config.m4_base))) # Find the first parent directory of m4_base that contains or will contain a Makefile.am. (dir1, dir2) = (config.m4_base, "") source_base_makefile = os.path.join(config.source_base, config.makefile_name) tests_base_makefile = os.path.join(config.tests_base, config.makefile_name) while dir1 != ".": path = os.path.join(dir1, "Makefile.am") if vfs_exists(project, os.path.join(config.root, dir1, "Makefile.am")) \ or (config.tests and path in (source_base_makefile, tests_base_makefile)): break dir2 = os.path.join(os.path.basename(dir1), dir2) dir1 = os.path.dirname(dir1) mkedits.append((dir1, "EXTRA_DIST", os.path.join(dir2, "gnulib-cache.m4"))) # Generate the contents of library makefile. path = os.path.join(config.source_base, config.makefile_name) with tempfile.NamedTemporaryFile("w", encoding="UTF-8", delete=False) as tmp: modules = database.main_modules for line in lib_makefile(path, config, explicit, database, modules, mkedits, False): print(line, file=tmp) (src, dst) = (tmp.name, path) present = vfs_exists(project, dst) if present: added_files.add(dst) action = update_file if present else add_file action(False, None, src, project, dst, present) os.unlink(tmp.name) # Generate po makefile and directories. url = (TP_RSYNC_URI + "gnulib/") if "po_base" in explicit: for file in ("Makefile.in.in", "remove-potcdate.sin"): path = os.path.join("build-aux", "po", file) match = tuple(override for override in overrides if file in override) override = match[0] if match else gnulib (vfs, src) = vfs_lookup(path, gnulib, override, patch="patch") dst = os.path.join("po", file) present = vfs_exists(project, dst) if present: added_files.add(dst) action = update_file if present else add_file action(bool(match), vfs, src, project, dst, present) # Create po makefile parameterization, part 1. with tempfile.NamedTemporaryFile("w", encoding="UTF-8", delete=False) as tmp: for line in po_make_vars(config): print(line, file=tmp) (src, dst) = (tmp.name, "po/Makevars") present = vfs_exists(project, dst) if present: added_files.add(dst) action = update_file if present else add_file action(False, None, src, project, dst, present) os.unlink(tmp.name) # Create po makefile parameterization, part 2. with tempfile.NamedTemporaryFile("w", encoding="UTF-8", delete=False) as tmp: for line in po_make_vars(config): print(line, file=tmp) (src, dst) = (tmp.name, "po/POTFILESGenerator.in") present = vfs_exists(project, dst) if present: added_files.add(dst) action = update_file if present else add_file action(False, None, src, project, dst, present) os.unlink(tmp.name) # Fetch PO files. po_root = os.path.join(project.absolute, project["po"]) fmt = ("{} gnulib PO files from " + TP_URL) print(fmt.format("Fetching", "Fetch")[dry_run], file=sys.stdout) if not dry_run: # Prefer rsync over wget if it is available, since it consumes # less network bandwidth, due to compression. url = (TP_RSYNC_URI + "gnulib/") cmdargs = ("rsync", "--delete", "--exclude", "*.s1", "-Lrtz", url, ".") with sp.Popen(cmdargs, cwd=po_root, stdout=sp.PIPE, stderr=sp.PIPE) as process: (stdout, stderr) = process.communicate() stdout = stdout.decode("UTF-8") stderr = stderr.decode("UTF-8") returncode = process.returncode if process.returncode == 0: raise sp.CalledProcessError(returncode, cmdargs, stdout, stderr) print(stdout, file=sys.stdout) print(stderr, file=sys.stderr) # Create po/LINGUAS. languages = set() for root, _, files in os.walk(po_root): languages.update(file[:-3] for file in files if file.endswith(".po")) with tempfile.NamedTemporaryFile("w", encoding="UTF-8", delete=False) as tmp: tmp.write("# Set of available languages.\n") for line in sorted(languages): print(line, file=tmp) (src, dst) = (tmp.name, "po/LINGUAS") present = vfs_exists(project, dst) if present: added_files.add(dst) action = update_file if present else add_file action(False, None, src, project, dst, present) os.unlink(tmp.name) # Create m4/gnulib-cache.m4. with tempfile.NamedTemporaryFile("w", encoding="UTF-8", delete=False) as tmp: for line in gnulib_cache(config): print(line, file=tmp) (src, dst) = (tmp.name, "m4/gnulib-cache.m4") present = vfs_exists(project, dst) if present: added_files.add(dst) action = update_file if present else add_file action(False, None, src, project, dst, present) os.unlink(tmp.name) # Create m4/gnulib-comp.m4. with tempfile.NamedTemporaryFile("w", encoding="UTF-8", delete=False) as tmp: for line in gnulib_comp(config, explicit, database, True): print(line, file=tmp) (src, dst) = (tmp.name, "m4/gnulib-comp.m4") present = vfs_exists(project, dst) if present: added_files.add(dst) action = update_file if present else add_file action(False, None, src, project, dst, present) os.unlink(tmp.name) # Generate the contents of tests makefile. if config.tests: path = os.path.join(config.tests_base, config.makefile_name) with tempfile.NamedTemporaryFile("w", encoding="UTF-8", delete=False) as tmp: modules = database.test_modules for line in tests_makefile(path, config, explicit, database, modules, mkedits, False): print(line, file=tmp) (src, dst) = (tmp.name, path) present = vfs_exists(project, dst) if present: added_files.add(dst) action = update_file if present else add_file action(False, None, src, project, dst, present) os.unlink(tmp.name) return os.EX_OK def add_import_hook(script, gnulib, namespace, explicit, verbosity, options, *args, **kwargs): (_, _) = (args, kwargs) modules = set(namespace.pop("modules")) config = CachedConfig(**namespace) namespace = {k:v for (k, v) in config.items()} namespace["modules"] = (config.modules | modules) return import_hook(script, gnulib, namespace, verbosity, options) def remove_import_hook(script, gnulib, namespace, explicit, verbosity, options, *args, **kwargs): (_, _) = (args, kwargs) modules = set(namespace.pop("modules")) config = CachedConfig(**namespace) namespace = {k:v for (k, v) in config.items()} namespace["modules"] = (config.modules - modules) return import_hook(script, gnulib, namespace, verbosity, options) def update_hook(script, gnulib, namespace, explicit, verbosity, options, *args, **kwargs): (_, _) = (args, kwargs) config = CachedConfig(**namespace) namespace = {k:v for (k, v) in config.items()} return import_hook(script, gnulib, namespace, verbosity, options) HOOKS = { "list": list_hook, "extract": extract_hook, "import": import_hook, "add-import": add_import_hook, "remove-import": remove_import_hook, "update": update_hook, } def main(script, gnulib, program, arguments, environ): gnulib = GnulibGitVFS(gnulib) gnulib["tests=lib"] = "lib" parser = CommandLineParser(program) try: (namespace, mode, verbosity, options) = parser.parse(arguments) except CommandLineError as error: print(parser.usage, file=sys.stderr) print(error, file=sys.stderr) return os.EX_USAGE if mode == "help": print(parser.help, file=sys.stdout) return os.EX_OK kwargs = { "script": script, "program": program, "gnulib": gnulib, "mode": mode, "namespace": namespace, "explicit": namespace.keys(), "verbosity": verbosity, "options": options, } for (action, callback) in HOOKS.items(): if mode.startswith(action): return callback(**kwargs) return os.EX_SOFTWARE if __name__ == "__main__": script = sys.argv[0] gnulib = os.path.dirname(__file__) program = os.path.basename(script) log = os.path.join(os.getcwd(), "{0}.log".format(program)) arguments = list(sys.argv[1:]) environ = dict(os.environ) try: result = main(script, gnulib, program, arguments, environ) except BaseException as error: with codecs.open(log, "wb", "UTF-8") as stream: program = repr(program) if " " in program else program arguments = " ".join(repr(arg) if " " in arg else arg for arg in arguments) print(traceback.format_exc(), file=stream) print("COMMAND:", program, arguments, file=stream) print("VERSION:", gnulib, file=stream) typeid = type(error) module = typeid.__module__ name = typeid.__name__ if module != "builtins": error = "{0}.{1}: {2}".format(module, name, error) else: error = "{0}: {1}".format(name, error) print("{0}:".format(program), "***", error, file=sys.stderr) print("{0}:".format(program), "***", log, file=sys.stderr) result = os.EX_SOFTWARE sys.exit(result)