| #!/usr/bin/python3 |
| # Build many configurations of glibc. |
| # Copyright (C) 2016-2018 Free Software Foundation, Inc. |
| # This file is part of the GNU C Library. |
| # |
| # The GNU C Library is free software; you can redistribute it and/or |
| # modify it under the terms of the GNU Lesser General Public |
| # License as published by the Free Software Foundation; either |
| # version 2.1 of the License, or (at your option) any later version. |
| # |
| # The GNU C Library is distributed in the hope that it will be useful, |
| # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| # Lesser General Public License for more details. |
| # |
| # You should have received a copy of the GNU Lesser General Public |
| # License along with the GNU C Library; if not, see |
| # <http://www.gnu.org/licenses/>. |
| |
| """Build many configurations of glibc. |
| |
| This script takes as arguments a directory name (containing a src |
| subdirectory with sources of the relevant toolchain components) and a |
| description of what to do: 'checkout', to check out sources into that |
| directory, 'bot-cycle', to run a series of checkout and build steps, |
| 'bot', to run 'bot-cycle' repeatedly, 'host-libraries', to build |
| libraries required by the toolchain, 'compilers', to build |
| cross-compilers for various configurations, or 'glibcs', to build |
| glibc for various configurations and run the compilation parts of the |
| testsuite. Subsequent arguments name the versions of components to |
| check out (<component>-<version), for 'checkout', or, for actions |
| other than 'checkout' and 'bot-cycle', name configurations for which |
| compilers or glibc are to be built. |
| |
| """ |
| |
| import argparse |
| import datetime |
| import email.mime.text |
| import email.utils |
| import json |
| import os |
| import re |
| import shutil |
| import smtplib |
| import stat |
| import subprocess |
| import sys |
| import time |
| import urllib.request |
| |
| try: |
| os.cpu_count |
| except: |
| import multiprocessing |
| os.cpu_count = lambda: multiprocessing.cpu_count() |
| |
| try: |
| re.fullmatch |
| except: |
| re.fullmatch = lambda p,s,f=0: re.match(p+"\\Z",s,f) |
| |
| try: |
| subprocess.run |
| except: |
| class _CompletedProcess: |
| def __init__(self, args, returncode, stdout=None, stderr=None): |
| self.args = args |
| self.returncode = returncode |
| self.stdout = stdout |
| self.stderr = stderr |
| |
| def _run(*popenargs, input=None, timeout=None, check=False, **kwargs): |
| assert(timeout is None) |
| with subprocess.Popen(*popenargs, **kwargs) as process: |
| try: |
| stdout, stderr = process.communicate(input) |
| except: |
| process.kill() |
| process.wait() |
| raise |
| returncode = process.poll() |
| if check and returncode: |
| raise subprocess.CalledProcessError(returncode, popenargs) |
| return _CompletedProcess(popenargs, returncode, stdout, stderr) |
| |
| subprocess.run = _run |
| |
| |
| class Context(object): |
| """The global state associated with builds in a given directory.""" |
| |
| def __init__(self, topdir, parallelism, keep, replace_sources, strip, |
| action): |
| """Initialize the context.""" |
| self.topdir = topdir |
| self.parallelism = parallelism |
| self.keep = keep |
| self.replace_sources = replace_sources |
| self.strip = strip |
| self.srcdir = os.path.join(topdir, 'src') |
| self.versions_json = os.path.join(self.srcdir, 'versions.json') |
| self.build_state_json = os.path.join(topdir, 'build-state.json') |
| self.bot_config_json = os.path.join(topdir, 'bot-config.json') |
| self.installdir = os.path.join(topdir, 'install') |
| self.host_libraries_installdir = os.path.join(self.installdir, |
| 'host-libraries') |
| self.builddir = os.path.join(topdir, 'build') |
| self.logsdir = os.path.join(topdir, 'logs') |
| self.logsdir_old = os.path.join(topdir, 'logs-old') |
| self.makefile = os.path.join(self.builddir, 'Makefile') |
| self.wrapper = os.path.join(self.builddir, 'wrapper') |
| self.save_logs = os.path.join(self.builddir, 'save-logs') |
| self.script_text = self.get_script_text() |
| if action != 'checkout': |
| self.build_triplet = self.get_build_triplet() |
| self.glibc_version = self.get_glibc_version() |
| self.configs = {} |
| self.glibc_configs = {} |
| self.makefile_pieces = ['.PHONY: all\n'] |
| self.add_all_configs() |
| self.load_versions_json() |
| self.load_build_state_json() |
| self.status_log_list = [] |
| self.email_warning = False |
| |
| def get_script_text(self): |
| """Return the text of this script.""" |
| with open(sys.argv[0], 'r') as f: |
| return f.read() |
| |
| def exec_self(self): |
| """Re-execute this script with the same arguments.""" |
| sys.stdout.flush() |
| os.execv(sys.executable, [sys.executable] + sys.argv) |
| |
| def get_build_triplet(self): |
| """Determine the build triplet with config.guess.""" |
| config_guess = os.path.join(self.component_srcdir('gcc'), |
| 'config.guess') |
| cg_out = subprocess.run([config_guess], stdout=subprocess.PIPE, |
| check=True, universal_newlines=True).stdout |
| return cg_out.rstrip() |
| |
| def get_glibc_version(self): |
| """Determine the glibc version number (major.minor).""" |
| version_h = os.path.join(self.component_srcdir('glibc'), 'version.h') |
| with open(version_h, 'r') as f: |
| lines = f.readlines() |
| starttext = '#define VERSION "' |
| for l in lines: |
| if l.startswith(starttext): |
| l = l[len(starttext):] |
| l = l.rstrip('"\n') |
| m = re.fullmatch('([0-9]+)\.([0-9]+)[.0-9]*', l) |
| return '%s.%s' % m.group(1, 2) |
| print('error: could not determine glibc version') |
| exit(1) |
| |
| def add_all_configs(self): |
| """Add all known glibc build configurations.""" |
| self.add_config(arch='aarch64', |
| os_name='linux-gnu', |
| extra_glibcs=[{'variant': 'disable-multi-arch', |
| 'cfg': ['--disable-multi-arch']}]) |
| self.add_config(arch='aarch64_be', |
| os_name='linux-gnu') |
| self.add_config(arch='alpha', |
| os_name='linux-gnu') |
| self.add_config(arch='arm', |
| os_name='linux-gnueabi') |
| self.add_config(arch='armeb', |
| os_name='linux-gnueabi') |
| self.add_config(arch='armeb', |
| os_name='linux-gnueabi', |
| variant='be8', |
| gcc_cfg=['--with-arch=armv7-a']) |
| self.add_config(arch='arm', |
| os_name='linux-gnueabihf', |
| gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'], |
| extra_glibcs=[{'variant': 'v7a', |
| 'ccopts': '-march=armv7-a -mfpu=vfpv3'}, |
| {'variant': 'v7a-disable-multi-arch', |
| 'ccopts': '-march=armv7-a -mfpu=vfpv3', |
| 'cfg': ['--disable-multi-arch']}]) |
| self.add_config(arch='armeb', |
| os_name='linux-gnueabihf', |
| gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s']) |
| self.add_config(arch='armeb', |
| os_name='linux-gnueabihf', |
| variant='be8', |
| gcc_cfg=['--with-float=hard', '--with-arch=armv7-a', |
| '--with-fpu=vfpv3']) |
| self.add_config(arch='hppa', |
| os_name='linux-gnu') |
| self.add_config(arch='i686', |
| os_name='gnu') |
| self.add_config(arch='ia64', |
| os_name='linux-gnu', |
| first_gcc_cfg=['--with-system-libunwind']) |
| self.add_config(arch='m68k', |
| os_name='linux-gnu', |
| gcc_cfg=['--disable-multilib']) |
| self.add_config(arch='m68k', |
| os_name='linux-gnu', |
| variant='coldfire', |
| gcc_cfg=['--with-arch=cf', '--disable-multilib']) |
| self.add_config(arch='m68k', |
| os_name='linux-gnu', |
| variant='coldfire-soft', |
| gcc_cfg=['--with-arch=cf', '--with-cpu=54455', |
| '--disable-multilib']) |
| self.add_config(arch='microblaze', |
| os_name='linux-gnu', |
| gcc_cfg=['--disable-multilib']) |
| self.add_config(arch='microblazeel', |
| os_name='linux-gnu', |
| gcc_cfg=['--disable-multilib']) |
| self.add_config(arch='mips64', |
| os_name='linux-gnu', |
| gcc_cfg=['--with-mips-plt'], |
| glibcs=[{'variant': 'n32'}, |
| {'arch': 'mips', |
| 'ccopts': '-mabi=32'}, |
| {'variant': 'n64', |
| 'ccopts': '-mabi=64'}]) |
| self.add_config(arch='mips64', |
| os_name='linux-gnu', |
| variant='soft', |
| gcc_cfg=['--with-mips-plt', '--with-float=soft'], |
| glibcs=[{'variant': 'n32-soft'}, |
| {'variant': 'soft', |
| 'arch': 'mips', |
| 'ccopts': '-mabi=32'}, |
| {'variant': 'n64-soft', |
| 'ccopts': '-mabi=64'}]) |
| self.add_config(arch='mips64', |
| os_name='linux-gnu', |
| variant='nan2008', |
| gcc_cfg=['--with-mips-plt', '--with-nan=2008', |
| '--with-arch-64=mips64r2', |
| '--with-arch-32=mips32r2'], |
| glibcs=[{'variant': 'n32-nan2008'}, |
| {'variant': 'nan2008', |
| 'arch': 'mips', |
| 'ccopts': '-mabi=32'}, |
| {'variant': 'n64-nan2008', |
| 'ccopts': '-mabi=64'}]) |
| self.add_config(arch='mips64', |
| os_name='linux-gnu', |
| variant='nan2008-soft', |
| gcc_cfg=['--with-mips-plt', '--with-nan=2008', |
| '--with-arch-64=mips64r2', |
| '--with-arch-32=mips32r2', |
| '--with-float=soft'], |
| glibcs=[{'variant': 'n32-nan2008-soft'}, |
| {'variant': 'nan2008-soft', |
| 'arch': 'mips', |
| 'ccopts': '-mabi=32'}, |
| {'variant': 'n64-nan2008-soft', |
| 'ccopts': '-mabi=64'}]) |
| self.add_config(arch='mips64el', |
| os_name='linux-gnu', |
| gcc_cfg=['--with-mips-plt'], |
| glibcs=[{'variant': 'n32'}, |
| {'arch': 'mipsel', |
| 'ccopts': '-mabi=32'}, |
| {'variant': 'n64', |
| 'ccopts': '-mabi=64'}]) |
| self.add_config(arch='mips64el', |
| os_name='linux-gnu', |
| variant='soft', |
| gcc_cfg=['--with-mips-plt', '--with-float=soft'], |
| glibcs=[{'variant': 'n32-soft'}, |
| {'variant': 'soft', |
| 'arch': 'mipsel', |
| 'ccopts': '-mabi=32'}, |
| {'variant': 'n64-soft', |
| 'ccopts': '-mabi=64'}]) |
| self.add_config(arch='mips64el', |
| os_name='linux-gnu', |
| variant='nan2008', |
| gcc_cfg=['--with-mips-plt', '--with-nan=2008', |
| '--with-arch-64=mips64r2', |
| '--with-arch-32=mips32r2'], |
| glibcs=[{'variant': 'n32-nan2008'}, |
| {'variant': 'nan2008', |
| 'arch': 'mipsel', |
| 'ccopts': '-mabi=32'}, |
| {'variant': 'n64-nan2008', |
| 'ccopts': '-mabi=64'}]) |
| self.add_config(arch='mips64el', |
| os_name='linux-gnu', |
| variant='nan2008-soft', |
| gcc_cfg=['--with-mips-plt', '--with-nan=2008', |
| '--with-arch-64=mips64r2', |
| '--with-arch-32=mips32r2', |
| '--with-float=soft'], |
| glibcs=[{'variant': 'n32-nan2008-soft'}, |
| {'variant': 'nan2008-soft', |
| 'arch': 'mipsel', |
| 'ccopts': '-mabi=32'}, |
| {'variant': 'n64-nan2008-soft', |
| 'ccopts': '-mabi=64'}]) |
| self.add_config(arch='nios2', |
| os_name='linux-gnu') |
| self.add_config(arch='powerpc', |
| os_name='linux-gnu', |
| gcc_cfg=['--disable-multilib', '--enable-secureplt'], |
| extra_glibcs=[{'variant': 'power4', |
| 'ccopts': '-mcpu=power4', |
| 'cfg': ['--with-cpu=power4']}]) |
| self.add_config(arch='powerpc', |
| os_name='linux-gnu', |
| variant='soft', |
| gcc_cfg=['--disable-multilib', '--with-float=soft', |
| '--enable-secureplt']) |
| self.add_config(arch='powerpc64', |
| os_name='linux-gnu', |
| gcc_cfg=['--disable-multilib', '--enable-secureplt']) |
| self.add_config(arch='powerpc64le', |
| os_name='linux-gnu', |
| gcc_cfg=['--disable-multilib', '--enable-secureplt']) |
| self.add_config(arch='powerpc', |
| os_name='linux-gnuspe', |
| gcc_cfg=['--disable-multilib', '--enable-secureplt', |
| '--enable-e500-double']) |
| self.add_config(arch='powerpc', |
| os_name='linux-gnuspe', |
| variant='e500v1', |
| gcc_cfg=['--disable-multilib', '--enable-secureplt']) |
| self.add_config(arch='riscv64', |
| os_name='linux-gnu', |
| variant='rv64imac-lp64', |
| gcc_cfg=['--with-arch=rv64imac', '--with-abi=lp64', |
| '--disable-multilib']) |
| self.add_config(arch='riscv64', |
| os_name='linux-gnu', |
| variant='rv64imafdc-lp64', |
| gcc_cfg=['--with-arch=rv64imafdc', '--with-abi=lp64', |
| '--disable-multilib']) |
| self.add_config(arch='riscv64', |
| os_name='linux-gnu', |
| variant='rv64imafdc-lp64d', |
| gcc_cfg=['--with-arch=rv64imafdc', '--with-abi=lp64d', |
| '--disable-multilib']) |
| self.add_config(arch='s390x', |
| os_name='linux-gnu', |
| glibcs=[{}, |
| {'arch': 's390', 'ccopts': '-m31'}]) |
| self.add_config(arch='sh3', |
| os_name='linux-gnu') |
| self.add_config(arch='sh3eb', |
| os_name='linux-gnu') |
| self.add_config(arch='sh4', |
| os_name='linux-gnu') |
| self.add_config(arch='sh4eb', |
| os_name='linux-gnu') |
| self.add_config(arch='sh4', |
| os_name='linux-gnu', |
| variant='soft', |
| gcc_cfg=['--without-fp']) |
| self.add_config(arch='sh4eb', |
| os_name='linux-gnu', |
| variant='soft', |
| gcc_cfg=['--without-fp']) |
| self.add_config(arch='sparc64', |
| os_name='linux-gnu', |
| glibcs=[{}, |
| {'arch': 'sparcv9', |
| 'ccopts': '-m32 -mlong-double-128'}], |
| extra_glibcs=[{'variant': 'disable-multi-arch', |
| 'cfg': ['--disable-multi-arch']}, |
| {'variant': 'disable-multi-arch', |
| 'arch': 'sparcv9', |
| 'ccopts': '-m32 -mlong-double-128', |
| 'cfg': ['--disable-multi-arch']}]) |
| self.add_config(arch='tilegx', |
| os_name='linux-gnu', |
| glibcs=[{}, |
| {'variant': '32', 'ccopts': '-m32'}]) |
| self.add_config(arch='tilegxbe', |
| os_name='linux-gnu', |
| glibcs=[{}, |
| {'variant': '32', 'ccopts': '-m32'}]) |
| self.add_config(arch='x86_64', |
| os_name='linux-gnu', |
| gcc_cfg=['--with-multilib-list=m64,m32,mx32'], |
| glibcs=[{}, |
| {'variant': 'x32', 'ccopts': '-mx32'}, |
| {'arch': 'i686', 'ccopts': '-m32 -march=i686'}], |
| extra_glibcs=[{'variant': 'disable-multi-arch', |
| 'cfg': ['--disable-multi-arch']}, |
| {'variant': 'static-pie', |
| 'cfg': ['--enable-static-pie']}, |
| {'variant': 'x32-static-pie', |
| 'ccopts': '-mx32', |
| 'cfg': ['--enable-static-pie']}, |
| {'variant': 'static-pie', |
| 'arch': 'i686', |
| 'ccopts': '-m32 -march=i686', |
| 'cfg': ['--enable-static-pie']}, |
| {'variant': 'disable-multi-arch', |
| 'arch': 'i686', |
| 'ccopts': '-m32 -march=i686', |
| 'cfg': ['--disable-multi-arch']}, |
| {'arch': 'i486', |
| 'ccopts': '-m32 -march=i486'}, |
| {'arch': 'i586', |
| 'ccopts': '-m32 -march=i586'}]) |
| |
| def add_config(self, **args): |
| """Add an individual build configuration.""" |
| cfg = Config(self, **args) |
| if cfg.name in self.configs: |
| print('error: duplicate config %s' % cfg.name) |
| exit(1) |
| self.configs[cfg.name] = cfg |
| for c in cfg.all_glibcs: |
| if c.name in self.glibc_configs: |
| print('error: duplicate glibc config %s' % c.name) |
| exit(1) |
| self.glibc_configs[c.name] = c |
| |
| def component_srcdir(self, component): |
| """Return the source directory for a given component, e.g. gcc.""" |
| return os.path.join(self.srcdir, component) |
| |
| def component_builddir(self, action, config, component, subconfig=None): |
| """Return the directory to use for a build.""" |
| if config is None: |
| # Host libraries. |
| assert subconfig is None |
| return os.path.join(self.builddir, action, component) |
| if subconfig is None: |
| return os.path.join(self.builddir, action, config, component) |
| else: |
| # glibc build as part of compiler build. |
| return os.path.join(self.builddir, action, config, component, |
| subconfig) |
| |
| def compiler_installdir(self, config): |
| """Return the directory in which to install a compiler.""" |
| return os.path.join(self.installdir, 'compilers', config) |
| |
| def compiler_bindir(self, config): |
| """Return the directory in which to find compiler binaries.""" |
| return os.path.join(self.compiler_installdir(config), 'bin') |
| |
| def compiler_sysroot(self, config): |
| """Return the sysroot directory for a compiler.""" |
| return os.path.join(self.compiler_installdir(config), 'sysroot') |
| |
| def glibc_installdir(self, config): |
| """Return the directory in which to install glibc.""" |
| return os.path.join(self.installdir, 'glibcs', config) |
| |
| def run_builds(self, action, configs): |
| """Run the requested builds.""" |
| if action == 'checkout': |
| self.checkout(configs) |
| return |
| if action == 'bot-cycle': |
| if configs: |
| print('error: configurations specified for bot-cycle') |
| exit(1) |
| self.bot_cycle() |
| return |
| if action == 'bot': |
| if configs: |
| print('error: configurations specified for bot') |
| exit(1) |
| self.bot() |
| return |
| if action == 'host-libraries' and configs: |
| print('error: configurations specified for host-libraries') |
| exit(1) |
| self.clear_last_build_state(action) |
| build_time = datetime.datetime.utcnow() |
| if action == 'host-libraries': |
| build_components = ('gmp', 'mpfr', 'mpc') |
| old_components = () |
| old_versions = {} |
| self.build_host_libraries() |
| elif action == 'compilers': |
| build_components = ('binutils', 'gcc', 'glibc', 'linux', 'mig', |
| 'gnumach', 'hurd') |
| old_components = ('gmp', 'mpfr', 'mpc') |
| old_versions = self.build_state['host-libraries']['build-versions'] |
| self.build_compilers(configs) |
| else: |
| build_components = ('glibc',) |
| old_components = ('gmp', 'mpfr', 'mpc', 'binutils', 'gcc', 'linux', |
| 'mig', 'gnumach', 'hurd') |
| old_versions = self.build_state['compilers']['build-versions'] |
| self.build_glibcs(configs) |
| self.write_files() |
| self.do_build() |
| if configs: |
| # Partial build, do not update stored state. |
| return |
| build_versions = {} |
| for k in build_components: |
| if k in self.versions: |
| build_versions[k] = {'version': self.versions[k]['version'], |
| 'revision': self.versions[k]['revision']} |
| for k in old_components: |
| if k in old_versions: |
| build_versions[k] = {'version': old_versions[k]['version'], |
| 'revision': old_versions[k]['revision']} |
| self.update_build_state(action, build_time, build_versions) |
| |
| @staticmethod |
| def remove_dirs(*args): |
| """Remove directories and their contents if they exist.""" |
| for dir in args: |
| shutil.rmtree(dir, ignore_errors=True) |
| |
| @staticmethod |
| def remove_recreate_dirs(*args): |
| """Remove directories if they exist, and create them as empty.""" |
| Context.remove_dirs(*args) |
| for dir in args: |
| os.makedirs(dir, exist_ok=True) |
| |
| def add_makefile_cmdlist(self, target, cmdlist, logsdir): |
| """Add makefile text for a list of commands.""" |
| commands = cmdlist.makefile_commands(self.wrapper, logsdir) |
| self.makefile_pieces.append('all: %s\n.PHONY: %s\n%s:\n%s\n' % |
| (target, target, target, commands)) |
| self.status_log_list.extend(cmdlist.status_logs(logsdir)) |
| |
| def write_files(self): |
| """Write out the Makefile and wrapper script.""" |
| mftext = ''.join(self.makefile_pieces) |
| with open(self.makefile, 'w') as f: |
| f.write(mftext) |
| wrapper_text = ( |
| '#!/bin/sh\n' |
| 'prev_base=$1\n' |
| 'this_base=$2\n' |
| 'desc=$3\n' |
| 'dir=$4\n' |
| 'path=$5\n' |
| 'shift 5\n' |
| 'prev_status=$prev_base-status.txt\n' |
| 'this_status=$this_base-status.txt\n' |
| 'this_log=$this_base-log.txt\n' |
| 'date > "$this_log"\n' |
| 'echo >> "$this_log"\n' |
| 'echo "Description: $desc" >> "$this_log"\n' |
| 'printf "%s" "Command:" >> "$this_log"\n' |
| 'for word in "$@"; do\n' |
| ' if expr "$word" : "[]+,./0-9@A-Z_a-z-]\\\\{1,\\\\}\\$" > /dev/null; then\n' |
| ' printf " %s" "$word"\n' |
| ' else\n' |
| ' printf " \'"\n' |
| ' printf "%s" "$word" | sed -e "s/\'/\'\\\\\\\\\'\'/"\n' |
| ' printf "\'"\n' |
| ' fi\n' |
| 'done >> "$this_log"\n' |
| 'echo >> "$this_log"\n' |
| 'echo "Directory: $dir" >> "$this_log"\n' |
| 'echo "Path addition: $path" >> "$this_log"\n' |
| 'echo >> "$this_log"\n' |
| 'record_status ()\n' |
| '{\n' |
| ' echo >> "$this_log"\n' |
| ' echo "$1: $desc" > "$this_status"\n' |
| ' echo "$1: $desc" >> "$this_log"\n' |
| ' echo >> "$this_log"\n' |
| ' date >> "$this_log"\n' |
| ' echo "$1: $desc"\n' |
| ' exit 0\n' |
| '}\n' |
| 'check_error ()\n' |
| '{\n' |
| ' if [ "$1" != "0" ]; then\n' |
| ' record_status FAIL\n' |
| ' fi\n' |
| '}\n' |
| 'if [ "$prev_base" ] && ! grep -q "^PASS" "$prev_status"; then\n' |
| ' record_status UNRESOLVED\n' |
| 'fi\n' |
| 'if [ "$dir" ]; then\n' |
| ' cd "$dir"\n' |
| ' check_error "$?"\n' |
| 'fi\n' |
| 'if [ "$path" ]; then\n' |
| ' PATH=$path:$PATH\n' |
| 'fi\n' |
| '"$@" < /dev/null >> "$this_log" 2>&1\n' |
| 'check_error "$?"\n' |
| 'record_status PASS\n') |
| with open(self.wrapper, 'w') as f: |
| f.write(wrapper_text) |
| # Mode 0o755. |
| mode_exec = (stat.S_IRWXU|stat.S_IRGRP|stat.S_IXGRP| |
| stat.S_IROTH|stat.S_IXOTH) |
| os.chmod(self.wrapper, mode_exec) |
| save_logs_text = ( |
| '#!/bin/sh\n' |
| 'if ! [ -f tests.sum ]; then\n' |
| ' echo "No test summary available."\n' |
| ' exit 0\n' |
| 'fi\n' |
| 'save_file ()\n' |
| '{\n' |
| ' echo "Contents of $1:"\n' |
| ' echo\n' |
| ' cat "$1"\n' |
| ' echo\n' |
| ' echo "End of contents of $1."\n' |
| ' echo\n' |
| '}\n' |
| 'save_file tests.sum\n' |
| 'non_pass_tests=$(grep -v "^PASS: " tests.sum | sed -e "s/^PASS: //")\n' |
| 'for t in $non_pass_tests; do\n' |
| ' if [ -f "$t.out" ]; then\n' |
| ' save_file "$t.out"\n' |
| ' fi\n' |
| 'done\n') |
| with open(self.save_logs, 'w') as f: |
| f.write(save_logs_text) |
| os.chmod(self.save_logs, mode_exec) |
| |
| def do_build(self): |
| """Do the actual build.""" |
| cmd = ['make', '-j%d' % self.parallelism] |
| subprocess.run(cmd, cwd=self.builddir, check=True) |
| |
| def build_host_libraries(self): |
| """Build the host libraries.""" |
| installdir = self.host_libraries_installdir |
| builddir = os.path.join(self.builddir, 'host-libraries') |
| logsdir = os.path.join(self.logsdir, 'host-libraries') |
| self.remove_recreate_dirs(installdir, builddir, logsdir) |
| cmdlist = CommandList('host-libraries', self.keep) |
| self.build_host_library(cmdlist, 'gmp') |
| self.build_host_library(cmdlist, 'mpfr', |
| ['--with-gmp=%s' % installdir]) |
| self.build_host_library(cmdlist, 'mpc', |
| ['--with-gmp=%s' % installdir, |
| '--with-mpfr=%s' % installdir]) |
| cmdlist.add_command('done', ['touch', os.path.join(installdir, 'ok')]) |
| self.add_makefile_cmdlist('host-libraries', cmdlist, logsdir) |
| |
| def build_host_library(self, cmdlist, lib, extra_opts=None): |
| """Build one host library.""" |
| srcdir = self.component_srcdir(lib) |
| builddir = self.component_builddir('host-libraries', None, lib) |
| installdir = self.host_libraries_installdir |
| cmdlist.push_subdesc(lib) |
| cmdlist.create_use_dir(builddir) |
| cfg_cmd = [os.path.join(srcdir, 'configure'), |
| '--prefix=%s' % installdir, |
| '--disable-shared'] |
| if extra_opts: |
| cfg_cmd.extend (extra_opts) |
| cmdlist.add_command('configure', cfg_cmd) |
| cmdlist.add_command('build', ['make']) |
| cmdlist.add_command('check', ['make', 'check']) |
| cmdlist.add_command('install', ['make', 'install']) |
| cmdlist.cleanup_dir() |
| cmdlist.pop_subdesc() |
| |
| def build_compilers(self, configs): |
| """Build the compilers.""" |
| if not configs: |
| self.remove_dirs(os.path.join(self.builddir, 'compilers')) |
| self.remove_dirs(os.path.join(self.installdir, 'compilers')) |
| self.remove_dirs(os.path.join(self.logsdir, 'compilers')) |
| configs = sorted(self.configs.keys()) |
| for c in configs: |
| self.configs[c].build() |
| |
| def build_glibcs(self, configs): |
| """Build the glibcs.""" |
| if not configs: |
| self.remove_dirs(os.path.join(self.builddir, 'glibcs')) |
| self.remove_dirs(os.path.join(self.installdir, 'glibcs')) |
| self.remove_dirs(os.path.join(self.logsdir, 'glibcs')) |
| configs = sorted(self.glibc_configs.keys()) |
| for c in configs: |
| self.glibc_configs[c].build() |
| |
| def load_versions_json(self): |
| """Load information about source directory versions.""" |
| if not os.access(self.versions_json, os.F_OK): |
| self.versions = {} |
| return |
| with open(self.versions_json, 'r') as f: |
| self.versions = json.load(f) |
| |
| def store_json(self, data, filename): |
| """Store information in a JSON file.""" |
| filename_tmp = filename + '.tmp' |
| with open(filename_tmp, 'w') as f: |
| json.dump(data, f, indent=2, sort_keys=True) |
| os.rename(filename_tmp, filename) |
| |
| def store_versions_json(self): |
| """Store information about source directory versions.""" |
| self.store_json(self.versions, self.versions_json) |
| |
| def set_component_version(self, component, version, explicit, revision): |
| """Set the version information for a component.""" |
| self.versions[component] = {'version': version, |
| 'explicit': explicit, |
| 'revision': revision} |
| self.store_versions_json() |
| |
| def checkout(self, versions): |
| """Check out the desired component versions.""" |
| default_versions = {'binutils': 'vcs-2.30', |
| 'gcc': 'vcs-7', |
| 'glibc': 'vcs-mainline', |
| 'gmp': '6.1.2', |
| 'linux': '4.15', |
| 'mpc': '1.1.0', |
| 'mpfr': '4.0.0', |
| 'mig': 'vcs-mainline', |
| 'gnumach': 'vcs-mainline', |
| 'hurd': 'vcs-mainline'} |
| use_versions = {} |
| explicit_versions = {} |
| for v in versions: |
| found_v = False |
| for k in default_versions.keys(): |
| kx = k + '-' |
| if v.startswith(kx): |
| vx = v[len(kx):] |
| if k in use_versions: |
| print('error: multiple versions for %s' % k) |
| exit(1) |
| use_versions[k] = vx |
| explicit_versions[k] = True |
| found_v = True |
| break |
| if not found_v: |
| print('error: unknown component in %s' % v) |
| exit(1) |
| for k in default_versions.keys(): |
| if k not in use_versions: |
| if k in self.versions and self.versions[k]['explicit']: |
| use_versions[k] = self.versions[k]['version'] |
| explicit_versions[k] = True |
| else: |
| use_versions[k] = default_versions[k] |
| explicit_versions[k] = False |
| os.makedirs(self.srcdir, exist_ok=True) |
| for k in sorted(default_versions.keys()): |
| update = os.access(self.component_srcdir(k), os.F_OK) |
| v = use_versions[k] |
| if (update and |
| k in self.versions and |
| v != self.versions[k]['version']): |
| if not self.replace_sources: |
| print('error: version of %s has changed from %s to %s, ' |
| 'use --replace-sources to check out again' % |
| (k, self.versions[k]['version'], v)) |
| exit(1) |
| shutil.rmtree(self.component_srcdir(k)) |
| update = False |
| if v.startswith('vcs-'): |
| revision = self.checkout_vcs(k, v[4:], update) |
| else: |
| self.checkout_tar(k, v, update) |
| revision = v |
| self.set_component_version(k, v, explicit_versions[k], revision) |
| if self.get_script_text() != self.script_text: |
| # Rerun the checkout process in case the updated script |
| # uses different default versions or new components. |
| self.exec_self() |
| |
| def checkout_vcs(self, component, version, update): |
| """Check out the given version of the given component from version |
| control. Return a revision identifier.""" |
| if component == 'binutils': |
| git_url = 'git://sourceware.org/git/binutils-gdb.git' |
| if version == 'mainline': |
| git_branch = 'master' |
| else: |
| trans = str.maketrans({'.': '_'}) |
| git_branch = 'binutils-%s-branch' % version.translate(trans) |
| return self.git_checkout(component, git_url, git_branch, update) |
| elif component == 'gcc': |
| if version == 'mainline': |
| branch = 'trunk' |
| else: |
| trans = str.maketrans({'.': '_'}) |
| branch = 'branches/gcc-%s-branch' % version.translate(trans) |
| svn_url = 'svn://gcc.gnu.org/svn/gcc/%s' % branch |
| return self.gcc_checkout(svn_url, update) |
| elif component == 'glibc': |
| git_url = 'git://sourceware.org/git/glibc.git' |
| if version == 'mainline': |
| git_branch = 'master' |
| else: |
| git_branch = 'release/%s/master' % version |
| r = self.git_checkout(component, git_url, git_branch, update) |
| self.fix_glibc_timestamps() |
| return r |
| elif component == 'gnumach': |
| git_url = 'git://git.savannah.gnu.org/hurd/gnumach.git' |
| git_branch = 'master' |
| r = self.git_checkout(component, git_url, git_branch, update) |
| subprocess.run(['autoreconf', '-i'], |
| cwd=self.component_srcdir(component), check=True) |
| return r |
| elif component == 'mig': |
| git_url = 'git://git.savannah.gnu.org/hurd/mig.git' |
| git_branch = 'master' |
| r = self.git_checkout(component, git_url, git_branch, update) |
| subprocess.run(['autoreconf', '-i'], |
| cwd=self.component_srcdir(component), check=True) |
| return r |
| elif component == 'hurd': |
| git_url = 'git://git.savannah.gnu.org/hurd/hurd.git' |
| git_branch = 'master' |
| r = self.git_checkout(component, git_url, git_branch, update) |
| subprocess.run(['autoconf'], |
| cwd=self.component_srcdir(component), check=True) |
| return r |
| else: |
| print('error: component %s coming from VCS' % component) |
| exit(1) |
| |
| def git_checkout(self, component, git_url, git_branch, update): |
| """Check out a component from git. Return a commit identifier.""" |
| if update: |
| subprocess.run(['git', 'remote', 'prune', 'origin'], |
| cwd=self.component_srcdir(component), check=True) |
| if self.replace_sources: |
| subprocess.run(['git', 'clean', '-dxfq'], |
| cwd=self.component_srcdir(component), check=True) |
| subprocess.run(['git', 'pull', '-q'], |
| cwd=self.component_srcdir(component), check=True) |
| else: |
| subprocess.run(['git', 'clone', '-q', '-b', git_branch, git_url, |
| self.component_srcdir(component)], check=True) |
| r = subprocess.run(['git', 'rev-parse', 'HEAD'], |
| cwd=self.component_srcdir(component), |
| stdout=subprocess.PIPE, |
| check=True, universal_newlines=True).stdout |
| return r.rstrip() |
| |
| def fix_glibc_timestamps(self): |
| """Fix timestamps in a glibc checkout.""" |
| # Ensure that builds do not try to regenerate generated files |
| # in the source tree. |
| srcdir = self.component_srcdir('glibc') |
| for dirpath, dirnames, filenames in os.walk(srcdir): |
| for f in filenames: |
| if (f == 'configure' or |
| f == 'preconfigure' or |
| f.endswith('-kw.h')): |
| to_touch = os.path.join(dirpath, f) |
| subprocess.run(['touch', to_touch], check=True) |
| |
| def gcc_checkout(self, svn_url, update): |
| """Check out GCC from SVN. Return the revision number.""" |
| if not update: |
| subprocess.run(['svn', 'co', '-q', svn_url, |
| self.component_srcdir('gcc')], check=True) |
| subprocess.run(['contrib/gcc_update', '--silent'], |
| cwd=self.component_srcdir('gcc'), check=True) |
| r = subprocess.run(['svnversion', self.component_srcdir('gcc')], |
| stdout=subprocess.PIPE, |
| check=True, universal_newlines=True).stdout |
| return r.rstrip() |
| |
| def checkout_tar(self, component, version, update): |
| """Check out the given version of the given component from a |
| tarball.""" |
| if update: |
| return |
| url_map = {'binutils': 'https://ftp.gnu.org/gnu/binutils/binutils-%(version)s.tar.bz2', |
| 'gcc': 'https://ftp.gnu.org/gnu/gcc/gcc-%(version)s/gcc-%(version)s.tar.bz2', |
| 'gmp': 'https://ftp.gnu.org/gnu/gmp/gmp-%(version)s.tar.xz', |
| 'linux': 'https://www.kernel.org/pub/linux/kernel/v4.x/linux-%(version)s.tar.xz', |
| 'mpc': 'https://ftp.gnu.org/gnu/mpc/mpc-%(version)s.tar.gz', |
| 'mpfr': 'https://ftp.gnu.org/gnu/mpfr/mpfr-%(version)s.tar.xz', |
| 'mig': 'https://ftp.gnu.org/gnu/mig/mig-%(version)s.tar.bz2', |
| 'gnumach': 'https://ftp.gnu.org/gnu/gnumach/gnumach-%(version)s.tar.bz2', |
| 'hurd': 'https://ftp.gnu.org/gnu/hurd/hurd-%(version)s.tar.bz2'} |
| if component not in url_map: |
| print('error: component %s coming from tarball' % component) |
| exit(1) |
| url = url_map[component] % {'version': version} |
| filename = os.path.join(self.srcdir, url.split('/')[-1]) |
| response = urllib.request.urlopen(url) |
| data = response.read() |
| with open(filename, 'wb') as f: |
| f.write(data) |
| subprocess.run(['tar', '-C', self.srcdir, '-x', '-f', filename], |
| check=True) |
| os.rename(os.path.join(self.srcdir, '%s-%s' % (component, version)), |
| self.component_srcdir(component)) |
| os.remove(filename) |
| |
| def load_build_state_json(self): |
| """Load information about the state of previous builds.""" |
| if os.access(self.build_state_json, os.F_OK): |
| with open(self.build_state_json, 'r') as f: |
| self.build_state = json.load(f) |
| else: |
| self.build_state = {} |
| for k in ('host-libraries', 'compilers', 'glibcs'): |
| if k not in self.build_state: |
| self.build_state[k] = {} |
| if 'build-time' not in self.build_state[k]: |
| self.build_state[k]['build-time'] = '' |
| if 'build-versions' not in self.build_state[k]: |
| self.build_state[k]['build-versions'] = {} |
| if 'build-results' not in self.build_state[k]: |
| self.build_state[k]['build-results'] = {} |
| if 'result-changes' not in self.build_state[k]: |
| self.build_state[k]['result-changes'] = {} |
| if 'ever-passed' not in self.build_state[k]: |
| self.build_state[k]['ever-passed'] = [] |
| |
| def store_build_state_json(self): |
| """Store information about the state of previous builds.""" |
| self.store_json(self.build_state, self.build_state_json) |
| |
| def clear_last_build_state(self, action): |
| """Clear information about the state of part of the build.""" |
| # We clear the last build time and versions when starting a |
| # new build. The results of the last build are kept around, |
| # as comparison is still meaningful if this build is aborted |
| # and a new one started. |
| self.build_state[action]['build-time'] = '' |
| self.build_state[action]['build-versions'] = {} |
| self.store_build_state_json() |
| |
| def update_build_state(self, action, build_time, build_versions): |
| """Update the build state after a build.""" |
| build_time = build_time.replace(microsecond=0) |
| self.build_state[action]['build-time'] = str(build_time) |
| self.build_state[action]['build-versions'] = build_versions |
| build_results = {} |
| for log in self.status_log_list: |
| with open(log, 'r') as f: |
| log_text = f.read() |
| log_text = log_text.rstrip() |
| m = re.fullmatch('([A-Z]+): (.*)', log_text) |
| result = m.group(1) |
| test_name = m.group(2) |
| assert test_name not in build_results |
| build_results[test_name] = result |
| old_build_results = self.build_state[action]['build-results'] |
| self.build_state[action]['build-results'] = build_results |
| result_changes = {} |
| all_tests = set(old_build_results.keys()) | set(build_results.keys()) |
| for t in all_tests: |
| if t in old_build_results: |
| old_res = old_build_results[t] |
| else: |
| old_res = '(New test)' |
| if t in build_results: |
| new_res = build_results[t] |
| else: |
| new_res = '(Test removed)' |
| if old_res != new_res: |
| result_changes[t] = '%s -> %s' % (old_res, new_res) |
| self.build_state[action]['result-changes'] = result_changes |
| old_ever_passed = {t for t in self.build_state[action]['ever-passed'] |
| if t in build_results} |
| new_passes = {t for t in build_results if build_results[t] == 'PASS'} |
| self.build_state[action]['ever-passed'] = sorted(old_ever_passed | |
| new_passes) |
| self.store_build_state_json() |
| |
| def load_bot_config_json(self): |
| """Load bot configuration.""" |
| with open(self.bot_config_json, 'r') as f: |
| self.bot_config = json.load(f) |
| |
| def part_build_old(self, action, delay): |
| """Return whether the last build for a given action was at least a |
| given number of seconds ago, or does not have a time recorded.""" |
| old_time_str = self.build_state[action]['build-time'] |
| if not old_time_str: |
| return True |
| old_time = datetime.datetime.strptime(old_time_str, |
| '%Y-%m-%d %H:%M:%S') |
| new_time = datetime.datetime.utcnow() |
| delta = new_time - old_time |
| return delta.total_seconds() >= delay |
| |
| def bot_cycle(self): |
| """Run a single round of checkout and builds.""" |
| print('Bot cycle starting %s.' % str(datetime.datetime.utcnow())) |
| self.load_bot_config_json() |
| actions = ('host-libraries', 'compilers', 'glibcs') |
| self.bot_run_self(['--replace-sources'], 'checkout') |
| self.load_versions_json() |
| if self.get_script_text() != self.script_text: |
| print('Script changed, re-execing.') |
| # On script change, all parts of the build should be rerun. |
| for a in actions: |
| self.clear_last_build_state(a) |
| self.exec_self() |
| check_components = {'host-libraries': ('gmp', 'mpfr', 'mpc'), |
| 'compilers': ('binutils', 'gcc', 'glibc', 'linux', |
| 'mig', 'gnumach', 'hurd'), |
| 'glibcs': ('glibc',)} |
| must_build = {} |
| for a in actions: |
| build_vers = self.build_state[a]['build-versions'] |
| must_build[a] = False |
| if not self.build_state[a]['build-time']: |
| must_build[a] = True |
| old_vers = {} |
| new_vers = {} |
| for c in check_components[a]: |
| if c in build_vers: |
| old_vers[c] = build_vers[c] |
| new_vers[c] = {'version': self.versions[c]['version'], |
| 'revision': self.versions[c]['revision']} |
| if new_vers == old_vers: |
| print('Versions for %s unchanged.' % a) |
| else: |
| print('Versions changed or rebuild forced for %s.' % a) |
| if a == 'compilers' and not self.part_build_old( |
| a, self.bot_config['compilers-rebuild-delay']): |
| print('Not requiring rebuild of compilers this soon.') |
| else: |
| must_build[a] = True |
| if must_build['host-libraries']: |
| must_build['compilers'] = True |
| if must_build['compilers']: |
| must_build['glibcs'] = True |
| for a in actions: |
| if must_build[a]: |
| print('Must rebuild %s.' % a) |
| self.clear_last_build_state(a) |
| else: |
| print('No need to rebuild %s.' % a) |
| if os.access(self.logsdir, os.F_OK): |
| shutil.rmtree(self.logsdir_old, ignore_errors=True) |
| shutil.copytree(self.logsdir, self.logsdir_old) |
| for a in actions: |
| if must_build[a]: |
| build_time = datetime.datetime.utcnow() |
| print('Rebuilding %s at %s.' % (a, str(build_time))) |
| self.bot_run_self([], a) |
| self.load_build_state_json() |
| self.bot_build_mail(a, build_time) |
| print('Bot cycle done at %s.' % str(datetime.datetime.utcnow())) |
| |
| def bot_build_mail(self, action, build_time): |
| """Send email with the results of a build.""" |
| if not ('email-from' in self.bot_config and |
| 'email-server' in self.bot_config and |
| 'email-subject' in self.bot_config and |
| 'email-to' in self.bot_config): |
| if not self.email_warning: |
| print("Email not configured, not sending.") |
| self.email_warning = True |
| return |
| |
| build_time = build_time.replace(microsecond=0) |
| subject = (self.bot_config['email-subject'] % |
| {'action': action, |
| 'build-time': str(build_time)}) |
| results = self.build_state[action]['build-results'] |
| changes = self.build_state[action]['result-changes'] |
| ever_passed = set(self.build_state[action]['ever-passed']) |
| versions = self.build_state[action]['build-versions'] |
| new_regressions = {k for k in changes if changes[k] == 'PASS -> FAIL'} |
| all_regressions = {k for k in ever_passed if results[k] == 'FAIL'} |
| all_fails = {k for k in results if results[k] == 'FAIL'} |
| if new_regressions: |
| new_reg_list = sorted(['FAIL: %s' % k for k in new_regressions]) |
| new_reg_text = ('New regressions:\n\n%s\n\n' % |
| '\n'.join(new_reg_list)) |
| else: |
| new_reg_text = '' |
| if all_regressions: |
| all_reg_list = sorted(['FAIL: %s' % k for k in all_regressions]) |
| all_reg_text = ('All regressions:\n\n%s\n\n' % |
| '\n'.join(all_reg_list)) |
| else: |
| all_reg_text = '' |
| if all_fails: |
| all_fail_list = sorted(['FAIL: %s' % k for k in all_fails]) |
| all_fail_text = ('All failures:\n\n%s\n\n' % |
| '\n'.join(all_fail_list)) |
| else: |
| all_fail_text = '' |
| if changes: |
| changes_list = sorted(changes.keys()) |
| changes_list = ['%s: %s' % (changes[k], k) for k in changes_list] |
| changes_text = ('All changed results:\n\n%s\n\n' % |
| '\n'.join(changes_list)) |
| else: |
| changes_text = '' |
| results_text = (new_reg_text + all_reg_text + all_fail_text + |
| changes_text) |
| if not results_text: |
| results_text = 'Clean build with unchanged results.\n\n' |
| versions_list = sorted(versions.keys()) |
| versions_list = ['%s: %s (%s)' % (k, versions[k]['version'], |
| versions[k]['revision']) |
| for k in versions_list] |
| versions_text = ('Component versions for this build:\n\n%s\n' % |
| '\n'.join(versions_list)) |
| body_text = results_text + versions_text |
| msg = email.mime.text.MIMEText(body_text) |
| msg['Subject'] = subject |
| msg['From'] = self.bot_config['email-from'] |
| msg['To'] = self.bot_config['email-to'] |
| msg['Message-ID'] = email.utils.make_msgid() |
| msg['Date'] = email.utils.format_datetime(datetime.datetime.utcnow()) |
| with smtplib.SMTP(self.bot_config['email-server']) as s: |
| s.send_message(msg) |
| |
| def bot_run_self(self, opts, action, check=True): |
| """Run a copy of this script with given options.""" |
| cmd = [sys.executable, sys.argv[0], '--keep=none', |
| '-j%d' % self.parallelism] |
| cmd.extend(opts) |
| cmd.extend([self.topdir, action]) |
| sys.stdout.flush() |
| subprocess.run(cmd, check=check) |
| |
| def bot(self): |
| """Run repeated rounds of checkout and builds.""" |
| while True: |
| self.load_bot_config_json() |
| if not self.bot_config['run']: |
| print('Bot exiting by request.') |
| exit(0) |
| self.bot_run_self([], 'bot-cycle', check=False) |
| self.load_bot_config_json() |
| if not self.bot_config['run']: |
| print('Bot exiting by request.') |
| exit(0) |
| time.sleep(self.bot_config['delay']) |
| if self.get_script_text() != self.script_text: |
| print('Script changed, bot re-execing.') |
| self.exec_self() |
| |
| |
| class Config(object): |
| """A configuration for building a compiler and associated libraries.""" |
| |
| def __init__(self, ctx, arch, os_name, variant=None, gcc_cfg=None, |
| first_gcc_cfg=None, glibcs=None, extra_glibcs=None): |
| """Initialize a Config object.""" |
| self.ctx = ctx |
| self.arch = arch |
| self.os = os_name |
| self.variant = variant |
| if variant is None: |
| self.name = '%s-%s' % (arch, os_name) |
| else: |
| self.name = '%s-%s-%s' % (arch, os_name, variant) |
| self.triplet = '%s-glibc-%s' % (arch, os_name) |
| if gcc_cfg is None: |
| self.gcc_cfg = [] |
| else: |
| self.gcc_cfg = gcc_cfg |
| if first_gcc_cfg is None: |
| self.first_gcc_cfg = [] |
| else: |
| self.first_gcc_cfg = first_gcc_cfg |
| if glibcs is None: |
| glibcs = [{'variant': variant}] |
| if extra_glibcs is None: |
| extra_glibcs = [] |
| glibcs = [Glibc(self, **g) for g in glibcs] |
| extra_glibcs = [Glibc(self, **g) for g in extra_glibcs] |
| self.all_glibcs = glibcs + extra_glibcs |
| self.compiler_glibcs = glibcs |
| self.installdir = ctx.compiler_installdir(self.name) |
| self.bindir = ctx.compiler_bindir(self.name) |
| self.sysroot = ctx.compiler_sysroot(self.name) |
| self.builddir = os.path.join(ctx.builddir, 'compilers', self.name) |
| self.logsdir = os.path.join(ctx.logsdir, 'compilers', self.name) |
| |
| def component_builddir(self, component): |
| """Return the directory to use for a (non-glibc) build.""" |
| return self.ctx.component_builddir('compilers', self.name, component) |
| |
| def build(self): |
| """Generate commands to build this compiler.""" |
| self.ctx.remove_recreate_dirs(self.installdir, self.builddir, |
| self.logsdir) |
| cmdlist = CommandList('compilers-%s' % self.name, self.ctx.keep) |
| cmdlist.add_command('check-host-libraries', |
| ['test', '-f', |
| os.path.join(self.ctx.host_libraries_installdir, |
| 'ok')]) |
| cmdlist.use_path(self.bindir) |
| self.build_cross_tool(cmdlist, 'binutils', 'binutils', |
| ['--disable-gdb', |
| '--disable-libdecnumber', |
| '--disable-readline', |
| '--disable-sim']) |
| if self.os.startswith('linux'): |
| self.install_linux_headers(cmdlist) |
| self.build_gcc(cmdlist, True) |
| if self.os == 'gnu': |
| self.install_gnumach_headers(cmdlist) |
| self.build_cross_tool(cmdlist, 'mig', 'mig') |
| self.install_hurd_headers(cmdlist) |
| for g in self.compiler_glibcs: |
| cmdlist.push_subdesc('glibc') |
| cmdlist.push_subdesc(g.name) |
| g.build_glibc(cmdlist, True) |
| cmdlist.pop_subdesc() |
| cmdlist.pop_subdesc() |
| self.build_gcc(cmdlist, False) |
| cmdlist.add_command('done', ['touch', |
| os.path.join(self.installdir, 'ok')]) |
| self.ctx.add_makefile_cmdlist('compilers-%s' % self.name, cmdlist, |
| self.logsdir) |
| |
| def build_cross_tool(self, cmdlist, tool_src, tool_build, extra_opts=None): |
| """Build one cross tool.""" |
| srcdir = self.ctx.component_srcdir(tool_src) |
| builddir = self.component_builddir(tool_build) |
| cmdlist.push_subdesc(tool_build) |
| cmdlist.create_use_dir(builddir) |
| cfg_cmd = [os.path.join(srcdir, 'configure'), |
| '--prefix=%s' % self.installdir, |
| '--build=%s' % self.ctx.build_triplet, |
| '--host=%s' % self.ctx.build_triplet, |
| '--target=%s' % self.triplet, |
| '--with-sysroot=%s' % self.sysroot] |
| if extra_opts: |
| cfg_cmd.extend(extra_opts) |
| cmdlist.add_command('configure', cfg_cmd) |
| cmdlist.add_command('build', ['make']) |
| # Parallel "make install" for GCC has race conditions that can |
| # cause it to fail; see |
| # <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=42980>. Such |
| # problems are not known for binutils, but doing the |
| # installation in parallel within a particular toolchain build |
| # (as opposed to installation of one toolchain from |
| # build-many-glibcs.py running in parallel to the installation |
| # of other toolchains being built) is not known to be |
| # significantly beneficial, so it is simplest just to disable |
| # parallel install for cross tools here. |
| cmdlist.add_command('install', ['make', '-j1', 'install']) |
| cmdlist.cleanup_dir() |
| cmdlist.pop_subdesc() |
| |
| def install_linux_headers(self, cmdlist): |
| """Install Linux kernel headers.""" |
| arch_map = {'aarch64': 'arm64', |
| 'alpha': 'alpha', |
| 'arm': 'arm', |
| 'hppa': 'parisc', |
| 'i486': 'x86', |
| 'i586': 'x86', |
| 'i686': 'x86', |
| 'i786': 'x86', |
| 'ia64': 'ia64', |
| 'm68k': 'm68k', |
| 'microblaze': 'microblaze', |
| 'mips': 'mips', |
| 'nios2': 'nios2', |
| 'powerpc': 'powerpc', |
| 's390': 's390', |
| 'riscv32': 'riscv', |
| 'riscv64': 'riscv', |
| 'sh': 'sh', |
| 'sparc': 'sparc', |
| 'tile': 'tile', |
| 'x86_64': 'x86'} |
| linux_arch = None |
| for k in arch_map: |
| if self.arch.startswith(k): |
| linux_arch = arch_map[k] |
| break |
| assert linux_arch is not None |
| srcdir = self.ctx.component_srcdir('linux') |
| builddir = self.component_builddir('linux') |
| headers_dir = os.path.join(self.sysroot, 'usr') |
| cmdlist.push_subdesc('linux') |
| cmdlist.create_use_dir(builddir) |
| cmdlist.add_command('install-headers', |
| ['make', '-C', srcdir, 'O=%s' % builddir, |
| 'ARCH=%s' % linux_arch, |
| 'INSTALL_HDR_PATH=%s' % headers_dir, |
| 'headers_install']) |
| cmdlist.cleanup_dir() |
| cmdlist.pop_subdesc() |
| |
| def install_gnumach_headers(self, cmdlist): |
| """Install GNU Mach headers.""" |
| srcdir = self.ctx.component_srcdir('gnumach') |
| builddir = self.component_builddir('gnumach') |
| cmdlist.push_subdesc('gnumach') |
| cmdlist.create_use_dir(builddir) |
| cmdlist.add_command('configure', |
| [os.path.join(srcdir, 'configure'), |
| '--build=%s' % self.ctx.build_triplet, |
| '--host=%s' % self.triplet, |
| '--prefix=', |
| 'CC=%s-gcc -nostdlib' % self.triplet]) |
| cmdlist.add_command('install', ['make', 'DESTDIR=%s' % self.sysroot, |
| 'install-data']) |
| cmdlist.cleanup_dir() |
| cmdlist.pop_subdesc() |
| |
| def install_hurd_headers(self, cmdlist): |
| """Install Hurd headers.""" |
| srcdir = self.ctx.component_srcdir('hurd') |
| builddir = self.component_builddir('hurd') |
| cmdlist.push_subdesc('hurd') |
| cmdlist.create_use_dir(builddir) |
| cmdlist.add_command('configure', |
| [os.path.join(srcdir, 'configure'), |
| '--build=%s' % self.ctx.build_triplet, |
| '--host=%s' % self.triplet, |
| '--prefix=', |
| '--disable-profile', '--without-parted', |
| 'CC=%s-gcc -nostdlib' % self.triplet]) |
| cmdlist.add_command('install', ['make', 'prefix=%s' % self.sysroot, |
| 'no_deps=t', 'install-headers']) |
| cmdlist.cleanup_dir() |
| cmdlist.pop_subdesc() |
| |
| def build_gcc(self, cmdlist, bootstrap): |
| """Build GCC.""" |
| # libsanitizer commonly breaks because of glibc header |
| # changes, or on unusual targets. libssp is of little |
| # relevance with glibc's own stack checking support. |
| # libcilkrts does not support GNU/Hurd (and has been removed |
| # in GCC 8, so --disable-libcilkrts can be removed once glibc |
| # no longer supports building with older GCC versions). |
| cfg_opts = list(self.gcc_cfg) |
| cfg_opts += ['--disable-libsanitizer', '--disable-libssp', |
| '--disable-libcilkrts'] |
| host_libs = self.ctx.host_libraries_installdir |
| cfg_opts += ['--with-gmp=%s' % host_libs, |
| '--with-mpfr=%s' % host_libs, |
| '--with-mpc=%s' % host_libs] |
| if bootstrap: |
| tool_build = 'gcc-first' |
| # Building a static-only, C-only compiler that is |
| # sufficient to build glibc. Various libraries and |
| # features that may require libc headers must be disabled. |
| # When configuring with a sysroot, --with-newlib is |
| # required to define inhibit_libc (to stop some parts of |
| # libgcc including libc headers); --without-headers is not |
| # sufficient. |
| cfg_opts += ['--enable-languages=c', '--disable-shared', |
| '--disable-threads', |
| '--disable-libatomic', |
| '--disable-decimal-float', |
| '--disable-libffi', |
| '--disable-libgomp', |
| '--disable-libitm', |
| '--disable-libmpx', |
| '--disable-libquadmath', |
| '--without-headers', '--with-newlib', |
| '--with-glibc-version=%s' % self.ctx.glibc_version |
| ] |
| cfg_opts += self.first_gcc_cfg |
| else: |
| tool_build = 'gcc' |
| cfg_opts += ['--enable-languages=c,c++', '--enable-shared', |
| '--enable-threads'] |
| self.build_cross_tool(cmdlist, 'gcc', tool_build, cfg_opts) |
| |
| |
| class Glibc(object): |
| """A configuration for building glibc.""" |
| |
| def __init__(self, compiler, arch=None, os_name=None, variant=None, |
| cfg=None, ccopts=None): |
| """Initialize a Glibc object.""" |
| self.ctx = compiler.ctx |
| self.compiler = compiler |
| if arch is None: |
| self.arch = compiler.arch |
| else: |
| self.arch = arch |
| if os_name is None: |
| self.os = compiler.os |
| else: |
| self.os = os_name |
| self.variant = variant |
| if variant is None: |
| self.name = '%s-%s' % (self.arch, self.os) |
| else: |
| self.name = '%s-%s-%s' % (self.arch, self.os, variant) |
| self.triplet = '%s-glibc-%s' % (self.arch, self.os) |
| if cfg is None: |
| self.cfg = [] |
| else: |
| self.cfg = cfg |
| self.ccopts = ccopts |
| |
| def tool_name(self, tool): |
| """Return the name of a cross-compilation tool.""" |
| ctool = '%s-%s' % (self.compiler.triplet, tool) |
| if self.ccopts and (tool == 'gcc' or tool == 'g++'): |
| ctool = '%s %s' % (ctool, self.ccopts) |
| return ctool |
| |
| def build(self): |
| """Generate commands to build this glibc.""" |
| builddir = self.ctx.component_builddir('glibcs', self.name, 'glibc') |
| installdir = self.ctx.glibc_installdir(self.name) |
| logsdir = os.path.join(self.ctx.logsdir, 'glibcs', self.name) |
| self.ctx.remove_recreate_dirs(installdir, builddir, logsdir) |
| cmdlist = CommandList('glibcs-%s' % self.name, self.ctx.keep) |
| cmdlist.add_command('check-compilers', |
| ['test', '-f', |
| os.path.join(self.compiler.installdir, 'ok')]) |
| cmdlist.use_path(self.compiler.bindir) |
| self.build_glibc(cmdlist, False) |
| self.ctx.add_makefile_cmdlist('glibcs-%s' % self.name, cmdlist, |
| logsdir) |
| |
| def build_glibc(self, cmdlist, for_compiler): |
| """Generate commands to build this glibc, either as part of a compiler |
| build or with the bootstrapped compiler (and in the latter case, run |
| tests as well).""" |
| srcdir = self.ctx.component_srcdir('glibc') |
| if for_compiler: |
| builddir = self.ctx.component_builddir('compilers', |
| self.compiler.name, 'glibc', |
| self.name) |
| installdir = self.compiler.sysroot |
| srcdir_copy = self.ctx.component_builddir('compilers', |
| self.compiler.name, |
| 'glibc-src', |
| self.name) |
| else: |
| builddir = self.ctx.component_builddir('glibcs', self.name, |
| 'glibc') |
| installdir = self.ctx.glibc_installdir(self.name) |
| srcdir_copy = self.ctx.component_builddir('glibcs', self.name, |
| 'glibc-src') |
| cmdlist.create_use_dir(builddir) |
| # glibc builds write into the source directory, and even if |
| # not intentionally there is a risk of bugs that involve |
| # writing into the working directory. To avoid possible |
| # concurrency issues, copy the source directory. |
| cmdlist.create_copy_dir(srcdir, srcdir_copy) |
| use_usr = self.os != 'gnu' |
| prefix = '/usr' if use_usr else '' |
| cfg_cmd = [os.path.join(srcdir_copy, 'configure'), |
| '--prefix=%s' % prefix, |
| '--enable-profile', |
| '--build=%s' % self.ctx.build_triplet, |
| '--host=%s' % self.triplet, |
| 'CC=%s' % self.tool_name('gcc'), |
| 'CXX=%s' % self.tool_name('g++'), |
| 'AR=%s' % self.tool_name('ar'), |
| 'AS=%s' % self.tool_name('as'), |
| 'LD=%s' % self.tool_name('ld'), |
| 'NM=%s' % self.tool_name('nm'), |
| 'OBJCOPY=%s' % self.tool_name('objcopy'), |
| 'OBJDUMP=%s' % self.tool_name('objdump'), |
| 'RANLIB=%s' % self.tool_name('ranlib'), |
| 'READELF=%s' % self.tool_name('readelf'), |
| 'STRIP=%s' % self.tool_name('strip')] |
| if self.os == 'gnu': |
| cfg_cmd += ['MIG=%s' % self.tool_name('mig')] |
| cfg_cmd += self.cfg |
| cmdlist.add_command('configure', cfg_cmd) |
| cmdlist.add_command('build', ['make']) |
| cmdlist.add_command('install', ['make', 'install', |
| 'install_root=%s' % installdir]) |
| # GCC uses paths such as lib/../lib64, so make sure lib |
| # directories always exist. |
| mkdir_cmd = ['mkdir', '-p', |
| os.path.join(installdir, 'lib')] |
| if use_usr: |
| mkdir_cmd += [os.path.join(installdir, 'usr', 'lib')] |
| cmdlist.add_command('mkdir-lib', mkdir_cmd) |
| if not for_compiler: |
| if self.ctx.strip: |
| cmdlist.add_command('strip', |
| ['sh', '-c', |
| ('%s $(find %s/lib* -name "*.so")' % |
| (self.tool_name('strip'), installdir))]) |
| cmdlist.add_command('check', ['make', 'check']) |
| cmdlist.add_command('save-logs', [self.ctx.save_logs], |
| always_run=True) |
| cmdlist.cleanup_dir('cleanup-src', srcdir_copy) |
| cmdlist.cleanup_dir() |
| |
| |
| class Command(object): |
| """A command run in the build process.""" |
| |
| def __init__(self, desc, num, dir, path, command, always_run=False): |
| """Initialize a Command object.""" |
| self.dir = dir |
| self.path = path |
| self.desc = desc |
| trans = str.maketrans({' ': '-'}) |
| self.logbase = '%03d-%s' % (num, desc.translate(trans)) |
| self.command = command |
| self.always_run = always_run |
| |
| @staticmethod |
| def shell_make_quote_string(s): |
| """Given a string not containing a newline, quote it for use by the |
| shell and make.""" |
| assert '\n' not in s |
| if re.fullmatch('[]+,./0-9@A-Z_a-z-]+', s): |
| return s |
| strans = str.maketrans({"'": "'\\''"}) |
| s = "'%s'" % s.translate(strans) |
| mtrans = str.maketrans({'$': '$$'}) |
| return s.translate(mtrans) |
| |
| @staticmethod |
| def shell_make_quote_list(l, translate_make): |
| """Given a list of strings not containing newlines, quote them for use |
| by the shell and make, returning a single string. If translate_make |
| is true and the first string is 'make', change it to $(MAKE).""" |
| l = [Command.shell_make_quote_string(s) for s in l] |
| if translate_make and l[0] == 'make': |
| l[0] = '$(MAKE)' |
| return ' '.join(l) |
| |
| def shell_make_quote(self): |
| """Return this command quoted for the shell and make.""" |
| return self.shell_make_quote_list(self.command, True) |
| |
| |
| class CommandList(object): |
| """A list of commands run in the build process.""" |
| |
| def __init__(self, desc, keep): |
| """Initialize a CommandList object.""" |
| self.cmdlist = [] |
| self.dir = None |
| self.path = None |
| self.desc = [desc] |
| self.keep = keep |
| |
| def desc_txt(self, desc): |
| """Return the description to use for a command.""" |
| return '%s %s' % (' '.join(self.desc), desc) |
| |
| def use_dir(self, dir): |
| """Set the default directory for subsequent commands.""" |
| self.dir = dir |
| |
| def use_path(self, path): |
| """Set a directory to be prepended to the PATH for subsequent |
| commands.""" |
| self.path = path |
| |
| def push_subdesc(self, subdesc): |
| """Set the default subdescription for subsequent commands (e.g., the |
| name of a component being built, within the series of commands |
| building it).""" |
| self.desc.append(subdesc) |
| |
| def pop_subdesc(self): |
| """Pop a subdescription from the list of descriptions.""" |
| self.desc.pop() |
| |
| def create_use_dir(self, dir): |
| """Remove and recreate a directory and use it for subsequent |
| commands.""" |
| self.add_command_dir('rm', None, ['rm', '-rf', dir]) |
| self.add_command_dir('mkdir', None, ['mkdir', '-p', dir]) |
| self.use_dir(dir) |
| |
| def create_copy_dir(self, src, dest): |
| """Remove a directory and recreate it as a copy from the given |
| source.""" |
| self.add_command_dir('copy-rm', None, ['rm', '-rf', dest]) |
| parent = os.path.dirname(dest) |
| self.add_command_dir('copy-mkdir', None, ['mkdir', '-p', parent]) |
| self.add_command_dir('copy', None, ['cp', '-a', src, dest]) |
| |
| def add_command_dir(self, desc, dir, command, always_run=False): |
| """Add a command to run in a given directory.""" |
| cmd = Command(self.desc_txt(desc), len(self.cmdlist), dir, self.path, |
| command, always_run) |
| self.cmdlist.append(cmd) |
| |
| def add_command(self, desc, command, always_run=False): |
| """Add a command to run in the default directory.""" |
| cmd = Command(self.desc_txt(desc), len(self.cmdlist), self.dir, |
| self.path, command, always_run) |
| self.cmdlist.append(cmd) |
| |
| def cleanup_dir(self, desc='cleanup', dir=None): |
| """Clean up a build directory. If no directory is specified, the |
| default directory is cleaned up and ceases to be the default |
| directory.""" |
| if dir is None: |
| dir = self.dir |
| self.use_dir(None) |
| if self.keep != 'all': |
| self.add_command_dir(desc, None, ['rm', '-rf', dir], |
| always_run=(self.keep == 'none')) |
| |
| def makefile_commands(self, wrapper, logsdir): |
| """Return the sequence of commands in the form of text for a Makefile. |
| The given wrapper script takes arguments: base of logs for |
| previous command, or empty; base of logs for this command; |
| description; directory; PATH addition; the command itself.""" |
| # prev_base is the base of the name for logs of the previous |
| # command that is not always-run (that is, a build command, |
| # whose failure should stop subsequent build commands from |
| # being run, as opposed to a cleanup command, which is run |
| # even if previous commands failed). |
| prev_base = '' |
| cmds = [] |
| for c in self.cmdlist: |
| ctxt = c.shell_make_quote() |
| if prev_base and not c.always_run: |
| prev_log = os.path.join(logsdir, prev_base) |
| else: |
| prev_log = '' |
| this_log = os.path.join(logsdir, c.logbase) |
| if not c.always_run: |
| prev_base = c.logbase |
| if c.dir is None: |
| dir = '' |
| else: |
| dir = c.dir |
| if c.path is None: |
| path = '' |
| else: |
| path = c.path |
| prelims = [wrapper, prev_log, this_log, c.desc, dir, path] |
| prelim_txt = Command.shell_make_quote_list(prelims, False) |
| cmds.append('\t@%s %s' % (prelim_txt, ctxt)) |
| return '\n'.join(cmds) |
| |
| def status_logs(self, logsdir): |
| """Return the list of log files with command status.""" |
| return [os.path.join(logsdir, '%s-status.txt' % c.logbase) |
| for c in self.cmdlist] |
| |
| |
| def get_parser(): |
| """Return an argument parser for this module.""" |
| parser = argparse.ArgumentParser(description=__doc__) |
| parser.add_argument('-j', dest='parallelism', |
| help='Run this number of jobs in parallel', |
| type=int, default=os.cpu_count()) |
| parser.add_argument('--keep', dest='keep', |
| help='Whether to keep all build directories, ' |
| 'none or only those from failed builds', |
| default='none', choices=('none', 'all', 'failed')) |
| parser.add_argument('--replace-sources', action='store_true', |
| help='Remove and replace source directories ' |
| 'with the wrong version of a component') |
| parser.add_argument('--strip', action='store_true', |
| help='Strip installed glibc libraries') |
| parser.add_argument('topdir', |
| help='Toplevel working directory') |
| parser.add_argument('action', |
| help='What to do', |
| choices=('checkout', 'bot-cycle', 'bot', |
| 'host-libraries', 'compilers', 'glibcs')) |
| parser.add_argument('configs', |
| help='Versions to check out or configurations to build', |
| nargs='*') |
| return parser |
| |
| |
| def main(argv): |
| """The main entry point.""" |
| parser = get_parser() |
| opts = parser.parse_args(argv) |
| topdir = os.path.abspath(opts.topdir) |
| ctx = Context(topdir, opts.parallelism, opts.keep, opts.replace_sources, |
| opts.strip, opts.action) |
| ctx.run_builds(opts.action, opts.configs) |
| |
| |
| if __name__ == '__main__': |
| main(sys.argv[1:]) |
