| #!/usr/bin/env python3 |
| # Copyright 2015-2016 Obsidian Research Corp. |
| # Licensed under BSD (MIT variant) or GPLv2. See COPYING. |
| # PYTHON_ARGCOMPLETE_OK |
| """cbuild - Build in a docker container |
| |
| This script helps using docker containers to run software builds. This allows |
| building for a wide range of distributions without having to install them. |
| |
| Each target distribution has a base docker image and a set of packages to |
| install. The first step is to build the customized docker container: |
| |
| $ buildlib/cbuild build-images fedora |
| |
| This will download the base image and customize it with the required packages. |
| |
| Next, a build can be performed 'in place'. This is useful to do edit/compile |
| cycles with an alternate distribution. |
| |
| $ buildlib/cbuild make fedora |
| |
| The build output will be placed in build-fcXX, where XX is latest fedora release. |
| |
| Finally, a full package build can be performed inside the container. Note this |
| mode actually creates a source tree inside the container based on the current |
| git HEAD commit, so any uncommitted edits will be lost. |
| |
| $ buildlib/cbuild pkg fedora |
| |
| In this case only the final package results are copied outside the container |
| (to ..) and everything else is discarded. |
| |
| In all cases the containers that are spun up are deleted after they are |
| finished, only the base container created during 'build-images' is kept. The |
| '--run-shell' option can be used to setup the container to the point of |
| running the build command and instead run an interactive bash shell. This is |
| useful for debugging certain kinds of build problems.""" |
| |
| from __future__ import print_function |
| import argparse |
| import collections |
| import filecmp |
| import grp |
| import inspect |
| import json |
| import multiprocessing |
| import os |
| import pwd |
| import re |
| import shutil |
| import subprocess |
| import sys |
| import tempfile |
| import yaml |
| from contextlib import contextmanager; |
| |
| project = "rdma-core"; |
| |
| def get_version(): |
| """Return the version string for the project, this gets automatically written |
| into the packaging files.""" |
| with open("CMakeLists.txt","r") as F: |
| for ln in F: |
| g = re.match(r'^set\(PACKAGE_VERSION "(.+)"\)',ln) |
| if g is None: |
| continue; |
| return g.group(1); |
| raise RuntimeError("Could not find version"); |
| |
| class DockerFile(object): |
| def __init__(self,src): |
| self.lines = ["FROM %s"%(src)]; |
| |
| class Environment(object): |
| azp_images = None; |
| pandoc = True; |
| python_cmd = "python3"; |
| aliases = set(); |
| use_make = False; |
| proxy = True; |
| build_pyverbs = True; |
| docker_opts = [] |
| |
| to_azp = False; |
| |
| def _get_azp_names(self): |
| if Environment.azp_images: |
| return Environment.azp_images; |
| |
| with open("buildlib/azure-pipelines.yml") as F: |
| azp = yaml.safe_load(F) |
| Environment.azp_images = set(I["image"] for I in azp["resources"]["containers"]) |
| return Environment.azp_images; |
| |
| def image_name(self): |
| if self.to_azp: |
| # Get the version number of the container out of the azp file. |
| prefix = "ucfconsort.azurecr.io/%s/%s:"%(project, self.name); |
| for I in self._get_azp_names(): |
| if I.startswith(prefix): |
| return I; |
| raise ValueError("Image is not used in buildlib/azure-pipelines.yml") |
| return "build-%s/%s"%(project,self.name); |
| |
| # ------------------------------------------------------------------------- |
| |
| class YumEnvironment(Environment): |
| is_rpm = True; |
| def get_docker_file(self,tmpdir): |
| res = DockerFile(self.docker_parent); |
| res.lines.append("RUN yum install -y %s && yum clean all"%( |
| " ".join(sorted(self.pkgs)))); |
| return res; |
| |
| class centos7(YumEnvironment): |
| docker_parent = "centos:7"; |
| pkgs = { |
| 'cmake', |
| 'gcc', |
| 'libnl3-devel', |
| 'libudev-devel', |
| 'make', |
| 'pkgconfig', |
| 'python', |
| 'python-argparse', |
| 'python-docutils', |
| 'rpm-build', |
| 'systemd-devel', |
| 'valgrind-devel', |
| } |
| name = "centos7"; |
| use_make = True; |
| pandoc = False; |
| build_pyverbs = False; |
| specfile = "redhat/rdma-core.spec"; |
| python_cmd = "python"; |
| to_azp = True; |
| |
| class centos7_epel(centos7): |
| pkgs = (centos7.pkgs - {"cmake","make"}) | { |
| "cmake3", |
| "ninja-build", |
| "pandoc", |
| "python34-setuptools", |
| 'python34-Cython', |
| 'python34-devel', |
| }; |
| name = "centos7_epel"; |
| build_pyverbs = True; |
| use_make = False; |
| pandoc = True; |
| ninja_cmd = "ninja-build"; |
| # Our spec file does not know how to cope with cmake3 |
| is_rpm = False; |
| to_azp = False; |
| |
| def get_docker_file(self,tmpdir): |
| res = YumEnvironment.get_docker_file(self,tmpdir); |
| res.lines.insert(1,"RUN yum install -y epel-release"); |
| res.lines.append("RUN ln -s /usr/bin/cmake3 /usr/local/bin/cmake && ln -sf /usr/bin/python3.4 /usr/bin/python3"); |
| return res; |
| |
| class amazonlinux2(YumEnvironment): |
| docker_parent = "amazonlinux:2"; |
| pkgs = centos7.pkgs; |
| name = "amazonlinux2"; |
| use_make = True; |
| pandoc = False; |
| build_pyverbs = False; |
| specfile = "redhat/rdma-core.spec"; |
| python_cmd = "python"; |
| to_azp = False; |
| |
| class centos8(Environment): |
| docker_parent = "quay.io/centos/centos:stream8" |
| pkgs = { |
| "pandoc", |
| "perl-generators", |
| "python3-Cython", |
| "python3-devel", |
| "python3-docutils", |
| 'cmake', |
| 'gcc', |
| 'libnl3-devel', |
| 'libudev-devel', |
| 'ninja-build', |
| 'pkgconfig', |
| 'rpm-build', |
| 'systemd-devel', |
| 'valgrind-devel', |
| }; |
| name = "centos8"; |
| specfile = "redhat/rdma-core.spec"; |
| is_rpm = True; |
| to_azp = True; |
| proxy = False; |
| |
| def get_docker_file(self,tmpdir): |
| res = DockerFile(self.docker_parent); |
| res.lines.append("RUN dnf config-manager --set-enabled powertools && " |
| "dnf install -y %s && dnf clean all" % |
| (" ".join(sorted(self.pkgs)))) |
| return res; |
| |
| class centos9(Environment): |
| docker_parent = "quay.io/centos/centos:stream9" |
| pkgs = centos8.pkgs |
| name = "centos9" |
| specfile = "redhat/rdma-core.spec" |
| ninja_cmd = "ninja-build" |
| is_rpm = True |
| to_azp = True |
| proxy = False |
| |
| def get_docker_file(self,tmpdir): |
| res = DockerFile(self.docker_parent); |
| res.lines.append("RUN dnf install -y 'dnf-command(config-manager)' epel-release &&" |
| "dnf config-manager --set-enabled crb && " |
| "dnf install -y %s && dnf clean all" % |
| (" ".join(sorted(self.pkgs)))) |
| return res |
| |
| class fc41(Environment): |
| docker_parent = "fedora:41"; |
| pkgs = centos8.pkgs | {"util-linux"} |
| name = "fc41"; |
| specfile = "redhat/rdma-core.spec"; |
| ninja_cmd = "ninja-build"; |
| is_rpm = True; |
| aliases = {"fedora"}; |
| to_azp = True; |
| |
| def get_docker_file(self,tmpdir): |
| res = DockerFile(self.docker_parent); |
| res.lines.append("RUN dnf install -y %s && dnf clean all"%( |
| " ".join(sorted(self.pkgs)))); |
| return res; |
| |
| # ------------------------------------------------------------------------- |
| |
| class APTEnvironment(Environment): |
| is_deb = True; |
| build_python = True; |
| def get_docker_file(self,tmpdir): |
| res = DockerFile(self.docker_parent); |
| res.lines.append("RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends %s && apt-get clean && rm -rf /usr/share/doc/ /usr/lib/debug /var/lib/apt/lists/"%( |
| " ".join(sorted(self.pkgs)))); |
| return res; |
| |
| def add_source_list(self,tmpdir,name,content): |
| sld = os.path.join(tmpdir,"etc","apt","sources.list.d"); |
| if not os.path.isdir(sld): |
| os.makedirs(sld); |
| with open(os.path.join(sld,name),"w") as F: |
| F.write(content + "\n"); |
| |
| def fix_https(self,tmpdir): |
| """The ubuntu image does not include ca-certificates, so if we want to use |
| HTTPS disable certificate validation.""" |
| cfgd = os.path.join(tmpdir,"etc","apt","apt.conf.d") |
| if not os.path.isdir(cfgd): |
| os.makedirs(cfgd) |
| with open(os.path.join(cfgd,"01nossl"),"w") as F: |
| F.write('Acquire::https { Verify-Peer "false"; };') |
| |
| class xenial(APTEnvironment): |
| docker_parent = "ubuntu:16.04" |
| pkgs = { |
| 'build-essential', |
| 'cmake', |
| 'debhelper', |
| 'dh-systemd', |
| 'fakeroot', # for AZP |
| 'gcc', |
| 'libnl-3-dev', |
| 'libnl-route-3-dev', |
| 'libsystemd-dev', |
| 'libudev-dev', |
| 'make', |
| 'ninja-build', |
| 'pandoc', |
| 'pkg-config', |
| 'python3', |
| 'python3-docutils', |
| 'valgrind', |
| }; |
| name = "ubuntu-16.04"; |
| aliases = {"xenial"}; |
| to_azp = True; |
| |
| class bionic(APTEnvironment): |
| docker_parent = "ubuntu:18.04" |
| pkgs = xenial.pkgs | { |
| 'cython3', |
| 'python3-dev', |
| }; |
| name = "ubuntu-18.04"; |
| aliases = {"bionic", "ubuntu"}; |
| to_azp = True |
| |
| class focal(APTEnvironment): |
| docker_parent = "ubuntu:20.04" |
| pkgs = bionic.pkgs | { |
| 'dh-python', |
| } |
| name = "ubuntu-20.04"; |
| aliases = {"focal", "ubuntu"}; |
| to_azp = True |
| |
| class jammy(APTEnvironment): |
| docker_parent = "ubuntu:22.04" |
| pkgs = (bionic.pkgs ^ {"dh-systemd"}) | { |
| 'dh-python', |
| } |
| name = "ubuntu-22.04"; |
| aliases = {"jammy", "ubuntu"}; |
| |
| class jessie(APTEnvironment): |
| docker_parent = "debian:8" |
| pkgs = xenial.pkgs; |
| name = "debian-8"; |
| aliases = {"jessie"}; |
| build_pyverbs = False; |
| |
| class stretch(APTEnvironment): |
| docker_parent = "debian:9" |
| pkgs = bionic.pkgs; |
| name = "debian-9"; |
| aliases = {"stretch"}; |
| |
| class bullseye(APTEnvironment): |
| docker_parent = "debian:11" |
| pkgs = { |
| 'build-essential', |
| 'cmake', |
| 'debhelper', |
| 'fakeroot', # for AZP |
| 'gcc', |
| 'libnl-3-dev', |
| 'libnl-route-3-dev', |
| 'libsystemd-dev', |
| 'libudev-dev', |
| 'make', |
| 'ninja-build', |
| 'pandoc', |
| 'pkg-config', |
| 'python3', |
| 'python3-docutils', |
| 'valgrind', |
| }; |
| name = "debian-11"; |
| aliases = {"bullseye"}; |
| |
| class bullseye_i386(APTEnvironment): |
| docker_parent = "debian:11" |
| pkgs = bullseye.pkgs | {"nodejs"} |
| name = "debian-11-i386"; |
| aliases = {"bullseye_i386"}; |
| docker_opts = ["--platform","linux/386"] |
| to_azp = True |
| |
| def get_docker_file(self,tmpdir): |
| res = json.loads(docker_cmd_str(args,"manifest","inspect",self.docker_parent)) |
| |
| # Docker is somewhat obnoxious in how it handles the multi-platform |
| # images since it does not store the manifest locally and thus |
| # overwrites the local tag. Figure out the tag we want by hash and |
| # use it directly. Docker will cache this. |
| for image in res["manifests"]: |
| platform = image["platform"] |
| if platform["architecture"] == "386" and platform["os"] == "linux": |
| base = f"{self.docker_parent}@{image['digest']}" |
| break |
| else: |
| raise RuntimeError("Docker manifest failed"); |
| res = DockerFile(base); |
| res.lines.append("RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends %s && apt-get clean && rm -rf /usr/share/doc/ /usr/lib/debug /var/lib/apt/lists/"%( |
| " ".join(sorted(self.pkgs)))); |
| res.lines.append('LABEL "com.azure.dev.pipelines.agent.handler.node.path"="/usr/bin/node"') |
| return res; |
| |
| class debian_experimental(APTEnvironment): |
| docker_parent = "debian:experimental" |
| pkgs = (stretch.pkgs ^ {"gcc"}) | {"gcc-9"}; |
| name = "debian-experimental"; |
| |
| def get_docker_file(self,tmpdir): |
| res = DockerFile(self.docker_parent); |
| res.lines.append("RUN apt-get update && apt-get -t experimental install -y --no-install-recommends %s && apt-get clean"%( |
| " ".join(sorted(self.pkgs)))); |
| return res; |
| |
| # ------------------------------------------------------------------------- |
| |
| class ZypperEnvironment(Environment): |
| proxy = False; |
| is_rpm = True; |
| def get_docker_file(self,tmpdir): |
| res = DockerFile(self.docker_parent); |
| res.lines.append("RUN zypper --non-interactive refresh"); |
| res.lines.append("RUN zypper --non-interactive dist-upgrade"); |
| res.lines.append("RUN zypper --non-interactive install %s"%( |
| " ".join(sorted(self.pkgs)))); |
| return res; |
| |
| class leap(ZypperEnvironment): |
| docker_parent = "opensuse/leap:15.0"; |
| specfile = "suse/rdma-core.spec"; |
| pkgs = { |
| 'cmake', |
| 'gcc', |
| 'libnl3-devel', |
| 'libudev-devel', |
| 'udev', |
| 'make', |
| 'ninja', |
| 'pandoc', |
| 'pkg-config', |
| 'python3', |
| 'rpm-build', |
| 'systemd-devel', |
| 'valgrind-devel', |
| 'python3-Cython', |
| 'python3-devel', |
| 'python3-docutils', |
| }; |
| rpmbuild_options = [ "--without=curlmini" ]; |
| to_azp = True; |
| name = "opensuse-15.0"; |
| aliases = {"leap"}; |
| |
| class tumbleweed(ZypperEnvironment): |
| docker_parent = "opensuse/tumbleweed:latest"; |
| pkgs = (leap.pkgs ^ {"valgrind-devel"}) | { |
| "valgrind-client-headers", |
| "perl" |
| }; |
| name = "tumbleweed"; |
| specfile = "suse/rdma-core.spec"; |
| rpmbuild_options = [ "--without=curlmini" ]; |
| |
| # ------------------------------------------------------------------------- |
| |
| class azure_pipelines(APTEnvironment): |
| docker_parent = "ubuntu:22.04" |
| pkgs = { |
| "abi-compliance-checker", |
| "abi-dumper", |
| "ca-certificates", |
| "clang-15", |
| "cmake", |
| "cython3", |
| "debhelper", |
| "dh-python", |
| "dpkg-dev", |
| "fakeroot", |
| "gcc-12", |
| "git", |
| "libc6-dev", |
| "libnl-3-dev", |
| "libnl-route-3-dev", |
| "libsystemd-dev", |
| "libudev-dev", |
| "lintian", |
| "make", |
| "ninja-build", |
| "pandoc", |
| "pkg-config", |
| "python3", |
| "python3-dev", |
| "python3-docutils", |
| "python3-pkg-resources", |
| "python3-yaml", |
| "sparse", |
| "valgrind", |
| } | { |
| # ARM 64 cross compiler |
| "gcc-12-aarch64-linux-gnu", |
| "libgcc-12-dev:arm64", |
| "libc6-dev:arm64", |
| "libnl-3-dev:arm64", |
| "libnl-route-3-dev:arm64", |
| "libsystemd-dev:arm64", |
| "libudev-dev:arm64", |
| } | { |
| # PPC 64 cross compiler |
| "gcc-12-powerpc64le-linux-gnu", |
| "libgcc-12-dev:ppc64el", |
| "libc6-dev:ppc64el", |
| "libnl-3-dev:ppc64el", |
| "libnl-route-3-dev:ppc64el", |
| "libsystemd-dev:ppc64el", |
| "libudev-dev:ppc64el", |
| } |
| to_azp = True; |
| name = "azure_pipelines"; |
| aliases = {"azp"} |
| |
| llvm_sources = """ |
| Types: deb |
| URIs: http://apt.llvm.org/jammy/ |
| Suites: llvm-toolchain-jammy-15 |
| Components: main |
| Architectures: amd64 |
| Signed-By: |
| -----BEGIN PGP PUBLIC KEY BLOCK----- |
| Version: GnuPG v1.4.12 (GNU/Linux) |
| . |
| mQINBFE9lCwBEADi0WUAApM/mgHJRU8lVkkw0CHsZNpqaQDNaHefD6Rw3S4LxNmM |
| EZaOTkhP200XZM8lVdbfUW9xSjA3oPldc1HG26NjbqqCmWpdo2fb+r7VmU2dq3NM |
| R18ZlKixiLDE6OUfaXWKamZsXb6ITTYmgTO6orQWYrnW6ckYHSeaAkW0wkDAryl2 |
| B5v8aoFnQ1rFiVEMo4NGzw4UX+MelF7rxaaregmKVTPiqCOSPJ1McC1dHFN533FY |
| Wh/RVLKWo6npu+owtwYFQW+zyQhKzSIMvNujFRzhIxzxR9Gn87MoLAyfgKEzrbbT |
| DhqqNXTxS4UMUKCQaO93TzetX/EBrRpJj+vP640yio80h4Dr5pAd7+LnKwgpTDk1 |
| G88bBXJAcPZnTSKu9I2c6KY4iRNbvRz4i+ZdwwZtdW4nSdl2792L7Sl7Nc44uLL/ |
| ZqkKDXEBF6lsX5XpABwyK89S/SbHOytXv9o4puv+65Ac5/UShspQTMSKGZgvDauU |
| cs8kE1U9dPOqVNCYq9Nfwinkf6RxV1k1+gwtclxQuY7UpKXP0hNAXjAiA5KS5Crq |
| 7aaJg9q2F4bub0mNU6n7UI6vXguF2n4SEtzPRk6RP+4TiT3bZUsmr+1ktogyOJCc |
| Ha8G5VdL+NBIYQthOcieYCBnTeIH7D3Sp6FYQTYtVbKFzmMK+36ERreL/wARAQAB |
| tD1TeWx2ZXN0cmUgTGVkcnUgLSBEZWJpYW4gTExWTSBwYWNrYWdlcyA8c3lsdmVz |
| dHJlQGRlYmlhbi5vcmc+iQI4BBMBAgAiBQJRPZQsAhsDBgsJCAcDAgYVCAIJCgsE |
| FgIDAQIeAQIXgAAKCRAVz00Yr090Ibx+EADArS/hvkDF8juWMXxh17CgR0WZlHCC |
| 9CTBWkg5a0bNN/3bb97cPQt/vIKWjQtkQpav6/5JTVCSx2riL4FHYhH0iuo4iAPR |
| udC7Cvg8g7bSPrKO6tenQZNvQm+tUmBHgFiMBJi92AjZ/Qn1Shg7p9ITivFxpLyX |
| wpmnF1OKyI2Kof2rm4BFwfSWuf8Fvh7kDMRLHv+MlnK/7j/BNpKdozXxLcwoFBmn |
| l0WjpAH3OFF7Pvm1LJdf1DjWKH0Dc3sc6zxtmBR/KHHg6kK4BGQNnFKujcP7TVdv |
| gMYv84kun14pnwjZcqOtN3UJtcx22880DOQzinoMs3Q4w4o05oIF+sSgHViFpc3W |
| R0v+RllnH05vKZo+LDzc83DQVrdwliV12eHxrMQ8UYg88zCbF/cHHnlzZWAJgftg |
| hB08v1BKPgYRUzwJ6VdVqXYcZWEaUJmQAPuAALyZESw94hSo28FAn0/gzEc5uOYx |
| K+xG/lFwgAGYNb3uGM5m0P6LVTfdg6vDwwOeTNIExVk3KVFXeSQef2ZMkhwA7wya |
| KJptkb62wBHFE+o9TUdtMCY6qONxMMdwioRE5BYNwAsS1PnRD2+jtlI0DzvKHt7B |
| MWd8hnoUKhMeZ9TNmo+8CpsAtXZcBho0zPGz/R8NlJhAWpdAZ1CmcPo83EW86Yq7 |
| BxQUKnNHcwj2ebkCDQRRPZQsARAA4jxYmbTHwmMjqSizlMJYNuGOpIidEdx9zQ5g |
| zOr431/VfWq4S+VhMDhs15j9lyml0y4ok215VRFwrAREDg6UPMr7ajLmBQGau0Fc |
| bvZJ90l4NjXp5p0NEE/qOb9UEHT7EGkEhaZ1ekkWFTWCgsy7rRXfZLxB6sk7pzLC |
| DshyW3zjIakWAnpQ5j5obiDy708pReAuGB94NSyb1HoW/xGsGgvvCw4r0w3xPStw |
| F1PhmScE6NTBIfLliea3pl8vhKPlCh54Hk7I8QGjo1ETlRP4Qll1ZxHJ8u25f/ta |
| RES2Aw8Hi7j0EVcZ6MT9JWTI83yUcnUlZPZS2HyeWcUj+8nUC8W4N8An+aNps9l/ |
| 21inIl2TbGo3Yn1JQLnA1YCoGwC34g8QZTJhElEQBN0X29ayWW6OdFx8MDvllbBV |
| ymmKq2lK1U55mQTfDli7S3vfGz9Gp/oQwZ8bQpOeUkc5hbZszYwP4RX+68xDPfn+ |
| M9udl+qW9wu+LyePbW6HX90LmkhNkkY2ZzUPRPDHZANU5btaPXc2H7edX4y4maQa |
| xenqD0lGh9LGz/mps4HEZtCI5CY8o0uCMF3lT0XfXhuLksr7Pxv57yue8LLTItOJ |
| d9Hmzp9G97SRYYeqU+8lyNXtU2PdrLLq7QHkzrsloG78lCpQcalHGACJzrlUWVP/ |
| fN3Ht3kAEQEAAYkCHwQYAQIACQUCUT2ULAIbDAAKCRAVz00Yr090IbhWEADbr50X |
| OEXMIMGRLe+YMjeMX9NG4jxs0jZaWHc/WrGR+CCSUb9r6aPXeLo+45949uEfdSsB |
| pbaEdNWxF5Vr1CSjuO5siIlgDjmT655voXo67xVpEN4HhMrxugDJfCa6z97P0+ML |
| PdDxim57uNqkam9XIq9hKQaurxMAECDPmlEXI4QT3eu5qw5/knMzDMZj4Vi6hovL |
| wvvAeLHO/jsyfIdNmhBGU2RWCEZ9uo/MeerPHtRPfg74g+9PPfP6nyHD2Wes6yGd |
| oVQwtPNAQD6Cj7EaA2xdZYLJ7/jW6yiPu98FFWP74FN2dlyEA2uVziLsfBrgpS4l |
| tVOlrO2YzkkqUGrybzbLpj6eeHx+Cd7wcjI8CalsqtL6cG8cUEjtWQUHyTbQWAgG |
| 5VPEgIAVhJ6RTZ26i/G+4J8neKyRs4vz+57UGwY6zI4AB1ZcWGEE3Bf+CDEDgmnP |
| LSwbnHefK9IljT9XU98PelSryUO/5UPw7leE0akXKB4DtekToO226px1VnGp3Bov |
| 1GBGvpHvL2WizEwdk+nfk8LtrLzej+9FtIcq3uIrYnsac47Pf7p0otcFeTJTjSq3 |
| krCaoG4Hx0zGQG2ZFpHrSrZTVy6lxvIdfi0beMgY6h78p6M9eYZHQHc02DjFkQXN |
| bXb5c6gCHESH5PXwPU4jQEE7Ib9J6sbk7ZT2Mw== |
| =j+4q |
| -----END PGP PUBLIC KEY BLOCK----- |
| """ |
| |
| gcc12_sources = """ |
| Types: deb |
| URIs: https://ppa.launchpadcontent.net/ubuntu-toolchain-r/test/ubuntu |
| Suites: jammy |
| Components: main |
| Architectures: amd64 arm64 ppc64el |
| Signed-By: |
| -----BEGIN PGP PUBLIC KEY BLOCK----- |
| . |
| xo0ESuBvRwEEAMi4cDba7xlKaaoXjO1n1HX8RKrkW+HEIl79nSOSJyvzysajs7zU |
| ow/OzCQp9NswqrDmNuH1+lPTTRNAGtK8r2ouq2rnXT1mTl23dpgHZ9spseR73s4Z |
| BGw/ag4bpU5dNUStvfmHhIjVCuiSpNn7cyy1JSSvSs3N2mxteKjXLBf7ABEBAAHN |
| GkxhdW5jaHBhZCBUb29sY2hhaW4gYnVpbGRzwrYEEwECACAFAkrgb0cCGwMGCwkI |
| BwMCBBUCCAMEFgIDAQIeAQIXgAAKCRAek3eiup7yfzGKA/4xzUqNACSlB+k+DxFF |
| HqkwKa/ziFiAlkLQyyhm+iqz80htRZr7Ls/ZRYZl0aSU56/hLe0V+TviJ1s8qdN2 |
| lamkKdXIAFfavA04nOnTzyIBJ82EAUT3Nh45skMxo4z4iZMNmsyaQpNl/m/lNtOL |
| hR64v5ZybofB2EWkMxUzX8D/FQ== |
| =xe+/ |
| -----END PGP PUBLIC KEY BLOCK----- |
| """ |
| |
| ports_sources = """ |
| Types: deb |
| URIS: http://ports.ubuntu.com |
| Suites: jammy jammy-security jammy-updates |
| Components: main universe |
| Architectures: arm64 ppc64el |
| """ |
| |
| amd64_sources = """ |
| Types: deb |
| URIS: http://archive.ubuntu.com/ubuntu |
| Suites: jammy jammy-security jammy-updates |
| Components: main universe |
| Architectures: amd64 |
| """ |
| |
| def get_docker_file(self,tmpdir): |
| res = focal.get_docker_file(self,tmpdir); |
| self.fix_https(tmpdir) |
| |
| self.add_source_list(tmpdir, "llvm.sources", self.llvm_sources) |
| self.add_source_list(tmpdir, |
| "ubuntu-toolchain-r-ubuntu-test-jammy.sources", |
| self.gcc12_sources) |
| self.add_source_list(tmpdir, "ports.sources", self.ports_sources) |
| |
| # Replace the main sources so we can limit the architecture |
| self.add_source_list(tmpdir, "amd64.sources", self.amd64_sources) |
| |
| res.lines.insert(1,"ADD etc/ /etc/"); |
| res.lines.insert(1,"RUN rm /etc/apt/sources.list && " |
| "dpkg --add-architecture ppc64el && " |
| "dpkg --add-architecture arm64") |
| return res; |
| |
| # ------------------------------------------------------------------------- |
| |
| environments = [centos7(), |
| centos7_epel(), |
| centos8(), |
| centos9(), |
| amazonlinux2(), |
| xenial(), |
| bionic(), |
| focal(), |
| jammy(), |
| jessie(), |
| stretch(), |
| fc41(), |
| leap(), |
| tumbleweed(), |
| debian_experimental(), |
| azure_pipelines(), |
| bullseye(), |
| bullseye_i386(), |
| ]; |
| |
| class ToEnvActionPkg(argparse.Action): |
| """argparse helper to parse environment lists into environment classes""" |
| def __call__(self, parser, namespace, values, option_string=None): |
| if not isinstance(values,list): |
| values = [values]; |
| |
| res = set(); |
| for I in values: |
| if I == "all": |
| for env in environments: |
| if env.name != "centos7_epel": |
| res.add(env); |
| else: |
| for env in environments: |
| if env.name == I or I in env.aliases: |
| res.add(env); |
| setattr(namespace, self.dest, sorted(res,key=lambda x:x.name)) |
| |
| |
| class ToEnvAction(argparse.Action): |
| """argparse helper to parse environment lists into environment classes""" |
| def __call__(self, parser, namespace, values, option_string=None): |
| if not isinstance(values,list): |
| values = [values]; |
| |
| res = set(); |
| for I in values: |
| if I == "all": |
| res.update(environments); |
| else: |
| for env in environments: |
| if env.name == I or I in env.aliases: |
| res.add(env); |
| setattr(namespace, self.dest, sorted(res,key=lambda x:x.name)) |
| |
| def env_choices_pkg(): |
| """All the names that can be used with ToEnvAction""" |
| envs = set(("all",)); |
| for I in environments: |
| if getattr(I,"is_deb",False) or getattr(I,"is_rpm",False): |
| envs.add(I.name); |
| envs.update(I.aliases); |
| return envs; |
| |
| def env_choices(): |
| """All the names that can be used with ToEnvAction""" |
| envs = set(("all",)); |
| for I in environments: |
| envs.add(I.name); |
| envs.update(I.aliases); |
| return envs; |
| |
| def docker_cmd(env,*cmd): |
| """Invoke docker""" |
| cmd = list(cmd); |
| if env.sudo: |
| return subprocess.check_call(["sudo","docker"] + cmd); |
| return subprocess.check_call(["docker"] + cmd); |
| |
| def docker_cmd_str(env,*cmd): |
| """Invoke docker""" |
| cmd = list(cmd); |
| if env.sudo: |
| return subprocess.check_output(["sudo","docker"] + cmd).decode(); |
| return subprocess.check_output(["docker"] + cmd).decode(); |
| |
| @contextmanager |
| def private_tmp(args): |
| """Simple version of Python 3's tempfile.TemporaryDirectory""" |
| dfn = tempfile.mkdtemp(); |
| try: |
| yield dfn; |
| finally: |
| try: |
| shutil.rmtree(dfn); |
| except: |
| # The debian builds result in root owned files because we don't use fakeroot |
| subprocess.check_call(['sudo','rm','-rf',dfn]); |
| |
| @contextmanager |
| def inDirectory(dir): |
| cdir = os.getcwd(); |
| try: |
| os.chdir(dir); |
| yield True; |
| finally: |
| os.chdir(cdir); |
| |
| def map_git_args(src_root,to): |
| """Return a list of docker arguments that will map the .git directory into the |
| container""" |
| git_dir = subprocess.check_output([ |
| "git", |
| "-C", src_root, |
| "rev-parse", |
| "--absolute-git-dir", |
| ]).decode().strip() |
| if ".git/worktrees" in git_dir: |
| with open(os.path.join(git_dir, "commondir")) as F: |
| git_dir = os.path.join(git_dir, F.read().strip()) |
| git_dir = os.path.abspath(git_dir) |
| res = ["-v", "%s:%s:ro" % (os.path.join(src_root, ".git"), os.path.join(to, ".git")), |
| "-v", "%s:%s:ro" % (git_dir, git_dir)] |
| else: |
| res = ["-v", "%s:%s:ro" % (git_dir, os.path.join(to, ".git"))] |
| |
| alternates = os.path.join(git_dir, "objects/info/alternates") |
| if os.path.exists(alternates): |
| with open(alternates) as F: |
| for I in F.readlines(): |
| I = os.path.normpath(I.strip()) |
| res.extend(["-v","%s:%s:ro"%(I,I)]); |
| |
| return res; |
| |
| def get_image_id(args,image_name): |
| img = json.loads(docker_cmd_str(args,"inspect",image_name)); |
| image_id = img[0]["Id"]; |
| # Newer dockers put a prefix |
| if ":" in image_id: |
| image_id = image_id.partition(':')[2]; |
| return image_id; |
| |
| # ------------------------------------------------------------------------- |
| |
| def get_tar_file(args,tarfn,pandoc_prebuilt=False): |
| """Create a tar file that matches what buildlib/github-release would do if it |
| was a tagged release""" |
| prefix = "%s-%s/"%(project,get_version()); |
| if not pandoc_prebuilt: |
| subprocess.check_call(["git","archive", |
| # This must match the prefix generated buildlib/github-release |
| "--prefix",prefix, |
| "--output",tarfn, |
| "HEAD"]); |
| return; |
| |
| # When the OS does not support pandoc we got through the extra step to |
| # build pandoc output in the azp container and include it in the |
| # tar. |
| if not args.use_prebuilt_pandoc: |
| subprocess.check_call(["buildlib/cbuild","make","azure_pipelines","docs"]); |
| |
| cmd_make_dist_tar(argparse.Namespace(BUILD="build-azure_pipelines",tarfn=tarfn, |
| script_pwd="",tag=None)); |
| |
| def run_rpm_build(args,spec_file,env): |
| with open(spec_file,"r") as F: |
| for ln in F: |
| if ln.startswith("Version:"): |
| ver = ln.strip().partition(' ')[2].strip(); |
| assert(ver == get_version()); |
| |
| if ln.startswith("Source:"): |
| tarfn = ln.strip().partition(' ')[2].strip(); |
| |
| image_id = get_image_id(args,env.image_name()); |
| with private_tmp(args) as tmpdir: |
| os.mkdir(os.path.join(tmpdir,"SOURCES")); |
| os.mkdir(os.path.join(tmpdir,"tmp")); |
| |
| get_tar_file(args,os.path.join(tmpdir,"SOURCES",tarfn), |
| pandoc_prebuilt=not env.pandoc); |
| |
| with open(spec_file,"r") as inF: |
| spec = list(inF); |
| tspec_file = os.path.basename(spec_file); |
| with open(os.path.join(tmpdir,tspec_file),"w") as outF: |
| outF.write("".join(spec)); |
| |
| home = os.path.join(os.path.sep,"home",os.getenv("LOGNAME")); |
| vdir = os.path.join(home,"rpmbuild"); |
| |
| opts = [ |
| "run", |
| "--rm=true", |
| "-v","%s:%s"%(tmpdir,vdir), |
| "-w",vdir, |
| "-h","builder-%s"%(image_id[:12]), |
| "-e","HOME=%s"%(home), |
| "-e","TMPDIR=%s"%(os.path.join(vdir,"tmp")), |
| ]; |
| |
| # rpmbuild complains if we do not have an entry in passwd and group |
| # for the user we are going to use to do the build. |
| with open(os.path.join(tmpdir,"go.py"),"w") as F: |
| print(""" |
| import os,subprocess; |
| with open("/etc/passwd","a") as F: |
| F.write({passwd!r} + "\\n"); |
| with open("/etc/group","a") as F: |
| F.write({group!r} + "\\n"); |
| os.setgid({gid:d}); |
| os.setuid({uid:d}); |
| |
| # Get RPM to tell us the expected tar filename. |
| for ln in subprocess.check_output(["rpmspec","-P",{tspec_file!r}]).splitlines(): |
| if ln.startswith(b"Source:"): |
| tarfn = ln.strip().partition(b' ')[2].strip(); |
| if tarfn != {tarfn!r}: |
| os.symlink({tarfn!r},os.path.join(b"SOURCES",tarfn)); |
| """.format(passwd=":".join(str(I) for I in pwd.getpwuid(os.getuid())), |
| group=":".join(str(I) for I in grp.getgrgid(os.getgid())), |
| uid=os.getuid(), |
| gid=os.getgid(), |
| tarfn=tarfn, |
| tspec_file=tspec_file), file=F); |
| |
| extra_opts = getattr(env,"rpmbuild_options", []) |
| bopts = ["-bb",tspec_file] + extra_opts; |
| for arg in args.with_flags: |
| bopts.extend(["--with", arg]); |
| for arg in args.without_flags: |
| bopts.extend(["--without", arg]); |
| if "pyverbs" not in args.with_flags + args.without_flags: |
| if env.build_pyverbs: |
| bopts.extend(["--with", "pyverbs"]); |
| |
| print('os.execlp("rpmbuild","rpmbuild",%s)'%( |
| ",".join(repr(I) for I in bopts)), file=F); |
| |
| if args.run_shell: |
| opts.append("-ti"); |
| opts.append(env.image_name()); |
| |
| if args.run_shell: |
| opts.append("/bin/bash"); |
| else: |
| opts.extend([env.python_cmd,"go.py"]); |
| |
| docker_cmd(args,*opts) |
| |
| print() |
| for path,jnk,files in os.walk(os.path.join(tmpdir,"RPMS")): |
| for I in files: |
| print("Final RPM: ",os.path.join("..",I)); |
| shutil.move(os.path.join(path,I), |
| os.path.join("..",I)); |
| |
| def run_deb_build(args,env): |
| image_id = get_image_id(args,env.image_name()); |
| with private_tmp(args) as tmpdir: |
| os.mkdir(os.path.join(tmpdir,"src")); |
| os.mkdir(os.path.join(tmpdir,"tmp")); |
| |
| opwd = os.getcwd(); |
| with inDirectory(os.path.join(tmpdir,"src")): |
| subprocess.check_call(["git", |
| "--git-dir",os.path.join(opwd,".git"), |
| "reset","--hard","HEAD"]); |
| |
| home = os.path.join(os.path.sep,"home",os.getenv("LOGNAME")); |
| |
| opts = [ |
| "run", |
| "--read-only", |
| "--rm=true", |
| "-v","%s:%s"%(tmpdir,home), |
| "-w",os.path.join(home,"src"), |
| "-h","builder-%s"%(image_id[:12]), |
| "-e","HOME=%s"%(home), |
| "-e","TMPDIR=%s"%(os.path.join(home,"tmp")), |
| "-e","DEB_BUILD_OPTIONS=parallel=%u"%(multiprocessing.cpu_count()), |
| ]; |
| |
| # Create a go.py that will let us run the compilation as the user and |
| # then switch to root only for the packaging step. |
| with open(os.path.join(tmpdir,"go.py"),"w") as F: |
| print(""" |
| import subprocess,os; |
| def to_user(): |
| os.setgid({gid:d}); |
| os.setuid({uid:d}); |
| subprocess.check_call(["debian/rules","debian/rules","build"], |
| preexec_fn=to_user); |
| subprocess.check_call(["debian/rules","debian/rules","binary"]); |
| """.format(uid=os.getuid(), |
| gid=os.getgid()), file=F); |
| |
| if args.run_shell: |
| opts.append("-ti"); |
| opts.append(env.image_name()); |
| |
| if args.run_shell: |
| opts.append("/bin/bash"); |
| else: |
| opts.extend(["python3",os.path.join(home,"go.py")]); |
| |
| docker_cmd(args,*opts); |
| |
| print() |
| for I in os.listdir(tmpdir): |
| if I.endswith(".deb"): |
| print("Final DEB: ",os.path.join("..",I)); |
| shutil.move(os.path.join(tmpdir,I), |
| os.path.join("..",I)); |
| |
| def copy_abi_files(src): |
| """Retrieve the current ABI files and place them in the source tree.""" |
| if not os.path.isdir(src): |
| return; |
| |
| for path,jnk,files in os.walk(src): |
| for I in files: |
| if not I.startswith("current-"): |
| continue; |
| |
| ref_fn = os.path.join("ABI",I[8:]); |
| cur_fn = os.path.join(src, path, I); |
| |
| if os.path.isfile(ref_fn) and filecmp.cmp(ref_fn,cur_fn,False): |
| continue; |
| |
| print("Changed ABI File: ", ref_fn); |
| shutil.copy(cur_fn, ref_fn); |
| |
| def run_azp_build(args,env): |
| # Load the commands from the pipelines file |
| with open("buildlib/azure-pipelines.yml") as F: |
| azp = yaml.safe_load(F); |
| for bst in azp["stages"]: |
| if bst["stage"] == "Build": |
| break; |
| else: |
| raise ValueError("No Build stage found"); |
| for job in bst["jobs"]: |
| if job["job"] == "Compile": |
| break; |
| else: |
| raise ValueError("No Compile job found"); |
| |
| script = ["#!/bin/bash"] |
| workdir = "/__w/1" |
| srcdir = os.path.join(workdir,"s"); |
| for I in job["steps"]: |
| script.append("echo ==================================="); |
| script.append("echo %s"%(I["displayName"])); |
| script.append("cd %s"%(srcdir)); |
| if "bash" in I: |
| script.append(I["bash"]); |
| elif I.get("task") == "PythonScript@0": |
| script.append("set -e"); |
| if "workingDirectory" in I["inputs"]: |
| script.append("cd %s"%(os.path.join(srcdir,I["inputs"]["workingDirectory"]))); |
| script.append("%s %s %s"%(I["inputs"]["pythonInterpreter"], |
| os.path.join(srcdir,I["inputs"]["scriptPath"]), |
| I["inputs"].get("arguments",""))); |
| else: |
| raise ValueError("Unknown stanza %r"%(I)); |
| |
| with private_tmp(args) as tmpdir: |
| os.mkdir(os.path.join(tmpdir,"s")); |
| os.mkdir(os.path.join(tmpdir,"tmp")); |
| |
| opwd = os.getcwd(); |
| with inDirectory(os.path.join(tmpdir,"s")): |
| subprocess.check_call(["git", |
| "--git-dir",os.path.join(opwd,".git"), |
| "reset","--hard","HEAD"]); |
| subprocess.check_call(["git", |
| "--git-dir",os.path.join(opwd,".git"), |
| "fetch", |
| "--no-tags", |
| "https://github.com/linux-rdma/rdma-core.git","HEAD", |
| "master"]); |
| base = subprocess.check_output(["git", |
| "--git-dir",os.path.join(opwd,".git"), |
| "merge-base", |
| "HEAD","FETCH_HEAD"]).decode().strip(); |
| |
| opts = [ |
| "run", |
| "--read-only", |
| "--rm=true", |
| "-v","%s:%s"%(tmpdir, workdir), |
| "-w",srcdir, |
| "-u",str(os.getuid()), |
| "-e","SYSTEM_PULLREQUEST_SOURCECOMMITID=HEAD", |
| # azp puts the branch name 'master' here, we need to put a commit ID.. |
| "-e","SYSTEM_PULLREQUEST_TARGETBRANCH=%s"%(base), |
| "-e","HOME=%s"%(workdir), |
| "-e","TMPDIR=%s"%(os.path.join(workdir,"tmp")), |
| ] + map_git_args(opwd,srcdir); |
| |
| if args.run_shell: |
| opts.append("-ti"); |
| opts.append(env.image_name()); |
| |
| with open(os.path.join(tmpdir,"go.sh"),"w") as F: |
| F.write("\n".join(script)) |
| |
| if args.run_shell: |
| opts.append("/bin/bash"); |
| else: |
| opts.extend(["/bin/bash",os.path.join(workdir,"go.sh")]); |
| |
| try: |
| docker_cmd(args,*opts); |
| except subprocess.CalledProcessError as e: |
| copy_abi_files(os.path.join(tmpdir, "s/ABI")); |
| raise; |
| copy_abi_files(os.path.join(tmpdir, "s/ABI")); |
| |
| def args_pkg(parser): |
| parser.add_argument("ENV",action=ToEnvActionPkg,choices=env_choices_pkg()); |
| parser.add_argument("--run-shell",default=False,action="store_true", |
| help="Instead of running the build, enter a shell"); |
| parser.add_argument("--use-prebuilt-pandoc",default=False,action="store_true", |
| help="Do not rebuild the pandoc cache in build-azure_pipelines/pandoc-prebuilt/"); |
| parser.add_argument("--with", default=[],action="append", dest="with_flags", |
| help="Enable specified feature in RPM builds"); |
| parser.add_argument("--without", default=[],action="append", dest="without_flags", |
| help="Disable specified feature in RPM builds"); |
| def cmd_pkg(args): |
| """Build a package in the given environment.""" |
| for env in args.ENV: |
| if env.name == "azure_pipelines": |
| run_azp_build(args,env); |
| elif getattr(env,"is_deb",False): |
| run_deb_build(args,env); |
| elif getattr(env,"is_rpm",False): |
| run_rpm_build(args, |
| getattr(env,"specfile","%s.spec"%(project)), |
| env); |
| else: |
| print("%s does not support packaging"%(env.name)); |
| |
| # ------------------------------------------------------------------------- |
| |
| def args_make(parser): |
| parser.add_argument("--run-shell",default=False,action="store_true", |
| help="Instead of running the build, enter a shell"); |
| parser.add_argument("ENV",action=ToEnvAction,choices=env_choices()); |
| parser.add_argument('ARGS', nargs=argparse.REMAINDER); |
| def cmd_make(args): |
| """Run cmake and ninja within a docker container. If cmake has not yet been |
| run then this runs it with the given environment variables, then invokes ninja. |
| Otherwise ninja is invoked without calling cmake.""" |
| SRC = os.getcwd(); |
| |
| for env in args.ENV: |
| BUILD = "build-%s"%(env.name) |
| if not os.path.exists(BUILD): |
| os.mkdir(BUILD); |
| |
| home = os.path.join(os.path.sep,"home",os.getenv("LOGNAME")); |
| |
| dirs = [os.getcwd(),"/tmp"]; |
| # Import the symlink target too if BUILD is a symlink |
| BUILD_r = os.path.realpath(BUILD); |
| if not BUILD_r.startswith(os.path.realpath(SRC)): |
| dirs.append(BUILD_r); |
| |
| cmake_args = [] |
| if not env.build_pyverbs: |
| cmake_args.extend(["-DNO_PYVERBS=1"]); |
| |
| cmake_envs = [] |
| ninja_args = [] |
| for I in args.ARGS: |
| if I.startswith("-D"): |
| cmake_args.append(I); |
| elif I.find('=') != -1: |
| cmake_envs.append(I); |
| else: |
| ninja_args.append(I); |
| if env.use_make: |
| need_cmake = not os.path.exists(os.path.join(BUILD_r,"Makefile")); |
| else: |
| need_cmake = not os.path.exists(os.path.join(BUILD_r,"build.ninja")); |
| opts = ["run", |
| "--read-only", |
| "--rm=true", |
| "-ti", |
| "-u",str(os.getuid()), |
| "-e","HOME=%s"%(home), |
| "-w",BUILD_r, |
| ]; |
| opts.extend(env.docker_opts) |
| for I in dirs: |
| opts.append("-v"); |
| opts.append("%s:%s"%(I,I)); |
| for I in cmake_envs: |
| opts.append("-e"); |
| opts.append(I); |
| if args.run_shell: |
| opts.append("-ti"); |
| opts.append(env.image_name()); |
| |
| if args.run_shell: |
| os.execlp("sudo","sudo","docker",*(opts + ["/bin/bash"])); |
| |
| if need_cmake: |
| if env.use_make: |
| prog_args = ["cmake",SRC] + cmake_args; |
| else: |
| prog_args = ["cmake","-GNinja",SRC] + cmake_args; |
| docker_cmd(args,*(opts + prog_args)); |
| |
| if env.use_make: |
| prog_args = ["make","-C",BUILD_r] + ninja_args; |
| else: |
| prog_args = [getattr(env,"ninja_cmd","ninja"), |
| "-C",BUILD_r] + ninja_args; |
| |
| if len(args.ENV) <= 1: |
| os.execlp("sudo","sudo","docker",*(opts + prog_args)); |
| else: |
| docker_cmd(args,*(opts + prog_args)); |
| |
| # ------------------------------------------------------------------------- |
| |
| def get_build_args(args,env): |
| """Return extra docker arguments for building. This is the system APT proxy.""" |
| res = []; |
| if args.pull: |
| res.append("--pull"); |
| |
| if env.proxy and os.path.exists("/etc/apt/apt.conf.d/01proxy"): |
| # The line in this file must be 'Acquire::http { Proxy "http://xxxx:3142"; };' |
| with open("/etc/apt/apt.conf.d/01proxy") as F: |
| proxy = F.read().strip().split('"')[1]; |
| res.append("--build-arg"); |
| res.append('http_proxy=%s'%(proxy)); |
| return res; |
| |
| def args_build_images(parser): |
| parser.add_argument("ENV",nargs="+",action=ToEnvAction,choices=env_choices()); |
| parser.add_argument("--no-pull",default=True,action="store_false", |
| dest="pull", |
| help="Instead of running the build, enter a shell"); |
| def cmd_build_images(args): |
| """Run from the top level source directory to make the docker images that are |
| needed for building. This only needs to be run once.""" |
| # Docker copies the permissions from the local host and we need this umask |
| # to be 022 or the container breaks |
| os.umask(0o22) |
| for env in args.ENV: |
| with private_tmp(args) as tmpdir: |
| df = env.get_docker_file(tmpdir); |
| fn = os.path.join(tmpdir,"Dockerfile"); |
| with open(fn,"wt") as F: |
| for ln in df.lines: |
| print(ln, file=F); |
| opts = (["build"] + |
| get_build_args(args,env) + |
| env.docker_opts + |
| ["-f",fn, |
| "-t",env.image_name(), |
| tmpdir]); |
| print(opts) |
| docker_cmd(args,*opts); |
| |
| # ------------------------------------------------------------------------- |
| |
| def args_push_azp_images(args): |
| pass |
| def cmd_push_azp_images(args): |
| """Push the images required for Azure Pipelines to the container |
| registry. Must have done 'az login' first""" |
| subprocess.check_call(["sudo","az","acr","login","--name","ucfconsort"]); |
| with private_tmp(args) as tmpdir: |
| nfn = os.path.join(tmpdir,"build.ninja"); |
| with open(nfn,"w") as F: |
| F.write("""rule push |
| command = docker push $img |
| description=Push $img\n"""); |
| |
| for env in environments: |
| name = env.image_name() |
| if "ucfconsort.azurecr.io" not in name: |
| continue |
| F.write("build push_%s : push\n img = %s\n"%(env.name,env.image_name())); |
| F.write("default push_%s\n"%(env.name)); |
| subprocess.check_call(["sudo","ninja"],cwd=tmpdir); |
| |
| # ------------------------------------------------------------------------- |
| def args_make_dist_tar(parser): |
| parser.add_argument("BUILD",help="Path to the build directory") |
| parser.add_argument("--tarfn",help="Output TAR filename") |
| parser.add_argument("--tag",help="git tag to sanity check against") |
| def cmd_make_dist_tar(args): |
| """Make the standard distribution tar. The BUILD argument must point to a build |
| output directory that has pandoc-prebuilt""" |
| ver = get_version(); |
| |
| if not args.tarfn: |
| args.tarfn = "%s-%s.tar.gz"%(project,ver) |
| |
| # The tag name and the cmake file must match. |
| if args.tag: |
| assert args.tag == "v" + ver; |
| |
| os.umask(0o22) |
| with private_tmp(args) as tmpdir: |
| tmp_tarfn = os.path.join(tmpdir,"tmp.tar"); |
| |
| prefix = "%s-%s/"%(project,get_version()); |
| subprocess.check_call(["git","archive", |
| "--prefix",prefix, |
| "--output",tmp_tarfn, |
| "HEAD"]); |
| |
| # Mangle the paths and append the prebuilt stuff to the tar file |
| if args.BUILD: |
| subprocess.check_call([ |
| "tar", |
| "-C",os.path.join(args.script_pwd,args.BUILD,"pandoc-prebuilt"), |
| "-rf",tmp_tarfn, |
| "./", |
| "--xform",r"s|^\.|%sbuildlib/pandoc-prebuilt|g"%(prefix)]); |
| |
| assert args.tarfn.endswith(".gz") or args.tarfn.endswith(".tgz"); |
| with open(os.path.join(args.script_pwd,args.tarfn),"w") as F: |
| subprocess.check_call(["gzip","-9c",tmp_tarfn],stdout=F); |
| |
| # ------------------------------------------------------------------------- |
| |
| if __name__ == '__main__': |
| parser = argparse.ArgumentParser(description='Operate docker for building this package') |
| subparsers = parser.add_subparsers(title="Sub Commands",dest="command"); |
| subparsers.required = True; |
| |
| funcs = globals(); |
| for k,v in list(funcs.items()): |
| if k.startswith("cmd_") and inspect.isfunction(v): |
| sparser = subparsers.add_parser(k[4:].replace('_','-'), |
| help=v.__doc__); |
| sparser.required = True; |
| funcs["args_" + k[4:]](sparser); |
| sparser.set_defaults(func=v); |
| |
| try: |
| import argcomplete; |
| argcomplete.autocomplete(parser); |
| except ImportError: |
| pass; |
| |
| args = parser.parse_args(); |
| args.sudo = True; |
| |
| # This script must always run from the top of the git tree, and a git |
| # checkout is mandatory. |
| git_top = subprocess.check_output(["git","rev-parse","--show-toplevel"]).strip(); |
| args.script_pwd = os.getcwd(); |
| os.chdir(git_top); |
| |
| args.func(args); |
