From 08cfd7ead1c885a11cd2fdf4ffdab4e5b6475935 Mon Sep 17 00:00:00 2001 From: Kenny Lee Sin Cheong <2530351+kleesc@users.noreply.github.com> Date: Tue, 14 Jul 2020 12:55:47 -0400 Subject: [PATCH] Update the executor image from Container Linux to Fedora CoreOS (#434) * Update the executor image from Container Linux to Fedora CoreOS * Move the container cloud config script for templating from devtable to quay's repo * Ignition config template * Move dockersystemd from devtable repo * Remove pinned dependency on devtable/container-cloud-config * Removes squashed image and logentries * Update builder image * Update mounted cert directory for Fedora * Removes old clouconfig template * Pass userdata as firmware config to qemu * Use CentOS:8 as base image --- buildman/container_cloud_config.py | 116 +++++++++++++++++++++++ buildman/manager/executor.py | 55 ++++++----- buildman/qemu-coreos/Dockerfile | 31 +++--- buildman/qemu-coreos/build.sh | 11 +++ buildman/qemu-coreos/start.sh | 9 +- buildman/templates/cloudconfig.json | 131 ++++++++++++++++++++++++++ buildman/templates/cloudconfig.yaml | 102 -------------------- buildman/templates/dockersystemd.json | 54 +++++++++++ requirements.txt | 1 - 9 files changed, 362 insertions(+), 148 deletions(-) create mode 100644 buildman/container_cloud_config.py create mode 100755 buildman/qemu-coreos/build.sh create mode 100644 buildman/templates/cloudconfig.json delete mode 100644 buildman/templates/cloudconfig.yaml create mode 100644 buildman/templates/dockersystemd.json diff --git a/buildman/container_cloud_config.py b/buildman/container_cloud_config.py new file mode 100644 index 000000000..1ed17df28 --- /dev/null +++ b/buildman/container_cloud_config.py @@ -0,0 +1,116 @@ +""" +Provides helper methods and templates for generating cloud config for running containers. + +Originally from https://github.com/DevTable/container-cloud-config +""" + +from functools import partial + +import base64 +import json +import os +import requests +import logging + +try: + # Python 3 + from urllib.request import HTTPRedirectHandler, build_opener, install_opener, urlopen, Request + from urllib.error import HTTPError + from urllib.parse import quote as urlquote +except ImportError: + # Python 2 + from urllib2 import ( + HTTPRedirectHandler, + build_opener, + install_opener, + urlopen, + Request, + HTTPError, + ) + from urllib import quote as urlquote + +from jinja2 import FileSystemLoader, Environment, StrictUndefined + +logger = logging.getLogger(__name__) + + +class CloudConfigContext(object): + """ Context object for easy generating of cloud config. """ + + def populate_jinja_environment(self, env): + """ Populates the given jinja environment with the methods defined in this context. """ + env.filters["registry"] = self.registry + env.filters["dataurl"] = self.data_url + env.filters["jsonify"] = json.dumps + env.globals["dockersystemd"] = self._dockersystemd_template + + def _dockersystemd_template( + self, + name, + container, + username="", + password="", + tag="latest", + extra_args="", + command="", + after_units=[], + exec_start_post=[], + exec_stop_post=[], + restart_policy="always", + oneshot=False, + env_file=None, + onfailure_units=[], + requires_units=[], + wants_units=[], + timeout_start_sec=None, + timeout_stop_sec=None, + autostart=True, + ): + + path = os.path.join(os.path.dirname(__file__), "templates") + env = Environment(loader=FileSystemLoader(path), undefined=StrictUndefined) + self.populate_jinja_environment(env) + template = env.get_template("dockersystemd.json") + return template.render( + name=name, + container=container, + username=username, + password=password, + tag=tag, + extra_args=extra_args, + command=command, + after_units=after_units, + requires_units=requires_units, + wants_units=wants_units, + onfailure_units=onfailure_units, + exec_start_post=exec_start_post, + exec_stop_post=exec_stop_post, + restart_policy=restart_policy, + oneshot=oneshot, + autostart=autostart, + timeout_start_sec=timeout_start_sec, + timeout_stop_sec=timeout_stop_sec, + env_file=env_file, + ) + + def data_url(self, content): + """ Encodes the content of an ignition file using RFC 2397. """ + data = "," + urlquote(content) + return "data:" + data + + + def registry(self, container_name): + """ Parse the registry from repositories of the following formats: + quay.io/quay/quay:tagname -> quay.io + localhost:5000/quay/quay:tagname -> localhost:5000 + localhost:5000/quay/quay -> localhost:5000 + quay/quay:latest -> '' + quay/quay -> '' + mysql:latest -> '' + mysql -> '' + """ + num_slashes = container_name.count("/") + if num_slashes == 2: + return container_name[: container_name.find("/")] + else: + return "" diff --git a/buildman/manager/executor.py b/buildman/manager/executor.py index a5820db0b..931c11d0b 100644 --- a/buildman/manager/executor.py +++ b/buildman/manager/executor.py @@ -1,6 +1,8 @@ import asyncio import datetime import hashlib +import io +import json import logging import os import socket @@ -16,7 +18,6 @@ import botocore import cachetools.func import requests -from container_cloud_config import CloudConfigContext from jinja2 import FileSystemLoader, Environment from prometheus_client import Histogram @@ -25,6 +26,7 @@ import release from _init import ROOT_DIR from app import app from buildman.asyncutil import AsyncWrapper +from buildman.container_cloud_config import CloudConfigContext logger = logging.getLogger(__name__) @@ -36,8 +38,8 @@ _TAG_RETRY_COUNT = 3 # Number of times to retry adding tags. _TAG_RETRY_SLEEP = 2 # Number of seconds to wait between tag retries. ENV = Environment(loader=FileSystemLoader(os.path.join(ROOT_DIR, "buildman/templates"))) -TEMPLATE = ENV.get_template("cloudconfig.yaml") CloudConfigContext().populate_jinja_environment(ENV) +TEMPLATE = ENV.get_template("cloudconfig.json") build_start_duration = Histogram( @@ -157,34 +159,37 @@ class BuilderExecutor(object): if quay_password is None: quay_password = self.executor_config["QUAY_PASSWORD"] - return TEMPLATE.render( - realm=realm, - token=token, - build_uuid=build_uuid, - quay_username=quay_username, - quay_password=quay_password, - manager_hostname=manager_hostname, - websocket_scheme=self.websocket_scheme, - coreos_channel=coreos_channel, - worker_image=self.executor_config.get( - "WORKER_IMAGE", "quay.io/coreos/registry-build-worker" - ), - worker_tag=self.executor_config["WORKER_TAG"], - logentries_token=self.executor_config.get("LOGENTRIES_TOKEN", None), - volume_size=self.executor_config.get("VOLUME_SIZE", "42G"), - max_lifetime_s=self.executor_config.get("MAX_LIFETIME_S", 10800), - ssh_authorized_keys=self.executor_config.get("SSH_AUTHORIZED_KEYS", []), + rendered_json = json.load( + io.StringIO(TEMPLATE.render( + realm=realm, + token=token, + build_uuid=build_uuid, + quay_username=quay_username, + quay_password=quay_password, + manager_hostname=manager_hostname, + websocket_scheme=self.websocket_scheme, + coreos_channel=coreos_channel, + worker_image=self.executor_config.get( + "WORKER_IMAGE", "quay.io/coreos/registry-build-worker" + ), + worker_tag=self.executor_config["WORKER_TAG"], + volume_size=self.executor_config.get("VOLUME_SIZE", "42G"), + max_lifetime_s=self.executor_config.get("MAX_LIFETIME_S", 10800), + ssh_authorized_keys=self.executor_config.get("SSH_AUTHORIZED_KEYS", []), + )) ) + return json.dumps(rendered_json) + class EC2Executor(BuilderExecutor): """ Implementation of BuilderExecutor which uses libcloud to start machines on a variety of cloud providers. """ - + COREOS_STACK_ARCHITECTURE = "x86_64" COREOS_STACK_URL = ( - "http://%s.release.core-os.net/amd64-usr/current/coreos_production_ami_hvm.txt" + "https://builds.coreos.fedoraproject.org/streams/%s.json" ) def __init__(self, *args, **kwargs): @@ -210,9 +215,9 @@ class EC2Executor(BuilderExecutor): """ Retrieve the CoreOS AMI id from the canonical listing. """ - stack_list_string = requests.get(EC2Executor.COREOS_STACK_URL % coreos_channel).text - stack_amis = dict([stack.split("=") for stack in stack_list_string.split("|")]) - return stack_amis[ec2_region] + stack_list_json = requests.get(EC2Executor.COREOS_STACK_URL % coreos_channel).json() + stack_amis = stack_list_json['architectures'][EC2Executor.COREOS_STACK_ARCHITECTURE]['images']['aws']['regions'] + return stack_amis[ec2_region]['image'] @async_observe(build_start_duration, "ec2") async def start_builder(self, realm, token, build_uuid): @@ -394,7 +399,7 @@ class KubernetesExecutor(BuilderExecutor): self._loop = asyncio.get_event_loop() self.namespace = self.executor_config.get("BUILDER_NAMESPACE", "builder") self.image = self.executor_config.get( - "BUILDER_VM_CONTAINER_IMAGE", "quay.io/quay/quay-builder-qemu-coreos:stable" + "BUILDER_VM_CONTAINER_IMAGE", "quay.io/quay/quay-builder-qemu-fedoracoreos:stable" ) async def _request(self, method, path, **kwargs): diff --git a/buildman/qemu-coreos/Dockerfile b/buildman/qemu-coreos/Dockerfile index cdad6f0fc..414184ccc 100644 --- a/buildman/qemu-coreos/Dockerfile +++ b/buildman/qemu-coreos/Dockerfile @@ -1,23 +1,24 @@ -FROM debian +FROM centos:8 as executor-img +ARG location +ARG channel +ARG version -RUN apt-get clean && apt-get update && apt-get upgrade -y # 03APR2017 -RUN apt-get install -y \ - bzip2 \ - curl \ - openssh-client \ - qemu-kvm +RUN [ -z "${location}" ] || [ -z "${channel}" ] || [ -z "${version}" ] && echo "ARG location, channel, version are required" && exit 1 || true -ARG channel=stable -ARG version=current +RUN echo "Downloading" ${location} +RUN curl -s -o coreos_production_qemu_image.qcow2.xz ${location} && unxz coreos_production_qemu_image.qcow2.xz -RUN echo "Downloading http://${channel}.release.core-os.net/amd64-usr/${version}/coreos_production_qemu_image.img.bz2" -RUN curl -s -O http://${channel}.release.core-os.net/amd64-usr/${version}/coreos_production_qemu_image.img.bz2 && \ - bzip2 -d coreos_production_qemu_image.img.bz2 -RUN apt-get remove -y curl bzip2 && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* +FROM centos:8 +ARG location +ARG channel +ARG version +RUN yum -y update && \ + yum -y install openssh-clients qemu-kvm && \ + yum -y clean all + +COPY --from=executor-img /coreos_production_qemu_image.qcow2 /coreos_production_qemu_image.qcow2 COPY start.sh /start.sh LABEL com.coreos.channel ${channel} diff --git a/buildman/qemu-coreos/build.sh b/buildman/qemu-coreos/build.sh new file mode 100755 index 000000000..2a5106fe2 --- /dev/null +++ b/buildman/qemu-coreos/build.sh @@ -0,0 +1,11 @@ +set -e +set -o nounset + +TAG=${TAG:-"stable"} + +CHANNEL=${CHANNEL:-"stable"} +CHANNEL_MANIFEST_JSON=`curl https://builds.coreos.fedoraproject.org/streams/stable.json` +LOCATION=`echo $CHANNEL_MANIFEST_JSON | jq '.architectures.x86_64.artifacts.qemu.formats."qcow2.xz".disk.location' | tr -d '"'` +VERSION=`echo $CHANNEL_MANIFEST_JSON | jq '.architectures.x86_64.artifacts.qemu.release' | tr -d '"'` + +time docker build --build-arg=channel=$CHANNEL --build-arg version=$VERSION --build-arg location=$LOCATION -t quay.io/quay/quay-builder-qemu-fedoracoreos:$TAG . diff --git a/buildman/qemu-coreos/start.sh b/buildman/qemu-coreos/start.sh index ccb1f63e1..054358bb9 100644 --- a/buildman/qemu-coreos/start.sh +++ b/buildman/qemu-coreos/start.sh @@ -10,15 +10,14 @@ set -o nounset mkdir -p /userdata/openstack/latest echo "${USERDATA}" > /userdata/openstack/latest/user_data -time qemu-img resize ./coreos_production_qemu_image.img "${VM_VOLUME_SIZE}" +time qemu-img resize ./coreos_production_qemu_image.qcow2 "${VM_VOLUME_SIZE}" -qemu-system-x86_64 \ +/usr/libexec/qemu-kvm \ -enable-kvm \ -cpu host \ - -device virtio-9p-pci,fsdev=conf,mount_tag=config-2 \ -nographic \ - -drive if=virtio,file=./coreos_production_qemu_image.img \ - -fsdev local,id=conf,security_model=none,readonly,path=/userdata \ + -drive if=virtio,file=./coreos_production_qemu_image.qcow2 \ + -fw_cfg name=opt/com.coreos/config,file=/userdata/openstack/latest/user_data \ -m "${VM_MEMORY}" \ -machine accel=kvm \ -net nic,model=virtio \ diff --git a/buildman/templates/cloudconfig.json b/buildman/templates/cloudconfig.json new file mode 100644 index 000000000..c5c2add02 --- /dev/null +++ b/buildman/templates/cloudconfig.json @@ -0,0 +1,131 @@ +{% macro overridelist() -%} + +REALM={{ realm }} +TOKEN={{ token }} +SERVER={{ websocket_scheme }}://{{ manager_hostname }} + +{%- endmacro %} + + +{% macro journalgatewayservice() -%} + +[Unit] +Description=Journal Gateway Service Socket +[Socket] +ListenStream=/var/run/journald.sock +Service=systemd-journal-gatewayd.service +[Install] +WantedBy=sockets.target + +{%- endmacro %} + + +{% macro disableawsmetadataservice() -%} + +[Unit] +Description=Disable AWS metadata service +Before=network-pre.target +Wants=network-pre.target +[Service] +Type=oneshot +ExecStart=/usr/local/bin/disable-aws-metadata.sh +RemainAfterExit=yes +[Install] +WantedBy=multi-user.target + +{%- endmacro %} + + +{% macro machinelifetimeservice() -%} + +[Unit] +Description=Machine Lifetime Service +[Service] +Type=oneshot +ExecStart=/bin/sh -xc "/bin/sleep {{ max_lifetime_s }}; /usr/bin/systemctl --no-block poweroff" +[Install] +WantedBy=multi-user.target + +{%- endmacro %} + + +{ + "ignition": { + "version": "3.0.0" + }, + "passwd": { + "users": [ + { + {% if ssh_authorized_keys -%} + "sshAuthorizedKeys": {{ ssh_authorized_keys | jsonify }}, + {%- endif %} + "groups": [ + "sudo", + "docker" + ], + "name": "core" + } + ] + }, + "storage": { + "files": [ + { + "path": "/etc/hostname", + "contents": { + "source": {{ build_uuid | default('quay-builder', True) | dataurl | jsonify }} + }, + "mode": 420 + }, + { + "overwrite": true, + "path": "/etc/zincati/config.d/90-disable-auto-updates.toml", + "contents": { + "source": {{ "[updates]\nenabled = false" | dataurl | jsonify }} + }, + "mode": 420 + }, + { + "path": "/usr/local/bin/disable-aws-metadata.sh", + "contents": { + "source": {{ "#!/bin/bash\niptables -t nat -I PREROUTING -p tcp -d 169.254.169.254 --dport 80 -j DNAT --to-destination 1.1.1.1" | dataurl | jsonify }} + }, + "mode": 493 + }, + { + "path": "/root/overrides.list", + "contents": { + "source": {{ overridelist() | dataurl | jsonify }} + }, + "mode": 420 + } + ] + }, + "systemd": { + "units": [ + {{ dockersystemd("quay-builder", + worker_image, + quay_username, + quay_password, + worker_tag, + extra_args='--net=host --privileged --env-file /root/overrides.list -v /var/run/docker.sock:/var/run/docker.sock -v /usr/share/pki/ca-trust-source/anchors:/etc/ssl/certs', + exec_stop_post=['/bin/sh -xc "/bin/sleep 120; /usr/bin/systemctl --no-block poweroff"'], + restart_policy='no' + ) | indent(6) }}, + { + "name": "systemd-journal-gatewayd.socket", + "enabled": true, + "contents": {{ journalgatewayservice() | jsonify }} + }, + { + "name": "disable-aws-metadata.service", + "enabled": true, + "contents": {{ disableawsmetadataservice() | jsonify }} + }, + { + "name": "machine-lifetime.service", + "enabled": true, + "contents": {{ machinelifetimeservice() | jsonify }} + } + ] + } +} diff --git a/buildman/templates/cloudconfig.yaml b/buildman/templates/cloudconfig.yaml deleted file mode 100644 index f90b784b3..000000000 --- a/buildman/templates/cloudconfig.yaml +++ /dev/null @@ -1,102 +0,0 @@ -#cloud-config - -hostname: {{ build_uuid | default('quay-builder', True) }} - -users: - groups: - - sudo - - docker - -{% if ssh_authorized_keys -%} -ssh_authorized_keys: -{% for ssh_key in ssh_authorized_keys -%} -- {{ ssh_key }} -{%- endfor %} -{%- endif %} - -write_files: -- path: /root/disable-aws-metadata.sh - permission: '0755' - content: | - iptables -t nat -I PREROUTING -p tcp -d 169.254.169.254 --dport 80 -j DNAT --to-destination 1.1.1.1 - -- path: /etc/docker/daemon.json - permission: '0644' - content: | - { - "storage-driver": "overlay2" - } - -- path: /root/overrides.list - permission: '0644' - content: | - REALM={{ realm }} - TOKEN={{ token }} - SERVER={{ websocket_scheme }}://{{ manager_hostname }} - {% if logentries_token -%} - LOGENTRIES_TOKEN={{ logentries_token }} - {%- endif %} - -coreos: - update: - reboot-strategy: off - group: {{ coreos_channel }} - - units: - - name: update-engine.service - command: stop - - name: locksmithd.service - command: stop - - name: systemd-journal-gatewayd.socket - command: start - enable: yes - content: | - [Unit] - Description=Journal Gateway Service Socket - [Socket] - ListenStream=/var/run/journald.sock - Service=systemd-journal-gatewayd.service - [Install] - WantedBy=sockets.target - {{ dockersystemd('quay-builder', - worker_image, - quay_username, - quay_password, - worker_tag, - extra_args='--net=host --privileged --env-file /root/overrides.list -v /var/run/docker.sock:/var/run/docker.sock -v /usr/share/ca-certificates:/etc/ssl/certs', - exec_stop_post=['/bin/sh -xc "/bin/sleep 120; /usr/bin/systemctl --no-block poweroff"'], - flattened=False, - restart_policy='no' - ) | indent(4) }} - {% if logentries_token -%} - # https://github.com/kelseyhightower/journal-2-logentries/pull/11 so moved journal-2-logentries to coreos - {{ dockersystemd('builder-logs', - 'quay.io/coreos/journal-2-logentries', - extra_args='--env-file /root/overrides.list -v /run/journald.sock:/run/journald.sock', - flattened=False, - after_units=['quay-builder.service'] - ) | indent(4) }} - {%- endif %} - - name: disable-aws-metadata.service - command: start - enable: yes - content: | - [Unit] - Description=Disable AWS metadata service - Before=network-pre.target - Wants=network-pre.target - [Service] - Type=oneshot - ExecStart=/root/disable-aws-metadata.sh - RemainAfterExit=yes - [Install] - WantedBy=multi-user.target - - name: machine-lifetime.service - command: start - enable: yes - content: | - [Unit] - Description=Machine Lifetime Service - [Service] - Type=oneshot - ExecStart=/bin/sh -xc "/bin/sleep {{ max_lifetime_s }}; /usr/bin/systemctl --no-block poweroff" diff --git a/buildman/templates/dockersystemd.json b/buildman/templates/dockersystemd.json new file mode 100644 index 000000000..a4c025ab1 --- /dev/null +++ b/buildman/templates/dockersystemd.json @@ -0,0 +1,54 @@ +{% macro dockerimageservice() -%} + +[Unit] +After=docker.service +Requires=docker.service +{% if onfailure_units -%} +OnFailure={% for failure in onfailure_units -%}{{ failure }} {% endfor %} +{%- endif %} +{% for after in after_units -%} +After={{ after }} +{% endfor %} +{% for requires in requires_units -%} +Requires={{ requires }} +{% endfor %} +{% for wants in wants_units -%} +Wants={{ wants }} +{% endfor %} + +[Service] +{% if env_file -%} +EnvironmentFile={{ env_file }} +{%- endif %} +{% if oneshot -%} +Type=oneshot +{% else -%} +Restart={{ restart_policy }} +{%- endif %} +TimeoutStartSec={{ timeout_start_sec|default(600) }} +TimeoutStopSec={{ timeout_stop_sec|default(2000) }} +{% if username and password %} +ExecStartPre=/usr/bin/docker login -u {{ username }} -p {{ password }} {{ container|registry }} +{% endif %} +ExecStart=/usr/bin/docker run --rm {{ extra_args }} --name {{ name }} {{ container }}:{{ tag }} {{ command }} +{% for start_post in exec_start_post -%} +ExecStartPost={{ start_post }} +{% endfor %} +{% if not oneshot -%} +ExecStop=/usr/bin/docker stop {{ name }} +{%- endif %} +{% for stop_post in exec_stop_post -%} +ExecStopPost={{ stop_post }} +{% endfor %} + +[Install] +WantedBy=multi-user.target + +{%- endmacro %} + + +{ + "name": "{{ name }}.service", + "enabled": true, + "contents": {{ dockerimageservice() | jsonify }} +} diff --git a/requirements.txt b/requirements.txt index 70c066db6..b8d8e86e9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,6 @@ -e git+https://github.com/quay/appr.git@58c88e4952e95935c0dd72d4a24b0c44f2249f5b#egg=cnr_server -e git+https://github.com/DevTable/aniso8601-fake.git@bd7762c7dea0498706d3f57db60cd8a8af44ba90#egg=aniso8601 -e git+https://github.com/DevTable/boto.git@a6a5c00bd199b1492e99199251b10451970b5b08#egg=boto --e git+https://github.com/DevTable/container-cloud-config.git@7d6c1545554b81ac65edd6d1bd1ab9f8c462209a#egg=container_cloud_config -e git+https://github.com/jarus/flask-testing.git@17f19d7fee0e1e176703fc7cb04917a77913ba1a#egg=Flask_Testing -e git+https://github.com/quay/mockldap.git@4265554a3d89fe39bf05b18e91607bec3fcf215a#egg=mockldap -e git+https://github.com/quay/py-bitbucket.git@85301693ce3682f8e1244e90bd8a903181844bde#egg=py_bitbucket