Update the executor image from Container Linux to Fedora CoreOS (#434)

* Update the executor image from Container Linux to Fedora CoreOS

* Move the container cloud config script for templating from devtable to quay's repo

* Ignition config template

* Move dockersystemd from devtable repo

* Remove pinned dependency on devtable/container-cloud-config

* Removes squashed image and logentries

* Update builder image

* Update mounted cert directory for Fedora

* Removes old clouconfig template

* Pass userdata as firmware config to qemu

* Use CentOS:8 as base image

buildman/container_cloud_config.py

@@ -0,0 +1,116 @@
+"""
+Provides helper methods and templates for generating cloud config for running containers.
+
+Originally from https://github.com/DevTable/container-cloud-config
+"""
+
+from functools import partial
+
+import base64
+import json
+import os
+import requests
+import logging
+
+try:
+    # Python 3
+    from urllib.request import HTTPRedirectHandler, build_opener, install_opener, urlopen, Request
+    from urllib.error import HTTPError
+    from urllib.parse import quote as urlquote
+except ImportError:
+    # Python 2
+    from urllib2 import (
+        HTTPRedirectHandler,
+        build_opener,
+        install_opener,
+        urlopen,
+        Request,
+        HTTPError,
+    )
+    from urllib import quote as urlquote
+
+from jinja2 import FileSystemLoader, Environment, StrictUndefined
+
+logger = logging.getLogger(__name__)
+
+
+class CloudConfigContext(object):
+    """ Context object for easy generating of cloud config. """
+
+    def populate_jinja_environment(self, env):
+        """ Populates the given jinja environment with the methods defined in this context. """
+        env.filters["registry"] = self.registry
+        env.filters["dataurl"] = self.data_url
+        env.filters["jsonify"] = json.dumps
+        env.globals["dockersystemd"] = self._dockersystemd_template
+
+    def _dockersystemd_template(
+        self,
+        name,
+        container,
+        username="",
+        password="",
+        tag="latest",
+        extra_args="",
+        command="",
+        after_units=[],
+        exec_start_post=[],
+        exec_stop_post=[],
+        restart_policy="always",
+        oneshot=False,
+        env_file=None,
+        onfailure_units=[],
+        requires_units=[],
+        wants_units=[],
+        timeout_start_sec=None,
+        timeout_stop_sec=None,
+        autostart=True,
+    ):
+
+        path = os.path.join(os.path.dirname(__file__), "templates")
+        env = Environment(loader=FileSystemLoader(path), undefined=StrictUndefined)
+        self.populate_jinja_environment(env)
+        template = env.get_template("dockersystemd.json")
+        return template.render(
+            name=name,
+            container=container,
+            username=username,
+            password=password,
+            tag=tag,
+            extra_args=extra_args,
+            command=command,
+            after_units=after_units,
+            requires_units=requires_units,
+            wants_units=wants_units,
+            onfailure_units=onfailure_units,
+            exec_start_post=exec_start_post,
+            exec_stop_post=exec_stop_post,
+            restart_policy=restart_policy,
+            oneshot=oneshot,
+            autostart=autostart,
+            timeout_start_sec=timeout_start_sec,
+            timeout_stop_sec=timeout_stop_sec,
+            env_file=env_file,
+        )
+
+    def data_url(self, content):
+        """ Encodes the content of an ignition file using RFC 2397. """
+        data = "," + urlquote(content)
+        return "data:" + data
+
+
+    def registry(self, container_name):
+        """ Parse the registry from repositories of the following formats:
+        quay.io/quay/quay:tagname -> quay.io
+        localhost:5000/quay/quay:tagname -> localhost:5000
+        localhost:5000/quay/quay -> localhost:5000
+        quay/quay:latest -> ''
+        quay/quay -> ''
+        mysql:latest -> ''
+        mysql -> ''
+    """
+        num_slashes = container_name.count("/")
+        if num_slashes == 2:
+            return container_name[: container_name.find("/")]
+        else:
+            return ""

buildman/manager/executor.py

@@ -1,6 +1,8 @@
 import asyncio
 import datetime
 import hashlib
+import io
+import json
 import logging
 import os
 import socket
@@ -16,7 +18,6 @@ import botocore
 import cachetools.func
 import requests
 
-from container_cloud_config import CloudConfigContext
 from jinja2 import FileSystemLoader, Environment
 from prometheus_client import Histogram
 
@@ -25,6 +26,7 @@ import release
 from _init import ROOT_DIR
 from app import app
 from buildman.asyncutil import AsyncWrapper
+from buildman.container_cloud_config import CloudConfigContext
 
 
 logger = logging.getLogger(__name__)
@@ -36,8 +38,8 @@ _TAG_RETRY_COUNT = 3  # Number of times to retry adding tags.
 _TAG_RETRY_SLEEP = 2  # Number of seconds to wait between tag retries.
 
 ENV = Environment(loader=FileSystemLoader(os.path.join(ROOT_DIR, "buildman/templates")))
-TEMPLATE = ENV.get_template("cloudconfig.yaml")
 CloudConfigContext().populate_jinja_environment(ENV)
+TEMPLATE = ENV.get_template("cloudconfig.json")
 
 
 build_start_duration = Histogram(
@@ -157,34 +159,37 @@ class BuilderExecutor(object):
         if quay_password is None:
             quay_password = self.executor_config["QUAY_PASSWORD"]
 
-        return TEMPLATE.render(
-            realm=realm,
-            token=token,
-            build_uuid=build_uuid,
-            quay_username=quay_username,
-            quay_password=quay_password,
-            manager_hostname=manager_hostname,
-            websocket_scheme=self.websocket_scheme,
-            coreos_channel=coreos_channel,
-            worker_image=self.executor_config.get(
-                "WORKER_IMAGE", "quay.io/coreos/registry-build-worker"
-            ),
-            worker_tag=self.executor_config["WORKER_TAG"],
-            logentries_token=self.executor_config.get("LOGENTRIES_TOKEN", None),
-            volume_size=self.executor_config.get("VOLUME_SIZE", "42G"),
-            max_lifetime_s=self.executor_config.get("MAX_LIFETIME_S", 10800),
-            ssh_authorized_keys=self.executor_config.get("SSH_AUTHORIZED_KEYS", []),
+        rendered_json = json.load(
+            io.StringIO(TEMPLATE.render(
+                realm=realm,
+                token=token,
+                build_uuid=build_uuid,
+                quay_username=quay_username,
+                quay_password=quay_password,
+                manager_hostname=manager_hostname,
+                websocket_scheme=self.websocket_scheme,
+                coreos_channel=coreos_channel,
+                worker_image=self.executor_config.get(
+                    "WORKER_IMAGE", "quay.io/coreos/registry-build-worker"
+                ),
+                worker_tag=self.executor_config["WORKER_TAG"],
+                volume_size=self.executor_config.get("VOLUME_SIZE", "42G"),
+                max_lifetime_s=self.executor_config.get("MAX_LIFETIME_S", 10800),
+                ssh_authorized_keys=self.executor_config.get("SSH_AUTHORIZED_KEYS", []),
+            ))
         )
 
+        return json.dumps(rendered_json)
+
 
 class EC2Executor(BuilderExecutor):
     """
     Implementation of BuilderExecutor which uses libcloud to start machines on a variety of cloud
     providers.
     """
-
+    COREOS_STACK_ARCHITECTURE = "x86_64"
     COREOS_STACK_URL = (
-        "http://%s.release.core-os.net/amd64-usr/current/coreos_production_ami_hvm.txt"
+        "https://builds.coreos.fedoraproject.org/streams/%s.json"
     )
 
     def __init__(self, *args, **kwargs):
@@ -210,9 +215,9 @@ class EC2Executor(BuilderExecutor):
         """
         Retrieve the CoreOS AMI id from the canonical listing.
         """
-        stack_list_string = requests.get(EC2Executor.COREOS_STACK_URL % coreos_channel).text
-        stack_amis = dict([stack.split("=") for stack in stack_list_string.split("|")])
-        return stack_amis[ec2_region]
+        stack_list_json = requests.get(EC2Executor.COREOS_STACK_URL % coreos_channel).json()
+        stack_amis = stack_list_json['architectures'][EC2Executor.COREOS_STACK_ARCHITECTURE]['images']['aws']['regions']
+        return stack_amis[ec2_region]['image']
 
     @async_observe(build_start_duration, "ec2")
     async def start_builder(self, realm, token, build_uuid):
@@ -394,7 +399,7 @@ class KubernetesExecutor(BuilderExecutor):
         self._loop = asyncio.get_event_loop()
         self.namespace = self.executor_config.get("BUILDER_NAMESPACE", "builder")
         self.image = self.executor_config.get(
-            "BUILDER_VM_CONTAINER_IMAGE", "quay.io/quay/quay-builder-qemu-coreos:stable"
+            "BUILDER_VM_CONTAINER_IMAGE", "quay.io/quay/quay-builder-qemu-fedoracoreos:stable"
         )
 
     async def _request(self, method, path, **kwargs):

buildman/qemu-coreos/Dockerfile

@@ -1,23 +1,24 @@
-FROM debian
+FROM centos:8 as executor-img
+ARG location
+ARG channel
+ARG version
 
-RUN apt-get clean && apt-get update && apt-get upgrade -y # 03APR2017
-RUN apt-get install -y \
-  bzip2 \
-  curl \
-  openssh-client \
-  qemu-kvm
+RUN [ -z "${location}" ] || [ -z "${channel}" ] || [ -z "${version}" ] && echo "ARG location, channel, version are required" && exit 1 || true
 
-ARG channel=stable
-ARG version=current
+RUN echo "Downloading" ${location}
+RUN curl -s -o coreos_production_qemu_image.qcow2.xz ${location} && unxz coreos_production_qemu_image.qcow2.xz
 
-RUN echo "Downloading http://${channel}.release.core-os.net/amd64-usr/${version}/coreos_production_qemu_image.img.bz2"
-RUN curl -s -O http://${channel}.release.core-os.net/amd64-usr/${version}/coreos_production_qemu_image.img.bz2 && \
-  bzip2 -d coreos_production_qemu_image.img.bz2
 
-RUN apt-get remove -y curl bzip2 && \
-  apt-get clean && \
-  rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+FROM centos:8
+ARG location
+ARG channel
+ARG version
 
+RUN yum -y update && \
+  yum -y install openssh-clients qemu-kvm && \
+  yum -y clean all
+
+COPY --from=executor-img /coreos_production_qemu_image.qcow2 /coreos_production_qemu_image.qcow2
 COPY start.sh /start.sh
 
 LABEL com.coreos.channel ${channel}

buildman/qemu-coreos/build.sh

@@ -0,0 +1,11 @@
+set -e
+set -o nounset
+
+TAG=${TAG:-"stable"}
+
+CHANNEL=${CHANNEL:-"stable"}
+CHANNEL_MANIFEST_JSON=`curl https://builds.coreos.fedoraproject.org/streams/stable.json`
+LOCATION=`echo $CHANNEL_MANIFEST_JSON | jq '.architectures.x86_64.artifacts.qemu.formats."qcow2.xz".disk.location' | tr -d '"'`
+VERSION=`echo $CHANNEL_MANIFEST_JSON | jq '.architectures.x86_64.artifacts.qemu.release' | tr -d '"'`
+
+time docker build --build-arg=channel=$CHANNEL --build-arg version=$VERSION --build-arg location=$LOCATION -t quay.io/quay/quay-builder-qemu-fedoracoreos:$TAG .

buildman/qemu-coreos/start.sh

@@ -10,15 +10,14 @@ set -o nounset
 mkdir -p /userdata/openstack/latest
 echo "${USERDATA}" > /userdata/openstack/latest/user_data
 
-time qemu-img resize ./coreos_production_qemu_image.img "${VM_VOLUME_SIZE}"
+time qemu-img resize ./coreos_production_qemu_image.qcow2 "${VM_VOLUME_SIZE}"
 
-qemu-system-x86_64 \
+/usr/libexec/qemu-kvm \
         -enable-kvm \
         -cpu host \
-        -device virtio-9p-pci,fsdev=conf,mount_tag=config-2 \
         -nographic \
-        -drive if=virtio,file=./coreos_production_qemu_image.img \
-        -fsdev local,id=conf,security_model=none,readonly,path=/userdata \
+        -drive if=virtio,file=./coreos_production_qemu_image.qcow2 \
+        -fw_cfg name=opt/com.coreos/config,file=/userdata/openstack/latest/user_data \
         -m "${VM_MEMORY}" \
         -machine accel=kvm \
         -net nic,model=virtio \

buildman/templates/cloudconfig.json

@@ -0,0 +1,131 @@
+{% macro overridelist() -%}
+
+REALM={{ realm }}
+TOKEN={{ token }}
+SERVER={{ websocket_scheme }}://{{ manager_hostname }}
+
+{%- endmacro %}
+
+
+{% macro journalgatewayservice() -%}
+
+[Unit]
+Description=Journal Gateway Service Socket
+[Socket]
+ListenStream=/var/run/journald.sock
+Service=systemd-journal-gatewayd.service
+[Install]
+WantedBy=sockets.target
+
+{%- endmacro %}
+
+
+{% macro disableawsmetadataservice() -%}
+
+[Unit]
+Description=Disable AWS metadata service
+Before=network-pre.target
+Wants=network-pre.target
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/disable-aws-metadata.sh
+RemainAfterExit=yes
+[Install]
+WantedBy=multi-user.target
+
+{%- endmacro %}
+
+
+{% macro machinelifetimeservice() -%}
+
+[Unit]
+Description=Machine Lifetime Service
+[Service]
+Type=oneshot
+ExecStart=/bin/sh -xc "/bin/sleep {{ max_lifetime_s }}; /usr/bin/systemctl --no-block poweroff"
+[Install]
+WantedBy=multi-user.target
+
+{%- endmacro %}
+
+
+{
+  "ignition": {
+    "version": "3.0.0"
+  },
+  "passwd": {
+    "users": [
+      {
+        {% if ssh_authorized_keys -%}
+        "sshAuthorizedKeys": {{ ssh_authorized_keys | jsonify }},
+        {%- endif %}
+        "groups": [
+          "sudo",
+          "docker"
+        ],
+        "name": "core"
+      }
+    ]
+  },
+  "storage": {
+    "files": [
+      {
+        "path": "/etc/hostname",
+        "contents": {
+          "source": {{ build_uuid | default('quay-builder', True) | dataurl | jsonify }}
+        },
+        "mode": 420
+      },
+      {
+        "overwrite": true,
+        "path": "/etc/zincati/config.d/90-disable-auto-updates.toml",
+        "contents": {
+          "source": {{ "[updates]\nenabled = false" | dataurl | jsonify }}
+        },
+        "mode": 420
+      },
+      {
+        "path": "/usr/local/bin/disable-aws-metadata.sh",
+        "contents": {
+          "source": {{ "#!/bin/bash\niptables -t nat -I PREROUTING -p tcp -d 169.254.169.254 --dport 80 -j DNAT --to-destination 1.1.1.1" | dataurl | jsonify }}
+        },
+        "mode": 493
+      },
+      {
+        "path": "/root/overrides.list",
+        "contents": {
+          "source": {{ overridelist() | dataurl | jsonify }}
+        },
+        "mode": 420
+      }
+    ]
+  },
+  "systemd": {
+    "units": [
+      {{ dockersystemd("quay-builder",
+                       worker_image,
+                       quay_username,
+                       quay_password,
+                       worker_tag,
+                       extra_args='--net=host --privileged --env-file /root/overrides.list -v /var/run/docker.sock:/var/run/docker.sock -v /usr/share/pki/ca-trust-source/anchors:/etc/ssl/certs',
+                       exec_stop_post=['/bin/sh -xc "/bin/sleep 120; /usr/bin/systemctl --no-block poweroff"'],
+                       restart_policy='no'
+                      ) | indent(6) }},
+      {
+        "name": "systemd-journal-gatewayd.socket",
+        "enabled": true,
+        "contents": {{ journalgatewayservice() | jsonify }}
+      },
+      {
+        "name": "disable-aws-metadata.service",
+        "enabled": true,
+        "contents": {{ disableawsmetadataservice() | jsonify }}
+      },
+      {
+        "name": "machine-lifetime.service",
+        "enabled": true,
+        "contents": {{ machinelifetimeservice() | jsonify }}
+      }
+    ]
+  }
+}

buildman/templates/cloudconfig.yaml

@@ -1,102 +0,0 @@
-#cloud-config
-
-hostname: {{ build_uuid | default('quay-builder', True) }}
-
-users:
-  groups:
-  - sudo
-  - docker
-
-{% if ssh_authorized_keys -%}
-ssh_authorized_keys:
-{% for ssh_key in ssh_authorized_keys -%}
-- {{ ssh_key }}
-{%- endfor %}
-{%- endif %}
-
-write_files:
-- path: /root/disable-aws-metadata.sh
-  permission: '0755'
-  content: |
-    iptables -t nat -I PREROUTING -p tcp -d 169.254.169.254 --dport 80 -j DNAT --to-destination 1.1.1.1
-
-- path: /etc/docker/daemon.json
-  permission: '0644'
-  content: |
-    {
-        "storage-driver": "overlay2"
-    }
-
-- path: /root/overrides.list
-  permission: '0644'
-  content: |
-    REALM={{ realm }}
-    TOKEN={{ token }}
-    SERVER={{ websocket_scheme }}://{{ manager_hostname }}
-    {% if logentries_token -%}
-    LOGENTRIES_TOKEN={{ logentries_token }}
-    {%- endif %}
-
-coreos:
-  update:
-    reboot-strategy: off
-    group: {{ coreos_channel }}
-
-  units:
-    - name: update-engine.service
-      command: stop
-    - name: locksmithd.service
-      command: stop
-    - name: systemd-journal-gatewayd.socket
-      command: start
-      enable: yes
-      content: |
-        [Unit]
-        Description=Journal Gateway Service Socket
-        [Socket]
-        ListenStream=/var/run/journald.sock
-        Service=systemd-journal-gatewayd.service
-        [Install]
-        WantedBy=sockets.target
-    {{ dockersystemd('quay-builder',
-                     worker_image,
-                     quay_username,
-                     quay_password,
-                     worker_tag,
-                     extra_args='--net=host --privileged --env-file /root/overrides.list -v /var/run/docker.sock:/var/run/docker.sock -v /usr/share/ca-certificates:/etc/ssl/certs',
-                     exec_stop_post=['/bin/sh -xc "/bin/sleep 120; /usr/bin/systemctl --no-block poweroff"'],
-                     flattened=False,
-                     restart_policy='no'
-                    ) | indent(4) }}
-    {% if logentries_token -%}
-    # https://github.com/kelseyhightower/journal-2-logentries/pull/11 so moved journal-2-logentries to coreos
-    {{ dockersystemd('builder-logs',
-                     'quay.io/coreos/journal-2-logentries',
-                     extra_args='--env-file /root/overrides.list -v /run/journald.sock:/run/journald.sock',
-                     flattened=False,
-                     after_units=['quay-builder.service']
-                     ) | indent(4) }}
-    {%- endif %}
-    - name: disable-aws-metadata.service
-      command: start
-      enable: yes
-      content: |
-        [Unit]
-        Description=Disable AWS metadata service
-        Before=network-pre.target
-        Wants=network-pre.target
-        [Service]
-        Type=oneshot
-        ExecStart=/root/disable-aws-metadata.sh
-        RemainAfterExit=yes
-        [Install]
-        WantedBy=multi-user.target
-    - name: machine-lifetime.service
-      command: start
-      enable: yes
-      content: |
-        [Unit]
-        Description=Machine Lifetime Service
-        [Service]
-        Type=oneshot
-        ExecStart=/bin/sh -xc "/bin/sleep {{ max_lifetime_s }}; /usr/bin/systemctl --no-block poweroff"

buildman/templates/dockersystemd.json

@@ -0,0 +1,54 @@
+{% macro dockerimageservice() -%}
+
+[Unit]
+After=docker.service
+Requires=docker.service
+{% if onfailure_units -%}
+OnFailure={% for failure in onfailure_units -%}{{ failure }} {% endfor %}
+{%- endif %}
+{% for after in after_units -%}
+After={{ after }}
+{% endfor %}
+{% for requires in requires_units -%}
+Requires={{ requires }}
+{% endfor %}
+{% for wants in wants_units -%}
+Wants={{ wants }}
+{% endfor %}
+
+[Service]
+{% if env_file -%}
+EnvironmentFile={{ env_file }}
+{%- endif %}
+{% if oneshot -%}
+Type=oneshot
+{% else -%}
+Restart={{ restart_policy }}
+{%- endif %}
+TimeoutStartSec={{ timeout_start_sec|default(600) }}
+TimeoutStopSec={{ timeout_stop_sec|default(2000) }}
+{% if username and password %}
+ExecStartPre=/usr/bin/docker login -u {{ username }} -p {{ password }} {{ container|registry }}
+{% endif %}
+ExecStart=/usr/bin/docker run --rm {{ extra_args }} --name {{ name }} {{ container }}:{{ tag }} {{ command }}
+{% for start_post in exec_start_post -%}
+ExecStartPost={{ start_post }}
+{% endfor %}
+{% if not oneshot -%}
+ExecStop=/usr/bin/docker stop {{ name }}
+{%- endif %}
+{% for stop_post in exec_stop_post -%}
+ExecStopPost={{ stop_post }}
+{% endfor %}
+
+[Install]
+WantedBy=multi-user.target
+
+{%- endmacro %}
+
+
+{
+  "name": "{{ name }}.service",
+  "enabled": true,
+  "contents": {{ dockerimageservice() | jsonify }}
+}

requirements.txt

@@ -1,7 +1,6 @@
 -e git+https://github.com/quay/appr.git@58c88e4952e95935c0dd72d4a24b0c44f2249f5b#egg=cnr_server
 -e git+https://github.com/DevTable/aniso8601-fake.git@bd7762c7dea0498706d3f57db60cd8a8af44ba90#egg=aniso8601
 -e git+https://github.com/DevTable/boto.git@a6a5c00bd199b1492e99199251b10451970b5b08#egg=boto
--e git+https://github.com/DevTable/container-cloud-config.git@7d6c1545554b81ac65edd6d1bd1ab9f8c462209a#egg=container_cloud_config
 -e git+https://github.com/jarus/flask-testing.git@17f19d7fee0e1e176703fc7cb04917a77913ba1a#egg=Flask_Testing
 -e git+https://github.com/quay/mockldap.git@4265554a3d89fe39bf05b18e91607bec3fcf215a#egg=mockldap
 -e git+https://github.com/quay/py-bitbucket.git@85301693ce3682f8e1244e90bd8a903181844bde#egg=py_bitbucket