proxygen/build/fbcode_builder/docker_builder.py

#!/usr/bin/env python
# Copyright (c) Facebook, Inc. and its affiliates.

"""

Extends FBCodeBuilder to produce Docker context directories.

In order to get the largest iteration-time savings from Docker's build
caching, you will want to:
 - Use fine-grained steps as appropriate (e.g. separate make & make install),
 - Start your action sequence with the lowest-risk steps, and with the steps
   that change the least often, and
 - Put the steps that you are debugging towards the very end.

"""
import logging
import os
import shutil
import tempfile

from fbcode_builder import FBCodeBuilder
from shell_quoting import raw_shell, shell_comment, shell_join, ShellQuoted, path_join
from utils import recursively_flatten_list, run_command


class DockerFBCodeBuilder(FBCodeBuilder):
    def _user(self):
        return self.option("user", "root")

    def _change_user(self):
        return ShellQuoted("USER {u}").format(u=self._user())

    def setup(self):
        # Please add RPM-based OSes here as appropriate.
        #
        # To allow exercising non-root installs -- we change users after the
        # system packages are installed.  TODO: For users not defined in the
        # image, we should probably `useradd`.
        return self.step(
            "Setup",
            [
                # Docker's FROM does not understand shell quoting.
                ShellQuoted("FROM {}".format(self.option("os_image"))),
                # /bin/sh syntax is a pain
                ShellQuoted('SHELL ["/bin/bash", "-c"]'),
            ]
            + self.install_debian_deps()
            + [self._change_user()]
            + [self.workdir(self.option("prefix"))]
            + self.create_python_venv()
            + self.python_venv()
            + self.rust_toolchain(),
        )

    def python_venv(self):
        # To both avoid calling venv activate on each RUN command AND to ensure
        # it is present when the resulting container is run add to PATH
        actions = []
        if self.option("PYTHON_VENV", "OFF") == "ON":
            actions = ShellQuoted("ENV PATH={p}:$PATH").format(
                p=path_join(self.option("prefix"), "venv", "bin")
            )
        return actions

    def step(self, name, actions):
        assert "\n" not in name, "Name {0} would span > 1 line".format(name)
        b = ShellQuoted("")
        return [ShellQuoted("### {0} ###".format(name)), b] + actions + [b]

    def run(self, shell_cmd):
        return ShellQuoted("RUN {cmd}").format(cmd=shell_cmd)

    def set_env(self, key, value):
        return ShellQuoted("ENV {key}={val}").format(key=key, val=value)

    def workdir(self, dir):
        return [
            # As late as Docker 1.12.5, this results in `build` being owned
            # by root:root -- the explicit `mkdir` works around the bug:
            #   USER nobody
            #   WORKDIR build
            ShellQuoted("USER root"),
            ShellQuoted("RUN mkdir -p {d} && chown {u} {d}").format(
                d=dir, u=self._user()
            ),
            self._change_user(),
            ShellQuoted("WORKDIR {dir}").format(dir=dir),
        ]

    def comment(self, comment):
        # This should not be a command since we don't want comment changes
        # to invalidate the Docker build cache.
        return shell_comment(comment)

    def copy_local_repo(self, repo_dir, dest_name):
        fd, archive_path = tempfile.mkstemp(
            prefix="local_repo_{0}_".format(dest_name),
            suffix=".tgz",
            dir=os.path.abspath(self.option("docker_context_dir")),
        )
        os.close(fd)
        run_command("tar", "czf", archive_path, ".", cwd=repo_dir)
        return [
            ShellQuoted("ADD {archive} {dest_name}").format(
                archive=os.path.basename(archive_path), dest_name=dest_name
            ),
            # Docker permissions make very little sense... see also workdir()
            ShellQuoted("USER root"),
            ShellQuoted("RUN chown -R {u} {d}").format(d=dest_name, u=self._user()),
            self._change_user(),
        ]

    def _render_impl(self, steps):
        return raw_shell(shell_join("\n", recursively_flatten_list(steps)))

    def debian_ccache_setup_steps(self):
        source_ccache_tgz = self.option("ccache_tgz", "")
        if not source_ccache_tgz:
            logging.info("Docker ccache not enabled")
            return []

        dest_ccache_tgz = os.path.join(self.option("docker_context_dir"), "ccache.tgz")

        try:
            try:
                os.link(source_ccache_tgz, dest_ccache_tgz)
            except OSError:
                logging.exception(
                    "Hard-linking {s} to {d} failed, falling back to copy".format(
                        s=source_ccache_tgz, d=dest_ccache_tgz
                    )
                )
                shutil.copyfile(source_ccache_tgz, dest_ccache_tgz)
        except Exception:
            logging.exception(
                "Failed to copy or link {s} to {d}, aborting".format(
                    s=source_ccache_tgz, d=dest_ccache_tgz
                )
            )
            raise

        return [
            # Separate layer so that in development we avoid re-downloads.
            self.run(ShellQuoted("apt-get install -yq ccache")),
            ShellQuoted("ADD ccache.tgz /"),
            ShellQuoted(
                # Set CCACHE_DIR before the `ccache` invocations below.
                "ENV CCACHE_DIR=/ccache "
                # No clang support for now, so it's easiest to hardcode gcc.
                'CC="ccache gcc" CXX="ccache g++" '
                # Always log for ease of debugging. For real FB projects,
                # this log is several megabytes, so dumping it to stdout
                # would likely exceed the Travis log limit of 4MB.
                #
                # On a local machine, `docker cp` will get you the data.  To
                # get the data out from Travis, I would compress and dump
                # uuencoded bytes to the log -- for Bistro this was about
                # 600kb or 8000 lines:
                #
                #   apt-get install sharutils
                #   bzip2 -9 < /tmp/ccache.log | uuencode -m ccache.log.bz2
                "CCACHE_LOGFILE=/tmp/ccache.log"
            ),
            self.run(
                ShellQuoted(
                    # Future: Skipping this part made this Docker step instant,
                    # saving ~1min of build time.  It's unclear if it is the
                    # chown or the du, but probably the chown -- since a large
                    # part of the cost is incurred at image save time.
                    #
                    # ccache.tgz may be empty, or may have the wrong
                    # permissions.
                    "mkdir -p /ccache && time chown -R nobody /ccache && "
                    "time du -sh /ccache && "
                    # Reset stats so `docker_build_with_ccache.sh` can print
                    # useful values at the end of the run.
                    "echo === Prev run stats === && ccache -s && ccache -z && "
                    # Record the current time to let travis_build.sh figure out
                    # the number of bytes in the cache that are actually used --
                    # this is crucial for tuning the maximum cache size.
                    "date +%s > /FBCODE_BUILDER_CCACHE_START_TIME && "
                    # The build running as `nobody` should be able to write here
                    "chown nobody /tmp/ccache.log"
                )
            ),
        ]