quay/endpoints/api/build.py

"""
Create, list, cancel and get status/logs of repository builds.
"""
import datetime
import hashlib
import json
import logging
import os
from urllib.parse import urlparse

from flask import request

import features
from app import build_logs, dockerfile_build_queue, log_archive
from app import userfiles as user_files
from auth.permissions import (
    AdministerOrganizationPermission,
    AdministerRepositoryPermission,
    ModifyRepositoryPermission,
    ReadRepositoryPermission,
    SuperUserPermission,
)
from buildtrigger.basehandler import BuildTriggerHandler
from data import database, model
from data.buildlogs import BuildStatusRetrievalError
from endpoints.api import (
    ApiResource,
    RepositoryParamResource,
    abort,
    allow_if_any_superuser,
    allow_if_global_readonly_superuser,
    allow_if_superuser,
    allow_if_superuser_with_full_access,
    api,
    disallow_for_app_repositories,
    disallow_for_non_normal_repositories,
    disallow_for_user_namespace,
    format_date,
    internal_only,
    log_action,
    nickname,
    parse_args,
    path_param,
    query_param,
    require_repo_admin,
    require_repo_read,
    require_repo_write,
    resource,
    validate_json_request,
)
from endpoints.building import (
    BuildTriggerDisabledException,
    MaximumBuildsQueuedException,
    PreparedBuild,
    start_build,
)
from endpoints.exception import InvalidRequest, NotFound, Unauthorized
from util.names import parse_robot_username
from util.request import get_request_ip

logger = logging.getLogger(__name__)


def get_trigger_config(trigger):
    try:
        return json.loads(trigger.config)
    except:
        return {}


def get_job_config(build_obj):
    try:
        return json.loads(build_obj.job_config)
    except:
        return {}


def user_view(user):
    return {
        "name": user.username,
        "kind": "user",
        "is_robot": user.robot,
    }


def trigger_view(trigger, can_read=False, can_admin=False, for_build=False):
    if trigger and trigger.uuid:
        build_trigger = BuildTriggerHandler.get_handler(trigger)
        build_source = build_trigger.config.get("build_source")

        repo_url = build_trigger.get_repository_url() if build_source else None
        can_read = can_read or can_admin

        trigger_data = {
            "id": trigger.uuid,
            "service": trigger.service.name,
            "is_active": build_trigger.is_active(),
            "build_source": build_source if can_read else None,
            "repository_url": repo_url if can_read else None,
            "config": build_trigger.config if can_admin else {},
            "can_invoke": can_admin,
            "enabled": trigger.enabled,
            "disabled_reason": trigger.disabled_reason.name if trigger.disabled_reason else None,
        }

        if not for_build and can_admin and trigger.pull_robot:
            trigger_data["pull_robot"] = user_view(trigger.pull_robot)

        return trigger_data

    return None


def _get_build_status(build_obj):
    """
    Returns the updated build phase, status and (if any) error for the build object.
    """
    phase = build_obj.phase
    status = {}
    error = None

    # If the build is currently running, then load its "real-time" status from Redis.
    if not database.BUILD_PHASE.is_terminal_phase(phase):
        try:
            status = build_logs.get_status(build_obj.uuid)
        except BuildStatusRetrievalError as bsre:
            phase = "cannot_load"
            if SuperUserPermission().can():
                error = str(bsre)
            else:
                error = "Redis may be down. Please contact support."

        if phase != "cannot_load":
            # If the status contains a heartbeat, then check to see if has been written in the last few
            # minutes. If not, then the build timed out.
            if status is not None and "heartbeat" in status and status["heartbeat"]:
                heartbeat = datetime.datetime.utcfromtimestamp(status["heartbeat"])
                if datetime.datetime.utcnow() - heartbeat > datetime.timedelta(minutes=1):
                    phase = database.BUILD_PHASE.INTERNAL_ERROR

    # If the phase is internal error, return 'expired' instead if the number of retries
    # on the queue item is 0.
    if phase == database.BUILD_PHASE.INTERNAL_ERROR:
        retry = build_obj.queue_id and dockerfile_build_queue.has_retries_remaining(
            build_obj.queue_id
        )
        if not retry:
            phase = "expired"

    return (phase, status, error)


def build_status_view(build_obj):
    phase, status, error = _get_build_status(build_obj)
    repo_namespace = build_obj.repository.namespace_user.username
    repo_name = build_obj.repository.name

    can_read = ReadRepositoryPermission(repo_namespace, repo_name).can()
    can_write = ModifyRepositoryPermission(repo_namespace, repo_name).can()
    can_admin = AdministerRepositoryPermission(repo_namespace, repo_name).can()

    job_config = get_job_config(build_obj)

    resp = {
        "id": build_obj.uuid,
        "phase": phase,
        "started": format_date(build_obj.started),
        "display_name": build_obj.display_name,
        "status": status or {},
        "subdirectory": job_config.get("build_subdir", ""),
        "dockerfile_path": job_config.get("build_subdir", ""),
        "context": job_config.get("context", ""),
        "tags": job_config.get("docker_tags", []),
        "manual_user": job_config.get("manual_user", None),
        "is_writer": can_write,
        "trigger": trigger_view(build_obj.trigger, can_read, can_admin, for_build=True),
        "trigger_metadata": job_config.get("trigger_metadata", None) if can_read else None,
        "resource_key": build_obj.resource_key,
        "pull_robot": user_view(build_obj.pull_robot) if build_obj.pull_robot else None,
        "repository": {"namespace": repo_namespace, "name": repo_name},
        "error": error,
    }

    if can_write or features.READER_BUILD_LOGS:
        if build_obj.resource_key is not None:
            resp["archive_url"] = user_files.get_file_url(
                build_obj.resource_key, get_request_ip(), requires_cors=True
            )
        elif job_config.get("archive_url", None):
            resp["archive_url"] = job_config["archive_url"]

    return resp


@resource("/v1/repository/<apirepopath:repository>/build/")
@path_param("repository", "The full path of the repository. e.g. namespace/name")
class RepositoryBuildList(RepositoryParamResource):
    """
    Resource related to creating and listing repository builds.
    """

    schemas = {
        "RepositoryBuildRequest": {
            "type": "object",
            "description": "Description of a new repository build.",
            "properties": {
                "file_id": {
                    "type": "string",
                    "description": "The file id that was generated when the build spec was uploaded",
                },
                "archive_url": {
                    "type": "string",
                    "description": 'The URL of the .tar.gz to build. Must start with "http" or "https".',
                },
                "subdirectory": {
                    "type": "string",
                    "description": "Subdirectory in which the Dockerfile can be found. You can only specify this or dockerfile_path",
                },
                "dockerfile_path": {
                    "type": "string",
                    "description": "Path to a dockerfile. You can only specify this or subdirectory.",
                },
                "context": {
                    "type": "string",
                    "description": "Pass in the context for the dockerfile. This is optional.",
                },
                "pull_robot": {
                    "type": "string",
                    "description": "Username of a Quay robot account to use as pull credentials",
                },
                "docker_tags": {
                    "type": "array",
                    "description": "The tags to which the built images will be pushed. "
                    + 'If none specified, "latest" is used.',
                    "items": {"type": "string"},
                    "minItems": 1,
                    "uniqueItems": True,
                },
            },
        },
    }

    @require_repo_read(allow_for_superuser=True, allow_for_global_readonly_superuser=True)
    @parse_args()
    @query_param("limit", "The maximum number of builds to return", type=int, default=5)
    @query_param(
        "since", "Returns all builds since the given unix timecode", type=int, default=None
    )
    @nickname("getRepoBuilds")
    @disallow_for_app_repositories
    def get(self, namespace, repository, parsed_args):
        """
        Get the list of repository builds.
        """
        limit = parsed_args.get("limit", 5)
        since = parsed_args.get("since", None)

        if since is not None:
            since = datetime.datetime.utcfromtimestamp(since)

        builds = model.build.list_repository_builds(namespace, repository, limit, since=since)
        return {"builds": [build_status_view(build) for build in builds]}

    @require_repo_write(allow_for_superuser=True)
    @nickname("requestRepoBuild")
    @disallow_for_app_repositories
    @disallow_for_non_normal_repositories
    @disallow_for_user_namespace
    @validate_json_request("RepositoryBuildRequest")
    def post(self, namespace, repository):
        """
        Request that a repository be built and pushed from the specified input.
        """
        logger.debug("User requested repository initialization.")
        request_json = request.get_json()

        dockerfile_id = request_json.get("file_id", None)
        archive_url = request_json.get("archive_url", None)

        if not dockerfile_id and not archive_url:
            raise InvalidRequest("file_id or archive_url required")

        if archive_url:
            archive_match = None
            try:
                archive_match = urlparse(archive_url)
            except ValueError:
                pass

            if not archive_match:
                raise InvalidRequest("Invalid Archive URL: Must be a valid URI")

            scheme = archive_match.scheme
            if scheme != "http" and scheme != "https":
                raise InvalidRequest("Invalid Archive URL: Must be http or https")

        context, subdir = self.get_dockerfile_context(request_json)
        tags = request_json.get("docker_tags", ["latest"])
        pull_robot_name = request_json.get("pull_robot", None)

        # Verify the security behind the pull robot.
        if pull_robot_name:
            result = parse_robot_username(pull_robot_name)
            if result:
                try:
                    model.user.lookup_robot(pull_robot_name)
                except model.InvalidRobotException:
                    raise NotFound()

                # Make sure the user has administer permissions for the robot's namespace.
                (robot_namespace, _) = result
                if (
                    not AdministerOrganizationPermission(robot_namespace).can()
                    and not allow_if_superuser_with_full_access()
                ):
                    raise Unauthorized()
            else:
                raise Unauthorized()

        # Check if the dockerfile resource has already been used. If so, then it
        # can only be reused if the user has access to the repository in which the
        # dockerfile was previously built.
        if dockerfile_id:
            associated_repository = model.build.get_repository_for_resource(dockerfile_id)
            if associated_repository:
                if (
                    not ModifyRepositoryPermission(
                        associated_repository.namespace_user.username, associated_repository.name
                    )
                    and not allow_if_superuser_with_full_access()
                ):
                    raise Unauthorized()

        # Start the build.
        repo = model.repository.get_repository(namespace, repository)
        if repo is None:
            raise NotFound()

        try:
            build_name = (
                user_files.get_file_checksum(dockerfile_id)
                if dockerfile_id
                else hashlib.sha224(archive_url.encode("ascii")).hexdigest()[0:7]
            )
        except IOError:
            raise InvalidRequest("File %s could not be found or is invalid" % dockerfile_id)

        prepared = PreparedBuild()
        prepared.build_name = build_name
        prepared.dockerfile_id = dockerfile_id
        prepared.archive_url = archive_url
        prepared.tags = tags
        prepared.subdirectory = subdir
        prepared.context = context
        prepared.is_manual = True
        prepared.metadata = {}
        try:
            build_request = start_build(repo, prepared, pull_robot_name=pull_robot_name)
        except MaximumBuildsQueuedException:
            abort(429, message="Maximum queued build rate exceeded.")
        except BuildTriggerDisabledException:
            abort(400, message="Build trigger is disabled")

        resp = build_status_view(build_request)
        repo_string = "%s/%s" % (namespace, repository)
        headers = {
            "Location": api.url_for(
                RepositoryBuildStatus, repository=repo_string, build_uuid=build_request.uuid
            ),
        }
        return resp, 201, headers

    @staticmethod
    def get_dockerfile_context(request_json):
        context = request_json["context"] if "context" in request_json else os.path.sep
        if "dockerfile_path" in request_json:
            subdir = request_json["dockerfile_path"]
            if "context" not in request_json:
                context = os.path.dirname(subdir)
            return context, subdir

        if "subdirectory" in request_json:
            subdir = request_json["subdirectory"]
            context = subdir
            if not subdir.endswith(os.path.sep):
                subdir += os.path.sep

            subdir += "Dockerfile"
        else:
            if context.endswith(os.path.sep):
                subdir = context + "Dockerfile"
            else:
                subdir = context + os.path.sep + "Dockerfile"

        return context, subdir


@resource("/v1/repository/<apirepopath:repository>/build/<build_uuid>")
@path_param("repository", "The full path of the repository. e.g. namespace/name")
@path_param("build_uuid", "The UUID of the build")
class RepositoryBuildResource(RepositoryParamResource):
    """
    Resource for dealing with repository builds.
    """

    @require_repo_read(allow_for_superuser=True, allow_for_global_readonly_superuser=True)
    @nickname("getRepoBuild")
    @disallow_for_app_repositories
    def get(self, namespace, repository, build_uuid):
        """
        Returns information about a build.
        """
        try:
            build = model.build.get_repository_build(build_uuid)
        except model.build.InvalidRepositoryBuildException:
            raise NotFound()

        if (
            build.repository.name != repository
            or build.repository.namespace_user.username != namespace
        ):
            raise NotFound()

        return build_status_view(build)

    @require_repo_admin(allow_for_superuser=True)
    @nickname("cancelRepoBuild")
    @disallow_for_app_repositories
    @disallow_for_non_normal_repositories
    @disallow_for_user_namespace
    def delete(self, namespace, repository, build_uuid):
        """
        Cancels a repository build.
        """
        try:
            build = model.build.get_repository_build(build_uuid)
        except model.build.InvalidRepositoryBuildException:
            raise NotFound()

        if (
            build.repository.name != repository
            or build.repository.namespace_user.username != namespace
        ):
            raise NotFound()

        if model.build.cancel_repository_build(build, dockerfile_build_queue):
            log_action("cancel_build", namespace, {"build_uuid": build_uuid}, repo_name=repository)
            return "Okay", 201
        else:
            raise InvalidRequest("Build is currently running or has finished")


@resource("/v1/repository/<apirepopath:repository>/build/<build_uuid>/status")
@path_param("repository", "The full path of the repository. e.g. namespace/name")
@path_param("build_uuid", "The UUID of the build")
class RepositoryBuildStatus(RepositoryParamResource):
    """
    Resource for dealing with repository build status.
    """

    @require_repo_read(allow_for_superuser=True, allow_for_global_readonly_superuser=True)
    @nickname("getRepoBuildStatus")
    @disallow_for_app_repositories
    def get(self, namespace, repository, build_uuid):
        """
        Return the status for the builds specified by the build uuids.
        """
        build = model.build.get_repository_build(build_uuid)
        if (
            not build
            or build.repository.name != repository
            or build.repository.namespace_user.username != namespace
        ):
            raise NotFound()

        return build_status_view(build)


def get_logs_or_log_url(build):
    # If the logs have been archived, just return a URL of the completed archive
    if build.logs_archived:
        return {
            "logs_url": log_archive.get_file_url(build.uuid, get_request_ip(), requires_cors=True)
        }
    start = int(request.args.get("start", 0))

    try:
        count, logs = build_logs.get_log_entries(build.uuid, start)
    except BuildStatusRetrievalError:
        count, logs = (0, [])

    response_obj = {}
    response_obj.update(
        {
            "start": start,
            "total": count,
            "logs": [log for log in logs],
        }
    )

    return response_obj


@resource("/v1/repository/<apirepopath:repository>/build/<build_uuid>/logs")
@path_param("repository", "The full path of the repository. e.g. namespace/name")
@path_param("build_uuid", "The UUID of the build")
class RepositoryBuildLogs(RepositoryParamResource):
    """
    Resource for loading repository build logs.
    """

    @require_repo_read(allow_for_superuser=True, allow_for_global_readonly_superuser=True)
    @nickname("getRepoBuildLogs")
    @disallow_for_app_repositories
    def get(self, namespace, repository, build_uuid):
        """
        Return the build logs for the build specified by the build uuid.
        """
        can_write = ModifyRepositoryPermission(namespace, repository).can()
        # Global readonly superusers can always view build logs, regular superusers need FULL_ACCESS
        if (
            not features.READER_BUILD_LOGS
            and not can_write
            and not allow_if_global_readonly_superuser()
            and not (features.SUPERUSERS_FULL_ACCESS and allow_if_superuser())
        ):
            raise Unauthorized()

        build = model.build.get_repository_build(build_uuid)
        if (
            not build
            or build.repository.name != repository
            or build.repository.namespace_user.username != namespace
        ):
            raise NotFound()

        return get_logs_or_log_url(build)


@resource("/v1/filedrop/")
@internal_only
class FileDropResource(ApiResource):
    """
    Custom verb for setting up a client side file transfer.
    """

    schemas = {
        "FileDropRequest": {
            "type": "object",
            "description": "Description of the file that the user wishes to upload.",
            "required": [
                "mimeType",
            ],
            "properties": {
                "mimeType": {
                    "type": "string",
                    "description": "Type of the file which is about to be uploaded",
                },
            },
        },
    }

    @nickname("getFiledropUrl")
    @validate_json_request("FileDropRequest")
    def post(self):
        """
        Request a URL to which a file may be uploaded.
        """
        mime_type = request.get_json()["mimeType"]
        (url, file_id) = user_files.prepare_for_drop(mime_type, requires_cors=True)
        return {
            "url": url,
            "file_id": str(file_id),
        }