import logging import os import re from calendar import timegm import dateutil.parser from bitbucket import BitBucket from jsonschema import validate from app import app, get_app_url from buildtrigger.basehandler import BuildTriggerHandler from buildtrigger.triggerutil import ( InvalidPayloadException, RepositoryReadException, SkipRequestException, TriggerActivationException, TriggerDeactivationException, TriggerProviderException, TriggerStartException, determine_build_ref, find_matching_branches, raise_if_skipped_build, ) from util.dict_wrappers import JSONPathDict, SafeDictSetter from util.security.ssh import generate_ssh_keypair logger = logging.getLogger(__name__) _BITBUCKET_COMMIT_URL = "https://bitbucket.org/%s/commits/%s" _RAW_AUTHOR_REGEX = re.compile(r".*<(.+)>") BITBUCKET_WEBHOOK_PAYLOAD_SCHEMA = { "type": "object", "properties": { "repository": { "type": "object", "properties": { "full_name": { "type": "string", }, }, "required": ["full_name"], }, # /Repository "push": { "type": "object", "properties": { "changes": { "type": "array", "items": { "type": "object", "properties": { "new": { "type": "object", "properties": { "target": { "type": "object", "properties": { "hash": {"type": "string"}, "message": {"type": "string"}, "date": {"type": "string"}, "author": { "type": "object", "properties": { "user": { "type": "object", "properties": { "display_name": { "type": "string", }, "account_id": { "type": "string", }, "links": { "type": "object", "properties": { "avatar": { "type": "object", "properties": { "href": { "type": "string", }, }, "required": ["href"], }, }, "required": ["avatar"], }, # /User }, }, # /Author }, }, }, "required": ["hash", "message", "date"], }, # /Target }, "required": ["name", "target"], }, # /New }, }, # /Changes item }, # /Changes }, "required": ["changes"], }, # / Push }, "actor": { "type": "object", "properties": { "account_id": { "type": "string", }, "display_name": { "type": "string", }, "links": { "type": "object", "properties": { "avatar": { "type": "object", "properties": { "href": { "type": "string", }, }, "required": ["href"], }, }, "required": ["avatar"], }, }, }, # /Actor "required": ["push", "repository"], } # /Root BITBUCKET_COMMIT_INFO_SCHEMA = { "type": "object", "properties": { "node": { "type": "string", }, "message": { "type": "string", }, "timestamp": { "type": "string", }, "raw_author": { "type": "string", }, }, "required": ["node", "message", "timestamp"], } def get_transformed_commit_info(bb_commit, ref, default_branch, repository_name, lookup_author): """ Returns the BitBucket commit information transformed into our own payload format. """ try: validate(bb_commit, BITBUCKET_COMMIT_INFO_SCHEMA) except Exception as exc: logger.exception( "Exception when validating Bitbucket commit information: %s from %s", exc.message, bb_commit, ) raise InvalidPayloadException(exc.message) commit = JSONPathDict(bb_commit) config = SafeDictSetter() config["commit"] = commit["node"] config["ref"] = ref config["default_branch"] = default_branch config["git_url"] = "git@bitbucket.org:%s.git" % repository_name config["commit_info.url"] = _BITBUCKET_COMMIT_URL % (repository_name, commit["node"]) config["commit_info.message"] = commit["message"] config["commit_info.date"] = commit["timestamp"] match = _RAW_AUTHOR_REGEX.match(commit["raw_author"]) if match: author = lookup_author(match.group(1)) author_info = JSONPathDict(author) if author is not None else None if author_info: config["commit_info.author.username"] = author_info["user.display_name"] config["commit_info.author.avatar_url"] = author_info["user.avatar"] return config.dict_value() def get_transformed_webhook_payload(bb_payload, default_branch=None): """ Returns the BitBucket webhook JSON payload transformed into our own payload format. If the bb_payload is not valid, returns None. """ try: validate(bb_payload, BITBUCKET_WEBHOOK_PAYLOAD_SCHEMA) except Exception as exc: logger.exception( "Exception when validating Bitbucket webhook payload: %s from %s", exc.message, bb_payload, ) raise InvalidPayloadException(exc.message) payload = JSONPathDict(bb_payload) change = payload["push.changes[-1].new"] if not change: raise SkipRequestException is_branch = change["type"] == "branch" ref = "refs/heads/" + change["name"] if is_branch else "refs/tags/" + change["name"] repository_name = payload["repository.full_name"] target = change["target"] config = SafeDictSetter() config["commit"] = target["hash"] config["ref"] = ref config["default_branch"] = default_branch config["git_url"] = "git@bitbucket.org:%s.git" % repository_name config["commit_info.url"] = target["links.html.href"] or "" config["commit_info.message"] = target["message"] config["commit_info.date"] = target["date"] config["commit_info.author.username"] = target["author.user.display_name"] config["commit_info.author.avatar_url"] = target["author.user.links.avatar.href"] config["commit_info.committer.username"] = payload["actor.display_name"] config["commit_info.committer.avatar_url"] = payload["actor.links.avatar.href"] return config.dict_value() class BitbucketBuildTrigger(BuildTriggerHandler): """ BuildTrigger for Bitbucket. """ @classmethod def service_name(cls): return "bitbucket" def _get_client(self): """ Returns a BitBucket API client for this trigger's config. """ key = app.config.get("BITBUCKET_TRIGGER_CONFIG", {}).get("CONSUMER_KEY", "") secret = app.config.get("BITBUCKET_TRIGGER_CONFIG", {}).get("CONSUMER_SECRET", "") trigger_uuid = self.trigger.uuid callback_url = "%s/oauth1/bitbucket/callback/trigger/%s" % (get_app_url(), trigger_uuid) return BitBucket(key, secret, callback_url, timeout=15) def _get_authorized_client(self): """ Returns an authorized API client. """ base_client = self._get_client() auth_token = self.auth_token or "invalid:invalid" token_parts = auth_token.split(":") if len(token_parts) != 2: token_parts = ["invalid", "invalid"] (access_token, access_token_secret) = token_parts return base_client.get_authorized_client(access_token, access_token_secret) def _get_repository_client(self): """ Returns an API client for working with this config's BB repository. """ source = self.config["build_source"] (namespace, name) = source.split("/") bitbucket_client = self._get_authorized_client() return bitbucket_client.for_namespace(namespace).repositories().get(name) def _get_default_branch(self, repository, default_value="master"): """ Returns the default branch for the repository or the value given. """ (result, data, _) = repository.get_main_branch() if result: return data["name"] return default_value def get_oauth_url(self): """ Returns the OAuth URL to authorize Bitbucket. """ bitbucket_client = self._get_client() (result, data, err_msg) = bitbucket_client.get_authorization_url() if not result: raise TriggerProviderException(err_msg) return data def exchange_verifier(self, verifier): """ Exchanges the given verifier token to setup this trigger. """ bitbucket_client = self._get_client() access_token = self.config.get("access_token", "") access_token_secret = self.auth_token # Exchange the verifier for a new access token. (result, data, _) = bitbucket_client.verify_token( access_token, access_token_secret, verifier ) if not result: return False # Save the updated access token and secret. self.set_auth_token(data[0] + ":" + data[1]) # Retrieve the current authorized user's information and store the username in the config. authorized_client = self._get_authorized_client() (result, data, _) = authorized_client.get_current_user() if not result: return False self.put_config_key("account_id", data["user"]["account_id"]) self.put_config_key("nickname", data["user"]["nickname"]) return True def is_active(self): return "webhook_id" in self.config def activate(self, standard_webhook_url): config = self.config # Add a deploy key to the repository. public_key, private_key = generate_ssh_keypair() config["credentials"] = [ { "name": "SSH Public Key", "value": public_key.decode("ascii"), }, ] repository = self._get_repository_client() (result, created_deploykey, err_msg) = repository.deploykeys().create( app.config["REGISTRY_TITLE"] + " webhook key", public_key.decode("ascii") ) if not result: msg = "Unable to add deploy key to repository: %s" % err_msg raise TriggerActivationException(msg) config["deploy_key_id"] = created_deploykey["pk"] # Add a webhook callback. description = "Webhook for invoking builds on %s" % app.config["REGISTRY_TITLE_SHORT"] webhook_events = ["repo:push"] (result, created_webhook, err_msg) = repository.webhooks().create( description, standard_webhook_url, webhook_events ) if not result: msg = "Unable to add webhook to repository: %s" % err_msg raise TriggerActivationException(msg) config["webhook_id"] = created_webhook["uuid"] self.config = config return config, {"private_key": private_key.decode("ascii")} def deactivate(self): config = self.config webhook_id = config.pop("webhook_id", None) deploy_key_id = config.pop("deploy_key_id", None) repository = self._get_repository_client() # Remove the webhook. if webhook_id is not None: (result, _, err_msg) = repository.webhooks().delete(webhook_id) if not result: msg = "Unable to remove webhook from repository: %s" % err_msg raise TriggerDeactivationException(msg) # Remove the public key. if deploy_key_id is not None: (result, _, err_msg) = repository.deploykeys().delete(deploy_key_id) if not result: msg = "Unable to remove deploy key from repository: %s" % err_msg raise TriggerDeactivationException(msg) return config def list_build_source_namespaces(self): bitbucket_client = self._get_authorized_client() (result, data, err_msg) = bitbucket_client.get_visible_repositories() if not result: raise RepositoryReadException("Could not read repository list: " + err_msg) namespaces = {} for repo in data: owner = repo["owner"] if owner in namespaces: namespaces[owner]["score"] = namespaces[owner]["score"] + 1 else: namespaces[owner] = { "personal": owner == self.config.get("nickname", self.config.get("username")), "id": owner, "title": owner, "avatar_url": repo["logo"], "url": "https://bitbucket.org/%s" % (owner), "score": 1, } return BuildTriggerHandler.build_namespaces_response(namespaces) def list_build_sources_for_namespace(self, namespace): def repo_view(repo): last_modified = dateutil.parser.parse(repo["utc_last_updated"]) return { "name": repo["slug"], "full_name": "%s/%s" % (repo["owner"], repo["slug"]), "description": repo["description"] or "", "last_updated": timegm(last_modified.utctimetuple()), "url": "https://bitbucket.org/%s/%s" % (repo["owner"], repo["slug"]), "has_admin_permissions": repo["read_only"] is False, "private": repo["is_private"], } bitbucket_client = self._get_authorized_client() (result, data, err_msg) = bitbucket_client.get_visible_repositories() if not result: raise RepositoryReadException("Could not read repository list: " + err_msg) repos = [repo_view(repo) for repo in data if repo["owner"] == namespace] return BuildTriggerHandler.build_sources_response(repos) def list_build_subdirs(self): config = self.config repository = self._get_repository_client() # Find the first matching branch. repo_branches = self.list_field_values("branch_name") or [] branches = find_matching_branches(config, repo_branches) if not branches: branches = [self._get_default_branch(repository)] (result, data, err_msg) = repository.get_path_contents("", revision=branches[0]) if not result: raise RepositoryReadException(err_msg) files = set([f["path"] for f in data["files"]]) return [ "/" + file_path for file_path in files if self.filename_is_dockerfile(os.path.basename(file_path)) ] def load_dockerfile_contents(self): repository = self._get_repository_client() path = self.get_dockerfile_path() (result, data, err_msg) = repository.get_raw_path_contents(path, revision="master") if not result: return None return data def list_field_values(self, field_name, limit=None): if "build_source" not in self.config: return None source = self.config["build_source"] (namespace, name) = source.split("/") bitbucket_client = self._get_authorized_client() repository = bitbucket_client.for_namespace(namespace).repositories().get(name) if field_name == "refs": (result, data, _) = repository.get_branches_and_tags() if not result: return None branches = [b["name"] for b in data["branches"]] tags = [t["name"] for t in data["tags"]] return [{"kind": "branch", "name": b} for b in branches] + [ {"kind": "tag", "name": tag} for tag in tags ] if field_name == "tag_name": (result, data, _) = repository.get_tags() if not result: return None tags = list(data.keys()) if limit: tags = tags[0:limit] return tags if field_name == "branch_name": (result, data, _) = repository.get_branches() if not result: return None branches = list(data.keys()) if limit: branches = branches[0:limit] return branches return None def get_repository_url(self): source = self.config["build_source"] (namespace, name) = source.split("/") return "https://bitbucket.org/%s/%s" % (namespace, name) def handle_trigger_request(self, request): payload = request.get_json() if payload is None: raise InvalidPayloadException("Missing payload") logger.debug("Got BitBucket request: %s", payload) repository = self._get_repository_client() default_branch = self._get_default_branch(repository) metadata = get_transformed_webhook_payload(payload, default_branch=default_branch) prepared = self.prepare_build(metadata) # Check if we should skip this build. raise_if_skipped_build(prepared, self.config) return prepared def manual_start(self, run_parameters=None): run_parameters = run_parameters or {} repository = self._get_repository_client() bitbucket_client = self._get_authorized_client() def get_branch_sha(branch_name): # Lookup the commit SHA for the branch. (result, data, _) = repository.get_branch(branch_name) if not result: raise TriggerStartException("Could not find branch in repository") return data["target"]["hash"] def get_tag_sha(tag_name): # Lookup the commit SHA for the tag. (result, data, _) = repository.get_tag(tag_name) if not result: raise TriggerStartException("Could not find tag in repository") return data["target"]["hash"] def lookup_author(email_address): (result, data, _) = bitbucket_client.accounts().get_profile(email_address) return data if result else None # Find the branch or tag to build. default_branch = self._get_default_branch(repository) (commit_sha, ref) = determine_build_ref( run_parameters, get_branch_sha, get_tag_sha, default_branch ) # Lookup the commit SHA in BitBucket. (result, commit_info, _) = repository.changesets().get(commit_sha) if not result: raise TriggerStartException("Could not lookup commit SHA") # Return a prepared build for the commit. repository_name = "%s/%s" % (repository.namespace, repository.repository_name) metadata = get_transformed_commit_info( commit_info, ref, default_branch, repository_name, lookup_author ) return self.prepare_build(metadata, is_manual=True)