diff --git a/data/logs_model/__init__.py b/data/logs_model/__init__.py index c8320e307..dfdcb6012 100644 --- a/data/logs_model/__init__.py +++ b/data/logs_model/__init__.py @@ -1,5 +1,6 @@ import logging +from data.logs_model.splunk_logs_model import SplunkLogsModel from data.logs_model.table_logs_model import TableLogsModel from data.logs_model.document_logs_model import DocumentLogsModel from data.logs_model.combined_model import CombinedLogsModel @@ -18,6 +19,7 @@ _LOG_MODELS = { "database": TableLogsModel, "transition_reads_both_writes_es": _transition_model, "elasticsearch": DocumentLogsModel, + "splunk": SplunkLogsModel, } _PULL_LOG_KINDS = {"pull_repo", "repo_verb"} @@ -43,7 +45,7 @@ logs_model = LogsModelProxy() def configure(app_config): - logger.debug("Configuring log lodel") + logger.debug("Configuring log model") model_name = app_config.get("LOGS_MODEL", "database") model_config = app_config.get("LOGS_MODEL_CONFIG", {}) diff --git a/data/logs_model/logs_producer/splunk_logs_producer.py b/data/logs_model/logs_producer/splunk_logs_producer.py new file mode 100644 index 000000000..e3f07621f --- /dev/null +++ b/data/logs_model/logs_producer/splunk_logs_producer.py @@ -0,0 +1,44 @@ +import logging + +from splunklib import client + +from data.logs_model.logs_producer import LogSendException +from data.logs_model.logs_producer.interface import LogProducerInterface + +logger = logging.getLogger(__name__) + + +class SplunkLogsProducer(LogProducerInterface): + """ + Log producer for writing log entries to Splunk + This implementation writes directly to Splunk without a streaming/queueing service. + """ + + def __init__( + self, + host, + port, + bearer_token, + url_scheme="https", + verify_ssl=True, + index_prefix=None, + ): + try: + service = client.connect( + host=host, port=port, token=bearer_token, scheme=url_scheme, verify=verify_ssl + ) + except Exception as ex: + logger.exception("Failed to connect to Splunk instance %s", ex) + raise ex + try: + self.index = service.indexes[index_prefix] + logger.info("splunk index %s", self.index) + except KeyError: + self.index = service.indexes.create(index_prefix) + + def send(self, log): + try: + self.index.submit(log, sourcetype="access_combined", host="quay") + except Exception as e: + logger.exception("SplunkLogsProducer exception sending log to Splunk: %s", e) + raise LogSendException("SplunkLogsProducer exception sending log to Splunk: %s" % e) diff --git a/data/logs_model/splunk_logs_model.py b/data/logs_model/splunk_logs_model.py new file mode 100644 index 000000000..701fd4ee2 --- /dev/null +++ b/data/logs_model/splunk_logs_model.py @@ -0,0 +1,144 @@ +import json +import logging + +from datetime import datetime + +from data import model +from data.logs_model.interface import ActionLogsDataInterface +from data.logs_model.logs_producer import LogProducerProxy, LogSendException +from data.logs_model.logs_producer.splunk_logs_producer import SplunkLogsProducer +from data.logs_model.shared import SharedModel +from data.model import config +from data.model.log import ACTIONS_ALLOWED_WITHOUT_AUDIT_LOGGING + +logger = logging.getLogger(__name__) + + +class SplunkLogsModel(SharedModel, ActionLogsDataInterface): + """ + SplunkLogsModel implements model for establishing connection and sending events to Splunk cluster + """ + + def __init__(self, producer, splunk_config, should_skip_logging=None): + self._should_skip_logging = should_skip_logging + self._logs_producer = LogProducerProxy() + if producer == "splunk": + self._logs_producer.initialize(SplunkLogsProducer(**splunk_config)) + else: + raise Exception("Invalid log producer: %s" % producer) + + def log_action( + self, + kind_name, + namespace_name=None, + performer=None, + ip=None, + metadata=None, + repository=None, + repository_name=None, + timestamp=None, + is_free_namespace=False, + ): + + if self._should_skip_logging and self._should_skip_logging( + kind_name, namespace_name, is_free_namespace + ): + return + + if repository_name is not None: + if repository is None or namespace_name is not None: + raise ValueError( + "Incorrect argument provided when logging action logs, namespace name should not be " + "empty" + ) + repository = model.repository.get_repository(namespace_name, repository_name) + + if timestamp is None: + timestamp = datetime.today() + + account_id = None + performer_id = None + repository_id = None + + if namespace_name is not None: + ns_user = model.user.get_namespace_user(namespace_name) + if ns_user is not None: + account_id = ns_user.id + + if performer is not None: + performer_id = performer.id + + if repository is not None: + repository_id = repository.id + + kind_id = model.log._get_log_entry_kind(kind_name) + + metadata_json = metadata or {} + + log_data = { + "kind": kind_id, + "account": account_id, + "performer": performer_id, + "repository": repository_id, + "ip": ip, + "metadata_json": metadata_json or {}, + "datetime": timestamp, + } + + try: + self._logs_producer.send(json.dumps(log_data, sort_keys=True, default=str)) + except LogSendException as lse: + strict_logging_disabled = config.app_config.get("ALLOW_PULLS_WITHOUT_STRICT_LOGGING") + logger.exception("log_action failed", extra=({"exception": lse}).update(log_data)) + if not (strict_logging_disabled and kind_name in ACTIONS_ALLOWED_WITHOUT_AUDIT_LOGGING): + raise + + def lookup_logs( + self, + start_datetime, + end_datetime, + performer_name=None, + repository_name=None, + namespace_name=None, + filter_kinds=None, + page_token=None, + max_page_count=None, + ): + raise NotImplementedError("Method not implemented, Splunk does not support log lookups") + + def lookup_latest_logs( + self, + performer_name=None, + repository_name=None, + namespace_name=None, + filter_kinds=None, + size=20, + ): + raise NotImplementedError("Method not implemented, Splunk does not support log lookups") + + def get_aggregated_log_counts( + self, + start_datetime, + end_datetime, + performer_name=None, + repository_name=None, + namespace_name=None, + filter_kinds=None, + ): + raise NotImplementedError("Method not implemented, Splunk does not support log lookups") + + def count_repository_actions(self, repository, day): + raise NotImplementedError("Method not implemented, Splunk does not support log lookups") + + def yield_logs_for_export( + self, + start_datetime, + end_datetime, + repository_id=None, + namespace_id=None, + max_query_time=None, + ): + raise NotImplementedError("Method not implemented, Splunk does not support log lookups") + + def yield_log_rotation_context(self, cutoff_date, min_logs_per_rotation): + raise NotImplementedError("Method not implemented, Splunk does not support log lookups") diff --git a/data/logs_model/test/test_splunk.py b/data/logs_model/test/test_splunk.py new file mode 100644 index 000000000..72cf7dabe --- /dev/null +++ b/data/logs_model/test/test_splunk.py @@ -0,0 +1,56 @@ +from unittest.mock import MagicMock + +import pytest +from dateutil.parser import parse +from mock import patch, Mock + +from .test_elasticsearch import logs_model, mock_db_model +from data.logs_model import configure +from test.fixtures import * +from data.model.repository import create_repository + +FAKE_SPLUNK_HOST = "fakesplunk" +FAKE_SPLUNK_PORT = 443 +FAKE_SPLUNK_TOKEN = None +FAKE_INDEX_PREFIX = "test_index_prefix" + + +@pytest.fixture() +def splunk_logs_model_config(): + conf = { + "LOGS_MODEL": "splunk", + "LOGS_MODEL_CONFIG": { + "producer": "splunk", + "splunk_config": { + "host": FAKE_SPLUNK_HOST, + "port": FAKE_SPLUNK_PORT, + "bearer_token": FAKE_SPLUNK_TOKEN, + "url_scheme": "https", + "verify_ssl": True, + "index_prefix": FAKE_INDEX_PREFIX, + }, + }, + } + return conf + + +def test_splunk_logs_producers(logs_model, splunk_logs_model_config, mock_db_model, initialized_db): + + producer_config = splunk_logs_model_config + with patch( + "data.logs_model.logs_producer.splunk_logs_producer.SplunkLogsProducer.send" + ) as mock_send, patch("splunklib.client.connect", MagicMock()): + repo = create_repository("devtable", "somenewrepo", None, repo_kind="image") + configure(producer_config) + logs_model.log_action( + "pull_repo", + "devtable", + Mock(id=1), + "192.168.1.1", + {"key": "value"}, + repo, + None, + parse("2019-01-01T03:30"), + ) + + mock_send.assert_called_once() diff --git a/endpoints/decorated.py b/endpoints/decorated.py index f9960526a..4fd0ff9a8 100644 --- a/endpoints/decorated.py +++ b/endpoints/decorated.py @@ -62,3 +62,11 @@ def handle_readonly(ex): ) response.status_code = 503 return response + + +@app.errorhandler(NotImplementedError) +def handle_not_implemented_error(ex): + logger.exception(ex) + response = jsonify({"message": str(ex)}) + response.status_code = 501 + return response diff --git a/requirements.txt b/requirements.txt index 4c42b13e1..24df149f2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -114,6 +114,7 @@ s3transfer==0.5.1 semantic-version==2.8.4 six==1.14.0 soupsieve==1.9.5 +splunk-sdk==1.7.3 SQLAlchemy==1.4.31 stevedore==1.31.0 stringscore==0.1.0 diff --git a/static/directives/logs-view.html b/static/directives/logs-view.html index 1e7f6b0c9..7fa97a03c 100644 --- a/static/directives/logs-view.html +++ b/static/directives/logs-view.html @@ -35,11 +35,12 @@
-
{{ splunkLogMsg }}
+
- +
-
+