diff --git a/requirements.txt b/requirements.txt index f7c8315b..89d78bfe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,5 @@ sphinx-copybutton == 0.2.12 git+https://github.com/ravindk89/sphinx-tabs recommonmark == 0.6.0 sphinx-markdown-tables == 0.0.15 -Sphinx-Substitution-Extensions == 2020.9.30.0 \ No newline at end of file +Sphinx-Substitution-Extensions == 2020.9.30.0 +sphinx-togglebutton === 0.2.3 \ No newline at end of file diff --git a/source/concepts/bucket-notifications.md b/source/concepts/bucket-notifications.md deleted file mode 100644 index b3e3fa97..00000000 --- a/source/concepts/bucket-notifications.md +++ /dev/null @@ -1,1416 +0,0 @@ -# MinIO Bucket Notification Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) - -Events occurring on objects in a bucket can be monitored using bucket event notifications. Event types supported by MinIO server are - -| Supported Event Types | | | -| :---------------------- | ------------------------------------------ | ------------------------ | -| `s3:ObjectCreated:Put` | `s3:ObjectCreated:CompleteMultipartUpload` | `s3:ObjectAccessed:Head` | -| `s3:ObjectCreated:Post` | `s3:ObjectRemoved:Delete` | | -| `s3:ObjectCreated:Copy` | `s3:ObjectAccessed:Get` | | - -Use client tools like `mc` to set and listen for event notifications using the [`event` sub-command](https://docs.min.io/docs/minio-client-complete-guide#events). MinIO SDK's [`BucketNotification` APIs](https://docs.min.io/docs/golang-client-api-reference#SetBucketNotification) can also be used. The notification message MinIO sends to publish an event is a JSON message with the following [structure](https://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html). - -Bucket events can be published to the following targets: - -| Supported Notification Targets | | | -| :-------------------------------- | --------------------------- | ------------------------------- | -| [`AMQP`](#AMQP) | [`Redis`](#Redis) | [`MySQL`](#MySQL) | -| [`MQTT`](#MQTT) | [`NATS`](#NATS) | [`Apache Kafka`](#apache-kafka) | -| [`Elasticsearch`](#Elasticsearch) | [`PostgreSQL`](#PostgreSQL) | [`Webhooks`](#webhooks) | -| [`NSQ`](#NSQ) | | | - -## Prerequisites - -- Install and configure MinIO Server from [here](https://docs.min.io/docs/minio-quickstart-guide). -- Install and configure MinIO Client from [here](https://docs.min.io/docs/minio-client-quickstart-guide). - -``` -$ mc admin config get myminio | grep notify -notify_webhook publish bucket notifications to webhook endpoints -notify_amqp publish bucket notifications to AMQP endpoints -notify_kafka publish bucket notifications to Kafka endpoints -notify_mqtt publish bucket notifications to MQTT endpoints -notify_nats publish bucket notifications to NATS endpoints -notify_nsq publish bucket notifications to NSQ endpoints -notify_mysql publish bucket notifications to MySQL databases -notify_postgres publish bucket notifications to Postgres databases -notify_elasticsearch publish bucket notifications to Elasticsearch endpoints -notify_redis publish bucket notifications to Redis datastores -``` - -> NOTE: -> - '\*' at the end of arg means its mandatory. -> - '\*' at the end of the values, means its the default value for the arg. -> - When configured using environment variables, the `:name` can be specified using this format `MINIO_NOTIFY_WEBHOOK_ENABLE_`. - - - -## Publish MinIO events via AMQP - -Install RabbitMQ from [here](https://www.rabbitmq.com/). - -### Step 1: Add AMQP endpoint to MinIO - -The AMQP configuration is located under the sub-system `notify_amqp` top-level key. Create a configuration key-value pair here for your AMQP instance. The key is a name for your AMQP endpoint, and the value is a collection of key-value parameters described in the table below. - -``` -KEY: -notify_amqp[:name] publish bucket notifications to AMQP endpoints - -ARGS: -url* (url) AMQP server endpoint e.g. `amqp://myuser:mypassword@localhost:5672` -exchange (string) name of the AMQP exchange -exchange_type (string) AMQP exchange type -routing_key (string) routing key for publishing -mandatory (on|off) quietly ignore undelivered messages when set to 'off', default is 'on' -durable (on|off) persist queue across broker restarts when set to 'on', default is 'off' -no_wait (on|off) non-blocking message delivery when set to 'on', default is 'off' -internal (on|off) set to 'on' for exchange to be not used directly by publishers, but only when bound to other exchanges -auto_deleted (on|off) auto delete queue when set to 'on', when there are no consumers -delivery_mode (number) set to '1' for non-persistent or '2' for persistent queue -queue_dir (path) staging dir for undelivered messages e.g. '/home/events' -queue_limit (number) maximum limit for undelivered messages, defaults to '100000' -comment (sentence) optionally add a comment to this setting -``` - -Or environment variables - -``` -KEY: -notify_amqp[:name] publish bucket notifications to AMQP endpoints - -ARGS: -MINIO_NOTIFY_AMQP_ENABLE* (on|off) enable notify_amqp target, default is 'off' -MINIO_NOTIFY_AMQP_URL* (url) AMQP server endpoint e.g. `amqp://myuser:mypassword@localhost:5672` -MINIO_NOTIFY_AMQP_EXCHANGE (string) name of the AMQP exchange -MINIO_NOTIFY_AMQP_EXCHANGE_TYPE (string) AMQP exchange type -MINIO_NOTIFY_AMQP_ROUTING_KEY (string) routing key for publishing -MINIO_NOTIFY_AMQP_MANDATORY (on|off) quietly ignore undelivered messages when set to 'off', default is 'on' -MINIO_NOTIFY_AMQP_DURABLE (on|off) persist queue across broker restarts when set to 'on', default is 'off' -MINIO_NOTIFY_AMQP_NO_WAIT (on|off) non-blocking message delivery when set to 'on', default is 'off' -MINIO_NOTIFY_AMQP_INTERNAL (on|off) set to 'on' for exchange to be not used directly by publishers, but only when bound to other exchanges -MINIO_NOTIFY_AMQP_AUTO_DELETED (on|off) auto delete queue when set to 'on', when there are no consumers -MINIO_NOTIFY_AMQP_DELIVERY_MODE (number) set to '1' for non-persistent or '2' for persistent queue -MINIO_NOTIFY_AMQP_QUEUE_DIR (path) staging dir for undelivered messages e.g. '/home/events' -MINIO_NOTIFY_AMQP_QUEUE_LIMIT (number) maximum limit for undelivered messages, defaults to '100000' -MINIO_NOTIFY_AMQP_COMMENT (sentence) optionally add a comment to this setting -``` - -MinIO supports persistent event store. The persistent store will backup events when the AMQP broker goes offline and replays it when the broker comes back online. The event store can be configured by setting the directory path in `queue_dir` field and the maximum limit of events in the queue_dir in `queue_limit` field. For eg, the `queue_dir` can be `/home/events` and `queue_limit` can be `1000`. By default, the `queue_limit` is set to 100000. - -To update the configuration, use `mc admin config get notify_amqp` command to get the current configuration for `notify_amqp`. - -```sh -$ mc admin config get myminio/ notify_amqp -notify_amqp:1 delivery_mode="0" exchange_type="" no_wait="off" queue_dir="" queue_limit="0" url="" auto_deleted="off" durable="off" exchange="" internal="off" mandatory="off" routing_key="" -``` - -Use `mc admin config set` command to update the configuration for the deployment.Restart the MinIO server to put the changes into effect. The server will print a line like `SQS ARNs: arn:minio:sqs::1:amqp` at start-up if there were no errors. - -An example configuration for RabbitMQ is shown below: - -```sh -$ mc admin config set myminio/ notify_amqp:1 exchange="bucketevents" exchange_type="fanout" mandatory="false" no_wait="false" url="amqp://myuser:mypassword@localhost:5672" auto_deleted="false" delivery_mode="0" durable="false" internal="false" routing_key="bucketlogs" -``` - -MinIO supports all the exchanges available in [RabbitMQ](https://www.rabbitmq.com/). For this setup, we are using `fanout` exchange. - -Note that, you can add as many AMQP server endpoint configurations as needed by providing an identifier (like "1" in the example above) for the AMQP instance and an object of per-server configuration parameters. - -### Step 2: Enable bucket notification using MinIO client - -We will enable bucket event notification to trigger whenever a JPEG image is uploaded or deleted `images` bucket on `myminio` server. Here ARN value is `arn:minio:sqs::1:amqp`. To understand more about ARN please follow [AWS ARN](http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html) documentation. - -``` -mc mb myminio/images -mc event add myminio/images arn:minio:sqs::1:amqp --suffix .jpg -mc event list myminio/images -arn:minio:sqs::1:amqp s3:ObjectCreated:*,s3:ObjectRemoved:* Filter: suffix=”.jpg” -``` - -### Step 3: Test on RabbitMQ - -The python program below waits on the queue exchange `bucketevents` and prints event notifications on the console. We use [Pika Python Client](https://www.rabbitmq.com/tutorials/tutorial-three-python.html) library to do this. - -```python -#!/usr/bin/env python -import pika - -connection = pika.BlockingConnection(pika.ConnectionParameters( - host='localhost')) -channel = connection.channel() - -channel.exchange_declare(exchange='bucketevents', - exchange_type='fanout') - -result = channel.queue_declare(exclusive=False) -queue_name = result.method.queue - -channel.queue_bind(exchange='bucketevents', - queue=queue_name) - -print(' [*] Waiting for logs. To exit press CTRL+C') - -def callback(ch, method, properties, body): - print(" [x] %r" % body) - -channel.basic_consume(callback, - queue=queue_name, - no_ack=False) - -channel.start_consuming() -``` - -Execute this example python program to watch for RabbitMQ events on the console. - -```python -python rabbit.py -``` - -Open another terminal and upload a JPEG image into `images` bucket. - -``` -mc cp myphoto.jpg myminio/images -``` - -You should receive the following event notification via RabbitMQ once the upload completes. - -```python -python rabbit.py -'{"Records":[{"eventVersion":"2.0","eventSource":"aws:s3","awsRegion":"","eventTime":"2016–09–08T22:34:38.226Z","eventName":"s3:ObjectCreated:Put","userIdentity":{"principalId":"minio"},"requestParameters":{"sourceIPAddress":"10.1.10.150:44576"},"responseElements":{},"s3":{"s3SchemaVersion":"1.0","configurationId":"Config","bucket":{"name":"images","ownerIdentity":{"principalId":"minio"},"arn":"arn:aws:s3:::images"},"object":{"key":"myphoto.jpg","size":200436,"sequencer":"147279EAF9F40933"}}}],"level":"info","msg":"","time":"2016–09–08T15:34:38–07:00"}' -``` - - - -## Publish MinIO events MQTT - -Install an MQTT Broker from [here](https://mosquitto.org/). - -### Step 1: Add MQTT endpoint to MinIO - -The MQTT configuration is located as `notify_mqtt` key. Create a configuration key-value pair here for your MQTT instance. The key is a name for your MQTT endpoint, and the value is a collection of key-value parameters described in the table below. - -``` -KEY: -notify_mqtt[:name] publish bucket notifications to MQTT endpoints - -ARGS: -broker* (uri) MQTT server endpoint e.g. `tcp://localhost:1883` -topic* (string) name of the MQTT topic to publish -username (string) MQTT username -password (string) MQTT password -qos (number) set the quality of service priority, defaults to '0' -keep_alive_interval (duration) keep-alive interval for MQTT connections in s,m,h,d -reconnect_interval (duration) reconnect interval for MQTT connections in s,m,h,d -queue_dir (path) staging dir for undelivered messages e.g. '/home/events' -queue_limit (number) maximum limit for undelivered messages, defaults to '100000' -comment (sentence) optionally add a comment to this setting -``` - -or environment variables - -``` -KEY: -notify_mqtt[:name] publish bucket notifications to MQTT endpoints - -ARGS: -MINIO_NOTIFY_MQTT_ENABLE* (on|off) enable notify_mqtt target, default is 'off' -MINIO_NOTIFY_MQTT_BROKER* (uri) MQTT server endpoint e.g. `tcp://localhost:1883` -MINIO_NOTIFY_MQTT_TOPIC* (string) name of the MQTT topic to publish -MINIO_NOTIFY_MQTT_USERNAME (string) MQTT username -MINIO_NOTIFY_MQTT_PASSWORD (string) MQTT password -MINIO_NOTIFY_MQTT_QOS (number) set the quality of service priority, defaults to '0' -MINIO_NOTIFY_MQTT_KEEP_ALIVE_INTERVAL (duration) keep-alive interval for MQTT connections in s,m,h,d -MINIO_NOTIFY_MQTT_RECONNECT_INTERVAL (duration) reconnect interval for MQTT connections in s,m,h,d -MINIO_NOTIFY_MQTT_QUEUE_DIR (path) staging dir for undelivered messages e.g. '/home/events' -MINIO_NOTIFY_MQTT_QUEUE_LIMIT (number) maximum limit for undelivered messages, defaults to '100000' -MINIO_NOTIFY_MQTT_COMMENT (sentence) optionally add a comment to this setting -``` - -MinIO supports persistent event store. The persistent store will backup events when the MQTT broker goes offline and replays it when the broker comes back online. The event store can be configured by setting the directory path in `queue_dir` field and the maximum limit of events in the queue_dir in `queue_limit` field. For eg, the `queue_dir` can be `/home/events` and `queue_limit` can be `1000`. By default, the `queue_limit` is set to 100000. - -To update the configuration, use `mc admin config get` command to get the current configuration. - -```sh -$ mc admin config get myminio/ notify_mqtt -notify_mqtt:1 broker="" password="" queue_dir="" queue_limit="0" reconnect_interval="0s" keep_alive_interval="0s" qos="0" topic="" username="" -``` - -Use `mc admin config set` command to update the configuration for the deployment. Restart the MinIO server to put the changes into effect. The server will print a line like `SQS ARNs: arn:minio:sqs::1:mqtt` at start-up if there were no errors. - -```sh -$ mc admin config set myminio notify_mqtt:1 broker="tcp://localhost:1883" password="" queue_dir="" queue_limit="0" reconnect_interval="0s" keep_alive_interval="0s" qos="1" topic="minio" username="" -``` - -MinIO supports any MQTT server that supports MQTT 3.1 or 3.1.1 and can connect to them over TCP, TLS, or a Websocket connection using `tcp://`, `tls://`, or `ws://` respectively as the scheme for the broker url. See the [Go Client](http://www.eclipse.org/paho/clients/golang/) documentation for more information. - -Note that, you can add as many MQTT server endpoint configurations as needed by providing an identifier (like "1" in the example above) for the MQTT instance and an object of per-server configuration parameters. - -### Step 2: Enable bucket notification using MinIO client - -We will enable bucket event notification to trigger whenever a JPEG image is uploaded or deleted `images` bucket on `myminio` server. Here ARN value is `arn:minio:sqs::1:mqtt`. - -``` -mc mb myminio/images -mc event add myminio/images arn:minio:sqs::1:mqtt --suffix .jpg -mc event list myminio/images -arn:minio:sqs::1:amqp s3:ObjectCreated:*,s3:ObjectRemoved:* Filter: suffix=”.jpg” -``` - -### Step 3: Test on MQTT - -The python program below waits on mqtt topic `/minio` and prints event notifications on the console. We use [paho-mqtt](https://pypi.python.org/pypi/paho-mqtt/) library to do this. - -```python -#!/usr/bin/env python3 -from __future__ import print_function -import paho.mqtt.client as mqtt - -# This is the Subscriber - -def on_connect(client, userdata, flags, rc): - print("Connected with result code "+str(rc)) - # qos level is set to 1 - client.subscribe("minio", 1) - -def on_message(client, userdata, msg): - print(msg.payload) - -# client_id is a randomly generated unique ID for the mqtt broker to identify the connection. -client = mqtt.Client(client_id="myclientid",clean_session=False) - -client.on_connect = on_connect -client.on_message = on_message - -client.connect("localhost",1883,60) -client.loop_forever() -``` - -Execute this example python program to watch for MQTT events on the console. - -```python -python mqtt.py -``` - -Open another terminal and upload a JPEG image into `images` bucket. - -``` -mc cp myphoto.jpg myminio/images -``` - -You should receive the following event notification via MQTT once the upload completes. - -```python -python mqtt.py -{“Records”:[{“eventVersion”:”2.0",”eventSource”:”aws:s3",”awsRegion”:”",”eventTime”:”2016–09–08T22:34:38.226Z”,”eventName”:”s3:ObjectCreated:Put”,”userIdentity”:{“principalId”:”minio”},”requestParameters”:{“sourceIPAddress”:”10.1.10.150:44576"},”responseElements”:{},”s3":{“s3SchemaVersion”:”1.0",”configurationId”:”Config”,”bucket”:{“name”:”images”,”ownerIdentity”:{“principalId”:”minio”},”arn”:”arn:aws:s3:::images”},”object”:{“key”:”myphoto.jpg”,”size”:200436,”sequencer”:”147279EAF9F40933"}}}],”level”:”info”,”msg”:””,”time”:”2016–09–08T15:34:38–07:00"} -``` - - - -## Publish MinIO events via Elasticsearch - -Install [Elasticsearch](https://www.elastic.co/downloads/elasticsearch) server. - -This notification target supports two formats: _namespace_ and _access_. - -When the _namespace_ format is used, MinIO synchronizes objects in the bucket with documents in the index. For each event in the MinIO, the server creates a document with the bucket and object name from the event as the document ID. Other details of the event are stored in the body of the document. Thus if an existing object is over-written in MinIO, the corresponding document in the Elasticsearch index is updated. If an object is deleted, the corresponding document is deleted from the index. - -When the _access_ format is used, MinIO appends events as documents in an Elasticsearch index. For each event, a document with the event details, with the timestamp of document set to the event's timestamp is appended to an index. The ID of the documented is randomly generated by Elasticsearch. No documents are deleted or modified in this format. - -The steps below show how to use this notification target in `namespace` format. The other format is very similar and is omitted for brevity. - -### Step 1: Ensure minimum requirements are met - -MinIO requires a 5.x series version of Elasticsearch. This is the latest major release series. Elasticsearch provides version upgrade migration guidelines [here](https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-upgrade.html). - -### Step 2: Add Elasticsearch endpoint to MinIO - -The Elasticsearch configuration is located in the `notify_elasticsearch` key. Create a configuration key-value pair here for your Elasticsearch instance. The key is a name for your Elasticsearch endpoint, and the value is a collection of key-value parameters described in the table below. - -``` -KEY: -notify_elasticsearch[:name] publish bucket notifications to Elasticsearch endpoints - -ARGS: -url* (url) Elasticsearch server's address, with optional authentication info -index* (string) Elasticsearch index to store/update events, index is auto-created -format* (namespace*|access) 'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace' -queue_dir (path) staging dir for undelivered messages e.g. '/home/events' -queue_limit (number) maximum limit for undelivered messages, defaults to '100000' -comment (sentence) optionally add a comment to this setting -``` - -or environment variables - -``` -KEY: -notify_elasticsearch[:name] publish bucket notifications to Elasticsearch endpoints - -ARGS: -MINIO_NOTIFY_ELASTICSEARCH_ENABLE* (on|off) enable notify_elasticsearch target, default is 'off' -MINIO_NOTIFY_ELASTICSEARCH_URL* (url) Elasticsearch server's address, with optional authentication info -MINIO_NOTIFY_ELASTICSEARCH_INDEX* (string) Elasticsearch index to store/update events, index is auto-created -MINIO_NOTIFY_ELASTICSEARCH_FORMAT* (namespace*|access) 'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace' -MINIO_NOTIFY_ELASTICSEARCH_QUEUE_DIR (path) staging dir for undelivered messages e.g. '/home/events' -MINIO_NOTIFY_ELASTICSEARCH_QUEUE_LIMIT (number) maximum limit for undelivered messages, defaults to '100000' -MINIO_NOTIFY_ELASTICSEARCH_COMMENT (sentence) optionally add a comment to this setting -``` - -For example: `http://localhost:9200` or with authentication info `http://elastic:MagicWord@127.0.0.1:9200`. - -MinIO supports persistent event store. The persistent store will backup events when the Elasticsearch broker goes offline and replays it when the broker comes back online. The event store can be configured by setting the directory path in `queue_dir` field and the maximum limit of events in the queue_dir in `queue_limit` field. For eg, the `queue_dir` can be `/home/events` and `queue_limit` can be `1000`. By default, the `queue_limit` is set to 100000. - -If Elasticsearch has authentication enabled, the credentials can be supplied to MinIO via the `url` parameter formatted as `PROTO://USERNAME:PASSWORD@ELASTICSEARCH_HOST:PORT`. - -To update the configuration, use `mc admin config get` command to get the current configuration. - -```sh -$ mc admin config get myminio/ notify_elasticsearch -notify_elasticsearch:1 queue_limit="0" url="" format="namespace" index="" queue_dir="" -``` - -Use `mc admin config set` command to update the configuration for the deployment. Restart the MinIO server to put the changes into effect. The server will print a line like `SQS ARNs: arn:minio:sqs::1:elasticsearch` at start-up if there were no errors. - -```sh -$ mc admin config set myminio notify_elasticsearch:1 queue_limit="0" url="http://127.0.0.1:9200" format="namespace" index="minio_events" queue_dir="" -``` - -Note that, you can add as many Elasticsearch server endpoint configurations as needed by providing an identifier (like "1" in the example above) for the Elasticsearch instance and an object of per-server configuration parameters. - -### Step 3: Enable bucket notification using MinIO client - -We will now enable bucket event notifications on a bucket named `images`. Whenever a JPEG image is created/overwritten, a new document is added or an existing document is updated in the Elasticsearch index configured above. When an existing object is deleted, the corresponding document is deleted from the index. Thus, the rows in the Elasticsearch index, reflect the `.jpg` objects in the `images` bucket. - -To configure this bucket notification, we need the ARN printed by MinIO in the previous step. Additional information about ARN is available [here](http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html). - -With the `mc` tool, the configuration is very simple to add. Let us say that the MinIO server is aliased as `myminio` in our mc configuration. Execute the following: - -``` -mc mb myminio/images -mc event add myminio/images arn:minio:sqs::1:elasticsearch --suffix .jpg -mc event list myminio/images -arn:minio:sqs::1:elasticsearch s3:ObjectCreated:*,s3:ObjectRemoved:* Filter: suffix=”.jpg” -``` - -### Step 4: Test on Elasticsearch - -Upload a JPEG image into `images` bucket. - -``` -mc cp myphoto.jpg myminio/images -``` - -Use curl to view contents of `minio_events` index. - -``` -$ curl "http://localhost:9200/minio_events/_search?pretty=true" -{ - "took" : 40, - "timed_out" : false, - "_shards" : { - "total" : 5, - "successful" : 5, - "failed" : 0 - }, - "hits" : { - "total" : 1, - "max_score" : 1.0, - "hits" : [ - { - "_index" : "minio_events", - "_type" : "event", - "_id" : "images/myphoto.jpg", - "_score" : 1.0, - "_source" : { - "Records" : [ - { - "eventVersion" : "2.0", - "eventSource" : "minio:s3", - "awsRegion" : "", - "eventTime" : "2017-03-30T08:00:41Z", - "eventName" : "s3:ObjectCreated:Put", - "userIdentity" : { - "principalId" : "minio" - }, - "requestParameters" : { - "sourceIPAddress" : "127.0.0.1:38062" - }, - "responseElements" : { - "x-amz-request-id" : "14B09A09703FC47B", - "x-minio-origin-endpoint" : "http://192.168.86.115:9000" - }, - "s3" : { - "s3SchemaVersion" : "1.0", - "configurationId" : "Config", - "bucket" : { - "name" : "images", - "ownerIdentity" : { - "principalId" : "minio" - }, - "arn" : "arn:aws:s3:::images" - }, - "object" : { - "key" : "myphoto.jpg", - "size" : 6474, - "eTag" : "a3410f4f8788b510d6f19c5067e60a90", - "sequencer" : "14B09A09703FC47B" - } - }, - "source" : { - "host" : "127.0.0.1", - "port" : "38062", - "userAgent" : "MinIO (linux; amd64) minio-go/2.0.3 mc/2017-02-15T17:57:25Z" - } - } - ] - } - } - ] - } -} -``` - -This output shows that a document has been created for the event in Elasticsearch. - -Here we see that the document ID is the bucket and object name. In case `access` format was used, the document ID would be automatically generated by Elasticsearch. - - - -## Publish MinIO events via Redis - -Install [Redis](http://redis.io/download) server. For illustrative purposes, we have set the database password as "yoursecret". - -This notification target supports two formats: _namespace_ and _access_. - -When the _namespace_ format is used, MinIO synchronizes objects in the bucket with entries in a hash. For each entry, the key is formatted as "bucketName/objectName" for an object that exists in the bucket, and the value is the JSON-encoded event data about the operation that created/replaced the object in MinIO. When objects are updated or deleted, the corresponding entry in the hash is also updated or deleted. - -When the _access_ format is used, MinIO appends events to a list using [RPUSH](https://redis.io/commands/rpush). Each item in the list is a JSON encoded list with two items, where the first item is a timestamp string, and the second item is a JSON object containing event data about the operation that happened in the bucket. No entries appended to the list are updated or deleted by MinIO in this format. - -The steps below show how to use this notification target in `namespace` and `access` format. - -### Step 1: Add Redis endpoint to MinIO - -The MinIO server configuration file is stored on the backend in json format.The Redis configuration is located in the `redis` key under the `notify` top-level key. Create a configuration key-value pair here for your Redis instance. The key is a name for your Redis endpoint, and the value is a collection of key-value parameters described in the table below. - -``` -KEY: -notify_redis[:name] publish bucket notifications to Redis datastores - -ARGS: -address* (address) Redis server's address. For example: `localhost:6379` -key* (string) Redis key to store/update events, key is auto-created -format* (namespace*|access) 'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace' -password (string) Redis server password -queue_dir (path) staging dir for undelivered messages e.g. '/home/events' -queue_limit (number) maximum limit for undelivered messages, defaults to '100000' -comment (sentence) optionally add a comment to this setting -``` - -or environment variables - -``` -KEY: -notify_redis[:name] publish bucket notifications to Redis datastores - -ARGS: -MINIO_NOTIFY_REDIS_ENABLE* (on|off) enable notify_redis target, default is 'off' -MINIO_NOTIFY_REDIS_KEY* (string) Redis key to store/update events, key is auto-created -MINIO_NOTIFY_REDIS_FORMAT* (namespace*|access) 'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace' -MINIO_NOTIFY_REDIS_PASSWORD (string) Redis server password -MINIO_NOTIFY_REDIS_QUEUE_DIR (path) staging dir for undelivered messages e.g. '/home/events' -MINIO_NOTIFY_REDIS_QUEUE_LIMIT (number) maximum limit for undelivered messages, defaults to '100000' -MINIO_NOTIFY_REDIS_COMMENT (sentence) optionally add a comment to this setting -``` - -MinIO supports persistent event store. The persistent store will backup events when the Redis broker goes offline and replays it when the broker comes back online. The event store can be configured by setting the directory path in `queue_dir` field and the maximum limit of events in the queue_dir in `queue_limit` field. For eg, the `queue_dir` can be `/home/events` and `queue_limit` can be `1000`. By default, the `queue_limit` is set to 100000. - -To update the configuration, use `mc admin config get` command to get the current configuration. - -```sh -$ mc admin config get myminio/ notify_redis -notify_redis:1 address="" format="namespace" key="" password="" queue_dir="" queue_limit="0" -``` - -Use `mc admin config set` command to update the configuration for the deployment.Restart the MinIO server to put the changes into effect. The server will print a line like `SQS ARNs: arn:minio:sqs::1:redis` at start-up if there were no errors. - -```sh -$ mc admin config set myminio/ notify_redis:1 address="127.0.0.1:6379" format="namespace" key="bucketevents" password="yoursecret" queue_dir="" queue_limit="0" -``` - -Note that, you can add as many Redis server endpoint configurations as needed by providing an identifier (like "1" in the example above) for the Redis instance and an object of per-server configuration parameters. - -### Step 2: Enable bucket notification using MinIO client - -We will now enable bucket event notifications on a bucket named `images`. Whenever a JPEG image is created/overwritten, a new key is added or an existing key is updated in the Redis hash configured above. When an existing object is deleted, the corresponding key is deleted from the Redis hash. Thus, the rows in the Redis hash, reflect the `.jpg` objects in the `images` bucket. - -To configure this bucket notification, we need the ARN printed by MinIO in the previous step. Additional information about ARN is available [here](http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html). - -With the `mc` tool, the configuration is very simple to add. Let us say that the MinIO server is aliased as `myminio` in our mc configuration. Execute the following: - -``` -mc mb myminio/images -mc event add myminio/images arn:minio:sqs::1:redis --suffix .jpg -mc event list myminio/images -arn:minio:sqs::1:redis s3:ObjectCreated:*,s3:ObjectRemoved:* Filter: suffix=”.jpg” -``` - -### Step 3: Test on Redis - -Start the `redis-cli` Redis client program to inspect the contents in Redis. Run the `monitor` Redis command. This prints each operation performed on Redis as it occurs. - -``` -redis-cli -a yoursecret -127.0.0.1:6379> monitor -OK -``` - -Open another terminal and upload a JPEG image into `images` bucket. - -``` -mc cp myphoto.jpg myminio/images -``` - -In the previous terminal, you will now see the operation that MinIO performs on Redis: - -``` -127.0.0.1:6379> monitor -OK -1490686879.650649 [0 172.17.0.1:44710] "PING" -1490686879.651061 [0 172.17.0.1:44710] "HSET" "minio_events" "images/myphoto.jpg" "{\"Records\":[{\"eventVersion\":\"2.0\",\"eventSource\":\"minio:s3\",\"awsRegion\":\"\",\"eventTime\":\"2017-03-28T07:41:19Z\",\"eventName\":\"s3:ObjectCreated:Put\",\"userIdentity\":{\"principalId\":\"minio\"},\"requestParameters\":{\"sourceIPAddress\":\"127.0.0.1:52234\"},\"responseElements\":{\"x-amz-request-id\":\"14AFFBD1ACE5F632\",\"x-minio-origin-endpoint\":\"http://192.168.86.115:9000\"},\"s3\":{\"s3SchemaVersion\":\"1.0\",\"configurationId\":\"Config\",\"bucket\":{\"name\":\"images\",\"ownerIdentity\":{\"principalId\":\"minio\"},\"arn\":\"arn:aws:s3:::images\"},\"object\":{\"key\":\"myphoto.jpg\",\"size\":2586,\"eTag\":\"5d284463f9da279f060f0ea4d11af098\",\"sequencer\":\"14AFFBD1ACE5F632\"}},\"source\":{\"host\":\"127.0.0.1\",\"port\":\"52234\",\"userAgent\":\"MinIO (linux; amd64) minio-go/2.0.3 mc/2017-02-15T17:57:25Z\"}}]}" -``` - -Here we see that MinIO performed `HSET` on `minio_events` key. - -In case, `access` format was used, then `minio_events` would be a list, and the MinIO server would have performed an `RPUSH` to append to the list. A consumer of this list would ideally use `BLPOP` to remove list items from the left-end of the list. - - - -## Publish MinIO events via NATS - -Install NATS from [here](http://nats.io/). - -### Step 1: Add NATS endpoint to MinIO - -MinIO supports persistent event store. The persistent store will backup events when the NATS broker goes offline and replays it when the broker comes back online. The event store can be configured by setting the directory path in `queue_dir` field and the maximum limit of events in the queue_dir in `queue_limit` field. For eg, the `queue_dir` can be `/home/events` and `queue_limit` can be `1000`. By default, the `queue_limit` is set to 100000. - -``` -KEY: -notify_nats[:name] publish bucket notifications to NATS endpoints - -ARGS: -address* (address) NATS server address e.g. '0.0.0.0:4222' -subject* (string) NATS subscription subject -username (string) NATS username -password (string) NATS password -token (string) NATS token -tls (on|off) set to 'on' to enable TLS -tls_skip_verify (on|off) trust server TLS without verification, defaults to "on" (verify) -ping_interval (duration) client ping commands interval in s,m,h,d. Disabled by default -streaming (on|off) set to 'on', to use streaming NATS server -streaming_async (on|off) set to 'on', to enable asynchronous publish -streaming_max_pub_acks_in_flight (number) number of messages to publish without waiting for ACKs -streaming_cluster_id (string) unique ID for NATS streaming cluster -cert_authority (string) path to certificate chain of the target NATS server -client_cert (string) client cert for NATS mTLS auth -client_key (string) client cert key for NATS mTLS auth -queue_dir (path) staging dir for undelivered messages e.g. '/home/events' -queue_limit (number) maximum limit for undelivered messages, defaults to '100000' -comment (sentence) optionally add a comment to this setting -``` - -or environment variables -``` -KEY: -notify_nats[:name] publish bucket notifications to NATS endpoints - -ARGS: -MINIO_NOTIFY_NATS_ENABLE* (on|off) enable notify_nats target, default is 'off' -MINIO_NOTIFY_NATS_ADDRESS* (address) NATS server address e.g. '0.0.0.0:4222' -MINIO_NOTIFY_NATS_SUBJECT* (string) NATS subscription subject -MINIO_NOTIFY_NATS_USERNAME (string) NATS username -MINIO_NOTIFY_NATS_PASSWORD (string) NATS password -MINIO_NOTIFY_NATS_TOKEN (string) NATS token -MINIO_NOTIFY_NATS_TLS (on|off) set to 'on' to enable TLS -MINIO_NOTIFY_NATS_TLS_SKIP_VERIFY (on|off) trust server TLS without verification, defaults to "on" (verify) -MINIO_NOTIFY_NATS_PING_INTERVAL (duration) client ping commands interval in s,m,h,d. Disabled by default -MINIO_NOTIFY_NATS_STREAMING (on|off) set to 'on', to use streaming NATS server -MINIO_NOTIFY_NATS_STREAMING_ASYNC (on|off) set to 'on', to enable asynchronous publish -MINIO_NOTIFY_NATS_STREAMING_MAX_PUB_ACKS_IN_FLIGHT (number) number of messages to publish without waiting for ACKs -MINIO_NOTIFY_NATS_STREAMING_CLUSTER_ID (string) unique ID for NATS streaming cluster -MINIO_NOTIFY_NATS_CERT_AUTHORITY (string) path to certificate chain of the target NATS server -MINIO_NOTIFY_NATS_CLIENT_CERT (string) client cert for NATS mTLS auth -MINIO_NOTIFY_NATS_CLIENT_KEY (string) client cert key for NATS mTLS auth -MINIO_NOTIFY_NATS_QUEUE_DIR (path) staging dir for undelivered messages e.g. '/home/events' -MINIO_NOTIFY_NATS_QUEUE_LIMIT (number) maximum limit for undelivered messages, defaults to '100000' -MINIO_NOTIFY_NATS_COMMENT (sentence) optionally add a comment to this setting -``` - -To update the configuration, use `mc admin config get` command to get the current configuration file for the minio deployment. - -```sh -$ mc admin config get myminio/ notify_nats -notify_nats:1 password="yoursecret" streaming_max_pub_acks_in_flight="10" subject="" address="0.0.0.0:4222" token="" username="yourusername" ping_interval="0" queue_limit="0" tls="off" tls_skip_verify="off" streaming_async="on" queue_dir="" streaming_cluster_id="test-cluster" streaming_enable="on" -``` - -Use `mc admin config set` command to update the configuration for the deployment.Restart MinIO server to reflect config changes. `bucketevents` is the subject used by NATS in this example. - -```sh -$ mc admin config set myminio notify_nats:1 password="yoursecret" streaming_max_pub_acks_in_flight="10" subject="" address="0.0.0.0:4222" token="" username="yourusername" ping_interval="0" queue_limit="0" tls="off" streaming_async="on" queue_dir="" streaming_cluster_id="test-cluster" streaming_enable="on" -``` - -MinIO server also supports [NATS Streaming mode](http://nats.io/documentation/streaming/nats-streaming-intro/) that offers additional functionality like `At-least-once-delivery`, and `Publisher rate limiting`. To configure MinIO server to send notifications to NATS Streaming server, update the MinIO server configuration file as follows: - -Read more about sections `cluster_id`, `client_id` on [NATS documentation](https://github.com/nats-io/nats-streaming-server/blob/master/README.md). Section `maxPubAcksInflight` is explained [here](https://github.com/nats-io/stan.go#publisher-rate-limiting). - -### Step 2: Enable bucket notification using MinIO client - -We will enable bucket event notification to trigger whenever a JPEG image is uploaded or deleted from `images` bucket on `myminio` server. Here ARN value is `arn:minio:sqs::1:nats`. To understand more about ARN please follow [AWS ARN](http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html) documentation. - -``` -mc mb myminio/images -mc event add myminio/images arn:minio:sqs::1:nats --suffix .jpg -mc event list myminio/images -arn:minio:sqs::1:nats s3:ObjectCreated:*,s3:ObjectRemoved:* Filter: suffix=”.jpg” -``` - -### Step 3: Test on NATS - -If you use NATS server, check out this sample program below to log the bucket notification added to NATS. - -```go -package main - -// Import Go and NATS packages -import ( - "log" - "runtime" - - "github.com/nats-io/nats.go" -) - -func main() { - - // Create server connection - natsConnection, _ := nats.Connect("nats://yourusername:yoursecret@localhost:4222") - log.Println("Connected") - - // Subscribe to subject - log.Printf("Subscribing to subject 'bucketevents'\n") - natsConnection.Subscribe("bucketevents", func(msg *nats.Msg) { - - // Handle the message - log.Printf("Received message '%s\n", string(msg.Data)+"'") - }) - - // Keep the connection alive - runtime.Goexit() -} -``` - -``` -go run nats.go -2016/10/12 06:39:18 Connected -2016/10/12 06:39:18 Subscribing to subject 'bucketevents' -``` - -Open another terminal and upload a JPEG image into `images` bucket. - -``` -mc cp myphoto.jpg myminio/images -``` - -The example `nats.go` program prints event notification to console. - -``` -go run nats.go -2016/10/12 06:51:26 Connected -2016/10/12 06:51:26 Subscribing to subject 'bucketevents' -2016/10/12 06:51:33 Received message '{"EventType":"s3:ObjectCreated:Put","Key":"images/myphoto.jpg","Records":[{"eventVersion":"2.0","eventSource":"aws:s3","awsRegion":"","eventTime":"2016-10-12T13:51:33Z","eventName":"s3:ObjectCreated:Put","userIdentity":{"principalId":"minio"},"requestParameters":{"sourceIPAddress":"[::1]:57106"},"responseElements":{},"s3":{"s3SchemaVersion":"1.0","configurationId":"Config","bucket":{"name":"images","ownerIdentity":{"principalId":"minio"},"arn":"arn:aws:s3:::images"},"object":{"key":"myphoto.jpg","size":56060,"eTag":"1d97bf45ecb37f7a7b699418070df08f","sequencer":"147CCD1AE054BFD0"}}}],"level":"info","msg":"","time":"2016-10-12T06:51:33-07:00"} -``` - -If you use NATS Streaming server, check out this sample program below to log the bucket notification added to NATS. - -```go -package main - -// Import Go and NATS packages -import ( - "fmt" - "runtime" - - "github.com/nats-io/stan.go" -) - -func main() { - - var stanConnection stan.Conn - - subscribe := func() { - fmt.Printf("Subscribing to subject 'bucketevents'\n") - stanConnection.Subscribe("bucketevents", func(m *stan.Msg) { - - // Handle the message - fmt.Printf("Received a message: %s\n", string(m.Data)) - }) - } - - - stanConnection, _ = stan.Connect("test-cluster", "test-client", stan.NatsURL("nats://yourusername:yoursecret@0.0.0.0:4222"), stan.SetConnectionLostHandler(func(c stan.Conn, _ error) { - go func() { - for { - // Reconnect if the connection is lost. - if stanConnection == nil || stanConnection.NatsConn() == nil || !stanConnection.NatsConn().IsConnected() { - stanConnection, _ = stan.Connect("test-cluster", "test-client", stan.NatsURL("nats://yourusername:yoursecret@0.0.0.0:4222"), stan.SetConnectionLostHandler(func(c stan.Conn, _ error) { - if c.NatsConn() != nil { - c.NatsConn().Close() - } - _ = c.Close() - })) - if stanConnection != nil { - subscribe() - } - - } - } - - }() - })) - - // Subscribe to subject - subscribe() - - // Keep the connection alive - runtime.Goexit() -} - -``` - -``` -go run nats.go -2017/07/07 11:47:40 Connected -2017/07/07 11:47:40 Subscribing to subject 'bucketevents' -``` - -Open another terminal and upload a JPEG image into `images` bucket. - -``` -mc cp myphoto.jpg myminio/images -``` - -The example `nats.go` program prints event notification to console. - -``` -Received a message: {"EventType":"s3:ObjectCreated:Put","Key":"images/myphoto.jpg","Records":[{"eventVersion":"2.0","eventSource":"minio:s3","awsRegion":"","eventTime":"2017-07-07T18:46:37Z","eventName":"s3:ObjectCreated:Put","userIdentity":{"principalId":"minio"},"requestParameters":{"sourceIPAddress":"192.168.1.80:55328"},"responseElements":{"x-amz-request-id":"14CF20BD1EFD5B93","x-minio-origin-endpoint":"http://127.0.0.1:9000"},"s3":{"s3SchemaVersion":"1.0","configurationId":"Config","bucket":{"name":"images","ownerIdentity":{"principalId":"minio"},"arn":"arn:aws:s3:::images"},"object":{"key":"myphoto.jpg","size":248682,"eTag":"f1671feacb8bbf7b0397c6e9364e8c92","contentType":"image/jpeg","userDefined":{"content-type":"image/jpeg"},"versionId":"1","sequencer":"14CF20BD1EFD5B93"}},"source":{"host":"192.168.1.80","port":"55328","userAgent":"MinIO (linux; amd64) minio-go/2.0.4 mc/DEVELOPMENT.GOGET"}}],"level":"info","msg":"","time":"2017-07-07T11:46:37-07:00"} -``` - - - -## Publish MinIO events via PostgreSQL - -> NOTE: Until release RELEASE.2020-04-10T03-34-42Z PostgreSQL notification used to support following options: -> -> ``` -> host (hostname) Postgres server hostname (used only if `connection_string` is empty) -> port (port) Postgres server port, defaults to `5432` (used only if `connection_string` is empty) -> username (string) database username (used only if `connection_string` is empty) -> password (string) database password (used only if `connection_string` is empty) -> database (string) database name (used only if `connection_string` is empty) -> ``` -> -> These are now deprecated, if you plan to upgrade to any releases after *RELEASE.2020-04-10T03-34-42Z* make sure -> to migrate to only using *connection_string* option. To migrate, once you have upgraded all the servers use the -> following command to update the existing notification targets. -> -> ``` -> mc admin config set myminio/ notify_postgres[:name] connection_string="host=hostname port=2832 username=psqluser password=psqlpass database=bucketevents" -> ``` -> -> Please make sure this step is carried out, without this step PostgreSQL notification targets will not work, -> an error message will be shown on the console upon server upgrade/restart, make sure to follow the above -> instructions appropriately. For further questions please join our https://slack.min.io - -Install [PostgreSQL](https://www.postgresql.org/) database server. For illustrative purposes, we have set the "postgres" user password as `password` and created a database called `minio_events` to store the events. - -This notification target supports two formats: _namespace_ and _access_. - -When the _namespace_ format is used, MinIO synchronizes objects in the bucket with rows in the table. It creates rows with two columns: key and value. The key is the bucket and object name of an object that exists in MinIO. The value is JSON encoded event data about the operation that created/replaced the object in MinIO. When objects are updated or deleted, the corresponding row from this table is updated or deleted respectively. - -When the _access_ format is used, MinIO appends events to a table. It creates rows with two columns: event_time and event_data. The event_time is the time at which the event occurred in the MinIO server. The event_data is the JSON encoded event data about the operation on an object. No rows are deleted or modified in this format. - -The steps below show how to use this notification target in `namespace` format. The other format is very similar and is omitted for brevity. - -### Step 1: Ensure minimum requirements are met - -MinIO requires PostgreSQL version 9.5 or above. MinIO uses the [`INSERT ON CONFLICT`](https://www.postgresql.org/docs/9.5/static/sql-insert.html#SQL-ON-CONFLICT) (aka UPSERT) feature, introduced in version 9.5 and the [JSONB](https://www.postgresql.org/docs/9.4/static/datatype-json.html) data-type introduced in version 9.4. - -### Step 2: Add PostgreSQL endpoint to MinIO - -The PostgreSQL configuration is located in the `notify_postgresql` key. Create a configuration key-value pair here for your PostgreSQL instance. The key is a name for your PostgreSQL endpoint, and the value is a collection of key-value parameters described in the table below. - -``` -KEY: -notify_postgres[:name] publish bucket notifications to Postgres databases - -ARGS: -connection_string* (string) Postgres server connection-string e.g. "host=localhost port=5432 dbname=minio_events user=postgres password=password sslmode=disable" -table* (string) DB table name to store/update events, table is auto-created -format* (namespace*|access) 'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace' -queue_dir (path) staging dir for undelivered messages e.g. '/home/events' -queue_limit (number) maximum limit for undelivered messages, defaults to '100000' -comment (sentence) optionally add a comment to this setting -``` - -or environment variables -``` -KEY: -notify_postgres[:name] publish bucket notifications to Postgres databases - -ARGS: -MINIO_NOTIFY_POSTGRES_ENABLE* (on|off) enable notify_postgres target, default is 'off' -MINIO_NOTIFY_POSTGRES_CONNECTION_STRING* (string) Postgres server connection-string e.g. "host=localhost port=5432 dbname=minio_events user=postgres password=password sslmode=disable" -MINIO_NOTIFY_POSTGRES_TABLE* (string) DB table name to store/update events, table is auto-created -MINIO_NOTIFY_POSTGRES_FORMAT* (namespace*|access) 'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace' -MINIO_NOTIFY_POSTGRES_QUEUE_DIR (path) staging dir for undelivered messages e.g. '/home/events' -MINIO_NOTIFY_POSTGRES_QUEUE_LIMIT (number) maximum limit for undelivered messages, defaults to '100000' -MINIO_NOTIFY_POSTGRES_COMMENT (sentence) optionally add a comment to this setting -``` - -MinIO supports persistent event store. The persistent store will backup events when the PostgreSQL connection goes offline and replays it when the broker comes back online. The event store can be configured by setting the directory path in `queue_dir` field and the maximum limit of events in the queue_dir in `queue_limit` field. For eg, the `queue_dir` can be `/home/events` and `queue_limit` can be `1000`. By default, the `queue_limit` is set to 100000. - -Note that for illustration here, we have disabled SSL. In the interest of security, for production this is not recommended. -To update the configuration, use `mc admin config get` command to get the current configuration. - -```sh -$ mc admin config get myminio notify_postgres -notify_postgres:1 queue_dir="" connection_string="" queue_limit="0" table="" format="namespace" -``` - -Use `mc admin config set` command to update the configuration for the deployment. Restart the MinIO server to put the changes into effect. The server will print a line like `SQS ARNs: arn:minio:sqs::1:postgresql` at start-up if there were no errors. - -```sh -$ mc admin config set myminio notify_postgres:1 connection_string="host=localhost port=5432 dbname=minio_events user=postgres password=password sslmode=disable" table="bucketevents" format="namespace" -``` - -Note that, you can add as many PostgreSQL server endpoint configurations as needed by providing an identifier (like "1" in the example above) for the PostgreSQL instance and an object of per-server configuration parameters. - -### Step 3: Enable bucket notification using MinIO client - -We will now enable bucket event notifications on a bucket named `images`. Whenever a JPEG image is created/overwritten, a new row is added or an existing row is updated in the PostgreSQL configured above. When an existing object is deleted, the corresponding row is deleted from the PostgreSQL table. Thus, the rows in the PostgreSQL table, reflect the `.jpg` objects in the `images` bucket. - -To configure this bucket notification, we need the ARN printed by MinIO in the previous step. Additional information about ARN is available [here](http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html). - -With the `mc` tool, the configuration is very simple to add. Let us say that the MinIO server is aliased as `myminio` in our mc configuration. Execute the following: - -``` -# Create bucket named `images` in myminio -mc mb myminio/images -# Add notification configuration on the `images` bucket using the MySQL ARN. The --suffix argument filters events. -mc event add myminio/images arn:minio:sqs::1:postgresql --suffix .jpg -# Print out the notification configuration on the `images` bucket. -mc event list myminio/images -mc event list myminio/images -arn:minio:sqs::1:postgresql s3:ObjectCreated:*,s3:ObjectRemoved:* Filter: suffix=”.jpg” -``` - -### Step 4: Test on PostgreSQL - -Open another terminal and upload a JPEG image into `images` bucket. - -``` -mc cp myphoto.jpg myminio/images -``` - -Open PostgreSQL terminal to list the rows in the `bucketevents` table. - -``` -$ psql -h 127.0.0.1 -U postgres -d minio_events -minio_events=# select * from bucketevents; - -key | value ---------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - images/myphoto.jpg | {"Records": [{"s3": {"bucket": {"arn": "arn:aws:s3:::images", "name": "images", "ownerIdentity": {"principalId": "minio"}}, "object": {"key": "myphoto.jpg", "eTag": "1d97bf45ecb37f7a7b699418070df08f", "size": 56060, "sequencer": "147CE57C70B31931"}, "configurationId": "Config", "s3SchemaVersion": "1.0"}, "awsRegion": "", "eventName": "s3:ObjectCreated:Put", "eventTime": "2016-10-12T21:18:20Z", "eventSource": "aws:s3", "eventVersion": "2.0", "userIdentity": {"principalId": "minio"}, "responseElements": {}, "requestParameters": {"sourceIPAddress": "[::1]:39706"}}]} -(1 row) -``` - - - -## Publish MinIO events via MySQL - -> NOTE: Until release RELEASE.2020-04-10T03-34-42Z MySQL notification used to support following options: -> -> ``` -> host (hostname) MySQL server hostname (used only if `dsn_string` is empty) -> port (port) MySQL server port (used only if `dsn_string` is empty) -> username (string) database username (used only if `dsn_string` is empty) -> password (string) database password (used only if `dsn_string` is empty) -> database (string) database name (used only if `dsn_string` is empty) -> ``` -> -> These are now deprecated, if you plan to upgrade to any releases after *RELEASE.2020-04-10T03-34-42Z* make sure -> to migrate to only using *dsn_string* option. To migrate, once you have upgraded all the servers use the -> following command to update the existing notification targets. -> -> ``` -> mc admin config set myminio/ notify_mysql[:name] dsn_string="mysqluser:mysqlpass@tcp(localhost:2832)/bucketevents" -> ``` -> -> Please make sure this step is carried out, without this step MySQL notification targets will not work, -> an error message will be shown on the console upon server upgrade/restart, make sure to follow the above -> instructions appropriately. For further questions please join our https://slack.min.io - -Install MySQL from [here](https://dev.mysql.com/downloads/mysql/). For illustrative purposes, we have set the root password as `password` and created a database called `miniodb` to store the events. - -This notification target supports two formats: _namespace_ and _access_. - -When the _namespace_ format is used, MinIO synchronizes objects in the bucket with rows in the table. It creates rows with two columns: key_name and value. The key_name is the bucket and object name of an object that exists in MinIO. The value is JSON encoded event data about the operation that created/replaced the object in MinIO. When objects are updated or deleted, the corresponding row from this table is updated or deleted respectively. - -When the _access_ format is used, MinIO appends events to a table. It creates rows with two columns: event_time and event_data. The event_time is the time at which the event occurred in the MinIO server. The event_data is the JSON encoded event data about the operation on an object. No rows are deleted or modified in this format. - -The steps below show how to use this notification target in `namespace` format. The other format is very similar and is omitted for brevity. - -### Step 1: Ensure minimum requirements are met - -MinIO requires MySQL version 5.7.8 or above. MinIO uses the [JSON](https://dev.mysql.com/doc/refman/5.7/en/json.html) data-type introduced in version 5.7.8. We tested this setup on MySQL 5.7.17. - -### Step 2: Add MySQL server endpoint configuration to MinIO - -The MySQL configuration is located in the `notify_mysql` key. Create a configuration key-value pair here for your MySQL instance. The key is a name for your MySQL endpoint, and the value is a collection of key-value parameters described in the table below. - -``` -KEY: -notify_mysql[:name] publish bucket notifications to MySQL databases. When multiple MySQL server endpoints are needed, a user specified "name" can be added for each configuration, (e.g."notify_mysql:myinstance"). - -ARGS: -dsn_string* (string) MySQL data-source-name connection string e.g. ":@tcp(:)/" -table* (string) DB table name to store/update events, table is auto-created -format* (namespace*|access) 'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace' -queue_dir (path) staging dir for undelivered messages e.g. '/home/events' -queue_limit (number) maximum limit for undelivered messages, defaults to '100000' -comment (sentence) optionally add a comment to this setting -``` - -or environment variables -``` -KEY: -notify_mysql[:name] publish bucket notifications to MySQL databases - -ARGS: -MINIO_NOTIFY_MYSQL_ENABLE* (on|off) enable notify_mysql target, default is 'off' -MINIO_NOTIFY_MYSQL_DSN_STRING* (string) MySQL data-source-name connection string e.g. ":@tcp(:)/" -MINIO_NOTIFY_MYSQL_TABLE* (string) DB table name to store/update events, table is auto-created -MINIO_NOTIFY_MYSQL_FORMAT* (namespace*|access) 'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace' -MINIO_NOTIFY_MYSQL_QUEUE_DIR (path) staging dir for undelivered messages e.g. '/home/events' -MINIO_NOTIFY_MYSQL_QUEUE_LIMIT (number) maximum limit for undelivered messages, defaults to '100000' -MINIO_NOTIFY_MYSQL_COMMENT (sentence) optionally add a comment to this setting -``` - -`dsn_string` is required and is of form `":@tcp(:)/"` - -MinIO supports persistent event store. The persistent store will backup events if MySQL connection goes offline and then replays the stored events when the broken connection comes back up. The event store can be configured by setting a directory path in `queue_dir` field, and the maximum number of events, which can be stored in a `queue_dir`, in `queue_limit` field. For example, `queue_dir` can be set to `/home/events` and `queue_limit` can be set to `1000`. By default, the `queue_limit` is set to `100000`. - -Before updating the configuration, let's start with `mc admin config get` command to get the current configuration. - -```sh -$ mc admin config get myminio/ notify_mysql -notify_mysql:myinstance enable=off format=namespace host= port= username= password= database= dsn_string= table= queue_dir= queue_limit=0 -``` - -Use `mc admin config set` command to update MySQL notification configuration for the deployment with `dsn_string` parameter: - -```sh -$ mc admin config set myminio notify_mysql:myinstance table="minio_images" dsn_string="root:xxxx@tcp(172.17.0.1:3306)/miniodb" -``` - -Note that, you can add as many MySQL server endpoint configurations as needed by providing an identifier (like "myinstance" in the example above) for each MySQL instance desired. - -Restart the MinIO server to put the changes into effect. The server will print a line like `SQS ARNs: arn:minio:sqs::myinstance:mysql` at start-up, if there are no errors. - -### Step 3: Enable bucket notification using MinIO client - -We will now setup bucket notifications on a bucket named `images`. Whenever a JPEG image object is created/overwritten, a new row is added or an existing row is updated in the MySQL table configured above. When an existing object is deleted, the corresponding row is deleted from the MySQL table. Thus, the rows in the MySQL table, reflect the `.jpg` objects in the `images` bucket. - -To configure this bucket notification, we need the ARN printed by MinIO in the previous step. Additional information about ARN is available [here](http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html). - -With the `mc` tool, the configuration is very simple to add. Let us say that the MinIO server is aliased as `myminio` in our mc configuration. Execute the following: - -``` -# Create bucket named `images` in myminio -mc mb myminio/images -# Add notification configuration on the `images` bucket using the MySQL ARN. The --suffix argument filters events. -mc event add myminio/images arn:minio:sqs::myinstance:mysql --suffix .jpg -# Print out the notification configuration on the `images` bucket. -mc event list myminio/images -arn:minio:sqs::myinstance:mysql s3:ObjectCreated:*,s3:ObjectRemoved:*,s3:ObjectAccessed:* Filter: suffix=”.jpg” -``` - -### Step 4: Test on MySQL - -Open another terminal and upload a JPEG image into `images` bucket: - -``` -mc cp myphoto.jpg myminio/images -``` - -Open MySQL terminal and list the rows in the `minio_images` table. - -``` -$ mysql -h 172.17.0.1 -P 3306 -u root -p miniodb -mysql> select * from minio_images; -+--------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| key_name | value | -+--------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| images/myphoto.jpg | {"Records": [{"s3": {"bucket": {"arn": "arn:aws:s3:::images", "name": "images", "ownerIdentity": {"principalId": "minio"}}, "object": {"key": "myphoto.jpg", "eTag": "467886be95c8ecfd71a2900e3f461b4f", "size": 26, "sequencer": "14AC59476F809FD3"}, "configurationId": "Config", "s3SchemaVersion": "1.0"}, "awsRegion": "", "eventName": "s3:ObjectCreated:Put", "eventTime": "2017-03-16T11:29:00Z", "eventSource": "aws:s3", "eventVersion": "2.0", "userIdentity": {"principalId": "minio"}, "responseElements": {"x-amz-request-id": "14AC59476F809FD3", "x-minio-origin-endpoint": "http://192.168.86.110:9000"}, "requestParameters": {"sourceIPAddress": "127.0.0.1:38260"}}]} | -+--------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -1 row in set (0.01 sec) - -``` - - - -## Publish MinIO events via Kafka - -Install Apache Kafka from [here](http://kafka.apache.org/). - -### Step 1: Ensure minimum requirements are met - -MinIO requires Kafka version 0.10 or 0.9. Internally MinIO uses the [Shopify/sarama](https://github.com/Shopify/sarama/) library and so has the same version compatibility as provided by this library. - -### Step 2: Add Kafka endpoint to MinIO - -MinIO supports persistent event store. The persistent store will backup events when the kafka broker goes offline and replays it when the broker comes back online. The event store can be configured by setting the directory path in `queue_dir` field and the maximum limit of events in the queue_dir in `queue_limit` field. For eg, the `queue_dir` can be `/home/events` and `queue_limit` can be `1000`. By default, the `queue_limit` is set to 100000. - -``` -KEY: -notify_kafka[:name] publish bucket notifications to Kafka endpoints - -ARGS: -brokers* (csv) comma separated list of Kafka broker addresses -topic (string) Kafka topic used for bucket notifications -sasl_username (string) username for SASL/PLAIN or SASL/SCRAM authentication -sasl_password (string) password for SASL/PLAIN or SASL/SCRAM authentication -sasl_mechanism (string) sasl authentication mechanism, default 'PLAIN' -tls_client_auth (string) clientAuth determines the Kafka server's policy for TLS client auth -sasl (on|off) set to 'on' to enable SASL authentication -tls (on|off) set to 'on' to enable TLS -tls_skip_verify (on|off) trust server TLS without verification, defaults to "on" (verify) -client_tls_cert (path) path to client certificate for mTLS auth -client_tls_key (path) path to client key for mTLS auth -queue_dir (path) staging dir for undelivered messages e.g. '/home/events' -queue_limit (number) maximum limit for undelivered messages, defaults to '100000' -version (string) specify the version of the Kafka cluster e.g '2.2.0' -comment (sentence) optionally add a comment to this setting -``` - -or environment variables -``` -KEY: -notify_kafka[:name] publish bucket notifications to Kafka endpoints - -ARGS: -MINIO_NOTIFY_KAFKA_ENABLE* (on|off) enable notify_kafka target, default is 'off' -MINIO_NOTIFY_KAFKA_BROKERS* (csv) comma separated list of Kafka broker addresses -MINIO_NOTIFY_KAFKA_TOPIC (string) Kafka topic used for bucket notifications -MINIO_NOTIFY_KAFKA_SASL_USERNAME (string) username for SASL/PLAIN or SASL/SCRAM authentication -MINIO_NOTIFY_KAFKA_SASL_PASSWORD (string) password for SASL/PLAIN or SASL/SCRAM authentication -MINIO_NOTIFY_KAFKA_SASL_MECHANISM (plain*|sha256|sha512) sasl authentication mechanism, default 'plain' -MINIO_NOTIFY_KAFKA_TLS_CLIENT_AUTH (string) clientAuth determines the Kafka server's policy for TLS client auth -MINIO_NOTIFY_KAFKA_SASL (on|off) set to 'on' to enable SASL authentication -MINIO_NOTIFY_KAFKA_TLS (on|off) set to 'on' to enable TLS -MINIO_NOTIFY_KAFKA_TLS_SKIP_VERIFY (on|off) trust server TLS without verification, defaults to "on" (verify) -MINIO_NOTIFY_KAFKA_CLIENT_TLS_CERT (path) path to client certificate for mTLS auth -MINIO_NOTIFY_KAFKA_CLIENT_TLS_KEY (path) path to client key for mTLS auth -MINIO_NOTIFY_KAFKA_QUEUE_DIR (path) staging dir for undelivered messages e.g. '/home/events' -MINIO_NOTIFY_KAFKA_QUEUE_LIMIT (number) maximum limit for undelivered messages, defaults to '100000' -MINIO_NOTIFY_KAFKA_COMMENT (sentence) optionally add a comment to this setting -MINIO_NOTIFY_KAFKA_VERSION (string) specify the version of the Kafka cluster e.g. '2.2.0' -``` - -To update the configuration, use `mc admin config get` command to get the current configuration. - -```sh -$ mc admin config get myminio/ notify_kafka -notify_kafka:1 tls_skip_verify="off" queue_dir="" queue_limit="0" sasl="off" sasl_password="" sasl_username="" tls_client_auth="0" tls="off" brokers="" topic="" client_tls_cert="" client_tls_key="" version="" -``` - -Use `mc admin config set` command to update the configuration for the deployment. Restart the MinIO server to put the changes into effect. The server will print a line like `SQS ARNs: arn:minio:sqs::1:kafka` at start-up if there were no errors.`bucketevents` is the topic used by kafka in this example. - -```sh -$ mc admin config set myminio notify_kafka:1 tls_skip_verify="off" queue_dir="" queue_limit="0" sasl="off" sasl_password="" sasl_username="" tls_client_auth="0" tls="off" client_tls_cert="" client_tls_key="" brokers="localhost:9092,localhost:9093" topic="bucketevents" version="" -``` - -### Step 3: Enable bucket notification using MinIO client - -We will enable bucket event notification to trigger whenever a JPEG image is uploaded or deleted from `images` bucket on `myminio` server. Here ARN value is `arn:minio:sqs::1:kafka`. To understand more about ARN please follow [AWS ARN](http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html) documentation. - -``` -mc mb myminio/images -mc event add myminio/images arn:minio:sqs::1:kafka --suffix .jpg -mc event list myminio/images -arn:minio:sqs::1:kafka s3:ObjectCreated:*,s3:ObjectRemoved:* Filter: suffix=”.jpg” -``` - -### Step 4: Test on Kafka - -We used [kafkacat](https://github.com/edenhill/kafkacat) to print all notifications on the console. - -``` -kafkacat -C -b localhost:9092 -t bucketevents -``` - -Open another terminal and upload a JPEG image into `images` bucket. - -``` -mc cp myphoto.jpg myminio/images -``` - -`kafkacat` prints the event notification to the console. - -``` -kafkacat -b localhost:9092 -t bucketevents -{ - "EventName": "s3:ObjectCreated:Put", - "Key": "images/myphoto.jpg", - "Records": [ - { - "eventVersion": "2.0", - "eventSource": "minio:s3", - "awsRegion": "", - "eventTime": "2019-09-10T17:41:54Z", - "eventName": "s3:ObjectCreated:Put", - "userIdentity": { - "principalId": "AKIAIOSFODNN7EXAMPLE" - }, - "requestParameters": { - "accessKey": "AKIAIOSFODNN7EXAMPLE", - "region": "", - "sourceIPAddress": "192.168.56.192" - }, - "responseElements": { - "x-amz-request-id": "15C3249451E12784", - "x-minio-deployment-id": "751a8ba6-acb2-42f6-a297-4cdf1cf1fa4f", - "x-minio-origin-endpoint": "http://192.168.97.83:9000" - }, - "s3": { - "s3SchemaVersion": "1.0", - "configurationId": "Config", - "bucket": { - "name": "images", - "ownerIdentity": { - "principalId": "AKIAIOSFODNN7EXAMPLE" - }, - "arn": "arn:aws:s3:::images" - }, - "object": { - "key": "myphoto.jpg", - "size": 6474, - "eTag": "430f89010c77aa34fc8760696da62d08-1", - "contentType": "image/jpeg", - "userMetadata": { - "content-type": "image/jpeg" - }, - "versionId": "1", - "sequencer": "15C32494527B46C5" - } - }, - "source": { - "host": "192.168.56.192", - "port": "", - "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:69.0) Gecko/20100101 Firefox/69.0" - } - } - ] -} -``` - - - -## Publish MinIO events via Webhooks - -[Webhooks](https://en.wikipedia.org/wiki/Webhook) are a way to receive information when it happens, rather than continually polling for that data. - -### Step 1: Add Webhook endpoint to MinIO - -MinIO supports persistent event store. The persistent store will backup events when the webhook goes offline and replays it when the broker comes back online. The event store can be configured by setting the directory path in `queue_dir` field and the maximum limit of events in the queue_dir in `queue_limit` field. For eg, the `queue_dir` can be `/home/events` and `queue_limit` can be `1000`. By default, the `queue_limit` is set to 100000. - -``` -KEY: -notify_webhook[:name] publish bucket notifications to webhook endpoints - -ARGS: -endpoint* (url) webhook server endpoint e.g. http://localhost:8080/minio/events -auth_token (string) opaque string or JWT authorization token -queue_dir (path) staging dir for undelivered messages e.g. '/home/events' -queue_limit (number) maximum limit for undelivered messages, defaults to '100000' -client_cert (string) client cert for Webhook mTLS auth -client_key (string) client cert key for Webhook mTLS auth -comment (sentence) optionally add a comment to this setting -``` - -or environment variables -``` -KEY: -notify_webhook[:name] publish bucket notifications to webhook endpoints - -ARGS: -MINIO_NOTIFY_WEBHOOK_ENABLE* (on|off) enable notify_webhook target, default is 'off' -MINIO_NOTIFY_WEBHOOK_ENDPOINT* (url) webhook server endpoint e.g. http://localhost:8080/minio/events -MINIO_NOTIFY_WEBHOOK_AUTH_TOKEN (string) opaque string or JWT authorization token -MINIO_NOTIFY_WEBHOOK_QUEUE_DIR (path) staging dir for undelivered messages e.g. '/home/events' -MINIO_NOTIFY_WEBHOOK_QUEUE_LIMIT (number) maximum limit for undelivered messages, defaults to '100000' -MINIO_NOTIFY_WEBHOOK_COMMENT (sentence) optionally add a comment to this setting -MINIO_NOTIFY_WEBHOOK_CLIENT_CERT (string) client cert for Webhook mTLS auth -MINIO_NOTIFY_WEBHOOK_CLIENT_KEY (string) client cert key for Webhook mTLS auth -``` - -```sh -$ mc admin config get myminio/ notify_webhook -notify_webhook:1 endpoint="" auth_token="" queue_limit="0" queue_dir="" client_cert="" client_key="" -``` - -Use `mc admin config set` command to update the configuration for the deployment. Here the endpoint is the server listening for webhook notifications. Save the settings and restart the MinIO server for changes to take effect. Note that the endpoint needs to be live and reachable when you restart your MinIO server. - -```sh -$ mc admin config set myminio notify_webhook:1 queue_limit="0" endpoint="http://localhost:3000" queue_dir="" -``` - -### Step 2: Enable bucket notification using MinIO client - -We will enable bucket event notification to trigger whenever a JPEG image is uploaded to `images` bucket on `myminio` server. Here ARN value is `arn:minio:sqs::1:webhook`. To learn more about ARN please follow [AWS ARN](http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html) documentation. - -``` -mc mb myminio/images -mc mb myminio/images-thumbnail -mc event add myminio/images arn:minio:sqs::1:webhook --event put --suffix .jpg -``` - -Check if event notification is successfully configured by - -``` -mc event list myminio/images -``` - -You should get a response like this - -``` -arn:minio:sqs::1:webhook s3:ObjectCreated:* Filter: suffix=".jpg" -``` - -### Step 3: Test with Thumbnailer - -We used [Thumbnailer](https://github.com/minio/thumbnailer) to listen for MinIO notifications when a new JPEG file is uploaded (HTTP PUT). Triggered by a notification, Thumbnailer uploads a thumbnail of new image to MinIO server. To start with, download and install Thumbnailer. - -``` -git clone https://github.com/minio/thumbnailer/ -npm install -``` - -Then open the Thumbnailer config file at `config/webhook.json` and add the configuration for your MinIO server and then start Thumbnailer by - -``` -NODE_ENV=webhook node thumbnail-webhook.js -``` - -Thumbnailer starts running at `http://localhost:3000/`. Next, configure the MinIO server to send notifications to this URL (as mentioned in step 1) and use `mc` to set up bucket notifications (as mentioned in step 2). Then upload a JPEG image to MinIO server by - -``` -mc cp ~/images.jpg myminio/images -.../images.jpg: 8.31 KB / 8.31 KB ┃▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓┃ 100.00% 59.42 KB/s 0s -``` - -Wait a few moments, then check the bucket’s contents with mc ls — you will see a thumbnail appear. - -``` -mc ls myminio/images-thumbnail -[2017-02-08 11:39:40 IST] 992B images-thumbnail.jpg -``` - - - -## Publish MinIO events to NSQ - -Install an NSQ Daemon from [here](https://nsq.io/). Or use the following Docker -command for starting an nsq daemon: - -``` -docker run --rm -p 4150-4151:4150-4151 nsqio/nsq /nsqd -``` - -### Step 1: Add NSQ endpoint to MinIO - -MinIO supports persistent event store. The persistent store will backup events when the NSQ broker goes offline and replays it when the broker comes back online. The event store can be configured by setting the directory path in `queue_dir` field and the maximum limit of events in the queue_dir in `queue_limit` field. For eg, the `queue_dir` can be `/home/events` and `queue_limit` can be `1000`. By default, the `queue_limit` is set to 100000. - -To update the configuration, use `mc admin config get` command to get the current configuration for `notify_nsq`. - -``` -KEY: -notify_nsq[:name] publish bucket notifications to NSQ endpoints - -ARGS: -nsqd_address* (address) NSQ server address e.g. '127.0.0.1:4150' -topic* (string) NSQ topic -tls (on|off) set to 'on' to enable TLS -tls_skip_verify (on|off) trust server TLS without verification, defaults to "on" (verify) -queue_dir (path) staging dir for undelivered messages e.g. '/home/events' -queue_limit (number) maximum limit for undelivered messages, defaults to '100000' -comment (sentence) optionally add a comment to this setting -``` - -or environment variables -``` -KEY: -notify_nsq[:name] publish bucket notifications to NSQ endpoints - -ARGS: -MINIO_NOTIFY_NSQ_ENABLE* (on|off) enable notify_nsq target, default is 'off' -MINIO_NOTIFY_NSQ_NSQD_ADDRESS* (address) NSQ server address e.g. '127.0.0.1:4150' -MINIO_NOTIFY_NSQ_TOPIC* (string) NSQ topic -MINIO_NOTIFY_NSQ_TLS (on|off) set to 'on' to enable TLS -MINIO_NOTIFY_NSQ_TLS_SKIP_VERIFY (on|off) trust server TLS without verification, defaults to "on" (verify) -MINIO_NOTIFY_NSQ_QUEUE_DIR (path) staging dir for undelivered messages e.g. '/home/events' -MINIO_NOTIFY_NSQ_QUEUE_LIMIT (number) maximum limit for undelivered messages, defaults to '100000' -MINIO_NOTIFY_NSQ_COMMENT (sentence) optionally add a comment to this setting -``` - -```sh -$ mc admin config get myminio/ notify_nsq -notify_nsq:1 nsqd_address="" queue_dir="" queue_limit="0" tls_enable="off" tls_skip_verify="off" topic="" -``` - -Use `mc admin config set` command to update the configuration for the deployment. Restart the MinIO server to put the changes into effect. The server will print a line like `SQS ARNs: arn:minio:sqs::1:nsq` at start-up if there were no errors. - -```sh -$ mc admin config set myminio notify_nsq:1 nsqd_address="127.0.0.1:4150" queue_dir="" queue_limit="0" tls_enable="off" tls_skip_verify="on" topic="minio" -``` - -Note that, you can add as many NSQ daemon endpoint configurations as needed by providing an identifier (like "1" in the example above) for the NSQ instance and an object of per-server configuration parameters. - -### Step 2: Enable bucket notification using MinIO client - -We will enable bucket event notification to trigger whenever a JPEG image is uploaded or deleted `images` bucket on `myminio` server. Here ARN value is `arn:minio:sqs::1:nsq`. - -``` -mc mb myminio/images -mc event add myminio/images arn:minio:sqs::1:nsq --suffix .jpg -mc event list myminio/images -arn:minio:sqs::1:nsq s3:ObjectCreated:*,s3:ObjectRemoved:* Filter: suffix=”.jpg” -``` - -### Step 3: Test on NSQ - -The simplest test is to download `nsq_tail` from [nsq github](https://github.com/nsqio/nsq/releases) - -``` -./nsq_tail -nsqd-tcp-address 127.0.0.1:4150 -topic minio -``` - -Open another terminal and upload a JPEG image into `images` bucket. - -``` -mc cp gopher.jpg myminio/images -``` - -You should receive the following event notification via NSQ once the upload completes. - -``` -{"EventName":"s3:ObjectCreated:Put","Key":"images/gopher.jpg","Records":[{"eventVersion":"2.0","eventSource":"minio:s3","awsRegion":"","eventTime":"2018-10-31T09:31:11Z","eventName":"s3:ObjectCreated:Put","userIdentity":{"principalId":"21EJ9HYV110O8NVX2VMS"},"requestParameters":{"sourceIPAddress":"10.1.1.1"},"responseElements":{"x-amz-request-id":"1562A792DAA53426","x-minio-origin-endpoint":"http://10.0.3.1:9000"},"s3":{"s3SchemaVersion":"1.0","configurationId":"Config","bucket":{"name":"images","ownerIdentity":{"principalId":"21EJ9HYV110O8NVX2VMS"},"arn":"arn:aws:s3:::images"},"object":{"key":"gopher.jpg","size":162023,"eTag":"5337769ffa594e742408ad3f30713cd7","contentType":"image/jpeg","userMetadata":{"content-type":"image/jpeg"},"versionId":"1","sequencer":"1562A792DAA53426"}},"source":{"host":"","port":"","userAgent":"MinIO (linux; amd64) minio-go/v6.0.8 mc/DEVELOPMENT.GOGET"}}]} -``` diff --git a/source/concepts/feature-overview.rst b/source/concepts/feature-overview.rst index 0b86cd7a..eebcde1c 100644 --- a/source/concepts/feature-overview.rst +++ b/source/concepts/feature-overview.rst @@ -16,7 +16,7 @@ The following table lists MinIO features and their corresponding documentation: * - Feature - Description - * - :doc:`Bucket Notifications ` + * - :doc:`Bucket Notifications ` - MinIO Bucket Notifications allows you to automatically publish notifications to one or more configured notification targets when specific events occur in a bucket. @@ -38,6 +38,5 @@ The following table lists MinIO features and their corresponding documentation: :titlesonly: :hidden: - /concepts/bucket-notifications /concepts/bucket-versioning /concepts/erasure-coding \ No newline at end of file diff --git a/source/conf.py b/source/conf.py index f264275d..f586f0fb 100644 --- a/source/conf.py +++ b/source/conf.py @@ -43,7 +43,8 @@ extensions = [ 'recommonmark', 'sphinx_markdown_tables', 'sphinx-prompt', - 'sphinx_substitution_extensions' + 'sphinx_substitution_extensions', + 'sphinx_togglebutton' ] # -- External Links @@ -57,7 +58,7 @@ extlinks = { 'github' : ('https://github.com/%s',''), 'kube-api' : ('https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/%s',''), 'aws-docs' : ('https://docs.aws.amazon.com/%s',''), - 's3-docs' : ('https://docs.aws.amazon.com/AmazonS3/latest/dev/%s',''), + 's3-docs' : ('https://docs.aws.amazon.com/AmazonS3/latest/userguide/%s',''), 's3-api' : ('https://docs.aws.amazon.com/AmazonS3/latest/API/%s',''), 'iam-docs' : ('https://docs.aws.amazon.com/IAM/latest/UserGuide/%s',''), 'release' : ('https://github.com/minio/mc/releases/tag/%s',''), diff --git a/source/includes/common-admonitions.rst b/source/includes/common-admonitions.rst new file mode 100644 index 00000000..411014e9 --- /dev/null +++ b/source/includes/common-admonitions.rst @@ -0,0 +1,18 @@ +.. Used in the following pages: + - /monitoring/bucket-notifications/publish-events-to-amqp.rst + +.. start-restart-downtime + +.. important:: + + This procedure *requires* restarting all :mc:`minio server` processes + associated to the deployment at the same time. There is typically a brief + period of time during which API operations are interrupted or may fail. + + Applications using an S3-compatible SDK with built-in retry logic *or* which + implement manual retry logic typically experience no notable interruption in + services. For applications which cannot use retry-logic, consider scheduling + a maintenance period to minimize interruption of services while performing + this procedure. + +.. end-restart-downtime \ No newline at end of file diff --git a/source/includes/common-mc-admin-config.rst b/source/includes/common-mc-admin-config.rst new file mode 100644 index 00000000..34b30417 --- /dev/null +++ b/source/includes/common-mc-admin-config.rst @@ -0,0 +1,986 @@ +.. Descriptions for AMQP bucket notification configurations. + Used in the following files: + - /source/reference/minio-server/minio-server.rst + - /source/reference/minio-cli/minio-mc-admin/mc-admin-config.rst + +.. start-minio-notify-amqp-enable + +Specify ``on`` to enable publishing bucket notifications to an AMQP endpoint. + +Defaults to ``off``. + +.. end-minio-notify-amqp-enable + + +.. start-minio-notify-amqp-url + +Specify the AMQP server endpoint to which MinIO publishes bucket events. +For example, ``amqp://myuser:mypassword@localhost:5672``. + +.. end-minio-notify-amqp-url + + +.. start-minio-notify-amqp-exchange + +Specify the name of the AMQP exchange to use. + +.. end-minio-notify-amqp-exchange + + +.. start-minio-notify-amqp-exchange-type + +Specify the type of the AMQP exchange. + +.. end-minio-notify-amqp-exchange-type + + +.. start-minio-notify-amqp-routing-key + +Specify the routing key for publishing events. + +.. end-minio-notify-amqp-routing-key + + +.. start-minio-notify-amqp-mandatory + +Specify ``off`` to ignore undelivered messages errors. Defaults to ``on``. + +.. end-minio-notify-amqp-mandatory + + +.. start-minio-notify-amqp-durable + +Specify ``on`` to persist the message queue across broker restarts. Defaults to +'off'. + +.. end-minio-notify-amqp-durable + + +.. start-minio-notify-amqp-no-wait + +Specify ``on`` to enable non-blocking message delivery. Defaults to 'off'. + +.. end-minio-notify-amqp-no-wait + + +.. start-minio-notify-amqp-internal + +Specify ``on`` to use the exchange only if it is bound to other exchanges. See +the RabbitMQ documentation on `Exchange to Exchange Bindings +`__ for more information on AMQP exchange +binding. + +.. end-minio-notify-amqp-internal + + +.. start-minio-notify-amqp-auto-deleted + +Specify ``on`` to automatically delete the message queue if there are no +consumers. Defaults to ``off``. + +.. end-minio-notify-amqp-auto-deleted + + +.. start-minio-notify-amqp-delivery-mode + +Specify ``1`` for set the delivery mode to non-persistent queue. + +Specify ``2`` to set the delivery mode to persistent queue. + +.. end-minio-notify-amqp-delivery-mode + + +.. start-minio-notify-amqp-queue-dir + +Specify the directory path to enable MinIO's persistent event store for +undelivered messages, such as ``/home/events``. + +MinIO stores undelivered events in the specified store while the AMQP +service is offline and replays the directory when connectivity resumes. + +.. end-minio-notify-amqp-queue-dir + + +.. start-minio-notify-amqp-queue-limit + +Specify the maximum limit for undelivered messages. Defaults to ``10000``. + +.. end-minio-notify-amqp-queue-limit + + +.. start-minio-notify-amqp-comment + +Specify a comment for the AMQP configuration. + +.. end-minio-notify-amqp-comment + +.. Descriptions for MQTT bucket notification configurations. + Used in the following files: + - /source/reference/minio-server/minio-server.rst + - /source/reference/minio-cli/minio-mc-admin/mc-admin-config.rst + +.. start-minio-notify-mqtt-enable + +Specify ``on`` to enable publishing bucket notifications to an MQTT endpoint. + +Defaults to ``off``. + +.. end-minio-notify-mqtt-enable + + +.. start-minio-notify-mqtt-broker + +Specify the MQTT server/broker endpoint. MinIO supports TCP, TLS, or Websocket +connections to the server/broker URL. For example: + +- ``tcp://mqtt.example.net:1883`` +- ``tls://mqtt.example.net:1883`` +- ``ws://mqtt.example.net:1883`` + +.. end-minio-notify-mqtt-broker + + +.. start-minio-notify-mqtt-topic + +Specify the name of the MQTT topic to associate with events published by +MinIO to the MQTT endpoint. + +.. end-minio-notify-mqtt-topic + + +.. start-minio-notify-mqtt-username + +Specify the MQTT username with which MinIO authenticates to the MQTT +server/broker. + +.. end-minio-notify-mqtt-username + + +.. start-minio-notify-mqtt-password + +Specify the password for the MQTT username with which MinIO authenticates to the +MQTT server/broker. + +.. end-minio-notify-mqtt-password + + +.. start-minio-notify-mqtt-qos + +Specify the Quality of Service priority for the published events. + +Defaults to ``0``. + +.. end-minio-notify-mqtt-qos + + +.. start-minio-notify-mqtt-keep-alive-interval + +Specify the keep-alive interval for the MQTT connections. MinIO +supports the following units of time measurement: + +- ``s`` - seconds, "60s" +- ``m`` - minutes, "60m" +- ``h`` - hours, "24h" +- ``d`` - days, "7d" + +.. end-minio-notify-mqtt-keep-alive-interval + + +.. start-minio-notify-mqtt-reconnect-interval + +Specify the reconnect interval for the MQTT connections. MinIO +supports the following units of time measurement: + +- ``s`` - seconds, "60s" +- ``m`` - minutes, "60m" +- ``h`` - hours, "24h" +- ``d`` - days, "7d" + +.. end-minio-notify-mqtt-reconnect-interval + + +.. start-minio-notify-mqtt-queue-dir + +Specify the directory path to enable MinIO's persistent event store for +undelivered messages, such as ``/home/events``. + +MinIO stores undelivered events in the specified store while the MQTT +server/broker is offline and replays the directory when connectivity resumes. + +.. end-minio-notify-mqtt-queue-dir + + +.. start-minio-notify-mqtt-queue-limit + +Specify the maximum limit for undelivered messages. Defaults to ``10000``. + +.. end-minio-notify-mqtt-queue-limit + + +.. start-minio-notify-mqtt-comment + +Specify a comment to associate with the MQTT configuration. + +.. end-minio-notify-mqtt-comment + +.. Descriptions for Elasticsearch bucket notification configurations. + Used in the following files: + - /source/reference/minio-server/minio-server.rst + - /source/reference/minio-cli/minio-mc-admin/mc-admin-config.rst + - /source/monitoring/bucket-notifications/publish-events-to-elasticsearch.rst + +.. start-minio-notify-elasticsearch-enable + +Specify ``on`` to enable publishing bucket notifications to an Elasticsearch +service endpoint. + +Defaults to ``off``. + +.. end-minio-notify-elasticsearch-enable + +.. start-minio-notify-elasticsearch-url + +Specify the Elasticsearch service endpoint to which MinIO publishes bucket +events. For example, ``https://elasticsearch.example.com:9200``. + +MinIO supports passing authentication information using as URL parameters +using the format ``PROTOCOL://USERNAME:PASSWORD@HOSTNAME:PORT``. + +.. end-minio-notify-elasticsearch-url + +.. start-minio-notify-elasticsearch-index + +Specify the name of the Elasticsearch index in which to store or update +MinIO bucket events. Elasticsearch automatically creates the index if it +does not exist. + +.. end-minio-notify-elasticsearch-index + +.. start-minio-notify-elasticsearch-format + +Specify the format of event data written to the Elasticsearch index. MinIO +supports the following values: + +``namespace`` + For each bucket event, the MinIO creates a JSON document with the bucket + and object name from the event as the document ID and the actual event as + part of the document body. Additional updates to that object modify the + existing index entry for that object. Similarly, deleting the object + also deletes the corresponding index entry. + +``access`` + For each bucket event, MinIO creates a JSON document with the event + details and appends it to the index with an Elasticsearch-generated + random ID. Additional updates to an object result in new index entries, + and existing entries remain unmodified. + +.. end-minio-notify-elasticsearch-format + +.. start-minio-notify-elasticsearch-username + +The username for connecting to an Elasticsearch service endpoint which +enforces authentication. + +.. end-minio-notify-elasticsearch-username + +.. start-minio-notify-elasticsearch-password + +The password for connecting to an Elasticsearch service endpoint which enforces +authentication. + +.. end-minio-notify-elasticsearch-password + +.. start-minio-notify-elasticsearch-queue-limit + +Specify the maximum limit for undelivered messages. Defaults to ``10000``. + +.. end-minio-notify-elasticsearch-queue-limit + +.. start-minio-notify-elasticsearch-queue-dir + +Specify the directory path to enable MinIO's persistent event store for +undelivered messages, such as ``/home/events``. + +MinIO stores undelivered events in the specified store while the Elasticsearch +service is offline and replays the directory when connectivity resumes. + +.. end-minio-notify-elasticsearch-queue-dir + +.. start-minio-notify-elasticsearch-comment + +Specify a comment to associate with the Elasticsearch configuration. + +.. end-minio-notify-elasticsearch-comment + +.. Descriptions for NSQ bucket notification configurations. + Used in the following files: + - /source/reference/minio-server/minio-server.rst + - /source/reference/minio-cli/minio-mc-admin/mc-admin-config.rst + - /source/monitoring/bucket-notifications/publish-events-to-nsq.rst + +.. start-minio-notify-nsq-enable + +Specify ``on`` to enable publishing bucket notifications to an NSQ endpoint. + +.. end-minio-notify-nsq-enable + +.. start-minio-notify-nsq-nsqd-address + +Specify the NSQ server address. For example: + +``https://nsq-endpoing.example.net:4150`` + +.. end-minio-notify-nsq-nsqd-address + +.. start-minio-notify-nsq-topic + +Specify the name of the NSQ topic MinIO uses when publishing events to the +broker. + +.. end-minio-notify-nsq-topic + +.. start-minio-notify-nsq-tls + +Specify ``on`` to enable TLS connectivity to the NSQ service broker. + +.. end-minio-notify-nsq-tls + +.. start-minio-notify-nsq-tls-skip-verify + +Enables or disables TLS verification of the NSQ service broker TLS certificates. + +- Specify ``on`` to disable TLS verification (Default). +- Specify ``off`` to enable TLS verification. + +.. end-minio-notify-nsq-tls-skip-verify + +.. start-minio-notify-nsq-queue-dir + +Specify the directory path to enable MinIO's persistent event store for +undelivered messages, such as ``/home/events``. + +MinIO stores undelivered events in the specified store while the NSQ +server/broker is offline and replays the directory when connectivity resumes. + +.. end-minio-notify-nsq-queue-dir + +.. start-minio-notify-nsq-queue-limit + +Specify the maximum limit for undelivered messages. Defaults to ``10000``. + +.. end-minio-notify-nsq-queue-limit + +.. start-minio-notify-nsq-comment + + +Specify a comment to associate with the NSQ configuration. + +.. end-minio-notify-nsq-comment + +.. Descriptions for Redis bucket notification configurations. + Used in the following files: + - /source/reference/minio-server/minio-server.rst + - /source/reference/minio-cli/minio-mc-admin/mc-admin-config.rst + - /source/monitoring/bucket-notifications/publish-events-to-redis.rst + +.. start-minio-notify-redis-enable + +Specify ``on`` to enable publishing bucket notifications to a Redis +service endpoint. + +Defaults to ``off``. + +.. end-minio-notify-redis-enable + +.. start-minio-notify-redis-address + +Specify the Redis service endpoint to which MinIO publishes bucket events. +For example, ``https://redis.example.com:6369``. + +.. end-minio-notify-redis-address + +.. start-minio-notify-redis-key + +Specify the Redis key to use for storing and updating events. Redis +auto-creates the key if it does not exist. + +.. end-minio-notify-redis-key + +.. start-minio-notify-redis-format + +Specify the format of event data written to the Redis service endpoint. MinIO +supports the following values: + +``namespace`` + For each bucket event, the MinIO creates a JSON document with the bucket + and object name from the event as the document ID and the actual event as + part of the document body. Additional updates to that object modify the + existing index entry for that object. Similarly, deleting the object + also deletes the corresponding index entry. + +``access`` + For each bucket event, MinIO creates a JSON document with the event + details and appends it to the key with a Redis-generated + random ID. Additional updates to an object result in new index entries, + and existing entries remain unmodified. + +.. end-minio-notify-redis-format + +.. start-minio-notify-redis-password + +Specify the password for the Redis server. + +.. end-minio-notify-redis-password + + +.. start-minio-notify-redis-queue-dir + +Specify the directory path to enable MinIO's persistent event store for +undelivered messages, such as ``/home/events``. + +MinIO stores undelivered events in the specified store while the Redis +server/broker is offline and replays the directory when connectivity resumes. + +.. end-minio-notify-redis-queue-dir + +.. start-minio-notify-redis-queue-limit + +Specify the maximum limit for undelivered messages. Defaults to ``10000``. + +.. end-minio-notify-redis-queue-limit + +.. start-minio-notify-redis-comment + + +Specify a comment to associate with the Redis configuration. + +.. end-minio-notify-redis-comment + +.. Descriptions for NATS bucket notification configurations. + Used in the following files: + - /source/reference/minio-server/minio-server.rst + - /source/reference/minio-cli/minio-mc-admin/mc-admin-config.rst + - /source/monitoring/bucket-notifications/publish-events-to-nats.rst + +.. start-minio-notify-nats-enable + +Specify ``on`` to enable publishing bucket notifications to an NATS +service endpoint. + +Defaults to ``off``. + +.. end-minio-notify-nats-enable + +.. start-minio-notify-nats-address + +Specify the NATS service endpoint to which MinIO publishes bucket events. +For example, ``https://nats-endpoint.example.com:4222``. + +.. end-minio-notify-nats-address + +.. start-minio-notify-nats-subject + +Specify the subscription to which MinIO associates events +published to the NATS endpoint. + +.. end-minio-notify-nats-subject + +.. start-minio-notify-nats-username + +Specify the username for connecting to the NATS service endpoint. + +.. end-minio-notify-nats-username + +.. start-minio-notify-nats-password + +Specify the passport for connecting to the NATS service endpoint. + +.. end-minio-notify-nats-password + +.. start-minio-notify-nats-token + +Specify the token for connecting to the NATS service endpoint. + +.. end-minio-notify-nats-token + +.. start-minio-notify-nats-tls + +Specify ``on`` to enable TLS connectivity to the NATS service endpoint. + +.. end-minio-notify-nats-tls + +.. start-minio-notify-nats-tls-skip-verify + +Enables or disables TLS verification of the NATS service endpoint TLS +certificates. + +- Specify ``on`` to disable TLS verification (Default). +- Specify ``off`` to enable TLS verification. + +.. end-minio-notify-nats-tls-skip-verify + +.. start-minio-notify-nats-ping-interval + +Specify the duration interval for client pings to the NATS server. +MinIO supports the following time units: + +- ``s`` - seconds, ``"60s"`` +- ``m`` - minutes, ``"5m"`` +- ``h`` - hours, ``"1h"`` +- ``d`` - days, ``"1d"`` + +.. end-minio-notify-nats-ping-interval + +.. start-minio-notify-nats-streaming + +Specify ``on`` to enable streaming events to the NATS service endpoint. + +.. end-minio-notify-nats-streaming + +.. start-minio-notify-nats-streaming-async + +Specify ``on`` to enable asynchronous publishing of events to the NATS service +endpoint. + +.. end-minio-notify-nats-streaming-async + +.. start-minio-notify-nats-streaming-max-pub-acks-in-flight + +Specify the number of messages to publish without waiting for an ACK +response from the NATS service endpoint. + +.. end-minio-notify-nats-streaming-max-pub-acks-in-flight + +.. start-minio-notify-nats-streaming-cluster-id + +Specify the unique ID for the NATS streaming cluster. + +.. end-minio-notify-nats-streaming-cluster-id + +.. start-minio-notify-nats-cert-authority + +Specify the path to the Certificate Authority chain used to sign the +NATS service endpoint TLS certificates. + +.. end-minio-notify-nats-cert-authority + +.. start-minio-notify-nats-client-cert + +Specify the path to the client certificate to use for performing +mTLS authentication to the NATS service endpoint. + +.. end-minio-notify-nats-client-cert + +.. start-minio-notify-nats-client-key + +Specify the path to the client private key to use for performing mTLS +authentication to the NATS service endpoint. + +.. end-minio-notify-nats-client-key + +.. start-minio-notify-nats-queue-dir + +Specify the directory path to enable MinIO's persistent event store for +undelivered messages, such as ``/home/events``. + +MinIO stores undelivered events in the specified store while the NATS +server/broker is offline and replays the directory when connectivity resumes. + +.. end-minio-notify-nats-queue-dir + +.. start-minio-notify-nats-queue-limit + +Specify the maximum limit for undelivered messages. Defaults to ``10000``. + +.. end-minio-notify-nats-queue-limit + +.. start-minio-notify-nats-comment + +Specify a comment to associate with the NATS configuration. + +.. end-minio-notify-nats-comment + +.. Descriptions for postgresql bucket notification configurations. + Used in the following files: + - /source/reference/minio-server/minio-server.rst + - /source/reference/minio-cli/minio-mc-admin/mc-admin-config.rst + - /source/monitoring/bucket-notifications/publish-events-to-postgresql.rst + +.. start-minio-notify-postgresql-enable + +Specify ``on`` to enable publishing bucket notifications to a PostgreSQL +service endpoint. + +Defaults to ``off``. + +.. end-minio-notify-postgresql-enable + +.. start-minio-notify-postgresql-connection-string + +Specify the `URI connection string +`__ +of the PostgreSQL service endpoint. MinIO supports ``key=value`` format for +the PostgreSQL connection string. For example: + +``"host=https://postgresql.example.com port=5432 ..."`` + +For more complete documentation on supported PostgreSQL connection +string parameters, see the `PostgreSQL COnnection Strings documentation +`__ +. + +.. end-minio-notify-postgresql-connection-string + +.. start-minio-notify-postgresql-table + +Specify the name of the PostgreSQL table to which MinIO publishes +event notifications. + +.. end-minio-notify-postgresql-table + +.. start-minio-notify-postgresql-format + +Specify the format of event data written to the PostgreSQL service endpoint. +MinIO supports the following values: + +``namespace`` + For each bucket event, the MinIO creates a JSON document with the bucket + and object name from the event as the document ID and the actual event as + part of the document body. Additional updates to that object modify the + existing table entry for that object. Similarly, deleting the object + also deletes the corresponding table entry. + +``access`` + For each bucket event, MinIO creates a JSON document with the event + details and appends it to the table with a PostgreSQL-generated + random ID. Additional updates to an object result in new index entries, + and existing entries remain unmodified. + +.. end-minio-notify-postgresql-format + +.. start-minio-notify-postgresql-max-open-connections + +Specify the maximum number of open connections to the PostgreSQL database. + +Defaults to ``2``. + +.. end-minio-notify-postgresql-max-open-connections + +.. start-minio-notify-postgresql-queue-dir + +Specify the directory path to enable MinIO's persistent event store for +undelivered messages, such as ``/home/events``. + +MinIO stores undelivered events in the specified store while the PostgreSQL +server/broker is offline and replays the directory when connectivity resumes. + +.. end-minio-notify-postgresql-queue-dir + +.. start-minio-notify-postgresql-queue-limit + +Specify the maximum limit for undelivered messages. Defaults to ``10000``. + +.. end-minio-notify-postgresql-queue-limit + +.. start-minio-notify-postgresql-comment + +Specify a comment to associate with the PostgreSQL configuration. + +.. end-minio-notify-postgresql-comment + + +.. Descriptions for MySQL bucket notification configurations. + Used in the following files: + - /source/reference/minio-server/minio-server.rst + - /source/reference/minio-cli/minio-mc-admin/mc-admin-config.rst + - /source/monitoring/bucket-notifications/publish-events-to-mysql.rst + +.. start-minio-notify-mysql-enable + +Specify ``on`` to enable publishing bucket notifications to a MySQL +service endpoint. + +Defaults to ``off``. + +.. end-minio-notify-mysql-enable + +.. start-minio-notify-mysql-dsn-string + +Specify the data source name (DSN) of the MySQL service endpoint. MinIO expects +the following format: + +``:@tcp(:)/`` + +For example: + +``"username:password@tcp(mysql.example.com:3306)/miniodb"`` + +.. end-minio-notify-mysql-dsn-string + +.. start-minio-notify-mysql-connection-string + +Specify the data source name (DSN) connection string for the MySQL service +endpoint. MinIO expects the following format: + +``:@tcp(:)/`` + +For example: + +``"username:password@tcp(mysql.example.com:3306)/miniodb"`` + +.. end-minio-notify-mysql-connection-string + +.. start-minio-notify-mysql-table + +Specify the name of the MySQL table to which MinIO publishes event +notifications. + +.. end-minio-notify-mysql-table + +.. start-minio-notify-mysql-format + +Specify the format of event data written to the MySQL service endpoint. +MinIO supports the following values: + +``namespace`` + For each bucket event, the MinIO creates a JSON document with the bucket + and object name from the event as the document ID and the actual event as + part of the document body. Additional updates to that object modify the + existing table entry for that object. Similarly, deleting the object + also deletes the corresponding table entry. + +``access`` + For each bucket event, MinIO creates a JSON document with the event + details and appends it to the table with a MySQL-generated + random ID. Additional updates to an object result in new index entries, + and existing entries remain unmodified. + +.. end-minio-notify-mysql-format + +.. start-minio-notify-mysql-max-open-connections + +Specify the maximum number of open connections to the MySQL database. + +Defaults to ``2``. + +.. end-minio-notify-mysql-max-open-connections + +.. start-minio-notify-mysql-queue-dir + +Specify the directory path to enable MinIO's persistent event store for +undelivered messages, such as ``/home/events``. + +MinIO stores undelivered events in the specified store while the MySQL +server/broker is offline and replays the directory when connectivity resumes. + +.. end-minio-notify-mysql-queue-dir + +.. start-minio-notify-mysql-queue-limit + +Specify the maximum limit for undelivered messages. Defaults to ``10000``. + +.. end-minio-notify-mysql-queue-limit + +.. start-minio-notify-mysql-comment + +Specify a comment to associate with the MySQL configuration. + +.. end-minio-notify-mysql-comment + + +.. Descriptions for Kafka bucket notification configurations. + Used in the following files: + - /source/reference/minio-server/minio-server.rst + - /source/reference/minio-cli/minio-mc-admin/mc-admin-config.rst + - /source/monitoring/bucket-notifications/publish-events-to-kafka.rst + +.. start-minio-notify-kafka-enable + +Specify ``on`` to enable publishing bucket notifications to a Kafka +service endpoint. + +Defaults to ``off``. + +.. end-minio-notify-kafka-enable + +.. start-minio-notify-kafka-brokers + +Specify a comma-separated list of Kafka broker addresses. For example: + +``"kafka1.example.com:2021,kafka2.example.com:2021"`` + +.. end-minio-notify-kafka-brokers + +.. start-minio-notify-kafka-topic + +Specify the name of the Kafka topic to which MinIO publishes +bucket events. + +.. end-minio-notify-kafka-topic + +.. start-minio-notify-kafka-sasl-username + +Specify the username for performing SASL/PLAIN or SASL/SCRAM authentication +to the Kafka broker(s). + +.. end-minio-notify-kafka-sasl-username + +.. start-minio-notify-kafka-sasl-password + +Specify the password for performing SASL/PLAIN or SASL/SCRAM authentication +to the Kafka broker(s). + +.. end-minio-notify-kafka-sasl-password + +.. start-minio-notify-kafka-sasl-mechanism + +Specify the SASL mechanism to use for authenticating to the Kafka broker(s). +MinIO supports the following mechanisms: + +- ``PLAIN`` (Default) +- ``SCRAM`` + +.. end-minio-notify-kafka-sasl-mechanism + +.. start-minio-notify-kafka-tls-client-auth + +Specify the client authentication policy of the Kafka broker(s). See +`ClientAuthType `__ for +more information on possible values for this field. + +.. https://pkg.go.dev/crypto/tls#ClientAuthType ? + +.. end-minio-notify-kafka-tls-client-auth + +.. start-minio-notify-kafka-sasl + +Specify ``on`` to enable SASL authentication. + +.. end-minio-notify-kafka-sasl + +.. start-minio-notify-kafka-tls + +Specify ``on`` to enable TLS connectivity to the Kafka broker(s) + +.. end-minio-notify-kafka-tls + +.. start-minio-notify-kafka-tls-skip-verify + +Enables or disables TLS verification of the NATS service endpoint TLS +certificates. + +- Specify ``on`` to disable TLS verification (Default). +- Specify ``off`` to enable TLS verification. + +.. end-minio-notify-kafka-tls-skip-verify + +.. start-minio-notify-kafka-client-tls-cert + +Specify the path to the client certificate to use for performing +mTLS authentication to the Kafka broker(s). + +.. end-minio-notify-kafka-client-tls-cert + +.. start-minio-notify-kafka-client-tls-key + +Specify the path to the client private key to use for performing +mTLS authentication to the Kafka broker(s). + +.. end-minio-notify-kafka-client-tls-key + +.. start-minio-notify-kafka-version + +Specify the version of the Kafka cluster to assume when performing operations +against that cluster. See the `sarama reference documentation +`__ for +more information on this field's behavior. + +.. end-minio-notify-kafka-version + +.. start-minio-notify-kafka-queue-dir + +Specify the directory path to enable MinIO's persistent event store for +undelivered messages, such as ``/home/events``. + +MinIO stores undelivered events in the specified store while the Kafka +server/broker is offline and replays the directory when connectivity resumes. + +.. end-minio-notify-kafka-queue-dir + +.. start-minio-notify-kafka-queue-limit + +Specify the maximum limit for undelivered messages. Defaults to ``10000``. + +.. end-minio-notify-kafka-queue-limit + +.. start-minio-notify-kafka-comment + +Specify a comment to associate with the Kafka configuration. + +.. end-minio-notify-kafka-comment + + +.. Descriptions for Webhook bucket notification configurations. + Used in the following files: + - /source/reference/minio-server/minio-server.rst + - /source/reference/minio-cli/minio-mc-admin/mc-admin-config.rst + - /source/monitoring/bucket-notifications/publish-events-to-webhook.rst + + +.. start-minio-notify-webhook-enable + +Specify ``on`` to enable publishing bucket notifications to a Webhook +service endpoint. + +Defaults to ``off``. + +.. end-minio-notify-webhook-enable + +.. start-minio-notify-webhook-endpoint + +Specify the URL for the webhook service. + +.. end-minio-notify-webhook-endpoint + +.. start-minio-notify-webhook-auth-token + +Specify the opaque string or JWT authorization token to use for +authenticating to the webhook service. + +.. end-minio-notify-webhook-auth-token + +.. start-minio-notify-webhook-client-cert + +Specify the path to the client certificate to use for performing +mTLS authentication to the webhook service. + +.. end-minio-notify-webhook-client-cert + +.. start-minio-notify-webhook-client-key + +Specify the path to the client private key to use for performing +mTLS authentication to the webhook service. + +.. end-minio-notify-webhook-client-key + +.. start-minio-notify-webhook-queue-dir + +Specify the directory path to enable MinIO's persistent event store for +undelivered messages, such as ``/home/events``. + +MinIO stores undelivered events in the specified store while the webhook +service is offline and replays the directory when connectivity resumes. + +.. end-minio-notify-webhook-queue-dir + +.. start-minio-notify-webhook-queue-limit + +Specify the maximum limit for undelivered messages. Defaults to ``10000``. + +.. end-minio-notify-webhook-queue-limit + +.. start-minio-notify-webhook-comment + +Specify a comment to associate with the Webhook configuration. + +.. end-minio-notify-webhook-comment diff --git a/source/index.rst b/source/index.rst index 8cc634d0..8a543670 100644 --- a/source/index.rst +++ b/source/index.rst @@ -22,6 +22,8 @@ Users deploying onto a Kubernetes cluster should start with our /concepts/feature-overview /tutorials/minio-installation /security/security-overview + /monitoring/monitoring-overview /reference/minio-cli/minio-mc /reference/minio-cli/minio-mc-admin /reference/minio-server/minio-server + /reference/minio-server/minio-gateway diff --git a/source/monitoring/bucket-notifications/bucket-notifications.rst b/source/monitoring/bucket-notifications/bucket-notifications.rst new file mode 100644 index 00000000..3a3e364e --- /dev/null +++ b/source/monitoring/bucket-notifications/bucket-notifications.rst @@ -0,0 +1,167 @@ +.. _minio-bucket-notifications: + +==================== +Bucket Notifications +==================== + +.. default-domain:: minio + +.. contents:: Table of Contents + :local: + :depth: 2 + +MinIO bucket notifications allow administrators to send notifications to +supported external services on certain object or bucket events. MinIO +supports bucket and object-level S3 events similar to the +:s3-docs:`Amazon S3 Event Notifications `. + +MinIO bucket notifications are available *only* with +:mc:`minio server` deployments. MinIO :ref:`Gateway ` +does *not* support bucket notifications. + +Supported Notification Targets +------------------------------ + +MinIO supports publishing event notifications to the following targets: + +.. list-table:: + :header-rows: 1 + :widths: 30 70 + :width: 100% + + * - Target + - Description + + * - :guilabel:`AMQP` (RabbitMQ) + - Publish notifications to an AMQP service such as + `RabbitMQ `__. + + See :ref:`minio-bucket-notifications-publish-amqp` for a tutorial. + + * - :guilabel:`MQTT` + - Publish notifications to an `MQTT `__ + service. + + See :ref:`minio-bucket-notifications-publish-mqtt` for a tutorial. + + * - :guilabel:`NATS` + - Publish notifications to a `NATS `__ service. + + See :ref:`minio-bucket-notifications-publish-nats` for a tutorial. + + * - :guilabel:`NSQ` + - Publish notifications to a `NSQ `__ service. + + See :ref:`minio-bucket-notifications-publish-nsq` for a tutorial + + + * - :guilabel:`Elasticsearch` + - Publish notifications to a `Elasticsearch `__ + service. + + See :ref:`minio-bucket-notifications-publish-elasticsearch` for a + tutorial. + + * - :guilabel:`Kafka` + - Publish notifications to a `Kafka `__ service. + + See :ref:`minio-bucket-notifications-publish-kafka` for a tutorial. + + * - :guilabel:`MySQL` + - Publish notifications to a `MySQL `__ service. + + See :ref:`minio-bucket-notifications-publish-mysql` for a tutorial. + + * - :guilabel:`PostgreSQL` + - Publish notifications to a `PostgreSQL `__ + service. + + See :ref:`minio-bucket-notifications-publish-postgresql` for a tutorial. + + * - :guilabel:`Redis` + - Publish notifications to a `Redis `__ service. + + See :ref:`minio-bucket-notifications-publish-redis` for a tutorial. + + * - :guilabel:`webhook` + - Publish notifications to a `Webhook + `__ service. + + See :ref:`minio-bucket-notifications-publish-webhook` for a tutorial. + + +.. _minio-bucket-notifications-event-types: + +Supported S3 Event Types +------------------------ + +MinIO bucket notifications are compatible with +:s3-docs:`Amazon S3 Event Notifications `. This +section lists all supported events. + +Object Events +~~~~~~~~~~~~~ + +MinIO supports triggering notifications on the following S3 object events: + +.. data:: s3:ObjectRemoved:DeleteMarkerCreated +.. data:: s3:ObjectRemoved:Delete +.. data:: s3:ObjectCreated:PutRetention +.. data:: s3:ObjectCreated:PutLegalHold +.. data:: s3:ObjectCreated:Put +.. data:: s3:ObjectCreated:Post +.. data:: s3:ObjectCreated:Copy +.. data:: s3:ObjectCreated:CompleteMultipartUpload +.. data:: s3:ObjectAccessed:Head +.. data:: s3:ObjectAccessed:GetRetention +.. data:: s3:ObjectAccessed:GetLegalHold +.. data:: s3:ObjectAccessed:Get + +Replication Events +~~~~~~~~~~~~~~~~~~ + +MinIO supports triggering notifications on the following S3 replication +events: + +.. data:: s3:Replication:OperationCompletedReplication +.. data:: s3:Replication:OperationFailedReplication +.. data:: s3:Replication:OperationMissedThreshold +.. data:: s3:Replication:OperationNotTracked +.. data:: s3:Replication:OperationReplicatedAfterThreshold + +ILM Transition Events +~~~~~~~~~~~~~~~~~~~~~ + +MinIO supports triggering notifications on the following S3 ILM transition +events: + +.. data:: s3:ObjectRestore:Post +.. data:: s3:ObjectRestore:Completed + +Global Events +~~~~~~~~~~~~~ + +MinIO supports triggering notifications on the following global events. +You can only listen to these events through the :legacy:`ListenNotification +` API: + +.. data:: s3:BucketCreated +.. data:: s3:BucketRemoved + + +.. todo + +.. toctree:: + :titlesonly: + :hidden: + + /monitoring/bucket-notifications/publish-events-to-amqp + /monitoring/bucket-notifications/publish-events-to-mqtt + /monitoring/bucket-notifications/publish-events-to-nats + /monitoring/bucket-notifications/publish-events-to-nsq + /monitoring/bucket-notifications/publish-events-to-elasticsearch + /monitoring/bucket-notifications/publish-events-to-kafka + /monitoring/bucket-notifications/publish-events-to-mysql + /monitoring/bucket-notifications/publish-events-to-postgresql + /monitoring/bucket-notifications/publish-events-to-redis + /monitoring/bucket-notifications/publish-events-to-webhook diff --git a/source/monitoring/bucket-notifications/publish-events-to-amqp.rst b/source/monitoring/bucket-notifications/publish-events-to-amqp.rst new file mode 100644 index 00000000..13db3a20 --- /dev/null +++ b/source/monitoring/bucket-notifications/publish-events-to-amqp.rst @@ -0,0 +1,382 @@ +.. _minio-bucket-notifications-publish-amqp: + +================================= +Publish Events to AMQP (RabbitMQ) +================================= + +.. default-domain:: minio + +.. contents:: Table of Contents + :local: + :depth: 1 + +MinIO supports publishing :ref:`bucket notification +` events to a `AMQP 0-9-1 `__ +service endpoint such as `RabbitMQ `__. + +MinIO relies on the :github:`streadway/amqp` project for AMQP connectivity. The +project is primarily tested against `RabbitMQ `__ +deployments, though other `AMQP 0-9-1-compatible `__ +services *may* also work. The procedures on this page assume a RabbitMQ +deployment using the AMQP 0-9-1 protocol as the service endpoint. + +Add an AMQP Endpoint to a MinIO Deployment +------------------------------------------ + +The following procedure adds a new AMQP service endpoint for supporting +:ref:`bucket notifications ` in a MinIO +deployment. + +.. include:: /includes/common-admonitions.rst + :start-after: start-restart-downtime + :end-before: end-restart-downtime + +Prerequisites +~~~~~~~~~~~~~~ + +AMQP 0-9-1 Service Endpoint ++++++++++++++++++++++++++++ + +MinIO relies on the :github:`streadway/amqp` project for AMQP connectivity. The +project is primarily tested against `RabbitMQ `__ +deployments, though other `AMQP 0-9-1-compatible `__ +services *may* also work. This procedure assumes a RabbitMQ deployment +using the 0-9-1 protocol as the service endpoint. + +If the AMQP service requires authentication, you *must* provide an appropriate +username and password during the configuration process to grant MinIO access +to the service. + +MinIO ``mc`` Command Line Tool +++++++++++++++++++++++++++++++ + +This procedure uses the :mc:`mc` command line tool for certain actions. +See the ``mc`` :ref:`Quickstart ` for installation instructions. + +1) Add the AMQP Endpoint to MinIO +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can configure a new AMQP service endpoint using either environment variables +*or* by setting runtime configuration settings. + +.. tabs:: + + .. tab:: Environment Variables + + MinIO supports specifying the AMQP service endpoint and associated + configuration settings using + :ref:`environment variables + `. The + :mc:`minio server` process applies the specified settings on its + next startup. + + The following example code sets *all* environment variables + related to configuring an AMQP service endpoint. The minimum + *required* variables are + :envvar:`MINIO_NOTIFY_AMQP_ENABLE` and :envvar:`MINIO_NOTIFY_AMQP_URL`: + + .. code-block:: shell + :class: copyable + + set MINIO_NOTIFY_AMQP_ENABLE_="on" + set MINIO_NOTIFY_AMQP_URL_="" + set MINIO_NOTIFY_AMQP_EXCHANGE_="" + set MINIO_NOTIFY_AMQP_EXCHANGE_TYPE_="" + set MINIO_NOTIFY_AMQP_ROUTING_KEY_="" + set MINIO_NOTIFY_AMQP_MANDATORY_="" + set MINIO_NOTIFY_AMQP_DURABLE_="" + set MINIO_NOTIFY_AMQP_NO_WAIT_="" + set MINIO_NOTIFY_AMQP_INTERNAL_="" + set MINIO_NOTIFY_AMQP_AUTO_DELETED_="" + set MINIO_NOTIFY_AMQP_DELIVERY_MODE_="" + set MINIO_NOTIFY_AMQP_QUEUE_DIR_="" + set MINIO_NOTIFY_AMQP_QUEUE_LIMIT_="" + set MINIO_NOTIFY_AMQP_COMMENT_="" + + - Replace ```` with a unique descriptive string for the + AMQP service endpoint. Use the same ```` value for all + environment variables related to the new AMQP service endpoint. + The following examples assume an identifier of ``PRIMARY``. + + If the specified ```` matches an existing AMQP service + endpoint on the MinIO deployment, the new settings *override* + any existing settings for that endpoint. Use + :mc-cmd:`mc admin config get notify_amqp ` to + review the currently configured AMQP endpoints on the MinIO deployment. + + - Replace ```` with the URL of the AMQP service endpoint. + For example: + + ``amqp://user:password@hostname:port`` + + See :ref:`AMQP Service for Bucket Notifications + ` for complete documentation + on each environment variable. + + .. tab:: Configuration Settings + + MinIO supports adding or updating AMQP endpoints on a running + :mc:`minio server` process using the :mc-cmd:`mc admin config set` command + and the :mc-conf:`notify_amqp` configuration key. You must restart the + :mc:`minio server` process to apply any new or updated configuration + settings. + + The following example code sets *all* settings related to configuring an + AMQP service endpoint. The minimum *required* setting is + :mc-conf:`notify_amqp url `: + + .. code-block:: shell + :class: copyable + + mc admin config set ALIAS/ notify_amqp:IDENTIFIER \ + url="ENDPOINT" \ + exchange="" \ + exchange_type="" \ + routing_key="" \ + mandatory="" \ + durable="" \ + no_wait="" \ + internal="" \ + auto_deleted="" \ + delivery_mode="" \ + queue_dir="" \ + queue_limit="" \ + comment="" + + - Replace ``IDENTIFIER`` with a unique descriptive string for the + AMQP service endpoint. The following examples in this procedure + assume an identifier of ``PRIMARY``. + + If the specified ``IDENTIFIER`` matches an existing AMQP service + endpoint on the MinIO deployment, the new settings *override* + any existing settings for that endpoint. Use + :mc-cmd:`mc admin config get notify_amqp ` to + review the currently configured AMQP endpoints on the MinIO deployment. + + - Replace ``ENDPOINT`` with the URL of the AMQP service endpoint. + For example: + + ``amqp://user:password@hostname:port`` + + See :ref:`AMQP Bucket Notification Configuration Settings + ` for complete + documentation on each setting. + +2) Restart the MinIO Deployment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You must restart the MinIO deployment to apply the configuration changes. +Use the :mc-cmd:`mc admin service restart` command to restart the deployment. + +.. important:: + + MinIO restarts *all* :mc:`minio server` processes associated to the + deployment at the same time. Applications may experience a brief period of + downtime during the restart process. + + Consider scheduling the restart during a maintenance period to minimize + interruption of services. + +.. code-block:: shell + :class: copyable + + mc admin service restart ALIAS + +Replace ``ALIAS`` with the :mc:`alias ` of the deployment to +restart. + +The :mc:`minio server` process prints a line on startup for each configured AMQP +target similar to the following: + +.. code-block:: shell + + SQS ARNs: arn:minio:sqs::primary:amqp + +You must specify the ARN resource when configuring bucket notifications with +the associated AMQP deployment as a target. + +3) Configure Bucket Notifications using the AMQP Endpoint as a Target +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Use the :mc-cmd:`mc event add` command to add a new bucket notification +event with the configured AMQP service as a target: + +.. code-block:: shell + :class: copyable + + mc event add ALIAS/BUCKET arn:minio:sqs::primary:amqp \ + --event EVENTS + +- Replace ``ALIAS`` with the :mc:`alias ` of a MinIO deployment. +- Replace ``BUCKET`` with the name of the bucket in which to configure the + event. +- Replace ``EVENTS`` with a comma-separated list of :ref:`events + ` for which MinIO triggers notifications. + +Use :mc-cmd:`mc event list` to view all configured bucket events for +a given notification target: + +.. code-block:: shell + :class: copyable + + mc event list ALIAS/BUCKET arn:minio:sqs::primary:amqp + +4) Validate the Configured Events +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Perform an action on the bucket for which you configured the new event and +check the AMQP service for the notification data. The action required +depends on which :mc-cmd:`events ` were specified +when configuring the bucket notification. + +For example, if the bucket notification configuration includes the +``s3:ObjectCreated:Put`` event, you can use the +:mc-cmd:`mc cp` command to create a new object in the bucket and trigger +a notification. + +.. code-block:: shell + :class: copyable + + mc cp ~/data/new-object.txt ALIAS/BUCKET + + + +Update an AMQP Endpoint in a MinIO Deployment +--------------------------------------------- + +The following procedure updates an existing AMQP service endpoint for supporting +:ref:`bucket notifications ` in a MinIO +deployment. + +.. include:: /includes/common-admonitions.rst + :start-after: start-restart-downtime + :end-before: end-restart-downtime + +Prerequisites +~~~~~~~~~~~~~~ + +AMQP 0-9-1 Service Endpoint ++++++++++++++++++++++++++++ + +MinIO relies on the :github:`streadway/amqp` project for AMQP connectivity. The +project is primarily tested against `RabbitMQ `__ +deployments, though other `AMQP 0-9-1-compatible `__ +services *may* also work. This procedure *assumes* a RabbitMQ deployment +as the service endpoint. + +If the AMQP service requires authentication, you *must* provide an appropriate +username and password during the configuration process to grant MinIO access +to the service. + +MinIO ``mc`` Command Line Tool +++++++++++++++++++++++++++++++ + +This procedure uses the :mc:`mc` command line tool for certain actions. +See the ``mc`` :ref:`Quickstart ` for installation instructions. + + +1) List Configured AMQP Endpoints In The Deployment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Use the :mc-cmd:`mc admin config get` command to list the currently +configured AMQP service endpoints in the deployment: + +.. code-block:: shell + :class: copyable + + mc admin config get ALIAS/ notify_amqp + +Replace ``ALIAS`` with the :mc:`alias ` of the MinIO deployment. + +The command output resembles the following: + +.. code-block:: shell + + notify_amqp:primary delivery_mode="0" exchange_type="" no_wait="off" queue_dir="" queue_limit="0" url="amqp://user:password@hostname:port" auto_deleted="off" durable="off" exchange="" internal="off" mandatory="off" routing_key="" + notify_amqp:secondary delivery_mode="0" exchange_type="" no_wait="off" queue_dir="" queue_limit="0" url="amqp://user:password@hostname:port" auto_deleted="off" durable="off" exchange="" internal="off" mandatory="off" routing_key="" + +The :mc-conf:`notify_amqp` key is the top-level configuration key for an +:ref:`minio-server-config-bucket-notification-amqp`. The +:mc-conf:`url ` key specifies the AMQP service endpoint +for the given `notify_amqp` key. The ``notify_amqp:`` suffix +describes the unique identifier for that AMQP service endpoint. + +Note the identifier for the AMQP service endpoint you want to update for +the next step. + +2) Update the AMQP Endpoint +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Use the :mc-cmd:`mc admin config set` command to set the new configuration +for the AMQP service endpoint: + +.. code-block:: shell + :class: copyable + + mc admin config set ALIAS/ notify_amqp: \ + url="amqp://user:password@hostname:port" \ + exchange="" \ + exchange_type="" \ + routing_key="" \ + mandatory="" \ + durable="" \ + no_wait="" \ + internal="" \ + auto_deleted="" \ + delivery_mode="" \ + queue_dir="" \ + queue_limit="" \ + comment="" + +The :mc-conf:`notify_amqp url ` configuration setting is the +*minimum* required for an AMQP service endpoint. All other configuration +settings are *optional*. See :ref:`minio-server-config-bucket-notification-amqp` +for a complete list of AMQP configuration settings. + +3) Restart the MinIO Deployment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You must restart the MinIO deployment to apply the configuration changes. +Use the :mc-cmd:`mc admin service restart` command to restart the deployment. + +.. important:: + + MinIO restarts *all* :mc:`minio server` processes associated to the + deployment at the same time. Applications may experience a brief period of + downtime during the restart process. + + Consider scheduling the restart during a maintenance period to minimize + interruption of services. + +.. code-block:: shell + :class: copyable + + mc admin service restart ALIAS + +Replace ``ALIAS`` with the :mc:`alias ` of the deployment to +restart. + +The :mc:`minio server` process prints a line on startup for each configured AMQP +target similar to the following: + +.. code-block:: shell + + SQS ARNs: arn:minio:sqs::primary:amqp + +3) Validate the Changes +~~~~~~~~~~~~~~~~~~~~~~~ + +Perform an action on a bucket which has an event configuration using the updated +AMQP service endpoint and check the AMQP service for the notification data. The +action required depends on which :mc-cmd:`events ` were +specified when configuring the bucket notification. + +For example, if the bucket notification configuration includes the +``s3:ObjectCreated:Put`` event, you can use the +:mc-cmd:`mc cp` command to create a new object in the bucket and trigger +a notification. + +.. code-block:: shell + :class: copyable + + mc cp ~/data/new-object.txt ALIAS/BUCKET diff --git a/source/monitoring/bucket-notifications/publish-events-to-elasticsearch.rst b/source/monitoring/bucket-notifications/publish-events-to-elasticsearch.rst new file mode 100644 index 00000000..7000e700 --- /dev/null +++ b/source/monitoring/bucket-notifications/publish-events-to-elasticsearch.rst @@ -0,0 +1,363 @@ +.. _minio-bucket-notifications-publish-elasticsearch: + +=============================== +Publish Events to Elasticsearch +=============================== + +.. default-domain:: minio + +.. contents:: Table of Contents + :local: + :depth: 1 + +MinIO supports publishing :ref:`bucket notification +` events to an +`Elasticsearch `__ service endpoint. + +MinIO relies on the :github:`olivere/elastic` v7 project for Elastic +connectivity. The ``elastic/v7`` library specifically targets Elasticsearch +v7.0 and is *not compatible with earlier Elasticsearch versions*. + +Add a Elasticsearch Endpoint to a MinIO Deployment +-------------------------------------------------- + +The following procedure adds a new Elasticsearch service endpoint for supporting +:ref:`bucket notifications ` in a MinIO +deployment. + +.. include:: /includes/common-admonitions.rst + :start-after: start-restart-downtime + :end-before: end-restart-downtime + +Prerequisites +~~~~~~~~~~~~~ + +Elasticsearch v7.0 and later +++++++++++++++++++++++++++++ + +MinIO relies on the :github:`olivere/elastic` v7 project for Elastic +connectivity. The ``elastic/v7`` library specifically targets Elasticsearch +v7.0 and is *not compatible with earlier Elasticsearch versions*. + +MinIO ``mc`` Command Line Tool +++++++++++++++++++++++++++++++ + +This procedure uses the :mc:`mc` command line tool for certain actions. +See the ``mc`` :ref:`Quickstart ` for installation instructions. + +1) Add the Elasticsearch Endpoint to MinIO +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can configure a new Elasticsearch service endpoint using either environment variables +*or* by setting runtime configuration settings. + +.. tabs:: + + .. tab:: Environment Variables + + MinIO supports specifying the Elasticsearch service endpoint and associated + configuration settings using + :ref:`environment variables + `. The + :mc:`minio server` process applies the specified settings on its + next startup. + + The following example code sets *all* environment variables + related to configuring an Elasticsearch service endpoint. The minimum + *required* variables are: + + - :envvar:`MINIO_NOTIFY_ELASTICSEARCH_ENABLE` + - :envvar:`MINIO_NOTIFY_ELASTICSEARCH_URL` + - :envvar:`MINIO_NOTIFY_ELASTICSEARCH_INDEX` + - :envvar:`MINIO_NOTIFY_ELASTICSEARCH_FORMAT` + + + .. code-block:: shell + :class: copyable + + set MINIO_NOTIFY_ELASTICSEARCH_ENABLE_="on" + set MINIO_NOTIFY_ELASTICSEARCH_URL_="" + set MINIO_NOTIFY_ELASTICSEARCH_INDEX_="" + set MINIO_NOTIFY_ELASTICSEARCH_FORMAT_="" + set MINIO_NOTIFY_ELASTICSEARCH_USERNAME_="" + set MINIO_NOTIFY_ELASTICSEARCH_PASSWORD_="" + set MINIO_NOTIFY_ELASTICSEARCH_QUEUE_DIR_="" + set MINIO_NOTIFY_ELASTICSEARCH_QUEUE_LIMIT_="" + set MINIO_NOTIFY_ELASTICSEARCH_COMMENT_="" + + - Replace ```` with a unique descriptive string for the + TARGET service endpoint. Use the same ```` value for all + environment variables related to the new target service endpoint. + The following examples assume an identifier of ``PRIMARY``. + + If the specified ```` matches an existing Elasticsearch + service endpoint on the MinIO deployment, the new settings *override* + any existing settings for that endpoint. Use + :mc-cmd:`mc admin config get notify_elasticsearch ` + to review the currently configured Elasticsearch endpoints on the MinIO + deployment. + + - Replace ```` with the URL of the Elasticsearch service endpoint. + For example: + + See :ref:`Elasticsearch Service for Bucket Notifications + ` for complete + documentation on each environment variable. + + .. tab:: Configuration Settings + + MinIO supports adding or updating Elasticsearch endpoints on a running + :mc:`minio server` process using the :mc-cmd:`mc admin config set` command + and the :mc-conf:`notify_elasticsearch` configuration key. You must restart the + :mc:`minio server` process to apply any new or updated configuration + settings. + + The following example code sets *all* settings related to configuring an + Elasticsearch service endpoint. The minimum *required* settings are: + + - :mc-conf:`url ` + - :mc-conf:`index ` + - :mc-conf:`format ` + + .. code-block:: shell + :class: copyable + + mc admin config set ALIAS/ notify_elasticsearch:IDENTIFIER \ + url="ENDPOINT" \ + index="" \ + format="" \ + username="" \ + password="" \ + queue_dir="" \ + queue_limit="" \ + comment="" + + + - Replace ``IDENTIFIER`` with a unique descriptive string for the + Elasticsearch service endpoint. The following examples in this procedure + assume an identifier of ``PRIMARY``. + + If the specified ``IDENTIFIER`` matches an existing Elasticsearch service + endpoint on the MinIO deployment, the new settings *override* + any existing settings for that endpoint. Use + :mc-cmd:`mc admin config get notify_elasticsearch ` + to review the currently configured Elasticsearch endpoints on the MinIO + deployment. + + - Replace ``ENDPOINT`` with the URL of the Elasticsearch service endpoint. + For example: + + ``https://user:password@hostname:port`` + + See :ref:`Elasticsearch Bucket Notification Configuration Settings + ` for complete + documentation on each setting. + +2) Restart the MinIO Deployment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You must restart the MinIO deployment to apply the configuration changes. +Use the :mc-cmd:`mc admin service restart` command to restart the deployment. + +.. important:: + + MinIO restarts *all* :mc:`minio server` processes associated to the + deployment at the same time. Applications may experience a brief period of + downtime during the restart process. + + Consider scheduling the restart during a maintenance period to minimize + interruption of services. + +.. code-block:: shell + :class: copyable + + mc admin service restart ALIAS + +Replace ``ALIAS`` with the :mc:`alias ` of the deployment to +restart. + +The :mc:`minio server` process prints a line on startup for each configured Elasticsearch +target similar to the following: + +.. code-block:: shell + + SQS ARNs: arn:minio:sqs::primary:elasticsearch + +You must specify the ARN resource when configuring bucket notifications with +the associated Elasticsearch deployment as a target. + +3) Configure Bucket Notifications using the Elasticsearch Endpoint as a Target +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Use the :mc-cmd:`mc event add` command to add a new bucket notification +event with the configured Elasticsearch service as a target: + +.. code-block:: shell + :class: copyable + + mc event add ALIAS/BUCKET arn:minio:sqs::primary:elasticsearch \ + --event EVENTS + +- Replace ``ALIAS`` with the :mc:`alias ` of a MinIO deployment. +- Replace ``BUCKET`` with the name of the bucket in which to configure the + event. +- Replace ``EVENTS`` with a comma-separated list of :ref:`events + ` for which MinIO triggers notifications. + +Use :mc-cmd:`mc event list` to view all configured bucket events for +a given notification target: + +.. code-block:: shell + :class: copyable + + mc event list ALIAS/BUCKET arn:minio:sqs::primary:Elasticsearch + +4) Validate the Configured Events +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Perform an action on the bucket for which you configured the new event and +check the Elasticsearch service for the notification data. The action required +depends on which :mc-cmd:`events ` were specified +when configuring the bucket notification. + +For example, if the bucket notification configuration includes the +``s3:ObjectCreated:Put`` event, you can use the +:mc-cmd:`mc cp` command to create a new object in the bucket and trigger +a notification. + +.. code-block:: shell + :class: copyable + + mc cp ~/data/new-object.txt ALIAS/BUCKET + +Update an Elasticsearch Endpoint in a MinIO Deployment +------------------------------------------------------ + +The following procedure updates an existing Elasticsearch service endpoint for supporting +:ref:`bucket notifications ` in a MinIO +deployment. + +.. include:: /includes/common-admonitions.rst + :start-after: start-restart-downtime + :end-before: end-restart-downtime + +Prerequisites +~~~~~~~~~~~~~~ + +Elasticsearch v7.0 and later +++++++++++++++++++++++++++++ + +MinIO relies on the :github:`olivere/elastic` v7 project for Elastic +connectivity. The ``elastic/v7`` library specifically targets Elasticsearch +v7.0 and is *not compatible with earlier Elasticsearch versions*. + +MinIO ``mc`` Command Line Tool +++++++++++++++++++++++++++++++ + +This procedure uses the :mc:`mc` command line tool for certain actions. +See the ``mc`` :ref:`Quickstart ` for installation instructions. + + +1) List Configured Elasticsearch Endpoints In The Deployment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Use the :mc-cmd:`mc admin config get` command to list the currently +configured Elasticsearch service endpoints in the deployment: + +.. code-block:: shell + :class: copyable + + mc admin config get ALIAS/ notify_elasticsearch + +Replace ``ALIAS`` with the :mc:`alias ` of the MinIO deployment. + +The command output resembles the following: + +.. code-block:: shell + + notify_elasticsearch:primary queue_dir="" queue_limit="0" url="https://user:password@hostname:port" format="namespace" index="" + notify_elasticsearch:secondary queue_dir="" queue_limit="0" url="https://user:password@hostname:port" format="namespace" index="" + +The :mc-conf:`notify_elasticsearch` key is the top-level configuration key for +an :ref:`minio-server-config-bucket-notification-elasticsearch`. The +:mc-conf:`url ` key specifies the Elasticsearch +service endpoint for the given `notify_elasticsearch` key. The +``notify_elasticsearch:`` suffix describes the unique identifier for +that Elasticsearch service endpoint. + +Note the identifier for the Elasticsearch service endpoint you want to update for +the next step. + +2) Update the Elasticsearch Endpoint +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Use the :mc-cmd:`mc admin config set` command to set the new configuration +for the Elasticsearch service endpoint: + +.. code-block:: shell + :class: copyable + + mc admin config set ALIAS/ notify_elasticsearch: \ + url="https://user:password@hostname:port" \ + index="" \ + format="" \ + username="" \ + password="" \ + queue_dir="" \ + queue_limit="" \ + comment="" + +The :mc-conf:`notify_elasticsearch url ` configuration +setting is the *minimum* required for an Elasticsearch service endpoint. All +other configuration settings are *optional*. See +:ref:`minio-server-config-bucket-notification-elasticsearch` for a complete list +of Elasticsearch configuration settings. + +3) Restart the MinIO Deployment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You must restart the MinIO deployment to apply the configuration changes. +Use the :mc-cmd:`mc admin service restart` command to restart the deployment. + +.. important:: + + MinIO restarts *all* :mc:`minio server` processes associated to the + deployment at the same time. Applications may experience a brief period of + downtime during the restart process. + + Consider scheduling the restart during a maintenance period to minimize + interruption of services. + +.. code-block:: shell + :class: copyable + + mc admin service restart ALIAS + +Replace ``ALIAS`` with the :mc:`alias ` of the deployment to +restart. + +The :mc:`minio server` process prints a line on startup for each configured +Elasticsearch target similar to the following: + +.. code-block:: shell + + SQS ARNs: arn:minio:sqs::primary:elasticsearch + +3) Validate the Changes +~~~~~~~~~~~~~~~~~~~~~~~ + +Perform an action on a bucket which has an event configuration using the updated +Elasticsearch service endpoint and check the Elasticsearch service for the +notification data. The action required depends on which +:mc-cmd:`events ` were specified when configuring the bucket +notification. + +For example, if the bucket notification configuration includes the +``s3:ObjectCreated:Put`` event, you can use the +:mc-cmd:`mc cp` command to create a new object in the bucket and trigger +a notification. + +.. code-block:: shell + :class: copyable + + mc cp ~/data/new-object.txt ALIAS/BUCKET \ No newline at end of file diff --git a/source/monitoring/bucket-notifications/publish-events-to-kafka.rst b/source/monitoring/bucket-notifications/publish-events-to-kafka.rst new file mode 100644 index 00000000..55bc4fd9 --- /dev/null +++ b/source/monitoring/bucket-notifications/publish-events-to-kafka.rst @@ -0,0 +1,375 @@ +.. _minio-bucket-notifications-publish-kafka: + +======================= +Publish Events to Kafka +======================= + +.. default-domain:: minio + +.. contents:: Table of Contents + :local: + :depth: 1 + +MinIO supports publishing :ref:`bucket notification +` events to a `Kafka `__ +service endpoint. + +MinIO relies on the :github:`Shopify/sarama` project for Kafka connectivity +and shares that project's Kafka support. See the +``sarama`` :github:`Compatibility and API stability +` section for more details. + +Add a Kafka Endpoint to a MinIO Deployment +------------------------------------------ + +The following procedure adds a new Kafka service endpoint for supporting +:ref:`bucket notifications ` in a MinIO +deployment. + +.. include:: /includes/common-admonitions.rst + :start-after: start-restart-downtime + :end-before: end-restart-downtime + +Prerequisites +~~~~~~~~~~~~~ + +Kafka Minimum Versions and Supported Versions ++++++++++++++++++++++++++++++++++++++++++++++ + +MinIO relies on the :github:`Shopify/sarama` project for Kafka connectivity +and shares that project's Kafka support. See the +``sarama`` :github:`Compatibility and API stability +` section for more details. + +MinIO ``mc`` Command Line Tool +++++++++++++++++++++++++++++++ + +This procedure uses the :mc:`mc` command line tool for certain actions. +See the ``mc`` :ref:`Quickstart ` for installation instructions. + +1) Add the Kafka Endpoint to MinIO +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can configure a new Kafka service endpoint using either environment variables +*or* by setting runtime configuration settings. + +.. tabs:: + + .. tab:: Environment Variables + + MinIO supports specifying the Kafka service endpoint and associated + configuration settings using + :ref:`environment variables + `. The + :mc:`minio server` process applies the specified settings on its + next startup. + + The following example code sets *all* environment variables + related to configuring a Kafka service endpoint. The minimum + *required* variables are + :envvar:`MINIO_NOTIFY_KAFKA_ENABLE` and + :envvar:`MINIO_NOTIFY_KAFKA_BROKERS`: + + .. code-block:: shell + :class: copyable + + set MINIO_NOTIFY_KAFKA_ENABLE_="on" + set MINIO_NOTIFY_KAFKA_BROKERS_="" + set MINIO_NOTIFY_KAFKA_TOPIC_="" + set MINIO_NOTIFY_KAFKA_SASL_USERNAME_="" + set MINIO_NOTIFY_KAFKA_SASL_PASSWORD_="" + set MINIO_NOTIFY_KAFKA_SASL_MECHANISM_="" + set MINIO_NOTIFY_KAFKA_TLS_CLIENT_AUTH_="" + set MINIO_NOTIFY_KAFKA_SASL_="" + set MINIO_NOTIFY_KAFKA_TLS_="" + set MINIO_NOTIFY_KAFKA_TLS_SKIP_VERIFY_="" + set MINIO_NOTIFY_KAFKA_CLIENT_TLS_CERT_="" + set MINIO_NOTIFY_KAFKA_CLIENT_TLS_KEY_="" + set MINIO_NOTIFY_KAFKA_QUEUE_DIR_="" + set MINIO_NOTIFY_KAFKA_QUEUE_LIMIT_="" + set MINIO_NOTIFY_KAFKA_VERSION_="" + set MINIO_NOTIFY_KAFKA_COMMENT_="" + + - Replace ```` with a unique descriptive string for the + Kafka service endpoint. Use the same ```` value for all + environment variables related to the new target service endpoint. + The following examples assume an identifier of ``PRIMARY``. + + If the specified ```` matches an existing Kafka service + endpoint on the MinIO deployment, the new settings *override* + any existing settings for that endpoint. Use + :mc-cmd:`mc admin config get notify_kafka ` to + review the currently configured Kafka endpoints on the MinIO deployment. + + - Replace ```` with a comma-separated list of Kafka brokers. + For example: + + ``"kafka1.example.com:2021,kafka2.example.com:2021"`` + + See :ref:`Kafka Service for Bucket Notifications + ` for complete documentation + on each environment variable. + + .. tab:: Configuration Settings + + MinIO supports adding or updating Kafka endpoints on a running + :mc:`minio server` process using the :mc-cmd:`mc admin config set` command + and the :mc-conf:`notify_kafka` configuration key. You must restart the + :mc:`minio server` process to apply any new or updated configuration + settings. + + The following example code sets *all* settings related to configuring an + Kafka service endpoint. The minimum *required* setting is + :mc-conf:`notify_kafka brokers `: + + .. code-block:: shell + :class: copyable + + mc admin config set ALIAS/ notify_kafka:IDENTIFIER \ + brokers="" \ + topic="" \ + sasl_username="" \ + sasl_password="" \ + sasl_mechanism="" \ + tls_client_auth="" \ + sasl="" \ + tls="" \ + tls_skip_verify="" \ + client_tls_cert="" \ + client_tls_key="" \ + version="" \ + queue_dir="" \ + queue_limit="" \ + comment="" + + - Replace ``IDENTIFIER`` with a unique descriptive string for the + Kafka service endpoint. The following examples in this procedure + assume an identifier of ``PRIMARY``. + + If the specified ``IDENTIFIER`` matches an existing Kafka service + endpoint on the MinIO deployment, the new settings *override* + any existing settings for that endpoint. Use + :mc-cmd:`mc admin config get notify_kafka ` to + review the currently configured Kafka endpoints on the MinIO deployment. + + - Replace ``ENDPOINT`` with a comma separated list of Kafka brokers. + For example: + + ``"kafka1.example.com:2021,kafka2.example.com:2021"`` + + See :ref:`Kafka Bucket Notification Configuration Settings + ` for complete + documentation on each setting. + +2) Restart the MinIO Deployment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You must restart the MinIO deployment to apply the configuration changes. +Use the :mc-cmd:`mc admin service restart` command to restart the deployment. + +.. important:: + + MinIO restarts *all* :mc:`minio server` processes associated to the + deployment at the same time. Applications may experience a brief period of + downtime during the restart process. + + Consider scheduling the restart during a maintenance period to minimize + interruption of services. + +.. code-block:: shell + :class: copyable + + mc admin service restart ALIAS + +Replace ``ALIAS`` with the :mc:`alias ` of the deployment to +restart. + +The :mc:`minio server` process prints a line on startup for each configured +Kafka target similar to the following: + +.. code-block:: shell + + SQS ARNs: arn:minio:sqs::primary:kafka + +You must specify the ARN resource when configuring bucket notifications with +the associated Kafka deployment as a target. + +3) Configure Bucket Notifications using the Kafka Endpoint as a Target +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Use the :mc-cmd:`mc event add` command to add a new bucket notification +event with the configured Kafka service as a target: + +.. code-block:: shell + :class: copyable + + mc event add ALIAS/BUCKET arn:minio:sqs::primary:kafka \ + --event EVENTS + +- Replace ``ALIAS`` with the :mc:`alias ` of a MinIO deployment. +- Replace ``BUCKET`` with the name of the bucket in which to configure the + event. +- Replace ``EVENTS`` with a comma-separated list of :ref:`events + ` for which MinIO triggers notifications. + +Use :mc-cmd:`mc event list` to view all configured bucket events for +a given notification target: + +.. code-block:: shell + :class: copyable + + mc event list ALIAS/BUCKET arn:minio:sqs::primary:kafka + +4) Validate the Configured Events +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Perform an action on the bucket for which you configured the new event and +check the Kafka service for the notification data. The action required +depends on which :mc-cmd:`events ` were specified +when configuring the bucket notification. + +For example, if the bucket notification configuration includes the +``s3:ObjectCreated:Put`` event, you can use the +:mc-cmd:`mc cp` command to create a new object in the bucket and trigger +a notification. + +.. code-block:: shell + :class: copyable + + mc cp ~/data/new-object.txt ALIAS/BUCKET + +Update a Kafka Endpoint in a MinIO Deployment +--------------------------------------------- + +The following procedure updates an existing Kafka service endpoint for supporting +:ref:`bucket notifications ` in a MinIO +deployment. + +.. include:: /includes/common-admonitions.rst + :start-after: start-restart-downtime + :end-before: end-restart-downtime + +Prerequisites +~~~~~~~~~~~~~~ + +Kafka Minimum Versions and Supported Versions ++++++++++++++++++++++++++++++++++++++++++++++ + +MinIO relies on the :github:`Shopify/sarama` project for Kafka connectivity +and shares that project's Kafka support. See the +``sarama`` :github:`Compatibility and API stability +` section for more details. + +MinIO ``mc`` Command Line Tool +++++++++++++++++++++++++++++++ + +This procedure uses the :mc:`mc` command line tool for certain actions. +See the ``mc`` :ref:`Quickstart ` for installation instructions. + + +1) List Configured Kafka Endpoints In The Deployment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Use the :mc-cmd:`mc admin config get` command to list the currently +configured Kafka service endpoints in the deployment: + +.. code-block:: shell + :class: copyable + + mc admin config get ALIAS/ notify_kafka + +Replace ``ALIAS`` with the :mc:`alias ` of the MinIO deployment. + +The command output resembles the following: + +.. code-block:: shell + + notify_kafka:primary tls_skip_verify="off" queue_dir="" queue_limit="0" sasl="off" sasl_password="" sasl_username="" tls_client_auth="0" tls="off" brokers="" topic="" client_tls_cert="" client_tls_key="" version="" + notify_kafka:secondary tls_skip_verify="off" queue_dir="" queue_limit="0" sasl="off" sasl_password="" sasl_username="" tls_client_auth="0" tls="off" brokers="" topic="" client_tls_cert="" client_tls_key="" version="" + +The :mc-conf:`notify_kafka` key is the top-level configuration key for an +:ref:`minio-server-config-bucket-notification-kafka`. The +:mc-conf:`brokers ` key specifies the Kafka service +endpoint for the given `notify_kafka` key. The ``notify_kafka:`` +suffix describes the unique identifier for that Kafka service endpoint. + +Note the identifier for the Kafka service endpoint you want to update for +the next step. + +2) Update the Kafka Endpoint +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Use the :mc-cmd:`mc admin config set` command to set the new configuration +for the Kafka service endpoint: + +.. code-block:: shell + :class: copyable + + mc admin config set ALIAS/ notify_kafka: \ + url="https://kafka1.example.net:9200, https://kafka2.example.net:9200" \ + exchange="" \ + exchange_type="" \ + routing_key="" \ + mandatory="" \ + durable="" \ + no_wait="" \ + internal="" \ + auto_deleted="" \ + delivery_mode="" \ + queue_dir="" \ + queue_limit="" \ + comment="" + +The :mc-conf:`notify_kafka brokers ` configuration setting +is the *minimum* required for a Kafka service endpoint. All other configuration +settings are *optional*. See +:ref:`minio-server-config-bucket-notification-kafka` for a complete list of +Kafka configuration settings. + +3) Restart the MinIO Deployment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You must restart the MinIO deployment to apply the configuration changes. +Use the :mc-cmd:`mc admin service restart` command to restart the deployment. + +.. important:: + + MinIO restarts *all* :mc:`minio server` processes associated to the + deployment at the same time. Applications may experience a brief period of + downtime during the restart process. + + Consider scheduling the restart during a maintenance period to minimize + interruption of services. + +.. code-block:: shell + :class: copyable + + mc admin service restart ALIAS + +Replace ``ALIAS`` with the :mc:`alias ` of the deployment to +restart. + +The :mc:`minio server` process prints a line on startup for each configured +Kafka target similar to the following: + +.. code-block:: shell + + SQS ARNs: arn:minio:sqs::primary:kafka + +3) Validate the Changes +~~~~~~~~~~~~~~~~~~~~~~~ + +Perform an action on a bucket which has an event configuration using the updated +Kafka service endpoint and check the Kafka service for the notification data. +The action required depends on which :mc-cmd:`events ` were +specified when configuring the bucket notification. + +For example, if the bucket notification configuration includes the +``s3:ObjectCreated:Put`` event, you can use the +:mc-cmd:`mc cp` command to create a new object in the bucket and trigger +a notification. + +.. code-block:: shell + :class: copyable + + mc cp ~/data/new-object.txt ALIAS/BUCKET \ No newline at end of file diff --git a/source/monitoring/bucket-notifications/publish-events-to-mqtt.rst b/source/monitoring/bucket-notifications/publish-events-to-mqtt.rst new file mode 100644 index 00000000..3ffd3d2f --- /dev/null +++ b/source/monitoring/bucket-notifications/publish-events-to-mqtt.rst @@ -0,0 +1,389 @@ +.. _minio-bucket-notifications-publish-mqtt: + +====================== +Publish Events to MQTT +====================== + +.. default-domain:: minio + +.. contents:: Table of Contents + :local: + :depth: 1 + +MinIO supports publishing :ref:`bucket notification +` events to `MQTT `__ +server/broker endpoint. + +Add an MQTT Endpoint to a MinIO Deployment +------------------------------------------ + +The following procedure adds a new MQTT service endpoint for supporting +:ref:`bucket notifications ` in a MinIO +deployment. + +.. include:: /includes/common-admonitions.rst + :start-after: start-restart-downtime + :end-before: end-restart-downtime + +Prerequisites +~~~~~~~~~~~~~~ + +MQTT 3.1 or 3.1.1 Server/Broker ++++++++++++++++++++++++++++++++ + +This procedure assumes an existing MQTT 3.1 or 3.1.1 server/broker to which the +MinIO deployment has connectivity. See the +`mqtt.org software listing `__ for a list of +MQTT-compatible server/brokers. + +If the MQTT service requires authentication, you *must* provide an appropriate +username and password during the configuration process to grant MinIO access +to the service. + +MinIO ``mc`` Command Line Tool +++++++++++++++++++++++++++++++ + +This procedure uses the :mc:`mc` command line tool for certain actions. +See the ``mc`` :ref:`Quickstart ` for installation instructions. + +1) Add the MQTT Endpoint to MinIO +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can configure a new MQTT service endpoint using either environment variables +*or* by setting runtime configuration settings. + +.. tabs:: + + .. tab:: Environment Variables + + MinIO supports specifying the MQTT service endpoint and associated + configuration settings using + :ref:`environment variables + `. The + :mc:`minio server` process applies the specified settings on its + next startup. + + The following example code sets *all* environment variables + related to configuring an MQTT service endpoint. The minimum *required* + variables are: + + - :envvar:`MINIO_NOTIFY_MQTT_ENABLE` + - :envvar:`MINIO_NOTIFY_MQTT_BROKER` + - :envvar:`MINIO_NOTIFY_MQTT_TOPIC` + - :envvar:`MINIO_NOTIFY_MQTT_USERNAME` *Required if the MQTT server/broker enforces authentication/authorization* + - :envvar:`MINIO_NOTIFY_MQTT_PASSWORD` *Required if the MQTT server/broker enforces authentication/authorization* + + .. code-block:: shell + :class: copyable + + set MINIO_NOTIFY_MQTT_ENABLE_="on" + set MINIO_NOTIFY_MQTT_BROKER_="ENDPOINT" + set MINIO_NOTIFY_MQTT_TOPIC_="TOPIC" + set MINIO_NOTIFY_MQTT_USERNAME_="" + set MINIO_NOTIFY_MQTT_PASSWORD_="" + set MINIO_NOTIFY_MQTT_QOS_="" + set MINIO_NOTIFY_MQTT_KEEP_ALIVE_INTERVAL_="" + set MINIO_NOTIFY_MQTT_RECONNECT_INTERVAL_="" + set MINIO_NOTIFY_MQTT_QUEUE_DIR_="" + set MINIO_NOTIFY_MQTT_QUEUE_LIMIT_="" + set MINIO_NOTIFY_MQTT_COMMENT_="" + + - Replace ```` with a unique descriptive string for the + MQTT service endpoint. Use the same ```` value for all + environment variables related to the new MQTT service endpoint. + The following examples assume an identifier of ``PRIMARY``. + + If the specified ```` matches an existing MQTT service + endpoint on the MinIO deployment, the new settings *override* + any existing settings for that endpoint. Use + :mc-cmd:`mc admin config get notify_mqtt ` to + review the currently configured MQTT endpoints on the MinIO deployment. + + - Replace ```` with the URL of the MQTT service endpoint. + For example: + + ``tcp://hostname:port`` + + - Replace ``TOPIC`` with the MQTT topic to which MinIO associates + events published to the server/broker. + + See :ref:`MQTT Service for Bucket Notifications + ` for complete documentation + on each environment variable. + + .. tab:: Configuration Settings + + MinIO supports adding or updating MQTT endpoints on a running + :mc:`minio server` process using the :mc-cmd:`mc admin config set` command + and the :mc-conf:`notify_mqtt` configuration key. You must restart the + :mc:`minio server` process to apply any new or updated configuration + settings. + + The following example code sets *all* settings related to configuring an + MQTT service endpoint. The following configuration settings are the + *minimum* required for an MQTT server/broker endpoint: + + - :mc-conf:`~notify_mqtt.broker` + - :mc-conf:`~notify_mqtt.topic` + - :mc-conf:`~notify_mqtt.username` *Required if the MQTT server/broker enforces authentication/authorization* + - :mc-conf:`~notify_mqtt.password` *Required if the MQTT server/broker enforces authentication/authorization* + + .. code-block:: shell + :class: copyable + + mc admin config set ALIAS/ notify_mqtt:IDENTIFIER \ + broker="ENDPOINT" \ + topic="TOPIC" \ + username="username" \ + password="password" \ + qos="" \ + keep_alive_interval="60s|m|h|d" + reconnect_interval="60s|m|h|d" + queue_dir="" \ + queue_limit="" \ + comment="" + + - Replace ``IDENTIFIER`` with a unique descriptive string for the + MQTT service endpoint. The following examples in this procedure + assume an identifier of ``PRIMARY``. + + If the specified ``IDENTIFIER`` matches an existing MQTT service + endpoint on the MinIO deployment, the new settings *override* + any existing settings for that endpoint. Use + :mc-cmd:`mc admin config get notify_mqtt ` to + review the currently configured MQTT endpoints on the MinIO deployment. + + - Replace ``ENDPOINT`` with the URL of the MQTT service endpoint. + For example: + + ``tcp://hostname:port`` + + - Replace ``TOPIC`` with the MQTT topic to which MinIO associates + events published to the server/broker. + + See :ref:`MQTT Bucket Notification Configuration Settings + ` for complete + documentation on each setting. + +2) Restart the MinIO Deployment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You must restart the MinIO deployment to apply the configuration changes. +Use the :mc-cmd:`mc admin service restart` command to restart the deployment. + +.. important:: + + MinIO restarts *all* :mc:`minio server` processes associated to the + deployment at the same time. Applications may experience a brief period of + downtime during the restart process. + + Consider scheduling the restart during a maintenance period to minimize + interruption of services. + +.. code-block:: shell + :class: copyable + + mc admin service restart ALIAS + +Replace ``ALIAS`` with the :mc:`alias ` of the deployment to +restart. + +The :mc:`minio server` process prints a line on startup for each configured MQTT +target similar to the following: + +.. code-block:: shell + + SQS ARNs: arn:minio:sqs::primary:mqtt + +You must specify the ARN resource when configuring bucket notifications with +the associated MQTT deployment as a target. + +3) Configure Bucket Notifications using the MQTT Endpoint as a Target +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Use the :mc-cmd:`mc event add` command to add a new bucket notification +event with the configured MQTT service as a target: + +.. code-block:: shell + :class: copyable + + mc event add ALIAS/BUCKET arn:minio:sqs::primary:mqtt \ + --event EVENTS + +- Replace ``ALIAS`` with the :mc:`alias ` of a MinIO deployment. +- Replace ``BUCKET`` with the name of the bucket in which to configure the + event. +- Replace ``EVENTS`` with a comma-separated list of :ref:`events + ` for which MinIO triggers notifications. + +Use :mc-cmd:`mc event list` to view all configured bucket events for +a given notification target: + +.. code-block:: shell + :class: copyable + + mc event list ALIAS/BUCKET arn:minio:sqs::primary:MQTT + +4) Validate the Configured Events +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Perform an action on the bucket for which you configured the new event and +check the MQTT service for the notification data. The action required +depends on which :mc-cmd:`events ` were specified +when configuring the bucket notification. + +For example, if the bucket notification configuration includes the +``s3:ObjectCreated:Put`` event, you can use the +:mc-cmd:`mc cp` command to create a new object in the bucket and trigger +a notification. + +.. code-block:: shell + :class: copyable + + mc cp ~/data/new-object.txt ALIAS/BUCKET + +Update an MQTT Endpoint in a MinIO Deployment +--------------------------------------------- + +The following procedure updates an existing MQTT service endpoint for supporting +:ref:`bucket notifications ` in a MinIO +deployment. + +.. include:: /includes/common-admonitions.rst + :start-after: start-restart-downtime + :end-before: end-restart-downtime + +Prerequisites +~~~~~~~~~~~~~~ + +MQTT 3.1 or 3.1.1 Server/Broker Endpoint +++++++++++++++++++++++++++++++++++++++++ + +This procedure assumes an existing MQTT 3.1 or 3.1.1 server/broker to which the +MinIO deployment has connectivity. See the +`mqtt.org software listing `__ for a list of +MQTT-compatible server/brokers. + +If the MQTT service requires authentication, you *must* provide an appropriate +username and password during the configuration process to grant MinIO access +to the service. + +MinIO ``mc`` Command Line Tool +++++++++++++++++++++++++++++++ + +This procedure uses the :mc:`mc` command line tool for certain actions. +See the ``mc`` :ref:`Quickstart ` for installation instructions. + + +1) List Configured MQTT Endpoints In The Deployment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Use the :mc-cmd:`mc admin config get` command to list the currently +configured MQTT service endpoints in the deployment: + +.. code-block:: shell + :class: copyable + + mc admin config get ALIAS/ notify_mqtt + +Replace ``ALIAS`` with the :mc:`alias ` of the MinIO deployment. + +The command output resembles the following: + +.. code-block:: shell + + notify_mqtt:primary broker="tcp://mqtt-primary.example.net:port" password="" queue_dir="" queue_limit="0" reconnect_interval="0s" keep_alive_interval="0s" qos="0" topic="" username="" + notify_mqtt:secondary broker="tcp://mqtt-primary.example.net:port" password="" queue_dir="" queue_limit="0" reconnect_interval="0s" keep_alive_interval="0s" qos="0" topic="" username="" + +The :mc-conf:`notify_mqtt` key is the top-level configuration key for an +:ref:`minio-server-config-bucket-notification-mqtt`. The +:mc-conf:`broker ` key specifies the MQTT server/broker endpoint +for the given `notify_mqtt` key. The ``notify_mqtt:`` suffix +describes the unique identifier for that MQTT service endpoint. + +Note the identifier for the MQTT service endpoint you want to update for +the next step. + +2) Update the MQTT Endpoint +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Use the :mc-cmd:`mc admin config set` command to set the new configuration +for the MQTT service endpoint: + +.. code-block:: shell + :class: copyable + + mc admin config set ALIAS/ notify_mqtt: \ + url="MQTT://user:password@hostname:port" \ + exchange="" \ + exchange_type="" \ + routing_key="" \ + mandatory="" \ + durable="" \ + no_wait="" \ + internal="" \ + auto_deleted="" \ + delivery_mode="" \ + queue_dir="" \ + queue_limit="" \ + comment="" + +The following configuration settings are the *minimum* required for an +MQTT server/broker endpoint: + +- :mc-conf:`~notify_mqtt.broker` +- :mc-conf:`~notify_mqtt.topic` +- :mc-conf:`~notify_mqtt.username` *Required if the MQTT server/broker enforces authentication/authorization* +- :mc-conf:`~notify_mqtt.password` *Required if the MQTT server/broker enforces authentication/authorization* + +All other configuration settings are *optional*. See +:ref:`minio-server-config-bucket-notification-mqtt` for a complete list of MQTT +configuration settings. + +3) Restart the MinIO Deployment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You must restart the MinIO deployment to apply the configuration changes. +Use the :mc-cmd:`mc admin service restart` command to restart the deployment. + +.. important:: + + MinIO restarts *all* :mc:`minio server` processes associated to the + deployment at the same time. Applications may experience a brief period of + downtime during the restart process. + + Consider scheduling the restart during a maintenance period to minimize + interruption of services. + +.. code-block:: shell + :class: copyable + + mc admin service restart ALIAS + +Replace ``ALIAS`` with the :mc:`alias ` of the deployment to +restart. + +The :mc:`minio server` process prints a line on startup for each configured MQTT +target similar to the following: + +.. code-block:: shell + + SQS ARNs: arn:minio:sqs::primary:mqtt + +3) Validate the Changes +~~~~~~~~~~~~~~~~~~~~~~~ + +Perform an action on a bucket which has an event configuration using the updated +MQTT service endpoint and check the MQTT service for the notification data. The +action required depends on which :mc-cmd:`events ` were +specified when configuring the bucket notification. + +For example, if the bucket notification configuration includes the +``s3:ObjectCreated:Put`` event, you can use the +:mc-cmd:`mc cp` command to create a new object in the bucket and trigger +a notification. + +.. code-block:: shell + :class: copyable + + mc cp ~/data/new-object.txt ALIAS/BUCKET diff --git a/source/monitoring/bucket-notifications/publish-events-to-mysql.rst b/source/monitoring/bucket-notifications/publish-events-to-mysql.rst new file mode 100644 index 00000000..ee8b5132 --- /dev/null +++ b/source/monitoring/bucket-notifications/publish-events-to-mysql.rst @@ -0,0 +1,362 @@ +.. _minio-bucket-notifications-publish-mysql: + +======================= +Publish Events to MySQL +======================= + +.. default-domain:: minio + +.. contents:: Table of Contents + :local: + :depth: 1 + +MinIO supports publishing :ref:`bucket notification +` events to a My`MySQL `__ +service endpoint. MinIO supports MySQL 5.7.8 and later *only*. + +Add a MySQL Endpoint to a MinIO Deployment +------------------------------------------ + +The following procedure adds a new MySQL service endpoint for supporting +:ref:`bucket notifications ` in a MinIO +deployment. + +.. include:: /includes/common-admonitions.rst + :start-after: start-restart-downtime + :end-before: end-restart-downtime + +Prerequisites +~~~~~~~~~~~~~ + +MySQL 5.7.8 and later ++++++++++++++++++++++ + +MinIO relies on features introduced with MySQL 5.7.8. + +MinIO ``mc`` Command Line Tool +++++++++++++++++++++++++++++++ + +This procedure uses the :mc:`mc` command line tool for certain actions. +See the ``mc`` :ref:`Quickstart ` for installation instructions. + +1) Add the MySQL Endpoint to MinIO +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can configure a new MySQL service endpoint using either environment variables +*or* by setting runtime configuration settings. + +.. tabs:: + + .. tab:: Environment Variables + + MinIO supports specifying the MySQL service endpoint and associated + configuration settings using + :ref:`environment variables + `. The + :mc:`minio server` process applies the specified settings on its + next startup. + + The following example code sets *all* environment variables + related to configuring a MySQL service endpoint. The minimum + *required* variables are: + + - :envvar:`MINIO_NOTIFY_MYSQL_ENABLE` + - :envvar:`MINIO_NOTIFY_MYSQL_DSN_STRING` + - :envvar:`MINIO_NOTIFY_MYSQL_TABLE` + - :envvar:`MINIO_NOTIFY_MYSQL_FORMAT` + + .. code-block:: shell + :class: copyable + + set MINIO_NOTIFY_MYSQL_DSN_STRING_="on" + set MINIO_NOTIFY_MYSQL_TABLE_="" + set MINIO_NOTIFY_MYSQL_FORMAT_="" + set MINIO_NOTIFY_MYSQL_MAX_OPEN_CONNECTIONS_="" + set MINIO_NOTIFY_MYSQL_QUEUE_DIR_="" + set MINIO_NOTIFY_MYSQL_QUEUE_LIMIT_="" + set MINIO_NOTIFY_MYSQL_COMMENT_="" + + - Replace ```` with a unique descriptive string for the + MySQL service endpoint. Use the same ```` value for all + environment variables related to the new target service endpoint. + The following examples assume an identifier of ``PRIMARY``. + + If the specified ```` matches an existing MySQL service + endpoint on the MinIO deployment, the new settings *override* + any existing settings for that endpoint. Use + :mc-cmd:`mc admin config get notify_mysql ` to + review the currently configured MySQL endpoints on the MinIO deployment. + + - Replace ```` with the DSN of the MySQL service endpoint. + MinIO expects the following format: + + ``:@tcp(:)/`` + + For example: + + ``"username:password@tcp(mysql.example.com:3306)/miniodb"`` + + + See :ref:`MySQL Service for Bucket Notifications + ` for complete documentation + on each environment variable. + + .. tab:: Configuration Settings + + MinIO supports adding or updating MySQL endpoints on a running + :mc:`minio server` process using the :mc-cmd:`mc admin config set` command + and the :mc-conf:`notify_mysql` configuration key. You must restart the + :mc:`minio server` process to apply any new or updated configuration + settings. + + The following example code sets *all* settings related to configuring an + MySQL service endpoint. The minimum *required* settings are: + + - :mc-conf:`notify_mysql dsn_string ` + - :mc-conf:`notify_mysql table ` + - :mc-conf:`notify_mysql format ` + + + .. code-block:: shell + :class: copyable + + mc admin config set ALIAS/ notify_mysql:IDENTIFIER \ + dsn_string="" \ + table="" \ + format="" \ + max_open_connections="" \ + queue_dir="" \ + queue_limit="" \ + comment="" + + - Replace ``IDENTIFIER`` with a unique descriptive string for the + MySQL service endpoint. The following examples in this procedure + assume an identifier of ``PRIMARY``. + + If the specified ``IDENTIFIER`` matches an existing MySQL service + endpoint on the MinIO deployment, the new settings *override* + any existing settings for that endpoint. Use + :mc-cmd:`mc admin config get notify_mysql ` to + review the currently configured MySQL endpoints on the MinIO deployment. + + - Replace ```` with the DSN of the MySQL service endpoint. + MinIO expects the following format: + + ``:@tcp(:)/`` + + For example: + + ``"username:password@tcp(mysql.example.com:3306)/miniodb"`` + + See :ref:`MySQL Bucket Notification Configuration Settings + ` for complete + documentation on each setting. + +2) Restart the MinIO Deployment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You must restart the MinIO deployment to apply the configuration changes. +Use the :mc-cmd:`mc admin service restart` command to restart the deployment. + +.. important:: + + MinIO restarts *all* :mc:`minio server` processes associated to the + deployment at the same time. Applications may experience a brief period of + downtime during the restart process. + + Consider scheduling the restart during a maintenance period to minimize + interruption of services. + +.. code-block:: shell + :class: copyable + + mc admin service restart ALIAS + +Replace ``ALIAS`` with the :mc:`alias ` of the deployment to +restart. + +The :mc:`minio server` process prints a line on startup for each configured MySQL +target similar to the following: + +.. code-block:: shell + + SQS ARNs: arn:minio:sqs::primary:mysql + +You must specify the ARN resource when configuring bucket notifications with +the associated MySQL deployment as a target. + +3) Configure Bucket Notifications using the MySQL Endpoint as a Target +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Use the :mc-cmd:`mc event add` command to add a new bucket notification +event with the configured MySQL service as a target: + +.. code-block:: shell + :class: copyable + + mc event add ALIAS/BUCKET arn:minio:sqs::primary:mysql \ + --event EVENTS + +- Replace ``ALIAS`` with the :mc:`alias ` of a MinIO deployment. +- Replace ``BUCKET`` with the name of the bucket in which to configure the + event. +- Replace ``EVENTS`` with a comma-separated list of :ref:`events + ` for which MinIO triggers notifications. + +Use :mc-cmd:`mc event list` to view all configured bucket events for +a given notification target: + +.. code-block:: shell + :class: copyable + + mc event list ALIAS/BUCKET arn:minio:sqs::primary:mysql + +4) Validate the Configured Events +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Perform an action on the bucket for which you configured the new event and +check the MySQL service for the notification data. The action required +depends on which :mc-cmd:`events ` were specified +when configuring the bucket notification. + +For example, if the bucket notification configuration includes the +``s3:ObjectCreated:Put`` event, you can use the +:mc-cmd:`mc cp` command to create a new object in the bucket and trigger +a notification. + +.. code-block:: shell + :class: copyable + + mc cp ~/data/new-object.txt ALIAS/BUCKET + +Update a MySQL Endpoint in a MinIO Deployment +--------------------------------------------- + +The following procedure updates an existing MySQL service endpoint for supporting +:ref:`bucket notifications ` in a MinIO +deployment. + +.. include:: /includes/common-admonitions.rst + :start-after: start-restart-downtime + :end-before: end-restart-downtime + +Prerequisites +~~~~~~~~~~~~~~ + +MySQL 5.7.8 and later ++++++++++++++++++++++ + +MinIO relies on features introduced with MySQL 5.7.8. + +MinIO ``mc`` Command Line Tool +++++++++++++++++++++++++++++++ + +This procedure uses the :mc:`mc` command line tool for certain actions. +See the ``mc`` :ref:`Quickstart ` for installation instructions. + + +1) List Configured MySQL Endpoints In The Deployment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Use the :mc-cmd:`mc admin config get` command to list the currently +configured MySQL service endpoints in the deployment: + +.. code-block:: shell + :class: copyable + + mc admin config get ALIAS/ notify_mysql + +Replace ``ALIAS`` with the :mc:`alias ` of the MinIO deployment. + +The command output resembles the following: + +.. code-block:: shell + + notify_mysql:primary format="namespace" table="minio_images" dsn_string="user:pass@tcp(mysql.example.com:3306)/miniodb" + notify_mysql:secondary format="namespace" table="minio_images" dsn_string="user:pass@tcp(mysql.example.com:3306)/miniodb" + +The :mc-conf:`notify_mysql` key is the top-level configuration key for an +:ref:`minio-server-config-bucket-notification-mysql`. The +:mc-conf:`dsn_string ` key specifies the MySQL service +endpoint for the given `notify_mysql` key. The ``notify_mysql:`` +suffix describes the unique identifier for that MySQL service endpoint. + +Note the identifier for the MySQL service endpoint you want to update for +the next step. + +2) Update the MySQL Endpoint +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Use the :mc-cmd:`mc admin config set` command to set the new configuration +for the MySQL service endpoint: + +.. code-block:: shell + :class: copyable + + mc admin config set ALIAS/ notify_mysql:IDENTIFIER \ + dsn_string="" \ + table="" \ + format="" \ + max_open_connections="" \ + queue_dir="" \ + queue_limit="" \ + comment="" + +The following configuration settings are the *minimum required* for a MySQL +service endpoint: + +- :mc-conf:`notify_mysql dsn_string ` +- :mc-conf:`notify_mysql table ` +- :mc-conf:`notify_mysql format ` + +All other configuration settings are *optional*. See +:ref:`minio-server-config-bucket-notification-mysql` for a complete list of +MySQL configuration settings. + +3) Restart the MinIO Deployment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You must restart the MinIO deployment to apply the configuration changes. +Use the :mc-cmd:`mc admin service restart` command to restart the deployment. + +.. important:: + + MinIO restarts *all* :mc:`minio server` processes associated to the + deployment at the same time. Applications may experience a brief period of + downtime during the restart process. + + Consider scheduling the restart during a maintenance period to minimize + interruption of services. + +.. code-block:: shell + :class: copyable + + mc admin service restart ALIAS + +Replace ``ALIAS`` with the :mc:`alias ` of the deployment to +restart. + +The :mc:`minio server` process prints a line on startup for each configured MySQL +target similar to the following: + +.. code-block:: shell + + SQS ARNs: arn:minio:sqs::primary:mysql + +3) Validate the Changes +~~~~~~~~~~~~~~~~~~~~~~~ + +Perform an action on a bucket which has an event configuration using the updated +MySQL service endpoint and check the MySQL service for the notification data. The +action required depends on which :mc-cmd:`events ` were +specified when configuring the bucket notification. + +For example, if the bucket notification configuration includes the +``s3:ObjectCreated:Put`` event, you can use the +:mc-cmd:`mc cp` command to create a new object in the bucket and trigger +a notification. + +.. code-block:: shell + :class: copyable + + mc cp ~/data/new-object.txt ALIAS/BUCKET \ No newline at end of file diff --git a/source/monitoring/bucket-notifications/publish-events-to-nats.rst b/source/monitoring/bucket-notifications/publish-events-to-nats.rst new file mode 100644 index 00000000..5c3b89a8 --- /dev/null +++ b/source/monitoring/bucket-notifications/publish-events-to-nats.rst @@ -0,0 +1,362 @@ +.. _minio-bucket-notifications-publish-nats: + +====================== +Publish Events to NATS +====================== + +.. default-domain:: minio + +.. contents:: Table of Contents + :local: + :depth: 1 + +MinIO supports publishing :ref:`bucket notification +` events to a +`NATS `__ service endpoint. + +Add a NATS Endpoint to a MinIO Deployment +----------------------------------------- + +The following procedure adds a new NATS service endpoint for supporting +:ref:`bucket notifications ` in a MinIO +deployment. + +.. include:: /includes/common-admonitions.rst + :start-after: start-restart-downtime + :end-before: end-restart-downtime + +Prerequisites +~~~~~~~~~~~~~ + +MinIO ``mc`` Command Line Tool +++++++++++++++++++++++++++++++ + +This procedure uses the :mc:`mc` command line tool for certain actions. +See the ``mc`` :ref:`Quickstart ` for installation instructions. + +1) Add the NATS Endpoint to MinIO +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can configure a new NATS service endpoint using either environment variables +*or* by setting runtime configuration settings. + +.. tabs:: + + .. tab:: Environment Variables + + MinIO supports specifying the NATS service endpoint and associated + configuration settings using + :ref:`environment variables + `. The + :mc:`minio server` process applies the specified settings on its + next startup. + + The following example code sets *all* environment variables + related to configuring an NATS service endpoint. The minimum + *required* variables are + :envvar:`MINIO_NOTIFY_NATS_ADDRESS` and + :envvar:`MINIO_NOTIFY_NATS_SUBJECT`: + + .. code-block:: shell + :class: copyable + + set MINIO_NOTIFY_NATS_ENABLE_="on" + set MINIO_NOTIFY_NATS_ADDRESS_="" + set MINIO_NOTIFY_NATS_SUBJECT_="" + set MINIO_NOTIFY_NATS_USERNAME_="" + set MINIO_NOTIFY_NATS_PASSWORD_="" + set MINIO_NOTIFY_NATS_TOKEN_="" + set MINIO_NOTIFY_NATS_TLS_="" + set MINIO_NOTIFY_NATS_TLS_SKIP_VERIFY_="" + set MINIO_NOTIFY_NATS_PING_INTERVAL_="" + set MINIO_NOTIFY_NATS_STREAMING_="" + set MINIO_NOTIFY_NATS_STREAMING_ASYNC_="" + set MINIO_NOTIFY_NATS_STREAMING_MAX_PUB_ACKS_IN_FLIGHT_="" + set MINIO_NOTIFY_NATS_STREAMING_CLUSTER_ID_="" + set MINIO_NOTIFY_NATS_CERT_AUTHORITY_="" + set MINIO_NOTIFY_NATS_CLIENT_CERT_="" + set MINIO_NOTIFY_NATS_CLIENT_KEY_="" + set MINIO_NOTIFY_NATS_QUEUE_DIR_="" + set MINIO_NOTIFY_NATS_QUEUE_LIMIT_="" + set MINIO_NOTIFY_NATS_COMMENT_="" + + - Replace ```` with a unique descriptive string for the + NATS service endpoint. Use the same ```` value for all + environment variables related to the new target service endpoint. + The following examples assume an identifier of ``PRIMARY``. + + If the specified ```` matches an existing NATS service + endpoint on the MinIO deployment, the new settings *override* + any existing settings for that endpoint. Use + :mc-cmd:`mc admin config get notify_nats ` to + review the currently configured NATS endpoints on the MinIO deployment. + + - Replace ```` with the URL of the NATS service endpoint. + For example: ``htpps://nats-endpoint.example.com:4222`` + + See :ref:`NATS Service for Bucket Notifications + ` for complete documentation + on each environment variable. + + .. tab:: Configuration Settings + + MinIO supports adding or updating NATS endpoints on a running + :mc:`minio server` process using the :mc-cmd:`mc admin config set` command + and the :mc-conf:`notify_nats` configuration key. You must restart the + :mc:`minio server` process to apply any new or updated configuration + settings. + + The following example code sets *all* settings related to configuring an + NATS service endpoint. The minimum *required* setting are + :mc-conf:`notify_nats address ` and + :mc-conf:`notify_nats subject `: + + .. code-block:: shell + :class: copyable + + mc admin config set ALIAS/ notify_nats:IDENTIFIER \ + address="HOSTNAME" \ + subject="" \ + username="" \ + password="" \ + token="" \ + tls="" \ + tls_skip_verify="" \ + ping_interval="" \ + streaming="" \ + streaming_async="" \ + streaming_max_pub_acks_in_flight="" \ + streaming_cluster_id="" \ + cert_authority="" \ + client_cert="" \ + client_key="" \ + queue_dir="" \ + queue_limit="" \ + comment="" + + + - Replace ``IDENTIFIER`` with a unique descriptive string for the + NATS service endpoint. The following examples in this procedure + assume an identifier of ``PRIMARY``. + + If the specified ``IDENTIFIER`` matches an existing NATS service + endpoint on the MinIO deployment, the new settings *override* + any existing settings for that endpoint. Use + :mc-cmd:`mc admin config get notify_nats ` to + review the currently configured NATS endpoints on the MinIO deployment. + + - Replace ``ENDPOINT`` with the URL of the NATS service endpoint. + For example: ``htpps://nats-endpoint.example.com:4222``. + + See :ref:`NATS Bucket Notification Configuration Settings + ` for complete + documentation on each setting. + +2) Restart the MinIO Deployment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You must restart the MinIO deployment to apply the configuration changes. +Use the :mc-cmd:`mc admin service restart` command to restart the deployment. + +.. important:: + + MinIO restarts *all* :mc:`minio server` processes associated to the + deployment at the same time. Applications may experience a brief period of + downtime during the restart process. + + Consider scheduling the restart during a maintenance period to minimize + interruption of services. + +.. code-block:: shell + :class: copyable + + mc admin service restart ALIAS + +Replace ``ALIAS`` with the :mc:`alias ` of the deployment to +restart. + +The :mc:`minio server` process prints a line on startup for each configured NATS +target similar to the following: + +.. code-block:: shell + + SQS ARNs: arn:minio:sqs::primary:nats + +You must specify the ARN resource when configuring bucket notifications with +the associated NATS deployment as a target. + +3) Configure Bucket Notifications using the NATS Endpoint as a Target +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Use the :mc-cmd:`mc event add` command to add a new bucket notification +event with the configured NATS service as a target: + +.. code-block:: shell + :class: copyable + + mc event add ALIAS/BUCKET arn:minio:sqs::primary:nats \ + --event EVENTS + +- Replace ``ALIAS`` with the :mc:`alias ` of a MinIO deployment. +- Replace ``BUCKET`` with the name of the bucket in which to configure the + event. +- Replace ``EVENTS`` with a comma-separated list of :ref:`events + ` for which MinIO triggers notifications. + +Use :mc-cmd:`mc event list` to view all configured bucket events for +a given notification target: + +.. code-block:: shell + :class: copyable + + mc event list ALIAS/BUCKET arn:minio:sqs::primary:nats + +4) Validate the Configured Events +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Perform an action on the bucket for which you configured the new event and +check the NATS service for the notification data. The action required +depends on which :mc-cmd:`events ` were specified +when configuring the bucket notification. + +For example, if the bucket notification configuration includes the +``s3:ObjectCreated:Put`` event, you can use the +:mc-cmd:`mc cp` command to create a new object in the bucket and trigger +a notification. + +.. code-block:: shell + :class: copyable + + mc cp ~/data/new-object.txt ALIAS/BUCKET + +Update an NATS Endpoint in a MinIO Deployment +--------------------------------------------- + +The following procedure updates an existing NATS service endpoint for supporting +:ref:`bucket notifications ` in a MinIO +deployment. + +.. include:: /includes/common-admonitions.rst + :start-after: start-restart-downtime + :end-before: end-restart-downtime + +Prerequisites +~~~~~~~~~~~~~~ + +MinIO ``mc`` Command Line Tool +++++++++++++++++++++++++++++++ + +This procedure uses the :mc:`mc` command line tool for certain actions. +See the ``mc`` :ref:`Quickstart ` for installation instructions. + + +1) List Configured NATS Endpoints In The Deployment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Use the :mc-cmd:`mc admin config get` command to list the currently +configured NATS service endpoints in the deployment: + +.. code-block:: shell + :class: copyable + + mc admin config get ALIAS/ notify_nats + +Replace ``ALIAS`` with the :mc:`alias ` of the MinIO deployment. + +The command output resembles the following: + +.. code-block:: shell + + notify_nats:primary password="yoursecret" streaming_max_pub_acks_in_flight="10" subject="" address="nats-endpoint.example.com:4222" token="" username="yourusername" ping_interval="0" queue_limit="0" tls="off" tls_skip_verify="off" streaming_async="on" queue_dir="" streaming_cluster_id="test-cluster" streaming_enable="on" + notify_nats:secondary password="yoursecret" streaming_max_pub_acks_in_flight="10" subject="" address="nats-endpoint.example.com:4222" token="" username="yourusername" ping_interval="0" queue_limit="0" tls="off" tls_skip_verify="off" streaming_async="on" queue_dir="" streaming_cluster_id="test-cluster" streaming_enable="on" + +The :mc-conf:`notify_nats` key is the top-level configuration key for an +:ref:`minio-server-config-bucket-notification-nats`. The +:mc-conf:`address ` key specifies the NATS service endpoint +for the given ``notify_nats`` key. The ``notify_nats:`` suffix +describes the unique identifier for that NATS service endpoint. + +Note the identifier for the NATS service endpoint you want to update for +the next step. + +2) Update the NATS Endpoint +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Use the :mc-cmd:`mc admin config set` command to set the new configuration +for the NATS service endpoint: + +.. code-block:: shell + :class: copyable + + mc admin config set ALIAS/ notify_nats:IDENTIFIER \ + address="HOSTNAME" \ + subject="" \ + username="" \ + password="" \ + token="" \ + tls="" \ + tls_skip_verify="" \ + ping_interval="" \ + streaming="" \ + streaming_async="" \ + streaming_max_pub_acks_in_flight="" \ + streaming_cluster_id="" \ + cert_authority="" \ + client_cert="" \ + client_key="" \ + queue_dir="" \ + queue_limit="" \ + comment="" + +The :mc-conf:`notify_nats address ` configuration setting +is the *minimum* required for an NATS service endpoint. All other configuration +settings are *optional*. See :ref:`minio-server-config-bucket-notification-nats` +for a complete list of NATS configuration settings. + +3) Restart the MinIO Deployment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You must restart the MinIO deployment to apply the configuration changes. +Use the :mc-cmd:`mc admin service restart` command to restart the deployment. + +.. important:: + + MinIO restarts *all* :mc:`minio server` processes associated to the + deployment at the same time. Applications may experience a brief period of + downtime during the restart process. + + Consider scheduling the restart during a maintenance period to minimize + interruption of services. + +.. code-block:: shell + :class: copyable + + mc admin service restart ALIAS + +Replace ``ALIAS`` with the :mc:`alias ` of the deployment to +restart. + +The :mc:`minio server` process prints a line on startup for each configured NATS +target similar to the following: + +.. code-block:: shell + + SQS ARNs: arn:minio:sqs::primary:nats + +3) Validate the Changes +~~~~~~~~~~~~~~~~~~~~~~~ + +Perform an action on a bucket which has an event configuration using the updated +NATS service endpoint and check the NATS service for the notification data. The +action required depends on which :mc-cmd:`events ` were +specified when configuring the bucket notification. + +For example, if the bucket notification configuration includes the +``s3:ObjectCreated:Put`` event, you can use the +:mc-cmd:`mc cp` command to create a new object in the bucket and trigger +a notification. + +.. code-block:: shell + :class: copyable + + mc cp ~/data/new-object.txt ALIAS/BUCKET \ No newline at end of file diff --git a/source/monitoring/bucket-notifications/publish-events-to-nsq.rst b/source/monitoring/bucket-notifications/publish-events-to-nsq.rst new file mode 100644 index 00000000..2611c886 --- /dev/null +++ b/source/monitoring/bucket-notifications/publish-events-to-nsq.rst @@ -0,0 +1,333 @@ +.. _minio-bucket-notifications-publish-nsq: + +===================== +Publish Events to NSQ +===================== + +.. default-domain:: minio + +.. contents:: Table of Contents + :local: + :depth: 1 + +MinIO supports publishing :ref:`bucket notification +` events to `NSQ `__ +service endpoint. + +Add a NSQ Endpoint to a MinIO Deployment +---------------------------------------- + +The following procedure adds a new NSQ service endpoint for supporting +:ref:`bucket notifications ` in a MinIO +deployment. + +.. include:: /includes/common-admonitions.rst + :start-after: start-restart-downtime + :end-before: end-restart-downtime + +Prerequisites +~~~~~~~~~~~~~ + +MinIO ``mc`` Command Line Tool +++++++++++++++++++++++++++++++ + +This procedure uses the :mc:`mc` command line tool for certain actions. +See the ``mc`` :ref:`Quickstart ` for installation instructions. + +1) Add the NSQ Endpoint to MinIO +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can configure a new NSQ service endpoint using either environment variables +*or* by setting runtime configuration settings. + +.. tabs:: + + .. tab:: Environment Variables + + MinIO supports specifying the NSQ service endpoint and associated + configuration settings using + :ref:`environment variables + `. The + :mc:`minio server` process applies the specified settings on its + next startup. + + The following example code sets *all* environment variables + related to configuring an NSQ service endpoint. The minimum + *required* variables are + :envvar:`MINIO_NOTIFY_NSQ_NSQD_ADDRESS` and + :envvar:`MINIO_NOTIFY_NSQ_TOPIC`: + + + .. code-block:: shell + :class: copyable + + set MINIO_NOTIFY_NSQ_ENABLE_="on" + set MINIO_NOTIFY_NSQ_NSQD_ADDRESS_="" + set MINIO_NOTIFY_NSQ_TOPIC_="" + set MINIO_NOTIFY_NSQ_TLS_="" + set MINIO_NOTIFY_NSQ_TLS_SKIP_VERIFY_="" + set MINIO_NOTIFY_NSQ_QUEUE_DIR_="" + set MINIO_NOTIFY_NSQ_QUEUE_LIMIT_="" + set MINIO_NOTIFY_NSQ_COMMENT_="" + + - Replace ```` with a unique descriptive string for the + TARGET service endpoint. Use the same ```` value for all + environment variables related to the new target service endpoint. + The following examples assume an identifier of ``PRIMARY``. + + If the specified ```` matches an existing NSQ service + endpoint on the MinIO deployment, the new settings *override* + any existing settings for that endpoint. Use + :mc-cmd:`mc admin config get notify_nsq ` to + review the currently configured NSQ endpoints on the MinIO deployment. + + - Replace ```` with the URL of the NSQ service endpoint. + For example, ``https://nsq-service.example.com:4150``. + + See :ref:`NSQ Service for Bucket Notifications + ` for complete documentation + on each environment variable. + + .. tab:: Configuration Settings + + MinIO supports adding or updating NSQ endpoints on a running + :mc:`minio server` process using the :mc-cmd:`mc admin config set` command + and the :mc-conf:`notify_nsq` configuration key. You must restart the + :mc:`minio server` process to apply any new or updated configuration + settings. + + The following example code sets *all* settings related to configuring an + NSQ service endpoint. The minimum *required* setting is + :mc-conf:`notify_nsq nsqd_address ` and + :mc-conf:`notify_nsq topic `: + + .. code-block:: shell + :class: copyable + + mc admin config set ALIAS/ notify_nsq:IDENTIFIER \ + nsqd_address="ENDPOINT" \ + topic="" \ + tls="" \ + tls_skip_verify="" \ + queue_dir="" \ + queue_limit="" \ + comment="" + + + - Replace ``IDENTIFIER`` with a unique descriptive string for the + NSQ service endpoint. The following examples in this procedure + assume an identifier of ``PRIMARY``. + + If the specified ``IDENTIFIER`` matches an existing NSQ service + endpoint on the MinIO deployment, the new settings *override* + any existing settings for that endpoint. Use + :mc-cmd:`mc admin config get notify_nsq ` to + review the currently configured NSQ endpoints on the MinIO deployment. + + - Replace ``ENDPOINT`` with the URL of the NSQ service endpoint. + For example: + + ``NSQ://user:password@hostname:port`` + + See :ref:`NSQ Bucket Notification Configuration Settings + ` for complete + documentation on each setting. + +2) Restart the MinIO Deployment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You must restart the MinIO deployment to apply the configuration changes. +Use the :mc-cmd:`mc admin service restart` command to restart the deployment. + +.. important:: + + MinIO restarts *all* :mc:`minio server` processes associated to the + deployment at the same time. Applications may experience a brief period of + downtime during the restart process. + + Consider scheduling the restart during a maintenance period to minimize + interruption of services. + +.. code-block:: shell + :class: copyable + + mc admin service restart ALIAS + +Replace ``ALIAS`` with the :mc:`alias ` of the deployment to +restart. + +The :mc:`minio server` process prints a line on startup for each configured NSQ +target similar to the following: + +.. code-block:: shell + + SQS ARNs: arn:minio:sqs::primary:nsq + +You must specify the ARN resource when configuring bucket notifications with +the associated NSQ deployment as a target. + +3) Configure Bucket Notifications using the NSQ Endpoint as a Target +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Use the :mc-cmd:`mc event add` command to add a new bucket notification +event with the configured NSQ service as a target: + +.. code-block:: shell + :class: copyable + + mc event add ALIAS/BUCKET arn:minio:sqs::primary:nsq \ + --event EVENTS + +- Replace ``ALIAS`` with the :mc:`alias ` of a MinIO deployment. +- Replace ``BUCKET`` with the name of the bucket in which to configure the + event. +- Replace ``EVENTS`` with a comma-separated list of :ref:`events + ` for which MinIO triggers notifications. + +Use :mc-cmd:`mc event list` to view all configured bucket events for +a given notification target: + +.. code-block:: shell + :class: copyable + + mc event list ALIAS/BUCKET arn:minio:sqs::primary:nsq + +4) Validate the Configured Events +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Perform an action on the bucket for which you configured the new event and +check the NSQ service for the notification data. The action required +depends on which :mc-cmd:`events ` were specified +when configuring the bucket notification. + +For example, if the bucket notification configuration includes the +``s3:ObjectCreated:Put`` event, you can use the +:mc-cmd:`mc cp` command to create a new object in the bucket and trigger +a notification. + +.. code-block:: shell + :class: copyable + + mc cp ~/data/new-object.txt ALIAS/BUCKET + +Update an NSQ Endpoint in a MinIO Deployment +-------------------------------------------- + +The following procedure updates an existing NSQ service endpoint for supporting +:ref:`bucket notifications ` in a MinIO +deployment. + +.. include:: /includes/common-admonitions.rst + :start-after: start-restart-downtime + :end-before: end-restart-downtime + +Prerequisites +~~~~~~~~~~~~~~ + +MinIO ``mc`` Command Line Tool +++++++++++++++++++++++++++++++ + +This procedure uses the :mc:`mc` command line tool for certain actions. +See the ``mc`` :ref:`Quickstart ` for installation instructions. + + +1) List Configured NSQ Endpoints In The Deployment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Use the :mc-cmd:`mc admin config get` command to list the currently +configured NSQ service endpoints in the deployment: + +.. code-block:: shell + :class: copyable + + mc admin config get ALIAS/ notify_nsq + +Replace ``ALIAS`` with the :mc:`alias ` of the MinIO deployment. + +The command output resembles the following: + +.. code-block:: shell + + notify_nsq:primary nsqd_address="https://nsq.example.com" queue_dir="" queue_limit="0" tls="off" tls_skip_verify="off" topic="" + notify_nsq:secondary nsqd_address="https://nsq.example.com" queue_dir="" queue_limit="0" tls="off" tls_skip_verify="off" topic="" + +The :mc-conf:`notify_nsq` key is the top-level configuration key for an +:ref:`minio-server-config-bucket-notification-nsq`. The :mc-conf:`nsqd_address +` key specifies the NSQ service endpoint for the given +`notify_nsq` key. The ``notify_nsq:`` suffix describes the unique +identifier for that NSQ service endpoint. + +Note the identifier for the NSQ service endpoint you want to update for +the next step. + +2) Update the NSQ Endpoint +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Use the :mc-cmd:`mc admin config set` command to set the new configuration +for the NSQ service endpoint: + +.. code-block:: shell + :class: copyable + + mc admin config set ALIAS/ notify_nsq: \ + nsqd_address="NSQ://user:password@hostname:port" \ + topic="" \ + tls="" \ + tls_skip_verify="" \ + queue_dir="" \ + queue_limit="" \ + comment="" + +The :mc-conf:`notify_nsq nsqd_address ` configuration +setting is the *minimum* required for an NSQ service endpoint. All other +configuration settings are *optional*. See +:ref:`minio-server-config-bucket-notification-nsq` for a complete list of NSQ +configuration settings. + +3) Restart the MinIO Deployment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You must restart the MinIO deployment to apply the configuration changes. +Use the :mc-cmd:`mc admin service restart` command to restart the deployment. + +.. important:: + + MinIO restarts *all* :mc:`minio server` processes associated to the + deployment at the same time. Applications may experience a brief period of + downtime during the restart process. + + Consider scheduling the restart during a maintenance period to minimize + interruption of services. + +.. code-block:: shell + :class: copyable + + mc admin service restart ALIAS + +Replace ``ALIAS`` with the :mc:`alias ` of the deployment to +restart. + +The :mc:`minio server` process prints a line on startup for each configured NSQ +target similar to the following: + +.. code-block:: shell + + SQS ARNs: arn:minio:sqs::primary:NSQ + +3) Validate the Changes +~~~~~~~~~~~~~~~~~~~~~~~ + +Perform an action on a bucket which has an event configuration using the updated +NSQ service endpoint and check the NSQ service for the notification data. The +action required depends on which :mc-cmd:`events ` were +specified when configuring the bucket notification. + +For example, if the bucket notification configuration includes the +``s3:ObjectCreated:Put`` event, you can use the +:mc-cmd:`mc cp` command to create a new object in the bucket and trigger +a notification. + +.. code-block:: shell + :class: copyable + + mc cp ~/data/new-object.txt ALIAS/BUCKET \ No newline at end of file diff --git a/source/monitoring/bucket-notifications/publish-events-to-postgresql.rst b/source/monitoring/bucket-notifications/publish-events-to-postgresql.rst new file mode 100644 index 00000000..98216790 --- /dev/null +++ b/source/monitoring/bucket-notifications/publish-events-to-postgresql.rst @@ -0,0 +1,367 @@ +.. _minio-bucket-notifications-publish-postgresql: + +============================ +Publish Events to PostgreSQL +============================ + +.. default-domain:: minio + +.. contents:: Table of Contents + :local: + :depth: 1 + +.. |postgresql-uri-reference| replace:: `PostgreSQL Connection String `__ + +MinIO supports publishing :ref:`bucket notification +` events to +`PostgreSQL `__. MinIO supports +PostgreSQL 9.5 and later *only*. + +Add a PostgreSQL Endpoint to a MinIO Deployment +----------------------------------------------- + +The following procedure adds a new PostgreSQL service endpoint for supporting +:ref:`bucket notifications ` in a MinIO +deployment. + +.. include:: /includes/common-admonitions.rst + :start-after: start-restart-downtime + :end-before: end-restart-downtime + +Prerequisites +~~~~~~~~~~~~~ + +PostgreSQL 9.5 and later +++++++++++++++++++++++++ + +MinIO relies on features introduced with PostgreSQL 9.5. + +MinIO ``mc`` Command Line Tool +++++++++++++++++++++++++++++++ + +This procedure uses the :mc:`mc` command line tool for certain actions. +See the ``mc`` :ref:`Quickstart ` for installation instructions. + +1) Add the PostgreSQL Endpoint to MinIO +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can configure a new PostgreSQL service endpoint using either environment +variables *or* by setting runtime configuration settings. + +.. tabs:: + + .. tab:: Environment Variables + + MinIO supports specifying the PostgreSQL service endpoint and associated + configuration settings using + :ref:`environment variables + `. The + :mc:`minio server` process applies the specified settings on its + next startup. + + The following example code sets *all* environment variables + related to configuring a PostgreSQL service endpoint. The minimum + *required* variables are: + + - :envvar:`MINIO_NOTIFY_POSTGRESQL_CONNECTION_STRING` + - :envvar:`MINIO_NOTIFY_POSTGRESQL_TABLE` + - :envvar:`MINIO_NOTIFY_POSTGRESQL_FORMAT` + + .. code-block:: shell + :class: copyable + + set MINIO_NOTIFY_CONNECTION_STRING_="on" + set MINIO_NOTIFY_TABLE_="ENDPOINT" + set MINIO_NOTIFY_FORMAT_="" + set MINIO_NOTIFY_MAX_OPEN_CONNECTIONS_="" + set MINIO_NOTIFY_QUEUE_DIR_="" + set MINIO_NOTIFY_QUEUE_LIMIT_="" + set MINIO_NOTIFY_COMMENT_="" + + + - Replace ```` with a unique descriptive string for the + PostgreSQL service endpoint. Use the same ```` value for all + environment variables related to the new target service endpoint. + The following examples assume an identifier of ``PRIMARY``. + + If the specified ```` matches an existing PostgreSQL service + endpoint on the MinIO deployment, the new settings *override* + any existing settings for that endpoint. Use + :mc-cmd:`mc admin config get notify_postgresql ` to + review the currently configured PostgreSQL endpoints on the MinIO deployment. + + - Replace ```` with the |postgresql-uri-reference| + for PostgreSQL service endpoint. MinIO supports ``key=value`` format for + the connection string. For example: + + ``"host=https://postgresql.example.com port=5432 ..."`` + + For more complete documentation on supported PostgreSQL connection + string parameters, see |postgresql-uri-reference|. + + See :ref:`PostgreSQL Service for Bucket Notifications + ` for complete + documentation on each environment variable. + + .. tab:: Configuration Settings + + MinIO supports adding or updating PostgreSQL endpoints on a running + :mc:`minio server` process using the :mc-cmd:`mc admin config set` command + and the :mc-conf:`notify_postgresql` configuration key. You must restart the + :mc:`minio server` process to apply any new or updated configuration + settings. + + The following example code sets *all* settings related to configuring an + PostgreSQL service endpoint. The minimum *required* setting are: + + - :mc-conf:`notify_postgresql connection_string + ` + - :mc-conf:`notify_postgresql table ` + - :mc-conf:`notify_postgresql format ` + + .. code-block:: shell + :class: copyable + + mc admin config set ALIAS/ notify_postgresql:IDENTIFIER \ + connection_string="ENDPOINT" \ + table="" \ + format="" \ + max_open_connections="" \ + queue_dir="" \ + queue_limit="" \ + comment="" + + - Replace ``IDENTIFIER`` with a unique descriptive string for the + PostgreSQL service endpoint. The following examples in this procedure + assume an identifier of ``PRIMARY``. + + If the specified ``IDENTIFIER`` matches an existing PostgreSQL service + endpoint on the MinIO deployment, the new settings *override* + any existing settings for that endpoint. Use + :mc-cmd:`mc admin config get notify_postgresql ` to + review the currently configured PostgreSQL endpoints on the MinIO deployment. + + - Replace ```` with the `PostgreSQL URI connection string + `__ + of the PostgreSQL service endpoint. MinIO supports ``key=value`` format + for the PostgreSQL connection string. For example: + + ``"host=https://postgresql.example.com port=5432 ..."`` + + For more complete documentation on supported PostgreSQL connection + string parameters, see |postgresql-uri-reference|. + + See :ref:`PostgreSQL Bucket Notification Configuration Settings + ` for complete + documentation on each setting. + +2) Restart the MinIO Deployment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You must restart the MinIO deployment to apply the configuration changes. +Use the :mc-cmd:`mc admin service restart` command to restart the deployment. + +.. important:: + + MinIO restarts *all* :mc:`minio server` processes associated to the + deployment at the same time. Applications may experience a brief period of + downtime during the restart process. + + Consider scheduling the restart during a maintenance period to minimize + interruption of services. + +.. code-block:: shell + :class: copyable + + mc admin service restart ALIAS + +Replace ``ALIAS`` with the :mc:`alias ` of the deployment to +restart. + +The :mc:`minio server` process prints a line on startup for each configured PostgreSQL +target similar to the following: + +.. code-block:: shell + + SQS ARNs: arn:minio:sqs::primary:postgresql + +You must specify the ARN resource when configuring bucket notifications with +the associated PostgreSQL deployment as a target. + +3) Configure Bucket Notifications using the PostgreSQL Endpoint as a Target +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Use the :mc-cmd:`mc event add` command to add a new bucket notification +event with the configured PostgreSQL service as a target: + +.. code-block:: shell + :class: copyable + + mc event add ALIAS/BUCKET arn:minio:sqs::primary:postgresql \ + --event EVENTS + +- Replace ``ALIAS`` with the :mc:`alias ` of a MinIO deployment. +- Replace ``BUCKET`` with the name of the bucket in which to configure the + event. +- Replace ``EVENTS`` with a comma-separated list of :ref:`events + ` for which MinIO triggers notifications. + +Use :mc-cmd:`mc event list` to view all configured bucket events for +a given notification target: + +.. code-block:: shell + :class: copyable + + mc event list ALIAS/BUCKET arn:minio:sqs::primary:postgresql + +4) Validate the Configured Events +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Perform an action on the bucket for which you configured the new event and +check the PostgreSQL service for the notification data. The action required +depends on which :mc-cmd:`events ` were specified +when configuring the bucket notification. + +For example, if the bucket notification configuration includes the +``s3:ObjectCreated:Put`` event, you can use the +:mc-cmd:`mc cp` command to create a new object in the bucket and trigger +a notification. + +.. code-block:: shell + :class: copyable + + mc cp ~/data/new-object.txt ALIAS/BUCKET + +Update a PostgreSQL Endpoint in a MinIO Deployment +--------------------------------------------------- + +The following procedure updates an existing PostgreSQL service endpoint for +supporting :ref:`bucket notifications ` in a MinIO +deployment. + +.. include:: /includes/common-admonitions.rst + :start-after: start-restart-downtime + :end-before: end-restart-downtime + +Prerequisites +~~~~~~~~~~~~~ + +PostgreSQL 9.5 and later +++++++++++++++++++++++++ + +MinIO relies on features introduced with PostgreSQL 9.5. + +MinIO ``mc`` Command Line Tool +++++++++++++++++++++++++++++++ + +This procedure uses the :mc:`mc` command line tool for certain actions. +See the ``mc`` :ref:`Quickstart ` for installation instructions. + + +1) List Configured PostgreSQL Endpoints In The Deployment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Use the :mc-cmd:`mc admin config get` command to list the currently +configured PostgreSQL service endpoints in the deployment: + +.. code-block:: shell + :class: copyable + + mc admin config get ALIAS/ notify_postgresql + +Replace ``ALIAS`` with the :mc:`alias ` of the MinIO deployment. + +The command output resembles the following: + +.. code-block:: shell + + notify_postgresql:primary queue_dir="" connection_string="postgresql://" queue_limit="0" table="" format="namespace" + notify_postgresql:secondary queue_dir="" connection_string="" queue_limit="0" table="" format="namespace" + +The :mc-conf:`notify_postgresql` key is the top-level configuration key for an +:ref:`minio-server-config-bucket-notification-postgresql`. The +:mc-conf:`connection_string ` key specifies +the PostgreSQL service endpoint for the given `notify_postgresql` key. The +``notify_postgresql:`` suffix describes the unique identifier for +that PostgreSQL service endpoint. + +Note the identifier for the PostgreSQL service endpoint you want to update for +the next step. + +2) Update the PostgreSQL Endpoint +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Use the :mc-cmd:`mc admin config set` command to set the new configuration +for the PostgreSQL service endpoint: + +.. code-block:: shell + :class: copyable + + mc admin config set ALIAS/ notify_postgresql:IDENTIFIER \ + connection_string="ENDPOINT" \ + table="" \ + format="" \ + max_open_connections="" \ + queue_dir="" \ + queue_limit="" \ + comment="" + +The following configuration settings are the *minimum* required for a +PostgreSQL service endpoint: + +- :mc-conf:`notify_postgresql connection_string + ` +- :mc-conf:`notify_postgresql table ` +- :mc-conf:`notify_postgresql format ` + +All other configuration settings are *optional*. See +:ref:`minio-server-config-bucket-notification-postgresql` for a complete list of +PostgreSQL configuration settings. + +3) Restart the MinIO Deployment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You must restart the MinIO deployment to apply the configuration changes. +Use the :mc-cmd:`mc admin service restart` command to restart the deployment. + +.. important:: + + MinIO restarts *all* :mc:`minio server` processes associated to the + deployment at the same time. Applications may experience a brief period of + downtime during the restart process. + + Consider scheduling the restart during a maintenance period to minimize + interruption of services. + +.. code-block:: shell + :class: copyable + + mc admin service restart ALIAS + +Replace ``ALIAS`` with the :mc:`alias ` of the deployment to +restart. + +The :mc:`minio server` process prints a line on startup for each configured PostgreSQL +target similar to the following: + +.. code-block:: shell + + SQS ARNs: arn:minio:sqs::primary:postgresql + +3) Validate the Changes +~~~~~~~~~~~~~~~~~~~~~~~ + +Perform an action on a bucket which has an event configuration using the updated +PostgreSQL service endpoint and check the PostgreSQL service for the notification data. The +action required depends on which :mc-cmd:`events ` were +specified when configuring the bucket notification. + +For example, if the bucket notification configuration includes the +``s3:ObjectCreated:Put`` event, you can use the +:mc-cmd:`mc cp` command to create a new object in the bucket and trigger +a notification. + +.. code-block:: shell + :class: copyable + + mc cp ~/data/new-object.txt ALIAS/BUCKET \ No newline at end of file diff --git a/source/monitoring/bucket-notifications/publish-events-to-redis.rst b/source/monitoring/bucket-notifications/publish-events-to-redis.rst new file mode 100644 index 00000000..4d99e995 --- /dev/null +++ b/source/monitoring/bucket-notifications/publish-events-to-redis.rst @@ -0,0 +1,335 @@ +.. _minio-bucket-notifications-publish-redis: + +======================= +Publish Events to Redis +======================= + +.. default-domain:: minio + +.. contents:: Table of Contents + :local: + :depth: 1 + +MinIO supports publishing :ref:`bucket notification +` events to a `Redis `__ +service endpoint. + +Add a Redis Endpoint to a MinIO Deployment +------------------------------------------- + +The following procedure adds a new Redis service endpoint for supporting +:ref:`bucket notifications ` in a MinIO +deployment. + +.. include:: /includes/common-admonitions.rst + :start-after: start-restart-downtime + :end-before: end-restart-downtime + +Prerequisites +~~~~~~~~~~~~~ + +MinIO ``mc`` Command Line Tool +++++++++++++++++++++++++++++++ + +This procedure uses the :mc:`mc` command line tool for certain actions. +See the ``mc`` :ref:`Quickstart ` for installation instructions. + +1) Add the Redis Endpoint to MinIO +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can configure a new Redis service endpoint using either environment variables +*or* by setting runtime configuration settings. + +.. tabs:: + + .. tab:: Environment Variables + + MinIO supports specifying the Redis service endpoint and associated + configuration settings using + :ref:`environment variables + `. The + :mc:`minio server` process applies the specified settings on its + next startup. + + The following example code sets *all* environment variables + related to configuring an Redis service endpoint. The minimum + *required* variables are: + + - :envvar:`MINIO_NOTIFY_REDIS_ENABLE` + - :envvar:`MINIO_NOTIFY_REDIS_ADDRESS` + - :envvar:`MINIO_NOTIFY_REDIS_KEY` + - :envvar:`MINIO_NOTIFY_REDIS_FORMAT` + + .. code-block:: shell + :class: copyable + + set MINIO_NOTIFY_REDIS_ENABLE_="on" + set MINIO_NOTIFY_REDIS_ADDRESS_="" + set MINIO_NOTIFY_REDIS_KEY_="" + set MINIO_NOTIFY_REDIS_FORMAT_="" + set MINIO_NOTIFY_REDIS_PASSWORD_="" + set MINIO_NOTIFY_REDIS_QUEUE_DIR_="" + set MINIO_NOTIFY_REDIS_QUEUE_LIMIT_="" + set MINIO_NOTIFY_REDIS_COMMENT_="" + + - Replace ```` with a unique descriptive string for the + TARGET service endpoint. Use the same ```` value for all + environment variables related to the new target service endpoint. + The following examples assume an identifier of ``PRIMARY``. + + If the specified ```` matches an existing Redis service + endpoint on the MinIO deployment, the new settings *override* + any existing settings for that endpoint. Use + :mc-cmd:`mc admin config get notify_redis ` to + review the currently configured Redis endpoints on the MinIO deployment. + + - Replace ```` with the URL of the Redis service endpoint. + For example: ``https://redis.example.com:6369`` + + + See :ref:`Redis Service for Bucket Notifications + ` for complete documentation + on each environment variable. + + .. tab:: Configuration Settings + + MinIO supports adding or updating Redis endpoints on a running + :mc:`minio server` process using the :mc-cmd:`mc admin config set` command + and the :mc-conf:`notify_redis` configuration key. You must restart the + :mc:`minio server` process to apply any new or updated configuration + settings. + + The following example code sets *all* settings related to configuring an + Redis service endpoint. The minimum *required* settings are: + + - :mc-conf:`notify_redis address ` + - :mc-conf:`notify_redis key ` + - :mc-conf:`notify_redis format ` + + .. code-block:: shell + :class: copyable + + mc admin config set ALIAS/ notify_redis:IDENTIFIER \ + address="ENDPOINT" \ + key="" \ + format="" \ + password="" \ + queue_dir="" \ + queue_limit="" \ + comment="" + + - Replace ``IDENTIFIER`` with a unique descriptive string for the + Redis service endpoint. The following examples in this procedure + assume an identifier of ``PRIMARY``. + + If the specified ``IDENTIFIER`` matches an existing Redis service + endpoint on the MinIO deployment, the new settings *override* + any existing settings for that endpoint. Use + :mc-cmd:`mc admin config get notify_redis ` to + review the currently configured Redis endpoints on the MinIO deployment. + + - Replace ``ENDPOINT`` with the URL of the Redis service endpoint. + For example: ``https://redis.example.com:6369`` + + See :ref:`Redis Bucket Notification Configuration Settings + ` for complete + documentation on each setting. + +2) Restart the MinIO Deployment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You must restart the MinIO deployment to apply the configuration changes. +Use the :mc-cmd:`mc admin service restart` command to restart the deployment. + +.. important:: + + MinIO restarts *all* :mc:`minio server` processes associated to the + deployment at the same time. Applications may experience a brief period of + downtime during the restart process. + + Consider scheduling the restart during a maintenance period to minimize + interruption of services. + +.. code-block:: shell + :class: copyable + + mc admin service restart ALIAS + +Replace ``ALIAS`` with the :mc:`alias ` of the deployment to +restart. + +The :mc:`minio server` process prints a line on startup for each configured Redis +target similar to the following: + +.. code-block:: shell + + SQS ARNs: arn:minio:sqs::primary:redis + +You must specify the ARN resource when configuring bucket notifications with +the associated Redis deployment as a target. + +3) Configure Bucket Notifications using the Redis Endpoint as a Target +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Use the :mc-cmd:`mc event add` command to add a new bucket notification +event with the configured Redis service as a target: + +.. code-block:: shell + :class: copyable + + mc event add ALIAS/BUCKET arn:minio:sqs::primary:redis \ + --event EVENTS + +- Replace ``ALIAS`` with the :mc:`alias ` of a MinIO deployment. +- Replace ``BUCKET`` with the name of the bucket in which to configure the + event. +- Replace ``EVENTS`` with a comma-separated list of :ref:`events + ` for which MinIO triggers notifications. + +Use :mc-cmd:`mc event list` to view all configured bucket events for +a given notification target: + +.. code-block:: shell + :class: copyable + + mc event list ALIAS/BUCKET arn:minio:sqs::primary:redis + +4) Validate the Configured Events +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Perform an action on the bucket for which you configured the new event and +check the Redis service for the notification data. The action required +depends on which :mc-cmd:`events ` were specified +when configuring the bucket notification. + +For example, if the bucket notification configuration includes the +``s3:ObjectCreated:Put`` event, you can use the +:mc-cmd:`mc cp` command to create a new object in the bucket and trigger +a notification. + +.. code-block:: shell + :class: copyable + + mc cp ~/data/new-object.txt ALIAS/BUCKET + +Update an Redis Endpoint in a MinIO Deployment +---------------------------------------------- + +The following procedure updates an existing Redis service endpoint for +supporting :ref:`bucket notifications ` in a MinIO +deployment. + +.. include:: /includes/common-admonitions.rst + :start-after: start-restart-downtime + :end-before: end-restart-downtime + +Prerequisites +~~~~~~~~~~~~~~ + +MinIO ``mc`` Command Line Tool +++++++++++++++++++++++++++++++ + +This procedure uses the :mc:`mc` command line tool for certain actions. +See the ``mc`` :ref:`Quickstart ` for installation instructions. + + +1) List Configured Redis Endpoints In The Deployment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Use the :mc-cmd:`mc admin config get` command to list the currently +configured Redis service endpoints in the deployment: + +.. code-block:: shell + :class: copyable + + mc admin config get ALIAS/ notify_redis + +Replace ``ALIAS`` with the :mc:`alias ` of the MinIO deployment. + +The command output resembles the following: + +.. code-block:: shell + + notify_redis:primary address="https://redis.example.com:6369" format="namespace" key="minioevent" password="" queue_dir="" queue_limit="0" + notify_redis:secondary address="https://redis.example.com:6369" format="namespace" key="minioevent" password="" queue_dir="" queue_limit="0" + +The :mc-conf:`notify_redis` key is the top-level configuration key for an +:ref:`minio-server-config-bucket-notification-redis`. The +:mc-conf:`address ` key specifies the Redis service endpoint +for the given `notify_redis` key. The ``notify_redis:`` suffix +describes the unique identifier for that Redis service endpoint. + +Note the identifier for the Redis service endpoint you want to update for +the next step. + +2) Update the Redis Endpoint +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Use the :mc-cmd:`mc admin config set` command to set the new configuration +for the Redis service endpoint: + +.. code-block:: shell + :class: copyable + + mc admin config set ALIAS/ notify_redis:IDENTIFIER \ + address="ENDPOINT" \ + key="" \ + format="" \ + password="" \ + queue_dir="" \ + queue_limit="" \ + comment="" + +The :mc-conf:`notify_redis address ` configuration setting +is the *minimum* required for an Redis service endpoint. All other configuration +settings are *optional*. See +:ref:`minio-server-config-bucket-notification-redis` for a complete list of +Redis configuration settings. + +3) Restart the MinIO Deployment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You must restart the MinIO deployment to apply the configuration changes. +Use the :mc-cmd:`mc admin service restart` command to restart the deployment. + +.. important:: + + MinIO restarts *all* :mc:`minio server` processes associated to the + deployment at the same time. Applications may experience a brief period of + downtime during the restart process. + + Consider scheduling the restart during a maintenance period to minimize + interruption of services. + +.. code-block:: shell + :class: copyable + + mc admin service restart ALIAS + +Replace ``ALIAS`` with the :mc:`alias ` of the deployment to +restart. + +The :mc:`minio server` process prints a line on startup for each configured Redis +target similar to the following: + +.. code-block:: shell + + SQS ARNs: arn:minio:sqs::primary:redis + +3) Validate the Changes +~~~~~~~~~~~~~~~~~~~~~~~ + +Perform an action on a bucket which has an event configuration using the updated +Redis service endpoint and check the Redis service for the notification data. The +action required depends on which :mc-cmd:`events ` were +specified when configuring the bucket notification. + +For example, if the bucket notification configuration includes the +``s3:ObjectCreated:Put`` event, you can use the +:mc-cmd:`mc cp` command to create a new object in the bucket and trigger +a notification. + +.. code-block:: shell + :class: copyable + + mc cp ~/data/new-object.txt ALIAS/BUCKET \ No newline at end of file diff --git a/source/monitoring/bucket-notifications/publish-events-to-webhook.rst b/source/monitoring/bucket-notifications/publish-events-to-webhook.rst new file mode 100644 index 00000000..36e3b99d --- /dev/null +++ b/source/monitoring/bucket-notifications/publish-events-to-webhook.rst @@ -0,0 +1,333 @@ +.. _minio-bucket-notifications-publish-webhook: + +========================= +Publish Events to Webhook +========================= + +.. default-domain:: minio + +.. contents:: Table of Contents + :local: + :depth: 1 + +MinIO supports publishing :ref:`bucket notification +` events to a +`Webhook `__ service endpoint. + +Add a Webhook Endpoint to a MinIO Deployment +-------------------------------------------- + +The following procedure adds a new Webhook service endpoint for supporting +:ref:`bucket notifications ` in a MinIO +deployment. + +.. include:: /includes/common-admonitions.rst + :start-after: start-restart-downtime + :end-before: end-restart-downtime + +Prerequisites +~~~~~~~~~~~~~ + +MinIO ``mc`` Command Line Tool +++++++++++++++++++++++++++++++ + +This procedure uses the :mc:`mc` command line tool for certain actions. +See the ``mc`` :ref:`Quickstart ` for installation instructions. + +1) Add the Webhook Endpoint to MinIO +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can configure a new Webhook service endpoint using either environment +variables *or* by setting runtime configuration settings. + +.. tabs:: + + .. tab:: Environment Variables + + MinIO supports specifying the Webhook service endpoint and associated + configuration settings using + :ref:`environment variables + `. The + :mc:`minio server` process applies the specified settings on its + next startup. + + The following example code sets *all* environment variables + related to configuring an Webhook service endpoint. The minimum + *required* variables are + :envvar:`MINIO_NOTIFY_WEBHOOK_ENABLE` and + :envvar:`MINIO_NOTIFY_WEBHOOK_ENDPOINT`: + + + .. code-block:: shell + :class: copyable + + set MINIO_NOTIFY_WEBHOOK_ENABLE__="on" + set MINIO_NOTIFY_WEBHOOK_ENDPOINT__="ENDPOINT" + set MINIO_NOTIFY_WEBHOOK_AUTH_TOKEN__="" + set MINIO_NOTIFY_WEBHOOK_QUEUE_DIR__="" + set MINIO_NOTIFY_WEBHOOK_QUEUE_LIMIT__="" + set MINIO_NOTIFY_WEBHOOK_CLIENT_CERT__="" + set MINIO_NOTIFY_WEBHOOK_CLIENT_KEY__="" + set MINIO_NOTIFY_WEBHOOK_COMMENT__="" + + - Replace ```` with a unique descriptive string for the + Webhook service endpoint. Use the same ```` value for all + environment variables related to the new target service endpoint. + The following examples assume an identifier of ``PRIMARY``. + + If the specified ```` matches an existing Webhook service + endpoint on the MinIO deployment, the new settings *override* + any existing settings for that endpoint. Use + :mc-cmd:`mc admin config get notify_webhook ` to + review the currently configured Webhook endpoints on the MinIO deployment. + + - Replace ```` with the URL of the Webhook service endpoint. + For example: + + ``https://webhook.example.com`` + + See :ref:`Webhook Service for Bucket Notifications + ` for complete documentation + on each environment variable. + + .. tab:: Configuration Settings + + MinIO supports adding or updating Webhook endpoints on a running + :mc:`minio server` process using the :mc-cmd:`mc admin config set` command + and the :mc-conf:`notify_webhook` configuration key. You must restart the + :mc:`minio server` process to apply any new or updated configuration + settings. + + The following example code sets *all* settings related to configuring an + Webhook service endpoint. The minimum *required* setting is + :mc-conf:`notify_webhook endpoint `: + + .. code-block:: shell + :class: copyable + + mc admin config set ALIAS/ notify_webhook:IDENTIFIER \ + endpoint="" \ + auth_token="" \ + queue_dir="" \ + queue_limit="" \ + client_cert="" \ + client_key="" \ + comment="" + + - Replace ``IDENTIFIER`` with a unique descriptive string for the + Webhook service endpoint. The following examples in this procedure + assume an identifier of ``PRIMARY``. + + If the specified ``IDENTIFIER`` matches an existing Webhook service + endpoint on the MinIO deployment, the new settings *override* + any existing settings for that endpoint. Use + :mc-cmd:`mc admin config get notify_webhook ` to + review the currently configured Webhook endpoints on the MinIO deployment. + + - Replace ``ENDPOINT`` with the URL of the Webhook service endpoint. + For example: + + ``https://webhook.example.com`` + + See :ref:`Webhook Bucket Notification Configuration Settings + ` for complete + documentation on each setting. + +2) Restart the MinIO Deployment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You must restart the MinIO deployment to apply the configuration changes. +Use the :mc-cmd:`mc admin service restart` command to restart the deployment. + +.. important:: + + MinIO restarts *all* :mc:`minio server` processes associated to the + deployment at the same time. Applications may experience a brief period of + downtime during the restart process. + + Consider scheduling the restart during a maintenance period to minimize + interruption of services. + +.. code-block:: shell + :class: copyable + + mc admin service restart ALIAS + +Replace ``ALIAS`` with the :mc:`alias ` of the deployment to +restart. + +The :mc:`minio server` process prints a line on startup for each configured +Webhook target similar to the following: + +.. code-block:: shell + + SQS ARNs: arn:minio:sqs::primary:webhook + +You must specify the ARN resource when configuring bucket notifications with +the associated Webhook deployment as a target. + +3) Configure Bucket Notifications using the Webhook Endpoint as a Target +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Use the :mc-cmd:`mc event add` command to add a new bucket notification +event with the configured Webhook service as a target: + +.. code-block:: shell + :class: copyable + + mc event add ALIAS/BUCKET arn:minio:sqs::primary:webhook \ + --event EVENTS + +- Replace ``ALIAS`` with the :mc:`alias ` of a MinIO deployment. +- Replace ``BUCKET`` with the name of the bucket in which to configure the + event. +- Replace ``EVENTS`` with a comma-separated list of :ref:`events + ` for which MinIO triggers notifications. + +Use :mc-cmd:`mc event list` to view all configured bucket events for +a given notification target: + +.. code-block:: shell + :class: copyable + + mc event list ALIAS/BUCKET arn:minio:sqs::primary:webhook + +4) Validate the Configured Events +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Perform an action on the bucket for which you configured the new event and +check the Webhook service for the notification data. The action required +depends on which :mc-cmd:`events ` were specified +when configuring the bucket notification. + +For example, if the bucket notification configuration includes the +``s3:ObjectCreated:Put`` event, you can use the +:mc-cmd:`mc cp` command to create a new object in the bucket and trigger +a notification. + +.. code-block:: shell + :class: copyable + + mc cp ~/data/new-object.txt ALIAS/BUCKET + +Update an Webhook Endpoint in a MinIO Deployment +------------------------------------------------ + +The following procedure updates an existing Webhook service endpoint for supporting +:ref:`bucket notifications ` in a MinIO +deployment. + +.. include:: /includes/common-admonitions.rst + :start-after: start-restart-downtime + :end-before: end-restart-downtime + +Prerequisites +~~~~~~~~~~~~~~ + +MinIO ``mc`` Command Line Tool +++++++++++++++++++++++++++++++ + +This procedure uses the :mc:`mc` command line tool for certain actions. +See the ``mc`` :ref:`Quickstart ` for installation instructions. + + +1) List Configured Webhook Endpoints In The Deployment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Use the :mc-cmd:`mc admin config get` command to list the currently +configured Webhook service endpoints in the deployment: + +.. code-block:: shell + :class: copyable + + mc admin config get ALIAS/ notify_webhook + +Replace ``ALIAS`` with the :mc:`alias ` of the MinIO deployment. + +The command output resembles the following: + +.. code-block:: shell + + notify_webhook:primary endpoint="https://webhook.example.com" auth_token="" queue_limit="0" queue_dir="" client_cert="" client_key="" + notify_webhook:secondary endpoint="https://webhook.example.com" auth_token="" queue_limit="0" queue_dir="" client_cert="" client_key="" + +The :mc-conf:`notify_webhook` key is the top-level configuration key for an +:ref:`minio-server-config-bucket-notification-webhook`. The +:mc-conf:`endpoint ` key specifies the Webhook service +endpoint for the given `notify_webhook` key. The ``notify_webhook:`` +suffix describes the unique identifier for that Webhook service endpoint. + +Note the identifier for the Webhook service endpoint you want to update for +the next step. + +2) Update the Webhook Endpoint +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Use the :mc-cmd:`mc admin config set` command to set the new configuration +for the Webhook service endpoint: + +.. code-block:: shell + :class: copyable + + mc admin config set ALIAS/ notify_webhook:IDENTIFIER \ + endpoint="" \ + auth_token="" \ + queue_dir="" \ + queue_limit="" \ + client_cert="" \ + client_key="" \ + comment="" + +The :mc-conf:`notify_webhook endpoint ` configuration +setting is the *minimum* required for an Webhook service endpoint. All other +configuration settings are *optional*. See +:ref:`minio-server-config-bucket-notification-webhook` for a complete list of +Webhook configuration settings. + +3) Restart the MinIO Deployment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You must restart the MinIO deployment to apply the configuration changes. +Use the :mc-cmd:`mc admin service restart` command to restart the deployment. + +.. important:: + + MinIO restarts *all* :mc:`minio server` processes associated to the + deployment at the same time. Applications may experience a brief period of + downtime during the restart process. + + Consider scheduling the restart during a maintenance period to minimize + interruption of services. + +.. code-block:: shell + :class: copyable + + mc admin service restart ALIAS + +Replace ``ALIAS`` with the :mc:`alias ` of the deployment to +restart. + +The :mc:`minio server` process prints a line on startup for each configured Webhook +target similar to the following: + +.. code-block:: shell + + SQS ARNs: arn:minio:sqs::primary:webhook + +3) Validate the Changes +~~~~~~~~~~~~~~~~~~~~~~~ + +Perform an action on a bucket which has an event configuration using the updated +Webhook service endpoint and check the Webhook service for the notification +data. The action required depends on which :mc-cmd:`events ` +were specified when configuring the bucket notification. + +For example, if the bucket notification configuration includes the +``s3:ObjectCreated:Put`` event, you can use the +:mc-cmd:`mc cp` command to create a new object in the bucket and trigger +a notification. + +.. code-block:: shell + :class: copyable + + mc cp ~/data/new-object.txt ALIAS/BUCKET \ No newline at end of file diff --git a/source/monitoring/monitoring-overview.rst b/source/monitoring/monitoring-overview.rst new file mode 100644 index 00000000..3673e7e6 --- /dev/null +++ b/source/monitoring/monitoring-overview.rst @@ -0,0 +1,36 @@ +========== +Monitoring +========== + +.. default-domain:: minio + +.. contents:: Table of Contents + :local: + :depth: 1 + +Bucket Notifications +-------------------- + +MinIO supports publishing bucket or object events to the following supported +targets on certain supported events. + +- :ref:`minio-bucket-notifications-publish-amqp` +- :ref:`minio-bucket-notifications-publish-mqtt` +- :ref:`minio-bucket-notifications-publish-nats` +- :ref:`minio-bucket-notifications-publish-nsq` +- :ref:`minio-bucket-notifications-publish-elasticsearch` +- :ref:`minio-bucket-notifications-publish-kafka` +- :ref:`minio-bucket-notifications-publish-mysql` +- :ref:`minio-bucket-notifications-publish-postgresql` +- :ref:`minio-bucket-notifications-publish-redis` +- :ref:`minio-bucket-notifications-publish-webhook` + +See :ref:`minio-bucket-notifications` +for more complete documentation on MinIO Bucket Notifications. + + +.. toctree:: + :titlesonly: + :hidden: + + /monitoring/bucket-notifications/bucket-notifications \ No newline at end of file diff --git a/source/reference/minio-cli/minio-mc-admin/mc-admin.config.rst b/source/reference/minio-cli/minio-mc-admin/mc-admin.config.rst index a9852f24..e6a9b795 100644 --- a/source/reference/minio-cli/minio-mc-admin/mc-admin.config.rst +++ b/source/reference/minio-cli/minio-mc-admin/mc-admin.config.rst @@ -31,4 +31,1763 @@ Examples -------- Syntax ------- \ No newline at end of file +------ + +.. mc-cmd:: set + :fullpath: + + Sets a :ref:`configuration key ` on the + MinIO deployment. + +.. mc-cmd:: get + :fullpath: + + Gets a :ref:`configuration key ` on the + MinIO deployment. + +.. _minio-server-configuration-settings: + +Configuration Settings +---------------------- + +The following configuration settings define runtime behavior of the +MinIO :mc:`server ` process: + +.. _minio-server-config-bucket-notification-amqp: + +AMQP Service for Bucket Notifications +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The following section documents settings for configuring an AMQP +service as a target for :doc:`MinIO Bucket Notifications +`. See +:ref:`minio-bucket-notifications-publish-amqp` for a tutorial on +using these environment variables. + +.. mc-conf:: notify_amqp + + The top-level configuration key for defining an AMQP service endpoint for use + with :ref:`MinIO bucket notifications `. + + Use :mc-cmd:`mc admin config set` to set or update an AMQP service endpoint. + The :mc-conf:`~notify_amqp.url` argument is *required* for each target. + Specify additional optional arguments as a whitespace (``" "``)-delimited + list. + + .. code-block:: shell + :class: copyable + + mc admin config set notify_amqp \ + url="amqp://user:password@endpoint:port" \ + [ARGUMENT="VALUE"] ... \ + + You can specify multiple AMQP service endpoints by appending ``[:name]`` to + the top level key. For example, the following commands set two distinct AMQP + service endpoints as ``primary`` and ``secondary`` respectively: + + .. code-block:: shell + + mc admin config set notify_amqp:primary \ + url="user:password@amqp://endpoint:port" [ARGUMENT=VALUE ...] + + mc admin config set notify_amqp:secondary \ + url="user:password@amqp://endpoint:port" [ARGUMENT=VALUE ...] + + The :mc-conf:`notify_amqp` configuration key supports the following + arguments: + + .. mc-conf:: url + :delimiter: " " + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-amqp-url + :end-before: end-minio-notify-amqp-url + + This key corresponds to the :envvar:`MINIO_NOTIFY_AMQP_URL` environment + variable. + + .. mc-conf:: exchange + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-amqp-exchange + :end-before: end-minio-notify-amqp-exchange + + This field corresponds to the :envvar:`MINIO_NOTIFY_AMQP_EXCHANGE` + environment variable. + + .. mc-conf:: exchange_type + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-amqp-exchange-type + :end-before: end-minio-notify-amqp-exchange-type + + This field corresponds to the :envvar:`MINIO_NOTIFY_AMQP_EXCHANGE_TYPE` + environment variable. + + .. mc-conf:: routing_key + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-amqp-routing-key + :end-before: end-minio-notify-amqp-routing-key + + This field corresponds to the :envvar:`MINIO_NOTIFY_AMQP_ROUTING_KEY` + environment variable. + + .. mc-conf:: mandatory + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-amqp-mandatory + :end-before: end-minio-notify-amqp-mandatory + + This field corresponds to the :envvar:`MINIO_NOTIFY_AMQP_MANDATORY` + environment variable. + + .. mc-conf:: durable + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-amqp-durable + :end-before: end-minio-notify-amqp-durable + + This field corresponds to the :envvar:`MINIO_NOTIFY_AMQP_DURABLE` + environment variable. + + .. mc-conf:: no_wait + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-amqp-no-wait + :end-before: end-minio-notify-amqp-no-wait + + This field corresponds to the :envvar:`MINIO_NOTIFY_AMQP_NO_WAIT` + environment variable. + + .. mc-conf:: internal + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-amqp-internal + :end-before: end-minio-notify-amqp-internal + + This field corresponds to the :envvar:`MINIO_NOTIFY_AMQP_INTERNAL` + environment variable. + + .. explanation is very unclear. Need to revisit this. + + .. mc-conf:: auto_deleted + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-amqp-auto-deleted + :end-before: end-minio-notify-amqp-auto-deleted + + This field corresponds to the :envvar:`MINIO_NOTIFY_AMQP_AUTO_DELETED` + environment variable. + + .. mc-conf:: delivery_mode + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-amqp-delivery-mode + :end-before: end-minio-notify-amqp-delivery-mode + + This field corresponds to the :envvar:`MINIO_NOTIFY_AMQP_DELIVERY_MODE` + environment variable. + + .. mc-conf:: queue_dir + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-amqp-queue-dir + :end-before: end-minio-notify-amqp-queue-dir + + This field corresponds to the :envvar:`MINIO_NOTIFY_AMQP_QUEUE_DIR` + environment variable. + + .. mc-conf:: queue_limit + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-amqp-queue-limit + :end-before: end-minio-notify-amqp-queue-limit + + This field corresponds to the :envvar:`MINIO_NOTIFY_AMQP_QUEUE_LIMIT` + environment variable. + + .. mc-conf:: comment + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-amqp-comment + :end-before: end-minio-notify-amqp-comment + + This field corresponds to the :envvar:`MINIO_NOTIFY_AMQP_COMMENT` + environment variable. + +.. _minio-server-config-bucket-notification-mqtt: + +MQTT Service for Bucket Notifications +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The following section documents settings for configuring an MQTT +server/broker as a publishing target for :doc:`MinIO Bucket Notifications +`. See +:ref:`minio-bucket-notifications-publish-mqtt` for a tutorial on +using these configuration settings. + +.. mc-conf:: notify_mqtt + + The top-level configuration key for defining an MQTT server/broker endpoint + for use with :ref:`MinIO bucket notifications `. + + Use :mc-cmd:`mc admin config set` to set or update an MQTT server/broker + endpoint. The following arguments are *required* for each endpoint: + + - :mc-conf:`~notify_mqtt.broker` + - :mc-conf:`~notify_mqtt.topic` + - :mc-conf:`~notify_mqtt.username` *Optional if MQTT server/broker does not enforce authentication/authorization* + - :mc-conf:`~notify_mqtt.password` *Optional if MQTT server/broker does not enforce authentication/authorization* + + Specify additional optional arguments as a whitespace (``" "``)-delimited + list. + + .. code-block:: shell + :class: copyable + + mc admin config set notify_mqtt \ + broker="tcp://endpoint:port" \ + topic="minio/bucket-name/events/" \ + username="username" \ + password="password" \ + [ARGUMENT="VALUE"] ... \ + + You can specify multiple MQTT server/broker endpoints by appending + ``[:name]`` to the top level key. For example, the following commands set two + distinct MQTT service endpoints as ``primary`` and ``secondary`` + respectively: + + .. code-block:: shell + + mc admin config set notify_mqtt:primary \ + broker="tcp://endpoint:port" \ + topic="minio/bucket-name/events/" \ + username="username" \ + password="password" \ + [ARGUMENT="VALUE"] ... \ + + mc admin config set notify_mqtt:secondary \ + broker="tcp://endpoint:port" \ + topic="minio/bucket-name/events/" \ + username="username" \ + password="password" \ + [ARGUMENT="VALUE"] ... \ + + The :mc-conf:`notify_mqtt` configuration key supports the following + arguments: + + .. mc-conf:: broker + :delimiter: " " + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-mqtt-broker + :end-before: end-minio-notify-mqtt-broker + + This field corresponds to the :envvar:`MINIO_NOTIFY_MQTT_BROKER` + environment variable. + + .. mc-conf:: topic + :delimiter: " " + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-mqtt-topic + :end-before: end-minio-notify-mqtt-topic + + This field corresponds to the :envvar:`MINIO_NOTIFY_MQTT_TOPIC` + environment variable. + + .. mc-conf:: username + :delimiter: " " + + *Required if the MQTT server/broker enforces authentication/authorization* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-mqtt-username + :end-before: end-minio-notify-mqtt-username + + This field corresponds to the :envvar:`MINIO_NOTIFY_MQTT_TOPIC` + environment variable. + + .. mc-conf:: password + :delimiter: " " + + *Required if the MQTT server/broker enforces authentication/authorization* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-mqtt-password + :end-before: end-minio-notify-mqtt-password + + This field corresponds to the :envvar:`MINIO_NOTIFY_MQTT_PASSWORD` + environment variable. + + .. mc-conf:: qos + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-mqtt-qos + :end-before: end-minio-notify-mqtt-qos + + This field corresponds to the :envvar:`MINIO_NOTIFY_MQTT_QOS` + environment variable. + + .. mc-conf:: keep_alive_interval + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-mqtt-keep-alive-interval + :end-before: end-minio-notify-mqtt-keep-alive-interval + + This field corresponds to the :envvar:`MINIO_NOTIFY_MQTT_KEEP_ALIVE_INTERVAL` + environment variable. + + .. mc-conf:: reconnect_interval + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-mqtt-reconnect-interval + :end-before: end-minio-notify-mqtt-reconnect-interval + + This field corresponds to the :envvar:`MINIO_NOTIFY_MQTT_RECONNECT_INTERVAL` + environment variable. + + .. mc-conf:: queue_dir + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-mqtt-queue-dir + :end-before: end-minio-notify-mqtt-queue-dir + + This field corresponds to the :envvar:`MINIO_NOTIFY_MQTT_QUEUE_DIR` + environment variable. + + .. mc-conf:: queue_limit + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-mqtt-queue-limit + :end-before: end-minio-notify-mqtt-queue-limit + + This field corresponds to the :envvar:`MINIO_NOTIFY_MQTT_QUEUE_LIMIT` + environment variable. + + .. mc-conf:: comment + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-mqtt-comment + :end-before: end-minio-notify-mqtt-comment + + This field corresponds to the :envvar:`MINIO_NOTIFY_MQTT_COMMENT` + environment variable. + +.. _minio-server-config-bucket-notification-elasticsearch: + +Elasticsearch Service for Bucket Notifications +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The following section documents settings for configuring an Elasticsearch +service as a target for :doc:`MinIO Bucket Notifications +`. See +:ref:`minio-bucket-notifications-publish-elasticsearch` for a tutorial on using +these configuration settings. + +.. mc-conf:: notify_elasticsearch + + The top-level configuration key for defining an Elasticsearch service + endpoint for use with :ref:`MinIO bucket notifications + `. + + Use :mc-cmd:`mc admin config set` to set or update an Elasticsearch service + endpoint. The following arguments are *required* for each target: + + - :mc-conf:`~notify_elasticsearch.url` + - :mc-conf:`~notify_elasticsearch.index` + - :mc-conf:`~notify_elasticsearch.format` + + Specify additional optional arguments as a whitespace (``" "``)-delimited + list. + + .. code-block:: shell + :class: copyable + + mc admin config set notify_elasticsearch \ + url="https://user:password@endpoint:port" \ + [ARGUMENT="VALUE"] ... \ + + You can specify multiple Elasticsearch service endpoints by appending + ``[:name]`` to the top level key. For example, the following commands set two + distinct Elasticsearch service endpoints as ``primary`` and ``secondary`` + respectively: + + .. code-block:: shell + + mc admin config set notify_elasticsearch:primary \ + url="user:password@https://endpoint:port" [ARGUMENT=VALUE ...] + + mc admin config set notify_elasticsearch:secondary \ + url="user:password@https://endpoint:port" [ARGUMENT=VALUE ...] + + The :mc-conf:`notify_elasticsearch` configuration key supports the following + arguments: + + .. mc-conf:: url + :delimiter: " " + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-elasticsearch-url + :end-before: end-minio-notify-elasticsearch-url + + This field corresponds to the + :envvar:`MINIO_NOTIFY_ELASTICSEARCH_URL` environment variable. + + .. mc-conf:: index + :delimiter: " " + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-elasticsearch-index + :end-before: end-minio-notify-elasticsearch-index + + This field corresponds to the + :envvar:`MINIO_NOTIFY_ELASTICSEARCH_INDEX` environment variable. + + .. mc-conf:: format + :delimiter: " " + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-elasticsearch-format + :end-before: end-minio-notify-elasticsearch-format + + This field corresponds to the + :envvar:`MINIO_NOTIFY_ELASTICSEARCH_FORMAT` environment variable. + + .. mc-conf:: username + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-elasticsearch-username + :end-before: end-minio-notify-elasticsearch-username + + This field corresponds to the + :envvar:`MINIO_NOTIFY_ELASTICSEARCH_USERNAME` environment variable. + + .. mc-conf:: password + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-elasticsearch-password + :end-before: end-minio-notify-elasticsearch-password + + This field corresponds to the + :envvar:`MINIO_NOTIFY_ELASTICSEARCH_PASSWORD` environment variable. + + + .. mc-conf:: queue_dir + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-elasticsearch-queue-dir + :end-before: end-minio-notify-elasticsearch-queue-dir + + This field corresponds to the + :envvar:`MINIO_NOTIFY_ELASTICSEARCH_QUEUE_DIR` environment variable. + + .. mc-conf:: queue_limit + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-elasticsearch-queue-limit + :end-before: end-minio-notify-elasticsearch-queue-limit + + This field corresponds to the + :envvar:`MINIO_NOTIFY_ELASTICSEARCH_QUEUE_LIMIT` environment variable. + + .. mc-conf:: comment + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-elasticsearch-comment + :end-before: end-minio-notify-elasticsearch-comment + + This field corresponds to the :envvar:`MINIO_NOTIFY_ELASTICSEARCH_COMMENT` + environment variable. + + +.. _minio-server-config-bucket-notification-nsq: + +NSQ Service for Bucket Notifications +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The following section documents settings for configuring an NSQ +server/broker as a publishing target for :doc:`MinIO Bucket Notifications +`. See +:ref:`minio-bucket-notifications-publish-nsq` for a tutorial on +using these configuration settings. + +.. mc-conf:: notify_nsq + + The top-level configuration key for defining an NSQ server/broker endpoint + for use with :ref:`MinIO bucket notifications `. + + Use :mc-cmd:`mc admin config set` to set or update an NSQ server/broker + endpoint. The following arguments are *required* for each endpoint: + + - :mc-conf:`~notify_nsq.nsqd_address` + - :mc-conf:`~notify_nsq.topic` + + Specify additional optional arguments as a whitespace (``" "``)-delimited + list. + + .. code-block:: shell + :class: copyable + + mc admin config set notify_nsq \ + nsqd_address="ENDPOINT" \ + topic="" \ + [ARGUMENT="VALUE"] ... \ + + You can specify multiple NSQ server/broker endpoints by appending + ``[:name]`` to the top level key. For example, the following commands set two + distinct NSQ service endpoints as ``primary`` and ``secondary`` + respectively: + + .. code-block:: shell + + mc admin config set notify_nsq:primary \ + nsqd_address="ENDPOINT" \ + topic="" \ + [ARGUMENT="VALUE"] ... \ + + mc admin config set notify_nsq:secondary \ + nsqd_address="ENDPOINT" \ + topic="" \ + [ARGUMENT="VALUE"] ... \ + + The :mc-conf:`notify_nsq` configuration key supports the following + arguments: + + + .. mc-conf:: nsqd_address + :delimiter: " " + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nsq-nsqd-address + :end-before: end-minio-notify-nsq-nsqd-address + + This configuration setting corresponds to the + :envvar:`MINIO_NOTIFY_NSQ_NSQD_ADDRESS` environment variable. + + .. mc-conf:: topic + :delimiter: " " + + *Required* + + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nsq-topic + :end-before: end-minio-notify-nsq-topic + + This configuration setting corresponds to the + :envvar:`MINIO_NOTIFY_NSQ_TOPIC` environment variable. + + .. mc-conf:: tls + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nsq-tls + :end-before: end-minio-notify-nsq-tls + + This configuration setting corresponds to the + :envvar:`MINIO_NOTIFY_NSQ_TLS` environment variable. + + + .. mc-conf:: tls_skip_verify + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nsq-tls-skip-verify + :end-before: end-minio-notify-nsq-tls-skip-verify + + This configuration setting corresponds to the + :envvar:`MINIO_NOTIFY_NSQ_TLS_SKIP_VERIFY` environment variable. + + + .. mc-conf:: queue_dir + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nsq-queue-dir + :end-before: end-minio-notify-nsq-queue-dir + + This configuration setting corresponds to the + :envvar:`MINIO_NOTIFY_NSQ_QUEUE_DIR` environment variable. + + + .. mc-conf:: queue_limit + :delimiter: " " + + *Optional* + + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nsq-queue-limit + :end-before: end-minio-notify-nsq-queue-limit + + This configuration setting corresponds to the + :envvar:`MINIO_NOTIFY_NSQ_QUEUE_LIMIT` environment variable. + + + .. mc-conf:: comment + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nsq-comment + :end-before: end-minio-notify-nsq-comment + + This configuration setting corresponds to the + :envvar:`MINIO_NOTIFY_NSQ_COMMENT` environment variable. + + +.. _minio-server-config-bucket-notification-redis: + +Redis Service for Bucket Notifications +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The following section documents settings for configuring an Redis +server/broker as a publishing target for :doc:`MinIO Bucket Notifications +`. See +:ref:`minio-bucket-notifications-publish-redis` for a tutorial on +using these configuration settings. + +.. mc-conf:: notify_redis + + The top-level configuration key for defining an Redis server/broker endpoint + for use with :ref:`MinIO bucket notifications `. + + Use :mc-cmd:`mc admin config set` to set or update an Redis server/broker + endpoint. The following arguments are *required* for each endpoint: + + - :mc-conf:`~notify_redis.address` + - :mc-conf:`~notify_redis.key` + - :mc-conf:`~notify_redis.format` + + Specify additional optional arguments as a whitespace (``" "``)-delimited + list. + + .. code-block:: shell + :class: copyable + + mc admin config set notify_redis \ + address="ENDPOINT" \ + key="" \ + format="" \ + [ARGUMENT="VALUE"] ... \ + + You can specify multiple Redis server/broker endpoints by appending + ``[:name]`` to the top level key. For example, the following commands set two + distinct Redis service endpoints as ``primary`` and ``secondary`` + respectively: + + .. code-block:: shell + + mc admin config set notify_redis:primary \ + address="ENDPOINT" \ + key="" \ + format="" \ + [ARGUMENT="VALUE"] ... \ + + mc admin config set notify_redis:secondary \ + address="ENDPOINT" \ + key="" \ + format="" \ + [ARGUMENT="VALUE"] ... \ + + The :mc-conf:`notify_redis` configuration key supports the following + arguments: + + .. mc-conf:: address + :delimiter: " " + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-redis-address + :end-before: end-minio-notify-redis-address + + This configuration setting corresponds to the + :envvar:`MINIO_NOTIFY_REDIS_ADDRESS` environment variable. + + .. mc-conf:: key + :delimiter: " " + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-redis-key + :end-before: end-minio-notify-redis-key + + This configuration setting corresponds to the + :envvar:`MINIO_NOTIFY_REDIS_KEY` environment variable. + + .. mc-conf:: format + :delimiter: " " + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-redis-format + :end-before: end-minio-notify-redis-format + + This configuration setting corresponds to the + :envvar:`MINIO_NOTIFY_REDIS_FORMAT` environment variable. + + .. mc-conf:: password + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-redis-password + :end-before: end-minio-notify-redis-password + + This configuration setting corresponds to the + :envvar:`MINIO_NOTIFY_REDIS_PASSWORD` environment variable. + + .. mc-conf:: queue_dir + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-redis-queue-dir + :end-before: end-minio-notify-redis-queue-dir + + This configuration setting corresponds to the + :envvar:`MINIO_NOTIFY_REDIS_QUEUE_DIR` environment variable. + + .. mc-conf:: queue_limit + :delimiter: " " + + *Optional* + + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-redis-queue-limit + :end-before: end-minio-notify-redis-queue-limit + + This configuration setting corresponds to the + :envvar:`MINIO_NOTIFY_REDIS_QUEUE_LIMIT` environment variable. + + + .. mc-conf:: comment + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-redis-comment + :end-before: end-minio-notify-redis-comment + + This configuration setting corresponds to the + :envvar:`MINIO_NOTIFY_REDIS_COMMENT` environment variable. + + + +.. _minio-server-config-bucket-notification-nats: + +NATS Service for Bucket Notifications +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The following section documents settings for configuring an NATS +service as a target for :doc:`MinIO Bucket Notifications +`. See +:ref:`minio-bucket-notifications-publish-nats` for a tutorial on +using these environment variables. + +.. mc-conf:: notify_nats + + The top-level configuration key for defining an NATS service endpoint for use + with :ref:`MinIO bucket notifications `. + + Use :mc-cmd:`mc admin config set` to set or update an NATS service endpoint. + The :mc-conf:`~notify_nats.address` and + :mc-conf:`~notify_nats.subject` arguments are *required* for each target. + Specify additional optional arguments as a whitespace (``" "``)-delimited + list. + + .. code-block:: shell + :class: copyable + + mc admin config set notify_nats \ + address="htpps://nats-endpoint.example.com:4222" \ + subject="minioevents" \ + [ARGUMENT="VALUE"] ... \ + + You can specify multiple NATS service endpoints by appending ``[:name]`` to + the top level key. For example, the following commands set two distinct NATS + service endpoints as ``primary`` and ``secondary`` respectively: + + .. code-block:: shell + + mc admin config set notify_nats:primary \ + address="htpps://nats-endpoint.example.com:4222" \ + subject="minioevents" \ + [ARGUMENT=VALUE ...] + + mc admin config set notify_nats:secondary \ + address="htpps://nats-endpoint.example.com:4222" \ + subject="minioevents" \ + [ARGUMENT=VALUE ...] + + The :mc-conf:`notify_nats` configuration key supports the following + arguments: + + .. mc-conf:: address + :delimiter: " " + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nats-address + :end-before: end-minio-notify-nats-address + + This configuration setting corresponds with the environment variable + :envvar:`MINIO_NOTIFY_NATS_ADDRESS`. + + .. mc-conf:: subject + :delimiter: " " + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nats-subject + :end-before: end-minio-notify-nats-subject + + This configuration setting corresponds with the environment variable + :envvar:`MINIO_NOTIFY_NATS_SUBJECT`. + + .. mc-conf:: username + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nats-username + :end-before: end-minio-notify-nats-username + + This configuration setting corresponds with the environment variable + :envvar:`MINIO_NOTIFY_NATS_USERNAME`. + + .. mc-conf:: password + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nats-password + :end-before: end-minio-notify-nats-password + + This configuration setting corresponds with the environment variable + :envvar:`MINIO_NOTIFY_NATS_PASSWORD`. + + .. mc-conf:: token + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nats-token + :end-before: end-minio-notify-nats-token + + This configuration setting corresponds with the environment variable + :envvar:`MINIO_NOTIFY_NATS_TOKEN`. + + .. mc-conf:: tls + :delimiter: " + + *Optional*" + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nats-tls + :end-before: end-minio-notify-nats-tls + + This configuration setting corresponds with the environment variable + :envvar:`MINIO_NOTIFY_NATS_TLS`. + + .. mc-conf:: tls_skip_verify + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nats-tls-skip-verify + :end-before: end-minio-notify-nats-tls-skip-verify + + This configuration setting corresponds with the environment variable + :envvar:`MINIO_NOTIFY_NATS_TLS_SKIP_VERIFY`. + + .. mc-conf:: ping_interval + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nats-ping-interval + :end-before: end-minio-notify-nats-ping-interval + + This configuration setting corresponds with the environment variable + :envvar:`MINIO_NOTIFY_NATS_PING_INTERVAL`. + + .. mc-conf:: streaming + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nats-streaming + :end-before: end-minio-notify-nats-streaming + + This configuration setting corresponds with the environment variable + :envvar:`MINIO_NOTIFY_NATS_STREAMING`. + + .. mc-conf:: streaming_async + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nats-streaming-async + :end-before: end-minio-notify-nats-streaming-async + + This configuration setting corresponds with the environment variable + :envvar:`MINIO_NOTIFY_NATS_STREAMING_ASYNC`. + + .. mc-conf:: streaming_max_pub_acks_in_flight + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nats-streaming-max-pub-acks-in-flight + :end-before: end-minio-notify-nats-streaming-max-pub-acks-in-flight + + This configuration setting corresponds with the environment variable + :envvar:`MINIO_NOTIFY_NATS_STREAMING_MAX_PUB_ACKS_IN_FLIGHT`. + + .. mc-conf:: streaming_cluster_id + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nats-streaming-cluster-id + :end-before: end-minio-notify-nats-streaming-cluster-id + + This configuration setting corresponds with the environment variable + :envvar:`MINIO_NOTIFY_NATS_STREAMING_CLUSTER_ID`. + + .. mc-conf:: cert_authority + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nats-cert-authority + :end-before: end-minio-notify-nats-cert-authority + + This configuration setting corresponds with the environment variable + :envvar:`MINIO_NOTIFY_NATS_CERT_AUTHORITY`. + + .. mc-conf:: client_cert + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nats-client-cert + :end-before: end-minio-notify-nats-client-cert + + This configuration setting corresponds with the environment variable + :envvar:`MINIO_NOTIFY_NATS_CLIENT_CERT`. + + .. mc-conf:: client_key + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nats-client-key + :end-before: end-minio-notify-nats-client-key + + This configuration setting corresponds with the environment variable + :envvar:`MINIO_NOTIFY_NATS_CLIENT_KEY`. + + + .. mc-conf:: queue_dir + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nats-queue-dir + :end-before: end-minio-notify-nats-queue-dir + + This configuration setting corresponds to the + :envvar:`MINIO_NOTIFY_NATS_QUEUE_DIR` environment variable. + + .. mc-conf:: queue_limit + :delimiter: " " + + *Optional* + + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nats-queue-limit + :end-before: end-minio-notify-nats-queue-limit + + This configuration setting corresponds to the + :envvar:`MINIO_NOTIFY_NATS_QUEUE_LIMIT` environment variable. + + + .. mc-conf:: comment + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nats-comment + :end-before: end-minio-notify-nats-comment + + This configuration setting corresponds to the + :envvar:`MINIO_NOTIFY_NATS_COMMENT` environment variable. + +.. _minio-server-config-bucket-notification-postgresql: + +PostgreSQL Service for Bucket Notifications +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The following section documents settings for configuring an PostgreSQL +service as a target for :doc:`MinIO Bucket Notifications +`. See +:ref:`minio-bucket-notifications-publish-postgresql` for a tutorial on +using these environment variables. + +.. mc-conf:: notify_postgresql + + The top-level configuration key for defining an PostgreSQL service endpoint for use + with :ref:`MinIO bucket notifications `. + + Use :mc-cmd:`mc admin config set` to set or update an PostgreSQL service endpoint. + The following arguments are *required* for each target: + + - :mc-conf:`~notify_postgresql.connection_string` + - :mc-conf:`~notify_postgresql.table` + - :mc-conf:`~notify_postgresql.format` + + Specify additional optional arguments as a whitespace (``" "``)-delimited + list. + + .. code-block:: shell + :class: copyable + + mc admin config set notify_postgresql \ + connection_string="host=postgresql.example.com port=5432..." + table="minioevents" \ + format="namespace" \ + [ARGUMENT="VALUE"] ... \ + + You can specify multiple PostgreSQL service endpoints by appending ``[:name]`` to + the top level key. For example, the following commands set two distinct PostgreSQL + service endpoints as ``primary`` and ``secondary`` respectively: + + .. code-block:: shell + + mc admin config set notify_postgresql:primary \ + connection_string="host=postgresql.example.com port=5432..." + table="minioevents" \ + format="namespace" \ + [ARGUMENT=VALUE ...] + + mc admin config set notify_postgresql:secondary \ + connection_string="host=postgresql.example.com port=5432..." + table="minioevents" \ + format="namespace" \ + [ARGUMENT=VALUE ...] + + The :mc-conf:`notify_postgresql` configuration key supports the following + arguments: + + .. mc-conf:: connection_string + :delimiter: " " + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-postgresql-connection-string + :end-before: end-minio-notify-postgresql-connection-string + + This configuration setting corresponds to the + :envvar:`MINIO_NOTIFY_POSTGRESQL_CONNECTION_STRING` environment + variable. + + .. mc-conf:: table + :delimiter: " " + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-postgresql-table + :end-before: end-minio-notify-postgresql-table + + This configuration setting corresponds to the + :envvar:`MINIO_NOTIFY_POSTGRESQL_TABLE` environment + variable. + + .. mc-conf:: format + :delimiter: " " + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-postgresql-format + :end-before: end-minio-notify-postgresql-format + + This configuration setting corresponds to the + :envvar:`MINIO_NOTIFY_POSTGRESQL_FORMAT` environment + variable. + + .. mc-conf:: max_open_connections + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-postgresql-max-open-connections + :end-before: end-minio-notify-postgresql-max-open-connections + + This configuration setting corresponds to the + :envvar:`MINIO_NOTIFY_POSTGRESQL_MAX_OPEN_CONNECTIONS` environment + variable. + + + .. mc-conf:: queue_dir + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-postgresql-queue-dir + :end-before: end-minio-notify-postgresql-queue-dir + + This configuration setting corresponds to the + :envvar:`MINIO_NOTIFY_POSTGRESQL_QUEUE_DIR` environment variable. + + .. mc-conf:: queue_limit + :delimiter: " " + + *Optional* + + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-postgresql-queue-limit + :end-before: end-minio-notify-postgresql-queue-limit + + This configuration setting corresponds to the + :envvar:`MINIO_NOTIFY_POSTGRESQL_QUEUE_LIMIT` environment variable. + + + .. mc-conf:: comment + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-postgresql-comment + :end-before: end-minio-notify-postgresql-comment + + This configuration setting corresponds to the + :envvar:`MINIO_NOTIFY_POSTGRESQL_COMMENT` environment variable. + +.. _minio-server-config-bucket-notification-mysql: + +MySQL Service for Bucket Notifications +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The following section documents settings for configuring an MySQL +service as a target for :doc:`MinIO Bucket Notifications +`. See +:ref:`minio-bucket-notifications-publish-mysql` for a tutorial on +using these environment variables. + +.. mc-conf:: notify_mysql + + The top-level configuration key for defining an MySQL service endpoint for use + with :ref:`MinIO bucket notifications `. + + Use :mc-cmd:`mc admin config set` to set or update an MySQL service endpoint. + The following arguments are *required* for each target: + + - :mc-conf:`~notify_mysql.dsn_string` + - :mc-conf:`~notify_mysql.table` + - :mc-conf:`~notify_mysql.format` + + Specify additional optional arguments as a whitespace (``" "``)-delimited + list. + + .. code-block:: shell + :class: copyable + + mc admin config set notify_mysql \ + dsn_string="username:password@tcp(mysql.example.com:3306)/miniodb" + table="minioevents" \ + format="namespace" \ + [ARGUMENT="VALUE"] ... \ + + You can specify multiple MySQL service endpoints by appending ``[:name]`` to + the top level key. For example, the following commands set two distinct MySQL + service endpoints as ``primary`` and ``secondary`` respectively: + + .. code-block:: shell + + mc admin config set notify_mysql:primary \ + dsn_string="username:password@tcp(mysql.example.com:3306)/miniodb" + table="minioevents" \ + format="namespace" \ + [ARGUMENT=VALUE ...] + + mc admin config set notify_mysql:secondary \ + dsn_string="username:password@tcp(mysql.example.com:3306)/miniodb" + table="minioevents" \ + format="namespace" \ + [ARGUMENT=VALUE ...] + + The :mc-conf:`notify_mysql` configuration key supports the following + arguments: + + .. mc-conf:: dsn_string + :delimiter: " " + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-mysql-connection-string + :end-before: end-minio-notify-mysql-connection-string + + This configuration setting corresponds to the + :envvar:`MINIO_NOTIFY_MYSQL_DSN_STRING` environment + variable. + + .. mc-conf:: table + :delimiter: " " + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-mysql-table + :end-before: end-minio-notify-mysql-table + + This configuration setting corresponds to the + :envvar:`MINIO_NOTIFY_MYSQL_TABLE` environment + variable. + + .. mc-conf:: format + :delimiter: " " + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-mysql-format + :end-before: end-minio-notify-mysql-format + + This configuration setting corresponds to the + :envvar:`MINIO_NOTIFY_MYSQL_FORMAT` environment + variable. + + .. mc-conf:: max_open_connections + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-mysql-max-open-connections + :end-before: end-minio-notify-mysql-max-open-connections + + This configuration setting corresponds to the + :envvar:`MINIO_NOTIFY_MYSQL_MAX_OPEN_CONNECTIONS` environment + variable. + + + .. mc-conf:: queue_dir + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-mysql-queue-dir + :end-before: end-minio-notify-mysql-queue-dir + + This configuration setting corresponds to the + :envvar:`MINIO_NOTIFY_MYSQL_QUEUE_DIR` environment variable. + + .. mc-conf:: queue_limit + :delimiter: " " + + *Optional* + + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-mysql-queue-limit + :end-before: end-minio-notify-mysql-queue-limit + + This configuration setting corresponds to the + :envvar:`MINIO_NOTIFY_MYSQL_QUEUE_LIMIT` environment variable. + + + .. mc-conf:: comment + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-mysql-comment + :end-before: end-minio-notify-mysql-comment + + This configuration setting corresponds to the + :envvar:`MINIO_NOTIFY_MYSQL_COMMENT` environment variable. + +.. _minio-server-config-bucket-notification-kafka: + +Kafka Service for Bucket Notifications +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The following section documents settings for configuring an Kafka +service as a target for :doc:`MinIO Bucket Notifications +`. See +:ref:`minio-bucket-notifications-publish-kafka` for a tutorial on +using these environment variables. + +.. mc-conf:: notify_kafka + + The top-level configuration key for defining an Kafka service endpoint for use + with :ref:`MinIO bucket notifications `. + + Use :mc-cmd:`mc admin config set` to set or update an Kafka service endpoint. + The :mc-conf:`~notify_kafka.brokers` argument is *required* for each target. + Specify additional optional arguments as a whitespace (``" "``)-delimited + list. + + .. code-block:: shell + :class: copyable + + mc admin config set notify_kafka \ + brokers="https://kafka1.example.net:9200, https://kafka2.example.net:9200" + [ARGUMENT="VALUE"] ... \ + + You can specify multiple Kafka service endpoints by appending ``[:name]`` to + the top level key. For example, the following commands set two distinct Kafka + service endpoints as ``primary`` and ``secondary`` respectively: + + .. code-block:: shell + + mc admin config set notify_kafka:primary \ + brokers="https://kafka1.example.net:9200, https://kafka2.example.net:9200" + [ARGUMENT=VALUE ...] + + mc admin config set notify_kafka:secondary \ + brokers="https://kafka1.example.net:9200, https://kafka2.example.net:9200" + [ARGUMENT=VALUE ...] + + The :mc-conf:`notify_kafka` configuration key supports the following + arguments: + + .. mc-conf:: brokers + :delimiter: " " + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-kafka-brokers + :end-before: end-minio-notify-kafka-brokers + + This configuration setting corresponds to the + :ENVVAR:`MINIO_NOTIFY_KAFKA_BROKERS` ENVIRONMENT VARIABLE + + .. mc-conf:: topic + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-kafka-topic + :end-before: end-minio-notify-kafka-topic + + This configuration setting corresponds to the + :ENVVAR:`MINIO_NOTIFY_KAFKA_TOPIC` ENVIRONMENT VARIABLE + + .. mc-conf:: sasl_username + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-kafka-sasl-username + :end-before: end-minio-notify-kafka-sasl-username + + This configuration setting corresponds to the + :ENVVAR:`MINIO_NOTIFY_KAFKA_SASL_USERNAME` ENVIRONMENT VARIABLE + + .. mc-conf:: sasl_password + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-kafka-sasl-password + :end-before: end-minio-notify-kafka-sasl-password + + This configuration setting corresponds to the + :ENVVAR:`MINIO_NOTIFY_KAFKA_SASL_PASSWORD` ENVIRONMENT VARIABLE + + .. mc-conf:: sasl_mechanism + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-kafka-sasl-mechanism + :end-before: end-minio-notify-kafka-sasl-mechanism + + This configuration setting corresponds to the + :ENVVAR:`MINIO_NOTIFY_KAFKA_SASL_MECHANISM` ENVIRONMENT VARIABLE + + .. mc-conf:: tls_client_auth + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-kafka-tls-client-auth + :end-before: end-minio-notify-kafka-tls-client-auth + + This configuration setting corresponds to the + :ENVVAR:`MINIO_NOTIFY_KAFKA_TLS_CLIENT_AUTH` ENVIRONMENT VARIABLE + + .. mc-conf:: sasl + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-kafka-sasl + :end-before: end-minio-notify-kafka-sasl + + This configuration setting corresponds to the + :ENVVAR:`MINIO_NOTIFY_KAFKA_SASL` ENVIRONMENT VARIABLE + + .. mc-conf:: tls + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-kafka-tls + :end-before: end-minio-notify-kafka-tls + + This configuration setting corresponds to the + :ENVVAR:`MINIO_NOTIFY_KAFKA_TLS` ENVIRONMENT VARIABLE + + .. mc-conf:: tls_skip_verify + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-kafka-tls-skip-verify + :end-before: end-minio-notify-kafka-tls-skip-verify + + This configuration setting corresponds to the + :ENVVAR:`MINIO_NOTIFY_KAFKA_TLS_SKIP_VERIFY` ENVIRONMENT VARIABLE + + .. mc-conf:: client_tls_cert + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-kafka-client-tls-cert + :end-before: end-minio-notify-kafka-client-tls-cert + + This configuration setting corresponds to the + :ENVVAR:`MINIO_NOTIFY_KAFKA_CLIENT_TLS_CERT` ENVIRONMENT VARIABLE + + .. mc-conf:: client_tls_key + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-kafka-client-tls-key + :end-before: end-minio-notify-kafka-client-tls-key + + This configuration setting corresponds to the + :ENVVAR:`MINIO_NOTIFY_KAFKA_CLIENT_TLS_KEY` ENVIRONMENT VARIABLE + + .. mc-conf:: version + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-kafka-version + :end-before: end-minio-notify-kafka-version + + This configuration setting corresponds to the + :ENVVAR:`MINIO_NOTIFY_KAFKA_VERSION` ENVIRONMENT VARIABLE + + + .. mc-conf:: queue_dir + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-kafka-queue-dir + :end-before: end-minio-notify-kafka-queue-dir + + This configuration setting corresponds to the + :envvar:`MINIO_NOTIFY_KAFKA_QUEUE_DIR` environment variable. + + .. mc-conf:: queue_limit + :delimiter: " " + + *Optional* + + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-kafka-queue-limit + :end-before: end-minio-notify-kafka-queue-limit + + This configuration setting corresponds to the + :envvar:`MINIO_NOTIFY_KAFKA_QUEUE_LIMIT` environment variable. + + + .. mc-conf:: comment + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-kafka-comment + :end-before: end-minio-notify-kafka-comment + + This configuration setting corresponds to the + :envvar:`MINIO_NOTIFY_KAFKA_COMMENT` environment variable. + +.. _minio-server-config-bucket-notification-webhook: + +Webhook Service for Bucket Notifications +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The following section documents settings for configuring an Webhook +service as a target for :doc:`MinIO Bucket Notifications +`. See +:ref:`minio-bucket-notifications-publish-webhook` for a tutorial on +using these environment variables. + +.. mc-conf:: notify_webhook + + The top-level configuration key for defining an Webhook service endpoint for use + with :ref:`MinIO bucket notifications `. + + Use :mc-cmd:`mc admin config set` to set or update an Webhook service endpoint. + The :mc-conf:`~notify_webhook.endpoint` argument is *required* for each target. + Specify additional optional arguments as a whitespace (``" "``)-delimited + list. + + .. code-block:: shell + :class: copyable + + mc admin config set notify_webhook \ + endpoint="https://webhook.example.net" + [ARGUMENT="VALUE"] ... \ + + You can specify multiple Webhook service endpoints by appending ``[:name]`` to + the top level key. For example, the following commands set two distinct Webhook + service endpoints as ``primary`` and ``secondary`` respectively: + + .. code-block:: shell + + mc admin config set notify_webhook:primary \ + endpoint="https://webhook1.example.net" + [ARGUMENT=VALUE ...] + + mc admin config set notify_webhook:secondary \ + endpoint="https://webhook2.example.net + [ARGUMENT=VALUE ...] + + The :mc-conf:`notify_webhook` configuration key supports the following + arguments: + + .. mc-conf:: endpoint + :delimiter: " " + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-webhook-endpoint + :end-before: end-minio-notify-webhook-endpoint + + This configuration setting corresponds with the + :envvar:`MINIO_NOTIFY_WEBHOOK_ENDPOINT` environment variable. + + .. mc-conf:: auth_token + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-webhook-auth-token + :end-before: end-minio-notify-webhook-auth-token + + This configuration setting corresponds with the + :envvar:`MINIO_NOTIFY_WEBHOOK_AUTH_TOKEN` environment variable. + + .. mc-conf:: queue_dir + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-webhook-queue-dir + :end-before: end-minio-notify-webhook-queue-dir + + This configuration setting corresponds with the + :envvar:`MINIO_NOTIFY_WEBHOOK_QUEUE_DIR` environment variable. + + .. mc-conf:: queue_limit + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-webhook-queue-limit + :end-before: end-minio-notify-webhook-queue-limit + + This configuration setting corresponds with the + :envvar:`MINIO_NOTIFY_WEBHOOK_QUEUE_LIMIT` environment variable. + + .. mc-conf:: client_cert + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-webhook-client-cert + :end-before: end-minio-notify-webhook-client-cert + + This configuration setting corresponds with the + :envvar:`MINIO_NOTIFY_WEBHOOK_CLIENT_CERT` environment variable. + + .. mc-conf:: client_key + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-webhook-client-key + :end-before: end-minio-notify-webhook-client-key + + This configuration setting corresponds with the + :envvar:`MINIO_NOTIFY_WEBHOOK_CLIENT_KEY` environment variable. + + .. mc-conf:: comment + :delimiter: " " + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-webhook-comment + :end-before: end-minio-notify-webhook-comment + + This configuration setting corresponds with the + :envvar:`MINIO_NOTIFY_WEBHOOK_COMMENT` environment variable. diff --git a/source/reference/minio-cli/minio-mc/mc-event.rst b/source/reference/minio-cli/minio-mc/mc-event.rst index 77929449..017482eb 100644 --- a/source/reference/minio-cli/minio-mc/mc-event.rst +++ b/source/reference/minio-cli/minio-mc/mc-event.rst @@ -19,10 +19,10 @@ The :mc:`mc event` command supports adding, removing, and listing the bucket event notifications. MinIO automatically sends triggered events to the configured notification -targets. MinIO supports notification targets like AMQP, Redis, ElasticSearch, -NATS and PostgreSQL. See -:doc:`MinIO Bucket Notifications ` -for more information. +targets. MinIO supports notification targets like AMQP (RabbitMQ), Redis, +ElasticSearch, NATS and PostgreSQL. See +:doc:`MinIO Bucket Notifications ` for more +information. .. end-mc-event-desc @@ -31,37 +31,30 @@ for more information. Supported Bucket Events ~~~~~~~~~~~~~~~~~~~~~~~ -The following table lists the supported S3 Event and the corresponding -:mc:`mc event` alias: +MinIO supports the following S3 event types: -.. list-table:: - :header-rows: 1 - :widths: 20 80 - :width: 100% - - * - MinIO Alias - - Corresponding S3 Event - - * - ``put`` - - ``s3:ObjectCreated:Put`` - - * - ``completeMultipartUpload`` - - ``s3:ObjectCreated:CompleteMultipartUpload`` - - * - ``head`` - - ``s3:ObjectAccessed:Head`` - - * - ``post`` - - ``s3:ObjectCreated:Post`` - - * - ``delete`` - - ``s3:ObjectRemoved:Delete`` - - * - ``copy`` - - ``s3:ObjectCreated:Copy`` - - * - ``get`` - - ``s3:ObjectAccessed:Get`` +- :data:`s3:ObjectRemoved:DeleteMarkerCreated` +- :data:`s3:ObjectRemoved:Delete` +- :data:`s3:ObjectCreated:PutRetention` +- :data:`s3:ObjectCreated:PutLegalHold` +- :data:`s3:ObjectCreated:Put` +- :data:`s3:ObjectCreated:Post` +- :data:`s3:ObjectCreated:Copy` +- :data:`s3:ObjectCreated:CompleteMultipartUpload` +- :data:`s3:ObjectAccessed:Head` +- :data:`s3:ObjectAccessed:GetRetention` +- :data:`s3:ObjectAccessed:GetLegalHold` +- :data:`s3:ObjectAccessed:Get` +- :data:`s3:Replication:OperationCompletedReplication` +- :data:`s3:Replication:OperationFailedReplication` +- :data:`s3:Replication:OperationMissedThreshold` +- :data:`s3:Replication:OperationNotTracked` +- :data:`s3:Replication:OperationReplicatedAfterThreshold` +- :data:`s3:ObjectRestore:Post` +- :data:`s3:ObjectRestore:Completed` +- :data:`s3:BucketCreated` +- :data:`s3:BucketRemoved` + For more complete documentation on the listed S3 events, see :s3-docs:`S3 Supported Event Types @@ -171,7 +164,7 @@ Syntax The MinIO server outputs an ARN for each configured notification target at server startup. See - :doc:`/concepts/bucket-notifications` for more + :doc:`/monitoring/bucket-notifications/bucket-notifications` for more information. .. mc-cmd:: event @@ -232,7 +225,7 @@ Syntax The MinIO server outputs an ARN for each configured notification target at server startup. See - :doc:`/concepts/bucket-notifications` for more information. + :doc:`/monitoring/bucket-notifications/bucket-notifications` for more information. .. mc-cmd:: force :option: @@ -302,7 +295,7 @@ Syntax The MinIO server outputs an ARN for each configured notification target at server startup. See - :doc:`/concepts/bucket-notifications` for more information. + :doc:`/monitoring/bucket-notifications/bucket-notifications` for more information. diff --git a/source/reference/minio-server/minio-gateway.rst b/source/reference/minio-server/minio-gateway.rst new file mode 100644 index 00000000..e5db2c21 --- /dev/null +++ b/source/reference/minio-server/minio-gateway.rst @@ -0,0 +1,101 @@ +============= +MinIO Gateway +============= + +.. default-domain:: minio + +.. contents:: Table of Contents + :local: + :depth: 2 + +.. _minio-gateway: + +MinIO Gateway +------------- + +Syntax +~~~~~~ + +.. mc:: minio gateway + +Starts the MinIO Gateway process. + +The command has the following syntax: + +.. code-block:: shell + :class: copyable + + minio gateway [FLAGS] SUBCOMMAND [ARGUMENTS] + +:mc:`minio gateway` supports the following flags: + +.. mc-cmd:: address + :option: + + *Optional* Binds the MinIO Gateway to a specific network address and port + number. Specify the address and port as ``ADDRESS:PORT``, where ``ADDRESS`` + is an IP address or hostname and ``PORT`` is a valid and open port on the + host system. + + To change the port number for all IP addresses or hostnames configured + on the host machine, specify ``:PORT`` where ``PORT`` is a valid + and open port on the host. + +.. mc-cmd:: certs-dir, -S + :option: + + *Optional* Specifies the path to the folder containing certificates the + MinIO Gateway process uses for configuring TLS/SSL connectivity. + + Omit to use the default directory paths: + + - Linux/OSX: ``${HOME}/.minio/certs`` + - Windows: ``%%USERPROFILE%%\.minio\certs``. + + See :ref:`minio-TLS` for more information on TLS/SSL connectivity. + +.. mc-cmd:: quiet + :option: + + *Optional* Disables startup information. + +.. mc-cmd:: anonymous + :option: + + *Optional* Hides sensitive information from logging. + +.. mc-cmd:: json + :option: + + *Optional* Outputs server logs and startup information in ``JSON`` + format. + +:mc:`minio gateway` supports the following subcommands: + +.. mc-cmd:: nas + :fullpath: + + Creates a MinIO Gateway process configured for Network-Attached Storage + (NAS). + +.. mc-cmd:: azure + :fullpath: + + Creates a MinIO Gateway process configured for Microsoft Azure Blob Storage. + +.. mc-cmd:: s3 + :fullpath: + + Creates a MinIO Gateway process configured for Amazon Simple Storage Service + (S3). + +.. mc-cmd:: hdfs + :fullpath: + + Creates a MinIO Gateway process configured for Hadoop Distributed File + System (HDFS). + +.. mc-cmd:: gcs + :fullpath: + + Creates a MinIO Gateway process configured for Google Cloud Storage. \ No newline at end of file diff --git a/source/reference/minio-server/minio-server.rst b/source/reference/minio-server/minio-server.rst index 2ab50571..e67e5ac7 100644 --- a/source/reference/minio-server/minio-server.rst +++ b/source/reference/minio-server/minio-server.rst @@ -1,6 +1,6 @@ -======================== -MinIO Server (``minio``) -======================== +============ +MinIO Server +============ .. default-domain:: minio @@ -10,9 +10,6 @@ MinIO Server (``minio``) .. mc:: minio -The :mc:`minio` command line executable starts either the MinIO Object Storage -process *or* the MinIO Gateway process. - MinIO Server ------------ @@ -29,7 +26,6 @@ see :ref:`minio-installation`. For examples of deploying :mc:`minio server` on a Kubernetes environment, see :docs-k8s:`Kubernetes documentation <>`. - Configuration Settings ~~~~~~~~~~~~~~~~~~~~~~ @@ -168,101 +164,13 @@ The command accepts the following arguments: *Optional* Outputs server logs and startup information in ``JSON`` format. -MinIO Gateway -------------- - -Syntax -~~~~~~ - -.. mc:: minio gateway - -Starts the MinIO Gateway process. - -The command has the following syntax: - -.. code-block:: shell - :class: copyable - - minio gateway [FLAGS] SUBCOMMAND [ARGUMENTS] - -:mc:`minio gateway` supports the following flags: - -.. mc-cmd:: address - :option: - - *Optional* Binds the MinIO Gateway to a specific network address and port - number. Specify the address and port as ``ADDRESS:PORT``, where ``ADDRESS`` - is an IP address or hostname and ``PORT`` is a valid and open port on the - host system. - - To change the port number for all IP addresses or hostnames configured - on the host machine, specify ``:PORT`` where ``PORT`` is a valid - and open port on the host. - -.. mc-cmd:: certs-dir, -S - :option: - - *Optional* Specifies the path to the folder containing certificates the - MinIO Gateway process uses for configuring TLS/SSL connectivity. - - Omit to use the default directory paths: - - - Linux/OSX: ``${HOME}/.minio/certs`` - - Windows: ``%%USERPROFILE%%\.minio\certs``. - - See :ref:`minio-TLS` for more information on TLS/SSL connectivity. - -.. mc-cmd:: quiet - :option: - - *Optional* Disables startup information. - -.. mc-cmd:: anonymous - :option: - - *Optional* Hides sensitive information from logging. - -.. mc-cmd:: json - :option: - - *Optional* Outputs server logs and startup information in ``JSON`` - format. - -:mc:`minio gateway` supports the following subcommands: - -.. mc-cmd:: nas - :fullpath: - - Creates a MinIO Gateway process configured for Network-Attached Storage - (NAS). - -.. mc-cmd:: azure - :fullpath: - - Creates a MinIO Gateway process configured for Microsoft Azure Blob Storage. - -.. mc-cmd:: s3 - :fullpath: - - Creates a MinIO Gateway process configured for Amazon Simple Storage Service - (S3). - -.. mc-cmd:: hdfs - :fullpath: - - Creates a MinIO Gateway process configured for Hadoop Distributed File - System (HDFS). - -.. mc-cmd:: gcs - :fullpath: - - Creates a MinIO Gateway process configured for Google Cloud Storage. +.. _minio-server-environment-variables: Environment Variables --------------------- -The :mc:`minio server` and :mc:`minio gateway` processes can use the following -environment variables when creating its configuration settings: +The :mc:`minio server` processes uses the following +environment variables during startup to set configuration settings. Root Credentials ~~~~~~~~~~~~~~~~ @@ -360,4 +268,1571 @@ refers to the specific storage tier on which to store a given object. .. envvar:: MINIO_STORAGE_CLASS_COMMENT - Adds a comment to the storage class settings. \ No newline at end of file + Adds a comment to the storage class settings. + +Bucket Notifications +~~~~~~~~~~~~~~~~~~~~ + +These environment variables configure notification targets for use with +:doc:`MinIO Bucket Notifications `: + +- :ref:`minio-server-envvar-bucket-notification-amqp` +- :ref:`minio-server-envvar-bucket-notification-mqtt` +- :ref:`minio-server-envvar-bucket-notification-elasticsearch` +- :ref:`minio-server-envvar-bucket-notification-nsq` +- :ref:`minio-server-envvar-bucket-notification-redis` +- :ref:`minio-server-envvar-bucket-notification-postgresql` +- :ref:`minio-server-envvar-bucket-notification-mysql` +- :ref:`minio-server-envvar-bucket-notification-kafka` +- :ref:`minio-server-envvar-bucket-notification-webhook` + +.. _minio-server-envvar-bucket-notification-amqp: + +AMQP Service for Bucket Notifications ++++++++++++++++++++++++++++++++++++++ + +The following section documents environment variables for configuring an AMQP +service as a target for :doc:`MinIO Bucket Notifications +`. See +:ref:`minio-bucket-notifications-publish-amqp` for a tutorial on +using these environment variables. + +You can specify multiple AMQP service endpoints by appending a unique identifier +``_ID`` for each set of related AMQP environment variables: +the top level key. For example, the following commands set two distinct AMQP +service endpoints as ``PRIMARY`` and ``SECONDARY`` respectively: + +.. code-block:: shell + :class: copyable + + set MINIO_NOTIFY_AMQP_ENABLE_PRIMARY="on" + set MINIO_NOTIFY_AMQP_URL_PRIMARY="amqp://user:password@amqp-endpoint.example.net:5672" + + set MINIO_NOTIFY_AMQP_ENABLE_SECONDARY="on" + set MINIO_NOTIFY_AMQP_URL_SECONDARY="amqp://user:password@amqp-endpoint.example.net:5672" + +For example, :envvar:`MINIO_NOTIFY_AMQP_ENABLE_PRIMARY +` indicates the environment variable is associated to +an AMQP service endpoint with ID of ``PRIMARY``. + +.. envvar:: MINIO_NOTIFY_AMQP_ENABLE + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-amqp-enable + :end-before: end-minio-notify-amqp-enable + + Requires specifying :envvar:`MINIO_NOTIFY_AMQP_URL` if set to ``on``. + +.. envvar:: MINIO_NOTIFY_AMQP_URL + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-amqp-url + :end-before: end-minio-notify-amqp-url + + This field is *required* if :envvar:`MINIO_NOTIFY_AMQP_ENABLE` is ``on``. + All other AMQP-related variables are optional. + + This variable corresponds to the :mc-conf:`notify_amqp url ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_AMQP_EXCHANGE + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-amqp-exchange + :end-before: end-minio-notify-amqp-exchange + + This variable corresponds to the :mc-conf:`notify_amqp exchange + ` configuration setting. + +.. envvar:: MINIO_NOTIFY_AMQP_EXCHANGE_TYPE + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-amqp-exchange-type + :end-before: end-minio-notify-amqp-exchange-type + + This variable corresponds to the :mc-conf:`notify_amqp exchange_type + ` configuration setting. + +.. envvar:: MINIO_NOTIFY_AMQP_ROUTING_KEY + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-amqp-routing-key + :end-before: end-minio-notify-amqp-routing-key + + This variable corresponds to the :mc-conf:`notify_amqp routing_key + ` configuration setting. + +.. envvar:: MINIO_NOTIFY_AMQP_MANDATORY + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-amqp-mandatory + :end-before: end-minio-notify-amqp-mandatory + + This variable corresponds to the :mc-conf:`notify_amqp mandatory + ` configuration setting. + +.. envvar:: MINIO_NOTIFY_AMQP_DURABLE + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-amqp-durable + :end-before: end-minio-notify-amqp-durable + + This variable corresponds to the :mc-conf:`notify_amqp durable + ` configuration setting. + +.. envvar:: MINIO_NOTIFY_AMQP_NO_WAIT + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-amqp-no-wait + :end-before: end-minio-notify-amqp-no-wait + + This variable corresponds to the :mc-conf:`notify_amqp no_wait + ` configuration setting. + +.. envvar:: MINIO_NOTIFY_AMQP_INTERNAL + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-amqp-internal + :end-before: end-minio-notify-amqp-internal + + This variable corresponds to the :mc-conf:`notify_amqp internal + ` configuration setting. + + .. explanation is very unclear. Need to revisit this. + +.. envvar:: MINIO_NOTIFY_AMQP_AUTO_DELETED + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-amqp-auto-deleted + :end-before: end-minio-notify-amqp-auto-deleted + + This variable corresponds to the :mc-conf:`notify_amqp auto_deleted + ` configuration setting. + +.. envvar:: MINIO_NOTIFY_AMQP_DELIVERY_MODE + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-amqp-delivery-mode + :end-before: end-minio-notify-amqp-delivery-mode + + This variable corresponds to the :mc-conf:`notify_amqp delivery_mode + ` configuration setting. + +.. envvar:: MINIO_NOTIFY_AMQP_QUEUE_DIR + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-amqp-queue-dir + :end-before: end-minio-notify-amqp-queue-dir + + This variable corresponds to the :mc-conf:`notify_amqp queue_dir + ` configuration setting. + +.. envvar:: MINIO_NOTIFY_AMQP_QUEUE_LIMIT + + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-amqp-queue-limit + :end-before: end-minio-notify-amqp-queue-limit + + This variable corresponds to the :mc-conf:`notify_amqp queue_limit + ` configuration setting. + +.. envvar:: MINIO_NOTIFY_AMQP_COMMENT + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-amqp-comment + :end-before: end-minio-notify-amqp-comment + + This variable corresponds to the :mc-conf:`notify_amqp comment + ` configuration setting. + +.. _minio-server-envvar-bucket-notification-mqtt: + +MQTT Service for Bucket Notifications ++++++++++++++++++++++++++++++++++++++ + +The following section documents environment variables for configuring an MQTT +service as a target for :doc:`MinIO Bucket Notifications +`. See +:ref:`minio-bucket-notifications-publish-mqtt` for a tutorial on +using these environment variables. + +You can specify multiple MQTT service endpoints by appending a unique identifier +``_ID`` for each set of related MQTT environment variables: +the top level key. For example, the following commands set two distinct MQTT +service endpoints as ``PRIMARY`` and ``SECONDARY`` respectively: + +.. code-block:: shell + :class: copyable + + set MINIO_NOTIFY_MQTT_ENABLE_PRIMARY="on" + set MINIO_NOTIFY_MQTT_BROKER_PRIMARY="tcp://user:password@mqtt-endpoint.example.net:1883" + + set MINIO_NOTIFY_MQTT_ENABLE_SECONDARY="on" + set MINIO_NOTIFY_MQTT_BROKER_SECONDARY="tcp://user:password@mqtt-endpoint.example.net:1883" + +For example, :envvar:`MINIO_NOTIFY_MQTT_ENABLE_PRIMARY +` indicates the environment variable is associated to +an MQTT service endpoint with ID of ``PRIMARY``. + +.. envvar:: MINIO_NOTIFY_MQTT_ENABLE + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-mqtt-enable + :end-before: end-minio-notify-mqtt-enable + + This variable corresponds to the + :mc-conf:`notify_mqtt ` configuration setting. + +.. envvar:: MINIO_NOTIFY_MQTT_BROKER + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-mqtt-broker + :end-before: end-minio-notify-mqtt-broker + + This variable corresponds to the + :mc-conf:`notify_mqtt broker ` configuration setting. + +.. envvar:: MINIO_NOTIFY_MQTT_TOPIC + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-mqtt-topic + :end-before: end-minio-notify-mqtt-topic + + This variable corresponds to the + :mc-conf:`notify_mqtt topic ` configuration setting. + +.. envvar:: MINIO_NOTIFY_MQTT_USERNAME + + *Required if the MQTT server/broker enforces authentication/authorization* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-mqtt-username + :end-before: end-minio-notify-mqtt-username + + This variable corresponds to the + :mc-conf:`notify_mqtt username ` configuration setting. + +.. envvar:: MINIO_NOTIFY_MQTT_PASSWORD + + *Required if the MQTT server/broker enforces authentication/authorization* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-mqtt-password + :end-before: end-minio-notify-mqtt-password + + This variable corresponds to the + :mc-conf:`notify_mqtt password ` configuration setting. + +.. envvar:: MINIO_NOTIFY_MQTT_QOS + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-mqtt-qos + :end-before: end-minio-notify-mqtt-qos + + This variable corresponds to the + :mc-conf:`notify_mqtt qos ` configuration setting. + +.. envvar:: MINIO_NOTIFY_MQTT_KEEP_ALIVE_INTERVAL + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-mqtt-keep-alive-interval + :end-before: end-minio-notify-mqtt-keep-alive-interval + + This variable corresponds to the + :mc-conf:`notify_mqtt keep_alive_interval ` configuration setting. + +.. envvar:: MINIO_NOTIFY_MQTT_RECONNECT_INTERVAL + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-mqtt-reconnect-interval + :end-before: end-minio-notify-mqtt-reconnect-interval + + This variable corresponds to the + :mc-conf:`notify_mqtt reconnect_interval ` configuration setting. + +.. envvar:: MINIO_NOTIFY_MQTT_QUEUE_DIR + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-mqtt-queue-dir + :end-before: end-minio-notify-mqtt-queue-dir + + This variable corresponds to the + :mc-conf:`notify_mqtt queue_dir ` configuration setting. + +.. envvar:: MINIO_NOTIFY_MQTT_QUEUE_LIMIT + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-mqtt-queue-limit + :end-before: end-minio-notify-mqtt-queue-limit + + This variable corresponds to the + :mc-conf:`notify_mqtt queue_limit ` configuration setting. + +.. envvar:: MINIO_NOTIFY_MQTT_COMMENT + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-mqtt-comment + :end-before: end-minio-notify-mqtt-comment + + This variable corresponds to the + :mc-conf:`notify_mqtt comment ` configuration setting. + +.. _minio-server-envvar-bucket-notification-elasticsearch: + +Elasticsearch Service for Bucket Notifications +++++++++++++++++++++++++++++++++++++++++++++++ + +The following section documents environment variables for configuring an +Elasticsearch service as a target for :doc:`MinIO Bucket Notifications +`. See +:ref:`minio-bucket-notifications-publish-elasticsearch` for a tutorial on using +these environment variables. + +You can specify multiple Elasticsearch service endpoints by appending a unique identifier +``_ID`` for each set of related Elasticsearch environment variables: +the top level key. For example, the following commands set two distinct Elasticsearch +service endpoints as ``PRIMARY`` and ``SECONDARY`` respectively: + +.. code-block:: shell + :class: copyable + + set MINIO_NOTIFY_ELASTICSEARCH_ENABLE_PRIMARY="on" + set MINIO_NOTIFY_ELASTICSEARCH_URL_PRIMARY="https://user:password@elasticsearch-endpoint.example.net:9200" + set MINIO_NOTIFY_ELASTICSEARCH_INDEX_PRIMARY="bucketevents" + set MINIO_NOTIFY_ELASTICSEARCH_FORMAT_PRIMARY="namespace" + + set MINIO_NOTIFY_ELASTICSEARCH_ENABLE_SECONDARY="on" + set MINIO_NOTIFY_ELASTICSEARCH_URL_SECONDARY="https://user:password@elasticsearch-endpoint.example.net:9200" + set MINIO_NOTIFY_ELASTICSEARCH_INDEX_SECONDARY="bucketevents" + set MINIO_NOTIFY_ELASTICSEARCH_FORMAT_SECONDARY="namespace" + + +.. envvar:: MINIO_NOTIFY_ELASTICSEARCH_ENABLE + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-elasticsearch-enable + :end-before: end-minio-notify-elasticsearch-enable + + Requires specifying the following additional environment variables if set to + ``on``: + + - :envvar:`MINIO_NOTIFY_ELASTICSEARCH_URL` + - :envvar:`MINIO_NOTIFY_ELASTICSEARCH_INDEX` + - :envvar:`MINIO_NOTIFY_ELASTICSEARCH_FORMAT` + + This variable corresponds to the :mc-conf:`notify_elasticsearch` + configuration setting. + +.. envvar:: MINIO_NOTIFY_ELASTICSEARCH_URL + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-elasticsearch-url + :end-before: end-minio-notify-elasticsearch-url + + This variable corresponds to the + :mc-conf:`notify_elasticsearch url ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_ELASTICSEARCH_INDEX + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-elasticsearch-index + :end-before: end-minio-notify-elasticsearch-index + + This variable corresponds to the + :mc-conf:`notify_elasticsearch index ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_ELASTICSEARCH_FORMAT + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-elasticsearch-format + :end-before: end-minio-notify-elasticsearch-format + + This variable corresponds to the + :mc-conf:`notify_elasticsearch format ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_ELASTICSEARCH_USERNAME + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-elasticsearch-username + :end-before: end-minio-notify-elasticsearch-username + + This variable corresponds to the + :mc-conf:`notify_elasticsearch username ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_ELASTICSEARCH_PASSWORD + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-elasticsearch-password + :end-before: end-minio-notify-elasticsearch-password + + This variable corresponds to the + :mc-conf:`notify_elasticsearch password ` + configuration setting. +.. envvar:: MINIO_NOTIFY_ELASTICSEARCH_QUEUE_DIR + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-elasticsearch-queue-dir + :end-before: end-minio-notify-elasticsearch-queue-dir + + This variable corresponds to the + :mc-conf:`notify_elasticsearch queue_dir ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_ELASTICSEARCH_QUEUE_LIMIT + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-elasticsearch-queue-limit + :end-before: end-minio-notify-elasticsearch-queue-limit + + This variable corresponds to the + :mc-conf:`notify_elasticsearch queue_limit ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_ELASTICSEARCH_COMMENT + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-elasticsearch-comment + :end-before: end-minio-notify-elasticsearch-comment + + This variable corresponds to the + :mc-conf:`notify_elasticsearch comment ` + configuration setting. + +.. _minio-server-envvar-bucket-notification-nsq: + +NSQ Service for Bucket Notifications +++++++++++++++++++++++++++++++++++++ + +The following section documents environment variables for configuring an +NSQ service as a target for :doc:`MinIO Bucket Notifications +`. See +:ref:`minio-bucket-notifications-publish-nsq` for a tutorial on using +these environment variables. + +You can specify multiple NSQ service endpoints by appending a unique +identifier ``_ID`` for each set of related NSQ environment variables: +the top level key. For example, the following commands set two distinct +NSQ service endpoints as ``PRIMARY`` and ``SECONDARY`` respectively: + +.. code-block:: shell + :class: copyable + + set MINIO_NOTIFY_NSQ_ENABLE_PRIMARY="on" + set MINIO_NOTIFY_NSQ_NSQD_ADDRESS_PRIMARY="https://user:password@nsq-endpoint.example.net:9200" + set MINIO_NOTIFY_NSQ_TOPIC_PRIMARY="bucketevents" + + set MINIO_NOTIFY_NSQ_ENABLE_SECONDARY="on" + set MINIO_NOTIFY_NSQ_NSQD_ADDRESS_SECONDARY="https://user:password@nsq-endpoint.example.net:9200" + set MINIO_NOTIFY_NSQ_TOPIC_SECONDARY="bucketevents" + +.. envvar:: MINIO_NOTIFY_NSQ_ENABLE + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nsq-enable + :end-before: end-minio-notify-nsq-enable + + This variable corresponds to the + :mc-conf:`notify_nsq ` configuration setting. + +.. envvar:: MINIO_NOTIFY_NSQ_NSQD_ADDRESS + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nsq-nsqd-address + :end-before: end-minio-notify-nsq-nsqd-address + + This variable corresponds to the + :mc-conf:`notify_nsq nsqd_address ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_NSQ_TOPIC + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nsq-topic + :end-before: end-minio-notify-nsq-topic + + This variable corresponds to the + :mc-conf:`notify_nsq topic ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_NSQ_TLS + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nsq-tls + :end-before: end-minio-notify-nsq-tls + + This variable corresponds to the + :mc-conf:`notify_nsq tls ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_NSQ_TLS_SKIP_VERIFY + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nsq-tls-skip-verify + :end-before: end-minio-notify-nsq-tls-skip-verify + + This variable corresponds to the + :mc-conf:`notify_nsq tls_skip_verify ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_NSQ_QUEUE_DIR + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nsq-queue-dir + :end-before: end-minio-notify-nsq-queue-dir + + This variable corresponds to the + :mc-conf:`notify_nsq queue_dir ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_NSQ_QUEUE_LIMIT + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nsq-queue-limit + :end-before: end-minio-notify-nsq-queue-limit + + This variable corresponds to the + :mc-conf:`notify_nsq queue_limit ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_NSQ_COMMENT + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nsq-comment + :end-before: end-minio-notify-nsq-comment + + This variable corresponds to the + :mc-conf:`notify_nsq comment ` + configuration setting. + +.. _minio-server-envvar-bucket-notification-redis: + +Redis Service for Bucket Notifications +++++++++++++++++++++++++++++++++++++++ + +The following section documents environment variables for configuring an +Redis service as a target for :doc:`MinIO Bucket Notifications +`. See +:ref:`minio-bucket-notifications-publish-redis` for a tutorial on using +these environment variables. + +You can specify multiple Redis service endpoints by appending a unique +identifier ``_ID`` for each set of related Redis environment variables: the top +level key. For example, the following commands set two distinct Redis service +endpoints as ``PRIMARY`` and ``SECONDARY`` respectively: + +.. code-block:: shell + :class: copyable + + set MINIO_NOTIFY_REDIS_ENABLE_PRIMARY="on" + set MINIO_NOTIFY_REDIS_REDIS_ADDRESS_PRIMARY="https://user:password@redis-endpoint.example.net:9200" + set MINIO_NOTIFY_REDIS_KEY_PRIMARY="bucketevents" + set MINIO_NOTIFY_REDIS_FORMAT_PRIMARY="namespace" + + + set MINIO_NOTIFY_REDIS_ENABLE_SECONDARY="on" + set MINIO_NOTIFY_REDIS_REDIS_ADDRESS_SECONDARY="https://user:password@redis-endpoint.example.net:9200" + set MINIO_NOTIFY_REDIS_KEY_SECONDARY="bucketevents" + set MINIO_NOTIFY_REDIS_FORMAT_SECONDARY="namespace" + +.. envvar:: MINIO_NOTIFY_REDIS_ENABLE + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-redis-enable + :end-before: end-minio-notify-redis-enable + + Requires specifying the following additional environment variables if set to + ``on``: + + - :envvar:`MINIO_NOTIFY_REDIS_ADDRESS` + - :envvar:`MINIO_NOTIFY_REDIS_KEY` + - :envvar:`MINIO_NOTIFY_REDIS_FORMAT` + + This variable corresponds to the + :mc-conf:`notify_redis ` configuration setting. + +.. envvar:: MINIO_NOTIFY_REDIS_ADDRESS + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-redis-address + :end-before: end-minio-notify-redis-address + + This variable corresponds to the + :mc-conf:`notify_redis address ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_REDIS_KEY + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-redis-key + :end-before: end-minio-notify-redis-key + + This variable corresponds to the + :mc-conf:`notify_redis key ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_REDIS_FORMAT + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-redis-format + :end-before: end-minio-notify-redis-format + + This variable corresponds to the + :mc-conf:`notify_redis format ` + configuration setting. + + +.. envvar:: MINIO_NOTIFY_REDIS_PASSWORD + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-redis-password + :end-before: end-minio-notify-redis-password + + This variable corresponds to the + :mc-conf:`notify_redis password ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_REDIS_QUEUE_DIR + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-redis-queue-dir + :end-before: end-minio-notify-redis-queue-dir + + This variable corresponds to the + :mc-conf:`notify_redis queue_dir ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_REDIS_QUEUE_LIMIT + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-redis-queue-limit + :end-before: end-minio-notify-redis-queue-limit + + This variable corresponds to the + :mc-conf:`notify_redis queue_limit ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_REDIS_COMMENT + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-redis-comment + :end-before: end-minio-notify-redis-comment + + This variable corresponds to the + :mc-conf:`notify_redis comment ` + configuration setting. + +.. _minio-server-envvar-bucket-notification-nats: + +NATS Service for Bucket Notifications ++++++++++++++++++++++++++++++++++++++ + +The following section documents environment variables for configuring an NATS +service as a target for :doc:`MinIO Bucket Notifications +`. See +:ref:`minio-bucket-notifications-publish-nats` for a tutorial on +using these environment variables. + +You can specify multiple NATS service endpoints by appending a unique identifier +``_ID`` for each set of related NATS environment variables: +the top level key. For example, the following commands set two distinct NATS +service endpoints as ``PRIMARY`` and ``SECONDARY`` respectively: + +.. code-block:: shell + :class: copyable + + set MINIO_NOTIFY_NATS_ENABLE_PRIMARY="on" + set MINIO_NOTIFY_NATS_ADDRESS_PRIMARY="https://nats-endpoint.example.net:4222" + set MINIO_NOTIFY_NATS_SUBJECT="minioevents" + + set MINIO_NOTIFY_NATS_ENABLE_SECONDARY="on" + set MINIO_NOTIFY_NATS_ADDRESS_SECONDARY="https://nats-endpoint.example.net:4222" + set MINIO_NOTIFY_NATS_SUBJECT="minioevents" + +For example, :envvar:`MINIO_NOTIFY_NATS_ENABLE_PRIMARY +` indicates the environment variable is associated to +an NATS service endpoint with ID of ``PRIMARY``. + +.. envvar:: MINIO_NOTIFY_NATS_ENABLE + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nats-enable + :end-before: end-minio-notify-nats-enable + + This environment variable corresponds with the + :mc-conf:`notify_nats ` configuration setting. + +.. envvar:: MINIO_NOTIFY_NATS_ADDRESS + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nats-address + :end-before: end-minio-notify-nats-address + + This environment variable corresponds with the + :mc-conf:`notify_nats address ` configuration setting. + +.. envvar:: MINIO_NOTIFY_NATS_SUBJECT + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nats-subject + :end-before: end-minio-notify-nats-subject + + This environment variable corresponds with the + :mc-conf:`notify_nats subject ` configuration setting. + +.. envvar:: MINIO_NOTIFY_NATS_USERNAME + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nats-username + :end-before: end-minio-notify-nats-username + + This environment variable corresponds with the + :mc-conf:`notify_nats username ` configuration setting. + +.. envvar:: MINIO_NOTIFY_NATS_PASSWORD + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nats-password + :end-before: end-minio-notify-nats-password + + This environment variable corresponds with the + :mc-conf:`notify_nats password ` configuration setting. + +.. envvar:: MINIO_NOTIFY_NATS_TOKEN + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nats-token + :end-before: end-minio-notify-nats-token + + This environment variable corresponds with the + :mc-conf:`notify_nats token ` configuration setting. + +.. envvar:: MINIO_NOTIFY_NATS_TLS + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nats-tls + :end-before: end-minio-notify-nats-tls + + This environment variable corresponds with the + :mc-conf:`notify_nats tls ` configuration setting. + +.. envvar:: MINIO_NOTIFY_NATS_TLS_SKIP_VERIFY + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nats-tls-skip-verify + :end-before: end-minio-notify-nats-tls-skip-verify + + This environment variable corresponds with the + :mc-conf:`notify_nats tls_skip_verify ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_NATS_PING_INTERVAL + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nats-ping-interval + :end-before: end-minio-notify-nats-ping-interval + + This environment variable corresponds with the + :mc-conf:`notify_nats ping_interval ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_NATS_STREAMING + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nats-streaming + :end-before: end-minio-notify-nats-streaming + + This environment variable corresponds with the + :mc-conf:`notify_nats streaming ` configuration + setting. + +.. envvar:: MINIO_NOTIFY_NATS_STREAMING_ASYNC + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nats-streaming-async + :end-before: end-minio-notify-nats-streaming-async + + This environment variable corresponds with the + :mc-conf:`notify_nats streaming_async ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_NATS_STREAMING_MAX_PUB_ACKS_IN_FLIGHT + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nats-streaming-max-pub-acks-in-flight + :end-before: end-minio-notify-nats-streaming-max-pub-acks-in-flight + + This environment variable corresponds with the + :mc-conf:`notify_nats streaming_max_pub_acks_in_flight + ` configuration setting. + +.. envvar:: MINIO_NOTIFY_NATS_STREAMING_CLUSTER_ID + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nats-streaming-cluster-id + :end-before: end-minio-notify-nats-streaming-cluster-id + + This environment variable corresponds with the + :mc-conf:`notify_nats streaming_cluster_id + ` configuration setting. + +.. envvar:: MINIO_NOTIFY_NATS_CERT_AUTHORITY + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nats-cert-authority + :end-before: end-minio-notify-nats-cert-authority + + This environment variable corresponds with the + :mc-conf:`notify_nats cert_authority ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_NATS_CLIENT_CERT + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nats-client-cert + :end-before: end-minio-notify-nats-client-cert + + This environment variable corresponds with the + :mc-conf:`notify_nats client_cert ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_NATS_CLIENT_KEY + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nats-client-key + :end-before: end-minio-notify-nats-client-key + + This environment variable corresponds with the + :mc-conf:`notify_nats client_key ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_NATS_QUEUE_DIR + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nats-queue-dir + :end-before: end-minio-notify-nats-queue-dir + + This environment variable corresponds with the + :mc-conf:`notify_nats queue_dir ` configuration + setting. + +.. envvar:: MINIO_NOTIFY_NATS_QUEUE_LIMIT + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nats-queue-limit + :end-before: end-minio-notify-nats-queue-limit + + This environment variable corresponds with the + :mc-conf:`notify_nats queue_limit ` configuration + setting. + +.. envvar:: MINIO_NOTIFY_NATS_COMMENT + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-nats-comment + :end-before: end-minio-notify-nats-comment + + This environment variable corresponds with the + :mc-conf:`notify_nats comment ` configuration setting. + + +.. _minio-server-envvar-bucket-notification-postgresql: + +PostgreSQL Service for Bucket Notifications ++++++++++++++++++++++++++++++++++++++++++++ + +The following section documents environment variables for configuring an POSTGRESQL +service as a target for :doc:`MinIO Bucket Notifications +`. See +:ref:`minio-bucket-notifications-publish-postgresql` for a tutorial on +using these environment variables. + +You can specify multiple PostgreSQL service endpoints by appending a unique identifier +``_ID`` for each set of related PostgreSQL environment variables: +the top level key. For example, the following commands set two distinct PostgreSQL +service endpoints as ``PRIMARY`` and ``SECONDARY`` respectively: + +.. code-block:: shell + :class: copyable + + set MINIO_NOTIFY_POSTGRESQL_ENABLE_PRIMARY="on" + set MINIO_NOTIFY_POSTGRESQL_CONNECTION_STRING_PRIMARY="host=postgresql-endpoint.example.net port=4222..." + set MINIO_NOTIFY_POSTGRESQL_TABLE_PRIMARY="minioevents" + set MINIO_NOTIFY_POSTGRESQL_FORMAT_PRIMARY="namespace" + + set MINIO_NOTIFY_POSTGRESQL_ENABLE_SECONDARY="on" + set MINIO_NOTIFY_POSTGRESQL_CONNECTION_STRING_SECONDARY="host=postgresql-endpoint.example.net port=4222..." + set MINIO_NOTIFY_POSTGRESQL_TABLE_SECONDARY="minioevents" + set MINIO_NOTIFY_POSTGRESQL_FORMAT_SECONDARY="namespace" + +For example, :envvar:`MINIO_NOTIFY_POSTGRESQL_ENABLE_PRIMARY +` indicates the environment variable is +associated to an PostgreSQL service endpoint with ID of ``PRIMARY``. + +.. envvar:: MINIO_NOTIFY_POSTGRESQL_ENABLE + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-postgresql-enable + :end-before: end-minio-notify-postgresql-enable + + Requires specifying the following additional environment variables if set + to ``on``: + + - :envvar:`MINIO_NOTIFY_POSTGRESQL_CONNECTION_STRING` + - :envvar:`MINIO_NOTIFY_POSTGRESQL_TABLE` + - :envvar:`MINIO_NOTIFY_POSTGRESQL_FORMAT` + + This environment variable corresponds with the + :mc-conf:`notify_postgresql ` configuration setting. + +.. envvar:: MINIO_NOTIFY_POSTGRESQL_CONNECTION_STRING + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-postgresql-connection-string + :end-before: end-minio-notify-postgresql-connection-string + + This environment variable corresponds with the + :mc-conf:`notify_postgresql connection_string ` + configuration setting. + + +.. envvar:: MINIO_NOTIFY_POSTGRESQL_TABLE + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-postgresql-table + :end-before: end-minio-notify-postgresql-table + + This environment variable corresponds with the + :mc-conf:`notify_postgresql table ` + configuration setting. + + +.. envvar:: MINIO_NOTIFY_POSTGRESQL_FORMAT + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-postgresql-format + :end-before: end-minio-notify-postgresql-format + + This environment variable corresponds with the + :mc-conf:`notify_postgresql format ` + configuration setting. + + +.. envvar:: MINIO_NOTIFY_POSTGRESQL_MAX_OPEN_CONNECTIONS + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-postgresql-max-open-connections + :end-before: end-minio-notify-postgresql-max-open-connections + + This environment variable corresponds with the + :mc-conf:`notify_postgresql max_open_connections + ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_POSTGRESQL_QUEUE_DIR + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-postgresql-queue-dir + :end-before: end-minio-notify-postgresql-queue-dir + + This environment variable corresponds with the + :mc-conf:`notify_postgresql queue_dir ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_POSTGRESQL_QUEUE_LIMIT + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-postgresql-queue-limit + :end-before: end-minio-notify-postgresql-queue-limit + + This environment variable corresponds with the + :mc-conf:`notify_postgresql queue_limit ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_POSTGRESQL_COMMENT + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-postgresql-comment + :end-before: end-minio-notify-postgresql-comment + + This environment variable corresponds with the + :mc-conf:`notify_postgresql comment ` + configuration setting. + +.. _minio-server-envvar-bucket-notification-mysql: + +MySQL Service for Bucket Notifications +++++++++++++++++++++++++++++++++++++++ + +The following section documents environment variables for configuring an MYSQL +service as a target for :doc:`MinIO Bucket Notifications +`. See +:ref:`minio-bucket-notifications-publish-mysql` for a tutorial on +using these environment variables. + +You can specify multiple MySQL service endpoints by appending a unique +identifier ``_ID`` for each set of related MySQL environment variables: the top +level key. For example, the following commands set two distinct MySQL service +endpoints as ``PRIMARY`` and ``SECONDARY`` respectively: + +.. code-block:: shell + :class: copyable + + set MINIO_NOTIFY_MYSQL_ENABLE_PRIMARY="on" + set MINIO_NOTIFY_MYSQL_DSN_STRING_PRIMARY="username:password@tcp(mysql.example.com:3306)/miniodb" + set MINIO_NOTIFY_MYSQL_TABLE_PRIMARY="minioevents" + set MINIO_NOTIFY_MYSQL_FORMAT_PRIMARY="namespace" + + set MINIO_NOTIFY_MYSQL_ENABLE_SECONDARY="on" + set MINIO_NOTIFY_MYSQL_DSN_STRING_SECONDARY="username:password@tcp(mysql.example.com:3306)/miniodb" + set MINIO_NOTIFY_MYSQL_TABLE_SECONDARY="minioevents" + set MINIO_NOTIFY_MYSQL_FORMAT_SECONDARY="namespace" + +For example, :envvar:`MINIO_NOTIFY_MYSQL_ENABLE_PRIMARY +` indicates the environment variable is +associated to an MySQL service endpoint with ID of ``PRIMARY``. + +.. envvar:: MINIO_NOTIFY_MYSQL_ENABLE + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-mysql-enable + :end-before: end-minio-notify-mysql-enable + + Requires specifying the following additional environment variables if set + to ``on``: + + - :envvar:`MINIO_NOTIFY_MYSQL_DSN_STRING` + - :envvar:`MINIO_NOTIFY_MYSQL_TABLE` + - :envvar:`MINIO_NOTIFY_MYSQL_FORMAT` + + This environment variable corresponds with the + :mc-conf:`notify_mysql ` configuration setting. + +.. envvar:: MINIO_NOTIFY_MYSQL_DSN_STRING + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-mysql-connection-string + :end-before: end-minio-notify-mysql-connection-string + + This environment variable corresponds with the + :mc-conf:`notify_mysql dsn_string ` + configuration setting. + + +.. envvar:: MINIO_NOTIFY_MYSQL_TABLE + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-mysql-table + :end-before: end-minio-notify-mysql-table + + This environment variable corresponds with the + :mc-conf:`notify_mysql table ` + configuration setting. + + +.. envvar:: MINIO_NOTIFY_MYSQL_FORMAT + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-mysql-format + :end-before: end-minio-notify-mysql-format + + This environment variable corresponds with the + :mc-conf:`notify_mysql format ` + configuration setting. + + +.. envvar:: MINIO_NOTIFY_MYSQL_MAX_OPEN_CONNECTIONS + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-mysql-max-open-connections + :end-before: end-minio-notify-mysql-max-open-connections + + This environment variable corresponds with the + :mc-conf:`notify_mysql max_open_connections + ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_MYSQL_QUEUE_DIR + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-mysql-queue-dir + :end-before: end-minio-notify-mysql-queue-dir + + This environment variable corresponds with the + :mc-conf:`notify_mysql queue_dir ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_MYSQL_QUEUE_LIMIT + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-mysql-queue-limit + :end-before: end-minio-notify-mysql-queue-limit + + This environment variable corresponds with the + :mc-conf:`notify_mysql queue_limit ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_MYSQL_COMMENT + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-mysql-comment + :end-before: end-minio-notify-mysql-comment + + This environment variable corresponds with the + :mc-conf:`notify_mysql comment ` + configuration setting. + + +.. _minio-server-envvar-bucket-notification-kafka: + +Kafka Service for Bucket Notifications +++++++++++++++++++++++++++++++++++++++ + +The following section documents environment variables for configuring an +Kafka service as a target for :doc:`MinIO Bucket Notifications +`. See +:ref:`minio-bucket-notifications-publish-kafka` for a tutorial on using +these environment variables. + +You can specify multiple Kafka service endpoints by appending a unique +identifier ``_ID`` for each set of related Kafka environment variables: the top +level key. For example, the following commands set two distinct Kafka service +endpoints as ``PRIMARY`` and ``SECONDARY`` respectively: + +.. code-block:: shell + :class: copyable + + set MINIO_NOTIFY_KAFKA_ENABLE_PRIMARY="on" + set MINIO_NOTIFY_KAFKA_BROKERS_PRIMARY="https://kafka1.example.net:9200, https://kafka2.example.net:9200" + + set MINIO_NOTIFY_KAFKA_ENABLE_SECONDARY="on" + set MINIO_NOTIFY_KAFKA_BROKERS_SECONDARY="https://kafka1.example.net:9200, https://kafka2.example.net:9200" + +.. envvar:: MINIO_NOTIFY_KAFKA_ENABLE + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-kafka-enable + :end-before: end-minio-notify-kafka-enable + +.. envvar:: MINIO_NOTIFY_KAFKA_BROKERS + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-kafka-brokers + :end-before: end-minio-notify-kafka-brokers + + This environment variable corresponds to the + :mc-conf:`notify_kafka brokers ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_KAFKA_TOPIC + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-kafka-topic + :end-before: end-minio-notify-kafka-topic + + This environment variable corresponds to the + :mc-conf:`notify_kafka topic ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_KAFKA_SASL_USERNAME + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-kafka-sasl-username + :end-before: end-minio-notify-kafka-sasl-username + + This environment variable corresponds to the + :mc-conf:`notify_kafka sasl_username ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_KAFKA_SASL_PASSWORD + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-kafka-sasl-password + :end-before: end-minio-notify-kafka-sasl-password + + This environment variable corresponds to the + :mc-conf:`notify_kafka sasl_password ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_KAFKA_SASL_MECHANISM + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-kafka-sasl-mechanism + :end-before: end-minio-notify-kafka-sasl-mechanism + + This environment variable corresponds to the + :mc-conf:`notify_kafka sasl_mechanism ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_KAFKA_TLS_CLIENT_AUTH + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-kafka-tls-client-auth + :end-before: end-minio-notify-kafka-tls-client-auth + + This environment variable corresponds to the + :mc-conf:`notify_kafka tls_client_auth ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_KAFKA_SASL + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-kafka-sasl + :end-before: end-minio-notify-kafka-sasl + + This environment variable corresponds to the + :mc-conf:`notify_kafka sasl ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_KAFKA_TLS + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-kafka-tls + :end-before: end-minio-notify-kafka-tls + + This environment variable corresponds to the + :mc-conf:`notify_kafka tls ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_KAFKA_TLS_SKIP_VERIFY + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-kafka-tls-skip-verify + :end-before: end-minio-notify-kafka-tls-skip-verify + + This environment variable corresponds to the + :mc-conf:`notify_kafka tls_skip_verify ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_KAFKA_CLIENT_TLS_CERT + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-kafka-client-tls-cert + :end-before: end-minio-notify-kafka-client-tls-cert + + This environment variable corresponds to the + :mc-conf:`notify_kafka client_tls_cert ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_KAFKA_CLIENT_TLS_KEY + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-kafka-client-tls-key + :end-before: end-minio-notify-kafka-client-tls-key + + This environment variable corresponds to the + :mc-conf:`notify_kafka client_tls_key ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_KAFKA_VERSION + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-kafka-version + :end-before: end-minio-notify-kafka-version + + This environment variable corresponds to the + :mc-conf:`notify_kafka version ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_KAFKA_QUEUE_DIR + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-kafka-queue-dir + :end-before: end-minio-notify-kafka-queue-dir + + This environment variable corresponds to the + :mc-conf:`notify_kafka queue_dir ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_KAFKA_QUEUE_LIMIT + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-kafka-queue-limit + :end-before: end-minio-notify-kafka-queue-limit + + This environment variable corresponds to the + :mc-conf:`notify_kafka queue_limit ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_KAFKA_COMMENT + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: start-minio-notify-kafka-comment + :end-before: end-minio-notify-kafka-comment + + This environment variable corresponds to the + :mc-conf:`notify_kafka comment ` + configuration setting. + +.. _minio-server-envvar-bucket-notification-webhook: + +Webhook Service for Bucket Notifications +++++++++++++++++++++++++++++++++++++++++ + +The following section documents environment variables for configuring an +Webhook service as a target for :doc:`MinIO Bucket Notifications +`. See +:ref:`minio-bucket-notifications-publish-webhook` for a tutorial on using +these environment variables. + +You can specify multiple Webhook service endpoints by appending a unique +identifier ``_ID`` for each set of related Webhook environment variables: the top +level key. For example, the following commands set two distinct Webhook service +endpoints as ``PRIMARY`` and ``SECONDARY`` respectively: + +.. code-block:: shell + :class: copyable + + set MINIO_NOTIFY_WEBHOOK_ENABLE_PRIMARY="on" + set MINIO_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY="https://webhook1.example.net" + + set MINIO_NOTIFY_WEBHOOK_ENABLE_SECONDARY="on" + set MINIO_NOTIFY_WEBHOOK_ENDPOINT_SECONDARY="https://webhook1.example.net" + +.. envvar:: MINIO_NOTIFY_WEBHOOK_ENABLE + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: minio-notify-webhook-enable + :end-before: minio-notify-webhook-enable + +.. envvar:: MINIO_NOTIFY_WEBHOOK_ENDPOINT + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: minio-notify-webhook-endpoint + :end-before: minio-notify-webhook-endpoint + + This environment variable corresponds with the + :mc-conf:`notify_webhook endpoint ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_WEBHOOK_AUTH_TOKEN + + *Required* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: minio-notify-webhook-auth-token + :end-before: minio-notify-webhook-auth-token + + This environment variable corresponds with the + :mc-conf:`notify_webhook auth_token ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_WEBHOOK_QUEUE_DIR + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: minio-notify-webhook-queue-dir + :end-before: minio-notify-webhook-queue-dir + + This environment variable corresponds with the + :mc-conf:`notify_webhook queue_dir ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_WEBHOOK_QUEUE_LIMIT + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: minio-notify-webhook-queue-limit + :end-before: minio-notify-webhook-queue-limit + + This environment variable corresponds with the + :mc-conf:`notify_webhook queue_limit ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_WEBHOOK_CLIENT_CERT + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: minio-notify-webhook-client-cert + :end-before: minio-notify-webhook-client-cert + + This environment variable corresponds with the + :mc-conf:`notify_webhook client_cert ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_WEBHOOK_CLIENT_KEY + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: minio-notify-webhook-client-key + :end-before: minio-notify-webhook-client-key + + This environment variable corresponds with the + :mc-conf:`notify_webhook client_key ` + configuration setting. + +.. envvar:: MINIO_NOTIFY_WEBHOOK_COMMENT + + *Optional* + + .. include:: /includes/common-mc-admin-config.rst + :start-after: minio-notify-webhook-comment + :end-before: minio-notify-webhook-comment + + This environment variable corresponds with the + :mc-conf:`notify_webhook comment ` + configuration setting. diff --git a/sphinxext/minio.py b/sphinxext/minio.py index 551651aa..e71801a1 100644 --- a/sphinxext/minio.py +++ b/sphinxext/minio.py @@ -268,6 +268,8 @@ class MinioObject(ObjectDescription): option_spec = { 'noindex': directives.flag, 'noindexentry': directives.flag, + 'noprefix': directives.flag, + 'delimiter': directives.unchanged, } def handle_signature(self, sig: str, signode: desc_signature) -> Tuple[str, str]: @@ -279,6 +281,13 @@ class MinioObject(ObjectDescription): """ sig = sig.strip() + noprefix = 'noprefix' in self.options + if ('delimiter' in self.options): + delimiter = self.options.get('delimiter').strip('"') + else: + delimiter = "." + + member = sig # If construct is nested, prefix the current prefix prefix = self.env.ref_context.get('minio:object', None) @@ -291,8 +300,9 @@ class MinioObject(ObjectDescription): signode['object'] = prefix signode['fullname'] = fullname - if prefix: - signode += addnodes.desc_addname(prefix + '.', prefix + '.') + if prefix and (noprefix == False): + # signode += addnodes.desc_addname(prefix + '.', prefix + '.') + signode += addnodes.desc_addname(prefix + delimiter, prefix + delimiter) signode += addnodes.desc_name(member, member) @@ -480,7 +490,8 @@ class MinIODomain(Domain): 'mc-cmd': ObjType(_('mc-cmd'), 'mc-cmd'), 'mc-cmd-option': ObjType(_('mc-cmd-option'), 'mc-cmd-option'), 'policy-action': ObjType(_('policy-action'), 'policy-action'), - 'envvar': ObjType(_('envvar'), 'envvar') + 'envvar': ObjType(_('envvar'), 'envvar'), + 'mc-conf': ObjType(_('mc-conf'), 'mc-conf') } directives = { 'data': MinioObject, @@ -490,7 +501,8 @@ class MinIODomain(Domain): 'mc': MinioMCCommand, 'mc-cmd': MinioMCObject, 'policy-action': MinioObject, - 'envvar': MinioObject + 'envvar': MinioObject, + 'mc-conf': MinioObject, } roles = { 'data': MinioXRefRole(), @@ -502,6 +514,7 @@ class MinIODomain(Domain): 'mc-cmd-option': MinioCMDOptionXRefRole(), 'policy-action': MinioXRefRole(), 'envvar': MinioXRefRole(), + 'mc-conf': MinioXRefRole(), } initial_data = {