diff --git a/advanced_examples/minikube-prom-graf/.gitignore b/advanced_examples/minikube-prom-graf/.gitignore new file mode 100644 index 0000000..172c657 --- /dev/null +++ b/advanced_examples/minikube-prom-graf/.gitignore @@ -0,0 +1,2 @@ +_build/ +rebar.lock diff --git a/advanced_examples/minikube-prom-graf/.travis.yml b/advanced_examples/minikube-prom-graf/.travis.yml new file mode 100644 index 0000000..2d251a0 --- /dev/null +++ b/advanced_examples/minikube-prom-graf/.travis.yml @@ -0,0 +1,56 @@ +sudo: required + +dist: xenial + +addons: + apt: + packages: + - curl + +env: +- CHANGE_MINIKUBE_NONE_USER=true + +before_script: +# Download minikube. +- MINIKUBE_VERSION=latest +- curl -Lo minikube https://storage.googleapis.com/minikube/releases/$MINIKUBE_VERSION/minikube-linux-amd64 && chmod +x minikube && sudo mv minikube /usr/local/bin/ +# Download kubectl, which is a requirement for using minikube. +- KUBERNETES_VERSION=$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt) +- curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/$KUBERNETES_VERSION/bin/linux/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/ +# Test that it works +- kubectl -h +- sudo minikube start -v 7 --logtostderr --vm-driver=none --kubernetes-version "$KUBERNETES_VERSION" +# Fix the kubectl context, as it's often stale. +- minikube update-context +# Wait for Kubernetes to be up and ready. +- JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'; until kubectl get nodes -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do sleep 1; done + +script: +- kubectl cluster-info +# kube-addon-manager is responsible for managing other kubernetes components, such as kube-dns, dashboard, storage-provisioner.. +- JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'; until kubectl -n kube-system get pods -lcomponent=kube-addon-manager -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do sleep 5;echo "waiting for kube-addon-manager to be available"; kubectl get pods --all-namespaces; done +# Wait for kube-dns to be ready. +- JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'; until kubectl -n kube-system get pods -lk8s-app=kube-dns -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do sleep 5;echo "waiting for kube-dns to be available"; kubectl get pods --all-namespaces; done +- kubectl create service nodeport dockerwatch --tcp=8080:8080 --tcp=8443:8443 +- kubectl get service +- ./create-certs $(minikube ip) +- kubectl create secret generic dockerwatch --from-file=ssl/ +- kubectl get secret +- docker build -t dockerwatch . +- kubectl apply -f dockerwatch-deploy.yaml +# Wait for dockerwatch to be ready. +- JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'; until kubectl -n default get pods -lapp=dockerwatch -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do sleep 5;echo "waiting for dockerwatch to be available"; kubectl get pods --all-namespaces; done +- HTTP=$(minikube service dockerwatch --url | head -1) +- HTTPS=$(minikube service dockerwatch --url --https | tail -1) +- "curl -v -H 'Content-Type: application/json' -X POST -d '' $HTTP/cnt" +- "curl -v -H 'Content-Type: application/json' -X POST -d '{}' $HTTP/cnt" +- "curl -v --cacert ssl/dockerwatch-ca.pem -H 'Accept: application/json' $HTTPS/" +- "curl -v --cacert ssl/dockerwatch-ca.pem -H 'Accept: application/json' $HTTPS/cnt" +- kubectl apply -f monitoring-namespace.yaml +- kubectl apply -f prometheus-config.yaml +- kubectl apply -f prometheus-deployment.yaml +- kubectl apply -f prometheus-service.yaml +- kubectl apply -f grafana-deployment.yaml +- kubectl apply -f grafana-service.yaml +- JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'; until kubectl -n monitoring get pods -lname=grafana -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do sleep 5;echo "waiting for grafana to be available"; kubectl get pods --all-namespaces; done +- JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'; until kubectl -n monitoring get pods -lname=prometheus -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do sleep 5;echo "waiting for prometheus to be available"; kubectl get pods --all-namespaces; done diff --git a/advanced_examples/minikube-prom-graf/Docker-Cheat-Sheet.md b/advanced_examples/minikube-prom-graf/Docker-Cheat-Sheet.md new file mode 100644 index 0000000..9d58795 --- /dev/null +++ b/advanced_examples/minikube-prom-graf/Docker-Cheat-Sheet.md @@ -0,0 +1,26 @@ +## Docker Cheatsheet + +* Remove all containers that are not running: + + $ docker rm $(docker ps -aq -f status=exited) + +* Remove dangling images: + + $ docker rmi $(docker images -f dangling=true -q) + +* Attach to running docker: + + $ docker exec -i -t NameOrId /bin/sh + +## Core generation + + * `/proc/sys/core_pattern` is clearly persisted on the host. Taking note of + its content before starting any endeavour is therefore highly encouraged. + * dockers `--privileged` is necessary for a gdb session to catch the stack, + without privileges, gdb just complains about No stack. Google still is + hardly knowledgeable about this phenomenon... + * setting ulimit on docker run works perfectly, for future googlers (syntax hard to find), + a docker-compose example: + + ulimits: + core: -1 diff --git a/advanced_examples/minikube-prom-graf/Dockerfile b/advanced_examples/minikube-prom-graf/Dockerfile new file mode 100644 index 0000000..21b5c9d --- /dev/null +++ b/advanced_examples/minikube-prom-graf/Dockerfile @@ -0,0 +1,42 @@ +# Build stage 0 +FROM erlang:alpine + +# Install some libs +RUN apk add --no-cache g++ && \ + apk add --no-cache make + +# Install Rebar3 +RUN mkdir -p /buildroot/rebar3/bin +ADD https://s3.amazonaws.com/rebar3/rebar3 /buildroot/rebar3/bin/rebar3 +RUN chmod a+x /buildroot/rebar3/bin/rebar3 + +# Setup Environment +ENV PATH=/buildroot/rebar3/bin:$PATH + +# Reset working directory +WORKDIR /buildroot + +# Copy our Erlang test application +COPY dockerwatch dockerwatch + +# And build the release +WORKDIR dockerwatch +RUN rebar3 as prod release + + +# Build stage 1 +FROM alpine + +# Install some libs +RUN apk add --no-cache openssl && \ + apk add --no-cache ncurses-libs && \ + apk add --no-cache libstdc++ + +# Install the released application +COPY --from=0 /buildroot/dockerwatch/_build/prod/rel/dockerwatch /dockerwatch + +# Expose relevant ports +EXPOSE 8080 +EXPOSE 8443 + +CMD ["/dockerwatch/bin/dockerwatch", "foreground"] diff --git a/advanced_examples/minikube-prom-graf/README-CERTS.md b/advanced_examples/minikube-prom-graf/README-CERTS.md new file mode 100644 index 0000000..10b9565 --- /dev/null +++ b/advanced_examples/minikube-prom-graf/README-CERTS.md @@ -0,0 +1,23 @@ +## Generating Certificate + +Generate certificates in subdirectory `ssl`. + +### Root CA + + $ openssl genrsa -out dockerwatch-ca.key 4096 + + $ openssl req -x509 -new -nodes -key dockerwatch-ca.key -sha256 -days 1024 -out dockerwatch-ca.pem + +### Server Certificate + + $ openssl genrsa -out dockerwatch-server.key 4096 + +Certificate signing request + + $ openssl req -new -key dockerwatch-server.key -out dockerwatch-server.csr + +The most important field: `Common Name (eg, YOUR name) []: localhost`. We use localhost in this example. + +### Sign it + + $ openssl x509 -req -in dockerwatch-server.csr -CA dockerwatch-ca.pem -CAkey dockerwatch-ca.key -CAcreateserial -out dockerwatch-server.pem -days 500 -sha256 diff --git a/advanced_examples/minikube-prom-graf/README.md b/advanced_examples/minikube-prom-graf/README.md new file mode 100644 index 0000000..380572d --- /dev/null +++ b/advanced_examples/minikube-prom-graf/README.md @@ -0,0 +1,221 @@ +# Using Minikube, Erlang and Prometheus + +This is a quick demo of using minikube to run an Erlang node with prometheus and grafana. +The example we will use is the +[Docker Watch](http://github.com/erlang/docker-erlang-example/tree/master) node. +This demo assumes that you have done the +[Using Minikube](http://github.com/erlang/docker-erlang-example/tree/minikube-simple) demo. + +This is only meant to be an example of how to get started. It is not the only, +nor neccesarily the best way to setup minikube with Erlang. + +# Other Demos + +* [Using Docker](http://github.com/erlang/docker-erlang-example/) +* [Using Docker: Logstash](http://github.com/erlang/docker-erlang-example/tree/logstash) +* [Using Docker Compose: Logstash/ElasticSearch/Kibana](http://github.com/erlang/docker-erlang-example/tree/elk) +* [Using Minikube: Simple](http://github.com/erlang/docker-erlang-example/tree/minikube-simple) +* [Using Minikube: Prometheus/Grafana](http://github.com/erlang/docker-erlang-example/tree/minikube-prom-graf) +* [Using Minikube: Distributed Erlang](http://github.com/erlang/docker-erlang-example/tree/minikube-dist) +* [Using Minikube: Encrypted Distributed Erlang](http://github.com/erlang/docker-erlang-example/tree/minikube-tls-dist) + +# Prerequisites + +To start with you should familiarize yourself with minikube through this guide: +https://kubernetes.io/docs/setup/minikube/ + +In a nutshell: + +## Install + + * [VirtualBox](https://www.virtualbox.org/wiki/Downloads) + * [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) + * [minikube](https://github.com/kubernetes/minikube/releases) + +## Start and test + + > minikube start + > kubectl run hello-minikube --image=k8s.gcr.io/echoserver:1.10 --port=8080 + > kubectl expose deployment hello-minikube --type=NodePort + > curl $(minikube service hello-minikube --url) + ## Should print a lot of text + > kubectl delete services hello-minikube + > kubectl delete deployment hello-minikube + > minikube stop + +# Deploying Dockerwatch and Prometheus + +In this demo we will be doing three things: + +* Extend dockerwatch with prometheus metrics support +* Create Deployments for Prometheus and Grafana +* Create a Service that will be used to access the dockerwatch API +* Create a Secret for our ssl keys +* Create a Deployment of dockerwatch that implements the Service + +First however, make sure that the minikube cluster is started: + + > minikube start + +and that you have cloned this repo and checked out this branch: + + > git clone https://github.com/erlang/docker-erlang-example + > cd docker-erlang-example + > git checkout minikube-simple + +## Extend dockerwatch + +In this demo we will be using the [prometheus](https://hex.pm/packages/prometheus), +[prometheus\_process\_collector](https://hex.pm/packages/prometheus_process_collector) +and [prometheus\_cowboy](https://hex.pm/packages/prometheus_cowboy) hex packages to get +the instrumentation we need. So we need to add those packages to the rebar.conf file. + +``` +{deps, [{jsone, "1.4.7"}, %% JSON Encode/Decode + {cowboy, "2.5.0"}, %% HTTP Server + {prometheus,"4.2.0"}, + {prometheus_process_collector,"1.4.0"} + {prometheus_cowboy,"0.1.4"}]}. +``` + +And also the corresponding modificaion to the app.src file: + +``` +{applications, [ + kernel, + stdlib, + jsone, + cowboy, + prometheus, + prometheus_process_collector, + prometheus_cowboy + ]}, +``` + +We then need to add a new http endpoint that takes requests from the prometheus +server and returns the correct results. This service traditionally runs on port +9000 so we add another child to the dockerwatch supervisor. + + +``` +PromConfig = + #{ env => #{ dispatch => + cowboy_router:compile( + [{'_', [{"/metrics/[:registry]", prometheus_cowboy2_handler, []}]}]) } + }, + +Prometheus = ranch:child_spec( + cowboy_prometheus, 100, ranch_tcp, + [{port, 9000}], + cowboy_clear, + PromConfig), +``` + +We also need to add the correct instrumentation to the cowboy servers so that we +can measure things like requests per minute and the 95th percentile latency of +requests. This is done by modifying the cowboy config to include a metric_callback +and two stream handlers. + +``` +CowConfig = #{ env => #{ dispatch => Dispatch }, + metrics_callback => fun prometheus_cowboy2_instrumenter:observe/1, + stream_handlers => [cowboy_metrics_h, cowboy_stream_h] }, +``` + +You can view the entire new supervisor module [here](dockerwatch/src/dockerwatch_sup.erl). + +## Deploy Dockerwatch + +Now we should deploy the dockerwatch service almost the same way as was done in +[Using Minikube: Simple](http://github.com/erlang/docker-erlang-example/tree/minikube-simple). +So: + +``` +> kubectl create service nodeport dockerwatch --tcp=8080:8080 --tcp=8443:8443 +service/dockerwatch created +> ./create-certs $(minikube ip) +...... +> kubectl create secret generic dockerwatch --from-file=ssl/ +secret/dockerwatch created +> eval $(minikube docker-env) +> docker build -t dockerwatch . +``` + +We will have to modify the deployment somewhat from the original example: + +``` +cat < kubectl apply -f monitoring-namespace.yaml +> kubectl apply -f prometheus-config.yaml +> kubectl apply -f prometheus-deployment.yaml +> kubectl apply -f prometheus-service.yaml +``` + +You should now be able to view the prometheus dashboard through the url given by: + + minikube service --namespace=monitoring prometheus + +Then we can start Grafana: + +``` +> kubectl apply -f grafana-deployment.yaml +> kubectl apply -f grafana-service.yaml +``` + +Grafana can then be found under: + + minikube service --namespace=monitoring grafana + +The username and password is `admin`. You then need to add a new prometheus datasource to grafana. +The url to prometheus within the cluster is http://prometheus.monitoring.svc.cluster.local:9090. + +There are some ready made dashboards at: https://github.com/deadtrickster/beam-dashboards that +can be imported to get something quick up and running. If done correcly it could look like this: + +![BEAM Dashboard](grafana-screenshot.png) diff --git a/advanced_examples/minikube-prom-graf/create-certs b/advanced_examples/minikube-prom-graf/create-certs new file mode 100755 index 0000000..69ffc4d --- /dev/null +++ b/advanced_examples/minikube-prom-graf/create-certs @@ -0,0 +1,22 @@ +#!/bin/sh + +set -e + +if [ ! -d ssl ]; then + mkdir ssl +fi + +# Create the root CA (Certificate Authority) +openssl genrsa -out ssl/dockerwatch-ca.key 4096 + +## Certificate signing request for root CA +openssl req -x509 -new -nodes -key ssl/dockerwatch-ca.key -sha256 -days 1024 -subj "/C=SE/" -out ssl/dockerwatch-ca.pem + +# Create the server certificate +openssl genrsa -out ssl/dockerwatch-server.key 4096 + +## Certificate signing request for server certificate +openssl req -new -key ssl/dockerwatch-server.key -subj "/C=SE/CN=$1/" -out ssl/dockerwatch-server.csr + +## Sign the server certificate using the root CA +openssl x509 -req -in ssl/dockerwatch-server.csr -CA ssl/dockerwatch-ca.pem -CAkey ssl/dockerwatch-ca.key -CAcreateserial -out ssl/dockerwatch-server.pem -days 500 -sha256 diff --git a/advanced_examples/minikube-prom-graf/dockerwatch-deploy.yaml b/advanced_examples/minikube-prom-graf/dockerwatch-deploy.yaml new file mode 100644 index 0000000..8783071 --- /dev/null +++ b/advanced_examples/minikube-prom-graf/dockerwatch-deploy.yaml @@ -0,0 +1,40 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + ## Name and labels of the Deployment + labels: + app: dockerwatch + name: dockerwatch +spec: + replicas: 1 + selector: + matchLabels: + app: dockerwatch + template: + metadata: + labels: + app: dockerwatch + annotations: ## These annotations will tell prometheus to scrape us + prometheus.io/scrape: "true" + prometheus.io/port: "9000" + spec: + containers: + ## The container to launch + - image: dockerwatch + name: dockerwatch + imagePullPolicy: Never ## Set to Never as we built the image in the cluster + ports: + - containerPort: 8080 + protocol: TCP + - containerPort: 8443 + protocol: TCP + - containerPort: 9000 ## Expose the prometheus port + protocol: TCP + volumeMounts: + - name: kube-keypair + readOnly: true + mountPath: /etc/ssl/certs + volumes: + - name: kube-keypair + secret: + secretName: dockerwatch \ No newline at end of file diff --git a/advanced_examples/minikube-prom-graf/dockerwatch/config/sys.config b/advanced_examples/minikube-prom-graf/dockerwatch/config/sys.config new file mode 100644 index 0000000..674fbc9 --- /dev/null +++ b/advanced_examples/minikube-prom-graf/dockerwatch/config/sys.config @@ -0,0 +1,5 @@ +[%% Kernel/logger + {kernel, [{logger,[{handler,default,logger_std_h,#{}}]} + %%,{logger_level,info} + ]} +]. diff --git a/advanced_examples/minikube-prom-graf/dockerwatch/config/vm.args b/advanced_examples/minikube-prom-graf/dockerwatch/config/vm.args new file mode 100644 index 0000000..3c1f4b6 --- /dev/null +++ b/advanced_examples/minikube-prom-graf/dockerwatch/config/vm.args @@ -0,0 +1,2 @@ +-sname dockerwatch + diff --git a/advanced_examples/minikube-prom-graf/dockerwatch/rebar.config b/advanced_examples/minikube-prom-graf/dockerwatch/rebar.config new file mode 100644 index 0000000..a3ff73c --- /dev/null +++ b/advanced_examples/minikube-prom-graf/dockerwatch/rebar.config @@ -0,0 +1,20 @@ + +{deps, [{jsone, "1.4.7"}, %% JSON Encode/Decode + {cowboy, "2.5.0"}, %% HTTP Server + {prometheus,"4.2.0"}, + {prometheus_process_collector,"1.4.0"}, + {prometheus_cowboy,"0.1.4"}]}. + +{relx, [{release, {"dockerwatch", "1.0.0"}, [dockerwatch]}, + {vm_args, "config/vm.args"}, + {sys_config, "config/sys.config"}, + {dev_mode, true}, + {include_erts, false}, + {extended_start_script, true} + ]}. + +{profiles, [{prod, [{relx, [{dev_mode, false}, + {include_erts, true}, + {include_src, false}]}]} + ]}. +%% vim: ft=erlang diff --git a/advanced_examples/minikube-prom-graf/dockerwatch/src/dockerwatch.app.src b/advanced_examples/minikube-prom-graf/dockerwatch/src/dockerwatch.app.src new file mode 100644 index 0000000..de620bd --- /dev/null +++ b/advanced_examples/minikube-prom-graf/dockerwatch/src/dockerwatch.app.src @@ -0,0 +1,19 @@ +%% Feel free to use, reuse and abuse the code in this file. + +{application, dockerwatch, [ + {description, "Cowboy REST Hello World example."}, + {vsn, "1.0.0"}, + {modules, []}, + {registered, [dockerwatch_sup]}, + {applications, [ + kernel, + stdlib, + jsone, + cowboy, + prometheus, + prometheus_process_collector, + prometheus_cowboy + ]}, + {mod, {dockerwatch_app, []}}, + {env, []} +]}. diff --git a/advanced_examples/minikube-prom-graf/dockerwatch/src/dockerwatch.erl b/advanced_examples/minikube-prom-graf/dockerwatch/src/dockerwatch.erl new file mode 100644 index 0000000..3e5b400 --- /dev/null +++ b/advanced_examples/minikube-prom-graf/dockerwatch/src/dockerwatch.erl @@ -0,0 +1,45 @@ +%% +%% Copyright (C) 2014 Björn-Egil Dahlberg +%% +%% File: dockerwatch.erl +%% Author: Björn-Egil Dahlberg +%% Created: 2014-09-10 +%% + +-module(dockerwatch). + +-export([start_link/0, all/0, create/1, get/1, increment/2, decrement/2]). + +-type counter() :: binary(). + +-spec start_link() -> {ok, pid()}. +start_link() -> + {ok, spawn_link(fun() -> ets:new(?MODULE, [named_table, public]), + receive after infinity -> ok end end)}. + +-spec all() -> [counter()]. +all() -> + ets:select(?MODULE, [{{'$1','_'},[],['$1']}]). + +-spec create(counter()) -> ok | already_exists. +create(CounterName) -> + case ets:insert_new(?MODULE, {CounterName, 0}) of + true -> + ok; + false -> + already_exists + end. + +-spec get(counter()) -> integer(). +get(CounterName) -> + ets:lookup_element(?MODULE, CounterName, 2). + +-spec increment(counter(), integer()) -> ok. +increment(CounterName, Howmuch) -> + _ = ets:update_counter(?MODULE, CounterName, [{2, Howmuch}]), + ok. + +-spec decrement(counter(), integer()) -> ok. +decrement(CounterName, Howmuch) -> + _ = ets:update_counter(?MODULE, CounterName, [{2, -1 * Howmuch}]), + ok. diff --git a/advanced_examples/minikube-prom-graf/dockerwatch/src/dockerwatch_app.erl b/advanced_examples/minikube-prom-graf/dockerwatch/src/dockerwatch_app.erl new file mode 100644 index 0000000..a85b5b6 --- /dev/null +++ b/advanced_examples/minikube-prom-graf/dockerwatch/src/dockerwatch_app.erl @@ -0,0 +1,19 @@ +%% +%% Copyright (C) 2014 Björn-Egil Dahlberg +%% +%% File: dockerwatch_app.erl +%% Author: Björn-Egil Dahlberg +%% Created: 2014-09-10 +%% + +-module(dockerwatch_app). +-behaviour(application). + +-export([start/2,stop/1]). +%% API. + +start(_Type, _Args) -> + dockerwatch_sup:start_link(). + +stop(_State) -> + ok. diff --git a/advanced_examples/minikube-prom-graf/dockerwatch/src/dockerwatch_handler.erl b/advanced_examples/minikube-prom-graf/dockerwatch/src/dockerwatch_handler.erl new file mode 100644 index 0000000..1ac15f9 --- /dev/null +++ b/advanced_examples/minikube-prom-graf/dockerwatch/src/dockerwatch_handler.erl @@ -0,0 +1,103 @@ +%% +%% Copyright (C) 2014 Björn-Egil Dahlberg +%% +%% File: dockerwatch_handler.erl +%% Author: Björn-Egil Dahlberg +%% Created: 2014-09-10 +%% + +-module(dockerwatch_handler). + +-export([init/2]). +-export([allowed_methods/2]). +-export([content_types_accepted/2]). +-export([content_types_provided/2]). +-export([handle_post/2]). +-export([to_html/2]). +-export([to_json/2]). +-export([to_text/2]). + +init(Req, []) -> + {cowboy_rest, Req, []}. + +%% Which HTTP methods are allowed +allowed_methods(Req, State) -> + {[<<"GET">>, <<"POST">>], Req, State}. + +%% Which content types are accepted by POST/PUT requests +content_types_accepted(Req, State) -> + {[{{<<"application">>, <<"json">>, []}, handle_post}], + Req, State}. + +%% Handle the POST/PUT request +handle_post(Req, State) -> + case cowboy_req:binding(counter_name, Req) of + undefined -> + {false, Req, State}; + Name -> + case cowboy_req:has_body(Req) of + true -> + {ok, Body, Req3} = cowboy_req:read_body(Req), + Json = jsone:decode(Body), + ActionBin = maps:get(<<"action">>, Json, <<"increment">>), + Value = maps:get(<<"value">>, Json, 1), + Action = list_to_atom(binary_to_list(ActionBin)), + ok = dockerwatch:Action(Name, Value), + {true, Req3, State}; + false -> + ok = dockerwatch:create(Name), + {true, Req, State} + end + end. + +%% Which content types we handle for GET/HEAD requests +content_types_provided(Req, State) -> + {[{<<"text/html">>, to_html}, + {<<"application/json">>, to_json}, + {<<"text/plain">>, to_text} + ], Req, State}. + + +%% Return counters/counter as json +to_json(Req, State) -> + Resp = case cowboy_req:binding(counter_name, Req) of + undefined -> + dockerwatch:all(); + Counter -> + #{ Counter => dockerwatch:get(Counter) } + end, + {jsone:encode(Resp), Req, State}. + +%% Return counters/counter as plain text +to_text(Req, State) -> + Resp = case cowboy_req:binding(counter_name, Req) of + undefined -> + [io_lib:format("~s~n",[Counter]) || Counter <- dockerwatch:all()]; + Counter -> + io_lib:format("~p",[dockerwatch:get(Counter)]) + end, + {Resp, Req, State}. + +%% Return counters/counter as html +to_html(Req, State) -> + Body = case cowboy_req:binding(counter_name, Req) of + undefined -> + Counters = dockerwatch:all(), + ["\n"]; + Counter -> + Value = dockerwatch:get(Counter), + io_lib:format("~s = ~p",[Counter, Value]) + end, + {[html_head(),Body,html_tail()], Req, State}. + +html_head() -> + <<" + + + dockerwatch + ">>. +html_tail() -> + <<" + ">>. diff --git a/advanced_examples/minikube-prom-graf/dockerwatch/src/dockerwatch_sup.erl b/advanced_examples/minikube-prom-graf/dockerwatch/src/dockerwatch_sup.erl new file mode 100644 index 0000000..bd1bd9d --- /dev/null +++ b/advanced_examples/minikube-prom-graf/dockerwatch/src/dockerwatch_sup.erl @@ -0,0 +1,66 @@ +%% +%% Copyright (C) 2014 Björn-Egil Dahlberg +%% +%% File: dockerwatch_sup.erl +%% Author: Björn-Egil Dahlberg +%% Created: 2014-09-10 +%% + +-module(dockerwatch_sup). +-behaviour(supervisor). + +-export([start_link/0,init/1]). + +%% API. + +-spec start_link() -> {ok, pid()}. +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +%% supervisor. + +init([]) -> + CertsDir = "/etc/ssl/certs/", + + Dispatch = cowboy_router:compile( + [ + {'_', [{"/[:counter_name]", dockerwatch_handler, []}]} + ]), + + CowConfig = #{ env => #{ dispatch => Dispatch }, + metrics_callback => fun prometheus_cowboy2_instrumenter:observe/1, + stream_handlers => [cowboy_metrics_h, cowboy_stream_h] }, + + HTTPS = ranch:child_spec( + cowboy_https, 100, ranch_ssl, + [{port, 8443}, + {cacertfile, filename:join(CertsDir, "dockerwatch-ca.pem")}, + {certfile, filename:join(CertsDir, "dockerwatch-server.pem")}, + {keyfile, filename:join(CertsDir, "dockerwatch-server.key")}], + cowboy_tls, + CowConfig), + + HTTP = ranch:child_spec( + cowboy_http, 100, ranch_tcp, + [{port, 8080}], + cowboy_clear, + CowConfig), + + PromConfig = + #{ env => #{ dispatch => + cowboy_router:compile( + [{'_', [{"/metrics/[:registry]", prometheus_cowboy2_handler, []}]}]) } + }, + + Prometheus = ranch:child_spec( + cowboy_prometheus, 100, ranch_tcp, + [{port, 9000}], + cowboy_clear, + PromConfig), + + Counter = {dockerwatch, {dockerwatch, start_link, []}, + permanent, 5000, worker, [dockerwatch]}, + + Procs = [Counter, HTTP, HTTPS, Prometheus], + + {ok, {{one_for_one, 10, 10}, Procs}}. diff --git a/advanced_examples/minikube-prom-graf/grafana-deployment.yaml b/advanced_examples/minikube-prom-graf/grafana-deployment.yaml new file mode 100644 index 0000000..536b02f --- /dev/null +++ b/advanced_examples/minikube-prom-graf/grafana-deployment.yaml @@ -0,0 +1,46 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + labels: + name: grafana + name: grafana + namespace: monitoring +spec: + replicas: 1 + selector: + matchLabels: + name: grafana + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + creationTimestamp: null + labels: + name: grafana + spec: + containers: + - image: grafana/grafana + imagePullPolicy: Always + name: grafana + ports: + - containerPort: 3000 + protocol: TCP + resources: + limits: + cpu: 500m + memory: 2500Mi + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - mountPath: /var/lib/grafana + name: data + restartPolicy: Always + securityContext: {} + terminationGracePeriodSeconds: 30 + volumes: + - emptyDir: {} + name: data diff --git a/advanced_examples/minikube-prom-graf/grafana-screenshot.png b/advanced_examples/minikube-prom-graf/grafana-screenshot.png new file mode 100644 index 0000000..0808d47 Binary files /dev/null and b/advanced_examples/minikube-prom-graf/grafana-screenshot.png differ diff --git a/advanced_examples/minikube-prom-graf/grafana-service.yaml b/advanced_examples/minikube-prom-graf/grafana-service.yaml new file mode 100644 index 0000000..868c113 --- /dev/null +++ b/advanced_examples/minikube-prom-graf/grafana-service.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: grafana + namespace: monitoring +spec: + ports: + - port: 3000 + protocol: TCP + targetPort: 3000 + selector: + name: grafana + type: NodePort diff --git a/advanced_examples/minikube-prom-graf/monitoring-namespace.yaml b/advanced_examples/minikube-prom-graf/monitoring-namespace.yaml new file mode 100644 index 0000000..ff7ae1b --- /dev/null +++ b/advanced_examples/minikube-prom-graf/monitoring-namespace.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: monitoring diff --git a/advanced_examples/minikube-prom-graf/prometheus-config.yaml b/advanced_examples/minikube-prom-graf/prometheus-config.yaml new file mode 100644 index 0000000..6aab13f --- /dev/null +++ b/advanced_examples/minikube-prom-graf/prometheus-config.yaml @@ -0,0 +1,280 @@ +apiVersion: v1 +data: + prometheus.yml: | + # A scrape configuration for running Prometheus on a Kubernetes cluster. + # This uses separate scrape configs for cluster components (i.e. API server, node) + # and services to allow each to use different authentication configs. + # + # Kubernetes labels will be added as Prometheus labels on metrics via the + # `labelmap` relabeling action. + # + # If you are using Kubernetes 1.7.2 or earlier, please take note of the comments + # for the kubernetes-cadvisor job; you will need to edit or remove this job. + + # Scrape config for API servers. + # + # Kubernetes exposes API servers as endpoints to the default/kubernetes + # service so this uses `endpoints` role and uses relabelling to only keep + # the endpoints associated with the default/kubernetes service using the + # default named port `https`. This works for single API server deployments as + # well as HA API server deployments. + scrape_configs: + - job_name: 'kubernetes-apiservers' + + kubernetes_sd_configs: + - role: endpoints + + # Default to scraping over https. If required, just disable this or change to + # `http`. + scheme: https + + # This TLS & bearer token file config is used to connect to the actual scrape + # endpoints for cluster components. This is separate to discovery auth + # configuration because discovery & scraping are two separate concerns in + # Prometheus. The discovery auth config is automatic if Prometheus runs inside + # the cluster. Otherwise, more config options have to be provided within the + # . + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + # If your node certificates are self-signed or use a different CA to the + # master CA, then disable certificate verification below. Note that + # certificate verification is an integral part of a secure infrastructure + # so this should only be disabled in a controlled environment. You can + # disable certificate verification by uncommenting the line below. + # + insecure_skip_verify: true + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + + # Keep only the default/kubernetes service endpoints for the https port. This + # will add targets for each API server which Kubernetes adds an endpoint to + # the default/kubernetes service. + relabel_configs: + - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] + action: keep + regex: default;kubernetes;https + + # Scrape config for nodes (kubelet). + # + # Rather than connecting directly to the node, the scrape is proxied though the + # Kubernetes apiserver. This means it will work if Prometheus is running out of + # cluster, or can't connect to nodes for some other reason (e.g. because of + # firewalling). + - job_name: 'kubernetes-nodes' + + # Default to scraping over https. If required, just disable this or change to + # `http`. + scheme: https + + # This TLS & bearer token file config is used to connect to the actual scrape + # endpoints for cluster components. This is separate to discovery auth + # configuration because discovery & scraping are two separate concerns in + # Prometheus. The discovery auth config is automatic if Prometheus runs inside + # the cluster. Otherwise, more config options have to be provided within the + # . + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + + kubernetes_sd_configs: + - role: node + + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/${1}/proxy/metrics + + # Scrape config for Kubelet cAdvisor. + # + # This is required for Kubernetes 1.7.3 and later, where cAdvisor metrics + # (those whose names begin with 'container_') have been removed from the + # Kubelet metrics endpoint. This job scrapes the cAdvisor endpoint to + # retrieve those metrics. + # + # In Kubernetes 1.7.0-1.7.2, these metrics are only exposed on the cAdvisor + # HTTP endpoint; use "replacement: /api/v1/nodes/${1}:4194/proxy/metrics" + # in that case (and ensure cAdvisor's HTTP server hasn't been disabled with + # the --cadvisor-port=0 Kubelet flag). + # + # This job is not necessary and should be removed in Kubernetes 1.6 and + # earlier versions, or it will cause the metrics to be scraped twice. + - job_name: 'kubernetes-cadvisor' + + # Default to scraping over https. If required, just disable this or change to + # `http`. + scheme: https + + # This TLS & bearer token file config is used to connect to the actual scrape + # endpoints for cluster components. This is separate to discovery auth + # configuration because discovery & scraping are two separate concerns in + # Prometheus. The discovery auth config is automatic if Prometheus runs inside + # the cluster. Otherwise, more config options have to be provided within the + # . + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + + kubernetes_sd_configs: + - role: node + + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor + + # Scrape config for service endpoints. + # + # The relabeling allows the actual service scrape endpoint to be configured + # via the following annotations: + # + # * `prometheus.io/scrape`: Only scrape services that have a value of `true` + # * `prometheus.io/scheme`: If the metrics endpoint is secured then you will need + # to set this to `https` & most likely set the `tls_config` of the scrape config. + # * `prometheus.io/path`: If the metrics path is not `/metrics` override this. + # * `prometheus.io/port`: If the metrics are exposed on a different port to the + # service then set this appropriately. + - job_name: 'kubernetes-service-endpoints' + + kubernetes_sd_configs: + - role: endpoints + + relabel_configs: + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme] + action: replace + target_label: __scheme__ + regex: (https?) + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port] + action: replace + target_label: __address__ + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_service_name] + action: replace + target_label: kubernetes_name + + # Example scrape config for probing services via the Blackbox Exporter. + # + # The relabeling allows the actual service scrape endpoint to be configured + # via the following annotations: + # + # * `prometheus.io/probe`: Only probe services that have a value of `true` + - job_name: 'kubernetes-services' + + metrics_path: /probe + params: + module: [http_2xx] + + kubernetes_sd_configs: + - role: service + + relabel_configs: + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe] + action: keep + regex: true + - source_labels: [__address__] + target_label: __param_target + - target_label: __address__ + replacement: blackbox-exporter.example.com:9115 + - source_labels: [__param_target] + target_label: instance + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_service_name] + target_label: kubernetes_name + + # Example scrape config for probing ingresses via the Blackbox Exporter. + # + # The relabeling allows the actual ingress scrape endpoint to be configured + # via the following annotations: + # + # * `prometheus.io/probe`: Only probe services that have a value of `true` + - job_name: 'kubernetes-ingresses' + + metrics_path: /probe + params: + module: [http_2xx] + + kubernetes_sd_configs: + - role: ingress + + relabel_configs: + - source_labels: [__meta_kubernetes_ingress_annotation_prometheus_io_probe] + action: keep + regex: true + - source_labels: [__meta_kubernetes_ingress_scheme,__address__,__meta_kubernetes_ingress_path] + regex: (.+);(.+);(.+) + replacement: ${1}://${2}${3} + target_label: __param_target + - target_label: __address__ + replacement: blackbox-exporter.example.com:9115 + - source_labels: [__param_target] + target_label: instance + - action: labelmap + regex: __meta_kubernetes_ingress_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_ingress_name] + target_label: kubernetes_name + + # Example scrape config for pods + # + # The relabeling allows the actual pod scrape endpoint to be configured via the + # following annotations: + # + # * `prometheus.io/scrape`: Only scrape pods that have a value of `true` + # * `prometheus.io/path`: If the metrics path is not `/metrics` override this. + # * `prometheus.io/port`: Scrape the pod on the indicated port instead of the + # pod's declared ports (default is a port-free target if none are declared). + - job_name: 'kubernetes-pods' + + kubernetes_sd_configs: + - role: pod + + relabel_configs: + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: kubernetes_pod_name +kind: ConfigMap +metadata: + name: prometheus-config + namespace: monitoring diff --git a/advanced_examples/minikube-prom-graf/prometheus-deployment.yaml b/advanced_examples/minikube-prom-graf/prometheus-deployment.yaml new file mode 100644 index 0000000..9b31830 --- /dev/null +++ b/advanced_examples/minikube-prom-graf/prometheus-deployment.yaml @@ -0,0 +1,101 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: prometheus +rules: +- apiGroups: [""] + resources: + - nodes + - nodes/proxy + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] +- apiGroups: + - extensions + resources: + - ingresses + verbs: ["get", "list", "watch"] +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: default + namespace: monitoring +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: prometheus +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus +subjects: +- kind: ServiceAccount + name: default + namespace: monitoring +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + labels: + name: prometheus + name: prometheus + namespace: monitoring +spec: + replicas: 1 + selector: + matchLabels: + name: prometheus + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + creationTimestamp: null + labels: + name: prometheus + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + spec: + containers: + - args: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention=24h + command: + - /bin/prometheus + image: prom/prometheus + imagePullPolicy: Always + name: prometheus + ports: + - containerPort: 9090 + protocol: TCP + resources: + limits: + cpu: 500m + memory: 2500Mi + requests: + cpu: 10m + memory: 100Mi + volumeMounts: + - mountPath: /prometheus + name: data + - mountPath: /etc/prometheus + name: config-volume + restartPolicy: Always + securityContext: {} + terminationGracePeriodSeconds: 30 + volumes: + - emptyDir: {} + name: data + - configMap: + name: prometheus-config + name: config-volume diff --git a/advanced_examples/minikube-prom-graf/prometheus-service.yaml b/advanced_examples/minikube-prom-graf/prometheus-service.yaml new file mode 100644 index 0000000..847dc2f --- /dev/null +++ b/advanced_examples/minikube-prom-graf/prometheus-service.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: prometheus + namespace: monitoring +spec: + ports: + - port: 9090 + protocol: TCP + targetPort: 9090 + selector: + name: prometheus + type: NodePort