From 6b805ff3438716b7b4661483eb2733a27fe929ad Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Mon, 20 May 2019 14:18:17 +0200 Subject: [PATCH] Refactoring: Copy Minikube, Erlang and Prometheus example This commit copies Lukas Larsson's (@garazdawi) and Siri Hansen's (@sirihansen) example from: https://github.com/erlang/docker-erlang-example/tree/minikube-prom-graf --- .../minikube-prom-graf/.gitignore | 2 + .../minikube-prom-graf/.travis.yml | 56 ++++ .../minikube-prom-graf/Docker-Cheat-Sheet.md | 26 ++ .../minikube-prom-graf/Dockerfile | 42 +++ .../minikube-prom-graf/README-CERTS.md | 23 ++ .../minikube-prom-graf/README.md | 221 ++++++++++++++ .../minikube-prom-graf/create-certs | 22 ++ .../dockerwatch-deploy.yaml | 40 +++ .../dockerwatch/config/sys.config | 5 + .../dockerwatch/config/vm.args | 2 + .../dockerwatch/rebar.config | 20 ++ .../dockerwatch/src/dockerwatch.app.src | 19 ++ .../dockerwatch/src/dockerwatch.erl | 45 +++ .../dockerwatch/src/dockerwatch_app.erl | 19 ++ .../dockerwatch/src/dockerwatch_handler.erl | 103 +++++++ .../dockerwatch/src/dockerwatch_sup.erl | 66 +++++ .../grafana-deployment.yaml | 46 +++ .../minikube-prom-graf/grafana-screenshot.png | Bin 0 -> 118295 bytes .../minikube-prom-graf/grafana-service.yaml | 13 + .../monitoring-namespace.yaml | 5 + .../minikube-prom-graf/prometheus-config.yaml | 280 ++++++++++++++++++ .../prometheus-deployment.yaml | 101 +++++++ .../prometheus-service.yaml | 13 + 23 files changed, 1169 insertions(+) create mode 100644 advanced_examples/minikube-prom-graf/.gitignore create mode 100644 advanced_examples/minikube-prom-graf/.travis.yml create mode 100644 advanced_examples/minikube-prom-graf/Docker-Cheat-Sheet.md create mode 100644 advanced_examples/minikube-prom-graf/Dockerfile create mode 100644 advanced_examples/minikube-prom-graf/README-CERTS.md create mode 100644 advanced_examples/minikube-prom-graf/README.md create mode 100755 advanced_examples/minikube-prom-graf/create-certs create mode 100644 advanced_examples/minikube-prom-graf/dockerwatch-deploy.yaml create mode 100644 advanced_examples/minikube-prom-graf/dockerwatch/config/sys.config create mode 100644 advanced_examples/minikube-prom-graf/dockerwatch/config/vm.args create mode 100644 advanced_examples/minikube-prom-graf/dockerwatch/rebar.config create mode 100644 advanced_examples/minikube-prom-graf/dockerwatch/src/dockerwatch.app.src create mode 100644 advanced_examples/minikube-prom-graf/dockerwatch/src/dockerwatch.erl create mode 100644 advanced_examples/minikube-prom-graf/dockerwatch/src/dockerwatch_app.erl create mode 100644 advanced_examples/minikube-prom-graf/dockerwatch/src/dockerwatch_handler.erl create mode 100644 advanced_examples/minikube-prom-graf/dockerwatch/src/dockerwatch_sup.erl create mode 100644 advanced_examples/minikube-prom-graf/grafana-deployment.yaml create mode 100644 advanced_examples/minikube-prom-graf/grafana-screenshot.png create mode 100644 advanced_examples/minikube-prom-graf/grafana-service.yaml create mode 100644 advanced_examples/minikube-prom-graf/monitoring-namespace.yaml create mode 100644 advanced_examples/minikube-prom-graf/prometheus-config.yaml create mode 100644 advanced_examples/minikube-prom-graf/prometheus-deployment.yaml create mode 100644 advanced_examples/minikube-prom-graf/prometheus-service.yaml diff --git a/advanced_examples/minikube-prom-graf/.gitignore b/advanced_examples/minikube-prom-graf/.gitignore new file mode 100644 index 0000000..172c657 --- /dev/null +++ b/advanced_examples/minikube-prom-graf/.gitignore @@ -0,0 +1,2 @@ +_build/ +rebar.lock diff --git a/advanced_examples/minikube-prom-graf/.travis.yml b/advanced_examples/minikube-prom-graf/.travis.yml new file mode 100644 index 0000000..2d251a0 --- /dev/null +++ b/advanced_examples/minikube-prom-graf/.travis.yml @@ -0,0 +1,56 @@ +sudo: required + +dist: xenial + +addons: + apt: + packages: + - curl + +env: +- CHANGE_MINIKUBE_NONE_USER=true + +before_script: +# Download minikube. +- MINIKUBE_VERSION=latest +- curl -Lo minikube https://storage.googleapis.com/minikube/releases/$MINIKUBE_VERSION/minikube-linux-amd64 && chmod +x minikube && sudo mv minikube /usr/local/bin/ +# Download kubectl, which is a requirement for using minikube. +- KUBERNETES_VERSION=$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt) +- curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/$KUBERNETES_VERSION/bin/linux/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/ +# Test that it works +- kubectl -h +- sudo minikube start -v 7 --logtostderr --vm-driver=none --kubernetes-version "$KUBERNETES_VERSION" +# Fix the kubectl context, as it's often stale. +- minikube update-context +# Wait for Kubernetes to be up and ready. +- JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'; until kubectl get nodes -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do sleep 1; done + +script: +- kubectl cluster-info +# kube-addon-manager is responsible for managing other kubernetes components, such as kube-dns, dashboard, storage-provisioner.. +- JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'; until kubectl -n kube-system get pods -lcomponent=kube-addon-manager -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do sleep 5;echo "waiting for kube-addon-manager to be available"; kubectl get pods --all-namespaces; done +# Wait for kube-dns to be ready. +- JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'; until kubectl -n kube-system get pods -lk8s-app=kube-dns -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do sleep 5;echo "waiting for kube-dns to be available"; kubectl get pods --all-namespaces; done +- kubectl create service nodeport dockerwatch --tcp=8080:8080 --tcp=8443:8443 +- kubectl get service +- ./create-certs $(minikube ip) +- kubectl create secret generic dockerwatch --from-file=ssl/ +- kubectl get secret +- docker build -t dockerwatch . +- kubectl apply -f dockerwatch-deploy.yaml +# Wait for dockerwatch to be ready. +- JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'; until kubectl -n default get pods -lapp=dockerwatch -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do sleep 5;echo "waiting for dockerwatch to be available"; kubectl get pods --all-namespaces; done +- HTTP=$(minikube service dockerwatch --url | head -1) +- HTTPS=$(minikube service dockerwatch --url --https | tail -1) +- "curl -v -H 'Content-Type: application/json' -X POST -d '' $HTTP/cnt" +- "curl -v -H 'Content-Type: application/json' -X POST -d '{}' $HTTP/cnt" +- "curl -v --cacert ssl/dockerwatch-ca.pem -H 'Accept: application/json' $HTTPS/" +- "curl -v --cacert ssl/dockerwatch-ca.pem -H 'Accept: application/json' $HTTPS/cnt" +- kubectl apply -f monitoring-namespace.yaml +- kubectl apply -f prometheus-config.yaml +- kubectl apply -f prometheus-deployment.yaml +- kubectl apply -f prometheus-service.yaml +- kubectl apply -f grafana-deployment.yaml +- kubectl apply -f grafana-service.yaml +- JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'; until kubectl -n monitoring get pods -lname=grafana -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do sleep 5;echo "waiting for grafana to be available"; kubectl get pods --all-namespaces; done +- JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'; until kubectl -n monitoring get pods -lname=prometheus -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do sleep 5;echo "waiting for prometheus to be available"; kubectl get pods --all-namespaces; done diff --git a/advanced_examples/minikube-prom-graf/Docker-Cheat-Sheet.md b/advanced_examples/minikube-prom-graf/Docker-Cheat-Sheet.md new file mode 100644 index 0000000..9d58795 --- /dev/null +++ b/advanced_examples/minikube-prom-graf/Docker-Cheat-Sheet.md @@ -0,0 +1,26 @@ +## Docker Cheatsheet + +* Remove all containers that are not running: + + $ docker rm $(docker ps -aq -f status=exited) + +* Remove dangling images: + + $ docker rmi $(docker images -f dangling=true -q) + +* Attach to running docker: + + $ docker exec -i -t NameOrId /bin/sh + +## Core generation + + * `/proc/sys/core_pattern` is clearly persisted on the host. Taking note of + its content before starting any endeavour is therefore highly encouraged. + * dockers `--privileged` is necessary for a gdb session to catch the stack, + without privileges, gdb just complains about No stack. Google still is + hardly knowledgeable about this phenomenon... + * setting ulimit on docker run works perfectly, for future googlers (syntax hard to find), + a docker-compose example: + + ulimits: + core: -1 diff --git a/advanced_examples/minikube-prom-graf/Dockerfile b/advanced_examples/minikube-prom-graf/Dockerfile new file mode 100644 index 0000000..21b5c9d --- /dev/null +++ b/advanced_examples/minikube-prom-graf/Dockerfile @@ -0,0 +1,42 @@ +# Build stage 0 +FROM erlang:alpine + +# Install some libs +RUN apk add --no-cache g++ && \ + apk add --no-cache make + +# Install Rebar3 +RUN mkdir -p /buildroot/rebar3/bin +ADD https://s3.amazonaws.com/rebar3/rebar3 /buildroot/rebar3/bin/rebar3 +RUN chmod a+x /buildroot/rebar3/bin/rebar3 + +# Setup Environment +ENV PATH=/buildroot/rebar3/bin:$PATH + +# Reset working directory +WORKDIR /buildroot + +# Copy our Erlang test application +COPY dockerwatch dockerwatch + +# And build the release +WORKDIR dockerwatch +RUN rebar3 as prod release + + +# Build stage 1 +FROM alpine + +# Install some libs +RUN apk add --no-cache openssl && \ + apk add --no-cache ncurses-libs && \ + apk add --no-cache libstdc++ + +# Install the released application +COPY --from=0 /buildroot/dockerwatch/_build/prod/rel/dockerwatch /dockerwatch + +# Expose relevant ports +EXPOSE 8080 +EXPOSE 8443 + +CMD ["/dockerwatch/bin/dockerwatch", "foreground"] diff --git a/advanced_examples/minikube-prom-graf/README-CERTS.md b/advanced_examples/minikube-prom-graf/README-CERTS.md new file mode 100644 index 0000000..10b9565 --- /dev/null +++ b/advanced_examples/minikube-prom-graf/README-CERTS.md @@ -0,0 +1,23 @@ +## Generating Certificate + +Generate certificates in subdirectory `ssl`. + +### Root CA + + $ openssl genrsa -out dockerwatch-ca.key 4096 + + $ openssl req -x509 -new -nodes -key dockerwatch-ca.key -sha256 -days 1024 -out dockerwatch-ca.pem + +### Server Certificate + + $ openssl genrsa -out dockerwatch-server.key 4096 + +Certificate signing request + + $ openssl req -new -key dockerwatch-server.key -out dockerwatch-server.csr + +The most important field: `Common Name (eg, YOUR name) []: localhost`. We use localhost in this example. + +### Sign it + + $ openssl x509 -req -in dockerwatch-server.csr -CA dockerwatch-ca.pem -CAkey dockerwatch-ca.key -CAcreateserial -out dockerwatch-server.pem -days 500 -sha256 diff --git a/advanced_examples/minikube-prom-graf/README.md b/advanced_examples/minikube-prom-graf/README.md new file mode 100644 index 0000000..380572d --- /dev/null +++ b/advanced_examples/minikube-prom-graf/README.md @@ -0,0 +1,221 @@ +# Using Minikube, Erlang and Prometheus + +This is a quick demo of using minikube to run an Erlang node with prometheus and grafana. +The example we will use is the +[Docker Watch](http://github.com/erlang/docker-erlang-example/tree/master) node. +This demo assumes that you have done the +[Using Minikube](http://github.com/erlang/docker-erlang-example/tree/minikube-simple) demo. + +This is only meant to be an example of how to get started. It is not the only, +nor neccesarily the best way to setup minikube with Erlang. + +# Other Demos + +* [Using Docker](http://github.com/erlang/docker-erlang-example/) +* [Using Docker: Logstash](http://github.com/erlang/docker-erlang-example/tree/logstash) +* [Using Docker Compose: Logstash/ElasticSearch/Kibana](http://github.com/erlang/docker-erlang-example/tree/elk) +* [Using Minikube: Simple](http://github.com/erlang/docker-erlang-example/tree/minikube-simple) +* [Using Minikube: Prometheus/Grafana](http://github.com/erlang/docker-erlang-example/tree/minikube-prom-graf) +* [Using Minikube: Distributed Erlang](http://github.com/erlang/docker-erlang-example/tree/minikube-dist) +* [Using Minikube: Encrypted Distributed Erlang](http://github.com/erlang/docker-erlang-example/tree/minikube-tls-dist) + +# Prerequisites + +To start with you should familiarize yourself with minikube through this guide: +https://kubernetes.io/docs/setup/minikube/ + +In a nutshell: + +## Install + + * [VirtualBox](https://www.virtualbox.org/wiki/Downloads) + * [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) + * [minikube](https://github.com/kubernetes/minikube/releases) + +## Start and test + + > minikube start + > kubectl run hello-minikube --image=k8s.gcr.io/echoserver:1.10 --port=8080 + > kubectl expose deployment hello-minikube --type=NodePort + > curl $(minikube service hello-minikube --url) + ## Should print a lot of text + > kubectl delete services hello-minikube + > kubectl delete deployment hello-minikube + > minikube stop + +# Deploying Dockerwatch and Prometheus + +In this demo we will be doing three things: + +* Extend dockerwatch with prometheus metrics support +* Create Deployments for Prometheus and Grafana +* Create a Service that will be used to access the dockerwatch API +* Create a Secret for our ssl keys +* Create a Deployment of dockerwatch that implements the Service + +First however, make sure that the minikube cluster is started: + + > minikube start + +and that you have cloned this repo and checked out this branch: + + > git clone https://github.com/erlang/docker-erlang-example + > cd docker-erlang-example + > git checkout minikube-simple + +## Extend dockerwatch + +In this demo we will be using the [prometheus](https://hex.pm/packages/prometheus), +[prometheus\_process\_collector](https://hex.pm/packages/prometheus_process_collector) +and [prometheus\_cowboy](https://hex.pm/packages/prometheus_cowboy) hex packages to get +the instrumentation we need. So we need to add those packages to the rebar.conf file. + +``` +{deps, [{jsone, "1.4.7"}, %% JSON Encode/Decode + {cowboy, "2.5.0"}, %% HTTP Server + {prometheus,"4.2.0"}, + {prometheus_process_collector,"1.4.0"} + {prometheus_cowboy,"0.1.4"}]}. +``` + +And also the corresponding modificaion to the app.src file: + +``` +{applications, [ + kernel, + stdlib, + jsone, + cowboy, + prometheus, + prometheus_process_collector, + prometheus_cowboy + ]}, +``` + +We then need to add a new http endpoint that takes requests from the prometheus +server and returns the correct results. This service traditionally runs on port +9000 so we add another child to the dockerwatch supervisor. + + +``` +PromConfig = + #{ env => #{ dispatch => + cowboy_router:compile( + [{'_', [{"/metrics/[:registry]", prometheus_cowboy2_handler, []}]}]) } + }, + +Prometheus = ranch:child_spec( + cowboy_prometheus, 100, ranch_tcp, + [{port, 9000}], + cowboy_clear, + PromConfig), +``` + +We also need to add the correct instrumentation to the cowboy servers so that we +can measure things like requests per minute and the 95th percentile latency of +requests. This is done by modifying the cowboy config to include a metric_callback +and two stream handlers. + +``` +CowConfig = #{ env => #{ dispatch => Dispatch }, + metrics_callback => fun prometheus_cowboy2_instrumenter:observe/1, + stream_handlers => [cowboy_metrics_h, cowboy_stream_h] }, +``` + +You can view the entire new supervisor module [here](dockerwatch/src/dockerwatch_sup.erl). + +## Deploy Dockerwatch + +Now we should deploy the dockerwatch service almost the same way as was done in +[Using Minikube: Simple](http://github.com/erlang/docker-erlang-example/tree/minikube-simple). +So: + +``` +> kubectl create service nodeport dockerwatch --tcp=8080:8080 --tcp=8443:8443 +service/dockerwatch created +> ./create-certs $(minikube ip) +...... +> kubectl create secret generic dockerwatch --from-file=ssl/ +secret/dockerwatch created +> eval $(minikube docker-env) +> docker build -t dockerwatch . +``` + +We will have to modify the deployment somewhat from the original example: + +``` +cat < kubectl apply -f monitoring-namespace.yaml +> kubectl apply -f prometheus-config.yaml +> kubectl apply -f prometheus-deployment.yaml +> kubectl apply -f prometheus-service.yaml +``` + +You should now be able to view the prometheus dashboard through the url given by: + + minikube service --namespace=monitoring prometheus + +Then we can start Grafana: + +``` +> kubectl apply -f grafana-deployment.yaml +> kubectl apply -f grafana-service.yaml +``` + +Grafana can then be found under: + + minikube service --namespace=monitoring grafana + +The username and password is `admin`. You then need to add a new prometheus datasource to grafana. +The url to prometheus within the cluster is http://prometheus.monitoring.svc.cluster.local:9090. + +There are some ready made dashboards at: https://github.com/deadtrickster/beam-dashboards that +can be imported to get something quick up and running. If done correcly it could look like this: + +![BEAM Dashboard](grafana-screenshot.png) diff --git a/advanced_examples/minikube-prom-graf/create-certs b/advanced_examples/minikube-prom-graf/create-certs new file mode 100755 index 0000000..69ffc4d --- /dev/null +++ b/advanced_examples/minikube-prom-graf/create-certs @@ -0,0 +1,22 @@ +#!/bin/sh + +set -e + +if [ ! -d ssl ]; then + mkdir ssl +fi + +# Create the root CA (Certificate Authority) +openssl genrsa -out ssl/dockerwatch-ca.key 4096 + +## Certificate signing request for root CA +openssl req -x509 -new -nodes -key ssl/dockerwatch-ca.key -sha256 -days 1024 -subj "/C=SE/" -out ssl/dockerwatch-ca.pem + +# Create the server certificate +openssl genrsa -out ssl/dockerwatch-server.key 4096 + +## Certificate signing request for server certificate +openssl req -new -key ssl/dockerwatch-server.key -subj "/C=SE/CN=$1/" -out ssl/dockerwatch-server.csr + +## Sign the server certificate using the root CA +openssl x509 -req -in ssl/dockerwatch-server.csr -CA ssl/dockerwatch-ca.pem -CAkey ssl/dockerwatch-ca.key -CAcreateserial -out ssl/dockerwatch-server.pem -days 500 -sha256 diff --git a/advanced_examples/minikube-prom-graf/dockerwatch-deploy.yaml b/advanced_examples/minikube-prom-graf/dockerwatch-deploy.yaml new file mode 100644 index 0000000..8783071 --- /dev/null +++ b/advanced_examples/minikube-prom-graf/dockerwatch-deploy.yaml @@ -0,0 +1,40 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + ## Name and labels of the Deployment + labels: + app: dockerwatch + name: dockerwatch +spec: + replicas: 1 + selector: + matchLabels: + app: dockerwatch + template: + metadata: + labels: + app: dockerwatch + annotations: ## These annotations will tell prometheus to scrape us + prometheus.io/scrape: "true" + prometheus.io/port: "9000" + spec: + containers: + ## The container to launch + - image: dockerwatch + name: dockerwatch + imagePullPolicy: Never ## Set to Never as we built the image in the cluster + ports: + - containerPort: 8080 + protocol: TCP + - containerPort: 8443 + protocol: TCP + - containerPort: 9000 ## Expose the prometheus port + protocol: TCP + volumeMounts: + - name: kube-keypair + readOnly: true + mountPath: /etc/ssl/certs + volumes: + - name: kube-keypair + secret: + secretName: dockerwatch \ No newline at end of file diff --git a/advanced_examples/minikube-prom-graf/dockerwatch/config/sys.config b/advanced_examples/minikube-prom-graf/dockerwatch/config/sys.config new file mode 100644 index 0000000..674fbc9 --- /dev/null +++ b/advanced_examples/minikube-prom-graf/dockerwatch/config/sys.config @@ -0,0 +1,5 @@ +[%% Kernel/logger + {kernel, [{logger,[{handler,default,logger_std_h,#{}}]} + %%,{logger_level,info} + ]} +]. diff --git a/advanced_examples/minikube-prom-graf/dockerwatch/config/vm.args b/advanced_examples/minikube-prom-graf/dockerwatch/config/vm.args new file mode 100644 index 0000000..3c1f4b6 --- /dev/null +++ b/advanced_examples/minikube-prom-graf/dockerwatch/config/vm.args @@ -0,0 +1,2 @@ +-sname dockerwatch + diff --git a/advanced_examples/minikube-prom-graf/dockerwatch/rebar.config b/advanced_examples/minikube-prom-graf/dockerwatch/rebar.config new file mode 100644 index 0000000..a3ff73c --- /dev/null +++ b/advanced_examples/minikube-prom-graf/dockerwatch/rebar.config @@ -0,0 +1,20 @@ + +{deps, [{jsone, "1.4.7"}, %% JSON Encode/Decode + {cowboy, "2.5.0"}, %% HTTP Server + {prometheus,"4.2.0"}, + {prometheus_process_collector,"1.4.0"}, + {prometheus_cowboy,"0.1.4"}]}. + +{relx, [{release, {"dockerwatch", "1.0.0"}, [dockerwatch]}, + {vm_args, "config/vm.args"}, + {sys_config, "config/sys.config"}, + {dev_mode, true}, + {include_erts, false}, + {extended_start_script, true} + ]}. + +{profiles, [{prod, [{relx, [{dev_mode, false}, + {include_erts, true}, + {include_src, false}]}]} + ]}. +%% vim: ft=erlang diff --git a/advanced_examples/minikube-prom-graf/dockerwatch/src/dockerwatch.app.src b/advanced_examples/minikube-prom-graf/dockerwatch/src/dockerwatch.app.src new file mode 100644 index 0000000..de620bd --- /dev/null +++ b/advanced_examples/minikube-prom-graf/dockerwatch/src/dockerwatch.app.src @@ -0,0 +1,19 @@ +%% Feel free to use, reuse and abuse the code in this file. + +{application, dockerwatch, [ + {description, "Cowboy REST Hello World example."}, + {vsn, "1.0.0"}, + {modules, []}, + {registered, [dockerwatch_sup]}, + {applications, [ + kernel, + stdlib, + jsone, + cowboy, + prometheus, + prometheus_process_collector, + prometheus_cowboy + ]}, + {mod, {dockerwatch_app, []}}, + {env, []} +]}. diff --git a/advanced_examples/minikube-prom-graf/dockerwatch/src/dockerwatch.erl b/advanced_examples/minikube-prom-graf/dockerwatch/src/dockerwatch.erl new file mode 100644 index 0000000..3e5b400 --- /dev/null +++ b/advanced_examples/minikube-prom-graf/dockerwatch/src/dockerwatch.erl @@ -0,0 +1,45 @@ +%% +%% Copyright (C) 2014 Björn-Egil Dahlberg +%% +%% File: dockerwatch.erl +%% Author: Björn-Egil Dahlberg +%% Created: 2014-09-10 +%% + +-module(dockerwatch). + +-export([start_link/0, all/0, create/1, get/1, increment/2, decrement/2]). + +-type counter() :: binary(). + +-spec start_link() -> {ok, pid()}. +start_link() -> + {ok, spawn_link(fun() -> ets:new(?MODULE, [named_table, public]), + receive after infinity -> ok end end)}. + +-spec all() -> [counter()]. +all() -> + ets:select(?MODULE, [{{'$1','_'},[],['$1']}]). + +-spec create(counter()) -> ok | already_exists. +create(CounterName) -> + case ets:insert_new(?MODULE, {CounterName, 0}) of + true -> + ok; + false -> + already_exists + end. + +-spec get(counter()) -> integer(). +get(CounterName) -> + ets:lookup_element(?MODULE, CounterName, 2). + +-spec increment(counter(), integer()) -> ok. +increment(CounterName, Howmuch) -> + _ = ets:update_counter(?MODULE, CounterName, [{2, Howmuch}]), + ok. + +-spec decrement(counter(), integer()) -> ok. +decrement(CounterName, Howmuch) -> + _ = ets:update_counter(?MODULE, CounterName, [{2, -1 * Howmuch}]), + ok. diff --git a/advanced_examples/minikube-prom-graf/dockerwatch/src/dockerwatch_app.erl b/advanced_examples/minikube-prom-graf/dockerwatch/src/dockerwatch_app.erl new file mode 100644 index 0000000..a85b5b6 --- /dev/null +++ b/advanced_examples/minikube-prom-graf/dockerwatch/src/dockerwatch_app.erl @@ -0,0 +1,19 @@ +%% +%% Copyright (C) 2014 Björn-Egil Dahlberg +%% +%% File: dockerwatch_app.erl +%% Author: Björn-Egil Dahlberg +%% Created: 2014-09-10 +%% + +-module(dockerwatch_app). +-behaviour(application). + +-export([start/2,stop/1]). +%% API. + +start(_Type, _Args) -> + dockerwatch_sup:start_link(). + +stop(_State) -> + ok. diff --git a/advanced_examples/minikube-prom-graf/dockerwatch/src/dockerwatch_handler.erl b/advanced_examples/minikube-prom-graf/dockerwatch/src/dockerwatch_handler.erl new file mode 100644 index 0000000..1ac15f9 --- /dev/null +++ b/advanced_examples/minikube-prom-graf/dockerwatch/src/dockerwatch_handler.erl @@ -0,0 +1,103 @@ +%% +%% Copyright (C) 2014 Björn-Egil Dahlberg +%% +%% File: dockerwatch_handler.erl +%% Author: Björn-Egil Dahlberg +%% Created: 2014-09-10 +%% + +-module(dockerwatch_handler). + +-export([init/2]). +-export([allowed_methods/2]). +-export([content_types_accepted/2]). +-export([content_types_provided/2]). +-export([handle_post/2]). +-export([to_html/2]). +-export([to_json/2]). +-export([to_text/2]). + +init(Req, []) -> + {cowboy_rest, Req, []}. + +%% Which HTTP methods are allowed +allowed_methods(Req, State) -> + {[<<"GET">>, <<"POST">>], Req, State}. + +%% Which content types are accepted by POST/PUT requests +content_types_accepted(Req, State) -> + {[{{<<"application">>, <<"json">>, []}, handle_post}], + Req, State}. + +%% Handle the POST/PUT request +handle_post(Req, State) -> + case cowboy_req:binding(counter_name, Req) of + undefined -> + {false, Req, State}; + Name -> + case cowboy_req:has_body(Req) of + true -> + {ok, Body, Req3} = cowboy_req:read_body(Req), + Json = jsone:decode(Body), + ActionBin = maps:get(<<"action">>, Json, <<"increment">>), + Value = maps:get(<<"value">>, Json, 1), + Action = list_to_atom(binary_to_list(ActionBin)), + ok = dockerwatch:Action(Name, Value), + {true, Req3, State}; + false -> + ok = dockerwatch:create(Name), + {true, Req, State} + end + end. + +%% Which content types we handle for GET/HEAD requests +content_types_provided(Req, State) -> + {[{<<"text/html">>, to_html}, + {<<"application/json">>, to_json}, + {<<"text/plain">>, to_text} + ], Req, State}. + + +%% Return counters/counter as json +to_json(Req, State) -> + Resp = case cowboy_req:binding(counter_name, Req) of + undefined -> + dockerwatch:all(); + Counter -> + #{ Counter => dockerwatch:get(Counter) } + end, + {jsone:encode(Resp), Req, State}. + +%% Return counters/counter as plain text +to_text(Req, State) -> + Resp = case cowboy_req:binding(counter_name, Req) of + undefined -> + [io_lib:format("~s~n",[Counter]) || Counter <- dockerwatch:all()]; + Counter -> + io_lib:format("~p",[dockerwatch:get(Counter)]) + end, + {Resp, Req, State}. + +%% Return counters/counter as html +to_html(Req, State) -> + Body = case cowboy_req:binding(counter_name, Req) of + undefined -> + Counters = dockerwatch:all(), + ["\n"]; + Counter -> + Value = dockerwatch:get(Counter), + io_lib:format("~s = ~p",[Counter, Value]) + end, + {[html_head(),Body,html_tail()], Req, State}. + +html_head() -> + <<" + + + dockerwatch + ">>. +html_tail() -> + <<" + ">>. diff --git a/advanced_examples/minikube-prom-graf/dockerwatch/src/dockerwatch_sup.erl b/advanced_examples/minikube-prom-graf/dockerwatch/src/dockerwatch_sup.erl new file mode 100644 index 0000000..bd1bd9d --- /dev/null +++ b/advanced_examples/minikube-prom-graf/dockerwatch/src/dockerwatch_sup.erl @@ -0,0 +1,66 @@ +%% +%% Copyright (C) 2014 Björn-Egil Dahlberg +%% +%% File: dockerwatch_sup.erl +%% Author: Björn-Egil Dahlberg +%% Created: 2014-09-10 +%% + +-module(dockerwatch_sup). +-behaviour(supervisor). + +-export([start_link/0,init/1]). + +%% API. + +-spec start_link() -> {ok, pid()}. +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +%% supervisor. + +init([]) -> + CertsDir = "/etc/ssl/certs/", + + Dispatch = cowboy_router:compile( + [ + {'_', [{"/[:counter_name]", dockerwatch_handler, []}]} + ]), + + CowConfig = #{ env => #{ dispatch => Dispatch }, + metrics_callback => fun prometheus_cowboy2_instrumenter:observe/1, + stream_handlers => [cowboy_metrics_h, cowboy_stream_h] }, + + HTTPS = ranch:child_spec( + cowboy_https, 100, ranch_ssl, + [{port, 8443}, + {cacertfile, filename:join(CertsDir, "dockerwatch-ca.pem")}, + {certfile, filename:join(CertsDir, "dockerwatch-server.pem")}, + {keyfile, filename:join(CertsDir, "dockerwatch-server.key")}], + cowboy_tls, + CowConfig), + + HTTP = ranch:child_spec( + cowboy_http, 100, ranch_tcp, + [{port, 8080}], + cowboy_clear, + CowConfig), + + PromConfig = + #{ env => #{ dispatch => + cowboy_router:compile( + [{'_', [{"/metrics/[:registry]", prometheus_cowboy2_handler, []}]}]) } + }, + + Prometheus = ranch:child_spec( + cowboy_prometheus, 100, ranch_tcp, + [{port, 9000}], + cowboy_clear, + PromConfig), + + Counter = {dockerwatch, {dockerwatch, start_link, []}, + permanent, 5000, worker, [dockerwatch]}, + + Procs = [Counter, HTTP, HTTPS, Prometheus], + + {ok, {{one_for_one, 10, 10}, Procs}}. diff --git a/advanced_examples/minikube-prom-graf/grafana-deployment.yaml b/advanced_examples/minikube-prom-graf/grafana-deployment.yaml new file mode 100644 index 0000000..536b02f --- /dev/null +++ b/advanced_examples/minikube-prom-graf/grafana-deployment.yaml @@ -0,0 +1,46 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + labels: + name: grafana + name: grafana + namespace: monitoring +spec: + replicas: 1 + selector: + matchLabels: + name: grafana + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + creationTimestamp: null + labels: + name: grafana + spec: + containers: + - image: grafana/grafana + imagePullPolicy: Always + name: grafana + ports: + - containerPort: 3000 + protocol: TCP + resources: + limits: + cpu: 500m + memory: 2500Mi + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - mountPath: /var/lib/grafana + name: data + restartPolicy: Always + securityContext: {} + terminationGracePeriodSeconds: 30 + volumes: + - emptyDir: {} + name: data diff --git a/advanced_examples/minikube-prom-graf/grafana-screenshot.png b/advanced_examples/minikube-prom-graf/grafana-screenshot.png new file mode 100644 index 0000000000000000000000000000000000000000..0808d478ef735cd6c2d3f1c49fc2e162b3501a7c GIT binary patch literal 118295 zcmb5W1zePC*FK7YNP~1sqoj0?=m63J(j`cDcL+$A3@xR!z|dWz4Bg!!-Q5ff=keX! zz2EPg-}#^OJ;d;P>aKOKxYo4@d#fmojX{oqf`WoA^9HPff`aCag7Q%O@qOS-WAZdL z@PTgsM#~8W1(V?S&pnhcX{5kOG-nwFNwj55Y&=5x0PLFz6qKhZGGK{!?lZgd9=Zf( zNVLO)K8Co5t+bB=9BGz=FvfHaTWgl*UHv9%&MWpNT-|2(YbI=M+-j?JikjzaTpFQb zRXTfnP&Go-ccRMoINv>7V_K5^D9@d;NE7&i^7(tbp+Q8uTwIFTy2pAq55eVnw_CCT z1wYWMKgUOUF1_+$q`yBFyaEf(g$3sZzQJu=SgLohJ%c}Yp`g?yOqEQT%Rt^h7-S%! z+cf+-kN&-A3oRjYuIa%ylfe<1g}(es_RpoS^v@cMvKe*yQW%U#Aq>H9=vK%6xrnm) zUXQOiSZ_*;PH5S>mbjGlpGz-8Hor)i2}L>ijFD;a{c{>cID&)BIggdr%p;FydEnpk z>0lCtQVC;3Y|>CdU8UYBnFs&B7j4TqQb+53^C~G_YuYpLP5$lq<{N7b8+V^3t*q;& zfir(IaQeM}9uxkY#mcX3`H2C-@Ce`RpVJ?sCpdW0%|hz@C^ilXtzi~puJviTQW5{$ zfbtbC=$m)4i+8a`?}-ggBvE*xc7&pzN$*>(u{Qh6OK+OuvSIzzh1T=O5S0v9gt|G;0Fw+>EQW ztrT3;cgBTu2FT44O>7m$J7%;I9(uLK+tJWwRkeA?+T(GDmsaU%n!)WZj~XzBEum0o zg1wkC9@?O@ug%w5_GNi+)s8g@@Z(f^+7 zAtuX&8EgJoX)lq7y%ppMGnzW|AeRxrl>Szz1yOIs0J%cQyZQ2Jw~RuaePWofz_hkm zRGil4WgO;Lwd{HH$=970UPIGr-H*&YguHhPjWhRFu5@XX5TTX3)&2|Mch5q>45d_K z3b*#hj94arP1B|(WhZ*T&{BVyHuSs_DwIfMtCpPops3{oZjW=+%Xmr8nGSzW!s?QeV`UP|?>AAXLA;EojWWmpTEa*Y^4P{5SE+*Ygu; zrA?z9tep;^yUW16@9jpzTh3@Wl)4BwE+5|+)!DvC#P#{!tn+N`g1$5F?Y2m5L`1I! zE3D4_Xu0>qz~+X$xw61+u_a8z&G&fgI1iD+yS1d4+s>r~{oiJ^f%>SE(b6od3eSL>>>zZNnHVgwuRS}v?s8y*XT zrAdvABE1z-9~2}cBuq@u(DY{G_2K3VDsTCOh9rHzXxQj`Hx( z4UUy|mlP8ggHJP6*HmBW=6y0h`0=%pjd^1^x(!UHQl#c{eVW2=WBJCst5sm3im0Hd zNT>RGwnNNs3vxSJH>DZ$3ZIymm=y1GYSLs?;&8SU^yauIkH&eC3WPuT+YkbWw8w{~ z9eOD&=B&ubNDY#kvk8H(8l@Tg`}@??)Sc1k1tlf6lMVOp-+cHWZDwjZJ~5%+;G7Z~ z5E!VZr{|urAvges2ZmmshW!%c;2S@}Ka8ahX>4pf5iM2u9{Pq*vbiNKR8+KK>uil) zI{J;md}@;0_gkFB>FF?&>21r`nwoFG;D`8YYimSAM8FaxB__IB^J^xWNDgOB?&xS{ zYkonHnu)+>H1$H$q(p6*E{* zl$n6SMZ_+LSyllj!LOg}(I=j^WeE5z2DdX4ohL{AuQqzxsudin=pe};of_L}F`uOv zKS}{xYfa6mdZ&$5M$^jamY0YK zfNbL66hfhDEQEC&f2~0PssM}tIx$@?V!fyM2%9`UB_(&<5)GTY&h4PZ6)McjyV@2O zNx;lah|ykV_!{2&N+{Y&n@vHprN9E`t%8hoOh_d-{xXp ztWI9S%s248mTT*$qh`-D2D+kgZBonCI!6H_7qwRm>m|?AFqZ)D*bK*S8rE zkM`qZ3<<9K`g$P|k&EqdtL?FF2*dYx&lAF7SF3W+@kFJn`?`ew85qGytC{^I*ymQT z)NPTn{P3k*zK!RZvwD{^NC@!P8pMB|DAjk8wL(M*>@l9mo>cfi7w>|BPS|6c0srjq z-1dse-{1cmEv<+v15`ax<17tL7_AjNP&B4H6 ze3FNKtn}thmy2w0Lj_HYBho$oq#4qD5DfcRe3`9ls!glstM{ywN%NIby zq@|@bc$^T&brTX2PSx7cIKM-npW(s2Sj?RmpCvz)n>{}~=;kXrfMpYW!=e^_6D^o2 z>~h_BFj1n1fETI({lC_6ajAawjg*H;y+v79gbZj!XxaPR`h2DuRIF3$yECykQGA80 zU3BeQvbz#_NOc_KU(@FOV2-H9VjzL(Md|f+9pzprHC1s5ZwhlMdlw6lb#D?4tHFzI zkrAD>5@V#mD6{MMnrxhlYBcQ+Jwup>oQ#I=Zf1?ZRvzXQiPe;;AIOjHIR}SDS=`RP z{Y{<2_#^nY*CBFpy-%3tDX55CHirxm_hoaJYxhI|8(IGA>({ijh~rn)+Ef%G*V`Kl zmyq8+J_^dqUO^EN0Yu$1Xm}XT+b6?EvY6GPqpu_GUOG9BTp|#S4Gj(T^$m4(=G8O6 zPGTdD`1!u8yI$}^M^#~A;r8}+PY>`}I5(#&!6i~?nQA-VFpcKQPAsRFE<-38B?Ui` zr5o8f)2T&HWX(MmI7<3%qh4w=QlgkY+v3v+?5OpP4IMV(PIaHdU*9sseD(+=rg{N8 zGdO5xk=AT=E4CmGjf$lF;V>6QKxQ^*+MB6=x;@Ur%bQbI2hJbeTS)gF6e9pMo2b)z z@whfk@bSe(CYPbE(Q-*|5@+obZEd}9*(RJ-ZPtew!t)Xj z)S>E1>B4JWvGgPyt5ue#IFzJXZ{NOEQQ0ow@UZ~T2R>j215rUqq7Y>87V)ug%)S3v zz8{&%u_FZ8BA22yw>}smEcqy2Tz^^MLeEEGF!6WHCmI~<}#a4dWHYN(9R zCZguskDMYlhhpiXO=?zD6D5welBUwLe4z6ii0Z%}l6cinb1-z6$rb!Ks zO&0j`asag0-_`Y=GoR0doUaL!wnMwcCRHMMcbQ{7o4HB6aaCzT^RcWpMi8Y75~i|ZvJ zR7*>%=IYonUfP!muu?9<>G}C)gBdMXSE$h4@O3~-V^fQ=fM8#Tv6d-|A!BW-f$3_2tG{T&XqXqQoWd}5-|K7*r1jbHTFg~cM-SRT_`nUcGVd= zBC$VFC8s+gS$)eGMJq$s@r{0zw$r8gc~d$Rd^$Lf5%>d7x%&H_g5|i$H^3Xg2x>Jd z&&j7*5e&KwW{R^vUj1HB@blh1WC$*eMa<0WR{>gy(4ry{@5?vY3=JPTqm_$FeML>H z!v*G2;enr^n8G05H;7EKV?_8_4`8!ce#Fr)T_pN{UA8D&=K_}<{Ijb``kQ*-;dfSunJg-6J_?!HOmGHhZAM@0=fFGmkH1hg0+ZCP+33a$y8LyA9}9zJ!eEPlXJN+;J*R4&o=!2DK0ZFS>TgRr$#wVi zoZ(KFD9*7y67&NR+8Ug~J-KnF6X1Gwai0f{hZ0`1KZOpi3e_x-l$8o$i z6mZCkq0pk@;s(UNd(bOoWo0r_(ziM~wLr)KfsAOj>6Hi~0{h)6Uo5)+QIKOwQy?c93mgB11QePsDzO5TSSUJ-wE`uwF z#GysoN*(KD7P9CxeQ;Lt?Iu~YjR@yl%fE*DmUPd7O=Hn=Z_WQmn$?1Pg^jGvg2HmCEw!PXdM+_73`+0x1vy6|A zkH+_={FdqXP4(@9sjfw65-MK`gx+ zBO@vrYEVdsAF`bUJhmB~721pWw4$a#q$dmBSMcSp)q! zew@^eFJ@-8(d>1BOD$?YRrU=Cf4t5aF7{@bnV4qIsUFuSGsw(0y1~4KK#YtomxqQg zUc3O}<2Z(HV&Zs)us490vomRzEbgN%C1t8UAwx&bXPFbD6yt+9x8nU`2oLU=cI=3z zQq|WlH|ebz8u}bdFCCw-)zfQ7`O#5@n~n9VQO51;$>0ttU7nI#ub11T?ch2xJ&ERr9_)p~k*T~w-3iI{frPsOt|B$H{E8E#qDkR~DzIx|48 zzSh3p!CCu+x6^p!f?K?{`q(&gf0-mC0qtb^r}%EcOU%wU)Z|uRx1HmvNB3mxn2Xem zd{gvxwv78()NgpS)ZX^%75&6v2&cn;me4e9y-;NlaVuF+GGN@;_+BI(OdPSME%OPi zT>Kqh*SKosNEm7IHX3y}wlK5J{cU8TE zWup(N>0P2u9g%=#Vb0Id&HwR*&kEQpbH@$Odsuf*>eIwsJdYah(u7U6x9won*wI#u z-VQD<&8$pP-`?_m!4NZ?6GVKieDxDwS<2MZ*B99qcGL)gFe#>)bEmi-$h0gqJ{>%X z5kC&y%Led`B;Uh(3;2J9G}aj&o+UTG8u_etNXpgK$8Q$zA_Ht@6%tB<43hzc37CgC zrr!~IccN`TsJFMb=LL>It!?1X_vSD`YeU0Wmhh-2D^Fq>WF9;^b#rrDgn^*9!Uj-v z#Pv15m|%n?8w*RF!}AS(A>W%%At50!7!Cf_)!B$SuOm+1=g;qjd@Z?CFfcJsj*o@y zHIGF>_V0}`%{&_G>s{WfMoB%u2V4wpEP5Fg&+Bs4{DH1{bJmspeMPW98q`2ty#@f_ zMpUTaTuGb;KyZ}DDq#~804RUmTH6Pm4!}nA?Y6YCqVQo+(bi6ki*rKWpa#8q;kX() zHKqL9Z7)a+M8i$#H)Uq7PD+|wf=l=I_L2q1Ru*ZfsZA%1?VNQ+Q<;%=3=VR#6kK^} z%R-EKC-7ps-h}u~WVW1LnTpjzassOUG zm`Ie~@!B;;KarwS0;_ta_R}?2N}D7K4$IzsF~Zq06*DNQfV}78Nhem)i)o!=SO$aOUK`%L3l9h8yei*-8Y>Zq%^jE{6O*W zt{IgfBqiNk8GY?*U9AlS7&do?==dIjOR$dmb!+m# zmPoe(Dc>6l*|U?Cs)K%KzjG0v(XhVqZ-qr(?+YsHFEScSt1UYsx4NG_o9ux~fO+zi z?9_(pW78r?!V27vA{lv)B3v>L_lD`s^0W$%ZE;a7!dW7 zC)`XNt_LkXQ5^=A9J+OfwSG-!F|g)mem_^?G2*t91|qIf*PlidnE;-K&N~^mSYQhakCAcE8G<9^W3>MyV*;6t1N7b8Dzl#R;jEUsMGQ~ zdw5D2O7yT&zx3kp@G%xvSARcG%0rSRL7<=Xg!jvg(N zN(2b~D%n`W>EK-XN}0pvyu-u80FJ@-?3ox44(aJp!OJvUN59%6ap?8*_a`W&jOGu5 zC9&0?Cq#)izhxoJVwqWIoz*v=eW3GuKfGRmyff&lS!i)8Sq(8H4R&yy;Kt%{Kl}=y z#61zKSi@mqVQ%w8JTBYD5_pQ4;`GcMFyB&wVH*Bp^)T^d+Z7yorfq4)F2jAcAv4#nKi58i=B$!a zngWX5O-9R-Hwve$?t5;fiM)P-qx0vmP^6gFX99w9wo&-BrwWdCMRC(fRj)TxD-^Tp zvR(@aS+5@D`Y7#ouf;hM9{YcC9~0+kTkeyav@L^kV49ei04U>pLw#pQ^h}G`u(%2d z{;*EV>~;!|+3c3*RKC(D_DyT8v|BULf`> zZ^ha-unr3{{miWScr6_**6HxuL$8?IQV*zN5pCA5{ctmNjV{I&I`>)m4Mvy~Re4>! zaNN$EX%)G+a(rTgAQg(qVdIAU7vxhEvpK`!sTEv@?dL_PDo{JYlh( zbt^W6#&>NE1^^a&vu;&oGSk!3eh36;LG$t&h*rH1#B;L$@@eX*q3VshD>2;M+;`X4 zuYwaa6J(;>s;qM78(fMC3O+N0ZCu@ZaGfgO3Jev9irbAGoSeqDPWO}nNIrIy3W%)S zTwH9lKdwx<_5&LdBWM|L${j~^yjD76Y+$>@Y`y3ERQt=zDl}+A<^a}P#yOO))OL09 z8(gH~d%R|MUF%3-g_gmWlA`6{Q2sgu2LSqOQVsRVBmvyhJkew@6J!UvsMm^(eemk_ zYrrKhoZOL)Bn6@nDR4$c2B7zWNN#Iko}Pol_VV!CY>f?Ug8C6oBp~$woPifBdNNZk z^iy)U=VX#1)kbj1FoF%k3(4p_Kja?=w6r8)nU>_s1tz^LtLP?~F8@g^P<_@* z9~BsiGLUiErTLTz{<>fVja^ox#z6&0X15fJESw}R8z zJM>e-q?^p01K9T}q8)^3H=LeVV!j?D`z#@IJE_qkIYjz<%)FnfFo}SERQVo{IrTfXf@}l^25LZ&HN(#Xf7;))p-V<%#+M=%Z;yWsD@Bq-QNdXbTpGhWr z2m0o)Iz85E$tne=IN4kXO+*$;INjh07Ti89>j7GZVt%9Z79=Jn=Jr+4D?`ln@GSF8 zeE=^tH8pj8AIPcO1!M@Y9zRazwNUZ&^i&*>6neO{vqL~dwYRlp$V-k(DLCf*8W$H= zN4~DG5CADE*k+p>%`Wyc?%nf0J_-&F{uCUXleZ6~U8}WsK6C{<#Z!L!_UqH+U3zCg z;~zumZmXC7Z~>sj>Q-RzL2K))&PT5SaL3yR0bt?wloXVd44k~F?CM3;W=e~ILj?rC zO-)UQhh>F@&*Nk^=S6l$a(e+d1pwrG`}_NQgrE6|zOR;L7Zo85zFKrxdvM#Y1RlR- z@AgCxw*LjY6*Z)>P6##zf@tsXy%nE=wJjKB6^=~mY0nH zFF;LGlO=otz`qfvVTZ@ZbXfo_NlZlKww*F?#w-Ip9gu>HP+*ti_vGT%V__B0e0Az5 zl%c)#T^p|*%^O|?&OSUd-urME)A8Mof0M1-?N?KUM&7Uujt;DrVq$!~pMn@jlK8G~ z<%j)+BmsGen_FDa40EqW(YWJ1FB+dFHa6b)P#{8DtZD!v6KQG5c_IR*Uq60IDO}qi z&g<%?_C#ED=VdLGQzbF!{>qHoi!>Ux1SE>qh{eIqGW7~P&p`_{j5e34RYCUxfYIFE z-sTRy0G;WAajL4G7NCL ztF?WA=Mqh%qR6`|tSv0Gw6)3MT!1-D;@pr1pYoGN0w`bg-YS54K#Zfp!VI z_&-n>ykMQ?A#EHUXPFGSZGCiSYHaMXv;M+*YA8HB90GX)