From dd858eaf070a1cfc9ae9db593dbbe52cf72a32e5 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 9 Nov 2022 15:16:13 +0200 Subject: [PATCH] chore: update otel example --- example/otel/README.md | 16 ++++ example/otel/config/alertmanager.yml | 53 +++++++++++ example/otel/{ => config}/otel-collector.yaml | 0 example/otel/{ => config}/vector.toml | 0 example/otel/docker-compose.yml | 27 ++++-- example/otel/image/metrics.png | Bin 0 -> 32347 bytes example/otel/image/redis-trace.png | Bin 0 -> 21536 bytes example/otel/uptrace.yml | 90 ++++++++++++++---- 8 files changed, 164 insertions(+), 22 deletions(-) create mode 100644 example/otel/config/alertmanager.yml rename example/otel/{ => config}/otel-collector.yaml (100%) rename example/otel/{ => config}/vector.toml (100%) create mode 100644 example/otel/image/metrics.png create mode 100644 example/otel/image/redis-trace.png diff --git a/example/otel/README.md b/example/otel/README.md index f2883207..5d1431e0 100644 --- a/example/otel/README.md +++ b/example/otel/README.md @@ -40,9 +40,25 @@ UPTRACE_DSN=http://project2_secret_token@localhost:14317/2 go run client.go trace: http://localhost:14318/traces/ee029d8782242c8ed38b16d961093b35 ``` +![Redis trace](./image/redis-trace.png) + You can also open Uptrace UI at [http://localhost:14318](http://localhost:14318) to view available spans, logs, and metrics. +## Redis monitoring + +You can also [monitor Redis performance](https://uptrace.dev/opentelemetry/redis-monitoring.html) +metrics By installing OpenTelemetry Collector. + +[OpenTelemetry Collector](https://uptrace.dev/opentelemetry/collector.html) is an agent that pulls +telemetry data from systems you want to monitor and sends it to APM tools using the OpenTelemetry +protocol (OTLP). + +When telemetry data reaches Uptrace, it automatically generates a Redis dashboard from a pre-defined +template. + +![Redis dashboard](./image/metrics.png) + ## Links - [Uptrace open-source APM](https://uptrace.dev/get/open-source-apm.html) diff --git a/example/otel/config/alertmanager.yml b/example/otel/config/alertmanager.yml new file mode 100644 index 00000000..9ea790db --- /dev/null +++ b/example/otel/config/alertmanager.yml @@ -0,0 +1,53 @@ +# See https://prometheus.io/docs/alerting/latest/configuration/ for details. + +global: + # The smarthost and SMTP sender used for mail notifications. + smtp_smarthost: 'mailhog:1025' + smtp_from: 'alertmanager@example.com' + smtp_require_tls: false + +receivers: + - name: 'team-X' + email_configs: + - to: 'some-receiver@example.com' + send_resolved: true + +# The root route on which each incoming alert enters. +route: + # The labels by which incoming alerts are grouped together. For example, + # multiple alerts coming in for cluster=A and alertname=LatencyHigh would + # be batched into a single group. + group_by: ['alertname', 'cluster', 'service'] + + # When a new group of alerts is created by an incoming alert, wait at + # least 'group_wait' to send the initial notification. + # This way ensures that you get multiple alerts for the same group that start + # firing shortly after another are batched together on the first + # notification. + group_wait: 30s + + # When the first notification was sent, wait 'group_interval' to send a batch + # of new alerts that started firing for that group. + group_interval: 5m + + # If an alert has successfully been sent, wait 'repeat_interval' to + # resend them. + repeat_interval: 3h + + # A default receiver + receiver: team-X + + # All the above attributes are inherited by all child routes and can + # overwritten on each. + + # The child route trees. + routes: + # This route matches error alerts created from spans or logs. + - matchers: + - alert_kind="error" + group_interval: 24h + receiver: team-X + +# The directory from which notification templates are read. +templates: + - '/etc/alertmanager/template/*.tmpl' diff --git a/example/otel/otel-collector.yaml b/example/otel/config/otel-collector.yaml similarity index 100% rename from example/otel/otel-collector.yaml rename to example/otel/config/otel-collector.yaml diff --git a/example/otel/vector.toml b/example/otel/config/vector.toml similarity index 100% rename from example/otel/vector.toml rename to example/otel/config/vector.toml diff --git a/example/otel/docker-compose.yml b/example/otel/docker-compose.yml index d4c5a695..786f6e1a 100644 --- a/example/otel/docker-compose.yml +++ b/example/otel/docker-compose.yml @@ -18,7 +18,7 @@ services: - '9000:9000' uptrace: - image: 'uptrace/uptrace:1.1.0' + image: 'uptrace/uptrace:1.2.0' #image: 'uptrace/uptrace-dev:latest' restart: on-failure volumes: @@ -36,11 +36,8 @@ services: otel-collector: image: otel/opentelemetry-collector-contrib:0.58.0 restart: on-failure - user: '0:0' # required for logs volumes: - - ./otel-collector.yaml:/etc/otelcol-contrib/config.yaml - - /var/lib/docker/containers:/var/lib/docker/containers:ro - - /var/log:/var/log:ro + - ./config/otel-collector.yaml:/etc/otelcol-contrib/config.yaml ports: - '4317:4317' - '4318:4318' @@ -48,7 +45,25 @@ services: vector: image: timberio/vector:0.24.X-alpine volumes: - - ./vector.toml:/etc/vector/vector.toml:ro + - ./config/vector.toml:/etc/vector/vector.toml:ro + + alertmanager: + image: prom/alertmanager:v0.24.0 + restart: on-failure + volumes: + - ./config/alertmanager.yml:/etc/alertmanager/config.yml + - alertmanager_data:/alertmanager + ports: + - 9093:9093 + command: + - '--config.file=/etc/alertmanager/config.yml' + - '--storage.path=/alertmanager' + + mailhog: + image: mailhog/mailhog:v1.0.1 + restart: on-failure + ports: + - '8025:8025' redis-server: image: redis diff --git a/example/otel/image/metrics.png b/example/otel/image/metrics.png new file mode 100644 index 0000000000000000000000000000000000000000..7c2beb4d6444a954184d6847bebd586339683f3d GIT binary patch literal 32347 zcmb@t2Ut^0(=d!(LB)cA(ky@?1O%lwMFr_nq=h2A<WJZEI`m=;&y6 zapz;~aC2wJN8ez0a%L}%utZqgA8?KB_KH4l`+|(@HktH`XR0pPrNjWcJfUk(kzec{ zK=zL&LMxGE<4IGP3a8nGc=(lYtq(uHM+DEsog(v~^_3-i2mbY#QIP$9{teKJ=F4ij z)4V$Nnyn``>{Ye%BykQY*p#e#)&&8g885f%+CZQUZ64*?5GIpccee4WD>K#A-IQ;f z_zDh*c5L*{S3Ev}b|_sTr$=dk*DKugk?cLDWQW?X2qdMke54xiyyDv7x)0eeLDyiN zZjYGzg=Uro7M4#Iw7b+Q`Fmrh#$wjRdEK-c)quIS2!eh(B3`ect?SZRuA9_*%Rs29 zG`=hQ%Is&LG!Nz6lLtbKThv_W*h(HSAIg(l{inNg?H0~>tQ9X>TmOah3V?N@9sL-4yyv*S^1eaI4737O-3%=k6f0=d-Cds z%N%)5hVtSh6We2Tnly7Q;vtqeP^0s;CflPRW>T zH2JP=Bw$a-Q2SyZl6l7xL9Kju$jdSmk=aMBy8^V6dNyV$ObN8ftb47a#mPDh)pG6Y zz$Bp(M6i@+$32(szMq=jsw+K40^rK#0pmmP!5Z)B;cj$CptmT zB-aRlrHU@jA7AM9V0a;S!Q2TB%sdSxj6C{)XWNFs-6iF#d<^eVXm~wtIpZQ-5Mp0z zDE(D_#{vxTE6ynG1uN5-*$e&n_z{x%BYbEhzJ5YHV4rM>?GIdW?=2&&@bDh&*yAdC zMWo7K%d|OWd0=SKgYY(O4G`JZPh0Wi)tK0*NyiGO<#e>=l~}!bY|k3Rj6akEr-Db_pI#=rxC|)P+7XE9MpzGALulMRsfY?60|cb zDM<=+6sjNpL;*2j3=7Nx`V}g$tf5U4 zoye+?JS=)Erknke+mvrRcXLztMCZdHjSpP_U|C}nJS)@0I$F|E5e#=zEp;GFDaWu*PT2{XxM>Ox+in9?l**>Abr zXZ$TG?Cl9aoFHSvMh2Yt^L(c!`}6#t0VF)&#E%)Q*NJ++_u!wsh4lMx+spq8P&l|< z&T*Zr`f8_LHG*KLc9QH6!V=4gDiakXC$nxCm-~n$OP)oH%o!A1u6O5QC^YLVqyFJ} zzg)d6@jLa8sj?X?SvBFoP2j};#_JDf&6C)Z9vVVhavfEd0OG8rW05zTaYL0 zap#R+j4KnlIZ_!oGFirzcQz~AY>wW#M=@NIG3<*ie};pIHRqis`%X`Fzn?b5 zk{658Jc*^z;mz&z_s~kdx%GD8`9c77gd1YHpVD$NM|(h{vm-!Gxo&Q68i)g{uDL}l zR6&ek77(bjp3%FV(rp*}Py6MdtgXrCkaUZ5!zn7~kJCI)kA(Kt=eR3NuD--Ep9Cw5 z_cf#{QXiNNY^Clf%an5=7Ftzx^0-*OTKNwv@fj?f_~PyIP9;$vV) zZfK^nWlwN`1(4-Z#hK?3iZ6c|Zm=vLA9@3g>n)g5MM4&I;V}pOW(-Q!YI!Hojmah3 zrdY-*HoesSJB9K7~P6E9a~z|oY0gQLLcQ{e$)4EOzT%KDkX4!xRt+3EW$TzZ; zMmrQz-0QyU{!#kQQ9}5#fG(RuCGjv=K%y)}%|6%A_<#jJh@v~~f;WOMmu*veME{t_ za>OaR{%AsmIBOBll||DC)MV*_?R?9_iv% zI0a6n5bDs|L?)^eDG=;QY@=_xf~Wo3WJEY|eO%Fn;z$E(X+1s|bg37~tTC5-wQXdE zb_EZ$$S!AQ3bJF{D;xTL*_yCypgrXD4Ek#ryXRLHvY-z{dP|nQE<-Luv2JvpN@Zqv zCsRx{1Dz4n#U7|TyoXazFGH>=0iJ=q{;VV03i05x3=T-Pq+b%PWy92%z z=O&%Ywq_(l_X;z<@W=%76b|6{qQyHCbwHiYm&M~f4Hm_2udYP`jnRllm!_;2YRvLw10M~fMx;2BKg348sl45}y#uk-S zBQ!V@7nTm@7>w%WDgz(-f;WLtL3!=ifG_y)ZP_I|`-Mi z=%qbZEgk93-FXW=tGuxJ^T}X0r$@W|;KSjji5t-Dz5;RXv7btOBf^qo+nJC9fw*m~>LI!|#zW4o08ZGExtYCUb60d`JY?TKc^qJFOG6g1$sgC>N^DhdD=XLh#Sh~oEIUT?x|5kAD6!S<$#m-S1!)feA%!3yul}^A?U1IU9HG0C3w4gdY)Z!E>c%U zTzSL+yMpk%*gLK^q0xbm9_p79yUKhU7DB(Aq9Mx6D7*C}ZEf!^TZq@r``L_M|D5fgIk z3J>9Xn10Zelh99BFn3Rb=kFA~JpUdd@s2A421g z+nY&RR{HL&GLrK;hSM9DV7939p6^P>^eIs1Ys8W6uz@%0%I9lhdpdiPFRRNWT#>NO zpwJUqeeisKfYii4bV)6Ks;9p5e+@e3?KeZm?1lzG>c;oY>d5Kfj7P(8u1=my^Y*;* zL~)V#@?oX}LE>XR@5LuNpmc@$HhdP)_FK%Z_TFKf(6-GtEV{q7Qp0s_Ob7ABZg$ynJ*U z!;m08`Kg! z)1gXUIA9iN9vID;Ppv3#Rg7?6zV+%7dTQB59sNx_%OT%EzqVh(dHeRW66-Jy@z8>N zr;vEhs7Pfwg#0J0GL6E$iicpU^jush_>wFZ8^tst*m(ln(=0!!% zD0gBQvjAfwy5sv-6r^Xl@PH7YG+9dpgfY_`{0%D+q4C=TMIL1>9B4qUt6Gh?HM)hwbJ$Q0LDiMf%O$>t>7~kCmm?0s zJx6X>yPLmMkzdM8psl5|$mZ;#o)D3v`xu8!(Q;5OwfHfEzd@+a%AvC`)tXXB&(o}A zO`P8=WdwD5w9tzVUyqt=>)_N5fjuk0KsHZ1OT_&3%aV$6-9J(x2OEAG>WaFZF`pMy zl$Eoa3ZGhFYqS;)$MuFRpa0~t!z9zQ)CsW6Y*6v4R}*>yd0C<%2-9q8U|Ud9-5^8f zTFw{Aytch}-H_OWlp?2;8xSKT5->;m-6Gy7pW}nh5}Mm@<^(|x&aoyz}!$HJTbuX|eHBsfCo zqBMlxhUYzNV@N8#Ie!v|RG`vi4=XcMhzC<2tK$Bpn~%; zt&SBi{_UmZr4?6P0gTkt#6-cFk(1=&Rp3FfoE_Z?=VR^(4V=Zi5M>iDZ0Ia>zhkNo z?@$p+sBy#V8pul{ii?jnX)<*~2(NWt4k`{nU{lKr0@iCBfKSQQ(o>ljqldFxdFWIY z!F8gq+pwBLf39IA5>ZVbAy#Eow~NMt+jbDky@i1tlykoqCgUlk9}tMhdWM3^F{Yap z(v5a-YJ|(#HG%Q`gX&8{`%gZa%Qzhgfjf(i0aMQbvIC#nn%ffj`66jBPrWuZt~~?5 zGo+%)KE^0CkL~BA!As)~_>{5)Ywz?%8QVaEli;-0;B4X;hVNZ7g@W1&{nv&QuDTB0$Z&DqcP=P z9I=fyoj9kOV1y}X++TUNjEW1G}{AKeRES1tL}x>mydDu_Rm@{qpfHE^%& z*R1N!n5`0axZdl+qXHMi8Bx4%`deb=&zEO+kbB}6b`3qFe`G#eUlDO>^5m#LoYyHWV=^9yeM{#o8}6(^jIRzmg*hxJJaFf}i4kMpn4pYLt+#obQpjK7_ht+wZNC4J zO2c22|Ln{EGr@-A?fQ5N|4rpjZu};$H18a%fbY5)rqZWam=Tm+1?k=F*L{mcsK8ce z^~Tj=(L@SRcF%P^cqX;T(UcNwyBMKv?Y?Jt($KZ&_DvXND57+{lw_W4N@J30pofMl z7t6+8q>{NX`nHJNCWQkyX~;>;*& z%;?fUUs+`{3I=n_#8D2LX<8ta0-$@JdnYU#5?laD`75!pW}8v`lY+o^zmp9k&@De^ zHGBXchVOAcaID(Kh}=6oy@jwcqqxTi9Laj1_MLlo@|cP6nrAJH-4EWQ;M{3HPB@J% ze8F!yLB%v>3krH0y>(z8L7U2Sgxhp3ta$Aa#}R{zhu4crN@~%9qBpCStOSVNGrk+Q zK7wXMgFr4<3d@RdTleT{`)_`Q$=wf1@%v3PUMq{I`8A`W_Ux`YCP5C>Gu7%)y?b8N z;pCs^(jnN|FF)~8<@sEk;t9pxJL?`5VIR?o+KgNz#KVvrXch6>5kwd+9D50y22f%T z>2f$0z0TR?zL|9h^`0>c0?piSajM(bhzL8uJ@zG`We4Vg!^_={A}z{hM4D%$vSuKB zxjR~lbOPlp)2ZT5j?Rn-2ER+Qxc2ce!Vn}3pMtV2v~bx==)5#;Qz!*E!k(S!%%**5 z>jKO)A>9?27u9e-@*x36R4v8GOVT5cZk7>rq4*6ZNhNw6=)$?_wpycYtX7@khs5 zbwtJ@CLlP?AAC9``4svTN@A+)%s=VnAINb9smj=%)F%?tuW?BCjiLqT^8w$fd%gmM z@lhC&LE>(i&wxdhlQpNJnT3!x2`w#R)S=u$6q^RwB>BCQx?hkm7|VD=k5w$u9DjrV z-FIrCe!iuAQj4r_lKQCKD{ubm>b$AW0oK_`Rfbl*j%F#W-!fJ=5x6WkQ}85 zeb3WJ?e-lEEk{*#oj0Q}9RhEjGArrmSK+}*C}Mrvp}pUA%qT?7JC9$9W>Bfo$iC^h zfD=}I)Cz?FgRQkw%+;Om)RpDdq^g17azB!2*mF%1L@mCVQ9Q9aP3k;}q2r$aNiI6L zy>NA6ISr{J3F@SLtZfIMXZ@Y4*hWnxg3?vm@|Nt*EQ#r1kWZM#_0(~HGZ0B#u=^w; z0yUu55FgU$SZ}_!^*P<#ZKG`^lYq`Y#m(N~+C67TmI_JG!*xH`9Yv9bOaixkr;wV| zb%8?g-0iC*SI%)tBTc;DKgTl=d-y1uaPNy`-vxSX<1UgkBydw+TNg>FKpoia@2;f3 zCoOqQeou8+L^uRAqwqAR(@TSpM2$Ub`kfj`JqO_Z-M#FrEQHRVD)Wi7sB)!^B+Uv5 znjHY&^yQdOS$4K2vh|OaI54M}955d}PwJYv()9!ssnW&^fK&78DjhfS4CfZsE2pMO znv0}0gO1f&l%e2v=l22DW0>hM2E}!h{efcWvKlAm)L)6+ix-aQN`c#_(vpv#6uF3kG%zycrK`Lp5-QxdJ3>SdfUoI zpz=zAB;$Qa1M1n8;RuBf;!pm`_Y>k;EZS>n0Kp3aq?Y=A6TW6PVDdCzrX9Bqb3d1@y~H0WsaY}YyK4fL)8hOZ<__UGzO>4M(8)! z^dTK(+$;H+WO0Ckz6g*>Cf?I@jVRGLS1)_ zdJ*S(`QcsEihM28jToG=7Msp}=gEyJ8oZ4*H?tG6QaFkN zJ2KsmjCtmyq^n0?zY-)LZ+CT0h;mxQYT`HPjjl}_Mq)p?9|=?^)l{`FHFYw3zpozT zL`)NHR#;l@&`A8;XC0L@C&btg?wbot-{+TiJ<~3WQjl@sD6wOkvQ6zkOj!LK_7(Kj z)bes|YJM}l)U>)O$@VFqs^&th0MfTk5o+g#O*mu&5Yklv#u4$jPts2BT}{(eaCC@) zDVArw=#C_*Lq%`c>#sV2YZcN&K99tV1C4|lO1+r7_-9Gt?2vQIO*-RKyZcgpn+?|r zy4SDBUDy!=6l`lcozkbWuHKi@+lKyq7r?3Cxa5;8_NSd;dFRCI#wjZelU(hjUxb-U zI!Ry!1()?{RKAv`?uD?n7*zw*KkQY=*a=p4GEa*JQ{6@f*(IhwiIeqt7!br1tvJV< zirz7BYgBTHf8}Cxr(nCvCN9G!H#kE`rXXXfDe;Xf9lNFUF0Tl}H1;RjW+Y+u3aZ@Y zhkv1CLbJHBa?^l%+rxOOdfcs~e)X8-AYmWv)$d-!Yr5kPe7?k9;_(>{DfF!?jF3}M zj2}RmRjIlfLEP{vjX0{gik{vu6_HpipIS;iT&J65sIqn~YbJ)_lQ0W1GTs31JgFxA zxQ=CX4!zad;vE;x!(eYYOvObw(RqDVMSAU8ZH7SprEoNr;)bV-YdTe0R=F>{WP);_ zjijok9Sd|w9f|}Qmso>)$X`=b@RU){UlI3mwM1lXgEe)tSe8|^Q6K!+tOAo8?E^#D zbNkP+>24RXv~2B6m;XQy!7ViK1&%c1)g3Kt)#D-)XgDSvQ{|gYMW@H9 zn^BF`PA!oSL_Bs(56;wWvJaHEFV-JrYNc>R%i0DN za)me$a7i(X`Op0rCdnIV72CTQ0C zQD$eXex9l;ga|TdLy0WaU#7XNffbH~BU#%5O{=`>8I+nxw|Ic`qH1bum0$jube^V;L|h3oBYg9b;X zCWqGFgu3->O05@w$%Qx(PZL-tjXAZjo8$DCN#0by$u2DH4;gB*tWE-Mfl1krvx<)OIU%Tv=tb_8&HeKI*0FFZHBdL7d^FyQJ&3$>Em2=~ionu?|F4EAA%6mw@KOl3jJ z6!ctvp$4WH9*Mk%h={_o{UWE|YTFp4alfCV7Dr)}#fCU8^V;@b0^*AK>Tpw{I+*Ly zJjaM_ZJ?OhsjJ$n@6o@^gpV{Dff6I#jHWWU@=kTzIUqdt7AsEzd8aC?P>T9OTrr7w zTsGIQhYcn6D0^V9X@~6=9gwn&1CZKJMa1g{kNyn~#E>2r&m`8$8-EyW4w@!xrjL_PIM68H>QMEiH1 zgib>J1ONSL>H$W=bof$|?y$;R3Rr5XVU=7C+|o?z$AY8im_b#FT|w$Bvq()+HXZD~ zk*a!oHmi%9jv_xM&Aj$)(?!L?xrMV(7Y|Xt$%Qkb49lSgm#0SKV<6e9n|4W;DP=}e zuH#RnuWni-T{i2uJr5(Xa~gK%{PBy#;9su~q%P<%KsmanY$@hpVNi}vp3C1|c|*e( ztv+QBuWv5roqqIdpVPWdhD@<=c40fnJtrm4^`auzH^QFfx>;;tYHo9R*uI1GpI zC9nYLAl|xtK{4!Y=WIw}+cth&fbj4#ELk?9?cA1YTMxXLP~dzERyA2NE zGO3zY&$(`M&3J_~TYunG@A~Gr5*)o3E3{H9|9EHtGxR9B%RO>w$oUG`&E)t#U#`{_ zVch0a^MQ~sJ3IKG?px>5!Phf=?z0croS`SvK%(KAw&I)gn4K6EudB$7SS5CKnfq&RD2;fY=XcNc8^grQO<gVOyFM#(&GX*2 z_<^Q0m|1)h4&FlM`Yt}S&xg}V;wvF#m->yJ( zn2^ie0h7v3v*2@}B`iqkGsA_7sWhRT_mn%BS;O}~y+VM3H;gQuY!C8XA74zQ7=3XL z8Zc$ER-qS(e*;rVRlAqX%aB^B+dIipo}l|$Lg8eZ?Dg~+cTIhERHph`UJk>~OTdEH zi~?!r9Z;C$l^|Ak8k#n#WR0Ju_PjaxO1K5VjQZT~>XdJGYOzd|u62ku&})6aI2Of1 z{<-2w%Rag|IzM)$Xr{bB@3l+!zHp6Ba1d+4jaFxtcAl1q@-!BeXf_YTMwy+zQ$g_! ze(ln+=u%#qbzGxc*5{JEs$w8w)%-66Y2sW?~p4L+`kz znXdM;X<($XYiT3rjYU> zg843~chZLlIt?IuZqg1$vf@Nd;FMZ@hy1I?M_t(mA1MoyA|4JEvOm%`J*W-`oA9>u z1Z_iH#X}6X#B`<+oMbG^TPi|kB}(4CKeVlPIRu}ZPW>>#e4%?=+75r6$XwwQasyVC zBM%uNa#~#Ivx~nKUA!r9bQGl;o-+{wI9U`>e1ucRYh5WjaAe?ic*EMRr0u7vt~`Zk z0)usQ*rqEa5LvK__wr3-u;J?u7bRd9r`1X>G91!L z%uOQNc-u_>CIgFW z{nQ;$r@eN>N`<{tH9KEch6&Md$FOK%8?=^*pa6A-Up6(+dItk@5`L!n7_2*tSEukZ z>Fy5toCN=k=5!2gQ&=$r8nfrrTU9)C)gVYu>bk=kki^&=!%Dt=zunxUx;j%VvuYgl z?CFBdw-!3r0HdlVVF0-hyt-ox>sT$gg4vc=Q8pw1hPeW{eN3$FYinsRo&CBM;A2-c(aGWvLo!eTI7uLB0mx~lMm{w$`C0&$jr~B{0+NpF! z7qvCA#Yg-{7TlIApU+m27*e;i!1YsKqCfMV9$z$5bgSteOhs_B9EDI~r)aC~D)yj5 zew56?{~R{o$5wax(Pt-nOaDBOF1h%iayLn_2ysd-?#Y$$N;+ObuAd{HDowXaY+2MP zHO2CWnye?)!k`*D?l~0mshkd*KHMyC^6GB3+>9>Vx+UI#jyLFd4#SVcQ;34;mTK0; z3{`Dn)9sn=3ixMu0<+TocrSz|BL zG3;YE;@{)*VnTEMJ-beAZF61oquF%Z!Jn>?Czj962TX9T^{@0+CFV5gNXNox_^6fZKre{Um-tSdWHjUx+{>e5SWPtBlz>eO{z3&RaQ;p>*_G>(>qG(vgRTSWM) z8bjf}6R0a7e?v?w*pQAo1Abp=K|`AH-b;x=5mk|VR1k=c)y_o+JT2&?px9RFGLIQ}XV`8ZP> z&$yLs9XaPG>+jTiVkpGQrvK9Y)zC+cg;4EBmxoCX*F%r25gd2x97mWn<>8^u=J^@y zw$8Hc&Nk_KPguA!Y9iU)>@ezH5=vMn+O)1GrQ47{)`MNAyk#tY4}S-G6`Mv-e>?=K z0&L}(u6KbGAL>Tz#wvYq2g~%bF>wpYtN0bOnYyYbsmTxY&gZi7j^J`6cOujJ$~o8y z@L=IkSxI#E;PCd2>iT{60ONDU-rG@C)>VZir-=En8E==;3HtLp?4u(#gmUQR+g>;m z9pUC*dC#(^$7-Ucibz3w7@>_)v%?GHw&Udf$t`&*PyDU{MV`psc~eiGkvYHYJxA8{ z$T*VcSuE~cp|Yn*)?ePgKjET(;ztz!4g8MeH~lC59a>sV$D(^+frDzmcLezJQ>c6P zchJ#9rR)vdP{%1Bi05J0QAh>wU3}0fTFc?HH5PQr2R`;f%a;lP{0&1XXfGFLCm+$Q zU6wgD-d=Vo@*qLf;VTee*B3(~)u(3gZ7Gg8q^2Um2rQKIXMpXHIq+7M+)}*z5MOjd zKb0uMPEX1)dDE(9S!D*w2vcEfzwOFJEN9|~0Wj3+|J4$J=7}D+M<{qqCyI3&c4E3` zei;F|nDtl4?6>3ok2C*ohyLHuwBHvt|E@ar^#7em2mZhO|9?J7kj5dV#zoEvIXmVP z&=z5#?&+3Q`106$Aw0rpbi_r6QDGkt+9pbjwx3tSy(V7ye$3;t8<~8=880XbZ;!+J zy$f}}-_NJW#(Z~K14lI_*H#m0E{iW1M@D;ykosi(3P6RrW94<-#h)c(XRDJ2Nic`s zu%7{;?gtimwJ{qSLd%KIa5`ss6T7?VM2 zpRH`nag<$YL%99G0zY5w%m|Fu*!IgPbq*y*R5dQtbq3weWrG_I!KN#uKbtTbcCEb_Q}eh+L_sJj^s38n^M#L5J! zBrTLXlOGF(Ft?Pd4t&f3NgRFziJ|VnxkBQoHzX}4_1CB1w4s@7Tg`QI_OirzJ?L+# zkrb$(c%f4KSk?}JGs8}i)9GXMv&ZDdg+T6%MeSou38+wMNID_nK}BdGB>4Z67GLB3 zM~fraj)Nb{(A24M8*IeFDpqoCH+Qr)jVrcmG?-6Da$ zR$lhIrd(s2bvs=~Lb_RcuBRT5R*x4Vp`ejaEr_0u@%QVGkyA_|$Tc^S+Q}ZG7Mp(~ z#lyg*@>i5;UJAOhbN%39K%{F=L5lT+tf%-xE^ArWxqUvtsNS7UV|R7-D$|(_`fAP> zcP7{_?=Yi8D&EQtv&y>e)@{2f)HtXlp}2-ewQZr}1vwO{ zsOh|QCpPh;lF<7DwvRodK~5~O8Q>W%j9~VB-oVqrw&>~{SJs)V72_>sri>o01H#6O25Co&Uu8a`V7wq7MKx}-)l&H(qjIV{S{ZMV}D2I z-72$y*tp7avC1gFZhOy1aPKOKj)#GhRuwvZ)Dr4}bq|MyP%)$+1MS~DJa5cz1APqTv;+s4X=$aXa=N*!ObN-{Rs47DEITW6GQ_)ITRO0NTV1GqD5{IP5un8I~T z@V{ampzPj+U^$;N>IAVdQ+P1h*6?QSdNteAz4>LhyXQJWHLcW4(C7I1D@8t{r>ssfLG?*a^>W8@Oio7NI?hfA7?TGc?*gKKI`_o)32E~q*fL*no z>E9;0?(qTbn~^-8=^*|j?i%A=YK9i^P>qtHwsn?!3|b;V<%1kFKq%^}HMC*O-#N&7 zhSwkf^6=>PPh(2l-SH$vM5nDz4PT>{GlGAnO4jj?#h4oBn`Y&>eJZzmjF@5xx?$XikEa@8rxmOOE8GFdKoD!vF-xtRXg+~HRxxw z#|Ugx+Ou`CY+YEs%W0`$%_;k}F6&AY&^%(X`C<+nW$jl0rZuc0t99jiS{eQ9M3pG^ z$lsM++@5;|Om3AIM`iDQH6%~HY-F&R%b^C(pMy9aL7719x|m^q{JkL`RP3Ftt=K0_ zozQ0BPK-ZpHa1oky1Qd0o5d8h;I11H&#YLvF{sXD=pNusW2?y{JtOjJ7@l}pCrob% zXQ-xAQqGBz_B7bWv7u(R*IzQ!3hZQzM$;`hx>+|@2@IAq^saz-(^>Ua^DX$nbi^rJi1^KX0K ze~v$p;=}*B;{Pfz5%7g%3;v03{QHg-`}aG&NaKTx5ev@&O=zO&QoF;b z9Fuux<)WyeV%C&N;C~rxvj1k`k;;GDawc(3vJ8Zw`N~&kK5_9; zBzKN-8trdjYagnt(8qbDfs<(-+|#nx+O0|)27ENUG;p!Tw;aVn<-`?noh-6R9p1I? zGQVEP12s2%lLxoo#GQSkHupS>DGRuianaxIVb5+9%Nd&VSgKBATh3-$=6T+kp7t!U z-T8>+uEadJ^kC=DMU96gmWPqt^OKF8WuBT@F&UXMSu5R}g`prmH#n&|>f@|K?opo( zq2*A)jd?3pc0Vl9UP(pmOuEg7kP5+H6GuZqpQxdkOVOpyhU7UMy1$+h7pz}N zeI(>~x|p+G3u6P9E38Pt3XKr`L|(3&0ROt`b2gVfEk&j{+`9DFmY74WwD0DhGW3;y z)QKJNHs;t_G!IDEF$$le@CG$RoVL$ktme#HX_IQv14Q+R?D|fcA%jBvRio$`&AjgS zt9`VfM*0KX&|!Hfc#2gbPxNqP(oZb{wc)TR;Q2N?6EvI ziEvIzTI+M#`?7EQXxbo!6v~j;DQj~jywC2JGZLD~zQg746uR)af*_eB)XTHFIh16z zs`9~n#>x`E?g#fXb9ujo`V-G^v~!xZ%u$?LV;sG2K%w{^(Ni<`%;SOHMf!0FxLQVv zXOuwQNr9Q!VjpbN?{Ed_RLtR<{l6S}>#3%-|0_?S0H? zu;4nAoGGgIO949^Y=XOpul?X_LOULgjzSs7`hwjD9C7@S)^y#Lfnmu@3l^}C8ZccO z@yF-N9qjR()eNNnO5rnq4112MMPaKe@7`+MKB2hMjCM?P&pdVSO-!Bfo-lrc z@@*$_#5#;{0^@ZVEOtom6C^iHA5GY|l?)FRW#~B(`FPFl@h#`s`Ufpa&pbDE{5=~4 zDj&?z?Ye(eWNWUd`?iwx%_y2hL(zJK3wMKRP&xHs#;6C{6FX8^7^0oqE_iE!$b2&i?ff z&PV|09DDiod3^7x2EA~n-)^m^I$!Q*hnRs4Tb1HVyY3JDQ5(->{i;n?Hxr*Mjw~fJ zx}fUOcJiHObj{l9JaLd4ggowN5Ywubg|cC^e!xIKRjXPrue1V0$rU({>@(Z8QTx7c z^PRf>Die*CN&oi#6}=UtaJYOdJbYa0$Kwv(E!VFxm)6+E_hGpBC1R85 zsl;A=e4^6_Sy;=DpSdp0F_Cp6VirGYwK5MQq+2&zw5=`*BB%JCxwl#vMzOX~&rNMV zvQ+7J6AJk?jkPGW#;1CGR$qC?Z(Ns^CF|%RGfr1-Dc34Iz5m~d#B#q-p7(a>hh4~Ptt!dST21ne=WbT+JTZ)2oXKg?ze%< z+>~tk;@~6H6B8JF34tSBuP-J~?F&+9-`8t$nqpy7eDH-m9mr`tz2MzuA-FRpp`kNL zGtj(MYI@4ADV|-U)4bWHK+N zquTjBqS;URA;Zq{bok}i znVDyEGhF1=hmJuC;`nP;cByK$>13vS*wB9G?z9-ouu-pOwW+VWX2dz)5gLeefRdMI zI6YC8)znt0Iuy_=-f}DpncK*TYkkc~C9JvkM1Al5%N~sn({Iud!@1kna4zVZFNn_o zf)Z?|0TG(usoTh}t)AY)H*`_ctjM*yeP$u2UBAEl7IZ`@w$9GkhW6Rq28MC-&5K>M zw;zzNGWZ4W<(nN7?Y?sc9PK3!u-eYifCY>6MbzicSuc73Diz6$vf(-kN?A3#`j&d? zudEH&flf%#mf(r#NjskRhfKd)y$|GXRs`f&>h^4Ov&)B*Ww*V z=~pyI`Dn(xVZXsdG1dochWzf-x=iVM4d`w?luc>L)*6?qN{=r~?p$$531}({(Q5&# zvwm#2O^-OVbz_@H%y2WPJ5)FgxL;EL@REM*8=Pv-G=8C~CBT$j{XlQ(CZ)`Al!#Sw3-7%A4GgJ&fhP|4(gi85ZUDhKmX)(ui~mNHd^xD=0`S9Rtdc z0`CwaU6M-wXaNC{j)58CrMpXz92i0*q`ULX@4wI9`+x0oKAr12A7|cq*LAP8o;A<2 zp1L8-$(-gd)*m~us%09VytrA+%>Hox*CI=6M19sZj|pD( zAaJUT-3vMmXNx=6Pi?$uFnudlrjKRew3t5Wzsv&u1yMCtF%muIS}!hM;D7Bi z++k@uSwjp%_o`fgJ}Bi`Z2FtjU3!Q6S%a!yxmwqrT`bflJPTc?2V%8Qni)8z^csF9C|e^osawvZ@rTx%17zyp1I4cc znA(x_;5cK)Sn?~((#sr#M$Pwywq!4niufZ*u*_U$pYXR`B|6_c!rSgIv-j!(L44|4 z7OvRE_gYh$wH{U0_KSAd;#E%G!QlDGZFBD8lRh$?W0;C7W;0UfGzuJZW{@SH*{fQt zgYV6|UP^g=?)T*Y!-b2_6XKo~tc!*`$jrpxzZjra#TsdJn6XdKH)=#7q;y)t<_)f- zx7YW9eRk7e8j0=ov*UX}!0?~Ng-nyFBPU$2^<>ym=30GbctC31B5?Z^kG~&j>&ck| z9rEPGcC`HJf(tI_)aiZ~?0MJmo10Fa?`88umj?9XvyQ{iJ6MsLu|t5r)@2yv0j6ep zdJ@K$r0Jb3STnPX%h~(OOc6M>E=_PNA7OG)8#>D+62^hNHeTK2pYNP`Y2Lu@ePt79 zPM2@T(A+0-Jv^Vv(-46Rt1Wh|zDL=6`D~j&&?T^if-*%a-+p6|RF!HuhqgX_Ke2kz zxUuE}utXzUQs~lmuGb!wYRjJ`gW;nzABxoLQA2tx3%iBGc8-KU+y2r_Uw6TeBH%Af zGyd|qMH32I2J8D+UXt3CsQX!Bn(RN68*_%xAX^jtV&4Ude;VjH7ipYRQMg@=ox6D| z(@+p$xtX9T-qpd^xwnEK>dtCQb_6IXPdDoG|58C2Q!6Y@p7jgXFt~n-jw3&o`%{_F zPjc6zS*Kr>%l@hM6!zs?=wmbS@o?2$C!i~(8D)0-1+zhuh08U!m9=d;$W@R6yZ5jq zBXHKY@M~Jt&aKaTMv?PF&3T@rpk!BvA2Z)aALRG@Geq9~4b+}?{r%Auy&b(>gK|@y z8G9_mnD#TW=sdIOkH|$+mG`Uu{Z@n-Hbu!92=*AYxXg5*ls)&*C_)N5?;J}fn11caz151d&*dU2>RJa zx7yk7@>l-CQ#@FM%cU)HLINy`MN7Z!xTmr8B+H8^aauZvI4=qMG6h$D4mPo{{V~uk zoW7|?!D&X%-M{fM0hEQl7bkq~s;K%r%*DrY^XY~4U(CG`A& zfUF-fd}hMo7#xFnJ#IjKN={8VHJzJfK8>W*_0CVNy4|Q@h|EA)r!WUpOZVO02gXj5 zB`h}Wb6ygE_2eHB9*)*hlrpgQt3R~$=#3fbU;ROHG++@l{z{CF!iskO&xmxZlyirZ1Y2~VM_Vhimmcy)rHME;MQg!U+?nmP$r@Q6`J_f(e}=QIFHcy&jO4rE-P_)r z=h|lFcTK}l`Vt`jG$|yCBo}@@l*Imni`MMB6;E#VSN7ehxSxyFXOIOtk(|IPv)M^m zeBgj8@&&TV!0YfXaNg~nBcJoBY7XQ(5SC=OHzF(5#LLYm@Dk=wpvX{TRdpFzq3jV6 zN?f&5Dct#N@k86=)i%m^d|n(7GMK-*{RH|PIflpO?{JSC6LigJ zl*f2!6K#?RJ_!*Wygff$Sga6IfaB~5dqEErp7>3b_{o{7ML&}Y7?mg&yp^FKj^ z)Q89}7<3-(*bNT$oD6ixBmgJOU;6J*M4PM;jbqpuYj6OsR55+BCDEI!N@rO&?wz)r z6VRfeqOneDbUjB6V&QWl>m|GBa=IR;|@MG>+Pa?Wrc&ZhYrGcIi1cX3(itlM!hEZ-Wp4!|`^C4=BhBE49 z&`J^6FI&^+?YqtFSL)idY5QDSmc(=#_I3Wgzg-6Jp3m%jfm9dtmA56crJkd8 zm@r3IQ^%2mCV-TfP@e`8BF}LH#75J^&S2=Fevj!AKW$UzLUx0;2QFU(p&4>z?K`Xb zcHEXF@>E_Gdgm|(MYA+4Mojvbu6kGmC~J%4>>k-}`8^ge3}Ar@!Aa}u4pSU+k2H(n z=oxMg^i07u5y%&yAJy<#9$j{tT8PD~@LD0ZGI)Mdt(&1uDh+7AqPOEWFL44eDP!YT zjv;>(3`=u1Vd@efW0FI3>1W4Ovup0{hrQAI!lo<}ayMPZ&7Ybnd8LPdN*a_$m zle&^&Jj(PTL!FEm>R|nObj138qScn1nmfwJAL#qEZ_4JU~13yIzcMhw+eoJWM5aiG(P%^k** zP+|noMA;Dcm%nYIWRk4{fI2pb+tLA>Z-b%86%oq;Yr7aBuDnj7ts;&yet>f&&h=~9 z(}X8S*>WyCx|i({!7)O6WfJEFKH~2T3fgI8dVj_sq*^5L<>p zNpwTee=GK`73@t5w|@t$E6H9hT1!7NSRBtrC(~hU9u3EC6V*)b=0Zk-sXGl`0Gc!) z$x_0|dqO_H#yB6q^>va*@KJP8a>Vwge(krAo!8J3dus6F`IEHc*)M1oGd!(c73|?` z+sys5L1V>W5$QL=h`cu-s?+o>cU=K zj0Nf@3lceyaJMGsc=WgcFiXy`I~)K!xuB$uZ9J(~R#XnGGpKIZlK8=>i*jS!gTw1K z>>7ZpY0`Cm%M$V)8r#Q*gvfiV4kfayq6U&Ys;yvur}$4YA3!Pwl4NC=aVgR8;W+w9 zksh$UG)*x3qw7TXh>IG^ckE)%^9~pKjbozFmb)9QKIgd)O$LKoGDAk@5Q^SNMYO%Y z?MCmcg?a<*vJ}n%UqV;#z+>?`rptQ&zGQ*A`U3-gNb8gwvvD?nwCeoB*PrCg4?pWF z5KmZ{t3yC^vAbNv$9_xoOBiN82cfcN& zh@{(DTXuN1S{G7Q`~?9WsBgLT5n|ssiuN+bSM1O`_RRF7Hw%wDCS~a z&0CF+8M>Cl9%SoGX#2vlt|#Y&{ouffNb>MjZ!Nb3QW zpN0=G=3f(^$t{28-ckCj*qwaC2Xa}r@hJhiwtLEeL)ls#kJ=%5pGGi}^9w@ZEq9|^ zNR}7j(Fa8{wkQZOsP@X5xl4Qx?_}NUdNJI`{OegRdh{2XD_k`SF-?x?XT^~aQp^=d@Q*;K72$F|hE_4Ov7kTrC?7QPkd6nlfhOIPR%bDtc?@PB zh{2%I7k8i=NHN!yRsEK?ufElC>+#4?L8a-SuDk$0qk!5AI;i=fLjI!rlM@BvLkdgc zhKKN@AcIyAYRit+A;4%2vN-?A#47T~pkqJB683IJq>XXd+ReHv;n+5L2WYDN`r(`L zsK_h+dm9moXafrkmno(#)DAK)lppr6t7;+K$I&eqU3_Q(y`h7u@y@%u1EiC|fhG72 z@6z%6XUOKm5Q?~BVUPk?$z34j+HdlHMlE-u^uwU@*lvUCqa6(*@G;CGrfbrSi9qjn z@j63JX068k>W>ZhrBqSK=&AcU8FoahFC<9XP#dU>TVI#)a34C1IoVG~CafCH(>rp%?fNsz_0AD9DoG-m&8&!*lDxEh>y(IiUXsSg;S|Z-;=Cb9>qr?_2WK$p#UFG zxTLDg-2vYfavjS7s-2)IL3R^TY$NN_wos%hCyYNNK>YMy}6ypVh0I{#paGa1b9NG9*52@NQ#XbA{{ z+h!{Y+sT0;vSv4Fg+Tp3gs(SIoUvKP;Crp9ix0F=fiFC!f0{ zKBgkq%z-9lBqNxdYO_H^V33YZkU73)lZqDXm;my zG>w_rp}Ci9!XtQ%xjOVHA_oN-2vI7Kr2vQT52Lmrde6C5U=E*lNKsw9)%yU>7x?2V z=MC-A_@`_Z>N2naxUzX^W8E7OPCDww)m(eNLs}jhWS$&SpU{E0?a*>;dl!M8EGNUv z?F*M7(+6?12uoYa5lePg^w zcUz^8qQ8V51(l4y@A-AGsyLJ=sAWe9CdNjmAM8uLe%2$NeU%|i4wk3MfudwKpSAQT z!>1X=y6HA#&FDA81XyZtpur4X*XhGbnoQ72T5wAkS0TeH`%ICi(Rsq|tzq|MF06zC ztm?)zkz!UCy@9M)cJtVov` zq=7z{1OT>q)G`dae1CHz*0P`Twiv=z?Pgv=ax$OE_otUXEZuj+5l8N+C%Z2F$I^ud z?e4z3Z=(<<^RwtK2y7t5l2JYw(u4zTCd2rwO}KAk7|;i8ZC|w1-xNe^%6qRB-qH}p z6_E*=nVJXk12I`5-+pepNc2B8nxE}-PcNt<#%#!f8zQ&V=!b%BnJ07y1+<^=f!#<8 zov3@%0#U6B@&bQeEXazr7BE(ZecvzuoM8jv;9no>dZ@?Z!D_er8(#jeSKGgaVd|{i z$u58XS~x18M?=xY1YIxC%ETDY=LO6S_Y!|)rsAPvYQj;`^vm)0{Jg98HCKk1(KiHW zZ+Pe%8jOwBId4rg^8D>q0_D!Py=}De+);4wM<$G?=r&p*KNN!FN0_P#DZ+Y5{-(z> zk^*^2V^w&VvhZm;j#M1*p2V#9LZsipa!0cKX9(1akC@~7a}YKl_d{-z1}sglr-&zt)A!U zKjA0@>|wFF5HWg`wN;!4U9WjXhVeWLzZ3s^Q}%*~Rgbb%KE=KMEA zt|4OVpPy{7m%)fXGEB0)76(04xGXMyHcLxk-nrIikC9`1klPRAN~70|zkehLee>nY zh%rIU@P@gmKnMms+3X__2GC=We@Lx!*cm12C|??z9H z);h&!(EeKRSYkeUwEJzzQ_%jo5wT%*R0MW!-MHAmLsbVHOl{`9f|ee{@Uup6iDl6b zTk08%TCqM1WsW^MK`YBAAf0os(Lzwd2QY3Pkfyt9UIgrpBE6%rF_&n(mX+ zcMXOW&5cX*UppyJiy)UMISSzo(m_^Yj80c>Z!p9X@ahElV&SJu6Vu&t2YX%K2~scK za7gQDsVORD{T?li)NpbBq2;Hi0pyKLtj)b$a2f9Xm|{_fJvGgThHgaAV+%bxi~@X& z5v%vjFFmW&8u--kC_`)%H0x^>PQ`ZQ9~#64K3&^aitlU01bgJZhLS(;P6p9$=UsLm zfo}tZcRY8eV3^DWQia>PLah@Mixk|wK!(Uh$xMVkj@L{?06D?> z@e}+xn(Xb))#|081*U2Bh@0Li7L~^UlVR+$j!>n)zA=!hUOUWs+Poa(7A846&v;u+HmD$rGEgvkLIrJwZelQE5!cLY-~#w z>Hu#HOIC(7ff|&lWkDL%#G)=**^gV*tGe1hf00X6;52kdt9aXK3A3tXM*`%?OZq=1 zX#4_^jsNBEP9+d#RpGElc3>5JQ9hw6)NwKma*p(;8wvtm(U1mhsZdA*Do+tz!3b?) zyL3Z~2T|!gz^$;M26nN(Scr4B1nd9@W~iBuA$PPP>=G)0wqF_YIYr@%o!qTHwoI3G z;CUB~pw-gEZEcJKf;KrkTGD*gy?YhIlq0aR_D%g4Wb@$r~_e9+T# zkL*Xf{U^ShYb&kH8m)t+@BXePckVDZ{>u?Sh^*GJ$uru)0mh^NYF>T>c>P%-NOd7* z8@Cx5sX?-El=eNcQBwGhjO0c^Xb?lE!l>={a~ArO16K=1!PeyZBYvI!bZDIiE(mKC zKS!=C7CTPT)WXf`3v-wy|;T#oKf5-FsOY8T$Ilk!6iO&?<5C}&kD9b3K3$j zOn3w?J$~d@PT^B1C`@j_NCDO!DX9W02dntt=tGhvh)|GE@85}Jiibzn=&;u?ViD3> z5-_SvYKVhSDdRDaVue3+B#xgmNuSrh6Rh_Xt88yx^Eo*#&ljP+wM7}mGdtWLDy+BlgrS-g!RuDignr1CAd4ots9siY;bPoz zcwS%~0%2`6Fe{k*c9;u4HK^7Pl79%=-H*`WEv|tv3bvixzrTR)O=sEJ|4RgP>@MvS zY;MqmJPdg>7G@Jq)_(QR)NhroW`>}ClkWp`ouRX4=mT1Fnex{!%RTHKFZR3J5CYsi z4G+MMtCzM&QbUHmLjzpZ76O^$JJJ8bgG+kKUWbpY`{W{^Zo2yZpgA z?>#Iw72{0Xj5b$;0^!*fI@c?t8IPm3e-^D+X{TX~5?7@VG_BRKUdZHLtKsio;4i0u zn9_GbV}7AQYV~j16%gZ8#%N`?En<{)2m((@J{-c)^N$erS{2hmfH^5Id3EADyCV;r zmxZ~J4p}jWqzdh)k>Zq3@%QdH9KBz4cXo-ysB*Q-vyDd9O*D3%SiLxxfDu>Xj7CAM zKi9s|$SjPBh6D(Kgiap6BT;Y(>&ceLw+DyAcTp2(759~ZK!s^iakMsDo+IM zD4_8PeybL|P3lu9_E2x;;z!B7{vq>5ykWyk(;zh$H@=2_HK2nC8UW7!W|{1n4TJ@a z%ntPJ3+iduUpY3S!(uI=9@W+9w~iit{KTcd4wvoIitZ$j2-fVNyu{IyxL}lhi@eC*N{H^j zJ@n-5#%ejK>KqMQktZG0EIK)h>(-@1FK|PGZN%d_2&VWeB{W+7A<(U|t zwodmYGi45lim+=e#4U8#LdHy93bk@b({`D_NQqu(x+BG&uE)p7q>nPW7HP40hV}#> zg6b@P3`FoA>6Ry_4cF(^VtSD9sa~hp5}BQ7k{dd-U^k-bkt#BVTCxVIX^o2RRt~Ga zWv&)PvUbyi6FnRUK3DIkt=^j48qkY${N9}Kta+_`e(BO<-aGjFZl}Q)9tw@AnM~mt z5t)MJ(HNh}iXX_kmG?1EFsZhVT3yL%76V!UNB{DSSF93hh&yK05SBDJ(Gw@8>rOkh z5L_E8CaME%ag<7>=F@Lv7>qG>t&#t*`RDO6-9T1Qo9G0(FU4cFGMV~bQw}p)IfpKh z7E_o=%<*OL63O{lY3~%N*-E$q^x>H#4a6p2k8i$~>-|&N0sibAeE&R{t-P}L(N<); zU7ABWyw=+M^+)YXT;|huy(wSl{M8+>a>l?X^U5vxG?;-v;>kw?(g8iSi^s|_Z zuPuZtI(AIi+Gci(`#!`hYN}cJ{T=_n3G%us2GiW>ck{+ZI|rY=F8omPwdU475!9WX z@Or3%7iQ%cW11=6c2H96K-_ei6h7Z^bc3y1c)52~kSOt27ahj@`o?qp1TVgLL+59W3=$jFIa3;wEbC1eXm$d*q;0pTx3LY z&gl(yX-*BkA%yx@ZJ&HQ_Jvwpnl{tpN8movHCfCM(V!aU%t4YxSphle%PK+miF{ifZ781*Hann^~|dygcwEIzj7H~5@N&X}5yro72mrbO&N z_;k;(PvJd58uABfZz)W_DsS`2lfWiojx{gJ!EntfclU?QDKOg zKtz}tn_!ngsbB6)LXU@ZUj))EVQI5ZA?t%rn)_?fyLz0}Z0w=g z%WR*ZCA{CFWL|!7Ia^xWvmys-|{Jw0~{8 zy)O^cZEKyH^OKdw8v9@A+($tyys_a?6c1?+gtmYqn8;6Q2%8TY@<|4RQ5teDO~(~| zxwzIIB$_(kCf+^%+iDi^o^!d-kWc!n9cZa;r%O=%%$%i3s3{lw8jBU==Qh@Bs zB;ULQx_+g^WxDpMJ@eEuHhI57p#RktYdJMQi`nl$JKmEP0s9zQg0?3;!{^Wy&8-hH zO1o-V=-uI+cJUgYt@17Km9wgfnwBH2q7Z0R(%!UP!N$eBOY|be(9yfECmOz?8|y%( zC^P*|EC&4wHMikSJt=O{B-VU;oYvAaL*)PR%!tu-&Nr9Y1Af1S`|!^r@DH|P-C0_# zj%Vm^%)ELmnnWO7rL=(v`edn3>K*f0z4XG_CaM}CHcyao!eL^IY;uA{cAF}!y$}d+ z*`j=-#CDn?Svq$Nk9t}}BO4b_;ZI-NE-JYN1K2PpOWKbhFs4@!d@3Fw(QG?!QC z_8D1lbNx6)yT;7E8%N3D=C-@jtJ}+4Lg>_GR04!|Dws2W@bIA%+brtV zZ#^}g7zOeSrz-UAcDXP2lp%o){GU5I^s9;N7>__{E@`xn{LnjnOP`Tu zB8B<3G_T#+nUR0JV>Vi$O(71S1vuSFvQ75y%&OJMG={Cs3QYn2HVBd6_=TOv-SfeN7NdGGft@;E_s2u!)Z5ZWpOYI`NKLU zEVc6O1P{;O+^3-fi;rk6gW+_AZ6AG~#URY^K9}wr!Ae}Ta%x0;|2f(h2a0~tb)mb& zcgIUo-|Q8Yce}ML(8#DH8E*+VK5Krc(Si$Q3d2+u#^;x^hTqBdfm89nu_h_ZH+XXM zp*voQp;p+=^v_dAv$)z4SvP6Dq+2>UQ1N}C4p(F6-`R(5izM^qZ(;5hF*q%1vxrMR-skj} z0KsmWgP9O5_@~f!ufFf2YT8-E2Yn380Jzv!v!rAa{F2SEU#DHD$U)+?I=Qf8$8I7_ z&%r(a_L)g_SdX+|7VvwLN;yR-p}HUtujl&5g*kCq-llfC0P`CTP3B)erQ-Lw=~ZiM ziYxMhvKZqNNE`bh-h}HtS4(oRE~ak}^Qx)cql6-AA+qnir)`o`ETBp2VdRE^uCJDK zOzW3MDSkYAl>Kz_`aBK~L3qSd$K)n>DPiRlFY8mWIQPk= z%W4`k%IjC`kD(>z2%@cqbAF&C51G&rG0{@=tBek1MW+sBti#0%DK#(0-cq|>c=NA4 z$Xr!WRHXQx-}QUY>esisX>BGA>~Y`HIy;Wm^y1a)qC%!_vu6EoN%#y;_bOJlxNf(m zU|5*^Xm1u#-=NE;o>6!T&~4?JE%eFwYTfx&&2gj+HoJr7w~ouHKAuj3|I@0kht{EJ@b zVO@po=6a}tiJh|R*Z1PGm%}WaA73xgzz|&-(nZRQ=@)I-jVPLrk(S67wpZ3Wtca$! z;as*xPg-A?H@r2f%ubGEz`TKq+5UMH6=Cwr3OBgAK3Y))YLspp6=DT3>RQ`wv6&}{0(=dGS!R6UcG16xQu)H)ZQzwr8XEaMH*kq1aw!e%X}qOO)+na#tQe$f z0wltXe@;pDxuquSLpwL1?p=();;SE}1?rTiM%pz$Zm3Z@w|iRo5*FX$X_My4vWukS zz8CHkyE+Z$3qEjUsjVGvWTc96_?$1^7Zh7oCo2!q#QPG1$7%IZZg`cvn6+p&Vv=i( zss>|0Vh(uJ$GKRI*N5Hv2AW(KA!fpP^Ap>8=Yz!r)-A}1d0t>s817(JJZ&Y_r7TyS z_f*L;iT77yeb+zUO6m@^rA9`=>G1N=fz@Tf$#BI)HVVcY_Ap-U$nR|d*5KCJAJ`_% zbAiLPg%YHpCBFnx&ca8w1iD~aW)k~H_p?Zo z^~iCN*NkCXSj@)>Wy6JzDc7M2^WwL99=8iCXajXvI%c=__mk}b_saAd@%U$f+vOT9 zd7b6$pfOQ{q@#oVvqcKny#25jaC$dy*=OLR+yLfaxH!OL$FTuA;o*CyIHmn&->vA0 zQ?&#DRC{dTy{XX%K;IZ55&wCBa^5xYDL`%~uajeL$-rmLH>Q$}rkzei^|#3Da?4AZ z=HLIc2w(61`*I{%*GOpXD-C4&|f3@g|O}FvIVu)f&h?^rL%rB7kAnhonWYy?e4lD_k?vx_6 zd-ElLCiOM(aiF1)jj|*ve`n!ga?>-W9LFoS;9ZU6>3$E|f1I<+6tlZOi+q)lR&wLJ zJTlCAOZCPzTfcyqjk2Q+H$VckXdmwU$k474Pv6Y_TwbQ#>!cuagw~}fRoLb?(NEoV zLNCq6G%-%@kTKbX-^F#BPhb5JpW~t~8!Y_c%U5+xG4wV3v^2Eq9_5XD!qcRYmb>q! zP1y1%M4yFi+h-TIou}Z4!870Q1cqDkth!k;@nn_i`y=(G9GZ zi>#u*(2)aXh%9{l$O2S0bh>C&Hx8pTLxiC3iF1Wi1B4}13ku12ElQ8%$MQ`#AI9- z8EyN(MAIwE8B*_H3$2abpCR6=%xPcRc!Buqh0(05XVTVgjny6vY zGB&bkIf#pDut%Z-=|Qyw@8@;0R!E}-C9`=tvQBsGR+9+|?_eQ?GMr(6M;)X$6a45S)}ZJpxgr zTQTtq>#H7_rk%?Tx@^C|U&R(NjQM~~C^xmR~{f|S!G=$~oji}#?keEv(60H)9wmGQHnb7*V4v`z*j$l6X)-V(m@1qS}w z^zb^R5zQGjAHWCd%4qDD@A}Hga11C4!-X6XG-a)3$WecIK`R{e&H`GN^fp(i*3opxZPv2ZQD-4KUrUi@QTY6%V&?b8MEWq?VTqIW z{UWOL@vB|+4-D;edA9=T^0b->9NyeDdqHrof>BR1lbskyJ3m-WQIipt)$o{;qv$B3 z!jsA>^s@PlO}TZr*tquF6|W`zRl3hxa0TH--J{(L#?qff$NU6 zKds_xXq}23MOpfX%8i8%3EGJq@+KcI zZ?yMBUVQK?bz3+SuBGYmU1Rd-p;pFuorO3TXgJHlG`xkLFCjxeA;4rDN*gX zeO9S*y1`y+h{&L>e|LQgE{Il3oZ6MtNM0VR254W~qb{{KXOk0r{Nb~-(F=eit7Bv1 zne$Sp+`a!_xF!Avau~T{;#c3p@o&e@|DvVp`tJ|_a#VQ|{coD8uK$CxYPiOe ziS=~E)-tldDH=Y%}Ki1eb=FvGx{)075_T^ti1UDJ_U;xds zYA0+E-ZkdY;Z7H?E`EC!*s1@Fu8127@y}W4JmGnrF(QNoo(_YO^bu-DOlLjiqHQ9C zeSEs?XB$P{CL{FfATr0XM-rbhPedYrmWyq7X4UPx!+dOci$C2=LwrwW+%K7KUWPj|_7At~^=sMBlK(Rzj?uz9`*{!j}Z9p&Dd#F*YAPT(i zWrFG&bCsJDshN;S6#y%$|9tg`x8oeD`#{d(#&|GE^{k{I-*Y2azr3f-d~u3~HnvF) z2eTxrd2A>vH#N3f{(iP3GOA?3qy`#HVc#7hhauwe zuQh2tXG*RooHMbCCfgpS%gqaiY^%h_9(-hg|BJfiSwvTOk*LRA$ zxRho0ss7ummKb2BpU^+x{ai40{4o$>c6d+5=6`&@f96fLgl$!OefXxIq1$#KkF&tj zTk7>~X`{tU)>3NaeQc}Rn{597L0RYb?FQF*w~#<#MTi2+7Jo-oN$X{qqIuB&1t}Q$ ABme*a literal 0 HcmV?d00001 diff --git a/example/otel/image/redis-trace.png b/example/otel/image/redis-trace.png new file mode 100644 index 0000000000000000000000000000000000000000..4f7115e1c515b40ee873c6ba6ed7a4c07bfe5ffb GIT binary patch literal 21536 zcmce8cT`l*vnPm>6i^XCq6&g!0f!s}1tjMIW(WczFhkBc2pEWhl9PbM;Sy&UGLi)( z4@1rphn$1J4!`ej-@ZNj-u|=a>^<0dx&_!EOj-a@gaTR~_T2ob5{_^q? zW^2C$lFIy%J3BjDJAL+qon!9c;_&d`QT*N|@|wbHC2#-t<&{-mzQ(n7{OTXZGJ6f- zaO)ErCm()=ysyC>Z?zna1rZ#h83+g-5-7cr)%6@-PYZX?7QOXhr>v!)dZsYF?h;YV zW@=7){p*h*RsHd?`=q0kJIkiAOyal9|BI_}h`o&AtH~qwQgXoD{2HR4iye7YlfyNt z;YJqO(iGP@Y;KIL{5aVC>>jhpdU#}WM)sux?Ut^avr;_w@!kjd8IMI z;3V^=K5R83Jfy^Yo~$vGqi*duOoE3U*}wv@fZhNCJU&0a)LX2om~Sa@yE2No?j9Lm z^y{Up<_sCZscZ70>t}}}EuKy+@A$5>?pl2F=Mv5X9T_`LL$n-&KGmYb@gL^c2gpQ<@_dl#wso3=oIlIaf zEjHg%M(XXGewQT(A3iPwKhIKq{`h)moTxNW$+l};PEfOZ4dKh}F{+1Cvx8+cT*gfK z-nQ4p7`_jXJWmUS$#3lBJp35>-1qeZZ@4ZQNH5QQNY;D7dyPvR6}k1AcOMcvm%Q3b zgl^!hu8pFA)bCa!5+cciq69oWzF~6;+xk1ghq_q@jyQh^86%r3kbE2_TSIF?1~xYb z)0yqu+-YF;vuYm9Wj?|3Mb!+3hap~#`-YOVGRHBt(IcOXZ`9sAZ<$ea0*$6U?=H{f zOZ9)OX8l3D%0>JU{fECoLxnpLFP0Y!GPmvMhF3q6EE-??89$hm<$824O6)WCvq|ow z{J!d0_71_7evM!nGln2hmn7^;s=lXdmdGY@V@I)WUW06_0#0tHob^(YfDKae@fad0=d&4y+*z@R5)yXH$inRNl3BqPxNh> zoB_qCI@f(x*9y^R1p*TD|)-u=?O| z=V@J>HAUCG!&@ewKgw;rXtMOa7TE1-6nn6``~A~G;&_Yp66H7XZ@UI&B_j}KBHstM zbN6q?UiY}yV`WootI7A7im>|r7sb$mD4Tk>q9Jn@fk*|{&ku~3L6+I6 z?$TfKe*1`D_giH9;nK9wr2L2|vW3!<>+xWm*}PQRl)=J)d41|cu%b~h_r1K-WI6pr z-B-?3k83x?ZEbMMyKp6E5@3ZiqmX+t69`fY)+#$IRFcl)iikh|^Wuza6_{sDcTWU= zGpd`7_@%+>6uqx5q(Ns(?93+;WJ#05SNm?xAhq>^yW*FOV7U;adO?d;_1BVrqRox< zX39{&b;{2IeNQzL^!0(=;Tx$1mM|UL*@wVyO6tM>wSq$5Fm(z`4)Zlm=Cdb0*SwVw z_STCcEy!zU6yIB7#N&~h>4U9BPEDd1gN3xuuR``HUmEXzv(Z>E*%otu zg?yZL^R>QFCQXUc{$1lhpAR>CwaSCM#fTUsS(<9DRX!B_U3Dkl(+{9tK@YiRLN)kg zn9(Nc3DoCj%2Chmg!_2DmEl;|Qhy-R5wRQn>@@i>(#Dx&_nmtjhk2}Z&zo!o7en1a zvEBtNCxU~=Xsp@#2L(Jzthr!MkAhd3gy0EArQ@|fb(bO3Liz^h9^{Oh?<~EP;>#nh zHXD`8L1yNj^wPndPtk$k)Hn9G8qk9Z8ppk;xH?ROPY%#RNIdb)rYE!)1&G=q@=?&>zro+RZ zht}}BKu$jx+EltN2%N+-BeEsz>x)k~9z3Uz`{O_VF~(3yjyUc+;w&Q3*d`6mtv3gC zPg-&|srQw|z2d7FkrXw;@hqDO*tn>3Gd^(6KebuDL`-jRI%GAu@K@Y8gXfElet03IiSaJH1u63nXJ(IxApLlG~k^AmI4Nq z8$^Yl3@HQZU>k~yrU$7KeNj)-pVihUJmLpQ=CDeoKYNn+T8kRUFD(;q4*lsf4v|ut z;fXhR8)9bp{eu3tV^|Z#NZohi# z(N7=CCzVK@u12P9!f~`YE0^rRt7K%J|A_~V$ktiQA{>!%`(e`n0`j&Oc|&5;)%W;~ z*!q1DTSp-UQUcX$T(zx#g&*qc=DJoqCQg5{u^+P2Q!5AObWNwG1lcp0*XNwO6)lIP zxHZf6ID+cL<#69VMgID{L&<9R>XV_8n3>d32&V{KTfHg=bo{N%c(Bl-hipOY&f20E zm?Ke8@Op9$r`TNL;E-B8nMU@$;39(4h4Ja2-xMTw)pai|g%4sgFo)^NwYO&K123Ln zF$vcGddADpo}@m}m+1~_@N7)}p)=5sW{+|gwcc=MNTU&=S%w{U8Yes$u8Je}1>MW5 zoUrJDCppI#d@|FMNJy~8pj69zZfSxZ|MGnfPm&Bjm9&q<+*a@$;;7I)v#ZE7Y+Yb=&hd>;D2yQ35Y1=3=u{9 zh>4gK)g0xBN_?=Ao~u#!^nROGI$AqOel&bQ^SpHfL-qGMfR5prOlh*IwBq+?%o_$p zZTmq|Y%$rnixQ%Z#!v^W5K{(O^?(M@V=+$7A51OIWl*hUxT6t%rraU!nHFI{nc zfC)ltCH-#q9D7L!==erjoV3rp-3S=$&N0011PggCL$b5bZ--v6y*6k+1atYs#V*Wq z-Rh;UIAxQQ(QFj|>GL4R--Mfn~8o@`I+~b9{=iCm(5jUR-npo#RmnSoY+m3(NfV@bgBA%>%xISE!5~%&~DKd(ta0avb?nHdBk<8ayxs>_4V(aWbNMQ6t6_N|+E?`*d77C~S}QwcS}tQ7(3+Ew}vzDfib za)LkzAwCgkauX21Zd|1oc-JqV<|_Rk9X*)`sz%HoW5|E%w5~1v)h6P0jV}N@iRuwWJ_49u2Bc)f$yW|pZxj%a zo*S|={-DO_b2snel0G&k~;dJSaSGhpLds0Prn{i z#bel)tzyzJU!x+!QO$4L>~Qgmt3v?KH~+H)1W2Z{!e1Y_sdl$}S8bvD;AXL4`-{t+ z+s{l+-6?e*kb`}BDX7*aGj%8|{I7RipcxRFN>lOD@ z#Fm|K9U9v`IfzkKEK)OMZq=5uHJk2z@H})>x6wQ4Q<0l-J+Key=^%cwnhD{s%H5rn zWD@h6wn<*Dr4PFy!+JGl2Mvh$=>_VWzw!(wshitei8i zj^Yt6WJot!6+VmnwV)0RsEn$xux z4y#-7jt?;1WSDy)l)$a$)|vIPz^slBMSf8PTO3j;7NTdxg^mqxw7&Z&@_STXekV;BgQ$6~goZ-!-9Eu|ag+(CWPKP`K`+t&P`*6P zgDcaUkABqTN9)ruZtbzx_Ihi&s5O2St!!0lUy|n*{i`==)`trfSIkf9QmdvnUfp@; zldrF(xGPPWh7RM~O}x*~<_A(CohZ=bFl9Ox>+{V#QUG-hNW<#Q>jUjv{$VI-iS7ROyy*PbGEAqG=rHgtU#%|yw;_mS)<&3En_JRob-1rcL zPTxmT_q0nESmx~xbhY4s&Zbk{pWxS`LusvkXJFIu+$}cs*Uxk8Kb)!17asvHIVAI7 zj`A$t0>)2zg$JtdVR8;b!>H|Y@RF>?^t%1tnX`iucdsM`X0=AXP3uFle#+;58UbXiVvqm15&Ae`MU+wL4Ffc|Z0cbhs!4LgqD8*5^ zs&V6k1C!V^h|cB@<~4>|k^{Rqw!(X5(+HIAFRd5$FecaA*0gIx$`tMAWsnO0J08i3 zJHW}Cnn!W1=QVKwZz%&Oxb!gN0JQT3-#CvGFH5IyTE5S^AGdsC(Ml@F)4$X{a*r<{ zYqJ>7Lo0WFFbDQUddY`8ojfdtj31W%EA9zvT2@LXeP{g>rWY~iDcuYxb!>_akF+6E>qW$-#+9Xi+!Q=7F^GzmJMx9}zlrGe~kh^+SWEr*A zsutC;Ipbru+#C!1^oPgi33>H}xYooV2fXF>W6#B3v4*_T(Ep%2sK(2(AS2wt)(~eg zs}4MuuVm;IP?NBPL2O)X#eZwN!6*bi^r9*2AuYmTlvHirKd@4OoP&7;Z}*oJhbJA$ zmeSt24!j%k-5tP-$Gk=#vzZuAGbQ90W@_|szuv~N>DR5qm}}dc9{`KyX&}nSb8o3= zi*vrZ{^77g6kLR}Nq-wn+VF#`&`}7|It6q&4PN)E`1NZ~C-o}|ZFpfEuXMNF*hbri zVr6{?}sWHo4*?~ESG2Zi*?7QF>!npbIP7w%O~sZ zm{kn5yj|qBU;L_tqi`re4=EXUWVtG>RGI;!M^VV`H5Kdmaw`&py4iXLj5vVak*6jj zz;w?CXi2xA7GW~_=rVCbR^8xkrIO%Xc|j*!a&V_Z2}TbhNLP+xllaVni*yf*>Z1E;wpmR)IC(jBG+= z814KOrB?1Q)c(uIgh8eYs;g5h-B|^43^UyZ>J$r0q=qB))#m)|yjk;Y3+?DiKd{(d=z0LuwoA@q6;m_gP>-2~-l!|e<#XfoF+74+ z_{Z~(Ee0S^HHhnhvs@aWNcZTVf8<8*986f%Epm9jE7!*G-jvcu< z!Bz2X6K`XHbq;!7?tZjvL^0c(@ipAo$+Vvw-Sq2<&F2bm4}_EXUp$x)aabe+wr4j! z7UI+8BUFQQ_NT@y;-5O6Ev8S?c4(CeJ&BKTGX#GbtLHC^2nj^5 z|A+PXZv*mwS(;a7=0BF^)nxy+H1WOu-O=&N68-;i9dnPqlF5>qS@&}J>w9!Yq+R;x zyuOwSd;$s)5n;Gv-&l_#48AGpQ}-21Eb&)-#%2=^Q**DPU^}kZP_U11s6E@;xWBRC ze&*&=AuM;Vn`$#mj^L zuSyi_T)?OV!{TebhvX`rq;_^^4()l* zNoNA1E#eOD4UcI)tKbG=ziGBucaIG(#*r?EyEC~uNb!^cEPEM!7F)kydK$Lb3FcjR zwitt`RxXARk_JmWbGYYsM*w%AlN9jx;8IIr{MIt;5s+mq!rHFi%h8gG1q0pOd&w z(G|2OVv*zW_OKFdLa%!)xcG}tywZD3IJZC=Sh()*r0irm7O!4GdVlLQN0|o-`-+3W zybD5wB1DQI`{ONIq(4-|0}x;!n6x9CtJ4GPWGip}3?yc3mv zAosB5oPTttlieCXDqD+d$%`>^vDbNNHU|wU4=_AKJ5{LCYH1gHjz!I7sAY}FnA>#n zh;cmwsQVr)o(w5RB}ArVLWxYzXtJrG(Pq_P+X5+pJ7L<`CF$TjOc1UNt$fyJ>*1Hc z`Ah!D2{DQG9-tKx{Z8}VKfHNnfcCl9lI}y&(uR)Lq(5oiHwY9lus$NtC%p!Ir@6}S zU*JBVHq^)WFY@E0J8~S>jQW)oNm#gs=4|R^zZIwdv22@)!Vf_= zY}tMac|^JZ?k7L@LAL*rNi~(*O$ylKp)T^sJid1=0YUEYEGnU9N{MFS{P#t{@Y?{6 z6mzkch!|tPhc>mxUI?xZEz@);cexc3Fm|wnV)cqGadK}LjSN-x1wI}KzTn=kwMI2y zSmMB6wY!uHE$=F42UeGZN1grE!J@OVmN9Wb;OTo(X<2blC2f%%aXUQ;pq>9fvpjF^ z{@dtpuGwE>*#d%FC!UXVd@iu&^RHAW&NC*qKG!_@)P0g>6)=PD1Y|B1Y!9{6J5m-{ zcfMAmjMIWWK)Vr3InwLsoUr?1K?I~KJ!wLv##&1$%E?N#KAS_)y8>Ol3 zU|ct76cb;6Y8jDap?OT`2Ld9-X&{kQwk8GMbQy2=-8A?(49man=hRsL(b8^vC-sW| zD|`i5xv_U@AY7~=AXp#H=U`(E3;WA>au2|~G)oAHvYT8B`8eOXYz@w`v^A`}lWX(M zI0EW%|M;Rx?~uo>~5c$_z?XS?oS#gJ9>G6KqUjl0Dh&8*^4hTZE={9^*W zDLJf}@?O^uPtVkWmW!&hpAc8c4xv%D& zfx26ip6wR%=LQ-JcPY`)NyDg|s4j0D=boD931O$OmU()6@rJ+cxpC=Tmf=Pk+)PB_ z4SWFZblt&Uwo(L@t!JM>tHU7;s-0}0r6m%#K=y651HL}*m}$=j&qcZ^X+GsF z3x*KdRfJLBsr0y{JmNQ|`<$o`31SC>A_c;^nXf zaWJwO|Gl5DBW6~+6rv9S?E6z4e{?Ml-tFb(kn}{{O#TsOKCE?=O6}Ynoe6?(a}Z!2 zMQNzdt20$7IaF#5Wn0^ebdS zHwFaaZ|EqZhh_ojSgDEPZQmw*RKh>808O#$Q<7(gIrc5XiVBZ zuA(S@Nf^#xV~jf}_u|!Lgl=+Du=XRD5;5j3pLYI$ejt$QUQuYP_tJ+^e?OY!$}n?X z%Fjv7X$%!srxmD5IK-Jvbl^RYbLVt$(&^}9v&}zFLf!r6fG){~nAvA&L0gII&QDU1 zM%@v1F5*wJYsL=G$N{o_c=h_bvs___T25T6fy2va5vt6uNO2{T-A^5Y*Cilc+ZDgK zuUXkpBy&Laf9||x$^o_Ft@Xx}xP<2RZ{y!%Nz``(ZqO}354O+;i_+acI4cCYznwfb zWV7^ZKqXXH0UA;*Al`S+H*hX$NcVDX6sGDCLK(*~lEjwgZb?5lSy=4&VEl-O7F z=ae)kk4|q3 zr(3ody{Id_FglS`d;VPQ$=ZDWYOG4bVL`1n-J+%|VG#I}!zo-(da{P+)${6=_evxw zmp(S)MT~X^xA>h6C=ETtZWebu7w>o`w8jRestZeO7n}|0~TuQ(2$Tgyfb5(>U`_#+?&eMmfff& zCb*y1i33lD+V{jzET?z@-CyYCEebu8{?Gzq1wCuGdm*|Z&EF49JCIrBO{d9-5k1$( z0d(vd!qrcRbKH;}GI|U}{J;d(-R^Ga3?puhB&DQDOv$s58_^`VeCt;w~>n@+d{ zc(V$G8m#Aph=S|p-t6cQ{+srFMAhU#(0)T_lAc=w#{+I?CPc{%HZOq8ru_XvFX!&3 zqos|gDI;H%@ku)jCkV~j#Bl;-B*SZrrmSELdE3fS2HGE3yXA7}h0w!E*LaR}%dZuHij9 zy!)02AL$gxHO^R<2;_+${(`>EUj(OpWc67OQCNAoZ{?%k$eu zUIvF&1h2!OZihe`ssZjg!C=|KGU7UYE8c3Az!?Kmo*b{uta$m&e(P)AMsR>7?AF8A zfIqlV#Z3rByxgTq_u>m=IMJ0J{>3QY?OD$ouXUn_@<8vhjiYB?mZ%lXt&0vr{!jq` zC`2vPEC%mx`?51$G@EG>h&gdz!5eu0Qcjdfi~A?LD4(y=zzi7oZH9_~+ZZ<}LU3V; zQa?E-D2+mB_mybFJGVFxEZ-G?B_^2KWpnH>)pmL5fe0+}W+n!7M1>n&6agcKYlo{^mh+!U^j;~K>a-&WO(@CL?jh&X) zMwLgyAcU3QvDB~|_&R;-)!p671{GlWuRp;T0L$6PLI4f{W@a{xVLkuxoZ--H2ub-V zp>P%${}&qy*gWQu2kKCut4UJ02TU`s32>~+N@E!(wiTqscH%V_>2#G|%`V zPzq_IR01cjFyT+XsEmwF>n{)gti@Q)Be<`2LCK-|dn6aU~ZQ8gC*HtFOy zwcwrmB!P%hltvWj;L5#Oy0vW;Pr-Ox;{}3apjb?cO%fWMj<@gSMqZ#C3neb9} zorQ%2Ltc90_tyO+P41uFqnY5qo)ADr4%5ZV7zpI+Rp%~vasL!#X@uUr5NWjFW9%;{ zA(;(=P|0EF{SbLC3F1dPwGd@?DwE{f_!B5?%vx8HlwfEvf56~LGiMTCAyM8 z#w-Y4YBxB&NK0Qq-p0;DNhV@Cua~w)v!QaEKN6Jwz@{;xyye}m|9tGEWS}|;i+i+p z`%Tb!M~6TEJ2g?RD``4gA0%SrYyjQ2lmiG<%-}|EW3#}^Nn}Jk2+#+>yqwj*@|pqL zB2%mv=|dLkX6$(olMBJ_SX&R^vl~KKdAqV*p8Z`j0aS}y$c*6Rk1D%t!?1yNDB*pK z_ZJ$pIELIY!B`7HHzRIPIY+>g58@a6yaO~S0t6mPG??pvpj$d`F`h0?eNc6Dln`}| zr{LhB4f`YtTM1;D64;C4XVmg)J_ZN+w94lOm#&lIXN(s3`uX=uf)g{i;oHZ1xA9Ex zenmG!{xGG*OX**wF&T6RAQ%+065P1qCnElYig2^gO`pBlyfDB+&1#h{kfFT?F!#mX z9!kvDBzJttH`JJbVyGX$lVNZlB4QMSH-D(2nKc;|=Oeqh3#7Nc3_OGJn#buNGAKj| zJQ8;)pKK_Q1Wu3&SP3dYECm6IXSBps<_V3}aZ;iL6Qs~^_!AAf7d4P0ANSvczc?%f z1*nNfLm*Vky6nE(fp37r%s;aUo)7;`E^P$jQDDbD<7<$j=q z=mVH>J^SQ^zq>3o-NNN$dVnGDZetJpff&}7?f~X~NrW=jz$jDG5uqBC{=K6J{_103 zMqbElz03rOq=BX)+y;kz>yj7mi18o)f(-D7r^S>7lm%$lr zky72Y$;O%^hl~txwR(ZS)+29!(OVczxFF@^6Rv-Lf`ZUS^}N@Yo|NuViInOaF;N(% zUfA=U&q;x|=iSDC{Nm~OI!{I)&GDbQau()BrVi(4@sq0Mzj^^+>rG+i?i3cM+ioN zuG#ydPSdSdQeQ}KJXgB&QV))AP6G{~1nO&W#8w$dEB~ViNZ%n(B!xQeAP8LZ{))6l zzMG)J6=^n9p<_j-#i|WT{U3I?xX&}_!&k5>C}Fjp=S|C>afU$gH2sb#T!z9%>agvU z?9JcO6-*xsEY({Mpj8X`A|P=Wkx_vs69VDz=@rjI)VTF*B;7JB!Kzr?<1X5Y1xI?+ zU+BZqiv5HO*V>o=wxR*-qx*Xf%L#)NC~Ro~nKAfU;llgT5M?#jvEs5EIv;ESNVKKA zQY#7|8{S1}s3r1!lrRSH(?@ObxyufZx!B6#ASm@i-!8g@LtAYE09V7tToA`ROn(i_ z-T_4Ffv%`UKoK2R)R@sr+==W5>NHK5$y()9y-D+_L~5wErpqAVkVkFbGOezcE?dfNb8zl6yYgf*pgHnH z3kXK_LZ}z8#d=&NWQaI;a~occf(5$r3t2L<$#Qd&HUKiysA7!FsS$knmb7RhK};n zTLbB^h09CCRz+a@zS?PwaLgz!S!_6e!^Y6Tosgq?EFHIdif+LD>E7^~3eMffS;;xT zjI{$g+K=sh?4fh$4{wv=@)kvwATv(&2hLdg6ZCT9z-h5}$C=A{xJ+YP4D`Bl!RxyqW{`83D-5Ga(y{fA#I)(|M=*!!FQB}jrNWWpov*19z; zj@viQVBuuEiF^G8126tCFR(VHC*{8cX0~tp9yyG5pf4|0JuVNwg5J;$z1tbeD4NRy zH`yQGaoN6IeQMKPidfvH@*giRpgD}gO(SF#EDffIUlo332Qe@p zCG%+tOWk)|Siu8RX>1hd%2HHSuxApIQc`JsTLq|1 z6U;c6lNXhX(f6`NPl)yS>HV&MEIoK^N;h+PcD6aJaTrk$mSz{RDB3o92tc-0kC9>* z{=kflj?5oOo@8b_sj0)qEOq%DEYbC0Z+V7L8a)J3NE-H9o$7vLS>j4U)kzk_a8i{9R!MTM|&(w&(g!sTdd}DicXiJz{X!tf+CeE+{0zEhACqy2uPOq@sb=7 zZtUNSptvva9+e#F4+UXw5?-d(Z5y`>04T~H#?U1QVuSH=adn~^ zM71eOX}bOAB6}qH=;zGHIGzO-PwnBnYX&xPGLrT^g!&85sCtF`yn#h9JYrSwi$&K* zQR9S;G7~tk&=0E5M~$U7NZCim4L|FU_Rz%j!g^jBHP+l!KdEhPo31ip{)Ji^P#sZm z8{QL}Tk}rw=(S_qi-;@Q1fkpM+TQSfeVdR_BM>+wo_`W#Kx(Sr{IW(m&2BklufCwy z>eg^gZH203fKPnER^^oA$d63!?Ac=ek`CbIOub~cYyGsiEn5C}hXA_f!(OLVlKnDP z)I2G?5?9FO-e_^lHY~^(&J>@5Gt71M^FH~C#sTta7MEcoju%1PqCH>W8+`hXxo&Uv z9$ffVvqSjMagHf0$tGCu)6ELi37y~ttVW_y_{cKWjUZ1FvgiFdx8^s{9lLr$uI}XL zJ>{0StdS*;#!J-7?&_z#0qDg|c);Q6r|J{^=$Fh}Uq6a;ziHSuKkm_BHYv#{Jb;gM z$$W=jjD02kM`xf4|D`T4g*7-Y;>Pr2}@eDd86i=qW1AYjcFdLiuH? z!)V;&aVaRQFy*>3@ZV9CF2wGkhfh4~CX1;YJScbG=e=yGXxMynvn35CmD;DZbmIF9 zmENZT&52oY_UWSXKXdOwNvDmB5FSCuOqT|p^gGN%O}z4kWJvre>7C&J=?|0Qk5l-8 z3x?NOp_)yWu~oHvp(CcOvpN`WQATy)XOm^v78JI{j8{aaT(vxJWBb5UtoyBhsPS)L zOvnED{bne71Ts=Rti(R51VFRkXpx%>-A)L%XY0#V2R0HUj1?<7Cgc(vx6UN(jf)HI zKbz1S7o(Yt975}R0F{F7-6LeWSs17L3C|E-YGyA`4UVTR%RzmGcn$dvC$4n-f70{A z|5EQ)D*i73Y4oE@VZxAKz5{NQa{PC!{AYqI*5O^l|NN!+pIkrnfT_jV)z+5rbzJ(` za`{*s^rI^E3@P^}0+vPc2@((6P%8R@OmWCigkd~eaJ3s(Lrj7BtFjZmN)Z}C7Q`k zqjFyz#*}d8<*CiU>X`=J!;iCZk{Gaj)kc6*w)dQhzgNTfn7Mwe(&_E(c>0w<}C>!B0N z!D;@V7O**3QTrYj`0z9%y3j2F)qV#SmwVN-lXbYp*ukzFnMz%(la!l-txD+AR*l)k z!vLtwGVJ1Y|FbNt-BYh05Rr-6A*pmMk@|wlz%tYFGQ40C_qis<6c!xLwL>&vIe$)8vcY;mHvD zzds;D+c+0Em&OtpbDD*1NY~y&TqfvVXaPg}I8=Oa+_Pw1_<1|dKM(u2%l^0p^07Sp zK@GgCDj~1UVz~9(!vBvePxm|P$HPmBgHB!i5ed(YFIylwswW)=H#3rlF0XETLd&u0$Y(QKt@p7ZPM6$$HNdejei$Zq+jf?Hn1Py4BztJhsARmzQ83Boh2%SKF8_g2u?7|H;h&S&mKDwl z115U?^Xh7m;`V|nV)3sNhc>-<>-5KE_OY_u!)aD3eLl4;{U%p)r-`* zv%qLg^=-s>?yiqmPFFIy`KD4Fn2>_}0UdA71k73R18$X9o zLB8$@#w5iiX62SfV7uO@30NH*+}zE`LJb<++mmNC>l2VPC`hS|thB?;iofeW6{=8W z8d5Ykt3r{+U!!0`wkvm3f#~_~@o*Fc33vqF>Tn$tf5-nhHM*M{2`;E6Zi)!^MmXmg zIG1yEttWdh1vY%Ktihkd+X@Eecq>7SrFZt7g~QABIwSX+9x(DN*$kJd^2HaI_~{ZE zr-6wMD2hmcyB+QL2qm#kJRGS9A^e%ePwAsXrp4cNdwb%4fARvyb=fDOt zYu$lj|Ech;hD?x7vHJX|ETR;BV-x8upjTjqBz_Pl%w=9a@mtVN7m_Jx)%`Yqu+ZB7 zm>GV~@6{dxUxbQvY z{!?#RT0B;C1!!HNVEBQLu8^=R6zpogDB>^O<|c(Fp5%VX6m>t4?8T9W z*dS*M4`Hf5h3`SiW>aq@j%N%SG*M+QsHvp z25F;KqS_Lq+%wSzfBUeQlM{QV?)R2thVk{6xiK#FNsrmoD8KDjFsLaS`kD3`jWbyd zhF(-i;|$t3Qsy`TDB0ozK>e?bmzZ&pQLEf}^8Owd-`1g?utKhz9n#qZ6nOii+FgY} z@JdR#>Q)Vc*}H)x!5)6skTNhi+_8=e4bxszY*0u{VJJqyE$4TIbkNRA;71 zaC`pxLTTQ2-Nu7!U8tVqv1acQup5Q-Y(vEpCWhC6Pi zYMlR`Pi3_y&rY7)LEZ8St@^nd1?DIdKZ~}RN;9Qtiz_VUfBtT#@4G=*eKDq0eItvb9XySyrPr-C{4%x@5wf_g5BYlQVR*loO2@>KcoZuS&5!v9$FTc_ zM{lBPm9~hY3nOOwyYB1MCg)u6aKyRTh;thpVSQY4?%VVT+PVqWIu#f5gZ=dALwcmA z`(YH()ZxXOIu1-o@*nGItv|hq4S647ZR~iWxb?UU{kna<`pNxh0q3HckBqR#9Pac! z;~-T?{m(bZ@fXjT7IVEG60xxeIM*H&sioF16TJboU$nh@mX3t%Nq(q5VZ3mMoDF{X zJR_0iiuV&A)|6~3q_lUXL`bbhVQm}3W##}mxRadKgk>UR7)sNVyE^*^875e}bZ+NN zN5W%$ZsHJE{I-?;@!~`Ay0{L|eV4~#Kdm@lv?_}PInaT1mb^j4y7yfV zEaQUC!r{~C*H!$y%*g3|K22y^%ERv&jQiu!E^|%LCwyBn>q67xt!g8G+#hMT18}}H z>?^5bn(iJX`}7e8A8}^4fE6jN9E|p?b-vpdQk+lPy5!BrTans1nLcudS&Y|ec0#Lk zi()K)>ZxovZf-Y*db6iT@y-nV>v0BUIKi@K~C>BL>zCyO%{V*m5EMGMP zm!*B4h^qYFhsyEP@nQ-H3Ij#;D!`|F3m;P0;oR&Uid;j{3g|C}P7R=v(o}IbFxivs zubmk8#iOPwMR|*3s0{raC3-aBT@NvZi~jWo*6GiwaDC2)Lh-A+>;}m0;*&dM@Pr~m_6^qhSpau2$a}jgf$WqTx#Zyxt+`dY@|CAAxlbT{fc~^c zNpba;W0aI0#`{N#&n*0Kq=aPZkK4L8f|DuXt-{`unucvdb-v|6?GkC94E+?>v=#%1 zl=DQzHN_maA-+Z0|53-4fJ4=`f0WlNWlcoo)oUq3R45WsvJ+ktbDA)AGee6ZdqQLx zWSto=*<&2Lu`dY~*_jz*8C$lolqH#xZ}fiOf4TnG|GJ-Zo$I;JbME^(=Xsvr?>X1G z??+#)lw*$!c$W9XT;t0MX8nU-b7E#~#C^!855UfaLKkiGs`n!7;i}y#=+c3x24+_9 zb-~X4H#XW3uVi^47V`EVxyOD0)RZ^EV}kz+h5tJOK^vmV`zt=-XQo?6EIYrK0PLXz z-(h$dbljWkpGbv&!}+3NmFl2KbL0N|vs^5Wxm1k4%n6Pxjp{eGIe_#Y)z787cPaq82q zsvdY%O$%KkGz|?mZnLuVqOm#Zg4*R}F{M9_|JZNwNS27-GFe~wdv!&*X~9*QA9YIS z@#~6E#EWLfw)I%ho#;uTUBq$QF@gmFbf2F?MtSuqb?LLDW;K#35D3@6DUpH(Jr%|t zw^+#<#^}W&)*o&KA^GhmLO|P7z<1?J2{h)D?|qF{a>SHaOeguVu{8?p4BxgluDvAl z153rv5{;jjzUHDbuK>Oq{A>VwnecxOHlSR1N^h*I^fmin}lZqCU;pLF|w&7@c^h2U*L{ z1@9&3_ODjVIK&S+UAZvlt=*|>^Nl2m=?h(TdiK!B(<0gufKuK!wJN#{MaRy+UDH_` z>GpkEN&N`bM9jD>-0^<#v!z_c85OSOinNXp&PuXKoqHL3=2y@FTPxF>I%;WUBmZ}3TU%#!R6<5FpR97%Ah zsO%&|JER9pYH}yjizxzedPACHs0{Dm6H1*6Ap-tFdER{BY25|oO&fC0p6vexw2LC*HWtYP^Dr) zyj%)(Il>JuOR>2tg!quXhZELIR3_0p7&%nG!AW|lKT>Z=6rvaD|0HO1~0o7!LI0`I>25oD%RsZY$_5r_*-oe zt-ekysuN17FF{9k+d&Q{_H7rbbf)I8yBXlmXVFu1am5HVE#LdsH?D_8|3ohpHG$83$80;D} zzo7R-1}xQf6gKw*Me@*lPp)orM#f}W3AofC_BF45?1%=}MV%^;56DJjcNMYgH1%t* z1ncovXT^oL4U=gLk(TlJo@@_#F+`0tx_nITVPFFjG+;ke z3ME7Y)o&Gm3RevlFCis7(2~Zb->J!i?#qUYB+}wl2S)Lj*Om(|>gA{AKYD0|Pzj;6 z^+H3p`L}%SvxVlvuZR@X$Y(;_#YqR3% zyTYp)tv2`iE>0&1AVVC>9Ht<1Uyor4bdIHotb`zYWn<%kMX9}Uu#Gdr4Nu*78UQ;7+(#)m_r*_P(UXpk*H!O z6LO?U3iNLs8STbWP=?vmvsuTG$4jqF+A)ryg=IqrsD4=PXpqzc_{DPH?F?H-6HWV5$HJc7 zSZKWNvK>koWk!X{@&!wMc)%N!0cyy`9FVZA%>mW^hrieCA*^()2jakP_+VzQtw?%I z!`Kn(fkc7nrw)1u6~04rXV+YK12ydwxq;$DKyXSd1VTVqJjxOh$AKK2|LJYkh5y<4 zEak7E@L!|h|0q~A<5?Ma(suZ4uplbjr(6BH!*9N=WKTup&I(Vc^1rhn;kKj0=Xyb+d}H^FhZi!?y)s!Pr=3jBP4m=I$R^OkrP^X; z7Al@!n2lgz+M0hnhtMbO6)O$iZ#(GgU0T)E0UnI*&f4tqZ3U7ah2ECejK;vjj{DUu z83|Rp!vhxW``xSk!fVOH=Z4OSk|!N84HT!0o{q{7b@E91mz&6ihOl9>%NLC)5mE|2 z{9NbK#5C0+{#ZpbxFaE#WS$8e;f^W2x0p$et3Pqgux?INCKh6-2}MwV5G7B zge%Rn_e)5ksjnrmGv(x#?QQ3B+nUj;;4hP1z9$r_mW_+Gus3bvRbx2cjfGQEyA1RmDulqPBUg^+I}pj#jeR$CIMg95K2i;S;0J>8^Pq)^xdo z1mc}l>&q?XA}}7(;zF$3DpLe_Yp>S&csvDarka8(ahDlh4ZC(?0~_k3c*hp6|3rMJ zl=pX!+_6`}9y)5{38pGLhO%%CIn2waavAMeJ>{JjV1#kM!Kj&<>Ny*j8+?Sfd9F^E z>|$~Uke09;`X16UXgFig;`MufPP-LWEqdyaFzhs4v5s;VCVd}vKJmTm-2^JnH)A&^ z{{W&M9QBzQTBX|G?!(q8*t* z_p6`-EoY?%=5z6b>S>KUphb^iMxF==Lx%Bw< z*y%gp4W0^AmRCyu%(&E3Lp$Dy?E1{@|OLKONeA`L~FxfR0j5o z4x>EhPeu4l*&njX{-aTGaYvjzBW}zaPQ9F*KYl+u_0=;g%r>~e0r5TGOK@qh6=XoJbhSu{Bo~B5#J2T++#ElDZlMchdKBz0fk54X~|6TiE7UtC0UaH z4tQ**|InlEN>_!I*bJ+^YwGrho6Hd}LF>v(Z3>bAZ2m?uN}%`FQl#*MWd~h;dpAgg zwv>+r-o2!*eGzebTuwHc01>N+NUj=~0!G@o4QAp2MBlPa-Vzk_l_UGkaazAz^ zSOK~s(P@-q?tSbi!PvUZS-{E4^~Fn`v#v5n1(oPHa_*b@qnbutWp1nQW~0{$mu?i0 z<06Mux=%!vhMlpw)n_CpecZ+;xtPm$ASwCN>%&gM*Ex9$lb{ks@@*V+!cIEQdf7hj zW>@`W{n<(Qqqt}&(Rp2C_$zPyy~W}D)ew3=GXnS{$Z=u%M}ha5l2zHVRI1%IdpO!3 z>p!5bulI;b5D;8E!s}@`ll!0*c4c}yQObDqc5ZcZwm_ZO?Q})|&{*gceUQl$|5G0jepbTT1cr^xK-ctE JiH_Z~e*xt@h?@Wa literal 0 HcmV?d00001 diff --git a/example/otel/uptrace.yml b/example/otel/uptrace.yml index 7c21e54f..9116d42d 100644 --- a/example/otel/uptrace.yml +++ b/example/otel/uptrace.yml @@ -13,6 +13,16 @@ ## foo: $$FOO_BAR ## +## +## ClickHouse database credentials. +## +ch: + # Connection string for ClickHouse database. For example: + # clickhouse://:@:/?sslmode=disable + # + # See https://clickhouse.uptrace.dev/guide/golang-clickhouse.html#options + dsn: 'clickhouse://default:@clickhouse:9000/uptrace?sslmode=disable' + ## ## A list of pre-configured projects. Each project is fully isolated. ## @@ -26,6 +36,10 @@ projects: - service.name - host.name - deployment.environment + # Group spans by deployment.environment attribute. + group_by_env: false + # Group funcs spans by service.name attribute. + group_funcs_by_service: false # Other projects can be used to monitor your applications. # To monitor micro-services or multiple related services, use a single project. @@ -36,6 +50,49 @@ projects: - service.name - host.name - deployment.environment + # Group spans by deployment.environment attribute. + group_by_env: false + # Group funcs spans by service.name attribute. + group_funcs_by_service: false + +## +## Create metrics from spans and events. +## +metrics_from_spans: + - name: uptrace.tracing.spans_duration + description: Spans duration (excluding events) + instrument: histogram + unit: microseconds + value: span.duration / 1000 + attrs: + - span.system as system + - service.name as service + - host.name as host + - span.status_code as status + where: not span.is_event + + - name: uptrace.tracing.spans + description: Spans count (excluding events) + instrument: counter + unit: 1 + value: span.count + attrs: + - span.system as system + - service.name as service + - host.name as host + - span.status_code as status + where: not span.is_event + + - name: uptrace.tracing.events + description: Events count (excluding spans) + instrument: counter + unit: 1 + value: span.count + attrs: + - span.system as system + - service.name as service + - host.name as host + where: span.is_event ## ## To require authentication, uncomment the following section. @@ -78,16 +135,6 @@ auth: # # Defaults to 'preferred_username'. # claim: preferred_username -## -## ClickHouse database credentials. -## -ch: - # Connection string for ClickHouse database. For example: - # clickhouse://:@:/?sslmode=disable - # - # See https://clickhouse.uptrace.dev/guide/golang-clickhouse.html#options - dsn: 'clickhouse://default:@clickhouse:9000/uptrace?sslmode=disable' - ## ## Alerting rules for monitoring metrics. ## @@ -102,8 +149,8 @@ alerting: - $net_errors > 0 group by host.name # for the last 5 minutes for: 5m - # in the project id=1 - projects: [1] + annotations: + summary: '{{ $labels.host_name }} has high number of net errors: {{ $values.net_errors }}' - name: Filesystem usage >= 90% metrics: @@ -114,7 +161,8 @@ alerting: - where device !~ "loop" - $fs_usage{state="used"} / $fs_usage >= 0.9 for: 5m - projects: [1] + annotations: + summary: '{{ $labels.host_name }} has high FS usage: {{ $values.fs_usage }}' - name: Uptrace is dropping spans metrics: @@ -122,7 +170,17 @@ alerting: query: - $spans{type=dropped} > 0 for: 1m - projects: [1] + annotations: + summary: 'Uptrace has dropped {{ $values.spans }} spans' + + - name: Always firing (for fun and testing) + metrics: + - process.runtime.go.goroutines as $goroutines + query: + - $goroutines >= 0 group by host.name + for: 1m + annotations: + summary: '{{ $labels.host_name }} has high number of goroutines: {{ $values.goroutines }}' # Create alerts from error logs and span events. create_alerts_from_spans: @@ -139,8 +197,8 @@ alerting: ## alertmanager_client: # AlertManager API endpoints that Uptrace uses to manage alerts. - # urls: - # - 'http://alertmanager:9093/api/v2/alerts' + urls: + - 'http://alertmanager:9093/api/v2/alerts' ## ## Various options to tweak ClickHouse schema.