mirror of
https://github.com/cs3org/wopiserver.git
synced 2025-04-18 13:04:00 +03:00
Moved monitoring scripts to a separate repo
This commit is contained in:
Giuseppe Lo Presti
parent
0ec21d7c0d
commit
54a3891d0f
2
Makefile
2
Makefile
@ -1,4 +1,4 @@
|
||||
FILES_TO_RPM = src mon tools wopiserver.conf wopiserver.service wopiserver.logrotate
|
||||
FILES_TO_RPM = src tools wopiserver.conf wopiserver.service wopiserver.logrotate
|
||||
SPECFILE = $(shell find . -type f -name *.spec)
|
||||
VERSREL = $(shell git describe | sed 's/^v//')
|
||||
VERSION = $(shell echo ${VERSREL} | cut -d\- -f 1)
|
||||
|
||||
@ -55,7 +55,6 @@ install -m 644 src/cs3iface.py %buildroot/%_python_lib/cs3iface.py
|
||||
install -m 644 wopiserver.service %buildroot/usr/lib/systemd/system/wopiserver.service
|
||||
install -m 644 wopiserver.conf %buildroot/etc/wopi/wopiserver.defaults.conf
|
||||
install -m 644 wopiserver.logrotate %buildroot/etc/logrotate.d/cernbox-wopi-server
|
||||
install -m 755 mon/wopi_grafana_feeder.py %buildroot/usr/bin/wopi_grafana_feeder.py
|
||||
install -m 755 tools/wopicheckfile.py %buildroot/usr/bin/wopicheckfile.py
|
||||
install -m 755 tools/wopilistopenfiles.sh %buildroot/usr/bin/wopilistopenfiles.sh
|
||||
install -m 755 tools/wopiopen.py %buildroot/usr/bin/wopiopen.py
|
||||
|
||||
@ -1,162 +0,0 @@
|
||||
#!/usr/bin/python
|
||||
'''
|
||||
wopi_grafana_feeder.py
|
||||
|
||||
A daemon pushing CERNBox WOPI monitoring data to Grafana.
|
||||
TODO: make it a collectd plugin. References:
|
||||
https://collectd.org/documentation/manpages/collectd-python.5.shtml
|
||||
https://blog.dbrgn.ch/2017/3/10/write-a-collectd-python-plugin/
|
||||
https://github.com/dbrgn/collectd-python-plugins
|
||||
|
||||
author: Giuseppe.LoPresti@cern.ch
|
||||
CERN/IT-ST
|
||||
'''
|
||||
|
||||
import fileinput
|
||||
import socket
|
||||
import time
|
||||
import pickle
|
||||
import struct
|
||||
import datetime
|
||||
import getopt
|
||||
import sys
|
||||
|
||||
CARBON_TCPPORT = 2004
|
||||
carbonHost = ''
|
||||
verbose = False
|
||||
prefix = 'cernbox.wopi.' + socket.gethostname().split('.')[0]
|
||||
epoch = datetime.datetime(1970, 1, 1)
|
||||
|
||||
|
||||
def usage(exitCode):
|
||||
'''prints usage'''
|
||||
print 'Usage : cat <logfile> | ' + sys.argv[0] + ' [-h|--help] -g|--grafanahost <hostname>'
|
||||
sys.exit(exitCode)
|
||||
|
||||
def send_metric(data):
|
||||
'''send data to grafana using the pickle protocol'''
|
||||
payload = pickle.dumps(data, protocol=2)
|
||||
header = struct.pack("!L", len(payload))
|
||||
message = header + payload
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
sock.connect((carbonHost, CARBON_TCPPORT))
|
||||
sock.sendall(message)
|
||||
sock.close()
|
||||
|
||||
def get_wopi_metrics(data):
|
||||
'''Parse WOPI usage metrics'''
|
||||
for line in data:
|
||||
if data.isfirstline():
|
||||
logdate = line.split('T')[0].split('-') # keeps the date until 'T', splits
|
||||
timestamp = (datetime.datetime(int(logdate[0]), int(logdate[1]), int(logdate[2]), 1, 0, 0) - epoch).total_seconds() + time.altzone
|
||||
errors = 0
|
||||
users = {}
|
||||
openfiles = {}
|
||||
openfiles['docx'] = {}
|
||||
openfiles['xlsx'] = {}
|
||||
openfiles['pptx'] = {}
|
||||
openfiles['odt'] = {}
|
||||
openfiles['ods'] = {}
|
||||
openfiles['odp'] = {}
|
||||
openfiles['md'] = {}
|
||||
openfiles['zmd'] = {}
|
||||
openfiles['txt'] = {}
|
||||
wrfiles = {}
|
||||
wrfiles['docx'] = {}
|
||||
wrfiles['xlsx'] = {}
|
||||
wrfiles['pptx'] = {}
|
||||
wrfiles['odt'] = {}
|
||||
wrfiles['ods'] = {}
|
||||
wrfiles['odp'] = {}
|
||||
wrfiles['md'] = {}
|
||||
wrfiles['zmd'] = {}
|
||||
wrfiles['txt'] = {}
|
||||
collab = 0
|
||||
try:
|
||||
if ' ERROR ' in line:
|
||||
errors += 1
|
||||
# all opened files
|
||||
elif 'CheckFileInfo' in line:
|
||||
# count of unique users
|
||||
l = line.split()
|
||||
u = l[4].split('=')[1]
|
||||
if u in users.keys():
|
||||
users[u] += 1
|
||||
else:
|
||||
users[u] = 1
|
||||
# count of open files per type: look for the file extension
|
||||
fname = line[line.find('filename=')+10:line.rfind('fileid=')-2]
|
||||
fext = fname[fname.rfind('.')+1:]
|
||||
if fext not in openfiles:
|
||||
openfiles[fext] = {}
|
||||
if fname in openfiles[fext]:
|
||||
openfiles[fext][fname] += 1
|
||||
else:
|
||||
openfiles[fext][fname] = 1
|
||||
# files opened for write
|
||||
elif 'successfully written' in line:
|
||||
# count of written files
|
||||
fname = line[line.find('filename=')+10:line.rfind('token=')-2]
|
||||
fext = fname[fname.rfind('.')+1:]
|
||||
if fname in wrfiles[fext]:
|
||||
wrfiles[fext][fname] += 1
|
||||
else:
|
||||
wrfiles[fext][fname] = 1
|
||||
# collaborative editing sessions
|
||||
elif 'Collaborative editing detected' in line:
|
||||
collab += 1
|
||||
# we could extract the filename and the users list for further statistics
|
||||
except Exception:
|
||||
if verbose:
|
||||
print 'Error occurred at line: %s' % line
|
||||
raise
|
||||
|
||||
if 'timestamp' not in locals():
|
||||
# the file was empty, nothing to do
|
||||
return
|
||||
# prepare data for grafana
|
||||
output = []
|
||||
output.append(( prefix + '.errors', (int(timestamp), errors) ))
|
||||
output.append(( prefix + '.users', (int(timestamp), len(users)) ))
|
||||
# get the top user by sorting the users dict by values instead of by keys
|
||||
if len(users) > 0:
|
||||
top = sorted(users.iteritems(), key=lambda (k, v): (v, k))[-1][1]
|
||||
output.append(( prefix + '.topuser', (int(timestamp), int(top)) ))
|
||||
for fext in openfiles:
|
||||
output.append(( prefix + '.openfiles.' + fext, (int(timestamp), len(openfiles[fext])) ))
|
||||
for fext in wrfiles:
|
||||
output.append(( prefix + '.writtenfiles.' + fext, (int(timestamp), len(wrfiles[fext])) ))
|
||||
output.append(( prefix + '.collab', (int(timestamp), collab) ))
|
||||
# send and print all collected data
|
||||
send_metric(output)
|
||||
if verbose:
|
||||
print output
|
||||
|
||||
|
||||
# first parse options
|
||||
try:
|
||||
options, args = getopt.getopt(sys.argv[1:], 'hvg:', ['help', 'verbose', 'grafanahost'])
|
||||
except Exception, e:
|
||||
print e
|
||||
usage(1)
|
||||
for f, v in options:
|
||||
if f == '-h' or f == '--help':
|
||||
usage(0)
|
||||
elif f == '-v' or f == '--verbose':
|
||||
verbose = True
|
||||
elif f == '-g' or f == '--grafanahost':
|
||||
carbonHost = v
|
||||
else:
|
||||
print "unknown option : " + f
|
||||
usage(1)
|
||||
if carbonHost == '':
|
||||
print 'grafanahost option is mandatory'
|
||||
usage(1)
|
||||
# now parse input and collect statistics
|
||||
try:
|
||||
get_wopi_metrics(fileinput.input('-'))
|
||||
except Exception, e:
|
||||
print 'Error with collecting metrics:', e
|
||||
if verbose:
|
||||
raise
|
||||
|
||||
@ -1,112 +0,0 @@
|
||||
#!/usr/bin/python
|
||||
'''
|
||||
wopi_max_concurrency.py
|
||||
|
||||
A daemon pushing CERNBox WOPI monitoring data to Grafana.
|
||||
TODO: make it a collectd plugin. References:
|
||||
https://collectd.org/documentation/manpages/collectd-python.5.shtml
|
||||
https://blog.dbrgn.ch/2017/3/10/write-a-collectd-python-plugin/
|
||||
https://github.com/dbrgn/collectd-python-plugins
|
||||
|
||||
author: Giuseppe.LoPresti@cern.ch
|
||||
CERN/IT-ST
|
||||
'''
|
||||
|
||||
import fileinput
|
||||
import socket
|
||||
import time
|
||||
import pickle
|
||||
import struct
|
||||
import datetime
|
||||
import getopt
|
||||
import sys
|
||||
|
||||
CARBON_TCPPORT = 2004
|
||||
carbonHost = ''
|
||||
verbose = False
|
||||
prefix = 'cernbox.wopi.' + socket.gethostname().split('.')[0]
|
||||
epoch = datetime.datetime(1970, 1, 1)
|
||||
|
||||
|
||||
def usage(exitCode):
|
||||
'''prints usage'''
|
||||
print 'Usage : cat <logfile> | ' + sys.argv[0] + ' [-h|--help] -g|--grafanahost <hostname>'
|
||||
sys.exit(exitCode)
|
||||
|
||||
def send_metric(data):
|
||||
'''send data to grafana using the pickle protocol'''
|
||||
payload = pickle.dumps(data, protocol=2)
|
||||
header = struct.pack("!L", len(payload))
|
||||
message = header + payload
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
sock.connect((carbonHost, CARBON_TCPPORT))
|
||||
sock.sendall(message)
|
||||
sock.close()
|
||||
|
||||
def get_wopi_metrics(data):
|
||||
'''Parse WOPI usage metrics'''
|
||||
for line in data:
|
||||
if data.isfirstline():
|
||||
logdate = line.split('T')[0].split('-') # keeps the date until 'T', splits
|
||||
timestamp = (datetime.datetime(int(logdate[0]), int(logdate[1]), int(logdate[2]), 1, 0, 0) - epoch).total_seconds() + time.altzone
|
||||
maxconc = 0
|
||||
tokens = set()
|
||||
try:
|
||||
if 'msg="Lock"' in line and 'INFO' in line and 'result' not in line:
|
||||
# +1 for this acc. token
|
||||
l = line.split()
|
||||
tok = l[-1].split('=')[1]
|
||||
tokens.add(tok)
|
||||
if len(tokens) > maxconc:
|
||||
maxconc += 1
|
||||
if 'msg="Unlock"' in line and 'INFO' in line:
|
||||
# -1 for this acc. token
|
||||
l = line.split()
|
||||
tok = l[-1].split('=')[1]
|
||||
try:
|
||||
tokens.remove(tok)
|
||||
except KeyError:
|
||||
pass
|
||||
except Exception:
|
||||
if verbose:
|
||||
print 'Error occurred at line: %s' % line
|
||||
raise
|
||||
|
||||
if 'tok' not in locals():
|
||||
# the file was empty, nothing to do
|
||||
return
|
||||
# prepare data for grafana
|
||||
output = []
|
||||
output.append(( prefix + '.maxconc', (int(timestamp), maxconc) ))
|
||||
send_metric(output)
|
||||
if verbose:
|
||||
print output
|
||||
|
||||
|
||||
# first parse options
|
||||
try:
|
||||
options, args = getopt.getopt(sys.argv[1:], 'hvg:', ['help', 'verbose', 'grafanahost'])
|
||||
except Exception, e:
|
||||
print e
|
||||
usage(1)
|
||||
for f, v in options:
|
||||
if f == '-h' or f == '--help':
|
||||
usage(0)
|
||||
elif f == '-v' or f == '--verbose':
|
||||
verbose = True
|
||||
elif f == '-g' or f == '--grafanahost':
|
||||
carbonHost = v
|
||||
else:
|
||||
print "unknown option : " + f
|
||||
usage(1)
|
||||
if carbonHost == '':
|
||||
print 'grafanahost option is mandatory'
|
||||
usage(1)
|
||||
# now parse input and collect statistics
|
||||
try:
|
||||
get_wopi_metrics(fileinput.input('-'))
|
||||
except Exception, e:
|
||||
print 'Error with collecting metrics:', e
|
||||
if verbose:
|
||||
raise
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user