diff --git a/Makefile b/Makefile index 590aa17..8bf09cd 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -FILES_TO_RPM = src mon tools wopiserver.conf wopiserver.service wopiserver.logrotate +FILES_TO_RPM = src tools wopiserver.conf wopiserver.service wopiserver.logrotate SPECFILE = $(shell find . -type f -name *.spec) VERSREL = $(shell git describe | sed 's/^v//') VERSION = $(shell echo ${VERSREL} | cut -d\- -f 1) diff --git a/cernbox-wopi-server.spec b/cernbox-wopi-server.spec index 07da8bf..03ceb5c 100644 --- a/cernbox-wopi-server.spec +++ b/cernbox-wopi-server.spec @@ -55,7 +55,6 @@ install -m 644 src/cs3iface.py %buildroot/%_python_lib/cs3iface.py install -m 644 wopiserver.service %buildroot/usr/lib/systemd/system/wopiserver.service install -m 644 wopiserver.conf %buildroot/etc/wopi/wopiserver.defaults.conf install -m 644 wopiserver.logrotate %buildroot/etc/logrotate.d/cernbox-wopi-server -install -m 755 mon/wopi_grafana_feeder.py %buildroot/usr/bin/wopi_grafana_feeder.py install -m 755 tools/wopicheckfile.py %buildroot/usr/bin/wopicheckfile.py install -m 755 tools/wopilistopenfiles.sh %buildroot/usr/bin/wopilistopenfiles.sh install -m 755 tools/wopiopen.py %buildroot/usr/bin/wopiopen.py diff --git a/mon/wopi_grafana_feeder.py2 b/mon/wopi_grafana_feeder.py2 deleted file mode 100755 index 2fbd1f0..0000000 --- a/mon/wopi_grafana_feeder.py2 +++ /dev/null @@ -1,162 +0,0 @@ -#!/usr/bin/python -''' -wopi_grafana_feeder.py - -A daemon pushing CERNBox WOPI monitoring data to Grafana. -TODO: make it a collectd plugin. References: -https://collectd.org/documentation/manpages/collectd-python.5.shtml -https://blog.dbrgn.ch/2017/3/10/write-a-collectd-python-plugin/ -https://github.com/dbrgn/collectd-python-plugins - -author: Giuseppe.LoPresti@cern.ch -CERN/IT-ST -''' - -import fileinput -import socket -import time -import pickle -import struct -import datetime -import getopt -import sys - -CARBON_TCPPORT = 2004 -carbonHost = '' -verbose = False -prefix = 'cernbox.wopi.' + socket.gethostname().split('.')[0] -epoch = datetime.datetime(1970, 1, 1) - - -def usage(exitCode): - '''prints usage''' - print 'Usage : cat | ' + sys.argv[0] + ' [-h|--help] -g|--grafanahost ' - sys.exit(exitCode) - -def send_metric(data): - '''send data to grafana using the pickle protocol''' - payload = pickle.dumps(data, protocol=2) - header = struct.pack("!L", len(payload)) - message = header + payload - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.connect((carbonHost, CARBON_TCPPORT)) - sock.sendall(message) - sock.close() - -def get_wopi_metrics(data): - '''Parse WOPI usage metrics''' - for line in data: - if data.isfirstline(): - logdate = line.split('T')[0].split('-') # keeps the date until 'T', splits - timestamp = (datetime.datetime(int(logdate[0]), int(logdate[1]), int(logdate[2]), 1, 0, 0) - epoch).total_seconds() + time.altzone - errors = 0 - users = {} - openfiles = {} - openfiles['docx'] = {} - openfiles['xlsx'] = {} - openfiles['pptx'] = {} - openfiles['odt'] = {} - openfiles['ods'] = {} - openfiles['odp'] = {} - openfiles['md'] = {} - openfiles['zmd'] = {} - openfiles['txt'] = {} - wrfiles = {} - wrfiles['docx'] = {} - wrfiles['xlsx'] = {} - wrfiles['pptx'] = {} - wrfiles['odt'] = {} - wrfiles['ods'] = {} - wrfiles['odp'] = {} - wrfiles['md'] = {} - wrfiles['zmd'] = {} - wrfiles['txt'] = {} - collab = 0 - try: - if ' ERROR ' in line: - errors += 1 - # all opened files - elif 'CheckFileInfo' in line: - # count of unique users - l = line.split() - u = l[4].split('=')[1] - if u in users.keys(): - users[u] += 1 - else: - users[u] = 1 - # count of open files per type: look for the file extension - fname = line[line.find('filename=')+10:line.rfind('fileid=')-2] - fext = fname[fname.rfind('.')+1:] - if fext not in openfiles: - openfiles[fext] = {} - if fname in openfiles[fext]: - openfiles[fext][fname] += 1 - else: - openfiles[fext][fname] = 1 - # files opened for write - elif 'successfully written' in line: - # count of written files - fname = line[line.find('filename=')+10:line.rfind('token=')-2] - fext = fname[fname.rfind('.')+1:] - if fname in wrfiles[fext]: - wrfiles[fext][fname] += 1 - else: - wrfiles[fext][fname] = 1 - # collaborative editing sessions - elif 'Collaborative editing detected' in line: - collab += 1 - # we could extract the filename and the users list for further statistics - except Exception: - if verbose: - print 'Error occurred at line: %s' % line - raise - - if 'timestamp' not in locals(): - # the file was empty, nothing to do - return - # prepare data for grafana - output = [] - output.append(( prefix + '.errors', (int(timestamp), errors) )) - output.append(( prefix + '.users', (int(timestamp), len(users)) )) - # get the top user by sorting the users dict by values instead of by keys - if len(users) > 0: - top = sorted(users.iteritems(), key=lambda (k, v): (v, k))[-1][1] - output.append(( prefix + '.topuser', (int(timestamp), int(top)) )) - for fext in openfiles: - output.append(( prefix + '.openfiles.' + fext, (int(timestamp), len(openfiles[fext])) )) - for fext in wrfiles: - output.append(( prefix + '.writtenfiles.' + fext, (int(timestamp), len(wrfiles[fext])) )) - output.append(( prefix + '.collab', (int(timestamp), collab) )) - # send and print all collected data - send_metric(output) - if verbose: - print output - - -# first parse options -try: - options, args = getopt.getopt(sys.argv[1:], 'hvg:', ['help', 'verbose', 'grafanahost']) -except Exception, e: - print e - usage(1) -for f, v in options: - if f == '-h' or f == '--help': - usage(0) - elif f == '-v' or f == '--verbose': - verbose = True - elif f == '-g' or f == '--grafanahost': - carbonHost = v - else: - print "unknown option : " + f - usage(1) -if carbonHost == '': - print 'grafanahost option is mandatory' - usage(1) -# now parse input and collect statistics -try: - get_wopi_metrics(fileinput.input('-')) -except Exception, e: - print 'Error with collecting metrics:', e - if verbose: - raise - diff --git a/mon/wopi_max_concurrency.py2 b/mon/wopi_max_concurrency.py2 deleted file mode 100755 index 1f74199..0000000 --- a/mon/wopi_max_concurrency.py2 +++ /dev/null @@ -1,112 +0,0 @@ -#!/usr/bin/python -''' -wopi_max_concurrency.py - -A daemon pushing CERNBox WOPI monitoring data to Grafana. -TODO: make it a collectd plugin. References: -https://collectd.org/documentation/manpages/collectd-python.5.shtml -https://blog.dbrgn.ch/2017/3/10/write-a-collectd-python-plugin/ -https://github.com/dbrgn/collectd-python-plugins - -author: Giuseppe.LoPresti@cern.ch -CERN/IT-ST -''' - -import fileinput -import socket -import time -import pickle -import struct -import datetime -import getopt -import sys - -CARBON_TCPPORT = 2004 -carbonHost = '' -verbose = False -prefix = 'cernbox.wopi.' + socket.gethostname().split('.')[0] -epoch = datetime.datetime(1970, 1, 1) - - -def usage(exitCode): - '''prints usage''' - print 'Usage : cat | ' + sys.argv[0] + ' [-h|--help] -g|--grafanahost ' - sys.exit(exitCode) - -def send_metric(data): - '''send data to grafana using the pickle protocol''' - payload = pickle.dumps(data, protocol=2) - header = struct.pack("!L", len(payload)) - message = header + payload - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.connect((carbonHost, CARBON_TCPPORT)) - sock.sendall(message) - sock.close() - -def get_wopi_metrics(data): - '''Parse WOPI usage metrics''' - for line in data: - if data.isfirstline(): - logdate = line.split('T')[0].split('-') # keeps the date until 'T', splits - timestamp = (datetime.datetime(int(logdate[0]), int(logdate[1]), int(logdate[2]), 1, 0, 0) - epoch).total_seconds() + time.altzone - maxconc = 0 - tokens = set() - try: - if 'msg="Lock"' in line and 'INFO' in line and 'result' not in line: - # +1 for this acc. token - l = line.split() - tok = l[-1].split('=')[1] - tokens.add(tok) - if len(tokens) > maxconc: - maxconc += 1 - if 'msg="Unlock"' in line and 'INFO' in line: - # -1 for this acc. token - l = line.split() - tok = l[-1].split('=')[1] - try: - tokens.remove(tok) - except KeyError: - pass - except Exception: - if verbose: - print 'Error occurred at line: %s' % line - raise - - if 'tok' not in locals(): - # the file was empty, nothing to do - return - # prepare data for grafana - output = [] - output.append(( prefix + '.maxconc', (int(timestamp), maxconc) )) - send_metric(output) - if verbose: - print output - - -# first parse options -try: - options, args = getopt.getopt(sys.argv[1:], 'hvg:', ['help', 'verbose', 'grafanahost']) -except Exception, e: - print e - usage(1) -for f, v in options: - if f == '-h' or f == '--help': - usage(0) - elif f == '-v' or f == '--verbose': - verbose = True - elif f == '-g' or f == '--grafanahost': - carbonHost = v - else: - print "unknown option : " + f - usage(1) -if carbonHost == '': - print 'grafanahost option is mandatory' - usage(1) -# now parse input and collect statistics -try: - get_wopi_metrics(fileinput.input('-')) -except Exception, e: - print 'Error with collecting metrics:', e - if verbose: - raise -