589 lines
22 KiB
Python
589 lines
22 KiB
Python
|
#!/usr/bin/env python
|
||
|
|
#
|
||
|
|
# (c)2011-2012 the Boeing Company
|
||
|
|
#
|
||
|
|
# perfmon.py - CORE server and node performace metrics logger and alarmer
|
||
|
|
# server metrics: loadave1, 5, 15, mem, used cpu% of total, cpu1, cpu2, ..., cpun
|
||
|
|
# node metrics: throughput, mem, cpu total, usr, sys, wait
|
||
|
|
#
|
||
|
|
import os, sys, time, re, optparse, signal, commands, pdb
|
||
|
|
|
||
|
|
def readfile(fname):
|
||
|
|
lines=[]
|
||
|
|
try:
|
||
|
|
f = open(fname, "r")
|
||
|
|
except:
|
||
|
|
if options.timestamp == True:
|
||
|
|
print str(time.time()),
|
||
|
|
print "ERROR: failed to open file %s\n" % fname
|
||
|
|
else :
|
||
|
|
lines = f.readlines()
|
||
|
|
f.close()
|
||
|
|
return lines
|
||
|
|
|
||
|
|
def numcpus():
|
||
|
|
lines = readfile("/proc/stat")
|
||
|
|
n = 0
|
||
|
|
for l in lines[1:]:
|
||
|
|
if l[:3] != "cpu":
|
||
|
|
break
|
||
|
|
n += 1
|
||
|
|
return n
|
||
|
|
|
||
|
|
def handler(signum, frame):
|
||
|
|
print "stop timestamp:", str(time.time()) + ", cyclecount=", cyclecount, ", caught signal", signum
|
||
|
|
sys.exit(0)
|
||
|
|
|
||
|
|
class ServerMetrics(object):
|
||
|
|
def __init__(self):
|
||
|
|
self.smetrics = { "serverloadavg1" : 0.0,
|
||
|
|
"serverloadavg5" : 0.0,
|
||
|
|
"serverloadavg15" : 0.0,
|
||
|
|
"serverusedmemory" : 0.0,
|
||
|
|
"serverusedcputime" : 0.0,
|
||
|
|
"processorusedcputime" : [] }
|
||
|
|
|
||
|
|
# set values from val = (nump, ldavg1, ldavg5, adavg15, mem, cpu, p1cpu, p2cpu...)
|
||
|
|
def setvalues(self, val):
|
||
|
|
self.smetrics["serverloadavg1"] = val[0]
|
||
|
|
self.smetrics["serverloadavg5"] = val[1]
|
||
|
|
self.smetrics["serverloadavg15"] = val[2]
|
||
|
|
self.smetrics["serverusedmemory"] = val[4]
|
||
|
|
self.smetrics["serverusedcputime"] = val[5]
|
||
|
|
#print self.smetrics.keys(), self.smetrics.values()
|
||
|
|
|
||
|
|
pcpu = []
|
||
|
|
for ind in range(5,len(val)):
|
||
|
|
pcpu.append(val[ind])
|
||
|
|
# print "[" + ",".join(map(lambda(x):str(round(x, 2)), pcpu)) +"]"
|
||
|
|
|
||
|
|
self.smetrics["processorusedcputime"] = pcpu
|
||
|
|
#print self.smetrics.keys(), self.smetrics.values()
|
||
|
|
|
||
|
|
def setvalue(self, key, val):
|
||
|
|
self.smetrics[key] = val
|
||
|
|
|
||
|
|
def getvalue(self, key):
|
||
|
|
return self.smetrics[key]
|
||
|
|
|
||
|
|
def getkeys(self):
|
||
|
|
return self.smetrics.keys()
|
||
|
|
|
||
|
|
def tocsv(self):
|
||
|
|
rv = "Server"
|
||
|
|
for k in self.smetrics:
|
||
|
|
# print k, self.smetrics[k]
|
||
|
|
if isinstance(self.smetrics[k], float):
|
||
|
|
rv += ", %.2f" % self.smetrics[k]
|
||
|
|
else:
|
||
|
|
if isinstance(self.smetrics[k], list):
|
||
|
|
rv += ", [" + \
|
||
|
|
", ".join(map(lambda(x):str(round(x, 2)), self.smetrics[k])) \
|
||
|
|
+ "]"
|
||
|
|
else:
|
||
|
|
rv += ", " + str(self.smetrics[k])
|
||
|
|
return rv
|
||
|
|
|
||
|
|
def readserverthresholds(filename):
|
||
|
|
if filename is None:
|
||
|
|
return
|
||
|
|
|
||
|
|
lines = readfile(filename)
|
||
|
|
for l in lines:
|
||
|
|
mval = l.strip().split('=')
|
||
|
|
#print "read line %s" % mval
|
||
|
|
if len(mval) > 1 :
|
||
|
|
thekey = mval[0].strip()
|
||
|
|
theval = mval[1].strip()
|
||
|
|
if thekey in serverthresholds.getkeys():
|
||
|
|
serverthresholds.setvalue(thekey, float(theval))
|
||
|
|
# print thekey," = %.2f" % float(theval)
|
||
|
|
|
||
|
|
def checkserverthreshold(metricval):
|
||
|
|
# print out an alarm if a ServerMetrics value crosses threshold
|
||
|
|
for key in serverthresholds.getkeys():
|
||
|
|
# print "checking threshold of key = ", key
|
||
|
|
if key == "processorusedcputime":
|
||
|
|
pcpus = metricval.getvalue(key)
|
||
|
|
# print key, pcpus, serverthresholds[key]
|
||
|
|
for ind in range(0, len(pcpus)):
|
||
|
|
# print ind, pcpus[ind]
|
||
|
|
if pcpus[ind] > serverthresholds.getvalue(key):
|
||
|
|
alarm = ["server", os.uname()[1], str(ind) + key,
|
||
|
|
"%.2f" % pcpus[ind], ">", serverthresholds.getvalue(key)]
|
||
|
|
if options.timestamp:
|
||
|
|
print str(time.time()) + ",",
|
||
|
|
print ", ".join(map(lambda(x):str(x), alarm))
|
||
|
|
else:
|
||
|
|
if metricval.getvalue(key) > serverthresholds.getvalue(key):
|
||
|
|
alarm = ["server", os.uname()[1], key,
|
||
|
|
"%.2f" % metricval.getvalue(key), ">", serverthresholds.getvalue(key)]
|
||
|
|
if options.timestamp:
|
||
|
|
print str(time.time()) + ",",
|
||
|
|
print ", ".join(map(lambda(x):str(x), alarm))
|
||
|
|
|
||
|
|
def collectservercputimes():
|
||
|
|
# return cpu times in ticks of this server total and each processor 3*(1+#cpu) columns
|
||
|
|
# (user+nice, sys, idle) from each /proc/stat cpu lines assume columns are:
|
||
|
|
# cpu# user nice sys idle iowait irq softirq steal guest (man 5 proc)
|
||
|
|
rval = {}
|
||
|
|
lines = readfile("/proc/stat")
|
||
|
|
for i in range(ncpus + 1):
|
||
|
|
items = lines[i].split()
|
||
|
|
(user, nice, sys, idle) = map(lambda(x): int(x), items[1:5])
|
||
|
|
rval[i] = [user+nice, sys, idle]
|
||
|
|
return rval
|
||
|
|
|
||
|
|
def csvservercputimes(cputimes):
|
||
|
|
# return a csv string of this server total and each processor's cpu times
|
||
|
|
# (usr, sys, idle) in ticks
|
||
|
|
rval = ''
|
||
|
|
for i in range(len(cputimes)):
|
||
|
|
rval += ', '.join(map(lambda(x):str(x), cputimes[i]))
|
||
|
|
return rval
|
||
|
|
|
||
|
|
def calcservercputimes(cputimea, cputimeb):
|
||
|
|
# return cpu used/total % of this server total and each processor (1+#cpu columns)
|
||
|
|
p = {}
|
||
|
|
for n in range(ncpus + 1):
|
||
|
|
# print cputimeb[n]
|
||
|
|
p[n] = []
|
||
|
|
for i in range(len(cputimea[n])):
|
||
|
|
p[n].append(cputimeb[n][i] - cputimea[n][i])
|
||
|
|
# print p[n]
|
||
|
|
total = sum(p[n]) # cpu times total delta
|
||
|
|
# print total
|
||
|
|
if total == 0:
|
||
|
|
p[n] = 0.0
|
||
|
|
else:
|
||
|
|
p[n] = 100 - ((100.0 * p[n][-1]) / total)
|
||
|
|
return p
|
||
|
|
|
||
|
|
def collectservermems():
|
||
|
|
# return memory (total, free) in KB from proc/meminfo
|
||
|
|
lines = readfile("/proc/meminfo")
|
||
|
|
mem = map(lambda(x):x.split(), lines[0:2])
|
||
|
|
return map(lambda(x):int(x), zip(*mem)[1])
|
||
|
|
|
||
|
|
def csvservermems(mems):
|
||
|
|
# return a csv string of this server memory (total, free)
|
||
|
|
return ", ".join(map(lambda x: str(x), mems))
|
||
|
|
|
||
|
|
def calcserverusedmem(mems):
|
||
|
|
# return int(100*(MemTotal-MemFree)/MemTotal) from /proc/meminfo
|
||
|
|
return 100 * (mems[0] - mems[1]) / mems[0]
|
||
|
|
|
||
|
|
def collectservermetrics(cputimes, mems, thresholdcheck):
|
||
|
|
# return ServerMetrics object with a dictionary of
|
||
|
|
# loadavg1,loadavg5,loadavg15, usedmem%, usedcpu% for total, cpu1, cpu2, ...
|
||
|
|
metricval = []
|
||
|
|
ldavgs=os.getloadavg()
|
||
|
|
for v in ldavgs:
|
||
|
|
metricval.append(v)
|
||
|
|
metricval.append(calcserverusedmem(mems))
|
||
|
|
|
||
|
|
for i in range(ncpus + 1):
|
||
|
|
metricval.append(cputimes[i])
|
||
|
|
# print cputimes[i]
|
||
|
|
|
||
|
|
srvmetrics = ServerMetrics()
|
||
|
|
srvmetrics.setvalues(metricval)
|
||
|
|
# print srvmetrics.tocsv()
|
||
|
|
|
||
|
|
if thresholdcheck:
|
||
|
|
checkserverthreshold(srvmetrics)
|
||
|
|
|
||
|
|
return srvmetrics
|
||
|
|
|
||
|
|
def csvservermetrics(srvmetrics):
|
||
|
|
# return a csv string of ServerMetrics.tocsv()
|
||
|
|
# loadavg1,loadavg5,loadavg15, usedmem%, usedcpu% for total, cpu1, cpu2, ...
|
||
|
|
rv = ""
|
||
|
|
if options.timestamp:
|
||
|
|
rv = str(time.time()) + ", "
|
||
|
|
rv += srvmetrics.tocsv()
|
||
|
|
return rv
|
||
|
|
|
||
|
|
def csvserverbaseline():
|
||
|
|
# return a csv string of raw server metrics data: memfree, memtotal, cpuused, cpusystem, cpuidle
|
||
|
|
return "memory (total, free) = " + csvservermems(collectservermems()) + "\ncputime (used, sys, idl) = " + csvservercputimes(collectservercputimes())
|
||
|
|
|
||
|
|
class NodeMetrics(object):
|
||
|
|
def __init__(self):
|
||
|
|
self.nmetrics = {"nodethroughput" : 0.0,
|
||
|
|
"nodeusedmemory" : 0.0,
|
||
|
|
"nodetotalcpu" : 0.0,
|
||
|
|
"nodeusercpu" : 0.0,
|
||
|
|
"nodesystemcpu" : 0.0,
|
||
|
|
"nodewaitcpu" : 0.0}
|
||
|
|
|
||
|
|
# set values from val = (throughput, mem, tcpu, ucpu, scpu, wcpu):
|
||
|
|
def setvalues(self, val):
|
||
|
|
self.nmetrics["nodethroughput"] = val[0]
|
||
|
|
self.nmetrics["nodeusedmemory"] = val[1]
|
||
|
|
self.nmetrics["nodetotalcpu"] = val[2]
|
||
|
|
self.nmetrics["nodeusercpu"] = val[3]
|
||
|
|
self.nmetrics["nodesystemcpu"] = val[4]
|
||
|
|
self.nmetrics["nodewaitcpu"] = val[5]
|
||
|
|
|
||
|
|
def setvalue(self, key, val):
|
||
|
|
self.nmetrics[key] = val
|
||
|
|
|
||
|
|
def getvalue(self, key):
|
||
|
|
return self.nmetrics[key]
|
||
|
|
|
||
|
|
def getkeys(self):
|
||
|
|
return self.nmetrics.keys()
|
||
|
|
|
||
|
|
def tocsv(self):
|
||
|
|
return ", ".join(map(lambda(x):str(x), self.nmetrics.values()))
|
||
|
|
|
||
|
|
|
||
|
|
class LogSession(object):
|
||
|
|
def __init__(self):
|
||
|
|
self.nodethresholds = NodeMetrics()
|
||
|
|
# set node threshold default values:
|
||
|
|
# nodethroughput=20.0, nodeusedmemory=15.0, nodetotalcpu=90.0,
|
||
|
|
# nodeusercpu=30.0, nodewaitcpu=50.0, nodesystemcpu=20.0}
|
||
|
|
self.nodethresholds.setvalues([20.0, 15.0, 90.0, 30.0, 50.0, 20.0])
|
||
|
|
if options.configfile is not None:
|
||
|
|
self.readnodethresholds(options.configfile)
|
||
|
|
self.pids = {}
|
||
|
|
self.nodemetricsA = {}
|
||
|
|
self.nodemetricsB = {}
|
||
|
|
self.nodemetricsC = {}
|
||
|
|
|
||
|
|
def getpids(self):
|
||
|
|
# return dict of all CORE session pids in a dict using node name as the keys
|
||
|
|
# parent pid (vnoded) is the first value
|
||
|
|
self.pids = {}
|
||
|
|
nodes = commands.getstatusoutput("ls /tmp/pycore.%s/*pid" % options.session)
|
||
|
|
if nodes[0] != 0:
|
||
|
|
# if options.timestamp == True:
|
||
|
|
# print str(time.time()),
|
||
|
|
# print "CORE session %s has not created nodes" % options.session
|
||
|
|
return
|
||
|
|
|
||
|
|
nodes = nodes[1].split('\n')
|
||
|
|
for nod in nodes:
|
||
|
|
nodename = nod.split('/')[-1].strip(".pid")
|
||
|
|
self.pids[nodename] = commands.getoutput("cat %s" % nod)
|
||
|
|
|
||
|
|
# do not expect failure of this command
|
||
|
|
procs = commands.getoutput('ps -eo ppid,pid,comm').split('\n')
|
||
|
|
|
||
|
|
# build self.pids dict with key=nodename and val="ppid,pid,cmd"
|
||
|
|
for nname in self.pids:
|
||
|
|
# print nname, self.pids[nname]
|
||
|
|
if self.pids[nname] == "":
|
||
|
|
if options.timestamp == True:
|
||
|
|
print str(time.time()),
|
||
|
|
print "ERROR: null vnoded pid of node: %s" % nname
|
||
|
|
else:
|
||
|
|
childprocs = []
|
||
|
|
ppid = self.pids[nname]
|
||
|
|
for proc in procs:
|
||
|
|
val=proc.split()
|
||
|
|
if ppid == val[1]:
|
||
|
|
childprocs.append([val[1], val[2]] )
|
||
|
|
if ppid == val[0]:
|
||
|
|
childprocs.append([val[1], val[2]])
|
||
|
|
self.pids[nname] = childprocs
|
||
|
|
# print nname, self.pids[nname]
|
||
|
|
return self.pids
|
||
|
|
|
||
|
|
def printsesspids(self):
|
||
|
|
if self.pids == {}:
|
||
|
|
return {}
|
||
|
|
for pp in self.pids:
|
||
|
|
if self.pids[pp] != []:
|
||
|
|
for ap in range(len(self.pids[pp]) - 1):
|
||
|
|
print ", " + self.pids[pp][ap][0], # ap pid
|
||
|
|
print ", " + self.pids[pp][ap][1], # ap cmd
|
||
|
|
procmetrics = map(lambda(x):str(x),self.pids[pp][ap][-1])
|
||
|
|
print ", " + ", ".join(procmetrics),
|
||
|
|
nodemetrics = map(lambda(x):str(x), self.pids[pp][-1])
|
||
|
|
print ", " + ", ".join(nodemetrics)
|
||
|
|
|
||
|
|
def getprocessmetrics(self, pid):
|
||
|
|
# return [cpu#, vsize(kb), ttime, utime, stime, wtime]
|
||
|
|
# from a /proc/pid/stat (a single line file) assume columns are:
|
||
|
|
# pid(0) comm(1) state ppid pgrp sess tty_nr tpgid flags
|
||
|
|
# minflt cmiflt majflt cmajflt # utime(12) stime cutime cstime
|
||
|
|
# priority nice num_threads itrealvalue starttime vsize(22) rss rsslim
|
||
|
|
# startcode endcode startstack kstkesp signal blocked sigignore sigcatch
|
||
|
|
# wchan nswap cnswap exit_signal processor(38) rt_priority
|
||
|
|
# policy ioblock guest_time cguest_time (man 5 proc)
|
||
|
|
#rval = ProcessMetrics()
|
||
|
|
#rval.__init__()
|
||
|
|
rval = {}
|
||
|
|
lines = readfile("/proc/" + pid + "/stat")
|
||
|
|
if lines == []:
|
||
|
|
return rval
|
||
|
|
items = lines[0].split()
|
||
|
|
(utime, stime, cutime, cstime) = map(lambda(x):int(x), items[13:17])
|
||
|
|
# print ">???", utime, stime, cutime, cstime
|
||
|
|
rval = (items[38], # last run processor
|
||
|
|
int(items[22])/1000, # process virtual mem in kb
|
||
|
|
utime + stime + cutime + cstime,# totoal time
|
||
|
|
utime, # user time
|
||
|
|
stime, # system time
|
||
|
|
cutime + cstime) # wait time
|
||
|
|
# print "pid --- processmetrics", rval
|
||
|
|
return rval
|
||
|
|
|
||
|
|
def getnodethroughput(self, pid):
|
||
|
|
# return node throughput of total receive and transmit packets in kb
|
||
|
|
lines = readfile("/proc/" + pid + "/net/dev")
|
||
|
|
if lines == []:
|
||
|
|
return -0.00
|
||
|
|
ifs = map(lambda(x): x.split(), lines[2:])
|
||
|
|
ifm = zip(*ifs)
|
||
|
|
rv = sum(map(lambda(x):int(x), ifm[1])) # received bytes
|
||
|
|
tr = sum(map(lambda(x):int(x), ifm[9])) # transmited bytes
|
||
|
|
#print 'node thruput :', rv, tr, (rv + tr)/1000
|
||
|
|
return (rv + tr)/1000
|
||
|
|
|
||
|
|
def getnodemetrics(self, mindex):
|
||
|
|
# return NodeMetrics with indexed by nodename, values are rows of
|
||
|
|
# [ [ppid, vnoded, [cpu#, vmem(kb), ttime, utime, stime, wtime]],
|
||
|
|
# [cpid, cmd, [cpu#, vmem(kb), ttime, utime, stime, wtime]], ... ,
|
||
|
|
# [thrput, vmem(kb), ttime, utime, stime, wtime]]
|
||
|
|
if mindex == 'a':
|
||
|
|
metricref = self.nodemetricsA
|
||
|
|
else:
|
||
|
|
metricref = self.nodemetricsB
|
||
|
|
|
||
|
|
self.getpids()
|
||
|
|
# print " inside getnodemetrics()", self.pids
|
||
|
|
if self.pids == {}:
|
||
|
|
return {}
|
||
|
|
|
||
|
|
|
||
|
|
for nod in self.pids:
|
||
|
|
nmetric = NodeMetrics()
|
||
|
|
nmetric.__init__()
|
||
|
|
nodeapps = {}
|
||
|
|
for ap in range(len(self.pids[nod])): # get each process metrics
|
||
|
|
procm = self.getprocessmetrics(self.pids[nod][ap][0])
|
||
|
|
if procm == []:
|
||
|
|
if options.timestamp == True:
|
||
|
|
print str(time.time()),
|
||
|
|
print "WARNING: transient process", self.pids[nod][ap][1], \
|
||
|
|
"/", self.pids[nod][ap][0], "on node %s" % nod
|
||
|
|
else:
|
||
|
|
nodeapps[ap] = procm
|
||
|
|
self.pids[nod][ap].append(nodeapps[ap])
|
||
|
|
processm = zip(*nodeapps.values()) # get overall node metrics
|
||
|
|
# print processm
|
||
|
|
if len(processm) > 0:
|
||
|
|
# if nod == 'n6':
|
||
|
|
# print nod, self.getnodethroughput(self.pids[nod][0][0])
|
||
|
|
nmetric.setvalues(( self.getnodethroughput(self.pids[nod][0][0]),
|
||
|
|
sum(map(lambda(x):int(x), processm[1])), # vsize(kb)
|
||
|
|
sum(map(lambda(x):int(x), processm[2])), # ttime
|
||
|
|
sum(map(lambda(x):int(x), processm[3])), # utime
|
||
|
|
sum(map(lambda(x):int(x), processm[4])), # stime
|
||
|
|
sum(map(lambda(x):int(x), processm[5])))) # wtime
|
||
|
|
metricref[nod] = nmetric
|
||
|
|
# print nod, self.pids[nod][0][0], metricref[nod].tocsv()
|
||
|
|
return metricref
|
||
|
|
|
||
|
|
def setnodemetricsC(self, key, val):
|
||
|
|
self.nodemetricsC[key] = val
|
||
|
|
|
||
|
|
def printnodemetrics(self, mindex):
|
||
|
|
if mindex == 'c':
|
||
|
|
mm = self.nodemetricsC
|
||
|
|
else:
|
||
|
|
if mindex == 'a':
|
||
|
|
mm = self.nodemetricsA
|
||
|
|
else:
|
||
|
|
mm = self.nodemetricsB
|
||
|
|
|
||
|
|
for k in self.nodemetricsC:
|
||
|
|
if options.timestamp:
|
||
|
|
print str(time.time()) + ",",
|
||
|
|
print k, ",", mm[k].tocsv()
|
||
|
|
|
||
|
|
def readnodethresholds(self, filename):
|
||
|
|
if filename is None:
|
||
|
|
return
|
||
|
|
lines = readfile(filename)
|
||
|
|
for l in lines:
|
||
|
|
mval = l.strip().split('=')
|
||
|
|
# print "read line %s" % mval
|
||
|
|
if len(mval) > 1 :
|
||
|
|
thekey = mval[0].strip()
|
||
|
|
theval = mval[1].strip()
|
||
|
|
if thekey in self.nodethresholds.getkeys():
|
||
|
|
self.nodethresholds.setvalue(thekey, float(theval))
|
||
|
|
#print thekey," = %.2f" % float(theval)
|
||
|
|
|
||
|
|
def checknodethresholds(self, nname):
|
||
|
|
# print "check node thresholds", nname
|
||
|
|
calcm = self.nodemetricsC[nname]
|
||
|
|
for keyname in self.nodethresholds.getkeys():
|
||
|
|
# print "HIII", keyname, calcm.getvalue(keyname), self.nodethresholds.getvalue(keyname)
|
||
|
|
if float(calcm.getvalue(keyname)) > float(self.nodethresholds.getvalue(keyname)):
|
||
|
|
# print calculatednodem.getvalue(m)
|
||
|
|
alarm = ["node", nname + "/" + self.pids[nname][0][0], keyname,\
|
||
|
|
calcm.getvalue(keyname), ">", self.nodethresholds.getvalue(keyname)]
|
||
|
|
if options.timestamp:
|
||
|
|
print str(time.time()) + ",",
|
||
|
|
print ", ".join(map(lambda(x):str(x), alarm))
|
||
|
|
|
||
|
|
def calcnodemetrics(self, cputimea, cputimeb, mems):
|
||
|
|
# return a dict of nodemetrics indexed by node name
|
||
|
|
# nodemetrics[nodename][-1] = node/host%
|
||
|
|
p = []
|
||
|
|
for i in range(len(cputimeb[0])):
|
||
|
|
p.append(cputimeb[0][i] - cputimea[0][i])
|
||
|
|
hostusedcpu = p[0] + p[1]
|
||
|
|
hostusedmem = mems[0] - mems[1]
|
||
|
|
if hostusedcpu == 0:
|
||
|
|
print "WARNING: host used cpu = 0, ", p[0], p[1]
|
||
|
|
hostusedcpu = 1
|
||
|
|
if hostusedmem == 0:
|
||
|
|
print "WARNING: host used mem = 0, ", mems[0], mems[1]
|
||
|
|
hostusedmem = 1
|
||
|
|
|
||
|
|
nodesa = self.nodemetricsA
|
||
|
|
nodesb = self.nodemetricsB
|
||
|
|
for nod in nodesb:
|
||
|
|
calcm = self.nodemetricsC
|
||
|
|
calcm = NodeMetrics()
|
||
|
|
calcm.__init__()
|
||
|
|
if (nod in nodesa):
|
||
|
|
try:
|
||
|
|
if (nodesb[nod] == []) | (nodesa[nod] == []) | \
|
||
|
|
( False == isinstance(nodesb[nod], NodeMetrics)) | \
|
||
|
|
( False == isinstance(nodesa[nod], NodeMetrics)):
|
||
|
|
if options.timestamp == True:
|
||
|
|
print str(time.time()),
|
||
|
|
print "Warning: nodes %s is not fully instanciated" % nod
|
||
|
|
else:
|
||
|
|
# calc throughput kbps
|
||
|
|
#print "node b : ", nodesb[nod].tocsv()
|
||
|
|
#print "node a : ", nodesa[nod].tocsv()
|
||
|
|
#if nod == 'n6':
|
||
|
|
#print nodesb[nod].getvalue("nodethroughput"), nodesa[nod].getvalue("nodethroughput")
|
||
|
|
calcm.setvalue("nodethroughput", "%.2f" % (8 * (nodesb[nod].getvalue("nodethroughput") \
|
||
|
|
- nodesa[nod].getvalue("nodethroughput")) / options.interval))
|
||
|
|
# calc mem node used / host used
|
||
|
|
calcm.setvalue("nodeusedmemory", "%.2f" % (100.0 * (nodesb[nod].getvalue("nodeusedmemory") / hostusedmem)))
|
||
|
|
|
||
|
|
# calc total cpu time node / host
|
||
|
|
calcm.setvalue("nodetotalcpu", "%.2f" % (100.0 * (nodesb[nod].getvalue("nodetotalcpu")\
|
||
|
|
- nodesa[nod].getvalue("nodetotalcpu")) /hostusedcpu))
|
||
|
|
# calc user cpu time node / host
|
||
|
|
calcm.setvalue("nodeusercpu", "%.2f" % (100.0 * (nodesb[nod].getvalue("nodeusercpu")\
|
||
|
|
- nodesa[nod].getvalue("nodeusercpu")) /hostusedcpu))
|
||
|
|
# calc system cpu time node / host
|
||
|
|
calcm.setvalue("nodesystemcpu", "%.2f" % (100.0 * (nodesb[nod].getvalue("nodesystemcpu")\
|
||
|
|
- nodesa[nod].getvalue("nodesystemcpu")) /hostusedcpu))
|
||
|
|
# calc waitcpu time node / host
|
||
|
|
calcm.setvalue("nodewaitcpu", "%.2f" % (100.0 * (nodesb[nod].getvalue("nodewaitcpu")\
|
||
|
|
- nodesa[nod].getvalue("nodewaitcpu")) /hostusedcpu))
|
||
|
|
|
||
|
|
#print nod, calcm.tocsv()
|
||
|
|
#print '=========================='
|
||
|
|
logsession.nodemetricsC[nod] = calcm
|
||
|
|
# logsession.printnodemetrics('c')
|
||
|
|
|
||
|
|
if options.alarm is not None:
|
||
|
|
logsession.checknodethresholds(nod)
|
||
|
|
except IndexError:
|
||
|
|
pass
|
||
|
|
else:
|
||
|
|
print "Warning: transient node %s " % nod
|
||
|
|
|
||
|
|
return nodesb
|
||
|
|
|
||
|
|
def main():
|
||
|
|
usagestr = "%prog [-h] [options] [args]\n\nLog server and optional CORE session metrics to stdout."
|
||
|
|
parser = optparse.OptionParser(usage = usagestr)
|
||
|
|
parser.set_defaults(interval=2, timestamp=False,
|
||
|
|
configfile = "/etc/core/perflogserver.conf",\
|
||
|
|
alarm = True, session = None)
|
||
|
|
parser.add_option("-i", "--interval", dest = "interval", type = int,
|
||
|
|
help = "seconds to wait between samples; default=%s" %
|
||
|
|
parser.defaults["interval"])
|
||
|
|
parser.add_option("-t", "--timestamp", action = "store_true",
|
||
|
|
dest = "timestamp",
|
||
|
|
help = "include timestamp on each line")
|
||
|
|
parser.add_option("-c", "--configfile", dest = "configfile",
|
||
|
|
type = "string",
|
||
|
|
help = "read threshold values from the specified file;\
|
||
|
|
default=%s" % parser.defaults["configfile"])
|
||
|
|
parser.add_option("-a", "--alarm", action = "store_true",
|
||
|
|
dest = "alarm",
|
||
|
|
help = "generate alarms based threshold check on each cycle")
|
||
|
|
parser.add_option("-s", "--session", dest = "session", type = int,
|
||
|
|
help = "CORE session id; default=%s" %
|
||
|
|
parser.defaults["session"])
|
||
|
|
global options
|
||
|
|
global ncpus
|
||
|
|
global serverthresholds
|
||
|
|
global logsession
|
||
|
|
global cyclecount
|
||
|
|
|
||
|
|
(options, args) = parser.parse_args()
|
||
|
|
# print options
|
||
|
|
|
||
|
|
signal.signal(signal.SIGINT, handler)
|
||
|
|
signal.signal(signal.SIGTERM, handler)
|
||
|
|
|
||
|
|
ncpus = numcpus()
|
||
|
|
|
||
|
|
# server threshold dictionary - a ServerMetrics instant with default values
|
||
|
|
serverthresholds = ServerMetrics()
|
||
|
|
# set to server threshold default values: serverloadavg1=3.5,
|
||
|
|
# serverloadavg5=3.5, serverloadavg15=3.5, serverusedmemory=80.0,
|
||
|
|
# serverusedcputime=80.0, processorusedcputime=90.0
|
||
|
|
serverthresholds.setvalues([3.5, 3.5, 3.5, 80.0, 80.0, 90.0])
|
||
|
|
if options.alarm is True:
|
||
|
|
# read server threshold values from configuration file
|
||
|
|
readserverthresholds(options.configfile)
|
||
|
|
|
||
|
|
if options.session is not None:
|
||
|
|
logsession = LogSession()
|
||
|
|
# print logsession
|
||
|
|
|
||
|
|
# mark host log baseline
|
||
|
|
print "server: ", ", ".join(map(lambda(x):str(x), os.uname())), ",", ncpus, "CPU cores"
|
||
|
|
print "start timestamp:", time.time(), ", baseline data: "
|
||
|
|
print csvserverbaseline()
|
||
|
|
print "server metrics: ", ", ".join(map(lambda(x):str(x), serverthresholds.getkeys()))
|
||
|
|
if options.session is not None:
|
||
|
|
print "node metrics: nodename, ", ", ".join(map(lambda(x):str(x), logsession.nodethresholds.getkeys()))
|
||
|
|
|
||
|
|
cyclecount = 0
|
||
|
|
while True:
|
||
|
|
cputimea = collectservercputimes()
|
||
|
|
if options.session is not None:
|
||
|
|
nodesa = logsession.getnodemetrics('a')
|
||
|
|
# print "nodes a:", nodesa
|
||
|
|
|
||
|
|
time.sleep(options.interval)
|
||
|
|
|
||
|
|
cputimeb = collectservercputimes()
|
||
|
|
mems = collectservermems()
|
||
|
|
|
||
|
|
calccputime = calcservercputimes(cputimea, cputimeb)
|
||
|
|
m = csvservermetrics(collectservermetrics(calccputime, mems, options.alarm))
|
||
|
|
print m
|
||
|
|
|
||
|
|
if options.session is not None:
|
||
|
|
nodesb = logsession.getnodemetrics('b')
|
||
|
|
# print "nodes b:", nodesb
|
||
|
|
if nodesb != {}:
|
||
|
|
logsession.calcnodemetrics(cputimea, cputimeb, mems)
|
||
|
|
logsession.printnodemetrics('c')
|
||
|
|
|
||
|
|
sys.stdout.flush()
|
||
|
|
cyclecount = cyclecount + 1
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
main()
|