#!/usr/bin/env python
'''Module for RRDtool interface to the pytomo data.
Necessary module: rrdtool (python-rrdtool,
http://oss.oetiker.ch/rrdtool/download.en.html)
Version: 0.1
Author: Ana Oprea
Date: 11.07.2012
Usage:
# first create a database - follow steps in lib_database
import pytomo.start_pytomo as start_pytomo
start_pytomo.configure_log_file('doc_test')
import pytomo.lib_database as lib_database
from pytomo.lib_plot import UNITS
import pytomo.lib_rrdtools as lib_rrdtools
pytomo_rrd = lib_rrdtools.PytomoRRD(db_name)
pytomo_rrd.update_pytomo_rrd()
pytomo_rrd.plot_pytomo_rrd()
>>> import time
>>> timestamp = time.strftime("%Y-%m-%d.%H_%M_%S")
>>> # to make sure a new file is created for every run we use
>>> # timestamp.
>>> db_name = 'doc_test_lib_db' + str(timestamp) + '.db'
>>> # import pytomo.lib_database as lib_database
>>> doc_db = lib_database.PytomoDatabase(db_name)
>>> doc_db.create_pytomo_table('doc_test_table')
>>> import datetime
>>> record = (datetime.datetime(2011, 5, 6, 15, 30, 50, 103775),
... 'Youtube', 'http://www.youtube.com/watch?v=RcmKbTR--iA',
... 'http://v15.lscache3.c.youtube.com',
... '173.194.20.56','default_10.193.225.12', None, None, None,
... 8.9944229125976562, 'mp4', 225, 115012833.0, 511168.14666666667,
... 9575411, 0, 1024 ,100, 0.99954795837402344, 7.9875903129577637,
... 35, 11.722306421319782, 1192528.8804511931, None)
>>> doc_db.insert_record(record)
>>> record = (datetime.datetime(2011, 5, 6, 15, 31, 10, 103775),
... 'Youtube', 'http://www.youtube.com/watch?v=RcmKbTR--iA',
... 'http://v15.lscache3.c.youtube.com',
... '173.194.20.56','default_10.193.225.12', None, None, None,
... 8.9944229125976562, 'mp4', 225, 115012833.0, 511168.14666666667,
... 9575411, 0, 1024, 100, 0.99954795837402344, 7.9875903129577637,
... 40, 11.722306421319782, 1192528.8804511931,
... 'http://www.youtube.com/fake_redirect')
>>> doc_db.insert_record(record)
>>> pytomo_rrd = PytomoRRD(db_name)
>>> pytomo_rrd.update_pytomo_rrd()
>>> pytomo_rrd.plot_pytomo_rrd()
>>> from os import unlink
>>> unlink(db_name)
'''
from __future__ import with_statement, absolute_import, print_function
import sys
# TODO Louis check if correct
if 'win' in sys.platform:
try:
import rrdtoolmodule as rrdtool
except ImportError:
from .rrdtool_win_x86_DLLs import rrdtoolmodule as rrdtool
else:
try:
import rrdtool
except ImportError:
print('Please install the RRDtool module on your system.\nWe suggest to'
' download from your distribution repository: python-rrdtool\n'
'For Debian based systems: sudo apt-get install python-rrdtool\n'
'For RHEL based systems: sudo yum install python-rrdtool\n')
import time
import math
from optparse import OptionParser
from operator import itemgetter
# only for logging
import logging
import os
# config file
try:
from . import config_pytomo
except ValueError:
import config_pytomo
# database data
try:
from . import lib_database
except ValueError:
import lib_database
from sqlite3 import Error
# parameters to extract from db and plot
try:
from .lib_plot import UNITS
except ValueError:
from lib_plot import UNITS
# file operations
try:
from .lib_io import TIMESTAMP_POSITION, NO_DB_RECORDS_UNITS, \
get_latest_file, rrd_filename, plot_filename
except ValueError:
from lib_io import TIMESTAMP_POSITION, NO_DB_RECORDS_UNITS, \
get_latest_file, rrd_filename, plot_filename
# RRD parameters
# RRD syncronise interval in seconds (plot cannot start exactly at start)
RRD_PLOT_SYNC_TIME = 300
# RRD syncronise interval in seconds (plot cannot start exactly at start)
RRD_STEP_SYNC_TIME = 1
# Consolidation function: AVERAGE, MIN, MAX, LAST
CF_AVG = 'AVERAGE'
CF_MAX = 'MAX'
CF_MIN = 'MIN'
CF_LAST = 'LAST'
# how many of primary DS are used to build a Consolidated Data Point which then
# goes into the archive (if all points need to be plotted set to 1)
# RRA_POINTS * RRD_STEP = time interval to apply CF
# specific for each CF, there can be different aggregations for the same CF
RRA_POINTS_AVG = 1
#RRA_POINTS_AVG3 = 3
RRA_POINTS_MIN = 1
RRA_POINTS_MAX = 1
RRA_POINTS_LAST = 1
# Data Source in rrd only allows for maximum 19 characters name
DS_NAME_MAX_LENGTH = 19
# percentage of allowed Unknown DS in a consolidation interval
RRA_U_PERCENTAGE = 0.99
# graph line thickness (mm)
RRD_PLOT_LINE = 2
# for some parameters that are generally zero the line is not very noticeable
# (problem noticed on WinXP_x86 and Ubuntu1204)
RRD_PLOT_LINE_GENERALLY_ZERO = 3
# line name for AVERAGE CF
RRD_PLOT_AVG_NAME = 'line_avg'
# line colour for AVERAGE CF in 0x
RRD_PLOT_AVG_COL = 'FF0000'
# line name for MIN CF
RRD_PLOT_MIN_NAME = 'line_min'
# line colour for AVERAGE CF in 0x
RRD_PLOT_MIN_COL = '00FF00'
# line name for MAX CF
RRD_PLOT_MAX_NAME = 'line_max'
# line colour for MAX CF in 0x
RRD_PLOT_MAX_COL = '0000FF'
# line name for LAST CF
RRD_PLOT_LAST_NAME = 'line_last'
# line colour for MAX CF in 0x
RRD_PLOT_LAST_COL = 'FFFF00'
# rrd plot characteristics
RRD_PLOT_CHARACTER = ['--lower-limit', '0',
'--rigid',
'--alt-autoscale-max',
'--font', 'WATERMARK:7']
# nr. of '=' displayed by log
NR_REPEAT = 28
# specific parameters
AVG_TH_PARAM = 'AvgThroughput'
INDICATOR_PARAM = 'ReceptionRatio'
DOWN_B_PARAM = 'DownloadBytes'
DOWN_T_PARAM = 'DownloadTime'
ENC_PARAM = 'EncodingRate'
DOWN_INT_PARAM = 'DownloadInterruptions'
VIDEO_PERCENTAGE_PARAM = 'VideosWithInterruptions'
# parameters to extract from db
PARAMETERS = sorted(UNITS.keys())
# parameters that do not exist in db
PARAM_NO_DB = sorted(NO_DB_RECORDS_UNITS.keys())
# all parameters to plot
ALL_PARAM = sorted(PARAMETERS + PARAM_NO_DB)
# parameters that are measured in bytes
PARAMETERS_IN_BYTES = ['DownloadBytes', 'InitialData', 'VideoLength']
# parameters that include buffering time
PARAMETERS_FOR_BUFFERING = ['BufferingDuration', 'BufferDurationAtEnd']
# parameters that should be genrally zero
PARAMETERS_GENERALLY_ZERO = ([DOWN_INT_PARAM, VIDEO_PERCENTAGE_PARAM] +
PARAMETERS_FOR_BUFFERING)
# indexes of parameters (for example {'DownloadTime': 0})
INDEX_DICT = dict((v, k) for (k, v) in enumerate(ALL_PARAM))
# power of 10 for Mega
MEGA_ORDER = 6
# needed to compute average throughput in kpbs
# 1 byte = 8 bits
BYTE_BIT_TRANS = 8
# kilo = 1000
KILO_TRANS = 1000
[docs]def create_DS_types(parameters, heartbeat):
'''Function to return a list of elements 'DS:ds-name:GAUGE:heartbeat:U:U'
>>> HEARTBEAT = 100
>>> create_DS_types(['BufferDurationAtEnd', 'PingMin',
... 'InitialData'], HEARTBEAT) #doctest: +NORMALIZE_WHITESPACE
['DS:BufferDurationAtEnd:GAUGE:100:U:U', 'DS:PingMin:GAUGE:100:U:U',
'DS:InitialData:GAUGE:100:U:U']
>>> create_DS_types([], HEARTBEAT)
[]
>>> create_DS_types(None, HEARTBEAT)
Traceback (most recent call last):
...
TypeError: 'NoneType' object is not iterable
'''
return [('DS:%s:GAUGE:%i:U:U' %
(parameter[:DS_NAME_MAX_LENGTH], heartbeat))
for parameter in parameters]
[docs]def generate_plot_names(parameters, timestamp):
''' Function to create a list of filenames for plots like:
RRD_PLOT_DIR/parameter_to_plot_timestamp.extension
TODO: redo doctest
>>> from time import time
>>> TIMESTAMP = '2012-07-20.11_44_27'
>>> generate_plot_names(['DownloadTime',
... 'PingMin'], TIMESTAMP) #doctest: +NORMALIZE_WHITESPACEi, +ELLIPSIS
['/home/capture/co/pytomo/trunk/Pytomo/images/s-spo-hti.2012-07-20.11_44_27.DownloadTime_pytomo_image.png',
'/home/capture/co/pytomo/trunk/Pytomo/images/s-spo-hti.2012-07-20.11_44_27.PingMin_pytomo_image.png']
>>> generate_plot_names([], TIMESTAMP)
[]
>>> generate_plot_names(None, TIMESTAMP)
Traceback (most recent call last):
...
TypeError: 'NoneType' object is not iterable
'''
try:
return [plot_filename(parameter, timestamp) for parameter in parameters]
except IOError:
return None
[docs]def update_data_types(parameters):
'''Function to return a string '%i:%s:...:%s' dependent on the number of DS
>>> update_data_types(['BufferDurationAtEnd', 'PingMin', 'InitialData'])
'%i:%s:%s:%s'
>>> update_data_types([])
'%i'
>>> update_data_types(None)
Traceback (most recent call last):
...
TypeError: object of type 'NoneType' has no len()
'''
return "".join(("%i", "".join((":%s")*len(parameters))))
[docs]def rrd_filename_escape_colon(rrd_file):
''' Escape the : in the filename of a rrd because this is not accepted in
the rrd_graph when defining a function (problem appears generally on
Windows)
>>> rrd_filename_escape_colon('/home/capture/co/pytomo/trunk/Pytomo/rrds/s-spo-hti.1350291171.pytomo.rrd')
'/home/capture/co/pytomo/trunk/Pytomo/rrds/s-spo-hti.1350291171.pytomo.rrd'
>>> rrd_filename_escape_colon('C:\Pytomo\rrds\s-spo-hti.1350291171.pytomo.rrd')
'C\\:\\Pytomo\rrds\\s-spo-hti.1350291171.pytomo.rrd'
'''
return rrd_file.replace(':','\:')
[docs]class PytomoRRD:
'''Pytomo class to interact with rrdtools
'''
has_values = None
def __init__(self, db_file):
'''Get the data and create the RRD (Round Robin Database).
Parameters:
--start = minimum timestamp in db - sync time
--step = STEP - by default, data is fed in the rrd every 300s (5min)
DS: ds-name : ds-type : heartbeat : min : max
Data Source type: GAUGE - current, absolute value is saved
Data Source heartbeat: HEARTBEAT - if no data is received after 5min, it
is set to Unknown
Data Source min: U - Unknown
Data Source max: U - Unknown
RRA: CF : cf-arguments
Round Robin Archive: consists of a number of data values or statistics
for each of the defined Data Sources (DS) and is defined within a line
RRA Consolidation Function: AVERAGE - the average of the data points
RRA CF-arguments: RRA_U_PERCENTAGE - percentage of values that can be U
RRA CF-arguments: RRD_STEP - the CF of every (RRD_STEP * RRA_POINTS_...)
is computed
RRA CF-arguments: rra_rows - how many generations of data values are
kept in an RRA; the CF is computed during
(RRD_STEP * RRA_POINTS_AVG) * self.rra_rows (= total time plotted)
'''
# Intialize the logger for standalone testing Logging
if not config_pytomo.LOG:
self.logger_rrd()
config_pytomo.LOG.info('=' * NR_REPEAT)
config_pytomo.LOG.info("Parameters plotted by rrd: %s", PARAMETERS)
# retrieve desired data from the database in a list of tuples
try:
self.data = lib_database.PytomoDatabase(db_file).\
fetch_all_parameters(PARAMETERS)
except Error, mes:
config_pytomo.LOG.error('Unable to extract data %s with error:'
'%s' % (str(PARAMETERS), mes))
return
config_pytomo.LOG.info(' '.join(("Retrieved data for rrd.",
"From database: ", os.path.basename(db_file))))
# if no data is extracted from sqlite, no other operation is performed
if not self.data:
self.has_values = False
config_pytomo.LOG.warn('No data could be extracted from the '
'%s for the parameters: %s' %
(os.path.basename(db_file), PARAMETERS))
return
# if there is only one point in the data, rrdtool cannot plot
if len(self.data) == 1:
self.has_values = False
config_pytomo.LOG.warn('Only one point could be extracted from the '
'%s for the parameters: %s' %
(os.path.basename(db_file), str(PARAMETERS)))
return
self.has_values = True
#config_pytomo.LOG.debug("Data: %s", str(self.data))
# number of unknown (None) values in the dataset for each parameter
self.unknown_values = [0] * len(PARAMETERS)
# all the timestamps in the database
timestamps = map(lib_database.time_to_epoch,
map(itemgetter(TIMESTAMP_POSITION), self.data))
# the timestamp is the last element of each tuple in the data list
# start time is the minimum timestamp (epoch) from the db - sync time
self.start_time = timestamps[0] - RRD_PLOT_SYNC_TIME
config_pytomo.LOG.debug("Start time rrd: %i", self.start_time)
# end time is the maximum timestamp (epoch) from the db + sync time
self.end_time = timestamps[-1] + RRD_PLOT_SYNC_TIME
config_pytomo.LOG.debug("End time rrd: %i", self.end_time)
# difference between two adjacent timestamps
timestamp_dif = [abs(a - b) for a, b in zip(timestamps, timestamps[1:])]
# time interval in seconds at which data will be fed into the RRD
# minimum time difference between the data inserted
# when time changes in winter the clock goes back 1h, difference is < 1s
min_time_diff = min(timestamp_dif) - RRD_STEP_SYNC_TIME
if min_time_diff < 1:
RRD_STEP = 1
else:
RRD_STEP = min_time_diff
# time interval in seconds after which data is considered Unknown
# maximum time difference between the data inserted
RRD_HEARTBEAT = max(timestamp_dif) + RRD_STEP_SYNC_TIME
# create the Data Sources for the rrd
self.data_sources = create_DS_types(PARAMETERS, RRD_HEARTBEAT)
config_pytomo.LOG.debug("Data sources: %s", self.data_sources)
# rrd file has the first db time to epoch timestamp
# RRA number of data generations kept
try:
self.rrd_file = rrd_filename(str(timestamps[0]))
except IOError:
self.rrd_file = None
config_pytomo.LOG.error("Could not get rrd file")
# (RRD_STEP * RRA_POINTS_AVG) * self.rra_rows = total time plotted
self.rra_rows = math.ceil((self.end_time - self.start_time) /
RRD_STEP * RRA_POINTS_AVG)
# if there is only one database entry or if the time difference between
# start, end is too small, there might be no generation of data kept
if self.rra_rows < 1:
self.rra_rows = 1
config_pytomo.LOG.debug("Number of data generations kept: %i",
self.rra_rows)
# all the points are plotted if RRA_POINTS_... = 1
# to uncomment if other CF need to be applied on aggregated data
# (min/max of 3 points for a specific interval, for example)
rrdtool.create(self.rrd_file,
'--start', '%i' % self.start_time,
'--step', '%i' % RRD_STEP,
self.data_sources,
'RRA:%s:%f:%i:%i' %
(CF_AVG, RRA_U_PERCENTAGE, RRA_POINTS_AVG,
self.rra_rows) #,
# 'RRA:AVERAGE%f:%i:%i' %
# (RRA_U_PERCENTAGE, RRA_POINTS_AVG3,
# self.rra_rows),
# 'RRA:%s:%f:%i:%i' %
# (CF_MIN, RRA_U_PERCENTAGE, RRA_POINTS_MIN,
# self.rra_rows),
# 'RRA:%s:%f:%i:%i' %
# (CF_MAX, RRA_U_PERCENTAGE, RRA_POINTS_MAX,
# self.rra_rows),
# 'RRA:%s:%f:%i:%i' %
# (CF_LAST, RRA_U_PERCENTAGE, RRA_POINTS_LAST,
# self.rra_rows)
)
config_pytomo.LOG.info(' '.join(("Created rrd: ",
os.path.basename(self.rrd_file))))
# image files have the first db time to epoch timestamp
self.rrd_plot_files = generate_plot_names(ALL_PARAM, timestamps[0])
[docs] def update_pytomo_rrd(self):
'''Insert data from the list of tuples (timestamp, parameter1, ...)
to the rrd.
'''
if not self.has_values:
config_pytomo.LOG.warn('RRD data update aborted')
return 1
# insert into rrd all the values for the extracted parameters to plot
# data[][TIMESTAMP_POSITION] is the timestamp
# data[][:TIMESTAMP_POSITION] represents the parameters to plot
for row in self.data:
# transform timestamp to epoch in local time
# TODO: check problems related to timezone
timestamp = row[TIMESTAMP_POSITION]
parameter_values = row[:TIMESTAMP_POSITION]
# function used in order to take advantage of the * operator and
# retrieve all the elements of an argument
identity = lambda *x: x
try:
rrdtool.update(self.rrd_file,
update_data_types(parameter_values) %
identity(lib_database.time_to_epoch(timestamp),
*format_null_values(*parameter_values)))
except rrdtool.error, mes:
config_pytomo.LOG.debug('Could not update the rrd with error'
' %s' % mes)
continue
#config_pytomo.LOG.debug('Updated rrd data: (%s, %s)' %
# (timestamp, str(format_null_values(*parameter_values))))
for index, parameter in enumerate(parameter_values):
if parameter is None:
self.unknown_values[index] += 1
#config_pytomo.LOG.debug('Updated unknown[%i]' %
# index)
config_pytomo.LOG.debug('Unknown values per parameter: %s' %
str(self.unknown_values))
[docs] def fetch_pytomo_rrd(self):
'''Fetch data from the rrd.
'''
if not self.has_values:
config_pytomo.LOG.warn('RRD data fetch aborted')
return 1
# fetch data
config_pytomo.LOG.debug('Fetched rrd %s data %s' % (CF_AVG,
str(rrdtool.fetch(self.rrd_file,
'%s' % CF_AVG,
'--start', '%i' % self.start_time,
'--end', '%i' % self.end_time #,
))))
[docs] def plot_pytomo_rrd(self):
'''Plot the time series parameters (at least 3 points must exist in the
database for the graphs to exist).
'''
if not self.has_values:
config_pytomo.LOG.warn('RRD data plot aborted')
return 1
# rrdtool graph does not accept ":" in the rrd filename path
def_rrd_file = rrd_filename_escape_colon(self.rrd_file)
# graphs that exist in db
for unknown_values_index, parameter in enumerate(PARAMETERS):
# set the line thickness of the plots (this needs to be thicker if
# values are generally zero)
line_thickness = RRD_PLOT_LINE
index = INDEX_DICT[parameter]
# display in Mega for parameters that are in bytes
if parameter in PARAMETERS_IN_BYTES:
units_exp = ['--units-exponent', '%s' % str(MEGA_ORDER)]
elif parameter in PARAMETERS_FOR_BUFFERING:
units_exp = ['--alt-y-grid']
else:
units_exp = ['--units-exponent', '0']
if parameter in PARAMETERS_GENERALLY_ZERO:
line_thickness = RRD_PLOT_LINE_GENERALLY_ZERO
# all the points are plotted if RRA_POINTS_... = 1
# to uncomment if other CF need to be applied on aggregated data
# (min/max of 3 points for a specific interval, for example)
rrdtool.graph(self.rrd_plot_files[index],
'--start', '%i' % self.start_time,
'--end', '%i' % self.end_time,
'--vertical-label', '%s' % UNITS[parameter],
'--title', 'Pytomo %s statistics' % parameter,
RRD_PLOT_CHARACTER,
units_exp,
# only display number of unknown data values
'--watermark', 'Number of unknown data values: %i'
% self.unknown_values[unknown_values_index],
#'--full-size-mode',
'DEF:%s=%s:%s:%s'
% (RRD_PLOT_AVG_NAME, def_rrd_file,
parameter[:DS_NAME_MAX_LENGTH], CF_AVG),
'LINE%i:%s#%s' % (line_thickness,
RRD_PLOT_AVG_NAME, RRD_PLOT_AVG_COL),
#'DEF:%s=%s:%s:%s'
#% (RRD_PLOT_MIN_NAME, self.rrd_file,
# parameter[:DS_NAME_MAX_LENGTH], CF_MIN),
#'LINE%i:%s#%s:Minimum' % (RRD_PLOT_LINE,
# RRD_PLOT_MIN_NAME, RRD_PLOT_MIN_COL),
#'DEF:%s=%s:%s:%s'
#% (RRD_PLOT_MAX_NAME, self.rrd_file,
# parameter[:DS_NAME_MAX_LENGTH], CF_MAX),
#'LINE%i:%s#%s:Maximum' % (RRD_PLOT_LINE,
# RRD_PLOT_MAX_NAME, RRD_PLOT_MAX_COL),
#'DEF:%s=%s:%s:%s'
#% (RRD_PLOT_LAST_NAME, self.rrd_file,
# parameter[:DS_NAME_MAX_LENGTH], CF_LAST),
#'LINE%i:%s#%s:Last' % (RRD_PLOT_LINE,
# RRD_PLOT_LAST_NAME, RRD_PLOT_LAST_COL)
)
config_pytomo.LOG.info(' '.join(("The rrd plot was updated: ",
os.path.basename(self.rrd_plot_files[index]))))
config_pytomo.LOG.info('=' * NR_REPEAT)
# graph for average throughput:
# avg_th [Kbps] = DownBytes [bytes] / DownTime [s] * 8 /1000
rrdtool.graph(self.rrd_plot_files[INDEX_DICT[AVG_TH_PARAM]],
'--start', '%i' % self.start_time,
'--end', '%i' % self.end_time,
'--vertical-label', '%s' %
NO_DB_RECORDS_UNITS[AVG_TH_PARAM],
'--title', 'Pytomo %s statistics' % AVG_TH_PARAM,
RRD_PLOT_CHARACTER,
'--units-exponent', '0',
'--watermark', 'Number of unknown data values: %i'
% self.unknown_values[INDEX_DICT[DOWN_B_PARAM]],
'DEF:%s=%s:%s:%s' % (DOWN_B_PARAM[:DS_NAME_MAX_LENGTH],
def_rrd_file,
DOWN_B_PARAM[:DS_NAME_MAX_LENGTH],
CF_AVG),
'DEF:%s=%s:%s:%s' % (DOWN_T_PARAM[:DS_NAME_MAX_LENGTH],
def_rrd_file,
DOWN_T_PARAM[:DS_NAME_MAX_LENGTH],
CF_AVG),
'CDEF:%s=%s,%s,/,%s,*,%s,/' %
(AVG_TH_PARAM[:DS_NAME_MAX_LENGTH],
DOWN_B_PARAM[:DS_NAME_MAX_LENGTH],
DOWN_T_PARAM[:DS_NAME_MAX_LENGTH],
str(BYTE_BIT_TRANS),
str(KILO_TRANS)
),
'LINE%i:%s#%s' % (RRD_PLOT_LINE,
AVG_TH_PARAM[:DS_NAME_MAX_LENGTH],
RRD_PLOT_AVG_COL)
)
config_pytomo.LOG.info(' '.join(("The rrd plot was updated: ",
os.path.basename(self.rrd_plot_files[INDEX_DICT[AVG_TH_PARAM]]))))
config_pytomo.LOG.info('=' * NR_REPEAT)
# graph for indicator:
# indicator = DownBytes[B] / DownTime[s] * 8 / 1000 / EncodingRate[kbps]
rrdtool.graph(self.rrd_plot_files[INDEX_DICT[INDICATOR_PARAM]],
'--start', '%i' % self.start_time,
'--end', '%i' % self.end_time,
'--vertical-label', '%s' %
NO_DB_RECORDS_UNITS[INDICATOR_PARAM],
'--title', 'Pytomo %s statistics' % INDICATOR_PARAM,
RRD_PLOT_CHARACTER,
'--units-exponent', '0',
'--watermark', 'Number of unknown data values: %i'
% self.unknown_values[INDEX_DICT[DOWN_B_PARAM]],
'DEF:%s=%s:%s:%s' % (DOWN_B_PARAM[:DS_NAME_MAX_LENGTH],
def_rrd_file,
DOWN_B_PARAM[:DS_NAME_MAX_LENGTH],
CF_AVG),
'DEF:%s=%s:%s:%s' % (DOWN_T_PARAM[:DS_NAME_MAX_LENGTH],
def_rrd_file,
DOWN_T_PARAM[:DS_NAME_MAX_LENGTH],
CF_AVG),
'DEF:%s=%s:%s:%s' % (ENC_PARAM[:DS_NAME_MAX_LENGTH],
def_rrd_file,
ENC_PARAM[:DS_NAME_MAX_LENGTH],
CF_AVG),
#'CDEF:%s=%s,%s,/,%s,*,%s,/' %
'CDEF:%s=%s,%s,/,%s,*,%s,/,%s,/' %
(INDICATOR_PARAM[:DS_NAME_MAX_LENGTH],
DOWN_B_PARAM[:DS_NAME_MAX_LENGTH],
DOWN_T_PARAM[:DS_NAME_MAX_LENGTH],
str(BYTE_BIT_TRANS),
str(KILO_TRANS),
ENC_PARAM[:DS_NAME_MAX_LENGTH]),
'LINE%i:%s#%s' % (RRD_PLOT_LINE,
INDICATOR_PARAM[:DS_NAME_MAX_LENGTH],
RRD_PLOT_AVG_COL)
)
config_pytomo.LOG.info(' '.join(("The rrd plot was updated: ",
os.path.basename(self.rrd_plot_files[INDEX_DICT[INDICATOR_PARAM]]))))
config_pytomo.LOG.info('=' * NR_REPEAT)
# graph for magic parameter:
# video_percentage = 1 IF DownInterruptions > 0, ELSE 0
rrdtool.graph(self.rrd_plot_files[INDEX_DICT[VIDEO_PERCENTAGE_PARAM]],
'--start', '%i' % self.start_time,
'--end', '%i' % self.end_time,
'--vertical-label', '%s' %
NO_DB_RECORDS_UNITS[VIDEO_PERCENTAGE_PARAM],
'--title', 'Pytomo %s statistics' %
VIDEO_PERCENTAGE_PARAM,
RRD_PLOT_CHARACTER,
'--units-exponent', '0',
'--watermark', 'Number of unknown data values: %i'
% self.unknown_values[INDEX_DICT[DOWN_INT_PARAM]],
'DEF:%s=%s:%s:%s' % (DOWN_INT_PARAM[:DS_NAME_MAX_LENGTH],
def_rrd_file,
DOWN_INT_PARAM[:DS_NAME_MAX_LENGTH],
CF_AVG),
# IF DownloadInterruptions > 0: ((DOWN_INT_PARAM, 0) GT)
# IF so, return 1, else return 0
'CDEF:%s=%s,0,GT,1,0,IF' %
(VIDEO_PERCENTAGE_PARAM[:DS_NAME_MAX_LENGTH],
DOWN_INT_PARAM[:DS_NAME_MAX_LENGTH]),
'LINE%i:%s#%s' % (RRD_PLOT_LINE_GENERALLY_ZERO,
VIDEO_PERCENTAGE_PARAM[:DS_NAME_MAX_LENGTH],
RRD_PLOT_AVG_COL)
)
config_pytomo.LOG.info(' '.join(("The rrd plot was updated: ",
os.path.basename(self.rrd_plot_files[INDEX_DICT[AVG_TH_PARAM]]))))
config_pytomo.LOG.info('=' * NR_REPEAT)
@staticmethod
[docs] def logger_rrd():
''' Initialze the logger'''
config_pytomo.LOG = logging.getLogger('pytomo_rrd')
# to not have console output
#config_pytomo.LOG.propagate = False
config_pytomo.LOG.setLevel(config_pytomo.LOG_LEVEL)
timestamp = time.strftime("%Y-%m-%d.%H_%M_%S")
if config_pytomo.LOG_FILE == '-':
handler = logging.StreamHandler(sys.stdout)
else:
log_file = os.path.sep.join((config_pytomo. LOG_DIR,
'.'.join((timestamp, config_pytomo.LOG_FILE))))
try:
with open(log_file, 'a') as _:
pass
except IOError:
raise IOError('Logfile %s could not be open for writing' %
log_file)
handler = logging.FileHandler(filename=log_file)
log_formatter = logging.Formatter("%(asctime)s - %(name)s - "
"%(levelname)s - %(message)s")
handler.setFormatter(log_formatter)
config_pytomo.LOG.addHandler(handler)
[docs]def create_options(parser):
''' Add the different options to the parser'''
parser.add_option('-v', '--verbose', dest='verbose',
action='store_true', default=False, help = 'run as verbose mode')
parser.add_option('-f', '--file', dest='db_name',
help = 'run on a specific database (by default the latest database'
' in the default database directory is selected)')
parser.add_option('-d', '--dir', dest='db_dir_name',
help = 'run on a specific directory where the '
'latest database will be selected')
[docs]def main(argv=None):
"Program wrapper"
if argv is None:
argv = sys.argv[1:]
usage = '%prog [-v] [-d database_directory] [-f database]'
parser = OptionParser(usage=usage)
create_options(parser)
(options, args) = parser.parse_args(argv)
# if no specific directory or database is specified, the latest one from
# DB_DIR is selected
if not options.db_name and not options.db_dir_name:
database = get_latest_file(config_pytomo.DATABASE_DIR)
# if database is specified
if options.db_name:
database = options.db_name
# if a directory is specified, the latest database there is selected
if options.db_dir_name:
database = get_latest_file(options.db_dir_name)
pytomo_rrd = PytomoRRD(database)
pytomo_rrd.update_pytomo_rrd()
pytomo_rrd.plot_pytomo_rrd()
return 0
if __name__ == '__main__':
import doctest
doctest.testmod()
sys.exit(main())