Start using classes
This commit is contained in:
Gregory Soutade
parent
7dada493ab
commit
c3c201fda1
303
iwla.py
303
iwla.py
|
|
@ -10,94 +10,89 @@ import gzip
|
|||
|
||||
from display import *
|
||||
|
||||
# Default configuration
|
||||
|
||||
DB_ROOT = './output/'
|
||||
DISPLAY_ROOT = './output/'
|
||||
|
||||
log_format = '$server_name:$server_port $remote_addr - $remote_user [$time_local] ' +\
|
||||
'"$request" $status $body_bytes_sent ' +\
|
||||
'"$http_referer" "$http_user_agent"'
|
||||
|
||||
time_format = '%d/%b/%Y:%H:%M:%S +0100'
|
||||
|
||||
pre_analysis_hooks = []
|
||||
post_analysis_hooks = []
|
||||
display_hooks = []
|
||||
|
||||
from default_conf import *
|
||||
from conf import *
|
||||
|
||||
print '==> Start'
|
||||
class IWLA(object):
|
||||
|
||||
meta_visit = {}
|
||||
analyse_started = False
|
||||
current_visits = {}
|
||||
cache_plugins = {}
|
||||
display = {}
|
||||
ANALYSIS_CLASS = 'HTTP'
|
||||
API_VERSION = 1
|
||||
|
||||
log_format_extracted = re.sub(r'([^\$\w])', r'\\\g<1>', log_format)
|
||||
log_format_extracted = re.sub(r'\$(\w+)', '(?P<\g<1>>.+)', log_format_extracted)
|
||||
http_request_extracted = re.compile(r'(?P<http_method>\S+) (?P<http_uri>\S+) (?P<http_version>\S+)')
|
||||
def __init__(self):
|
||||
print '==> Start'
|
||||
|
||||
log_re = re.compile(log_format_extracted)
|
||||
uri_re = re.compile(r'(?P<extract_uri>[^\?]*)[\?(?P<extract_parameters>.*)]?')
|
||||
pages_extensions = ['/', 'html', 'xhtml', 'py', 'pl', 'rb', 'php']
|
||||
viewed_http_codes = [200]
|
||||
self.meta_infos = {}
|
||||
self.analyse_started = False
|
||||
self.current_analysis = {}
|
||||
self.cache_plugins = {}
|
||||
self.display = {}
|
||||
self.valid_visitors = None
|
||||
|
||||
HOOKS_ROOT = './plugins/'
|
||||
PRE_HOOK_DIRECTORY = HOOKS_ROOT + 'pre_analysis/'
|
||||
POST_HOOK_DIRECTORY = HOOKS_ROOT + 'post_analysis/'
|
||||
DISPLAY_HOOK_DIRECTORY = HOOKS_ROOT + 'display/'
|
||||
META_PATH = DB_ROOT + 'meta.db'
|
||||
DB_FILENAME = 'iwla.db'
|
||||
self.log_format_extracted = re.sub(r'([^\$\w])', r'\\\g<1>', log_format)
|
||||
self.log_format_extracted = re.sub(r'\$(\w+)', '(?P<\g<1>>.+)', self.log_format_extracted)
|
||||
self.http_request_extracted = re.compile(r'(?P<http_method>\S+) (?P<http_uri>\S+) (?P<http_version>\S+)')
|
||||
self.log_re = re.compile(self.log_format_extracted)
|
||||
self.uri_re = re.compile(r'(?P<extract_uri>[^\?]*)[\?(?P<extract_parameters>.*)]?')
|
||||
self.plugins = {PRE_HOOK_DIRECTORY : pre_analysis_hooks,
|
||||
POST_HOOK_DIRECTORY : post_analysis_hooks,
|
||||
DISPLAY_HOOK_DIRECTORY : display_hooks}
|
||||
|
||||
plugins = {PRE_HOOK_DIRECTORY : pre_analysis_hooks, POST_HOOK_DIRECTORY : post_analysis_hooks, DISPLAY_HOOK_DIRECTORY : display_hooks}
|
||||
|
||||
ANALYSIS_CLASS = 'HTTP'
|
||||
API_VERSION = 1
|
||||
|
||||
def preloadPlugins():
|
||||
def _preloadPlugins(self):
|
||||
ret = True
|
||||
for root in plugins.keys():
|
||||
for plugin_name in plugins[root]:
|
||||
for root in self.plugins.keys():
|
||||
for plugin_name in self.plugins[root]:
|
||||
p = root + '/' + plugin_name
|
||||
try:
|
||||
fp, pathname, description = imp.find_module(plugin_name, [root])
|
||||
cache_plugins[p] = imp.load_module(plugin_name, fp, pathname, description)
|
||||
#cache_plugins[p] = imp.load_module(p,None,p,("py","r",imp.PKG_DIRECTORY))
|
||||
#cache_plugins[p] = imp.load_source(p, p)
|
||||
mod = cache_plugins[p]
|
||||
#print dir(mod)
|
||||
#print "Register %s -> %s" % (p, mod)
|
||||
self.cache_plugins[p] = imp.load_module(plugin_name, fp, pathname, description)
|
||||
mod = self.cache_plugins[p]
|
||||
infos = mod.get_plugins_infos()
|
||||
if infos['class'] != ANALYSIS_CLASS or \
|
||||
API_VERSION < infos['min_version'] or\
|
||||
(infos['max_version'] != -1 and (API_VERSION > infos['max_version'])):
|
||||
del cache_plugins[p]
|
||||
if infos['class'] != IWLA.ANALYSIS_CLASS or \
|
||||
IWLA.API_VERSION < infos['min_version'] or\
|
||||
(infos['max_version'] != -1 and (IWLA.API_VERSION > infos['max_version'])):
|
||||
del self.cache_plugins[p]
|
||||
elif not mod.load():
|
||||
del cache_plugins[p]
|
||||
del self.cache_plugins[p]
|
||||
except Exception as e:
|
||||
print 'Error loading \'%s\' => %s' % (p, e)
|
||||
ret = False
|
||||
return ret
|
||||
|
||||
def _clearVisits(self):
|
||||
self.current_analysis = {
|
||||
'days_stats' : {},
|
||||
'month_stats' : {},
|
||||
'visits' : {}
|
||||
}
|
||||
self.valid_visitors = None
|
||||
return self.current_analysis
|
||||
|
||||
def createEmptyVisits():
|
||||
visits = {'days_stats' : {}, 'month_stats' : {}, 'visits' : {}}
|
||||
return visits
|
||||
def getDaysStats(self):
|
||||
return self.current_analysis['days_stats']
|
||||
|
||||
def createEmptyMeta():
|
||||
meta = {'last_time' : None}
|
||||
return meta
|
||||
def getMonthStatsStats(self):
|
||||
return self.current_analysis['month_stats']
|
||||
|
||||
def createEmptyDisplay():
|
||||
display = {}
|
||||
return display
|
||||
def getCurrentVisists(self):
|
||||
return self.current_analysis['visits']
|
||||
|
||||
def getDBFilename(time):
|
||||
def getValidVisitors(self):
|
||||
return self.current_analysis['visits']
|
||||
|
||||
def _clearMeta(self):
|
||||
self.meta_infos = {
|
||||
'last_time' : None
|
||||
}
|
||||
return self.meta_infos
|
||||
|
||||
def _clearDisplay(self):
|
||||
self.display = {}
|
||||
return self.display
|
||||
|
||||
def getDBFilename(self, time):
|
||||
return (DB_ROOT + '%d/%d_%s') % (time.tm_year, time.tm_mon, DB_FILENAME)
|
||||
|
||||
def serialize(obj, filename):
|
||||
def _serialize(self, obj, filename):
|
||||
base = os.path.dirname(filename)
|
||||
if not os.path.exists(base):
|
||||
os.makedirs(base)
|
||||
|
|
@ -112,7 +107,7 @@ def serialize(obj, filename):
|
|||
fzip.write(f.read())
|
||||
os.remove(filename + '.tmp')
|
||||
|
||||
def deserialize(filename):
|
||||
def _deserialize(self, filename):
|
||||
if not os.path.exists(filename):
|
||||
return None
|
||||
|
||||
|
|
@ -120,31 +115,31 @@ def deserialize(filename):
|
|||
return pickle.load(f)
|
||||
return None
|
||||
|
||||
def callPlugins(root, *args):
|
||||
def _callPlugins(self, root, *args):
|
||||
print '==> Call plugins (%s)' % root
|
||||
for p in plugins[root]:
|
||||
for p in self.plugins[root]:
|
||||
print '\t%s' % (p)
|
||||
mod = cache_plugins[root + '/' + p]
|
||||
mod = self.cache_plugins[root + '/' + p]
|
||||
mod.hook(*args)
|
||||
|
||||
def isPage(request):
|
||||
def isPage(self, request):
|
||||
for e in pages_extensions:
|
||||
if request.endswith(e):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def appendHit(hit):
|
||||
def _appendHit(self, hit):
|
||||
remote_addr = hit['remote_addr']
|
||||
|
||||
if not remote_addr in current_visits['visits'].keys():
|
||||
createUser(hit)
|
||||
if not remote_addr in self.current_analysis['visits'].keys():
|
||||
self._createUser(hit)
|
||||
return
|
||||
|
||||
super_hit = current_visits['visits'][remote_addr]
|
||||
super_hit = self.current_analysis['visits'][remote_addr]
|
||||
super_hit['requests'].append(hit)
|
||||
super_hit['bandwidth'] += int(hit['body_bytes_sent'])
|
||||
super_hit['last_access'] = meta_visit['last_time']
|
||||
super_hit['last_access'] = self.meta_infos['last_time']
|
||||
|
||||
request = hit['extract_request']
|
||||
|
||||
|
|
@ -153,7 +148,7 @@ def appendHit(hit):
|
|||
else:
|
||||
uri = request['http_uri']
|
||||
|
||||
hit['is_page'] = isPage(uri)
|
||||
hit['is_page'] = self.isPage(uri)
|
||||
|
||||
# Don't count 3xx status
|
||||
status = int(hit['status'])
|
||||
|
|
@ -172,28 +167,28 @@ def appendHit(hit):
|
|||
else:
|
||||
super_hit[hit_key] += 1
|
||||
|
||||
def createUser(hit):
|
||||
super_hit = current_visits['visits'][hit['remote_addr']] = {}
|
||||
def _createUser(self, hit):
|
||||
super_hit = self.current_analysis['visits'][hit['remote_addr']] = {}
|
||||
super_hit['remote_addr'] = hit['remote_addr']
|
||||
super_hit['viewed_pages'] = 0
|
||||
super_hit['viewed_hits'] = 0
|
||||
super_hit['not_viewed_pages'] = 0
|
||||
super_hit['not_viewed_hits'] = 0
|
||||
super_hit['bandwidth'] = 0
|
||||
super_hit['last_access'] = meta_visit['last_time']
|
||||
super_hit['last_access'] = self.meta_infos['last_time']
|
||||
super_hit['requests'] = []
|
||||
super_hit['robot'] = False
|
||||
super_hit['hit_only'] = 0
|
||||
appendHit(hit)
|
||||
self._appendHit(hit)
|
||||
|
||||
def decodeHTTPRequest(hit):
|
||||
def _decodeHTTPRequest(self, hit):
|
||||
if not 'request' in hit.keys(): return False
|
||||
|
||||
groups = http_request_extracted.match(hit['request'])
|
||||
groups = self.http_request_extracted.match(hit['request'])
|
||||
|
||||
if groups:
|
||||
hit['extract_request'] = groups.groupdict()
|
||||
uri_groups = uri_re.match(hit['extract_request']['http_uri'])
|
||||
uri_groups = self.uri_re.match(hit['extract_request']['http_uri'])
|
||||
if uri_groups:
|
||||
d = uri_groups.groupdict()
|
||||
hit['extract_request']['extract_uri'] = d['extract_uri']
|
||||
|
|
@ -203,41 +198,39 @@ def decodeHTTPRequest(hit):
|
|||
print "Bad request extraction " + hit['request']
|
||||
return False
|
||||
|
||||
referer_groups = uri_re.match(hit['http_referer'])
|
||||
referer_groups = self.uri_re.match(hit['http_referer'])
|
||||
if referer_groups:
|
||||
referer = hit['extract_referer'] = referer_groups.groupdict()
|
||||
return True
|
||||
|
||||
def decodeTime(hit):
|
||||
t = hit['time_local']
|
||||
def _decodeTime(self, hit):
|
||||
hit['time_decoded'] = time.strptime(hit['time_local'], time_format)
|
||||
|
||||
hit['time_decoded'] = time.strptime(t, time_format)
|
||||
|
||||
def getDisplayIndex():
|
||||
cur_time = meta_visit['last_time']
|
||||
def getDisplayIndex(self):
|
||||
cur_time = self.meta_infos['last_time']
|
||||
filename = '%d/index_%d.html' % (cur_time.tm_year, cur_time.tm_mon)
|
||||
|
||||
return display.get(filename, None)
|
||||
return self.display.get(filename, None)
|
||||
|
||||
def generateDisplayDaysStat():
|
||||
cur_time = meta_visit['last_time']
|
||||
def _generateDisplayDaysStat(self):
|
||||
cur_time = self.meta_infos['last_time']
|
||||
title = 'Stats %d/%d' % (cur_time.tm_mon, cur_time.tm_year)
|
||||
filename = '%d/index_%d.html' % (cur_time.tm_year, cur_time.tm_mon)
|
||||
page = createPage(display, filename, title)
|
||||
page = createPage(self.display, filename, title)
|
||||
|
||||
days = createTable('By day', ['Day', 'Visits', 'Pages', 'Hits', 'Bandwidth', 'Robot Bandwidth'])
|
||||
|
||||
keys = current_visits['days_stats'].keys()
|
||||
keys = self.current_analysis['days_stats'].keys()
|
||||
keys.sort()
|
||||
nb_visits = 0
|
||||
for k in keys:
|
||||
stats = current_visits['days_stats'][k]
|
||||
stats = self.current_analysis['days_stats'][k]
|
||||
row = [k, stats['nb_visitors'], stats['viewed_pages'], stats['viewed_hits'], stats['viewed_bandwidth'], stats['not_viewed_bandwidth']]
|
||||
row = map(lambda(v): str(v), row)
|
||||
appendRowToTable(days, row)
|
||||
nb_visits += stats['nb_visitors']
|
||||
|
||||
stats = current_visits['month_stats']
|
||||
stats = self.current_analysis['month_stats']
|
||||
|
||||
nb_days = len(keys)
|
||||
row = [0, nb_visits, stats['viewed_pages'], stats['viewed_hits'], stats['viewed_bandwidth'], stats['not_viewed_bandwidth']]
|
||||
|
|
@ -254,12 +247,12 @@ def generateDisplayDaysStat():
|
|||
appendRowToTable(days, row)
|
||||
appendBlockToPage(page, days)
|
||||
|
||||
def generateDisplay():
|
||||
generateDisplayDaysStat()
|
||||
callPlugins(DISPLAY_HOOK_DIRECTORY, current_visits, display)
|
||||
buildPages(DISPLAY_ROOT, display)
|
||||
def _generateDisplay(self):
|
||||
self._generateDisplayDaysStat()
|
||||
self._callPlugins(DISPLAY_HOOK_DIRECTORY, self.current_analysis, self.display)
|
||||
buildPages(DISPLAY_ROOT, self.display)
|
||||
|
||||
def generateStats(visits):
|
||||
def _generateStats(self, visits):
|
||||
stats = {}
|
||||
stats['viewed_bandwidth'] = 0
|
||||
stats['not_viewed_bandwidth'] = 0
|
||||
|
|
@ -289,51 +282,51 @@ def generateStats(visits):
|
|||
|
||||
return stats
|
||||
|
||||
def generateMonthStats():
|
||||
display = createEmptyDisplay()
|
||||
def _generateMonthStats(self):
|
||||
self._clearDisplay()
|
||||
|
||||
visits = current_visits['visits']
|
||||
visits = self.current_analysis['visits']
|
||||
|
||||
stats = generateStats(visits)
|
||||
stats = self._generateStats(visits)
|
||||
|
||||
cur_time = meta_visit['last_time']
|
||||
cur_time = self.meta_infos['last_time']
|
||||
print "== Stats for %d/%d ==" % (cur_time.tm_year, cur_time.tm_mon)
|
||||
print stats
|
||||
|
||||
valid_visitors = {k: v for (k,v) in visits.items() if not visits[k]['robot']}
|
||||
callPlugins(POST_HOOK_DIRECTORY, valid_visitors, stats)
|
||||
self.valid_visitors = {k: v for (k,v) in visits.items() if not visits[k]['robot']}
|
||||
self._callPlugins(POST_HOOK_DIRECTORY, valid_visitors, stats)
|
||||
|
||||
current_visits['month_stats'] = stats
|
||||
self.current_analysis['month_stats'] = stats
|
||||
|
||||
path = getDBFilename(cur_time)
|
||||
path = self.getDBFilename(cur_time)
|
||||
if os.path.exists(path):
|
||||
os.remove(path)
|
||||
|
||||
print "==> Serialize to %s" % path
|
||||
|
||||
serialize(current_visits, path)
|
||||
self._serialize(self.current_analysis, path)
|
||||
|
||||
generateDisplay()
|
||||
self._generateDisplay()
|
||||
|
||||
def generateDayStats():
|
||||
visits = current_visits['visits']
|
||||
def _generateDayStats(self):
|
||||
visits = self.current_analysis['visits']
|
||||
|
||||
callPlugins(PRE_HOOK_DIRECTORY, visits)
|
||||
self._callPlugins(PRE_HOOK_DIRECTORY, visits)
|
||||
|
||||
stats = generateStats(visits)
|
||||
stats = self._generateStats(visits)
|
||||
|
||||
cur_time = meta_visit['last_time']
|
||||
cur_time = self.meta_infos['last_time']
|
||||
print "== Stats for %d/%d/%d ==" % (cur_time.tm_year, cur_time.tm_mon, cur_time.tm_mday)
|
||||
|
||||
if cur_time.tm_mday > 1:
|
||||
last_day = cur_time.tm_mday - 1
|
||||
while last_day:
|
||||
if last_day in current_visits['days_stats'].keys():
|
||||
if last_day in self.current_analysis['days_stats'].keys():
|
||||
break
|
||||
last_day -= 1
|
||||
if last_day:
|
||||
for k in stats.keys():
|
||||
stats[k] -= current_visits['days_stats'][last_day][k]
|
||||
stats[k] -= self.current_analysis['days_stats'][last_day][k]
|
||||
stats['nb_visitors'] = 0
|
||||
for k in visits.keys():
|
||||
if visits[k]['robot']: continue
|
||||
|
|
@ -341,71 +334,71 @@ def generateDayStats():
|
|||
stats['nb_visitors'] += 1
|
||||
print stats
|
||||
|
||||
current_visits['days_stats'][cur_time.tm_mday] = stats
|
||||
self.current_analysis['days_stats'][cur_time.tm_mday] = stats
|
||||
|
||||
def newHit(hit):
|
||||
global current_visits
|
||||
global analyse_started
|
||||
|
||||
decodeTime(hit)
|
||||
def _newHit(self, hit):
|
||||
self._decodeTime(hit)
|
||||
|
||||
t = hit['time_decoded']
|
||||
|
||||
cur_time = meta_visit['last_time']
|
||||
cur_time = self.meta_infos['last_time']
|
||||
|
||||
if cur_time == None:
|
||||
current_visits = deserialize(getDBFilename(t)) or createEmptyVisits()
|
||||
analyse_started = True
|
||||
self.current_analysis = self._deserialize(self.getDBFilename(t)) or self._clearVisits()
|
||||
self.analyse_started = True
|
||||
else:
|
||||
if not analyse_started:
|
||||
if not self.analyse_started:
|
||||
if time.mktime(cur_time) >= time.mktime(t):
|
||||
return
|
||||
else:
|
||||
analyse_started = True
|
||||
self.analyse_started = True
|
||||
if cur_time.tm_mon != t.tm_mon:
|
||||
generateMonthStats()
|
||||
current_visits = deserialize(getDBFilename(t)) or createEmptyVisits()
|
||||
self._generateMonthStats()
|
||||
self.current_analysis = self._deserialize(self.getDBFilename(t)) or self._clearVisits()
|
||||
elif cur_time.tm_mday != t.tm_mday:
|
||||
generateDayStats()
|
||||
self._generateDayStats()
|
||||
|
||||
meta_visit['last_time'] = t
|
||||
self.meta_infos['last_time'] = t
|
||||
|
||||
if not decodeHTTPRequest(hit): return False
|
||||
if not self._decodeHTTPRequest(hit): return False
|
||||
|
||||
for k in hit.keys():
|
||||
if hit[k] == '-': hit[k] = ''
|
||||
|
||||
appendHit(hit)
|
||||
self._appendHit(hit)
|
||||
|
||||
return True
|
||||
|
||||
preloadPlugins()
|
||||
def start(self):
|
||||
self._preloadPlugins()
|
||||
|
||||
print '==> Analysing log'
|
||||
print '==> Analysing log'
|
||||
|
||||
meta_visit = deserialize(META_PATH) or createEmptyMeta()
|
||||
if meta_visit['last_time']:
|
||||
current_visits = deserialize(getDBFilename(meta_visit['last_time'])) or createEmptyVisits()
|
||||
else:
|
||||
current_visits = createEmptyVisits()
|
||||
self.meta_infos = self._deserialize(META_PATH) or self._clearMeta()
|
||||
if self.meta_infos['last_time']:
|
||||
self.current_analysis = self._deserialize(self.getDBFilename(self.meta_infos['last_time'])) or self._clearVisits()
|
||||
else:
|
||||
self._clearVisits()
|
||||
|
||||
f = open(analyzed_filename)
|
||||
for l in f:
|
||||
with open(analyzed_filename) as f:
|
||||
for l in f:
|
||||
# print "line " + l
|
||||
|
||||
groups = log_re.match(l)
|
||||
groups = self.log_re.match(l)
|
||||
|
||||
if groups:
|
||||
if not newHit(groups.groupdict()):
|
||||
if not self._newHit(groups.groupdict()):
|
||||
break
|
||||
else:
|
||||
print "No match " + l
|
||||
f.close()
|
||||
print "No match for " + l
|
||||
|
||||
if analyse_started:
|
||||
generateDayStats()
|
||||
generateMonthStats()
|
||||
serialize(meta_visit, META_PATH)
|
||||
else:
|
||||
if self.analyse_started:
|
||||
self._generateDayStats()
|
||||
self._generateMonthStats()
|
||||
self._serialize(meta_infos, META_PATH)
|
||||
else:
|
||||
print '==> Analyse not started : nothing to do'
|
||||
generateMonthStats()
|
||||
self._generateMonthStats()
|
||||
|
||||
iwla = IWLA()
|
||||
iwla.start()
|
||||
|
|
|
|||
Loading…
Reference in New Issue