Merge branch 'master' of soutade.fr:iwla

This commit is contained in:
Gregory Soutade 2014-12-15 22:32:47 +01:00
commit fd6b05475b
4 changed files with 52 additions and 23 deletions

2
TODO
View File

@ -4,7 +4,7 @@ doc auto generation
doc enhancement doc enhancement
Limit hits/pages/downloads by rate Limit hits/pages/downloads by rate
Automatic tests Automatic tests
quiet mode
Add Licence Add Licence
Free memory as soon as possible Free memory as soon as possible
gzip output files gzip output files
different debug output levels

View File

@ -24,3 +24,5 @@ hit_to_page_conf = [r'^.+/category/.+$', r'^.+/tag/.+$', r'^.+/archive/.+$', r'^
# Because it's too long to build HTML when there is too much entries # Because it's too long to build HTML when there is too much entries
max_hits_displayed = 100 max_hits_displayed = 100
max_downloads_displayed = 100 max_downloads_displayed = 100
compress_output_files = ['html', 'css', 'js']

View File

@ -48,3 +48,6 @@ resources_path = ['resources']
icon_path = '%s/%s' % (os.path.basename(resources_path[0]), 'icon') icon_path = '%s/%s' % (os.path.basename(resources_path[0]), 'icon')
# CSS path (you can add yours) # CSS path (you can add yours)
css_path = ['%s/%s/%s' % (os.path.basename(resources_path[0]), 'css', 'iwla.css')] css_path = ['%s/%s/%s' % (os.path.basename(resources_path[0]), 'css', 'iwla.css')]
# Extensions to compress in gzip during display build
compress_output_files = []

64
iwla.py
View File

@ -31,6 +31,7 @@ from display import *
# Conf values needed : # Conf values needed :
# analyzed_filename # analyzed_filename
# domain_name # domain_name
# compress_output_files*
# #
# Output files : # Output files :
# DB_ROOT/meta.db # DB_ROOT/meta.db
@ -40,7 +41,7 @@ from display import *
# #
# Statistics creation : # Statistics creation :
# #
# meta => # meta :
# last_time # last_time
# start_analysis_time # start_analysis_time
# stats => # stats =>
@ -50,6 +51,7 @@ from display import *
# not_viewed_bandwidth # not_viewed_bandwidth
# viewed_pages # viewed_pages
# viewed_hits # viewed_hits
# nb_visits
# nb_visitors # nb_visitors
# #
# month_stats : # month_stats :
@ -57,7 +59,7 @@ from display import *
# not_viewed_bandwidth # not_viewed_bandwidth
# viewed_pages # viewed_pages
# viewed_hits # viewed_hits
# nb_visitors # nb_visits
# #
# days_stats : # days_stats :
# day => # day =>
@ -65,6 +67,7 @@ from display import *
# not_viewed_bandwidth # not_viewed_bandwidth
# viewed_pages # viewed_pages
# viewed_hits # viewed_hits
# nb_visits
# nb_visitors # nb_visitors
# #
# visits : # visits :
@ -338,18 +341,18 @@ class IWLA(object):
page = self.display.createPage(title, filename, conf.css_path) page = self.display.createPage(title, filename, conf.css_path)
_, nb_month_days = monthrange(cur_time.tm_year, cur_time.tm_mon) _, nb_month_days = monthrange(cur_time.tm_year, cur_time.tm_mon)
days = self.display.createBlock(DisplayHTMLBlockTableWithGraph, 'By day', ['Day', 'Visitors', 'Pages', 'Hits', 'Bandwidth', 'Not viewed Bandwidth'], None, nb_month_days, range(1,6)) days = self.display.createBlock(DisplayHTMLBlockTableWithGraph, 'By day', ['Day', 'Visits', 'Pages', 'Hits', 'Bandwidth', 'Not viewed Bandwidth'], None, nb_month_days, range(1,6))
days.setColsCSSClass(['', 'iwla_visitor', 'iwla_page', 'iwla_hit', 'iwla_bandwidth', 'iwla_bandwidth']) days.setColsCSSClass(['', 'iwla_visit', 'iwla_page', 'iwla_hit', 'iwla_bandwidth', 'iwla_bandwidth'])
nb_visits = 0 nb_visits = 0
nb_days = 0 nb_days = 0
for i in range(1, nb_month_days+1): for i in range(1, nb_month_days+1):
day = '%d<br/>%s' % (i, time.strftime('%b', cur_time)) day = '%d<br/>%s' % (i, time.strftime('%b', cur_time))
full_day = '%d %s %d' % (i, time.strftime('%b', cur_time), cur_time.tm_year) full_day = '%02d %s %d' % (i, time.strftime('%b', cur_time), cur_time.tm_year)
if i in self.current_analysis['days_stats'].keys(): if i in self.current_analysis['days_stats'].keys():
stats = self.current_analysis['days_stats'][i] stats = self.current_analysis['days_stats'][i]
row = [full_day, stats['nb_visitors'], stats['viewed_pages'], stats['viewed_hits'], row = [full_day, stats['nb_visits'], stats['viewed_pages'], stats['viewed_hits'],
stats['viewed_bandwidth'], stats['not_viewed_bandwidth']] stats['viewed_bandwidth'], stats['not_viewed_bandwidth']]
nb_visits += stats['nb_visitors'] nb_visits += stats['nb_visits']
nb_days += 1 nb_days += 1
else: else:
row = [full_day, 0, 0, 0, 0, 0] row = [full_day, 0, 0, 0, 0, 0]
@ -391,10 +394,10 @@ class IWLA(object):
cur_time = time.localtime() cur_time = time.localtime()
months_name = ['', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'June', 'July', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'] months_name = ['', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'June', 'July', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
title = 'Summary %d' % (year) title = 'Summary %d' % (year)
cols = ['Month', 'Visitors', 'Pages', 'Hits', 'Bandwidth', 'Not viewed Bandwidth', 'Details'] cols = ['Month', 'Visitors', 'Visits', 'Pages', 'Hits', 'Bandwidth', 'Not viewed Bandwidth', 'Details']
graph_cols=range(1,6) graph_cols=range(1,7)
months = self.display.createBlock(DisplayHTMLBlockTableWithGraph, title, cols, None, 12, graph_cols) months = self.display.createBlock(DisplayHTMLBlockTableWithGraph, title, cols, None, 12, graph_cols)
months.setColsCSSClass(['', 'iwla_visitor', 'iwla_page', 'iwla_hit', 'iwla_bandwidth', 'iwla_bandwidth', '']) months.setColsCSSClass(['', 'iwla_visitor', 'iwla_visit', 'iwla_page', 'iwla_hit', 'iwla_bandwidth', 'iwla_bandwidth', ''])
total = [0] * len(cols) total = [0] * len(cols)
for i in range(1, 13): for i in range(1, 13):
month = '%s<br/>%d' % (months_name[i], year) month = '%s<br/>%d' % (months_name[i], year)
@ -402,15 +405,15 @@ class IWLA(object):
if i in month_stats.keys(): if i in month_stats.keys():
stats = month_stats[i] stats = month_stats[i]
link = '<a href="%d/%02d/index.html">Details</a>' % (year, i) link = '<a href="%d/%02d/index.html">Details</a>' % (year, i)
row = [full_month, stats['nb_visitors'], stats['viewed_pages'], stats['viewed_hits'], row = [full_month, stats['nb_visitors'], stats['nb_visits'], stats['viewed_pages'], stats['viewed_hits'],
stats['viewed_bandwidth'], stats['not_viewed_bandwidth'], link] stats['viewed_bandwidth'], stats['not_viewed_bandwidth'], link]
for j in graph_cols: for j in graph_cols:
total[j] += row[j] total[j] += row[j]
else: else:
row = [full_month, 0, 0, 0, 0, 0, ''] row = [full_month, 0, 0, 0, 0, 0, 0, '']
months.appendRow(row) months.appendRow(row)
months.setCellValue(i-1, 4, bytesToStr(row[4]))
months.setCellValue(i-1, 5, bytesToStr(row[5])) months.setCellValue(i-1, 5, bytesToStr(row[5]))
months.setCellValue(i-1, 6, bytesToStr(row[6]))
months.appendShortTitle(month) months.appendShortTitle(month)
if year == cur_time.tm_year and i == cur_time.tm_mon: if year == cur_time.tm_year and i == cur_time.tm_mon:
css = months.getCellCSSClass(i-1, 0) css = months.getCellCSSClass(i-1, 0)
@ -419,8 +422,8 @@ class IWLA(object):
months.setCellCSSClass(i-1, 0, css) months.setCellCSSClass(i-1, 0, css)
total[0] = 'Total' total[0] = 'Total'
total[4] = bytesToStr(total[4])
total[5] = bytesToStr(total[5]) total[5] = bytesToStr(total[5])
total[6] = bytesToStr(total[6])
months.appendRow(total) months.appendRow(total)
page.appendBlock(months) page.appendBlock(months)
@ -431,19 +434,40 @@ class IWLA(object):
page = self.display.createPage(title, filename, conf.css_path) page = self.display.createPage(title, filename, conf.css_path)
last_update = '<b>Last update</b> %s<br />' % (time.strftime('%d %b %Y %H:%M', time.localtime())) last_update = '<b>Last update</b> %s<br />' % (time.strftime('%02d %b %Y %H:%M', time.localtime()))
page.appendBlock(self.display.createBlock(DisplayHTMLRaw, last_update)) page.appendBlock(self.display.createBlock(DisplayHTMLRaw, last_update))
for year in self.meta_infos['stats'].keys(): for year in sorted(self.meta_infos['stats'].keys(), reverse=True):
self._generateDisplayMonthStats(page, year, self.meta_infos['stats'][year]) self._generateDisplayMonthStats(page, year, self.meta_infos['stats'][year])
self.display.addPage(page) self.display.addPage(page)
def _compressFile(self, build_time, root, filename):
path = os.path.join(root, filename)
gz_path = path + '.gz'
#print 'Compress %s => %s' % (path, gz_path)
if not os.path.exists(gz_path) or\
os.stat(path).st_mtime > build_time:
with open(path, 'rb') as f_in:
with gzip.open(gz_path, 'wb') as f_out:
f_out.write(f_in.read())
def _compressFiles(self, build_time, root):
if not conf.compress_output_files: return
for rootdir, subdirs, files in os.walk(root, followlinks=True):
for f in files:
for ext in conf.compress_output_files:
if f.endswith(ext):
self._compressFile(build_time, rootdir, f)
break
def _generateDisplay(self): def _generateDisplay(self):
self._generateDisplayDaysStats() self._generateDisplayDaysStats()
self._callPlugins(conf.DISPLAY_HOOK_DIRECTORY) self._callPlugins(conf.DISPLAY_HOOK_DIRECTORY)
self._generateDisplayWholeMonthStats() self._generateDisplayWholeMonthStats()
build_time = time.localtime()
self.display.build(conf.DISPLAY_ROOT) self.display.build(conf.DISPLAY_ROOT)
self._compressFiles(build_time, conf.DISPLAY_ROOT)
def _createEmptyStats(self): def _createEmptyStats(self):
stats = {} stats = {}
@ -451,7 +475,7 @@ class IWLA(object):
stats['not_viewed_bandwidth'] = 0 stats['not_viewed_bandwidth'] = 0
stats['viewed_pages'] = 0 stats['viewed_pages'] = 0
stats['viewed_hits'] = 0 stats['viewed_hits'] = 0
stats['nb_visitors'] = 0 stats['nb_visits'] = 0
return stats return stats
@ -485,7 +509,7 @@ class IWLA(object):
continue continue
self.valid_visitors[k] = v self.valid_visitors[k] = v
duplicated_stats['visitors'] = stats['visitors'] = len(self.valid_visitors.keys()) duplicated_stats['nb_visitors'] = stats['nb_visitors'] = len(self.valid_visitors.keys())
self._callPlugins(conf.POST_HOOK_DIRECTORY) self._callPlugins(conf.POST_HOOK_DIRECTORY)
@ -535,9 +559,9 @@ class IWLA(object):
if (conf.count_hit_only_visitors or\ if (conf.count_hit_only_visitors or\
viewed_pages) and\ viewed_pages) and\
not super_hit['robot']: not super_hit['robot']:
stats['nb_visitors'] += 1 stats['nb_visits'] += 1
print "== Stats for %d/%02d/%d ==" % (cur_time.tm_year, cur_time.tm_mon, cur_time.tm_mday) print "== Stats for %d/%02d/%02d ==" % (cur_time.tm_year, cur_time.tm_mon, cur_time.tm_mday)
print stats print stats