soutade
/
iwla
1
0
Fork 0
Code Issues Pull Requests Releases 2 Activity

Merge branch 'master' of soutade.fr:iwla

This commit is contained in:
Gregory Soutade 2014-12-17 21:32:19 +01:00
parent a21786c0e2 a74a6b8469
commit 9f0770ec52
16 changed files with 678 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
TODO
View File

@ -1,10 +1,7 @@
Other when pages truncated Other when pages truncated
translations
doc auto generation doc auto generation
doc enhancement doc enhancement
Limit hits/pages/downloads by rate Limit hits/pages/downloads by rate
Automatic tests Automatic tests
Add Licence Add Licence
Free memory as soon as possible Free memory as soon as possible
gzip output files
different debug output levels

3
conf.py
View File

@ -26,3 +26,6 @@ max_hits_displayed = 100
max_downloads_displayed = 100 max_downloads_displayed = 100
compress_output_files = ['html', 'css', 'js'] compress_output_files = ['html', 'css', 'js']
locale = 'fr'
# locale = 'fr_FR.utf8'

6
default_conf.py
View File

@ -51,3 +51,9 @@ css_path = ['%s/%s/%s' % (os.path.basename(resources_path[0]), 'css', 'iwla.css'
# Extensions to compress in gzip during display build # Extensions to compress in gzip during display build
compress_output_files = [] compress_output_files = []
# Path to locales files
locales_path = './locales'
# Default locale (english)
locale = 'en_EN'

17
display.py
View File

@ -1,6 +1,7 @@
import os import os
import codecs import codecs
import time import time
import logging
# #
# Create output HTML files # Create output HTML files
@ -22,14 +23,8 @@ class DisplayHTMLRaw(object):
if html: f.write(html) if html: f.write(html)
def build(self, f): def build(self, f):
# t1 = time.time()
self._buildHTML() self._buildHTML()
# t2 = time.time()
# print 'Time for _buildHTML : %d seconds' % (t2-t1)
# t1 = time.time()
self._build(f, self.html) self._build(f, self.html)
# t2 = time.time()
# print 'Time for _build : %d seconds' % (t2-t1)
class DisplayHTMLBlock(DisplayHTMLRaw): class DisplayHTMLBlock(DisplayHTMLRaw):
@ -252,6 +247,7 @@ class DisplayHTMLPage(object):
self.filename = filename self.filename = filename
self.blocks = [] self.blocks = []
self.css_path = listToStr(css_path) self.css_path = listToStr(css_path)
self.logger = logging.getLogger(self.__class__.__name__)
def getFilename(self): def getFilename(self):
return self.filename; return self.filename;
@ -272,6 +268,8 @@ class DisplayHTMLPage(object):
if not os.path.exists(base): if not os.path.exists(base):
os.makedirs(base) os.makedirs(base)
self.logger.debug('Write %s' % (filename))
f = codecs.open(filename, 'w', 'utf-8') f = codecs.open(filename, 'w', 'utf-8')
f.write(u'<!DOCTYPE html>') f.write(u'<!DOCTYPE html>')
f.write(u'<html>') f.write(u'<html>')
@ -321,9 +319,7 @@ class DisplayHTMLBuild(object):
os.symlink(target, link_name) os.symlink(target, link_name)
for page in self.pages: for page in self.pages:
# print 'Build %s' % (page.filename)
page.build(root) page.build(root)
# print 'Built'
# #
# Global functions # Global functions
@ -352,3 +348,8 @@ def generateHTMLLink(url, name=None, max_length=100, prefix=u'http'):
if not name: name = unicode(url) if not name: name = unicode(url)
if not url.startswith(prefix): url = u'%s://%s' % (prefix, url) if not url.startswith(prefix): url = u'%s://%s' % (prefix, url)
return u'<a href="%s">%s</a>' % (url, name[:max_length]) return u'<a href="%s">%s</a>' % (url, name[:max_length])
def createCurTitle(iwla, title):
title = iwla._(title) + time.strftime(u' - %B %Y', iwla.getCurTime())
return title

17
iplugin.py
View File

@ -1,6 +1,6 @@
import importlib import importlib
import inspect import inspect
import traceback import logging
# #
# IWLA Plugin interface # IWLA Plugin interface
@ -47,7 +47,9 @@ def validConfRequirements(conf_requirements, iwla, plugin_path):
def preloadPlugins(plugins, iwla): def preloadPlugins(plugins, iwla):
cache_plugins = {} cache_plugins = {}
print "==> Preload plugins" logger = logging.getLogger(__name__)
logger.info("==> Preload plugins")
for (root, plugins_filenames) in plugins: for (root, plugins_filenames) in plugins:
for plugin_filename in plugins_filenames: for plugin_filename in plugins_filenames:
@ -61,7 +63,7 @@ def preloadPlugins(plugins, iwla):
] ]
if not classes: if not classes:
print 'No plugin defined in %s' % (plugin_path) logger.warning('No plugin defined in %s' % (plugin_path))
continue continue
plugin = classes[0](iwla) plugin = classes[0](iwla)
@ -86,18 +88,17 @@ def preloadPlugins(plugins, iwla):
requirement_validated = True requirement_validated = True
break break
if not requirement_validated: if not requirement_validated:
print 'Missing requirements \'%s\' for plugin %s' % (r, plugin_path) logger.error('Missing requirements \'%s\' for plugin %s' % (r, plugin_path))
break break
if requirements and not requirement_validated: continue if requirements and not requirement_validated: continue
if not plugin.load(): if not plugin.load():
print 'Plugin %s load failed' % (plugin_path) logger.error('Plugin %s load failed' % (plugin_path))
continue continue
print '\tRegister %s' % (plugin_path) logger.info('\tRegister %s' % (plugin_path))
cache_plugins[plugin_path] = plugin cache_plugins[plugin_path] = plugin
except Exception as e: except Exception as e:
print 'Error loading %s => %s' % (plugin_path, e) logger.exception('Error loading %s => %s' % (plugin_path, e))
traceback.print_exc()
return cache_plugins return cache_plugins

245
iwla.pot Normal file
View File

@ -0,0 +1,245 @@
# SOME DESCRIPTIVE TITLE.
# Copyright (C) YEAR ORGANIZATION
# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
#
msgid ""
msgstr ""
"Project-Id-Version: iwla\n"
"POT-Creation-Date: 2014-12-16 21:36+CET\n"
"PO-Revision-Date: 2014-12-17 19:07+0100\n"
"Last-Translator: Soutadé <soutade@gmail.com>\n"
"Language-Team: iwla <iwla>\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Generated-By: pygettext.py 1.5\n"
"Plural-Forms: nplurals=2; plural=(n != 1);\n"
"Language: en\n"
"X-Generator: Poedit 1.6.10\n"
"X-Poedit-SourceCharset: UTF-8\n"
#: iwla.py:343
msgid "Statistics"
msgstr ""
#: iwla.py:349
msgid "By day"
msgstr ""
#: iwla.py:349
msgid "Day"
msgstr ""
#: iwla.py:349 iwla.py:402
msgid "Not viewed Bandwidth"
msgstr ""
#: iwla.py:349 iwla.py:402
msgid "Visits"
msgstr ""
#: iwla.py:349 iwla.py:402 plugins/display/all_visits.py:51
#: plugins/display/referers.py:78 plugins/display/referers.py:118
#: plugins/display/top_downloads.py:74 plugins/display/top_visitors.py:53
msgid "Hits"
msgstr ""
#: iwla.py:349 iwla.py:402 plugins/display/all_visits.py:51
#: plugins/display/referers.py:78 plugins/display/referers.py:118
#: plugins/display/top_visitors.py:53
msgid "Pages"
msgstr ""
#: iwla.py:349 iwla.py:402 plugins/display/all_visits.py:51
#: plugins/display/top_visitors.py:53
msgid "Bandwidth"
msgstr ""
#: iwla.py:386
msgid "Average"
msgstr ""
#: iwla.py:391 iwla.py:429
msgid "Total"
msgstr ""
#: iwla.py:400
msgid "Apr"
msgstr ""
#: iwla.py:400
msgid "Aug"
msgstr ""
#: iwla.py:400
msgid "Dec"
msgstr ""
#: iwla.py:400
msgid "Feb"
msgstr ""
#: iwla.py:400
msgid "Jan"
msgstr ""
#: iwla.py:400
msgid "July"
msgstr ""
#: iwla.py:400
msgid "June"
msgstr ""
#: iwla.py:400
msgid "Mar"
msgstr ""
#: iwla.py:400
msgid "May"
msgstr ""
#: iwla.py:400
msgid "Nov"
msgstr ""
#: iwla.py:400
msgid "Oct"
msgstr ""
#: iwla.py:400
msgid "Sep"
msgstr ""
#: iwla.py:401
msgid "Summary"
msgstr ""
#: iwla.py:402
msgid "Month"
msgstr ""
#: iwla.py:402
msgid "Visitors"
msgstr ""
#: iwla.py:402 iwla.py:412
msgid "Details"
msgstr ""
#: iwla.py:436
msgid "Statistics for"
msgstr ""
#: iwla.py:443
msgid "Last update"
msgstr ""
#: plugins/display/all_visits.py:45 plugins/display/all_visits.py:73
msgid "All visits"
msgstr ""
#: plugins/display/all_visits.py:51 plugins/display/top_visitors.py:53
msgid "Host"
msgstr ""
#: plugins/display/all_visits.py:51 plugins/display/top_visitors.py:53
msgid "Last seen"
msgstr ""
#: plugins/display/all_visits.py:74 plugins/display/top_visitors.py:53
msgid "Top visitors"
msgstr ""
#: plugins/display/referers.py:72 plugins/display/referers.py:78
msgid "Connexion from"
msgstr ""
#: plugins/display/referers.py:78 plugins/display/referers.py:118
msgid "Origin"
msgstr ""
#: plugins/display/referers.py:82 plugins/display/referers.py:121
msgid "Search Engine"
msgstr ""
#: plugins/display/referers.py:91 plugins/display/referers.py:132
msgid "External URL"
msgstr ""
#: plugins/display/referers.py:100 plugins/display/referers.py:143
msgid "External URL (robot)"
msgstr ""
#: plugins/display/referers.py:112
msgid "Top Referers"
msgstr ""
#: plugins/display/referers.py:114
msgid "All Referers"
msgstr ""
#: plugins/display/referers.py:128 plugins/display/referers.py:139
#: plugins/display/referers.py:150 plugins/display/referers.py:187
#: plugins/display/top_downloads.py:80 plugins/display/top_hits.py:79
#: plugins/display/top_pages.py:79 plugins/display/top_visitors.py:73
msgid "Others"
msgstr ""
#: plugins/display/referers.py:158
msgid "Key Phrases"
msgstr ""
#: plugins/display/referers.py:165 plugins/display/referers.py:175
msgid "Top key phrases"
msgstr ""
#: plugins/display/referers.py:165 plugins/display/referers.py:181
msgid "Key phrase"
msgstr ""
#: plugins/display/referers.py:165 plugins/display/referers.py:181
msgid "Search"
msgstr ""
#: plugins/display/referers.py:177
msgid "All key phrases"
msgstr ""
#: plugins/display/top_downloads.py:51 plugins/display/top_downloads.py:54
#: plugins/display/top_downloads.py:68
msgid "All Downloads"
msgstr ""
#: plugins/display/top_downloads.py:54
msgid "Hit"
msgstr ""
#: plugins/display/top_downloads.py:54 plugins/display/top_downloads.py:74
#: plugins/display/top_hits.py:54 plugins/display/top_hits.py:73
#: plugins/display/top_pages.py:54 plugins/display/top_pages.py:73
msgid "URI"
msgstr ""
#: plugins/display/top_downloads.py:66
msgid "Top Downloads"
msgstr ""
#: plugins/display/top_hits.py:49 plugins/display/top_hits.py:54
#: plugins/display/top_hits.py:67
msgid "All Hits"
msgstr ""
#: plugins/display/top_hits.py:54 plugins/display/top_hits.py:73
#: plugins/display/top_pages.py:54 plugins/display/top_pages.py:73
msgid "Entrance"
msgstr ""
#: plugins/display/top_pages.py:49 plugins/display/top_pages.py:54
#: plugins/display/top_pages.py:67
msgid "All Pages"
msgstr ""
#: plugins/display/top_pages.py:65
msgid "Top Pages"
msgstr ""

99
iwla.py
View File

@ -9,6 +9,8 @@ import pickle
import gzip import gzip
import importlib import importlib
import argparse import argparse
import logging
import gettext
from calendar import monthrange from calendar import monthrange
from datetime import date from datetime import date
@ -31,6 +33,7 @@ from display import *
# Conf values needed : # Conf values needed :
# analyzed_filename # analyzed_filename
# domain_name # domain_name
# locales_path
# compress_output_files* # compress_output_files*
# #
# Output files : # Output files :
@ -108,9 +111,7 @@ class IWLA(object):
API_VERSION = 1 API_VERSION = 1
IWLA_VERSION = '0.1' IWLA_VERSION = '0.1'
def __init__(self): def __init__(self, logLevel):
print '==> Start'
self.meta_infos = {} self.meta_infos = {}
self.analyse_started = False self.analyse_started = False
self.current_analysis = {} self.current_analysis = {}
@ -123,10 +124,22 @@ class IWLA(object):
self.http_request_extracted = re.compile(r'(?P<http_method>\S+) (?P<http_uri>\S+) (?P<http_version>\S+)') self.http_request_extracted = re.compile(r'(?P<http_method>\S+) (?P<http_uri>\S+) (?P<http_version>\S+)')
self.log_re = re.compile(self.log_format_extracted) self.log_re = re.compile(self.log_format_extracted)
self.uri_re = re.compile(r'(?P<extract_uri>[^\?]+)(\?(?P<extract_parameters>.+))?') self.uri_re = re.compile(r'(?P<extract_uri>[^\?]+)(\?(?P<extract_parameters>.+))?')
self.domain_name_re = re.compile(r'.*%s' % conf.domain_name)
self.plugins = [(conf.PRE_HOOK_DIRECTORY , conf.pre_analysis_hooks), self.plugins = [(conf.PRE_HOOK_DIRECTORY , conf.pre_analysis_hooks),
(conf.POST_HOOK_DIRECTORY , conf.post_analysis_hooks), (conf.POST_HOOK_DIRECTORY , conf.post_analysis_hooks),
(conf.DISPLAY_HOOK_DIRECTORY , conf.display_hooks)] (conf.DISPLAY_HOOK_DIRECTORY , conf.display_hooks)]
logging.basicConfig(format='%(name)s %(message)s', level=logLevel)
self.logger = logging.getLogger(self.__class__.__name__)
self.logger.info('==> Start')
try:
t = gettext.translation('iwla', localedir=conf.locales_path, languages=[conf.locale], codeset='utf8')
self.logger.info('\tUsing locale %s' % (conf.locale))
except IOError:
t = gettext.NullTranslations()
self.logger.info('\tUsing default locale en_EN')
self._ = t.ugettext
def getVersion(self): def getVersion(self):
return IWLA.IWLA_VERSION return IWLA.IWLA_VERSION
@ -223,13 +236,13 @@ class IWLA(object):
return None return None
def _callPlugins(self, target_root, *args): def _callPlugins(self, target_root, *args):
print '==> Call plugins (%s)' % target_root self.logger.info('==> Call plugins (%s)' % (target_root))
for (root, plugins) in self.plugins: for (root, plugins) in self.plugins:
if root != target_root: continue if root != target_root: continue
for p in plugins: for p in plugins:
mod = self.cache_plugins.get(root + '.' + p, None) mod = self.cache_plugins.get(root + '.' + p, None)
if mod: if mod:
print '\t%s' % (p) self.logger.info('\t%s' % (p))
mod.hook(*args) mod.hook(*args)
def isPage(self, request): def isPage(self, request):
@ -299,7 +312,7 @@ class IWLA(object):
if 'extract_parameters' in d.keys(): if 'extract_parameters' in d.keys():
hit['extract_request']['extract_parameters'] = d['extract_parameters'] hit['extract_request']['extract_parameters'] = d['extract_parameters']
else: else:
print "Bad request extraction " + hit['request'] self.logger.warning("Bad request extraction %s" % (hit['request']))
return False return False
if hit['http_referer']: if hit['http_referer']:
@ -335,13 +348,13 @@ class IWLA(object):
def _generateDisplayDaysStats(self): def _generateDisplayDaysStats(self):
cur_time = self.meta_infos['last_time'] cur_time = self.meta_infos['last_time']
title = 'Stats %d/%02d' % (cur_time.tm_year, cur_time.tm_mon) title = '%s %d/%02d' % (self._('Statistics'), cur_time.tm_year, cur_time.tm_mon)
filename = self.getCurDisplayPath('index.html') filename = self.getCurDisplayPath('index.html')
print '==> Generate display (%s)' % (filename) self.logger.info('==> Generate display (%s)' % (filename))
page = self.display.createPage(title, filename, conf.css_path) page = self.display.createPage(title, filename, conf.css_path)
_, nb_month_days = monthrange(cur_time.tm_year, cur_time.tm_mon) _, nb_month_days = monthrange(cur_time.tm_year, cur_time.tm_mon)
days = self.display.createBlock(DisplayHTMLBlockTableWithGraph, 'By day', ['Day', 'Visits', 'Pages', 'Hits', 'Bandwidth', 'Not viewed Bandwidth'], None, nb_month_days, range(1,6)) days = self.display.createBlock(DisplayHTMLBlockTableWithGraph, self._('By day'), [self._('Day'), self._('Visits'), self._('Pages'), self._('Hits'), self._('Bandwidth'), self._('Not viewed Bandwidth')], None, nb_month_days, range(1,6))
days.setColsCSSClass(['', 'iwla_visit', 'iwla_page', 'iwla_hit', 'iwla_bandwidth', 'iwla_bandwidth']) days.setColsCSSClass(['', 'iwla_visit', 'iwla_page', 'iwla_hit', 'iwla_bandwidth', 'iwla_bandwidth'])
nb_visits = 0 nb_visits = 0
nb_days = 0 nb_days = 0
@ -378,12 +391,12 @@ class IWLA(object):
else: else:
average_row = map(lambda(v): 0, row) average_row = map(lambda(v): 0, row)
average_row[0] = 'Average' average_row[0] = self._('Average')
average_row[4] = bytesToStr(average_row[4]) average_row[4] = bytesToStr(average_row[4])
average_row[5] = bytesToStr(average_row[5]) average_row[5] = bytesToStr(average_row[5])
days.appendRow(average_row) days.appendRow(average_row)
row[0] = 'Total' row[0] = self._('Total')
row[4] = bytesToStr(row[4]) row[4] = bytesToStr(row[4])
row[5] = bytesToStr(row[5]) row[5] = bytesToStr(row[5])
days.appendRow(row) days.appendRow(row)
@ -392,9 +405,9 @@ class IWLA(object):
def _generateDisplayMonthStats(self, page, year, month_stats): def _generateDisplayMonthStats(self, page, year, month_stats):
cur_time = time.localtime() cur_time = time.localtime()
months_name = ['', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'June', 'July', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'] months_name = ['', self._('Jan'), self._('Feb'), self._('Mar'), self._('Apr'), self._('May'), self._('June'), self._('July'), self._('Aug'), self._('Sep'), self._('Oct'), self._('Nov'), self._('Dec')]
title = 'Summary %d' % (year) title = '%s %d' % (self._('Summary'), year)
cols = ['Month', 'Visitors', 'Visits', 'Pages', 'Hits', 'Bandwidth', 'Not viewed Bandwidth', 'Details'] cols = [self._('Month'), self._('Visitors'), self._('Visits'), self._('Pages'), self._('Hits'), self._('Bandwidth'), self._('Not viewed Bandwidth'), self._('Details')]
graph_cols=range(1,7) graph_cols=range(1,7)
months = self.display.createBlock(DisplayHTMLBlockTableWithGraph, title, cols, None, 12, graph_cols) months = self.display.createBlock(DisplayHTMLBlockTableWithGraph, title, cols, None, 12, graph_cols)
months.setColsCSSClass(['', 'iwla_visitor', 'iwla_visit', 'iwla_page', 'iwla_hit', 'iwla_bandwidth', 'iwla_bandwidth', '']) months.setColsCSSClass(['', 'iwla_visitor', 'iwla_visit', 'iwla_page', 'iwla_hit', 'iwla_bandwidth', 'iwla_bandwidth', ''])
@ -404,7 +417,7 @@ class IWLA(object):
full_month = '%s %d' % (months_name[i], year) full_month = '%s %d' % (months_name[i], year)
if i in month_stats.keys(): if i in month_stats.keys():
stats = month_stats[i] stats = month_stats[i]
link = '<a href="%d/%02d/index.html">Details</a>' % (year, i) link = '<a href="%d/%02d/index.html">%s</a>' % (year, i, self._('Details'))
row = [full_month, stats['nb_visitors'], stats['nb_visits'], stats['viewed_pages'], stats['viewed_hits'], row = [full_month, stats['nb_visitors'], stats['nb_visits'], stats['viewed_pages'], stats['viewed_hits'],
stats['viewed_bandwidth'], stats['not_viewed_bandwidth'], link] stats['viewed_bandwidth'], stats['not_viewed_bandwidth'], link]
for j in graph_cols: for j in graph_cols:
@ -421,20 +434,21 @@ class IWLA(object):
else: css = 'iwla_curday' else: css = 'iwla_curday'
months.setCellCSSClass(i-1, 0, css) months.setCellCSSClass(i-1, 0, css)
total[0] = 'Total' total[0] = self._('Total')
total[5] = bytesToStr(total[5]) total[5] = bytesToStr(total[5])
total[6] = bytesToStr(total[6]) total[6] = bytesToStr(total[6])
months.appendRow(total) months.appendRow(total)
page.appendBlock(months) page.appendBlock(months)
def _generateDisplayWholeMonthStats(self): def _generateDisplayWholeMonthStats(self):
title = 'Stats for %s' % (conf.domain_name) title = '%s %s' % (self._('Statistics for'), conf.domain_name)
filename = 'index.html' filename = 'index.html'
print '==> Generate main page (%s)' % (filename)
self.logger.info('==> Generate main page (%s)' % (filename))
page = self.display.createPage(title, filename, conf.css_path) page = self.display.createPage(title, filename, conf.css_path)
last_update = '<b>Last update</b> %s<br />' % (time.strftime('%02d %b %Y %H:%M', time.localtime())) last_update = '<b>%s</b> %s<br />' % (self._('Last update'), time.strftime('%02d %b %Y %H:%M', time.localtime()))
page.appendBlock(self.display.createBlock(DisplayHTMLRaw, last_update)) page.appendBlock(self.display.createBlock(DisplayHTMLRaw, last_update))
for year in sorted(self.meta_infos['stats'].keys(), reverse=True): for year in sorted(self.meta_infos['stats'].keys(), reverse=True):
@ -445,7 +459,9 @@ class IWLA(object):
def _compressFile(self, build_time, root, filename): def _compressFile(self, build_time, root, filename):
path = os.path.join(root, filename) path = os.path.join(root, filename)
gz_path = path + '.gz' gz_path = path + '.gz'
#print 'Compress %s => %s' % (path, gz_path)
self.logger.debug('Compress %s => %s' % (path, gz_path))
if not os.path.exists(gz_path) or\ if not os.path.exists(gz_path) or\
os.stat(path).st_mtime > build_time: os.stat(path).st_mtime > build_time:
with open(path, 'rb') as f_in: with open(path, 'rb') as f_in:
@ -492,8 +508,8 @@ class IWLA(object):
duplicated_stats = {k:v for (k,v) in stats.items()} duplicated_stats = {k:v for (k,v) in stats.items()}
cur_time = self.meta_infos['last_time'] cur_time = self.meta_infos['last_time']
print "== Stats for %d/%02d ==" % (cur_time.tm_year, cur_time.tm_mon) self.logger.info("== Stats for %d/%02d ==" % (cur_time.tm_year, cur_time.tm_mon))
print stats self.logger.info(stats)
if not 'month_stats' in self.current_analysis.keys(): if not 'month_stats' in self.current_analysis.keys():
self.current_analysis['month_stats'] = stats self.current_analysis['month_stats'] = stats
@ -517,7 +533,7 @@ class IWLA(object):
if os.path.exists(path): if os.path.exists(path):
os.remove(path) os.remove(path)
print "==> Serialize to %s" % path self.logger.info("==> Serialize to %s" % (path))
self._serialize(self.current_analysis, path) self._serialize(self.current_analysis, path)
# Save month stats # Save month stats
@ -561,13 +577,15 @@ class IWLA(object):
not super_hit['robot']: not super_hit['robot']:
stats['nb_visits'] += 1 stats['nb_visits'] += 1
print "== Stats for %d/%02d/%02d ==" % (cur_time.tm_year, cur_time.tm_mon, cur_time.tm_mday) self.logger.info("== Stats for %d/%02d/%02d ==" % (cur_time.tm_year, cur_time.tm_mon, cur_time.tm_mday))
self.logger.info(stats)
print stats
self.current_analysis['days_stats'][cur_time.tm_mday] = stats self.current_analysis['days_stats'][cur_time.tm_mday] = stats
def _newHit(self, hit): def _newHit(self, hit):
if not self.domain_name_re.match(hit['server_name']):
return False
t = self._decodeTime(hit) t = self._decodeTime(hit)
cur_time = self.meta_infos['last_time'] cur_time = self.meta_infos['last_time']
@ -595,18 +613,18 @@ class IWLA(object):
for k in hit.keys(): for k in hit.keys():
if hit[k] == '-' or hit[k] == '*': if hit[k] == '-' or hit[k] == '*':
hit[k] = '' hit[k] = ''
self._appendHit(hit) self._appendHit(hit)
return True return True
def start(self, _file): def start(self, _file):
print '==> Load previous database' self.logger.info('==> Load previous database')
self.meta_infos = self._deserialize(conf.META_PATH) or self._clearMeta() self.meta_infos = self._deserialize(conf.META_PATH) or self._clearMeta()
if self.meta_infos['last_time']: if self.meta_infos['last_time']:
print 'Last time' self.logger.info('Last time')
print self.meta_infos['last_time'] self.logger.info(self.meta_infos['last_time'])
self.current_analysis = self._deserialize(self.getDBFilename(self.meta_infos['last_time'])) or self._clearVisits() self.current_analysis = self._deserialize(self.getDBFilename(self.meta_infos['last_time'])) or self._clearVisits()
else: else:
self._clearVisits() self._clearVisits()
@ -615,7 +633,7 @@ class IWLA(object):
self.cache_plugins = preloadPlugins(self.plugins, self) self.cache_plugins = preloadPlugins(self.plugins, self)
print '==> Analysing log' self.logger.info('==> Analysing log')
for l in _file: for l in _file:
# print "line " + l # print "line " + l
@ -623,10 +641,9 @@ class IWLA(object):
groups = self.log_re.match(l) groups = self.log_re.match(l)
if groups: if groups:
if not self._newHit(groups.groupdict()): self._newHit(groups.groupdict())
continue
else: else:
print "No match for " + l self.logger.warning("No match for %s" % (l))
#break #break
if self.analyse_started: if self.analyse_started:
@ -635,7 +652,7 @@ class IWLA(object):
del self.meta_infos['start_analysis_time'] del self.meta_infos['start_analysis_time']
self._serialize(self.meta_infos, conf.META_PATH) self._serialize(self.meta_infos, conf.META_PATH)
else: else:
print '==> Analyse not started : nothing new' self.logger.info('==> Analyse not started : nothing new')
if __name__ == '__main__': if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Intelligent Web Log Analyzer') parser = argparse.ArgumentParser(description='Intelligent Web Log Analyzer')
@ -651,14 +668,22 @@ if __name__ == '__main__':
parser.add_argument('-f', '--file', dest='file', parser.add_argument('-f', '--file', dest='file',
help='Analyse this log file') help='Analyse this log file')
parser.add_argument('-d', '--log-level', dest='loglevel',
default='INFO', type=str,
help='Loglevel in %s, default : %s' % (['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], 'INFO'))
args = parser.parse_args() args = parser.parse_args()
if args.clean_output: if args.clean_output:
if os.path.exists(conf.DB_ROOT): shutil.rmtree(conf.DB_ROOT) if os.path.exists(conf.DB_ROOT): shutil.rmtree(conf.DB_ROOT)
if os.path.exists(conf.DISPLAY_ROOT): shutil.rmtree(conf.DISPLAY_ROOT) if os.path.exists(conf.DISPLAY_ROOT): shutil.rmtree(conf.DISPLAY_ROOT)
iwla = IWLA() loglevel = getattr(logging, args.loglevel.upper(), None)
if not isinstance(loglevel, int):
raise ValueError('Invalid log level: %s' % (args.loglevel))
iwla = IWLA(loglevel)
required_conf = ['analyzed_filename', 'domain_name'] required_conf = ['analyzed_filename', 'domain_name']
if not validConfRequirements(required_conf, iwla, 'Main Conf'): if not validConfRequirements(required_conf, iwla, 'Main Conf'):
sys.exit(0) sys.exit(0)

20
iwla_get.sh Executable file
View File

@ -0,0 +1,20 @@
#!/bin/bash
rm -f mega_log
for i in `seq 53 -1 2` ; do
[ ! -f "cybelle/soutade.fr_access.log.$i.gz" ] && continue
zcat "cybelle/soutade.fr_access.log.$i.gz" >> mega_log
echo "cybelle/soutade.fr_access.log.$i.gz"
done
#cat "cybelle/soutade.fr_access.log.2" >> mega_log
echo "cybelle/soutade.fr_access.log.1"
cat "cybelle/soutade.fr_access.log.1" >> mega_log
# cat "cybelle/soutade.fr_access.log" >> mega_log
cat mega_log | ./iwla.py -i -c
# cat "cybelle/soutade.fr_access.log.1" | ./iwla.py -i
# cat "cybelle/soutade.fr_access.log" | ./iwla.py -i
# TODO : reverse analysis
# -f option
# other when pages truncated

BIN
locales/fr_FR/LC_MESSAGES/iwla.mo Normal file
View File

Binary file not shown.

244
locales/fr_FR/LC_MESSAGES/iwla.pot Normal file
View File

@ -0,0 +1,244 @@
# SOME DESCRIPTIVE TITLE.
# Copyright (C) YEAR ORGANIZATION
# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
#
msgid ""
msgstr ""
"Project-Id-Version: iwla\n"
"POT-Creation-Date: 2014-12-16 21:36+CET\n"
"PO-Revision-Date: 2014-12-17 19:06+0100\n"
"Last-Translator: Soutadé <soutade@gmail.com>\n"
"Language-Team: iwla\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Generated-By: pygettext.py 1.5\n"
"Language: fr_FR\n"
"X-Generator: Poedit 1.6.10\n"
"X-Poedit-SourceCharset: UTF-8\n"
#: iwla.py:343
msgid "Statistics"
msgstr "Statistiques"
#: iwla.py:349
msgid "By day"
msgstr "Par jour"
#: iwla.py:349
msgid "Day"
msgstr "Jour"
#: iwla.py:349 iwla.py:402
msgid "Not viewed Bandwidth"
msgstr "Traffic non vu"
#: iwla.py:349 iwla.py:402
msgid "Visits"
msgstr "Visites"
#: iwla.py:349 iwla.py:402 plugins/display/all_visits.py:51
#: plugins/display/referers.py:78 plugins/display/referers.py:118
#: plugins/display/top_downloads.py:74 plugins/display/top_visitors.py:53
msgid "Hits"
msgstr "Hits"
#: iwla.py:349 iwla.py:402 plugins/display/all_visits.py:51
#: plugins/display/referers.py:78 plugins/display/referers.py:118
#: plugins/display/top_visitors.py:53
msgid "Pages"
msgstr "Pages"
#: iwla.py:349 iwla.py:402 plugins/display/all_visits.py:51
#: plugins/display/top_visitors.py:53
msgid "Bandwidth"
msgstr "Bande passante"
#: iwla.py:386
msgid "Average"
msgstr "Moyenne"
#: iwla.py:391 iwla.py:429
msgid "Total"
msgstr "Total"
#: iwla.py:400
msgid "Apr"
msgstr "Avr"
#: iwla.py:400
msgid "Aug"
msgstr "Août"
#: iwla.py:400
msgid "Dec"
msgstr "Déc"
#: iwla.py:400
msgid "Feb"
msgstr "Fév"
#: iwla.py:400
msgid "Jan"
msgstr "Jan"
#: iwla.py:400
msgid "July"
msgstr "Jui"
#: iwla.py:400
msgid "June"
msgstr "Juin"
#: iwla.py:400
msgid "Mar"
msgstr "Mars"
#: iwla.py:400
msgid "May"
msgstr "Mai"
#: iwla.py:400
msgid "Nov"
msgstr "Nov"
#: iwla.py:400
msgid "Oct"
msgstr "Oct"
#: iwla.py:400
msgid "Sep"
msgstr "Sep"
#: iwla.py:401
msgid "Summary"
msgstr "Résumé"
#: iwla.py:402
msgid "Month"
msgstr "Mois"
#: iwla.py:402
msgid "Visitors"
msgstr "Visiteurs"
#: iwla.py:402 iwla.py:412
msgid "Details"
msgstr "Détails"
#: iwla.py:436
msgid "Statistics for"
msgstr "Statistiques pour"
#: iwla.py:443
msgid "Last update"
msgstr "Dernière mise à jour"
#: plugins/display/all_visits.py:45 plugins/display/all_visits.py:73
msgid "All visits"
msgstr "Toutes les visites"
#: plugins/display/all_visits.py:51 plugins/display/top_visitors.py:53
msgid "Host"
msgstr "Hôte"
#: plugins/display/all_visits.py:51 plugins/display/top_visitors.py:53
msgid "Last seen"
msgstr "Dernière visite"
#: plugins/display/all_visits.py:74 plugins/display/top_visitors.py:53
msgid "Top visitors"
msgstr "Top visiteurs"
#: plugins/display/referers.py:72 plugins/display/referers.py:78
msgid "Connexion from"
msgstr "Connexion depuis"
#: plugins/display/referers.py:78 plugins/display/referers.py:118
msgid "Origin"
msgstr "Origine"
#: plugins/display/referers.py:82 plugins/display/referers.py:121
msgid "Search Engine"
msgstr "Moteur de recherche"
#: plugins/display/referers.py:91 plugins/display/referers.py:132
msgid "External URL"
msgstr "URL externe"
#: plugins/display/referers.py:100 plugins/display/referers.py:143
msgid "External URL (robot)"
msgstr "URL externe (robot)"
#: plugins/display/referers.py:112
msgid "Top Referers"
msgstr "Top Origines"
#: plugins/display/referers.py:114
msgid "All Referers"
msgstr "Toutes les origines"
#: plugins/display/referers.py:128 plugins/display/referers.py:139
#: plugins/display/referers.py:150 plugins/display/referers.py:187
#: plugins/display/top_downloads.py:80 plugins/display/top_hits.py:79
#: plugins/display/top_pages.py:79 plugins/display/top_visitors.py:73
msgid "Others"
msgstr "Autres"
#: plugins/display/referers.py:158
msgid "Key Phrases"
msgstr "Phrases clé"
#: plugins/display/referers.py:165 plugins/display/referers.py:175
msgid "Top key phrases"
msgstr "Top phrases clé"
#: plugins/display/referers.py:165 plugins/display/referers.py:181
msgid "Key phrase"
msgstr "Phrase clé"
#: plugins/display/referers.py:165 plugins/display/referers.py:181
msgid "Search"
msgstr "Recherche"
#: plugins/display/referers.py:177
msgid "All key phrases"
msgstr "Toutes les phrases clé"
#: plugins/display/top_downloads.py:51 plugins/display/top_downloads.py:54
#: plugins/display/top_downloads.py:68
msgid "All Downloads"
msgstr "Tous les téléchargements"
#: plugins/display/top_downloads.py:54
msgid "Hit"
msgstr "Hit"
#: plugins/display/top_downloads.py:54 plugins/display/top_downloads.py:74
#: plugins/display/top_hits.py:54 plugins/display/top_hits.py:73
#: plugins/display/top_pages.py:54 plugins/display/top_pages.py:73
msgid "URI"
msgstr "URI"
#: plugins/display/top_downloads.py:66
msgid "Top Downloads"
msgstr "Top Téléchargements"
#: plugins/display/top_hits.py:49 plugins/display/top_hits.py:54
#: plugins/display/top_hits.py:67
msgid "All Hits"
msgstr "Tous les hits"
#: plugins/display/top_hits.py:54 plugins/display/top_hits.py:73
#: plugins/display/top_pages.py:54 plugins/display/top_pages.py:73
msgid "Entrance"
msgstr "Entrées"
#: plugins/display/top_pages.py:49 plugins/display/top_pages.py:54
#: plugins/display/top_pages.py:67
msgid "All Pages"
msgstr "Toutes les pages"
#: plugins/display/top_pages.py:65
msgid "Top Pages"
msgstr "Top Pages"

8
plugins/display/all_visits.py
View File

@ -41,13 +41,13 @@ class IWLADisplayAllVisits(IPlugin):
last_access = sorted(hits.values(), key=lambda t: t['last_access'], reverse=True) last_access = sorted(hits.values(), key=lambda t: t['last_access'], reverse=True)
title = time.strftime('All visits - %B %Y', self.iwla.getCurTime()) title = time.strftime(self.iwla._(u'All visits') + u' - %B %Y', self.iwla.getCurTime())
filename = 'all_visits.html' filename = 'all_visits.html'
path = self.iwla.getCurDisplayPath(filename) path = self.iwla.getCurDisplayPath(filename)
page = display.createPage(title, path, self.iwla.getConfValue('css_path', [])) page = display.createPage(title, path, self.iwla.getConfValue('css_path', []))
table = display.createBlock(DisplayHTMLBlockTable, 'Last seen', ['Host', 'Pages', 'Hits', 'Bandwidth', 'Last seen']) table = display.createBlock(DisplayHTMLBlockTable, self.iwla._(u'Last seen'), [self.iwla._(u'Host'), self.iwla._(u'Pages'), self.iwla._(u'Hits'), self.iwla._(u'Bandwidth'), self.iwla._(u'Last seen')])
table.setColsCSSClass(['', 'iwla_page', 'iwla_hit', 'iwla_bandwidth', '']) table.setColsCSSClass(['', 'iwla_page', 'iwla_hit', 'iwla_bandwidth', ''])
for super_hit in last_access: for super_hit in last_access:
@ -69,8 +69,8 @@ class IWLADisplayAllVisits(IPlugin):
display.addPage(page) display.addPage(page)
index = self.iwla.getDisplayIndex() index = self.iwla.getDisplayIndex()
link = '<a href=\'%s\'>All visits</a>' % (filename) link = '<a href=\'%s\'>%s</a>' % (filename, self.iwla._(u'All visits'))
block = index.getBlock('Top visitors') block = index.getBlock(self.iwla._(u'Top visitors'))
if block: if block:
block.setTitle('%s - %s' % (block.getTitle(), link)) block.setTitle('%s - %s' % (block.getTitle(), link))
else: else:

60
plugins/display/referers.py
View File

@ -1,5 +1,3 @@
import time
from iwla import IWLA from iwla import IWLA
from iplugin import IPlugin from iplugin import IPlugin
from display import * from display import *
@ -68,85 +66,103 @@ class IWLADisplayReferers(IPlugin):
# All referers in a file # All referers in a file
if self.create_all_referers: if self.create_all_referers:
title = time.strftime('Connexion from - %B %Y', cur_time) title = createCurTitle(self.iwla, u'Connexion from')
filename = 'referers.html' filename = 'referers.html'
path = self.iwla.getCurDisplayPath(filename) path = self.iwla.getCurDisplayPath(filename)
page = display.createPage(title, path, self.iwla.getConfValue('css_path', [])) page = display.createPage(title, path, self.iwla.getConfValue('css_path', []))
table = display.createBlock(DisplayHTMLBlockTable, 'Connexion from', ['Origin', 'Pages', 'Hits']) table = display.createBlock(DisplayHTMLBlockTable, self.iwla._(u'Connexion from'), [self.iwla._(u'Origin'), self.iwla._(u'Pages'), self.iwla._(u'Hits')])
table.setColsCSSClass(['', 'iwla_page', 'iwla_hit']) table.setColsCSSClass(['', 'iwla_page', 'iwla_hit'])
total_search = [0]*3 total_search = [0]*3
table.appendRow(['<b>Search Engine</b>', '', '']) table.appendRow(['<b>%s</b>' % (self.iwla._(u'Search Engine')), '', ''])
new_list = self.max_referers and top_search_engine_referers[:self.max_referers] or top_search_engine_referers new_list = self.max_referers and top_search_engine_referers[:self.max_referers] or top_search_engine_referers
for r,_ in new_list: for r,_ in new_list:
row = [r, search_engine_referers[r]['pages'], search_engine_referers[r]['hits']] row = [r, search_engine_referers[r]['pages'], search_engine_referers[r]['hits']]
total_search[1] += search_engine_referers[r]['pages'] total_search[1] += search_engine_referers[r]['pages']
total_search[2] += search_engine_referers[r]['hits'] total_search[2] += search_engine_referers[r]['hits']
table.appendRow(row) table.appendRow(row)
if self.max_referers:
others = 0
for (uri, entrance) in top_referers[self.max_referers:]:
others += entrance
table.appendRow([self.iwla._(u'Others'), others])
table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others')
total_external = [0]*3 total_external = [0]*3
table.appendRow(['<b>External URL</b>', '', '']) table.appendRow(['<b>%s</b>' % (self.iwla._(u'External URL')), '', ''])
new_list = self.max_referers and top_referers[:self.max_referers] or top_referers new_list = self.max_referers and top_referers[:self.max_referers] or top_referers
for r,_ in new_list: for r,_ in new_list:
row = [generateHTMLLink(r), referers[r]['pages'], referers[r]['hits']] row = [generateHTMLLink(r), referers[r]['pages'], referers[r]['hits']]
total_external[1] += referers[r]['pages'] total_external[1] += referers[r]['pages']
total_external[2] += referers[r]['hits'] total_external[2] += referers[r]['hits']
table.appendRow(row) table.appendRow(row)
if self.max_referers:
others = 0
for (uri, entrance) in top_referers[self.max_referers:]:
others += entrance
table.appendRow([self.iwla._(u'Others'), others])
table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others')
total_robot = [0]*3 total_robot = [0]*3
table.appendRow(['<b>External URL (robot)</b>', '', '']) table.appendRow(['<b>%s</b>' % (self.iwla._(u'External URL (robot)')), '', ''])
new_list = self.max_referers and top_robots_referers[:self.max_referers] or top_robots_referers new_list = self.max_referers and top_robots_referers[:self.max_referers] or top_robots_referers
for r,_ in new_list: for r,_ in new_list:
row = [generateHTMLLink(r), robots_referers[r]['pages'], robots_referers[r]['hits']] row = [generateHTMLLink(r), robots_referers[r]['pages'], robots_referers[r]['hits']]
total_robot[1] += robots_referers[r]['pages'] total_robot[1] += robots_referers[r]['pages']
total_robot[2] += robots_referers[r]['hits'] total_robot[2] += robots_referers[r]['hits']
table.appendRow(row) table.appendRow(row)
if self.max_referers:
others = 0
for (uri, entrance) in top_referers[self.max_referers:]:
others += entrance
table.appendRow([self.iwla._(u'Others'), others])
table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others')
page.appendBlock(table) page.appendBlock(table)
display.addPage(page) display.addPage(page)
title = 'Top Referers' title = self.iwla._(u'Top Referers')
if self.create_all_referers: if self.create_all_referers:
link = '<a href=\'%s\'>All Referers</a>' % (filename) link = '<a href=\'%s\'>%s</a>' % (filename, self.iwla._(u'All Referers'))
title = '%s - %s' % (title, link) title = '%s - %s' % (title, link)
# Top referers in index # Top referers in index
table = display.createBlock(DisplayHTMLBlockTable, title, ['Origin', 'Pages', 'Hits']) table = display.createBlock(DisplayHTMLBlockTable, title, [self.iwla._(u'Origin'), self.iwla._(u'Pages'), self.iwla._(u'Hits')])
table.setColsCSSClass(['', 'iwla_page', 'iwla_hit']) table.setColsCSSClass(['', 'iwla_page', 'iwla_hit'])
table.appendRow(['<b>Search Engine</b>', '', '']) table.appendRow(['<b>%s</b>' % (self.iwla._(u'Search Engine')), '', ''])
for r,_ in top_search_engine_referers[:10]: for r,_ in top_search_engine_referers[:10]:
row = [r, search_engine_referers[r]['pages'], search_engine_referers[r]['hits']] row = [r, search_engine_referers[r]['pages'], search_engine_referers[r]['hits']]
total_search[1] -= search_engine_referers[r]['pages'] total_search[1] -= search_engine_referers[r]['pages']
total_search[2] -= search_engine_referers[r]['hits'] total_search[2] -= search_engine_referers[r]['hits']
table.appendRow(row) table.appendRow(row)
if total_search[1] or total_search[2]: if total_search[1] or total_search[2]:
total_search[0] = 'Others' total_search[0] = self.iwla._(u'Others')
table.appendRow(total_search) table.appendRow(total_search)
table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others') table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others')
table.appendRow(['<b>External URL</b>', '', '']) table.appendRow(['<b>%s</b>' % (self.iwla._(u'External URL')), '', ''])
for r,_ in top_referers[:10]: for r,_ in top_referers[:10]:
row = [generateHTMLLink(r), referers[r]['pages'], referers[r]['hits']] row = [generateHTMLLink(r), referers[r]['pages'], referers[r]['hits']]
total_external[1] -= referers[r]['pages'] total_external[1] -= referers[r]['pages']
total_external[2] -= referers[r]['hits'] total_external[2] -= referers[r]['hits']
table.appendRow(row) table.appendRow(row)
if total_external[1] or total_external[2]: if total_external[1] or total_external[2]:
total_external[0] = 'Others' total_external[0] = self.iwla._(u'Others')
table.appendRow(total_external) table.appendRow(total_external)
table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others') table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others')
table.appendRow(['<b>External URL (robot)</b>', '', '']) table.appendRow(['<b>%s</b>' % (self.iwla._(u'External URL (robot)')), '', ''])
for r,_ in top_robots_referers[:10]: for r,_ in top_robots_referers[:10]:
row = [generateHTMLLink(r), robots_referers[r]['pages'], robots_referers[r]['hits']] row = [generateHTMLLink(r), robots_referers[r]['pages'], robots_referers[r]['hits']]
total_robot[1] -= robots_referers[r]['pages'] total_robot[1] -= robots_referers[r]['pages']
total_robot[2] -= robots_referers[r]['hits'] total_robot[2] -= robots_referers[r]['hits']
table.appendRow(row) table.appendRow(row)
if total_robot[1] or total_robot[2]: if total_robot[1] or total_robot[2]:
total_robot[0] = 'Others' total_robot[0] = self.iwla._(u'Others')
table.appendRow(total_robot) table.appendRow(total_robot)
table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others') table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others')
@ -154,14 +170,14 @@ class IWLADisplayReferers(IPlugin):
# All key phrases in a file # All key phrases in a file
if self.create_all_key_phrases: if self.create_all_key_phrases:
title = time.strftime('Key Phrases - %B %Y', cur_time) title = createCurTitle(self.iwla, u'Key Phrases')
filename = 'key_phrases.html' filename = 'key_phrases.html'
path = self.iwla.getCurDisplayPath(filename) path = self.iwla.getCurDisplayPath(filename)
total_search = [0]*2 total_search = [0]*2
page = display.createPage(title, path, self.iwla.getConfValue('css_path', [])) page = display.createPage(title, path, self.iwla.getConfValue('css_path', []))
table = display.createBlock(DisplayHTMLBlockTable, 'Top key phrases', ['Key phrase', 'Search']) table = display.createBlock(DisplayHTMLBlockTable, self.iwla._(u'Top key phrases'), [self.iwla._(u'Key phrase'), self.iwla._(u'Search')])
table.setColsCSSClass(['', 'iwla_search']) table.setColsCSSClass(['', 'iwla_search'])
new_list = self.max_key_phrases and top_key_phrases[:self.max_key_phrases] or top_key_phrases new_list = self.max_key_phrases and top_key_phrases[:self.max_key_phrases] or top_key_phrases
for phrase in new_list: for phrase in new_list:
@ -171,19 +187,19 @@ class IWLADisplayReferers(IPlugin):
display.addPage(page) display.addPage(page)
title = 'Top key phrases' title = self.iwla._(u'Top key phrases')
if self.create_all_key_phrases: if self.create_all_key_phrases:
link = '<a href=\'%s\'>All key phrases</a>' % (filename) link = '<a href=\'%s\'>%s</a>' % (filename, self.iwla._(u'All key phrases'))
title = '%s - %s' % (title, link) title = '%s - %s' % (title, link)
# Top key phrases in index # Top key phrases in index
table = display.createBlock(DisplayHTMLBlockTable, title, ['Key phrase', 'Search']) table = display.createBlock(DisplayHTMLBlockTable, title, [self.iwla._(u'Key phrase'), self.iwla._(u'Search')])
table.setColsCSSClass(['', 'iwla_search']) table.setColsCSSClass(['', 'iwla_search'])
for phrase in top_key_phrases[:10]: for phrase in top_key_phrases[:10]:
table.appendRow([phrase[0], phrase[1]]) table.appendRow([phrase[0], phrase[1]])
total_search[1] -= phrase[1] total_search[1] -= phrase[1]
if total_search[1]: if total_search[1]:
total_search[0] = 'Others' total_search[0] = self.iwla._(u'Others')
table.appendRow(total_search) table.appendRow(total_search)
table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others') table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others')
index.appendBlock(table) index.appendBlock(table)

20
plugins/display/top_downloads.py
View File

@ -1,5 +1,3 @@
import time
from iwla import IWLA from iwla import IWLA
from iplugin import IPlugin from iplugin import IPlugin
from display import * from display import *
@ -47,10 +45,10 @@ class IWLADisplayTopDownloads(IPlugin):
if self.create_all_downloads: if self.create_all_downloads:
filename = 'top_downloads.html' filename = 'top_downloads.html'
path = self.iwla.getCurDisplayPath(filename) path = self.iwla.getCurDisplayPath(filename)
title = time.strftime('All Downloads - %B %Y', self.iwla.getCurTime()) title = createCurTitle(self.iwla, u'All Downloads')
page = display.createPage(title, path, self.iwla.getConfValue('css_path', [])) page = display.createPage(title, path, self.iwla.getConfValue('css_path', []))
table = display.createBlock(DisplayHTMLBlockTable, 'All Downloads', ['URI', 'Hit']) table = display.createBlock(DisplayHTMLBlockTable, self.iwla._(u'All Downloads'), [self.iwla._(u'URI'), self.iwla._(u'Hit')])
table.setColsCSSClass(['', 'iwla_hit']) table.setColsCSSClass(['', 'iwla_hit'])
total_entrance = [0]*2 total_entrance = [0]*2
@ -58,25 +56,31 @@ class IWLADisplayTopDownloads(IPlugin):
for (uri, entrance) in new_list: for (uri, entrance) in new_list:
table.appendRow([generateHTMLLink(uri), entrance]) table.appendRow([generateHTMLLink(uri), entrance])
total_entrance[1] += entrance total_entrance[1] += entrance
if self.max_downloads:
others = 0
for (uri, entrance) in top_downloads[self.max_downloads:]:
others += entrance
table.appendRow([self.iwla._(u'Others'), others])
table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others')
page.appendBlock(table) page.appendBlock(table)
display.addPage(page) display.addPage(page)
title = 'Top Downloads' title = self.iwla._(u'Top Downloads')
if self.create_all_downloads: if self.create_all_downloads:
link = '<a href=\'%s\'>All Downloads</a>' % (filename) link = '<a href=\'%s\'>%s</a>' % (filename, self.iwla._(u'All Downloads'))
title = '%s - %s' % (title, link) title = '%s - %s' % (title, link)
# Top in index # Top in index
index = self.iwla.getDisplayIndex() index = self.iwla.getDisplayIndex()
table = display.createBlock(DisplayHTMLBlockTable, title, ['URI', 'Hits']) table = display.createBlock(DisplayHTMLBlockTable, title, [self.iwla._(u'URI'), self.iwla._(u'Hits')])
table.setColsCSSClass(['', 'iwla_hit']) table.setColsCSSClass(['', 'iwla_hit'])
for (uri, entrance) in top_downloads[:10]: for (uri, entrance) in top_downloads[:10]:
table.appendRow([generateHTMLLink(uri), entrance]) table.appendRow([generateHTMLLink(uri), entrance])
total_entrance[1] -= entrance total_entrance[1] -= entrance
if total_entrance[1]: if total_entrance[1]:
total_entrance[0] = 'Others' total_entrance[0] = self.iwla._(u'Others')
table.appendRow(total_entrance) table.appendRow(total_entrance)
table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others') table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others')
index.appendBlock(table) index.appendBlock(table)

19
plugins/display/top_hits.py
View File

@ -1,5 +1,3 @@
import time
from iwla import IWLA from iwla import IWLA
from iplugin import IPlugin from iplugin import IPlugin
from display import * from display import *
@ -45,37 +43,44 @@ class IWLADisplayTopHits(IPlugin):
# All in a file # All in a file
if self.create_all_hits: if self.create_all_hits:
title = time.strftime('All Hits - %B %Y', self.iwla.getCurTime()) title = createCurTitle(self.iwla, u'All Hits')
filename = 'top_hits.html' filename = 'top_hits.html'
path = self.iwla.getCurDisplayPath(filename) path = self.iwla.getCurDisplayPath(filename)
page = display.createPage(title, path, self.iwla.getConfValue('css_path', [])) page = display.createPage(title, path, self.iwla.getConfValue('css_path', []))
table = display.createBlock(DisplayHTMLBlockTable, 'All Hits', ['URI', 'Entrance']) table = display.createBlock(DisplayHTMLBlockTable, self.iwla._(u'All Hits'), [self.iwla._(u'URI'), self.iwla._(u'Entrance')])
table.setColsCSSClass(['', 'iwla_hit']) table.setColsCSSClass(['', 'iwla_hit'])
total_hits = [0]*2 total_hits = [0]*2
new_list = self.max_hits and top_hits[:self.max_hits] or top_hits new_list = self.max_hits and top_hits[:self.max_hits] or top_hits
for (uri, entrance) in new_list: for (uri, entrance) in new_list:
table.appendRow([generateHTMLLink(uri), entrance]) table.appendRow([generateHTMLLink(uri), entrance])
total_hits[1] += entrance total_hits[1] += entrance
if self.max_hits:
others = 0
for (uri, entrance) in top_hits[self.max_hits:]:
others += entrance
table.appendRow([self.iwla._(u'Others'), others])
table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others')
page.appendBlock(table) page.appendBlock(table)
display.addPage(page) display.addPage(page)
title = 'Top Hits' title = 'Top Hits'
if self.create_all_hits: if self.create_all_hits:
link = '<a href=\'%s\'>All Hits</a>' % (filename) link = '<a href=\'%s\'>%s</a>' % (filename, self.iwla._(u'All Hits'))
title = '%s - %s' % (title, link) title = '%s - %s' % (title, link)
# Top in index # Top in index
index = self.iwla.getDisplayIndex() index = self.iwla.getDisplayIndex()
table = display.createBlock(DisplayHTMLBlockTable, title, ['URI', 'Entrance']) table = display.createBlock(DisplayHTMLBlockTable, title, [self.iwla._(u'URI'), self.iwla._(u'Entrance')])
table.setColsCSSClass(['', 'iwla_hit']) table.setColsCSSClass(['', 'iwla_hit'])
for (uri, entrance) in top_hits[:10]: for (uri, entrance) in top_hits[:10]:
table.appendRow([generateHTMLLink(uri), entrance]) table.appendRow([generateHTMLLink(uri), entrance])
total_hits[1] -= entrance total_hits[1] -= entrance
if total_hits[1]: if total_hits[1]:
total_hits[0] = 'Others' total_hits[0] = self.iwla._(u'Others')
table.appendRow(total_hits) table.appendRow(total_hits)
table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others') table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others')
index.appendBlock(table) index.appendBlock(table)

20
plugins/display/top_pages.py
View File

@ -1,5 +1,3 @@
import time
from iwla import IWLA from iwla import IWLA
from iplugin import IPlugin from iplugin import IPlugin
from display import * from display import *
@ -45,37 +43,43 @@ class IWLADisplayTopPages(IPlugin):
# All in a page # All in a page
if self.create_all_pages: if self.create_all_pages:
title = time.strftime('All Pages - %B %Y', self.iwla.getCurTime()) title = createCurTitle(self.iwla, u'All Pages')
filename = 'top_pages.html' filename = 'top_pages.html'
path = self.iwla.getCurDisplayPath(filename) path = self.iwla.getCurDisplayPath(filename)
page = display.createPage(title, path, self.iwla.getConfValue('css_path', [])) page = display.createPage(title, path, self.iwla.getConfValue('css_path', []))
table = display.createBlock(DisplayHTMLBlockTable, 'All Pages', ['URI', 'Entrance']) table = display.createBlock(DisplayHTMLBlockTable, self.iwla._(u'All Pages'), [self.iwla._(u'URI'), self.iwla._(u'Entrance')])
table.setColsCSSClass(['', 'iwla_hit']) table.setColsCSSClass(['', 'iwla_hit'])
total_hits = [0]*2 total_hits = [0]*2
new_list = self.max_pages and top_pages[:self.max_pages] or top_pages new_list = self.max_pages and top_pages[:self.max_pages] or top_pages
for (uri, entrance) in new_list: for (uri, entrance) in new_list:
table.appendRow([generateHTMLLink(uri), entrance]) table.appendRow([generateHTMLLink(uri), entrance])
total_hits[1] += entrance total_hits[1] += entrance
if self.max_pages:
others = 0
for (uri, entrance) in top_pages[self.max_pages:]:
others += entrance
table.appendRow([self.iwla._(u'Others'), others])
table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others')
page.appendBlock(table) page.appendBlock(table)
display.addPage(page) display.addPage(page)
title = 'Top Pages' title = self.iwla._(u'Top Pages')
if self.create_all_pages: if self.create_all_pages:
link = '<a href=\'%s\'>All Pages</a>' % (filename) link = '<a href=\'%s\'>%s</a>' % (filename, self.iwla._(u'All Pages'))
title = '%s - %s' % (title, link) title = '%s - %s' % (title, link)
# Top in index # Top in index
index = self.iwla.getDisplayIndex() index = self.iwla.getDisplayIndex()
table = display.createBlock(DisplayHTMLBlockTable, title, ['URI', 'Entrance']) table = display.createBlock(DisplayHTMLBlockTable, title, [self.iwla._(u'URI'), self.iwla._(u'Entrance')])
table.setColsCSSClass(['', 'iwla_hit']) table.setColsCSSClass(['', 'iwla_hit'])
for (uri, entrance) in top_pages[:10]: for (uri, entrance) in top_pages[:10]:
table.appendRow([generateHTMLLink(uri), entrance]) table.appendRow([generateHTMLLink(uri), entrance])
total_hits[1] -= entrance total_hits[1] -= entrance
if total_hits[1]: if total_hits[1]:
total_hits[0] = 'Others' total_hits[0] = self.iwla._(u'Others')
table.appendRow(total_hits) table.appendRow(total_hits)
table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others') table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others')
index.appendBlock(table) index.appendBlock(table)

4
plugins/display/top_visitors.py
View File

@ -49,7 +49,7 @@ class IWLADisplayTopVisitors(IPlugin):
top_visitors = [hits[h[0]] for h in top_bandwidth[:10]] top_visitors = [hits[h[0]] for h in top_bandwidth[:10]]
index = self.iwla.getDisplayIndex() index = self.iwla.getDisplayIndex()
table = display.createBlock(DisplayHTMLBlockTable, 'Top visitors', ['Host', 'Pages', 'Hits', 'Bandwidth', 'Last seen']) table = display.createBlock(DisplayHTMLBlockTable, self.iwla._(u'Top visitors'), [self.iwla._(u'Host'), self.iwla._(u'Pages'), self.iwla._(u'Hits'), self.iwla._(u'Bandwidth'), self.iwla._(u'Last seen')])
table.setColsCSSClass(['', 'iwla_page', 'iwla_hit', 'iwla_bandwidth', '']) table.setColsCSSClass(['', 'iwla_page', 'iwla_hit', 'iwla_bandwidth', ''])
for super_hit in top_visitors: for super_hit in top_visitors:
address = super_hit['remote_addr'] address = super_hit['remote_addr']
@ -69,7 +69,7 @@ class IWLADisplayTopVisitors(IPlugin):
total[3] -= super_hit['bandwidth'] total[3] -= super_hit['bandwidth']
table.appendRow(row) table.appendRow(row)
if total[1] or total[2] or total[3]: if total[1] or total[2] or total[3]:
total[0] = 'Others' total[0] = self.iwla._(u'Others')
total[3] = bytesToStr(total[3]) total[3] = bytesToStr(total[3])
total[4] = '' total[4] = ''
table.appendRow(total) table.appendRow(total)