iwla

iwla Commit Details

Date:2014-12-17 19:00:42 (6 years 7 months ago)
Author:Grégory Soutadé
Branch:dev, master
Commit:c278ff30f0d83278844b96b087aa61d973042278
Parents: b4fc831f0607b9a38f6f1525405592f82ce9524f
Message:Use g.gettext for translations

Changes:
MTODO (1 diff)
Mconf.py (1 diff)
Mdefault_conf.py (1 diff)
Miwla.py (9 diffs)
Mplugins/display/all_visits.py (3 diffs)
Mplugins/display/referers.py (7 diffs)
Mplugins/display/top_downloads.py (3 diffs)
Mplugins/display/top_hits.py (3 diffs)
Mplugins/display/top_pages.py (3 diffs)
Mplugins/display/top_visitors.py (3 diffs)

File differences

TODO
66
77
88
9
9
Automatic tests
Add Licence
Free memory as soon as possible
different debug output levels
filter log with domain name
conf.py
2626
2727
2828
29
30
31
32
max_downloads_displayed = 100
compress_output_files = ['html', 'css', 'js']
# locale = 'fr'
locale = 'fr_FR'
# locale = 'fr_FR.utf8'
default_conf.py
5151
5252
5353
54
55
56
57
58
59
# Extensions to compress in gzip during display build
compress_output_files = []
# Path to locales files
locales_path = './locales'
# Default locale (english)
locale = 'en_EN'
iwla.py
1010
1111
1212
13
1314
1415
1516
......
3233
3334
3435
36
3537
3638
3739
......
130132
131133
132134
135
136
137
138
139
140
141
142
143
144
145
146
147
133148
134149
135150
......
338353
339354
340355
341
356
342357
343358
344359
345360
346361
347
362
348363
349364
350365
......
381396
382397
383398
384
399
385400
386401
387402
388403
389
404
390405
391406
392407
......
395410
396411
397412
398
399
400
413
414
415
401416
402417
403418
......
407422
408423
409424
410
425
411426
412427
413428
......
424439
425440
426441
427
442
428443
429444
430445
431446
432447
433448
434
449
435450
436451
437452
438453
439454
440455
441
456
442457
443458
444459
......
669684
670685
671686
672
687
673688
674689
675690
import importlib
import argparse
import logging
import gettext as g
from calendar import monthrange
from datetime import date
# Conf values needed :
# analyzed_filename
# domain_name
# locales_path
# compress_output_files*
#
# Output files :
self.logger = logging.getLogger(self.__class__.__name__)
self.logger.info('==> Start')
# print conf.locales_path
# print conf.locale
#print g.find('iwla', localedir=conf.locales_path, all=False)
# print g.find('iwla', localedir=conf.locales_path, languages=[conf.locale], all=True)
#g.install('iwla', localedir=conf.locales_path)
t = g.translation('iwla', localedir=conf.locales_path, languages=[conf.locale])
self._ = t.gettext
#t.install()
# g.install('iwla', localedir=conf.locales_path, names=conf.locale)
print(self._('Statistics'))
# print(_('Statistics'))
def getVersion(self):
return IWLA.IWLA_VERSION
def _generateDisplayDaysStats(self):
cur_time = self.meta_infos['last_time']
title = 'Stats %d/%02d' % (cur_time.tm_year, cur_time.tm_mon)
title = '%s %d/%02d' % (g.gettext('Statistics'), cur_time.tm_year, cur_time.tm_mon)
filename = self.getCurDisplayPath('index.html')
self.logger.info('==> Generate display (%s)' % (filename))
page = self.display.createPage(title, filename, conf.css_path)
_, nb_month_days = monthrange(cur_time.tm_year, cur_time.tm_mon)
days = self.display.createBlock(DisplayHTMLBlockTableWithGraph, 'By day', ['Day', 'Visits', 'Pages', 'Hits', 'Bandwidth', 'Not viewed Bandwidth'], None, nb_month_days, range(1,6))
days = self.display.createBlock(DisplayHTMLBlockTableWithGraph, g.gettext('By day'), [g.gettext('Day'), g.gettext('Visits'), g.gettext('Pages'), g.gettext('Hits'), g.gettext('Bandwidth'), g.gettext('Not viewed Bandwidth')], None, nb_month_days, range(1,6))
days.setColsCSSClass(['', 'iwla_visit', 'iwla_page', 'iwla_hit', 'iwla_bandwidth', 'iwla_bandwidth'])
nb_visits = 0
nb_days = 0
else:
average_row = map(lambda(v): 0, row)
average_row[0] = 'Average'
average_row[0] = g.gettext('Average')
average_row[4] = bytesToStr(average_row[4])
average_row[5] = bytesToStr(average_row[5])
days.appendRow(average_row)
row[0] = 'Total'
row[0] = g.gettext('Total')
row[4] = bytesToStr(row[4])
row[5] = bytesToStr(row[5])
days.appendRow(row)
def _generateDisplayMonthStats(self, page, year, month_stats):
cur_time = time.localtime()
months_name = ['', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'June', 'July', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
title = 'Summary %d' % (year)
cols = ['Month', 'Visitors', 'Visits', 'Pages', 'Hits', 'Bandwidth', 'Not viewed Bandwidth', 'Details']
months_name = ['', g.gettext('Jan'), g.gettext('Feb'), g.gettext('Mar'), g.gettext('Apr'), g.gettext('May'), g.gettext('June'), g.gettext('July'), g.gettext('Aug'), g.gettext('Sep'), g.gettext('Oct'), g.gettext('Nov'), g.gettext('Dec')]
title = '%s %d' % (g.gettext('Summary'), year)
cols = [g.gettext('Month'), g.gettext('Visitors'), g.gettext('Visits'), g.gettext('Pages'), g.gettext('Hits'), g.gettext('Bandwidth'), g.gettext('Not viewed Bandwidth'), g.gettext('Details')]
graph_cols=range(1,7)
months = self.display.createBlock(DisplayHTMLBlockTableWithGraph, title, cols, None, 12, graph_cols)
months.setColsCSSClass(['', 'iwla_visitor', 'iwla_visit', 'iwla_page', 'iwla_hit', 'iwla_bandwidth', 'iwla_bandwidth', ''])
full_month = '%s %d' % (months_name[i], year)
if i in month_stats.keys():
stats = month_stats[i]
link = '<a href="%d/%02d/index.html">Details</a>' % (year, i)
link = '<a href="%d/%02d/index.html">%s</a>' % (year, i, g.gettext('Details'))
row = [full_month, stats['nb_visitors'], stats['nb_visits'], stats['viewed_pages'], stats['viewed_hits'],
stats['viewed_bandwidth'], stats['not_viewed_bandwidth'], link]
for j in graph_cols:
else: css = 'iwla_curday'
months.setCellCSSClass(i-1, 0, css)
total[0] = 'Total'
total[0] = g.gettext('Total')
total[5] = bytesToStr(total[5])
total[6] = bytesToStr(total[6])
months.appendRow(total)
page.appendBlock(months)
def _generateDisplayWholeMonthStats(self):
title = 'Stats for %s' % (conf.domain_name)
title = '%s %s' % (g.gettext('Statistics for'), conf.domain_name)
filename = 'index.html'
self.logger.info('==> Generate main page (%s)' % (filename))
page = self.display.createPage(title, filename, conf.css_path)
last_update = '<b>Last update</b> %s<br />' % (time.strftime('%02d %b %Y %H:%M', time.localtime()))
last_update = '<b>%s</b> %s<br />' % (g.gettext('Last update'), time.strftime('%02d %b %Y %H:%M', time.localtime()))
page.appendBlock(self.display.createBlock(DisplayHTMLRaw, last_update))
for year in sorted(self.meta_infos['stats'].keys(), reverse=True):
loglevel = getattr(logging, args.loglevel.upper(), None)
if not isinstance(loglevel, int):
raise ValueError('Invalid log level: %s' % (args.loglevel))
iwla = IWLA(loglevel)
required_conf = ['analyzed_filename', 'domain_name']
plugins/display/all_visits.py
11
2
23
34
45
......
4142
4243
4344
44
45
4546
4647
4748
4849
4950
50
51
5152
5253
5354
......
6970
7071
7172
72
73
73
74
7475
7576
7677
import time
import gettext as g
from iwla import IWLA
from iplugin import IPlugin
last_access = sorted(hits.values(), key=lambda t: t['last_access'], reverse=True)
title = time.strftime('All visits - %B %Y', self.iwla.getCurTime())
title = time.strftime(g.gettext('All visits') + ' - %B %Y', self.iwla.getCurTime())
filename = 'all_visits.html'
path = self.iwla.getCurDisplayPath(filename)
page = display.createPage(title, path, self.iwla.getConfValue('css_path', []))
table = display.createBlock(DisplayHTMLBlockTable, 'Last seen', ['Host', 'Pages', 'Hits', 'Bandwidth', 'Last seen'])
table = display.createBlock(DisplayHTMLBlockTable, g.gettext('Last seen'), [g.gettext('Host'), g.gettext('Pages'), g.gettext('Hits'), g.gettext('Bandwidth'), g.gettext('Last seen')])
table.setColsCSSClass(['', 'iwla_page', 'iwla_hit', 'iwla_bandwidth', ''])
for super_hit in last_access:
display.addPage(page)
index = self.iwla.getDisplayIndex()
link = '<a href=\'%s\'>All visits</a>' % (filename)
block = index.getBlock('Top visitors')
link = '<a href=\'%s\'>%s</a>' % (g.gettext('All visits'), filename)
block = index.getBlock(g.gettext('Top visitors'))
if block:
block.setTitle('%s - %s' % (block.getTitle(), link))
else:
plugins/display/referers.py
11
2
23
34
45
......
6869
6970
7071
71
72
7273
7374
7475
7576
7677
77
78
7879
7980
8081
81
82
8283
8384
8485
......
8788
8889
8990
90
91
9192
9293
9394
......
9697
9798
9899
99
100
100101
101102
102103
......
108109
109110
110111
111
112
112113
113
114
114115
115116
116117
117
118
118119
119120
120
121
121122
122123
123124
124125
125126
126127
127
128
128129
129130
130131
131
132
132133
133134
134135
135136
136137
137138
138
139
139140
140141
141142
142
143
143144
144145
145146
146147
147148
148149
149
150
150151
151152
152153
......
154155
155156
156157
157
158
158159
159160
160161
161162
162163
163164
164
165
165166
166167
167168
......
171172
172173
173174
174
175
175176
176
177
177178
178179
179180
180
181
181182
182183
183184
184185
185186
186
187
187188
188189
189190
import time
import gettext as g
from iwla import IWLA
from iplugin import IPlugin
# All referers in a file
if self.create_all_referers:
title = time.strftime('Connexion from - %B %Y', cur_time)
title = time.strftime(g.gettext('Connexion from') + ' - %B %Y', cur_time)
filename = 'referers.html'
path = self.iwla.getCurDisplayPath(filename)
page = display.createPage(title, path, self.iwla.getConfValue('css_path', []))
table = display.createBlock(DisplayHTMLBlockTable, 'Connexion from', ['Origin', 'Pages', 'Hits'])
table = display.createBlock(DisplayHTMLBlockTable, g.gettext('Connexion from'), [g.gettext('Origin'), g.gettext('Pages'), g.gettext('Hits')])
table.setColsCSSClass(['', 'iwla_page', 'iwla_hit'])
total_search = [0]*3
table.appendRow(['<b>Search Engine</b>', '', ''])
table.appendRow(['<b>%s</b>' % (g.gettext('Search Engine')), '', ''])
new_list = self.max_referers and top_search_engine_referers[:self.max_referers] or top_search_engine_referers
for r,_ in new_list:
row = [r, search_engine_referers[r]['pages'], search_engine_referers[r]['hits']]
table.appendRow(row)
total_external = [0]*3
table.appendRow(['<b>External URL</b>', '', ''])
table.appendRow(['<b>%s</b>' % (g.gettext('External URL')), '', ''])
new_list = self.max_referers and top_referers[:self.max_referers] or top_referers
for r,_ in new_list:
row = [generateHTMLLink(r), referers[r]['pages'], referers[r]['hits']]
table.appendRow(row)
total_robot = [0]*3
table.appendRow(['<b>External URL (robot)</b>', '', ''])
table.appendRow(['<b>%s</b>' % (g.gettext('External URL (robot)')), '', ''])
new_list = self.max_referers and top_robots_referers[:self.max_referers] or top_robots_referers
for r,_ in new_list:
row = [generateHTMLLink(r), robots_referers[r]['pages'], robots_referers[r]['hits']]
display.addPage(page)
title = 'Top Referers'
title = g.gettext('Top Referers')
if self.create_all_referers:
link = '<a href=\'%s\'>All Referers</a>' % (filename)
link = '<a href=\'%s\'>%s</a>' % (filename, g.gettext('All Referers'))
title = '%s - %s' % (title, link)
# Top referers in index
table = display.createBlock(DisplayHTMLBlockTable, title, ['Origin', 'Pages', 'Hits'])
table = display.createBlock(DisplayHTMLBlockTable, title, [g.gettext('Origin'), g.gettext('Pages'), g.gettext('Hits')])
table.setColsCSSClass(['', 'iwla_page', 'iwla_hit'])
table.appendRow(['<b>Search Engine</b>', '', ''])
table.appendRow(['<b>%s</b>' % (g.gettext('Search Engine')), '', ''])
for r,_ in top_search_engine_referers[:10]:
row = [r, search_engine_referers[r]['pages'], search_engine_referers[r]['hits']]
total_search[1] -= search_engine_referers[r]['pages']
total_search[2] -= search_engine_referers[r]['hits']
table.appendRow(row)
if total_search[1] or total_search[2]:
total_search[0] = 'Others'
total_search[0] = g.gettext('Others')
table.appendRow(total_search)
table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others')
table.appendRow(['<b>External URL</b>', '', ''])
table.appendRow(['<b>%s</b>' % (g.gettext('External URL')), '', ''])
for r,_ in top_referers[:10]:
row = [generateHTMLLink(r), referers[r]['pages'], referers[r]['hits']]
total_external[1] -= referers[r]['pages']
total_external[2] -= referers[r]['hits']
table.appendRow(row)
if total_external[1] or total_external[2]:
total_external[0] = 'Others'
total_external[0] = g.gettext('Others')
table.appendRow(total_external)
table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others')
table.appendRow(['<b>External URL (robot)</b>', '', ''])
table.appendRow(['<b>%s</b>' % (g.gettext('External URL (robot)')), '', ''])
for r,_ in top_robots_referers[:10]:
row = [generateHTMLLink(r), robots_referers[r]['pages'], robots_referers[r]['hits']]
total_robot[1] -= robots_referers[r]['pages']
total_robot[2] -= robots_referers[r]['hits']
table.appendRow(row)
if total_robot[1] or total_robot[2]:
total_robot[0] = 'Others'
total_robot[0] = g.gettext('Others')
table.appendRow(total_robot)
table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others')
# All key phrases in a file
if self.create_all_key_phrases:
title = time.strftime('Key Phrases - %B %Y', cur_time)
title = time.strftime(g.gettext('Key Phrases') + ' - %B %Y', cur_time)
filename = 'key_phrases.html'
path = self.iwla.getCurDisplayPath(filename)
total_search = [0]*2
page = display.createPage(title, path, self.iwla.getConfValue('css_path', []))
table = display.createBlock(DisplayHTMLBlockTable, 'Top key phrases', ['Key phrase', 'Search'])
table = display.createBlock(DisplayHTMLBlockTable, g.gettext('Top key phrases'), [g.gettext('Key phrase'), g.gettext('Search')])
table.setColsCSSClass(['', 'iwla_search'])
new_list = self.max_key_phrases and top_key_phrases[:self.max_key_phrases] or top_key_phrases
for phrase in new_list:
display.addPage(page)
title = 'Top key phrases'
title = g.gettext('Top key phrases')
if self.create_all_key_phrases:
link = '<a href=\'%s\'>All key phrases</a>' % (filename)
link = '<a href=\'%s\'>%s</a>' % (filename, g.gettext('All key phrases'))
title = '%s - %s' % (title, link)
# Top key phrases in index
table = display.createBlock(DisplayHTMLBlockTable, title, ['Key phrase', 'Search'])
table = display.createBlock(DisplayHTMLBlockTable, title, [g.gettext('Key phrase'), g.gettext('Search')])
table.setColsCSSClass(['', 'iwla_search'])
for phrase in top_key_phrases[:10]:
table.appendRow([phrase[0], phrase[1]])
total_search[1] -= phrase[1]
if total_search[1]:
total_search[0] = 'Others'
total_search[0] = g.gettext('Others')
table.appendRow(total_search)
table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others')
index.appendBlock(table)
plugins/display/top_downloads.py
11
2
23
34
45
......
4748
4849
4950
50
51
5152
5253
53
54
5455
5556
5657
......
6263
6364
6465
65
66
6667
67
68
6869
6970
7071
7172
7273
73
74
7475
7576
7677
7778
7879
79
80
8081
8182
8283
import time
import gettext as g
from iwla import IWLA
from iplugin import IPlugin
if self.create_all_downloads:
filename = 'top_downloads.html'
path = self.iwla.getCurDisplayPath(filename)
title = time.strftime('All Downloads - %B %Y', self.iwla.getCurTime())
title = time.strftime(g.gettext('All Downloads') + ' - %B %Y', self.iwla.getCurTime())
page = display.createPage(title, path, self.iwla.getConfValue('css_path', []))
table = display.createBlock(DisplayHTMLBlockTable, 'All Downloads', ['URI', 'Hit'])
table = display.createBlock(DisplayHTMLBlockTable, g.gettext('All Downloads'), [g.gettext('URI'), g.gettext('Hit')])
table.setColsCSSClass(['', 'iwla_hit'])
total_entrance = [0]*2
display.addPage(page)
title = 'Top Downloads'
title = g.gettext('Top Downloads')
if self.create_all_downloads:
link = '<a href=\'%s\'>All Downloads</a>' % (filename)
link = '<a href=\'%s\'>%s</a>' % (filename, g.gettext('All Downloads'))
title = '%s - %s' % (title, link)
# Top in index
index = self.iwla.getDisplayIndex()
table = display.createBlock(DisplayHTMLBlockTable, title, ['URI', 'Hits'])
table = display.createBlock(DisplayHTMLBlockTable, title, [g.gettext('URI'), g.gettext('Hits')])
table.setColsCSSClass(['', 'iwla_hit'])
for (uri, entrance) in top_downloads[:10]:
table.appendRow([generateHTMLLink(uri), entrance])
total_entrance[1] -= entrance
if total_entrance[1]:
total_entrance[0] = 'Others'
total_entrance[0] = g.gettext('Others')
table.appendRow(total_entrance)
table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others')
index.appendBlock(table)
plugins/display/top_hits.py
11
2
23
34
45
......
4546
4647
4748
48
49
4950
5051
5152
5253
53
54
5455
5556
5657
......
6364
6465
6566
66
67
6768
6869
6970
7071
7172
72
73
7374
7475
7576
7677
7778
78
79
7980
8081
8182
import time
import gettext as g
from iwla import IWLA
from iplugin import IPlugin
# All in a file
if self.create_all_hits:
title = time.strftime('All Hits - %B %Y', self.iwla.getCurTime())
title = time.strftime(g.gettext('All Hits') + ' - %B %Y', self.iwla.getCurTime())
filename = 'top_hits.html'
path = self.iwla.getCurDisplayPath(filename)
page = display.createPage(title, path, self.iwla.getConfValue('css_path', []))
table = display.createBlock(DisplayHTMLBlockTable, 'All Hits', ['URI', 'Entrance'])
table = display.createBlock(DisplayHTMLBlockTable, g.gettext('All Hits'), [g.gettext('URI'), g.gettext('Entrance')])
table.setColsCSSClass(['', 'iwla_hit'])
total_hits = [0]*2
new_list = self.max_hits and top_hits[:self.max_hits] or top_hits
title = 'Top Hits'
if self.create_all_hits:
link = '<a href=\'%s\'>All Hits</a>' % (filename)
link = '<a href=\'%s\'>%s</a>' % (filename, g.gettext('All Hits'))
title = '%s - %s' % (title, link)
# Top in index
index = self.iwla.getDisplayIndex()
table = display.createBlock(DisplayHTMLBlockTable, title, ['URI', 'Entrance'])
table = display.createBlock(DisplayHTMLBlockTable, title, [g.gettext('URI'), g.gettext('Entrance')])
table.setColsCSSClass(['', 'iwla_hit'])
for (uri, entrance) in top_hits[:10]:
table.appendRow([generateHTMLLink(uri), entrance])
total_hits[1] -= entrance
if total_hits[1]:
total_hits[0] = 'Others'
total_hits[0] = g.gettext('Others')
table.appendRow(total_hits)
table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others')
index.appendBlock(table)
plugins/display/top_pages.py
11
2
23
34
45
......
4546
4647
4748
48
49
4950
5051
5152
5253
53
54
5455
5556
5657
......
6162
6263
6364
64
65
6566
66
67
6768
6869
6970
7071
7172
72
73
7374
7475
7576
7677
7778
78
79
7980
8081
8182
import time
import gettext as g
from iwla import IWLA
from iplugin import IPlugin
# All in a page
if self.create_all_pages:
title = time.strftime('All Pages - %B %Y', self.iwla.getCurTime())
title = time.strftime(g.gettext('All Pages') + ' - %B %Y', self.iwla.getCurTime())
filename = 'top_pages.html'
path = self.iwla.getCurDisplayPath(filename)
page = display.createPage(title, path, self.iwla.getConfValue('css_path', []))
table = display.createBlock(DisplayHTMLBlockTable, 'All Pages', ['URI', 'Entrance'])
table = display.createBlock(DisplayHTMLBlockTable, g.gettext('All Pages'), [g.gettext('URI'), g.gettext('Entrance')])
table.setColsCSSClass(['', 'iwla_hit'])
total_hits = [0]*2
new_list = self.max_pages and top_pages[:self.max_pages] or top_pages
display.addPage(page)
title = 'Top Pages'
title = g.gettext('Top Pages')
if self.create_all_pages:
link = '<a href=\'%s\'>All Pages</a>' % (filename)
link = '<a href=\'%s\'>%s</a>' % (filename, g.gettext('All Pages'))
title = '%s - %s' % (title, link)
# Top in index
index = self.iwla.getDisplayIndex()
table = display.createBlock(DisplayHTMLBlockTable, title, ['URI', 'Entrance'])
table = display.createBlock(DisplayHTMLBlockTable, title, [g.gettext('URI'), g.gettext('Entrance')])
table.setColsCSSClass(['', 'iwla_hit'])
for (uri, entrance) in top_pages[:10]:
table.appendRow([generateHTMLLink(uri), entrance])
total_hits[1] -= entrance
if total_hits[1]:
total_hits[0] = 'Others'
total_hits[0] = g.gettext('Others')
table.appendRow(total_hits)
table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others')
index.appendBlock(table)
plugins/display/top_visitors.py
11
2
23
34
45
......
4950
5051
5152
52
53
5354
5455
5556
......
6970
7071
7172
72
73
7374
7475
7576
import time
import gettext as g
from iwla import IWLA
from iplugin import IPlugin
top_visitors = [hits[h[0]] for h in top_bandwidth[:10]]
index = self.iwla.getDisplayIndex()
table = display.createBlock(DisplayHTMLBlockTable, 'Top visitors', ['Host', 'Pages', 'Hits', 'Bandwidth', 'Last seen'])
table = display.createBlock(DisplayHTMLBlockTable, g.gettext('Top visitors'), [g.gettext('Host'), g.gettext('Pages'), g.gettext('Hits'), g.gettext('Bandwidth'), g.gettext('Last seen')])
table.setColsCSSClass(['', 'iwla_page', 'iwla_hit', 'iwla_bandwidth', ''])
for super_hit in top_visitors:
address = super_hit['remote_addr']
total[3] -= super_hit['bandwidth']
table.appendRow(row)
if total[1] or total[2] or total[3]:
total[0] = 'Others'
total[0] = g.gettext('Others')
total[3] = bytesToStr(total[3])
total[4] = ''
table.appendRow(total)

Archive Download the corresponding diff file

Branches

Tags