iwla

iwla Commit Details

Date:2014-12-04 21:04:41 (6 years 7 months ago)
Author:Grégory Soutadé
Branch:dev, master
Commit:2362fd1fd26460556dd06b11c55c179575d8eb30
Parents: 5f72a9c91236654100c0e694e2f653d8f7eebfcd
Message:Fix unicode problems Add generateHTMLLink()

Changes:
Mdisplay.py (11 diffs)
Miwla.py (1 diff)
Mplugins/display/referers.py (3 diffs)
Mplugins/display/top_downloads.py (2 diffs)
Mplugins/display/top_hits.py (2 diffs)
Mplugins/display/top_pages.py (2 diffs)
Mplugins/post_analysis/referers.py (2 diffs)

File differences

display.py
11
2
23
34
45
5
6
67
78
89
......
2324
2425
2526
26
27
28
27
28
29
2930
3031
3132
3233
3334
34
35
3536
3637
37
38
3839
3940
40
41
4142
4243
43
44
4445
4546
46
47
4748
48
49
50
49
50
51
5152
5253
5354
......
5556
5657
5758
58
59
5960
60
61
6162
62
63
6364
6465
6566
66
67
6768
6869
6970
......
7778
7879
7980
80
81
8182
8283
8384
8485
8586
8687
87
88
8889
8990
9091
......
103104
104105
105106
106
107
107108
108109
109110
110111
111112
112
113
113114
114115
115116
116117
117118
118
119
119120
120121
121
122
123
122
123
124
124125
125
126
126127
127128
128129
129
130
131
130
131
132
132133
133134
134
135
135136
136137
137138
138
139
140
141
139
140
141
142
142143
143144
144145
......
149150
150151
151152
153
152154
153155
154
156
155157
156158
157159
158
159
160
161
160162
161163
162164
......
164166
165167
166168
167
169
168170
169171
170
172
171173
172174
173175
......
180182
181183
182184
183
184
185
186
187
185
186
187
188
189
188190
189191
190
192
191193
192194
193195
194196
195
196
197
198
197
198
199
200
199201
200202
201
202
203
203
204
205
204206
205207
206208
207209
208
209
210
211
210212
211213
212214
213215
214
215
216
217
216
217
218
219
218220
219221
220
221
222
223
222
223
224
225
224226
225227
226228
......
229231
230232
231233
232
234
233235
234236
235
237
236238
237239
238240
......
253255
254256
255257
256
257
258
259
260
258
259
260
261
262
261263
262
264
263265
264
265
266
267
266268
267269
268
270
269271
270272
271273
......
295297
296298
297299
298
300
299301
300302
301303
302304
303305
304306
305
307
306308
307
309
308310
309311
310
312
311313
312314
313315
316
317
318
319
320
321
import os
import codecs
class DisplayHTMLRaw(object):
def __init__(self, html=''):
def __init__(self, html=u''):
self.html = html
def setRawHTML(self, html):
def __init__(self, title=''):
super(DisplayHTMLBlock, self).__init__(html='')
self.title = title
self.cssclass = 'iwla_block'
self.title_cssclass = 'iwla_block_title'
self.value_cssclass = 'iwla_block_value'
self.cssclass = u'iwla_block'
self.title_cssclass = u'iwla_block_title'
self.value_cssclass = u'iwla_block_value'
def getTitle(self):
return self.title
def setTitle(self, value):
self.title = value
self.title = unicode(value)
def setCSSClass(self, cssclass):
self.cssclass = cssclass
self.cssclass = unicode(cssclass)
def setTitleCSSClass(self, cssclass):
self.title_cssclass = cssclass
self.title_cssclass = unicode(cssclass)
def setValueCSSClass(self, cssclass):
self.value_cssclass = cssclass
self.value_cssclass = unicode(cssclass)
def _buildHTML(self):
html = '<div class="%s">' % (self.cssclass)
html = u'<div class="%s">' % (self.cssclass)
if self.title:
html += '<div class="%s">%s</div>' % (self.title_cssclass, self.title)
html += '<div class="%s">%s</div>' % (self.value_cssclass, self.html)
html += '</div>'
html += u'<div class="%s">%s</div>' % (self.title_cssclass, self.title)
html += u'<div class="%s">%s</div>' % (self.value_cssclass, self.html)
html += u'</div>'
self.html = html
def __init__(self, title, cols):
super(DisplayHTMLBlockTable, self).__init__(title=title)
self.cols = cols
self.cols = listToStr(cols)
self.rows = []
self.cols_cssclasses = [''] * len(cols)
self.cols_cssclasses = [u''] * len(cols)
self.rows_cssclasses = []
self.table_css = 'iwla_table'
self.table_css = u'iwla_table'
def appendRow(self, row):
self.rows.append(listToStr(row))
self.rows_cssclasses.append([''] * len(row))
self.rows_cssclasses.append([u''] * len(row))
def getCellValue(self, row, col):
if row < 0 or col < 0 or\
row >= len(self.rows) or col >= len(self.cols):
raise ValueError('Invalid indices %d,%d' % (row, col))
self.rows[row][col] = value
self.rows[row][col] = unicode(value)
def setCellCSSClass(self, row, col, value):
if row < 0 or col < 0 or\
row >= len(self.rows) or col >= len(self.cols):
raise ValueError('Invalid indices %d,%d' % (row, col))
self.rows_cssclasses[row][col] = value
self.rows_cssclasses[row][col] = unicode(value)
def getCellCSSClass(self, row, col):
if row < 0 or col < 0 or\
if row < 0 or row >= len(self.rows):
raise ValueError('Invalid indice %d' % (row))
self.rows_cssclasses[row] = [value] * len(self.rows_cssclasses[row])
self.rows_cssclasses[row] = [unicode(value)] * len(self.rows_cssclasses[row])
def setColCSSClass(self, col, value):
if col < 0 or col >= len(self.cols):
raise ValueError('Invalid indice %d' % (col))
self.cols_cssclasses[col] = value
self.cols_cssclasses[col] = unicode(value)
def setColsCSSClass(self, values):
if len(values) != len(self.cols):
raise ValueError('Invalid values size')
self.cols_cssclasses = values
self.cols_cssclasses = [unicode(values)] * len(self.cols)
def _buildHTML(self):
style = ''
if self.table_css: style = ' class="%s"' % (self.table_css)
html = '<table%s>' % (style)
style = u''
if self.table_css: style = u' class="%s"' % (self.table_css)
html = u'<table%s>' % (style)
if self.cols:
html += '<tr>'
html += u'<tr>'
for i in range (0, len(self.cols)):
title = self.cols[i]
style = self.getColCSSClass(i)
if style: style = ' class="%s"' % (style)
html += '<th%s>%s</th>' % (style, title)
html += '</tr>'
if style: style = u' class="%s"' % (style)
html += u'<th%s>%s</th>' % (style, title)
html += u'</tr>'
for i in range(0, len(self.rows)):
row = self.rows[i]
html += '<tr>'
html += u'<tr>'
for j in range(0, len(row)):
v = row[j]
style = self.getCellCSSClass(i, j)
if style: style = ' class="%s"' % (style)
html += '<td%s>%s</td>' % (style, v)
html += '</tr>'
html += '</table>'
if style: style = u' class="%s"' % (style)
html += u'<td%s>%s</td>' % (style, v)
html += u'</tr>'
html += u'</table>'
self.html += html
def __init__(self, title, cols, short_titles=None, nb_valid_rows=0, graph_cols=None):
super(DisplayHTMLBlockTableWithGraph, self).__init__(title=title, cols=cols)
self.short_titles = short_titles or []
self.short_titles = listToStr(self.short_titles)
self.nb_valid_rows = nb_valid_rows
# TOFIX
self.icon_path = 'resources/icon'
self.icon_path = u'resources/icon'
# self.icon_path = self.iwla.getConfValue('icon_path', '/')
self.raw_rows = []
self.maxes = [0] * len(cols)
self.table_graph_css = 'iwla_graph_table'
self.td_img_css = 'iwla_td_img'
self.table_graph_css = u'iwla_graph_table'
self.td_img_css = u'iwla_td_img'
self.graph_cols = graph_cols or []
def appendRow(self, row):
super(DisplayHTMLBlockTableWithGraph, self).appendRow(row)
def appendShortTitle(self, short_title):
self.short_titles.append(short_title)
self.short_titles.append(unicode(short_title))
def setShortTitle(self, short_titles):
self.short_titles = short_titles
self.short_titles = listToStr(short_titles)
def setNbValidRows(self, nb_valid_rows):
self.nb_valid_rows = nb_valid_rows
self.maxes[j] = row[j]
def _getIconFromStyle(self, style):
if style.startswith('iwla_page'): icon = 'vp.png'
elif style.startswith('iwla_hit'): icon = 'vh.png'
elif style.startswith('iwla_bandwidth'): icon = 'vk.png'
elif style.startswith('iwla_visit'): icon = 'vv.png'
elif style.startswith('iwla_search'): icon = 'vu.png'
if style.startswith(u'iwla_page'): icon = u'vp.png'
elif style.startswith(u'iwla_hit'): icon = u'vh.png'
elif style.startswith(u'iwla_bandwidth'): icon = u'vk.png'
elif style.startswith(u'iwla_visitor'): icon = u'vu.png'
elif style.startswith(u'iwla_visit'): icon = u'vv.png'
else: return ''
return '%s/%s' % (self.icon_path, icon)
return u'%s/%s' % (self.icon_path, icon)
def _buildHTML(self):
self._computeMax()
style = ''
if self.table_graph_css: style = ' class="%s"' % (self.table_graph_css)
html = '<table%s>' % (style)
html += '<tr>'
style = u''
if self.table_graph_css: style = u' class="%s"' % (self.table_graph_css)
html = u'<table%s>' % (style)
html += u'<tr>'
for i in range(0, self.nb_valid_rows):
row = self.rows[i]
css = ''
if self.td_img_css: css=' class="%s"' % (self.td_img_css)
html += '<td%s>' % (css)
css = u''
if self.td_img_css: css=u' class="%s"' % (self.td_img_css)
html += u'<td%s>' % (css)
for j in self.graph_cols:
style = self.getColCSSClass(j)
icon = self._getIconFromStyle(style)
if not icon: continue
if style: style = ' class="%s"' % (style)
alt = '%s: %s' % (row[j], self.cols[j])
if style: style = u' class="%s"' % (style)
alt = u'%s: %s' % (row[j], self.cols[j])
if self.maxes[j]:
height = int((self.raw_rows[i][j] * 100) / self.maxes[j]) or 1
else:
height = 1
html += '<img%s src="%s" height="%d" width="6" alt="%s" title="%s" />' % (style, icon, height, alt, alt)
html += '</td>'
html += '</tr>'
html += '<tr>'
html += u'<img%s src="%s" height="%d" width="6" alt="%s" title="%s" />' % (style, icon, height, alt, alt)
html += u'</td>'
html += u'</tr>'
html += u'<tr>'
for i in range(0, len(self.short_titles)):
style = self.getCellCSSClass(i, 0)
if style: style = ' class="%s"' % (style)
html += '<td%s>%s</td>' % (style, self.short_titles[i])
html += '</tr>'
html += '</table>'
if style: style = u' class="%s"' % (style)
html += u'<td%s>%s</td>' % (style, self.short_titles[i])
html += u'</tr>'
html += u'</table>'
self.html += html
class DisplayHTMLPage(object):
def __init__(self, title, filename, css_path):
self.title = title
self.title = unicode(title)
self.filename = filename
self.blocks = []
self.css_path = css_path
self.css_path = listToStr(css_path)
def getFilename(self):
return self.filename;
if not os.path.exists(base):
os.makedirs(base)
f = open(filename, 'w')
f.write('<!DOCTYPE html>')
f.write('<html>')
f.write('<head>')
f.write('<meta http-equiv="Content-type" content="text/html; charset=UTF-8" />')
f = codecs.open(filename, 'w', 'utf-8')
f.write(u'<!DOCTYPE html>')
f.write(u'<html>')
f.write(u'<head>')
f.write(u'<meta http-equiv="Content-type" content="text/html; charset=UTF-8" />')
for css in self.css_path:
f.write('<link rel="stylesheet" href="%s"/>' % (css))
f.write(u'<link rel="stylesheet" href="%s"/>' % (css))
if self.title:
f.write('<title>%s</title>' % (self.title))
f.write('</head>')
f.write(u'<title>%s</title>' % (self.title))
f.write(u'</head>')
for block in self.blocks:
block.build(f)
f.write('</body></html>')
f.write(u'</body></html>')
f.close()
class DisplayHTMLBuild(object):
page.build(root)
def bytesToStr(bytes):
suffixes = ['', ' kB', ' MB', ' GB', ' TB']
suffixes = [u'', u' kB', u' MB', u' GB', u' TB']
for i in range(0, len(suffixes)):
if bytes < 1024: break
bytes /= 1024.0
if i:
return '%.02f%s' % (bytes, suffixes[i])
return u'%.02f%s' % (bytes, suffixes[i])
else:
return '%d%s' % (bytes, suffixes[i])
return u'%d%s' % (bytes, suffixes[i])
def _toStr(v):
if type(v) != str: return str(v)
if type(v) != unicode: return unicode(v)
else: return v
def listToStr(l): return map(lambda(v) : _toStr(v), l)
def generateHTMLLink(url, name=None, max_length=100, prefix=u'http'):
url = unicode(url)
if not name: name = unicode(url)
if not url.startswith(prefix): url = u'%s://%s' % (prefix, url)
return u'<a href="%s">%s</a>' % (url, name[:max_length])
iwla.py
301301
302302
303303
304
304
305305
306306
307307
cols = ['Month', 'Visitors', 'Pages', 'Hits', 'Bandwidth', 'Not viewed Bandwidth', 'Details']
graph_cols=range(1,6)
months = DisplayHTMLBlockTableWithGraph(title, cols, nb_valid_rows=12, graph_cols=graph_cols)
months.setColsCSSClass(['', 'iwla_visit', 'iwla_page', 'iwla_hit', 'iwla_bandwidth', 'iwla_bandwidth', ''])
months.setColsCSSClass(['', 'iwla_visitor', 'iwla_page', 'iwla_hit', 'iwla_bandwidth', 'iwla_bandwidth', ''])
total = [0] * len(cols)
for i in range(1, 13):
month = '%s<br/>%d' % (months_name[i], year)
plugins/display/referers.py
4949
5050
5151
52
52
5353
5454
5555
5656
57
57
5858
5959
6060
......
7777
7878
7979
80
80
8181
8282
8383
8484
85
85
8686
8787
8888
......
9999
100100
101101
102
102
103103
104104
105105
table.appendRow(['<b>External URL</b>', '', ''])
for r,_ in top_referers:
row = [r, referers[r]['pages'], referers[r]['hits']]
row = [generateHTMLLink(r), referers[r]['pages'], referers[r]['hits']]
table.appendRow(row)
table.appendRow(['<b>External URL (robot)</b>', '', ''])
for r,_ in top_robots_referers:
row = [r, robots_referers[r]['pages'], robots_referers[r]['hits']]
row = [generateHTMLLink(r), robots_referers[r]['pages'], robots_referers[r]['hits']]
table.appendRow(row)
page.appendBlock(table)
table.appendRow(['<b>External URL</b>', '', ''])
for r,_ in top_referers[:10]:
row = [r, referers[r]['pages'], referers[r]['hits']]
row = [generateHTMLLink(r), referers[r]['pages'], referers[r]['hits']]
table.appendRow(row)
table.appendRow(['<b>External URL (robot)</b>', '', ''])
for r,_ in top_robots_referers[:10]:
row = [r, robots_referers[r]['pages'], robots_referers[r]['hits']]
row = [generateHTMLLink(r), robots_referers[r]['pages'], robots_referers[r]['hits']]
table.appendRow(row)
index.appendBlock(table)
for phrase in top_key_phrases:
table.appendRow([phrase[0], phrase[1]])
page.appendBlock(table)
display.addPage(page)
link = '<a href=\'%s\'>All key phrases</a>' % (filename)
plugins/display/top_downloads.py
2323
2424
2525
26
26
2727
2828
2929
......
3737
3838
3939
40
40
4141
table = DisplayHTMLBlockTable('All Downloads', ['URI', 'Hit'])
table.setColsCSSClass(['', 'iwla_hit'])
for (uri, entrance) in top_downloads:
table.appendRow([uri, entrance])
table.appendRow([generateHTMLLink(uri), entrance])
page.appendBlock(table)
self.iwla.getDisplay().addPage(page)
table = DisplayHTMLBlockTable(title, ['URI', 'Hits'])
table.setColsCSSClass(['', 'iwla_hit'])
for (uri, entrance) in top_downloads[:10]:
table.appendRow([uri, entrance])
table.appendRow([generateHTMLLink(uri), entrance])
index.appendBlock(table)
plugins/display/top_hits.py
2323
2424
2525
26
26
2727
2828
2929
......
3737
3838
3939
40
40
4141
table = DisplayHTMLBlockTable('All Hits', ['URI', 'Entrance'])
table.setColsCSSClass(['', 'iwla_hit'])
for (uri, entrance) in top_hits:
table.appendRow([uri, entrance])
table.appendRow([generateHTMLLink(uri), entrance])
page.appendBlock(table)
self.iwla.getDisplay().addPage(page)
table = DisplayHTMLBlockTable(title, ['URI', 'Entrance'])
table.setColsCSSClass(['', 'iwla_hit'])
for (uri, entrance) in top_hits[:10]:
table.appendRow([uri, entrance])
table.appendRow([generateHTMLLink(uri), entrance])
index.appendBlock(table)
plugins/display/top_pages.py
2323
2424
2525
26
26
2727
2828
2929
......
3737
3838
3939
40
40
4141
table = DisplayHTMLBlockTable('All Pages', ['URI', 'Entrance'])
table.setColsCSSClass(['', 'iwla_hit'])
for (uri, entrance) in top_pages:
table.appendRow([uri, entrance])
table.appendRow([generateHTMLLink(uri), entrance])
page.appendBlock(table)
self.iwla.getDisplay().addPage(page)
table = DisplayHTMLBlockTable(title, ['URI', 'Entrance'])
table.setColsCSSClass(['', 'iwla_hit'])
for (uri, entrance) in top_pages[:10]:
table.appendRow([uri, entrance])
table.appendRow([generateHTMLLink(uri), entrance])
index.appendBlock(table)
plugins/post_analysis/referers.py
11
2
2
33
44
55
......
5757
5858
5959
60
61
60
6261
6362
6463
import re
import xml.sax.saxutils as saxutils
import urllib
from iwla import IWLA
from iplugin import IPlugin
groups = key_phrase_re.match(p)
if groups:
key_phrase = groups.groupdict()['key_phrase']
key_phrase = key_phrase.replace('+', ' ').lower()
key_phrase = saxutils.unescape(key_phrase)
key_phrase = urllib.unquote_plus(key_phrase).decode('utf8')
if not key_phrase in key_phrases.keys():
key_phrases[key_phrase] = 1
else:

Archive Download the corresponding diff file

Branches

Tags