iwla

iwla Commit Details

Date:2020-10-30 14:42:56 (8 months 29 days ago)
Author:Grégory Soutadé
Branch:dev
Commit:f457f4e390e29db7a665355db7d5b8aaaf463f06
Parents: 6f9622bb911a41aba36910a8f08475201d28da7d
Message:Update code for Python3

Changes:
Mdisplay.py (1 diff)
Miplugin.py (2 diffs)
Miwla.py (15 diffs)
Mplugins/display/referers_diff.py (1 diff)
Mplugins/display/top_downloads_diff.py (1 diff)
Mplugins/display/top_pages_diff.py (1 diff)
Mplugins/post_analysis/ip_to_geo.py (2 diffs)
Mplugins/post_analysis/iptogeo.py (2 diffs)
Mplugins/post_analysis/referers.py (2 diffs)
Mplugins/pre_analysis/page_to_hit.py (1 diff)
Mplugins/pre_analysis/robots.py (1 diff)

File differences

display.py
437437
438438
439439
440
440
441441
442442
443443
if type(v) != unicode: return unicode(v)
else: return v
def listToStr(l): return map(lambda(v) : _toStr(v), l)
def listToStr(l): return map(lambda v : _toStr(v), l)
def generateHTMLLink(url, name=None, max_length=100, prefix=u'http'):
url = unicode(url)
iplugin.py
5959
6060
6161
62
62
6363
6464
6565
......
8989
9090
9191
92
92
9393
9494
9595
def validConfRequirements(conf_requirements, iwla, plugin_path):
for r in conf_requirements:
if iwla.getConfValue(r, None) is None:
print '\'%s\' conf value required for %s' % (r, plugin_path)
print('\'%s\' conf value required for %s' % (r, plugin_path))
return False
return True
if len(classes) > 1:
logger.warning('More than one class found in %s, loading may fail. Selecting %s' % (plugin_path, classes[0]))
print classes
print(classes)
continue
plugin = classes[0](iwla)
iwla.py
1
1
22
33
44
......
2424
2525
2626
27
27
2828
2929
3030
......
165165
166166
167167
168
168
169169
170170
171171
172172
173
173
174174
175175
176176
......
239239
240240
241241
242
242
243243
244244
245245
......
261261
262262
263263
264
264
265265
266266
267267
......
275275
276276
277277
278
278
279279
280280
281281
......
326326
327327
328328
329
329
330330
331331
332332
......
336336
337337
338338
339
339
340340
341341
342342
......
345345
346346
347347
348
348
349349
350350
351351
......
417417
418418
419419
420
420
421421
422
422
423423
424424
425425
......
482482
483483
484484
485
485
486486
487
487
488488
489489
490490
......
592592
593593
594594
595
595
596596
597597
598598
......
658658
659659
660660
661
661
662662
663663
664664
......
708708
709709
710710
711
711
712712
713713
714714
......
809809
810810
811811
812
812
813813
814814
815815
#!/usr/bin/env python
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright Grégory Soutadé 2015
import sys
import re
import time
import cPickle
import pickle
import gzip
import importlib
import argparse
else:
self.logger.info('==> Start')
try:
t = gettext.translation('iwla', localedir=conf.locales_path, languages=[conf.locale], codeset='utf8')
t = gettext.translation('iwla', localedir=conf.locales_path, languages=[conf.locale])
self.logger.info('\tUsing locale %s' % (conf.locale))
except IOError:
t = gettext.NullTranslations()
self.logger.info('\tUsing default locale en_EN')
self._ = t.ugettext
self._ = t.gettext
def getVersion(self):
return IWLA.IWLA_VERSION
def _clearDisplay(self):
self.display.clear()
return self.display
return self.display
def getDBFilename(self, time):
return os.path.join(conf.DB_ROOT, str(time.tm_year), '%02d' % (time.tm_mon), conf.DB_FILENAME)
shutil.copy(filename, filename + '.bak')
with open(filename + '.tmp', 'wb+') as f, self._openDB(filename, 'w') as fzip:
cPickle.dump(obj, f)
pickle.dump(obj, f)
f.seek(0)
fzip.write(f.read())
os.fsync(fzip)
res = None
with self._openDB(filename) as f:
res = cPickle.load(f)
res = pickle.load(f)
return res
def _callPlugins(self, target_root, *args):
if not remote_addr in self.current_analysis['visits'].keys():
self._createVisitor(hit)
super_hit = self.current_analysis['visits'][remote_addr]
# Don't keep all requests for robots
if not super_hit['robot']:
if self.hasBeenViewed(hit):
super_hit['bandwidth'][day] = super_hit['bandwidth'].get(day, 0) + int(hit['body_bytes_sent'])
super_hit['bandwidth'][0] += int(hit['body_bytes_sent'])
super_hit['last_access'] = self.meta_infos['last_time']
super_hit['last_access'] = self.meta_infos['last_time']
request = hit['extract_request']
hit['is_page'] = self.isPage(uri)
if super_hit['robot'] or\
not self.hasBeenViewed(hit):
not self.hasBeenViewed(hit):
page_key = 'not_viewed_pages'
hit_key = 'not_viewed_hits'
else:
def _decodeTime(self, hit):
try:
hit['time_decoded'] = time.strptime(hit['time_local'], conf.time_format)
except ValueError, e:
except ValueError as e:
if sys.version_info < (3, 2):
# Try without UTC value at the end (%z not recognized)
# Try without UTC value at the end (%z not recognized)
gmt_offset_str = hit['time_local'][-5:]
gmt_offset_hours = int(gmt_offset_str[1:3])*60*60
gmt_offset_minutes = int(gmt_offset_str[3:5])*60
row = [0, nb_visits, stats['viewed_pages'], stats['viewed_hits'], stats['viewed_bandwidth'], stats['not_viewed_bandwidth']]
if nb_days:
average_row = map(lambda(v): int(v/nb_days), row)
average_row = map(lambda v: int(v/nb_days), row)
else:
average_row = map(lambda(v): 0, row)
average_row = map(lambda v: 0, row)
average_row[0] = self._('Average')
average_row[4] = bytesToStr(average_row[4])
if f.endswith(ext):
self._compressFile(rootdir, f)
break
def _generateDisplay(self):
self._generateDisplayDaysStats()
self._callPlugins(conf.DISPLAY_HOOK_DIRECTORY)
if not year in self.meta_infos['stats'].keys():
self.meta_infos['stats'][year] = {}
self.meta_infos['stats'][year][month] = duplicated_stats
self.logger.info("==> Serialize to %s" % (conf.META_PATH))
self._serialize(self.meta_infos, conf.META_PATH)
self.analyse_started = True
else:
if not self.analyse_started and\
time.mktime(t) <= time.mktime(cur_time):
time.mktime(t) <= time.mktime(cur_time):
self.logger.debug("Not in time")
return False
self.analyse_started = True
self.filenames = [f for f in filenames.split(',') if f]
for f in self.filenames:
if not os.path.exists(f):
print 'No such file \'%s\'' % (f)
print('No such file \'%s\'' % (f))
sys.exit(-1)
self.cur_file = None
self._openNextFile()
plugins/display/referers_diff.py
1919
2020
2121
22
22
2323
2424
2525
#
from iwla import IWLA
from istats_diff import IWLADisplayStatsDiff
from .istats_diff import IWLADisplayStatsDiff
from display import *
"""
plugins/display/top_downloads_diff.py
1919
2020
2121
22
22
2323
2424
2525
#
from iwla import IWLA
from istats_diff import IWLADisplayStatsDiff
from .istats_diff import IWLADisplayStatsDiff
from display import *
"""
plugins/display/top_pages_diff.py
1919
2020
2121
22
22
2323
2424
2525
#
from iwla import IWLA
from istats_diff import IWLADisplayStatsDiff
from .istats_diff import IWLADisplayStatsDiff
from display import *
"""
plugins/post_analysis/ip_to_geo.py
2121
2222
2323
24
24
2525
2626
2727
......
8888
8989
9090
91
92
91
92
9393
9494
from iwla import IWLA
from iplugin import IPlugin
from iptogeo import IPToGeo
from .iptogeo import IPToGeo
"""
Post analysis hook
geo[cc] += 1
else:
geo[cc] = 1
except Exception, e:
print e
except Exception as e:
print(e)
month_stats['geo'] = geo
plugins/post_analysis/iptogeo.py
8181
8282
8383
84
84
8585
8686
8787
......
124124
125125
126126
127
127
128128
129129
130130
return res
def _create_request(self, ip, ip_type):
packet = ''
packet = b''
packet += struct.pack('<IBBBBI', IPToGeo.MAGIC, IPToGeo.VERSION, IPToGeo.REQ,
0, #err
ip_type, # ip type
if not packet:
raise socket.timeout
return packet
except socket.timeout, e:
except socket.timeout as e:
if second_chance:
self._nb_requests_sent = self.MAX_REQUESTS
return self._send_request(packet, False)
plugins/post_analysis/referers.py
7878
7979
8080
81
81
8282
8383
8484
......
114114
115115
116116
117
117
118118
119119
120120
domain_name = self.iwla.getConfValue('domain_name', '')
if not domain_name:
print 'domain_name must not be empty !'
print('domain_name must not be empty !')
return False
self.own_domain_re = re.compile(r'.*%s.*' % (domain_name))
key_phrase = groups.groupdict()['key_phrase']
try:
key_phrase = urllib.unquote_plus(key_phrase).decode('utf8')
except Exception, e:
except Exception as e:
print(e)
continue
if not key_phrase in key_phrases.keys():
plugins/pre_analysis/page_to_hit.py
5959
6060
6161
62
62
6363
6464
6565
66
66
6767
6868
6969
def load(self):
# Page to hit
self.ph_regexps = self.iwla.getConfValue('page_to_hit_conf', [])
self.ph_regexps = map(lambda(r): re.compile(r), self.ph_regexps)
self.ph_regexps = map(lambda r: re.compile(r), self.ph_regexps)
# Hit to page
self.hp_regexps = self.iwla.getConfValue('hit_to_page_conf', [])
self.hp_regexps = map(lambda(r): re.compile(r), self.hp_regexps)
self.hp_regexps = map(lambda r: re.compile(r), self.hp_regexps)
self.logger = logging.getLogger(self.__class__.__name__)
return True
plugins/pre_analysis/robots.py
6060
6161
6262
63
63
6464
6565
6666
self.API_VERSION = 1
def load(self):
self.awstats_robots = map(lambda (x) : re.compile(('.*%s.*') % (x), re.IGNORECASE), awstats_data.robots)
self.awstats_robots = map(lambda x : re.compile(('.*%s.*') % (x), re.IGNORECASE), awstats_data.robots)
self.robot_re = re.compile(r'.*bot.*', re.IGNORECASE)
self.crawl_re = re.compile(r'.*crawl.*', re.IGNORECASE)
self.logger = logging.getLogger(self.__class__.__name__)

Archive Download the corresponding diff file

Branches

Tags