iwla

iwla Commit Details

Date:2014-12-16 20:23:33 (6 years 7 months ago)
Author:Grégory Soutadé
Branch:dev, master
Commit:7afbf4fa96f2cb120bd107b70c7c3fa2472574d6
Parents: e69af5e675d80a76ded9abc6c8413a32a6f288f2
Message:Logging seems OK, except names

Changes:
Mdisplay.py (5 diffs)
Miplugin.py (4 diffs)
Miwla.py (16 diffs)

File differences

display.py
11
22
33
4
45
56
67
......
2223
2324
2425
25
2626
27
28
29
3027
31
32
3328
3429
3530
......
252247
253248
254249
250
255251
256252
257253
......
272268
273269
274270
271
272
275273
276274
277275
......
321319
322320
323321
324
325322
326
327323
328324
329325
import os
import codecs
import time
import logging
#
# Create output HTML files
if html: f.write(html)
def build(self, f):
# t1 = time.time()
self._buildHTML()
# t2 = time.time()
# print 'Time for _buildHTML : %d seconds' % (t2-t1)
# t1 = time.time()
self._build(f, self.html)
# t2 = time.time()
# print 'Time for _build : %d seconds' % (t2-t1)
class DisplayHTMLBlock(DisplayHTMLRaw):
self.filename = filename
self.blocks = []
self.css_path = listToStr(css_path)
self.logger = logging.getLogger(self.__class__.__name__)
def getFilename(self):
return self.filename;
if not os.path.exists(base):
os.makedirs(base)
self.logger.debug('Write %s' % (filename))
f = codecs.open(filename, 'w', 'utf-8')
f.write(u'<!DOCTYPE html>')
f.write(u'<html>')
os.symlink(target, link_name)
for page in self.pages:
# print 'Build %s' % (page.filename)
page.build(root)
# print 'Built'
#
# Global functions
iplugin.py
11
22
3
3
44
55
66
......
4747
4848
4949
50
50
51
52
5153
5254
5355
......
6163
6264
6365
64
66
6567
6668
6769
......
8688
8789
8890
89
91
9092
9193
9294
9395
94
96
9597
9698
97
99
98100
99101
100
101
102
102103
103104
import importlib
import inspect
import traceback
import logging
#
# IWLA Plugin interface
def preloadPlugins(plugins, iwla):
cache_plugins = {}
print "==> Preload plugins"
logger = logging.getLogger(__name__)
logger.info("==> Preload plugins")
for (root, plugins_filenames) in plugins:
for plugin_filename in plugins_filenames:
]
if not classes:
print 'No plugin defined in %s' % (plugin_path)
logger.warning('No plugin defined in %s' % (plugin_path))
continue
plugin = classes[0](iwla)
requirement_validated = True
break
if not requirement_validated:
print 'Missing requirements \'%s\' for plugin %s' % (r, plugin_path)
logger.error('Missing requirements \'%s\' for plugin %s' % (r, plugin_path))
break
if requirements and not requirement_validated: continue
if not plugin.load():
print 'Plugin %s load failed' % (plugin_path)
logger.error('Plugin %s load failed' % (plugin_path))
continue
print '\tRegister %s' % (plugin_path)
logger.info('\tRegister %s' % (plugin_path))
cache_plugins[plugin_path] = plugin
except Exception as e:
print 'Error loading %s => %s' % (plugin_path, e)
traceback.print_exc()
logger.exception('Error loading %s => %s' % (plugin_path, e))
return cache_plugins
iwla.py
109109
110110
111111
112
113
114
112
115113
116114
117115
......
128126
129127
130128
131
132
133
134
135
129
130
131
136132
137133
138134
......
230226
231227
232228
233
229
234230
235231
236232
237233
238234
239
235
240236
241237
242238
......
306302
307303
308304
309
305
310306
311307
312308
......
344340
345341
346342
347
343
348344
349345
350346
......
437433
438434
439435
440
436
437
441438
442439
443440
......
452449
453450
454451
455
452
453
454
456455
457456
458457
......
499498
500499
501500
502
503
501
502
504503
505504
506505
......
524523
525524
526525
527
526
528527
529528
530529
......
568567
569568
570569
571
572
573
570
571
574572
575573
576574
......
608606
609607
610608
611
609
612610
613611
614612
615
616
613
614
617615
618616
619617
......
622620
623621
624622
625
623
626624
627625
628626
......
633631
634632
635633
636
634
637635
638636
639637
......
642640
643641
644642
645
643
646644
647645
648646
......
659657
660658
661659
662
663
660
661
664662
665663
666664
......
668666
669667
670668
671
672
673
674
669
670
675671
676672
677
673
678674
679675
680676
API_VERSION = 1
IWLA_VERSION = '0.1'
def __init__(self):
print '==> Start'
def __init__(self, logLevel):
self.meta_infos = {}
self.analyse_started = False
self.current_analysis = {}
(conf.POST_HOOK_DIRECTORY , conf.post_analysis_hooks),
(conf.DISPLAY_HOOK_DIRECTORY , conf.display_hooks)]
self.logger = logging.getLogger('iwla')
self.logger.setFormatter(logging.Formatter('%(name)s %(message)s'))
def setLoggerLevel(self, level):
self.logger.setLevel(level)
logging.basicConfig(format='%(name)s %(message)s', level=logLevel)
self.logger = logging.getLogger(self.__class__.__name__)
self.logger.info('==> Start')
def getVersion(self):
return IWLA.IWLA_VERSION
return None
def _callPlugins(self, target_root, *args):
print '==> Call plugins (%s)' % target_root
self.logger.info('==> Call plugins (%s)' % (target_root))
for (root, plugins) in self.plugins:
if root != target_root: continue
for p in plugins:
mod = self.cache_plugins.get(root + '.' + p, None)
if mod:
print '\t%s' % (p)
self.logger.info('\t%s' % (p))
mod.hook(*args)
def isPage(self, request):
if 'extract_parameters' in d.keys():
hit['extract_request']['extract_parameters'] = d['extract_parameters']
else:
print "Bad request extraction " + hit['request']
self.logger.warning("Bad request extraction %s" % (hit['request']))
return False
if hit['http_referer']:
cur_time = self.meta_infos['last_time']
title = 'Stats %d/%02d' % (cur_time.tm_year, cur_time.tm_mon)
filename = self.getCurDisplayPath('index.html')
print '==> Generate display (%s)' % (filename)
self.logger.info('==> Generate display (%s)' % (filename))
page = self.display.createPage(title, filename, conf.css_path)
_, nb_month_days = monthrange(cur_time.tm_year, cur_time.tm_mon)
def _generateDisplayWholeMonthStats(self):
title = 'Stats for %s' % (conf.domain_name)
filename = 'index.html'
print '==> Generate main page (%s)' % (filename)
self.logger.info('==> Generate main page (%s)' % (filename))
page = self.display.createPage(title, filename, conf.css_path)
def _compressFile(self, build_time, root, filename):
path = os.path.join(root, filename)
gz_path = path + '.gz'
#print 'Compress %s => %s' % (path, gz_path)
self.logger.debug('Compress %s => %s' % (path, gz_path))
if not os.path.exists(gz_path) or\
os.stat(path).st_mtime > build_time:
with open(path, 'rb') as f_in:
duplicated_stats = {k:v for (k,v) in stats.items()}
cur_time = self.meta_infos['last_time']
print "== Stats for %d/%02d ==" % (cur_time.tm_year, cur_time.tm_mon)
print stats
self.logger.info("== Stats for %d/%02d ==" % (cur_time.tm_year, cur_time.tm_mon))
self.logger.info(stats)
if not 'month_stats' in self.current_analysis.keys():
self.current_analysis['month_stats'] = stats
if os.path.exists(path):
os.remove(path)
print "==> Serialize to %s" % path
self.logger.info("==> Serialize to %s" % (path))
self._serialize(self.current_analysis, path)
# Save month stats
not super_hit['robot']:
stats['nb_visits'] += 1
print "== Stats for %d/%02d/%02d ==" % (cur_time.tm_year, cur_time.tm_mon, cur_time.tm_mday)
print stats
self.logger.info("== Stats for %d/%02d/%02d ==" % (cur_time.tm_year, cur_time.tm_mon, cur_time.tm_mday))
self.logger.info(stats)
self.current_analysis['days_stats'][cur_time.tm_mday] = stats
return True
def start(self, _file):
print '==> Load previous database'
self.logger.info('==> Load previous database')
self.meta_infos = self._deserialize(conf.META_PATH) or self._clearMeta()
if self.meta_infos['last_time']:
print 'Last time'
print self.meta_infos['last_time']
self.logger.info('Last time')
self.logger.info(self.meta_infos['last_time'])
self.current_analysis = self._deserialize(self.getDBFilename(self.meta_infos['last_time'])) or self._clearVisits()
else:
self._clearVisits()
self.cache_plugins = preloadPlugins(self.plugins, self)
print '==> Analysing log'
self.logger.info('==> Analysing log')
for l in _file:
# print "line " + l
if not self._newHit(groups.groupdict()):
continue
else:
print "No match for " + l
self.logger.warning("No match for %s" % (l))
#break
if self.analyse_started:
del self.meta_infos['start_analysis_time']
self._serialize(self.meta_infos, conf.META_PATH)
else:
print '==> Analyse not started : nothing new'
self.logger.info('==> Analyse not started : nothing new')
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Intelligent Web Log Analyzer')
help='Analyse this log file')
parser.add_argument('-d', '--log-level', dest='loglevel',
default=logging.INFO,
help='Loglevel')
default='INFO', type=str,
help='Loglevel in %s, default : %s' % (['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], 'INFO'))
args = parser.parse_args()
if os.path.exists(conf.DB_ROOT): shutil.rmtree(conf.DB_ROOT)
if os.path.exists(conf.DISPLAY_ROOT): shutil.rmtree(conf.DISPLAY_ROOT)
iwla = IWLA()
numeric_level = getattr(logging, args.loglevel.upper(), None)
if not isinstance(numeric_level, int):
loglevel = getattr(logging, args.loglevel.upper(), None)
if not isinstance(loglevel, int):
raise ValueError('Invalid log level: %s' % (args.loglevel))
iwla.setLoggerLevel(numeric_level)
iwla = IWLA(loglevel)
required_conf = ['analyzed_filename', 'domain_name']
if not validConfRequirements(required_conf, iwla, 'Main Conf'):

Archive Download the corresponding diff file

Branches

Tags