Logging seems OK, except names
This commit is contained in:
parent
e69af5e675
commit
7afbf4fa96
12
display.py
12
display.py
|
|
@ -1,6 +1,7 @@
|
|||
import os
|
||||
import codecs
|
||||
import time
|
||||
import logging
|
||||
|
||||
#
|
||||
# Create output HTML files
|
||||
|
|
@ -22,14 +23,8 @@ class DisplayHTMLRaw(object):
|
|||
if html: f.write(html)
|
||||
|
||||
def build(self, f):
|
||||
# t1 = time.time()
|
||||
self._buildHTML()
|
||||
# t2 = time.time()
|
||||
# print 'Time for _buildHTML : %d seconds' % (t2-t1)
|
||||
# t1 = time.time()
|
||||
self._build(f, self.html)
|
||||
# t2 = time.time()
|
||||
# print 'Time for _build : %d seconds' % (t2-t1)
|
||||
|
||||
class DisplayHTMLBlock(DisplayHTMLRaw):
|
||||
|
||||
|
|
@ -252,6 +247,7 @@ class DisplayHTMLPage(object):
|
|||
self.filename = filename
|
||||
self.blocks = []
|
||||
self.css_path = listToStr(css_path)
|
||||
self.logger = logging.getLogger(self.__class__.__name__)
|
||||
|
||||
def getFilename(self):
|
||||
return self.filename;
|
||||
|
|
@ -272,6 +268,8 @@ class DisplayHTMLPage(object):
|
|||
if not os.path.exists(base):
|
||||
os.makedirs(base)
|
||||
|
||||
self.logger.debug('Write %s' % (filename))
|
||||
|
||||
f = codecs.open(filename, 'w', 'utf-8')
|
||||
f.write(u'<!DOCTYPE html>')
|
||||
f.write(u'<html>')
|
||||
|
|
@ -321,9 +319,7 @@ class DisplayHTMLBuild(object):
|
|||
os.symlink(target, link_name)
|
||||
|
||||
for page in self.pages:
|
||||
# print 'Build %s' % (page.filename)
|
||||
page.build(root)
|
||||
# print 'Built'
|
||||
|
||||
#
|
||||
# Global functions
|
||||
|
|
|
|||
17
iplugin.py
17
iplugin.py
|
|
@ -1,6 +1,6 @@
|
|||
import importlib
|
||||
import inspect
|
||||
import traceback
|
||||
import logging
|
||||
|
||||
#
|
||||
# IWLA Plugin interface
|
||||
|
|
@ -47,7 +47,9 @@ def validConfRequirements(conf_requirements, iwla, plugin_path):
|
|||
def preloadPlugins(plugins, iwla):
|
||||
cache_plugins = {}
|
||||
|
||||
print "==> Preload plugins"
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
logger.info("==> Preload plugins")
|
||||
|
||||
for (root, plugins_filenames) in plugins:
|
||||
for plugin_filename in plugins_filenames:
|
||||
|
|
@ -61,7 +63,7 @@ def preloadPlugins(plugins, iwla):
|
|||
]
|
||||
|
||||
if not classes:
|
||||
print 'No plugin defined in %s' % (plugin_path)
|
||||
logger.warning('No plugin defined in %s' % (plugin_path))
|
||||
continue
|
||||
|
||||
plugin = classes[0](iwla)
|
||||
|
|
@ -86,18 +88,17 @@ def preloadPlugins(plugins, iwla):
|
|||
requirement_validated = True
|
||||
break
|
||||
if not requirement_validated:
|
||||
print 'Missing requirements \'%s\' for plugin %s' % (r, plugin_path)
|
||||
logger.error('Missing requirements \'%s\' for plugin %s' % (r, plugin_path))
|
||||
break
|
||||
if requirements and not requirement_validated: continue
|
||||
|
||||
if not plugin.load():
|
||||
print 'Plugin %s load failed' % (plugin_path)
|
||||
logger.error('Plugin %s load failed' % (plugin_path))
|
||||
continue
|
||||
|
||||
print '\tRegister %s' % (plugin_path)
|
||||
logger.info('\tRegister %s' % (plugin_path))
|
||||
cache_plugins[plugin_path] = plugin
|
||||
except Exception as e:
|
||||
print 'Error loading %s => %s' % (plugin_path, e)
|
||||
traceback.print_exc()
|
||||
logger.exception('Error loading %s => %s' % (plugin_path, e))
|
||||
|
||||
return cache_plugins
|
||||
|
|
|
|||
62
iwla.py
62
iwla.py
|
|
@ -109,9 +109,7 @@ class IWLA(object):
|
|||
API_VERSION = 1
|
||||
IWLA_VERSION = '0.1'
|
||||
|
||||
def __init__(self):
|
||||
print '==> Start'
|
||||
|
||||
def __init__(self, logLevel):
|
||||
self.meta_infos = {}
|
||||
self.analyse_started = False
|
||||
self.current_analysis = {}
|
||||
|
|
@ -128,11 +126,9 @@ class IWLA(object):
|
|||
(conf.POST_HOOK_DIRECTORY , conf.post_analysis_hooks),
|
||||
(conf.DISPLAY_HOOK_DIRECTORY , conf.display_hooks)]
|
||||
|
||||
self.logger = logging.getLogger('iwla')
|
||||
self.logger.setFormatter(logging.Formatter('%(name)s %(message)s'))
|
||||
|
||||
def setLoggerLevel(self, level):
|
||||
self.logger.setLevel(level)
|
||||
logging.basicConfig(format='%(name)s %(message)s', level=logLevel)
|
||||
self.logger = logging.getLogger(self.__class__.__name__)
|
||||
self.logger.info('==> Start')
|
||||
|
||||
def getVersion(self):
|
||||
return IWLA.IWLA_VERSION
|
||||
|
|
@ -230,13 +226,13 @@ class IWLA(object):
|
|||
return None
|
||||
|
||||
def _callPlugins(self, target_root, *args):
|
||||
print '==> Call plugins (%s)' % target_root
|
||||
self.logger.info('==> Call plugins (%s)' % (target_root))
|
||||
for (root, plugins) in self.plugins:
|
||||
if root != target_root: continue
|
||||
for p in plugins:
|
||||
mod = self.cache_plugins.get(root + '.' + p, None)
|
||||
if mod:
|
||||
print '\t%s' % (p)
|
||||
self.logger.info('\t%s' % (p))
|
||||
mod.hook(*args)
|
||||
|
||||
def isPage(self, request):
|
||||
|
|
@ -306,7 +302,7 @@ class IWLA(object):
|
|||
if 'extract_parameters' in d.keys():
|
||||
hit['extract_request']['extract_parameters'] = d['extract_parameters']
|
||||
else:
|
||||
print "Bad request extraction " + hit['request']
|
||||
self.logger.warning("Bad request extraction %s" % (hit['request']))
|
||||
return False
|
||||
|
||||
if hit['http_referer']:
|
||||
|
|
@ -344,7 +340,7 @@ class IWLA(object):
|
|||
cur_time = self.meta_infos['last_time']
|
||||
title = 'Stats %d/%02d' % (cur_time.tm_year, cur_time.tm_mon)
|
||||
filename = self.getCurDisplayPath('index.html')
|
||||
print '==> Generate display (%s)' % (filename)
|
||||
self.logger.info('==> Generate display (%s)' % (filename))
|
||||
page = self.display.createPage(title, filename, conf.css_path)
|
||||
|
||||
_, nb_month_days = monthrange(cur_time.tm_year, cur_time.tm_mon)
|
||||
|
|
@ -437,7 +433,8 @@ class IWLA(object):
|
|||
def _generateDisplayWholeMonthStats(self):
|
||||
title = 'Stats for %s' % (conf.domain_name)
|
||||
filename = 'index.html'
|
||||
print '==> Generate main page (%s)' % (filename)
|
||||
|
||||
self.logger.info('==> Generate main page (%s)' % (filename))
|
||||
|
||||
page = self.display.createPage(title, filename, conf.css_path)
|
||||
|
||||
|
|
@ -452,7 +449,9 @@ class IWLA(object):
|
|||
def _compressFile(self, build_time, root, filename):
|
||||
path = os.path.join(root, filename)
|
||||
gz_path = path + '.gz'
|
||||
#print 'Compress %s => %s' % (path, gz_path)
|
||||
|
||||
self.logger.debug('Compress %s => %s' % (path, gz_path))
|
||||
|
||||
if not os.path.exists(gz_path) or\
|
||||
os.stat(path).st_mtime > build_time:
|
||||
with open(path, 'rb') as f_in:
|
||||
|
|
@ -499,8 +498,8 @@ class IWLA(object):
|
|||
duplicated_stats = {k:v for (k,v) in stats.items()}
|
||||
|
||||
cur_time = self.meta_infos['last_time']
|
||||
print "== Stats for %d/%02d ==" % (cur_time.tm_year, cur_time.tm_mon)
|
||||
print stats
|
||||
self.logger.info("== Stats for %d/%02d ==" % (cur_time.tm_year, cur_time.tm_mon))
|
||||
self.logger.info(stats)
|
||||
|
||||
if not 'month_stats' in self.current_analysis.keys():
|
||||
self.current_analysis['month_stats'] = stats
|
||||
|
|
@ -524,7 +523,7 @@ class IWLA(object):
|
|||
if os.path.exists(path):
|
||||
os.remove(path)
|
||||
|
||||
print "==> Serialize to %s" % path
|
||||
self.logger.info("==> Serialize to %s" % (path))
|
||||
self._serialize(self.current_analysis, path)
|
||||
|
||||
# Save month stats
|
||||
|
|
@ -568,9 +567,8 @@ class IWLA(object):
|
|||
not super_hit['robot']:
|
||||
stats['nb_visits'] += 1
|
||||
|
||||
print "== Stats for %d/%02d/%02d ==" % (cur_time.tm_year, cur_time.tm_mon, cur_time.tm_mday)
|
||||
|
||||
print stats
|
||||
self.logger.info("== Stats for %d/%02d/%02d ==" % (cur_time.tm_year, cur_time.tm_mon, cur_time.tm_mday))
|
||||
self.logger.info(stats)
|
||||
|
||||
self.current_analysis['days_stats'][cur_time.tm_mday] = stats
|
||||
|
||||
|
|
@ -608,12 +606,12 @@ class IWLA(object):
|
|||
return True
|
||||
|
||||
def start(self, _file):
|
||||
print '==> Load previous database'
|
||||
self.logger.info('==> Load previous database')
|
||||
|
||||
self.meta_infos = self._deserialize(conf.META_PATH) or self._clearMeta()
|
||||
if self.meta_infos['last_time']:
|
||||
print 'Last time'
|
||||
print self.meta_infos['last_time']
|
||||
self.logger.info('Last time')
|
||||
self.logger.info(self.meta_infos['last_time'])
|
||||
self.current_analysis = self._deserialize(self.getDBFilename(self.meta_infos['last_time'])) or self._clearVisits()
|
||||
else:
|
||||
self._clearVisits()
|
||||
|
|
@ -622,7 +620,7 @@ class IWLA(object):
|
|||
|
||||
self.cache_plugins = preloadPlugins(self.plugins, self)
|
||||
|
||||
print '==> Analysing log'
|
||||
self.logger.info('==> Analysing log')
|
||||
|
||||
for l in _file:
|
||||
# print "line " + l
|
||||
|
|
@ -633,7 +631,7 @@ class IWLA(object):
|
|||
if not self._newHit(groups.groupdict()):
|
||||
continue
|
||||
else:
|
||||
print "No match for " + l
|
||||
self.logger.warning("No match for %s" % (l))
|
||||
#break
|
||||
|
||||
if self.analyse_started:
|
||||
|
|
@ -642,7 +640,7 @@ class IWLA(object):
|
|||
del self.meta_infos['start_analysis_time']
|
||||
self._serialize(self.meta_infos, conf.META_PATH)
|
||||
else:
|
||||
print '==> Analyse not started : nothing new'
|
||||
self.logger.info('==> Analyse not started : nothing new')
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description='Intelligent Web Log Analyzer')
|
||||
|
|
@ -659,8 +657,8 @@ if __name__ == '__main__':
|
|||
help='Analyse this log file')
|
||||
|
||||
parser.add_argument('-d', '--log-level', dest='loglevel',
|
||||
default=logging.INFO,
|
||||
help='Loglevel')
|
||||
default='INFO', type=str,
|
||||
help='Loglevel in %s, default : %s' % (['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], 'INFO'))
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
|
|
@ -668,13 +666,11 @@ if __name__ == '__main__':
|
|||
if os.path.exists(conf.DB_ROOT): shutil.rmtree(conf.DB_ROOT)
|
||||
if os.path.exists(conf.DISPLAY_ROOT): shutil.rmtree(conf.DISPLAY_ROOT)
|
||||
|
||||
iwla = IWLA()
|
||||
|
||||
numeric_level = getattr(logging, args.loglevel.upper(), None)
|
||||
if not isinstance(numeric_level, int):
|
||||
loglevel = getattr(logging, args.loglevel.upper(), None)
|
||||
if not isinstance(loglevel, int):
|
||||
raise ValueError('Invalid log level: %s' % (args.loglevel))
|
||||
|
||||
iwla.setLoggerLevel(numeric_level)
|
||||
iwla = IWLA(loglevel)
|
||||
|
||||
required_conf = ['analyzed_filename', 'domain_name']
|
||||
if not validConfRequirements(required_conf, iwla, 'Main Conf'):
|
||||
|
|
|
|||
Loading…
Reference in New Issue