Externalize plugins

This commit is contained in:
Gregory Soutade 2014-11-20 16:15:57 +01:00
parent 4cc29487a2
commit f3cb04b16c
7 changed files with 77 additions and 89 deletions

View File

@ -10,3 +10,5 @@ analyzed_filename = 'access.log'
DB_ROOT = './output/' DB_ROOT = './output/'
DISPLAY_ROOT = './output/' DISPLAY_ROOT = './output/'
pre_analysis_hooks = ['H002_soutade.py', 'H001_robot.py']

View File

@ -1,22 +0,0 @@
import re
# Remove logo from indefero
logo_re = re.compile(r'^.+/logo/$')
# Basic rule to detect robots
def hook(hits):
for k in hits.keys():
super_hit = hits[k]
if super_hit['robot']: continue
for p in super_hit['pages']:
if not p['is_page']: continue
if int(p['status']) != 200: continue
if logo_re.match(p['extract_request']['extract_uri']):
p['is_page'] = False
if super_hit['viewed_pages']:
super_hit['viewed_pages'] -= 1
super_hit['viewed_hits'] += 1

View File

@ -1,42 +0,0 @@
# Basic rule to detect robots
def hook(hits):
for k in hits.keys():
super_hit = hits[k]
if super_hit['robot']: continue
isRobot = False
referers = 0
# 1) no pages view --> robot
# if not super_hit['viewed_pages']:
# super_hit['robot'] = 1
# continue
# 2) pages without hit --> robot
if not super_hit['viewed_hits']:
super_hit['robot'] = 1
continue
elif not super_hit['viewed_pages']:
# Hit only
super_hit['hit_only'] = 1
for hit in super_hit['pages']:
# 3) /robots.txt read
if hit['extract_request']['http_uri'] == '/robots.txt':
isRobot = True
break
# 4) Any referer for hits
if not hit['is_page'] and hit['http_referer']:
referers += 1
if isRobot:
super_hit['robot'] = 1
continue
if super_hit['viewed_hits'] and not referers:
super_hit['robot'] = 1
continue

62
iwla.py
View File

@ -8,8 +8,6 @@ import imp
import pickle import pickle
import gzip import gzip
from robots import awstats_robots;
# Default configuration # Default configuration
DB_ROOT = './output/' DB_ROOT = './output/'
@ -21,6 +19,10 @@ log_format = '$server_name:$server_port $remote_addr - $remote_user [$time_local
time_format = '%d/%b/%Y:%H:%M:%S +0100' time_format = '%d/%b/%Y:%H:%M:%S +0100'
pre_analysis_hooks = []
post_analysis_hooks = []
display_hooks = []
from conf import * from conf import *
print '==> Start' print '==> Start'
@ -40,16 +42,36 @@ uri_re = re.compile(r'(?P<extract_uri>[^\?]*)[\?(?P<extract_parameters>.*)]?')
pages_extensions = ['/', 'html', 'xhtml', 'py', 'pl', 'rb', 'php'] pages_extensions = ['/', 'html', 'xhtml', 'py', 'pl', 'rb', 'php']
viewed_http_codes = [200] viewed_http_codes = [200]
HOOKS_ROOT = './hooks/' HOOKS_ROOT = './plugins/'
PRE_HOOK_DIRECTORY = HOOKS_ROOT + 'pre_analysis/*.py' PRE_HOOK_DIRECTORY = HOOKS_ROOT + 'pre_analysis/'
POST_HOOK_DIRECTORY = HOOKS_ROOT + 'post_analysis/*.py' POST_HOOK_DIRECTORY = HOOKS_ROOT + 'post_analysis/'
DISPLAY_HOOK_DIRECTORY = HOOKS_ROOT + 'display/*.py' DISPLAY_HOOK_DIRECTORY = HOOKS_ROOT + 'display/'
META_PATH = DB_ROOT + 'meta.db' META_PATH = DB_ROOT + 'meta.db'
DB_FILENAME = 'iwla.db' DB_FILENAME = 'iwla.db'
print '==> Generating robot dictionary' plugins = {PRE_HOOK_DIRECTORY : pre_analysis_hooks, POST_HOOK_DIRECTORY : post_analysis_hooks, DISPLAY_HOOK_DIRECTORY : display_hooks}
awstats_robots = map(lambda (x) : re.compile(x, re.IGNORECASE), awstats_robots) ANALYSIS_CLASS = 'HTTP'
API_VERSION = 1
def preloadPlugins():
for root in plugins.keys():
for plugin_name in plugins[root]:
p = root + '/' + plugin_name
try:
mod = cache_plugins[p] = imp.load_source('hook', p)
infos = mod.get_plugins_infos()
if infos['class'] != ANALYSIS_CLASS or \
API_VERSION < infos['min_version'] or\
(infos['max_version'] != -1 and (API_VERSION > infos['max_version'])):
del cache_plugins[p]
elif not mod.load():
del cache_plugins[p]
except Exception as e:
print 'Error loading \'%s\' => %s' % (p, e)
return False
return True
def createEmptyVisits(): def createEmptyVisits():
visits = {'days_stats' : {}, 'month_stats' : {}, 'visits' : {}} visits = {'days_stats' : {}, 'month_stats' : {}, 'visits' : {}}
@ -97,17 +119,11 @@ def deserialize(filename):
return pickle.load(f) return pickle.load(f)
return None return None
def callPlugins(path, *kwargs): def callPlugins(root, *kwargs):
print '==> Call plugins (%s)' % path print '==> Call plugins (%s)' % root
plugins = glob.glob(path) for p in plugins[root]:
plugins.sort()
for p in plugins:
print '\t%s' % (p) print '\t%s' % (p)
if not p in cache_plugins: mod = cache_plugins[root + '/' + p]
mod = imp.load_source('hook', p)
cache_plugins[p] = mod
else:
mod = cache_plugins[p]
mod.hook(*kwargs) mod.hook(*kwargs)
def isPage(request): def isPage(request):
@ -164,16 +180,10 @@ def createUser(hit):
super_hit['bandwith'] = 0; super_hit['bandwith'] = 0;
super_hit['last_access'] = meta_visit['last_time'] super_hit['last_access'] = meta_visit['last_time']
super_hit['pages'] = []; super_hit['pages'] = [];
super_hit['robot'] = isRobot(hit); super_hit['robot'] = False
super_hit['hit_only'] = 0; super_hit['hit_only'] = 0;
appendHit(hit) appendHit(hit)
def isRobot(hit):
for r in awstats_robots:
if r.match(hit['http_user_agent']):
return True
return False
def decodeHTTPRequest(hit): def decodeHTTPRequest(hit):
if not 'request' in hit.keys(): return False if not 'request' in hit.keys(): return False
@ -385,6 +395,8 @@ def newHit(hit):
return True return True
preloadPlugins()
print '==> Analysing log' print '==> Analysing log'
meta_visit = deserialize(META_PATH) or createEmptyMeta() meta_visit = deserialize(META_PATH) or createEmptyMeta()

View File

@ -1,3 +1,23 @@
import re
from awstats_robots_data import awstats_robots
PLUGIN_CLASS = 'HTTP'
API_VERSION = 1
def get_plugins_infos():
infos = {'class' : PLUGIN_CLASS,
'min_version' : API_VERSION,
'max_version' : -1}
return infos
def load():
global awstats_robots
print '==> Generating robot dictionary'
awstats_robots = map(lambda (x) : re.compile(x, re.IGNORECASE), awstats_robots)
return True
# Basic rule to detect robots # Basic rule to detect robots
@ -10,6 +30,11 @@ def hook(hits):
isRobot = False isRobot = False
referers = 0 referers = 0
for r in awstats_robots:
if r.match(super_hit['pages'][0]['http_user_agent']):
super_hit['robot'] = 1
continue
# 1) no pages view --> robot # 1) no pages view --> robot
if not super_hit['viewed_pages']: if not super_hit['viewed_pages']:
super_hit['robot'] = 1 super_hit['robot'] = 1

View File

@ -3,6 +3,18 @@ import re
# Remove logo from indefero # Remove logo from indefero
logo_re = re.compile(r'^.+/logo/$') logo_re = re.compile(r'^.+/logo/$')
PLUGIN_CLASS = 'HTTP'
API_VERSION = 1
def get_plugins_infos():
infos = {'class' : PLUGIN_CLASS,
'min_version' : API_VERSION,
'max_version' : -1}
return infos
def load():
return True
# Basic rule to detect robots # Basic rule to detect robots
def hook(hits): def hook(hits):
@ -13,6 +25,7 @@ def hook(hits):
for p in super_hit['pages']: for p in super_hit['pages']:
if not p['is_page']: continue if not p['is_page']: continue
if int(p['status']) != 200: continue
if logo_re.match(p['extract_request']['extract_uri']): if logo_re.match(p['extract_request']['extract_uri']):
p['is_page'] = False p['is_page'] = False
super_hit['viewed_pages'] -= 1 super_hit['viewed_pages'] -= 1