Initial commit

This commit is contained in:
Gregory Soutade 2014-11-18 20:18:53 +01:00
commit 26688e4bf7
3 changed files with 214 additions and 0 deletions

178
iwla.py Executable file
View File

@ -0,0 +1,178 @@
#!/usr/bin/env python
import os
import re
import time
import glob
import imp
from robots import awstats_robots;
print '==> Start'
current_visit = {}
log_format = '$server_name:$server_port $remote_addr - $remote_user [$time_local] ' +\
'"$request" $status $body_bytes_sent ' +\
'"$http_referer" "$http_user_agent"';
log_format_extracted = re.sub(r'([^\$\w])', r'\\\g<1>', log_format);
log_format_extracted = re.sub(r'\$(\w+)', '(?P<\g<1>>.+)', log_format_extracted)
http_request_extracted = re.compile(r'(?P<http_method>\S+) (?P<http_uri>\S+) (?P<http_version>\S+)')
#09/Nov/2014:06:35:16 +0100
time_format = '%d/%b/%Y:%H:%M:%S +0100'
#print "Log format : " + log_format_extracted
log_re = re.compile(log_format_extracted)
uri_re = re.compile(r'(?P<extract_uri>[^\?]*)\?(?P<extract_parameters>.*)')
pages_extensions = ['/', 'html', 'xhtml', 'py', 'pl', 'rb', 'php']
viewed_http_codes = [200]
cur_time = None
print '==> Generating robot dictionary'
awstats_robots = map(lambda (x) : re.compile(x, re.IGNORECASE), awstats_robots)
def isPage(request):
for e in pages_extensions:
if request.endswith(e):
return True
return False
def appendHit(hit):
super_hit = current_visit[hit['remote_addr']]
super_hit['pages'].append(hit)
super_hit['bandwith'] += int(hit['body_bytes_sent'])
request = hit['extract_request']
if 'extract_uri' in request.keys():
uri = request['extract_uri']
else:
uri = request['http_uri']
hit['is_page'] = isPage(uri)
# Don't count redirect status
if int(hit['status']) == 302: return
if super_hit['robot'] or\
not int(hit['status']) in viewed_http_codes:
page_key = 'not_viewed_pages'
hit_key = 'not_viewed_hits'
else:
page_key = 'viewed_pages'
hit_key = 'viewed_hits'
if hit['is_page']:
super_hit[page_key] += 1
else:
super_hit[hit_key] += 1
def createGeneric(hit):
super_hit = current_visit[hit['remote_addr']] = {}
super_hit['viewed_pages'] = 0;
super_hit['viewed_hits'] = 0;
super_hit['not_viewed_pages'] = 0;
super_hit['not_viewed_hits'] = 0;
super_hit['bandwith'] = 0;
super_hit['pages'] = [];
return super_hit
def createUser(hit, robot):
super_hit = createGeneric(hit)
super_hit['robot'] = robot;
appendHit(hit)
def isRobot(hit):
for r in awstats_robots:
if r.match(hit['http_user_agent']):
return True
return False
def decode_http_request(hit):
if not 'request' in hit.keys(): return False
groups = http_request_extracted.match(hit['request'])
if groups:
hit['extract_request'] = groups.groupdict()
uri_groups = uri_re.match(hit['extract_request']['http_uri']);
if uri_groups:
hit['extract_request']['extract_uri'] = uri_groups.group('extract_uri')
hit['extract_request']['extract_parameters'] = uri_groups.group('extract_parameters')
else:
print "Bad request extraction " + hit['request']
return False
referer_groups = uri_re.match(hit['http_referer']);
if referer_groups:
hit['extract_referer']['extract_uri'] = referer_groups.group('extract_uri')
hit['extract_referer']['extract_parameters'] = referer_groups.group('extract_parameters')
return True
def decode_time(hit):
t = hit['time_local']
hit['time_decoded'] = time.strptime(t, time_format)
def newHit(hit):
global cur_time
if not decode_http_request(hit): return
for k in hit.keys():
if hit[k] == '-': hit[k] = ''
decode_time(hit)
t = hit['time_decoded']
current_visit['last_time'] = t
if cur_time == None:
cur_time = t
else:
if cur_time.tm_mday != t.tm_mday:
return False
remote_addr = hit['remote_addr']
if remote_addr in current_visit.keys():
appendHit(hit)
else:
createUser(hit, isRobot(hit))
return True
print '==> Analysing log'
f = open("access.log")
for l in f:
# print "line " + l;
groups = log_re.match(l)
if groups:
if not newHit(groups.groupdict()):
break
else:
print "No match " + l
f.close();
print '==> Call plugins'
plugins = glob.glob('./hooks_pre/*.py')
plugins.sort()
for p in plugins:
print '\t%s' % (p)
mod = imp.load_source('hook', p)
mod.hook(current_visit)
for ip in current_visit.keys():
hit = current_visit[ip]
if hit['robot']: continue
print "%s =>" % (ip)
for k in hit.keys():
if k != 'pages':
print "\t%s : %s" % (k, current_visit[ip][k])

34
iwla_convert.pl Executable file
View File

@ -0,0 +1,34 @@
#!/usr/bin/perl
my $awstats_lib_root = '/usr/share/awstats/lib/';
my @awstats_libs = ('browsers.pm', 'browsers_phone.pm', 'mime.pm', 'referer_spam.pm', 'search_engines.pm', 'operating_systems.pm', 'robots.pm', 'worms.pm');
foreach $lib (@awstats_libs) {require $awstats_lib_root . $lib;}
open($FIC,">", "robots.py") or die $!;
print $FIC "awstats_robots = [";
$first = 0;
foreach $r (@RobotsSearchIDOrder_list1)
{
$r =~ s/\'/\\\'/g;
if ($first != 0)
{
print $FIC ", ";
}
else
{
$first = 1;
}
print $FIC "'.*$r.*'";
}
foreach $r (@RobotsSearchIDOrder_list2)
{
$r =~ s/\'/\\\'/g;
print $FIC ", '.*$r.*'";
}
print $FIC "]\n\n";
close($FIC);

2
robots.py Normal file

File diff suppressed because one or more lines are too long