Do a lot of things

This commit is contained in:
Grégory Soutadé 2014-11-26 16:17:16 +01:00
parent 6505ca3ee5
commit 81b3eee552
12 changed files with 333 additions and 48 deletions

12
awstats_data.py Normal file

File diff suppressed because one or more lines are too long

View File

@ -8,12 +8,17 @@ time_format = '%d/%b/%Y:%H:%M:%S +0100'
analyzed_filename = 'access.log'
domain_name = 'soutade.fr'
display_visitor_ip = True
DB_ROOT = './output/'
DISPLAY_ROOT = './output/'
pre_analysis_hooks = ['page_to_hit', 'robots']
post_analysis_hooks = ['top_visitors', 'reverse_dns']
display_hooks = ['top_visitors', 'all_visits']
post_analysis_hooks = ['top_visitors']
# post_analysis_hooks = ['top_visitors', 'reverse_dns']
display_hooks = ['top_visitors', 'all_visits', 'referers']
reverse_dns_timeout = 0.2
page_to_hit_conf = [r'^.+/logo/$']

View File

@ -21,4 +21,4 @@ post_analysis_hooks = []
display_hooks = []
pages_extensions = ['/', 'html', 'xhtml', 'py', 'pl', 'rb', 'php']
viewed_http_codes = [200]
viewed_http_codes = [200, 304]

22
iwla.py
View File

@ -34,7 +34,7 @@ class IWLA(object):
self.log_format_extracted = re.sub(r'\$(\w+)', '(?P<\g<1>>.+)', self.log_format_extracted)
self.http_request_extracted = re.compile(r'(?P<http_method>\S+) (?P<http_uri>\S+) (?P<http_version>\S+)')
self.log_re = re.compile(self.log_format_extracted)
self.uri_re = re.compile(r'(?P<extract_uri>[^\?]*)[\?(?P<extract_parameters>.*)]?')
self.uri_re = re.compile(r'(?P<extract_uri>[^\?]+)(\?(?P<extract_parameters>.+))?')
self.plugins = {conf.PRE_HOOK_DIRECTORY : conf.pre_analysis_hooks,
conf.POST_HOOK_DIRECTORY : conf.post_analysis_hooks,
conf.DISPLAY_HOOK_DIRECTORY : conf.display_hooks}
@ -143,9 +143,9 @@ class IWLA(object):
hit['is_page'] = self.isPage(uri)
# Don't count 3xx status
status = int(hit['status'])
if status >= 300 and status < 400: return
if status not in conf.viewed_http_codes:
return
if super_hit['robot'] or\
not status in conf.viewed_http_codes:
@ -163,6 +163,7 @@ class IWLA(object):
def _createVisitor(self, hit):
super_hit = self.current_analysis['visits'][hit['remote_addr']] = {}
super_hit['remote_addr'] = hit['remote_addr']
super_hit['remote_ip'] = hit['remote_addr']
super_hit['viewed_pages'] = 0
super_hit['viewed_hits'] = 0
super_hit['not_viewed_pages'] = 0
@ -191,9 +192,10 @@ class IWLA(object):
print "Bad request extraction " + hit['request']
return False
referer_groups = self.uri_re.match(hit['http_referer'])
if referer_groups:
referer = hit['extract_referer'] = referer_groups.groupdict()
if hit['http_referer']:
referer_groups = self.uri_re.match(hit['http_referer'])
if referer_groups:
hit['extract_referer'] = referer_groups.groupdict()
return True
def _decodeTime(self, hit):
@ -229,13 +231,13 @@ class IWLA(object):
nb_days = len(keys)
row = [0, nb_visits, stats['viewed_pages'], stats['viewed_hits'], stats['viewed_bandwidth'], stats['not_viewed_bandwidth']]
if nb_days:
average_row = map(lambda(v): str(int(v/nb_days)), row)
average_row = map(lambda(v): int(v/nb_days), row)
else:
average_row = map(lambda(v): '0', row)
average_row = map(lambda(v): 0, row)
average_row[0] = 'Average'
average_row[4] = bytesToStr(row[4])
average_row[5] = bytesToStr(row[5])
average_row[4] = bytesToStr(average_row[4])
average_row[5] = bytesToStr(average_row[5])
days.appendRow(average_row)
row[0] = 'Total'

View File

@ -1,34 +1,79 @@
#!/usr/bin/perl
my $awstats_lib_root = '/usr/share/awstats/lib/';
my @awstats_libs = ('browsers.pm', 'browsers_phone.pm', 'mime.pm', 'referer_spam.pm', 'search_engines.pm', 'operating_systems.pm', 'robots.pm', 'worms.pm');
my $awstats_lib_root = './';
my @awstats_libs = ('search_engines.pm', 'robots.pm');
# my $awstats_lib_root = '/usr/share/awstats/lib/';
# my @awstats_libs = ('browsers.pm', 'browsers_phone.pm', 'mime.pm', 'referer_spam.pm', 'search_engines.pm', 'operating_systems.pm', 'robots.pm', 'worms.pm');
foreach $lib (@awstats_libs) {require $awstats_lib_root . $lib;}
open($FIC,">", "robots.py") or die $!;
sub dumpList {
my @list = @{$_[0]};
my $FIC = $_[1];
my $first = $_[2];
print $FIC "awstats_robots = [";
$first = 0;
foreach $r (@RobotsSearchIDOrder_list1)
{
$r =~ s/\'/\\\'/g;
if ($first != 0)
foreach $r (@list)
{
print $FIC ", ";
$r =~ s/\'/\\\'/g;
if ($first == 0)
{
print $FIC ", ";
}
else
{
$first = 0;
}
print $FIC "'.*$r.*'";
}
else
{
$first = 1;
}
sub dumpHash {
my %hash = %{$_[0]};
my $FIC = $_[1];
my $first = $_[2];
while( my ($k,$v) = each(%hash) ) {
$k =~ s/\'/\\\'/g;
$v =~ s/\'/\\\'/g;
if ($first == 0)
{
print $FIC ", ";
}
else
{
$first = 0;
}
print $FIC "'.*$k.*' : '$v'";
}
print $FIC "'.*$r.*'";
}
foreach $r (@RobotsSearchIDOrder_list2)
{
$r =~ s/\'/\\\'/g;
print $FIC ", '.*$r.*'";
}
# Robots
open($FIC,">", "awstats_data.py") or die $!;
print $FIC "robots = [";
dumpList(\@RobotsSearchIDOrder_list1, $FIC, 1);
dumpList(\@RobotsSearchIDOrder_list2, $FIC, 0);
print $FIC "]\n\n";
print $FIC "search_engines = [";
dumpList(\@SearchEnginesSearchIDOrder_list1, $FIC, 1);
print $FIC "]\n\n";
print $FIC "search_engines_2 = [";
dumpList(\@SearchEnginesSearchIDOrder_list2, $FIC, 1);
print $FIC "]\n\n";
print $FIC "not_search_engines_keys = {";
dumpHash(\%NotSearchEnginesKeys, $FIC, 1);
print $FIC "}\n\n";
print $FIC "search_engines_hashid = {";
dumpHash(\%SearchEnginesHashID, $FIC, 1);
print $FIC "}\n\n";
print $FIC "search_engines_knwown_url = {";
dumpHash(\%SearchEnginesKnownUrl, $FIC, 1);
print $FIC "}\n\n";
close($FIC);

View File

@ -14,7 +14,7 @@ class IWLADisplayAllVisits(IPlugin):
last_access = sorted(hits.values(), key=lambda t: t['last_access'], reverse=True)
cur_time = self.iwla.getCurTime()
title = time.strftime('All visits %B %Y', cur_time)
title = time.strftime('All visits - %B %Y', cur_time)
filename = 'all_visits_%d.html' % (cur_time.tm_mon)
path = '%d/%s' % (cur_time.tm_year, filename)
@ -22,8 +22,13 @@ class IWLADisplayAllVisits(IPlugin):
page = DisplayHTMLPage(title, path)
table = DisplayHTMLBlockTable('Last seen', ['Host', 'Pages', 'Hits', 'Bandwidth', 'Last seen'])
for super_hit in last_access:
address = super_hit['remote_addr']
if self.iwla.getConfValue('display_visitor_ip', False) and\
super_hit.get('dns_name_replaced', False):
address = '%s [%s]' % (address, super_hit['remote_ip'])
row = [
super_hit['remote_addr'],
address,
super_hit['viewed_pages'],
super_hit['viewed_hits'],
bytesToStr(super_hit['bandwidth']),

199
plugins/display/referers.py Normal file
View File

@ -0,0 +1,199 @@
import time
import re
import HTMLParser
from iwla import IWLA
from iplugin import IPlugin
from display import *
import awstats_data
class IWLADisplayReferers(IPlugin):
def __init__(self, iwla):
super(IWLADisplayReferers, self).__init__(iwla)
self.API_VERSION = 1
def load(self):
domain_name = self.iwla.getConfValue('domain_name', '')
if not domain_name:
print 'domain_name required in conf'
return False
self.own_domain_re = re.compile('.*%s.*' % (domain_name))
self.search_engines = {}
for engine in awstats_data.search_engines:
self.search_engines[engine] = {
're' : re.compile(engine, re.IGNORECASE)
}
for (engine, not_engine) in awstats_data.not_search_engines_keys.items():
if not engine in self.search_engines: continue
self.search_engines[engine]['not_search_engine'] = \
re.compile(not_engine, re.IGNORECASE)
for (engine, name) in awstats_data.search_engines_hashid.items():
if not engine in self.search_engines: continue
self.search_engines[engine]['name'] = name
for (engine, knwown_url) in awstats_data.search_engines_knwown_url.items():
engine = engin[2:-2]
if not engine in self.search_engines: continue
print knwown_url
self.search_engines[engine]['known_url'] = re.compile(known_url + '(?P<key_phrase>.+)')
self.html_parser = HTMLParser.HTMLParser()
return True
def _extractKeyPhrase(self, key_phrase_re, parameters, key_phrases):
if not parameters or not key_phrase_re: return
for p in parameters.split('&'):
groups = key_phrase_re.match(p)
if groups:
print groups.groupddict()
key_phrase = self.html_parser.unescape(groups.groupddict()['key_phrase']).lower()
if not key_phrase in key_phrases.keys():
key_phrases[key_phrase] = 1
else:
key_phrases[key_phrase] += 1
def hook(self, iwla):
stats = iwla.getCurrentVisists()
referers = {}
robots_referers = {}
search_engine_referers = {}
key_phrases = {}
for (k, super_hit) in stats.items():
for r in super_hit['requests']:
if not r['http_referer']: continue
uri = r['extract_referer']['extract_uri']
is_search_engine = False
if self.own_domain_re.match(uri): continue
for e in self.search_engines.values():
if e['re'].match(uri):
not_engine = e.get('not_search_engine', None)
# Try not engine
if not_engine and not_engine.match(uri): break
is_search_engine = True
uri = e['name']
parameters = r['extract_referer'].get('extract_parameters', None)
key_phrase_re = e.get('known_url', None)
print parameters
print key_phrase_re
self._extractKeyPhrase(key_phrase_re, parameters, key_phrases)
break
if is_search_engine:
dictionary = search_engine_referers
elif super_hit['robot']:
dictionary = robots_referers
# print '%s => %s' % (uri, super_hit['remote_ip'])
else:
dictionary = referers
if r['is_page']:
key = 'pages'
else:
key = 'hits'
if not uri in dictionary: dictionary[uri] = {'pages':0, 'hits':0}
dictionary[uri][key] += 1
top_referers = [(k, referers[k]['pages']) for k in referers.keys()]
top_referers = sorted(top_referers, key=lambda t: t[1], reverse=True)
top_robots_referers = [(k, robots_referers[k]['pages']) for k in robots_referers.keys()]
top_robots_referers = sorted(top_robots_referers, key=lambda t: t[1], reverse=True)
top_search_engine_referers = [(k, search_engine_referers[k]['pages']) for k in search_engine_referers.keys()]
top_search_engine_referers = sorted(top_search_engine_referers, key=lambda t: t[1], reverse=True)
top_key_phrases = key_phrases.items()
top_key_phrases = sorted(top_key_phrases, key=lambda t: t[1], reverse=True)
# Top referers in index
index = self.iwla.getDisplayIndex()
table = DisplayHTMLBlockTable('Connexion from', ['Origin', 'Pages', 'Hits'])
table.appendRow(['<b>Search Engine</b>', '', ''])
for r,_ in top_search_engine_referers[:10]:
row = [r, search_engine_referers[r]['pages'], search_engine_referers[r]['hits']]
table.appendRow(row)
table.appendRow(['<b>External URL</b>', '', ''])
for r,_ in top_referers[:10]:
row = [r, referers[r]['pages'], referers[r]['hits']]
table.appendRow(row)
table.appendRow(['<b>External URL (robot)</b>', '', ''])
for r,_ in top_robots_referers[:10]:
row = [r, robots_referers[r]['pages'], robots_referers[r]['hits']]
table.appendRow(row)
index.appendBlock(table)
# All referers in a file
cur_time = self.iwla.getCurTime()
title = time.strftime('Connexion from - %B %Y', cur_time)
filename = 'referers_%d.html' % (cur_time.tm_mon)
path = '%d/%s' % (cur_time.tm_year, filename)
page = DisplayHTMLPage(title, path)
table = DisplayHTMLBlockTable('Connexion from', ['Origin', 'Pages', 'Hits'])
table.appendRow(['<b>Search Engine</b>', '', ''])
for r,_ in top_search_engine_referers:
row = [r, search_engine_referers[r]['pages'], search_engine_referers[r]['hits']]
table.appendRow(row)
table.appendRow(['<b>External URL</b>', '', ''])
for r,_ in top_referers:
row = [r, referers[r]['pages'], referers[r]['hits']]
table.appendRow(row)
table.appendRow(['<b>External URL (robot)</b>', '', ''])
for r,_ in top_robots_referers:
row = [r, robots_referers[r]['pages'], robots_referers[r]['hits']]
table.appendRow(row)
page.appendBlock(table)
display = self.iwla.getDisplay()
display.addPage(page)
block = DisplayHTMLRawBlock()
block.setRawHTML('<a href=\'%s\'>All referers</a>' % (filename))
index.appendBlock(block)
# Top key phrases in index
table = DisplayHTMLBlockTable('Top key phrases', ['Key phrase', 'Search'])
for phrase in top_key_phrases[:10]:
table.appendRow([phrase[0], phrase[1]])
index.appendBlock(table)
# All key phrases in a file
cur_time = self.iwla.getCurTime()
title = time.strftime('Key Phrases - %B %Y', cur_time)
filename = 'key_phrases_%d.html' % (cur_time.tm_mon)
path = '%d/%s' % (cur_time.tm_year, filename)
page = DisplayHTMLPage(title, path)
table = DisplayHTMLBlockTable('Top key phrases', ['Key phrase', 'Search'])
for phrase in top_key_phrases:
table.appendRow([phrase[0], phrase[1]])
page.appendBlock(table)
display.addPage(page)

View File

@ -16,8 +16,13 @@ class IWLADisplayTopVisitors(IPlugin):
index = iwla.getDisplayIndex()
table = DisplayHTMLBlockTable('Top visitors', ['Host', 'Pages', 'Hits', 'Bandwidth', 'Last seen'])
for super_hit in stats['top_visitors']:
address = super_hit['remote_addr']
if self.iwla.getConfValue('display_visitor_ip', False) and\
super_hit.get('dns_name_replaced', False):
address = '%s [%s]' % (address, super_hit['remote_ip'])
row = [
super_hit['remote_addr'],
address,
super_hit['viewed_pages'],
super_hit['viewed_hits'],
bytesToStr(super_hit['bandwidth']),

View File

@ -20,6 +20,7 @@ class IWLAPostAnalysisReverseDNS(IPlugin):
try:
name, _, _ = socket.gethostbyaddr(k)
hit['remote_addr'] = name
hit['dns_name_replaced'] = True
except:
pass
finally:

View File

@ -9,7 +9,7 @@ class IWLAPostAnalysisTopVisitors(IPlugin):
def hook(self, iwla):
hits = iwla.getValidVisitors()
stats = iwla.getMonthStats()
top_bandwidth = [(k,hits[k]['bandwidth']) for (k,v) in hits.items()]
top_bandwidth = [(k,hits[k]['bandwidth']) for k in hits.keys()]
top_bandwidth = sorted(top_bandwidth, key=lambda t: t[1], reverse=True)
stats['top_visitors'] = [hits[h[0]] for h in top_bandwidth[:10]]

View File

@ -3,7 +3,7 @@ import re
from iwla import IWLA
from iplugin import IPlugin
from awstats_robots_data import awstats_robots
import awstats_data
class IWLAPreAnalysisRobots(IPlugin):
def __init__(self, iwla):
@ -11,9 +11,7 @@ class IWLAPreAnalysisRobots(IPlugin):
self.API_VERSION = 1
def load(self):
global awstats_robots
self.awstats_robots = map(lambda (x) : re.compile(x, re.IGNORECASE), awstats_robots)
self.awstats_robots = map(lambda (x) : re.compile(x, re.IGNORECASE), awstats_data.robots)
return True
@ -32,13 +30,17 @@ class IWLAPreAnalysisRobots(IPlugin):
if first_page['time_decoded'].tm_mday == super_hit['last_access'].tm_mday:
for r in self.awstats_robots:
if r.match(first_page['http_user_agent']):
super_hit['robot'] = 1
continue
isRobot = True
break
if isRobot:
super_hit['robot'] = 1
continue
# 1) no pages view --> robot
if not super_hit['viewed_pages']:
super_hit['robot'] = 1
continue
# if not super_hit['viewed_pages']:
# super_hit['robot'] = 1
# continue
# 2) pages without hit --> robot
if not super_hit['viewed_hits']:
@ -59,6 +61,7 @@ class IWLAPreAnalysisRobots(IPlugin):
super_hit['robot'] = 1
continue
if super_hit['viewed_hits'] and not referers:
if not super_hit['viewed_pages'] and \
(super_hit['viewed_hits'] and not referers):
super_hit['robot'] = 1
continue

8
search_engines.py Normal file

File diff suppressed because one or more lines are too long