iwla

iwla Commit Details

Date:2016-02-06 14:45:09 (5 years 5 months ago)
Author:Grégory Soutadé
Branch:dev, master
Commit:12cc80208dec00eb3db4e4c3b321fc4b6c8c6701
Parents: 992c3cee934ec47c3913dd983d661d6a706d8a46
Message:Do merge

Changes:
Aplugins/display/ip_to_geo.py (full)
Aplugins/post_analysis/ip_to_geo.py (full)
Aplugins/post_analysis/iptogeo.py (full)
Mconf.py (1 diff)
Mdisplay.py (8 diffs)
Miwla.py (2 diffs)
Mplugins/display/feeds.py (1 diff)
Mplugins/display/track_users.py (2 diffs)
Mplugins/post_analysis/feeds.py (1 diff)
Mplugins/post_analysis/hours_stats.py (1 diff)
Mplugins/post_analysis/referers.py (1 diff)
Mplugins/post_analysis/reverse_dns.py (1 diff)
Mplugins/post_analysis/top_hits.py (1 diff)
Mplugins/post_analysis/top_pages.py (1 diff)
Mplugins/pre_analysis/page_to_hit.py (1 diff)
Mplugins/pre_analysis/robots.py (1 diff)

File differences

conf.py
1010
1111
1212
13
14
13
14
1515
1616
1717
# Hooks used
pre_analysis_hooks = ['page_to_hit', 'robots']
post_analysis_hooks = ['referers', 'top_pages', 'top_downloads', 'operating_systems', 'browsers', 'feeds', 'hours_stats', 'reverse_dns']
display_hooks = ['track_users', 'top_visitors', 'all_visits', 'referers', 'top_pages', 'top_downloads', 'referers_diff', 'operating_systems', 'browsers', 'feeds', 'hours_stats', 'top_downloads_diff']
post_analysis_hooks = ['referers', 'top_pages', 'top_downloads', 'operating_systems', 'browsers', 'feeds', 'hours_stats', 'reverse_dns', 'ip_to_geo']
display_hooks = ['track_users', 'top_visitors', 'all_visits', 'referers', 'top_pages', 'top_downloads', 'referers_diff', 'ip_to_geo', 'operating_systems', 'browsers', 'feeds', 'hours_stats', 'top_downloads_diff']
# Reverse DNS timeout
reverse_dns_timeout = 0.2
display.py
4848
4949
5050
51
51
52
5253
5354
5455
56
57
58
59
60
61
62
5563
5664
5765
......
189197
190198
191199
200
201
202
203
204
205
206
207
208
209
210
211
192212
193213
194214
......
315335
316336
317337
318
338
339
340
341
319342
320343
321344
322
345
323346
324347
325348
......
339362
340363
341364
342
365
343366
344367
345368
......
349372
350373
351374
352
353375
376
377
378
379
380
354381
355382
356383
......
364391
365392
366393
394
395
396
367397
368398
369399
......
378408
379409
380410
381
411
412
413
414
415
382416
383417
384418
......
417451
418452
419453
420
def _build(self, f, html):
if html: f.write(html)
def build(self, f):
def build(self, f, filters=None):
if filters: self.filter(filters)
self._buildHTML()
self._build(f, self.html)
def _filter(self, function, **kwargs):
pass
def filter(self, filters):
for (args, function) in filters:
self._filter(function, **args)
def getTitle(self):
return ''
val = r[column] and int(r[column]) or 0
self.setCellValue(index, column_insertion, '%.1f%%' % (float(val*100)/float(total)))
def _filter(self, function, column, args):
target_col = None
for col in range(0, len(self.cols)):
if self.cols[col] == column:
target_col = col
break
if target_col is None: return
for row in self.rows:
res = function(row[target_col], **args)
if res:
row[target_col] = res
def _buildHTML(self):
style = u''
if self.table_css: style = u' class="%s"' % (self.table_css)
if title == b.getTitle():
return b
return None
def getAllBlocks(self):
return self.blocks
def appendBlock(self, block):
self.blocks.append(block)
def build(self, root, displayVersion=True):
def build(self, root, displayVersion=True, filters=None):
filename = os.path.join(root, self.filename)
base = os.path.dirname(filename)
f.write(u'<title>%s</title>' % (self.title))
f.write(u'</head><body>')
for block in self.blocks:
block.build(f)
block.build(f, filters=filters)
if displayVersion:
f.write(u'<center>Generated by <a href="%s">IWLA %s</a></center>' %
("http://indefero.soutade.fr/p/iwla", self.iwla.getVersion()))
class DisplayHTMLBuild(object):
def __init__(self, iwla):
self.pages = []
self.iwla = iwla
self.filters = []
self.clear()
def clear(self):
self.pages = []
def createPage(self, *args):
return DisplayHTMLPage(self.iwla, *args)
return page
return None
def getAllPages(self):
return self.pages
def addPage(self, page):
self.pages.append(page)
os.symlink(target, link_name)
for page in self.pages:
page.build(root)
page.build(root, filters=self.filters)
def addColumnFilter(self, column, function, args):
self.filters.append(({'column':column, 'args':args}, function))
#
# Global functions
if domain_name:
title += u' - %s' % (domain_name)
return title
iwla.py
188188
189189
190190
191
191
192192
193193
194194
......
230230
231231
232232
233
233
234234
235235
236236
def getMonthStats(self):
return self.current_analysis['month_stats']
def getCurrentVisists(self):
def getCurrentVisits(self):
return self.current_analysis['visits']
def getValidVisitors(self):
return self.meta_infos
def _clearDisplay(self):
self.display = DisplayHTMLBuild(self)
self.display.clear()
return self.display
def getDBFilename(self, time):
plugins/display/feeds.py
6262
6363
6464
65
65
6666
6767
6868
from plugins.post_analysis.feeds import IWLAPostAnalysisFeeds
display = self.iwla.getDisplay()
hits = self.iwla.getCurrentVisists()
hits = self.iwla.getCurrentVisits()
nb_feeds_parsers = 0
# All in a page
plugins/display/ip_to_geo.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
# -*- coding: utf-8 -*-
#
# Copyright Grégory Soutadé 2015
# This file is part of iwla
# iwla is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# iwla is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with iwla. If not, see <http://www.gnu.org/licenses/>.
#
import re
from iwla import IWLA
from iplugin import IPlugin
from display import *
import awstats_data
"""
Display hook
Add geo statistics
Plugin requirements :
post_analysis/ip_to_geo
Conf values needed :
create_geo_page*
Output files :
OUTPUT_ROOT/year/month/index.html
Statistics creation :
None
Statistics update :
None
Statistics deletion :
None
"""
class IWLADisplayTopGeo(IPlugin):
def __init__(self, iwla):
super(IWLADisplayTopGeo, self).__init__(iwla)
self.API_VERSION = 1
self.requires = ['IWLAPostAnalysisIPToGeo']
def load(self):
self.icon_path = self.iwla.getConfValue('icon_path', '/')
self.create_geo_page = self.iwla.getConfValue('create_geo_page_page', True)
display = self.iwla.getDisplay()
display.addColumnFilter(self.iwla._(u'Host'), self.FlagFilter, {'self':self})
return True
@staticmethod # Needed to have unbound methd
def FlagFilter(host, self):
cc = None
host_name = host.split(' ')[0] # hostname or ip
if host_name in self.valid_visitors.keys():
cc = self.valid_visitors[host_name]['country_code']
else:
for visitor in self.valid_visitors.values():
if visitor['remote_addr'] == host_name:
cc = visitor['country_code']
break
if not cc or cc == 'ip': return None
icon = '<img src="/%s/flags/%s.png"/>' % (self.icon_path, cc)
return '%s %s' % (icon ,host)
def hook(self):
display = self.iwla.getDisplay()
geo = self.iwla.getMonthStats()['geo']
geo = sorted(geo.items(), key=lambda t: t[1], reverse=True)
self.valid_visitors = self.iwla.getValidVisitors()
# All in a page
if self.create_geo_page:
title = createCurTitle(self.iwla, u'All Coutries')
filename = 'geo.html'
path = self.iwla.getCurDisplayPath(filename)
page = display.createPage(title, path, self.iwla.getConfValue('css_path', []))
table = display.createBlock(DisplayHTMLBlockTable, self.iwla._(u'Countries'), ['', self.iwla._(u'Country'), self.iwla._(u'Visitors')])
table.setColsCSSClass(['', '', 'iwla_hit'])
for (cc, visitors) in geo:
icon = '<img src="/%s/flags/%s.png"/>' % (self.icon_path, cc)
table.appendRow([icon, cc, visitors])
table.computeRatio(2)
page.appendBlock(table)
display.addPage(page)
# Countries in index
title = self.iwla._(u'Countries')
if self.create_geo_page:
link = '<a href=\'%s\'>%s</a>' % (filename, self.iwla._(u'Details'))
title = '%s - %s' % (title, link)
index = self.iwla.getDisplayIndex()
table = display.createBlock(DisplayHTMLBlockTable, title, ['', self.iwla._(u'Countries'), self.iwla._(u'Visitors')])
table.setColsCSSClass(['', '', 'iwla_hit'])
for (cc, visitors) in geo[:10]:
icon = '<img src="/%s/flags/%s.png"/>' % (self.icon_path, cc)
table.appendRow([icon, cc, visitors])
table.computeRatio(2)
index.appendBlock(table)
plugins/display/track_users.py
6464
6565
6666
67
67
6868
6969
7070
......
110110
111111
112112
113
113
114114
115115
116116
def hook(self):
display = self.iwla.getDisplay()
hits = self.iwla.getCurrentVisists()
hits = self.iwla.getCurrentVisits()
stats = {}
# All in a page
index = self.iwla.getDisplayIndex()
table = display.createBlock(DisplayHTMLBlockTable, title, [self.iwla._(u'IP'), self.iwla._(u'Last Access'), self.iwla._(u'Pages'), self.iwla._(u'Hits')])
table = display.createBlock(DisplayHTMLBlockTable, title, [self.iwla._(u'Host'), self.iwla._(u'Last Access'), self.iwla._(u'Pages'), self.iwla._(u'Hits')])
table.setColsCSSClass(['', '', 'iwla_page', 'iwla_hit'])
for ip in self.tracked_ip:
if not ip in hits.keys(): continue
plugins/post_analysis/feeds.py
8585
8686
8787
88
88
8989
9090
9191
hit['feed_parser'] = isFeedParser
def hook(self):
hits = self.iwla.getCurrentVisists()
hits = self.iwla.getCurrentVisits()
one_hit_only = {}
for hit in hits.values():
isFeedParser = hit.get('feed_parser', None)
plugins/post_analysis/hours_stats.py
6262
6363
6464
65
65
6666
6767
6868
self.API_VERSION = 1
def hook(self):
stats = self.iwla.getCurrentVisists()
stats = self.iwla.getCurrentVisits()
month_stats = self.iwla.getMonthStats()
hours_stats = month_stats.get('hours_stats', {})
plugins/post_analysis/ip_to_geo.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
# -*- coding: utf-8 -*-
#
# Copyright Grégory Soutadé 2016
# This file is part of iwla
# iwla is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# iwla is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with iwla. If not, see <http://www.gnu.org/licenses/>.
#
from iwla import IWLA
from iplugin import IPlugin
from iptogeo import IPToGeo
"""
Post analysis hook
Get country code from IP address
Plugin requirements :
None
Conf values needed :
iptogeo_remote_addr*
iptogeo_remote_port*
Output files :
None
Statistics creation :
geo =>
country_code => count
None
Statistics update :
valid_visitors:
country_code
Statistics deletion :
None
"""
class IWLAPostAnalysisIPToGeo(IPlugin):
def __init__(self, iwla):
super(IWLAPostAnalysisIPToGeo, self).__init__(iwla)
self.API_VERSION = 1
def load(self):
remote_addr = self.iwla.getConfValue('iptogeo_remote_addr',
None)
remote_port = self.iwla.getConfValue('iptogeo_remote_port',
None)
args = {}
if remote_addr: args['remote_addr'] = remote_addr
if remote_port: args['remote_port'] = remote_port
self.iptogeo = IPToGeo(**args)
return True
def hook(self):
visitors = self.iwla.getValidVisitors()
month_stats = self.iwla.getMonthStats()
geo = month_stats.get('geo', {})
for (ip, visitor) in visitors.items():
if visitor.get('country_code', False): continue
try:
(_, cc) = self.iptogeo.ip_to_geo(ip)
cc = cc and cc or 'ip'
visitor['country_code'] = cc
if cc in geo.keys():
geo[cc] += 1
else:
geo[cc] = 1
except Exception, e:
print e
month_stats['geo'] = geo
plugins/post_analysis/iptogeo.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import socket
import struct
class IPToGeoException(Exception):
pass
class IPToGeo(object):
MAGIC = 0x179E08EF
VERSION = 1
REQ = 1
RESP = 0
IPV4 = 4
IPV6 = 16
IP_NOT_FOUND = 6
PACKET_SIZE = 32
ERRORS = {1 : 'Bad magic',
2 : 'Bad version',
3 : 'Bad request field' ,
4 : 'Bad IP version',
5 : 'Unsupported IP version',
6 : 'IP not found'}
def __init__(self, remote_addr='127.0.0.1', remote_port=53333, timeout=None):
self._remote_addr = remote_addr
self._remote_port = remote_port
self._timeout = timeout
self._create_socket()
def _create_socket(self):
self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
if not self._timeout is None:
self._socket.settimeout(self._timeout)
self._socket.connect((self._remote_addr, self._remote_port))
def _create_request(self, ip):
packet = ''
packet += struct.pack('<IBBBBI', IPToGeo.MAGIC, IPToGeo.VERSION, IPToGeo.REQ,
0, #err
IPToGeo.IPV4, # ip type
0) # flags
packet += struct.pack('<BBBB', ip[0], ip[1], ip[2], ip[3]) # ipv4
packet += struct.pack('<III', 0, 0, 0) # ipv6
packet += struct.pack('<I', 0) # country code
return packet
def _check_request(self, packet):
(magic, version, req, err, ip_type, flags, ipv4, ipv6b, ipv6c, ipv6d) = struct.unpack_from('<IBBBBIIIII', packet, 0)
if magic != IPToGeo.MAGIC:
raise IPToGeoException('Invalid magic %08x' % (magic))
if err == IPToGeo.IP_NOT_FOUND: return (ipv4, None) # IP not found
if err != 0:
raise IPToGeoException(IPToGeo.ERRORS[err])
(cc0, cc1, cc2, cc3) = struct.unpack_from('BBBB', packet, 7*4)
return (ipv4, '%c%c%c%c' % (cc0, cc1, cc2, cc3))
def ip_to_geo(self, ip):
splitted_ip = [int(a) for a in ip.split('.')]
packet = self._create_request(splitted_ip)
try:
self._socket.send(packet)
except IOError, e:
# Give another chance (we may have been disconnected due to timeout)
self._create_socket()
self._socket.send(packet)
packet = self._socket.recv(IPToGeo.PACKET_SIZE)
if not packet:
raise IPToGeoException('Error, empty packet')
(ip, country_code) = self._check_request(packet)
if country_code:
# convert to string
country_code = '%c%c' % (country_code[0], country_code[1])
return (ip, country_code)
def close(self):
self._socket.close()
plugins/post_analysis/referers.py
120120
121121
122122
123
123
124124
125125
126126
break
def hook(self):
stats = self.iwla.getCurrentVisists()
stats = self.iwla.getCurrentVisits()
month_stats = self.iwla.getMonthStats()
referers = month_stats.get('referers', {})
plugins/post_analysis/reverse_dns.py
6464
6565
6666
67
67
6868
6969
7070
return True
def hook(self):
hits = self.iwla.getCurrentVisists()
hits = self.iwla.getCurrentVisits()
for (k, hit) in hits.items():
if hit.get('dns_analysed', False): continue
if not hit['feed_parser'] and\
plugins/post_analysis/top_hits.py
5353
5454
5555
56
56
5757
5858
5959
self.API_VERSION = 1
def hook(self):
stats = self.iwla.getCurrentVisists()
stats = self.iwla.getCurrentVisits()
month_stats = self.iwla.getMonthStats()
top_hits = month_stats.get('top_hits', {})
plugins/post_analysis/top_pages.py
5959
6060
6161
62
62
6363
6464
6565
return True
def hook(self):
stats = self.iwla.getCurrentVisists()
stats = self.iwla.getCurrentVisits()
month_stats = self.iwla.getMonthStats()
top_pages = month_stats.get('top_pages', {})
plugins/pre_analysis/page_to_hit.py
6969
7070
7171
72
72
7373
7474
7575
return True
def hook(self):
hits = self.iwla.getCurrentVisists()
hits = self.iwla.getCurrentVisits()
for (k, super_hit) in hits.items():
if super_hit['robot']: continue
plugins/pre_analysis/robots.py
7676
7777
7878
79
79
8080
8181
8282
# Basic rule to detect robots
def hook(self):
hits = self.iwla.getCurrentVisists()
hits = self.iwla.getCurrentVisits()
for (k, super_hit) in hits.items():
if super_hit['robot']:
self.logger.debug('%s is a robot' % (k))

Archive Download the corresponding diff file

Branches

Tags