Add IPToGeo plugin
4
conf.py
|
@ -10,8 +10,8 @@ display_visitor_ip = True
|
||||||
|
|
||||||
# Hooks used
|
# Hooks used
|
||||||
pre_analysis_hooks = ['page_to_hit', 'robots']
|
pre_analysis_hooks = ['page_to_hit', 'robots']
|
||||||
post_analysis_hooks = ['referers', 'top_pages', 'top_downloads', 'operating_systems', 'browsers', 'feeds', 'hours_stats', 'reverse_dns']
|
post_analysis_hooks = ['referers', 'top_pages', 'top_downloads', 'operating_systems', 'browsers', 'feeds', 'hours_stats', 'reverse_dns', 'ip_to_geo']
|
||||||
display_hooks = ['track_users', 'top_visitors', 'all_visits', 'referers', 'top_pages', 'top_downloads', 'referers_diff', 'operating_systems', 'browsers', 'feeds', 'hours_stats', 'top_downloads_diff']
|
display_hooks = ['track_users', 'top_visitors', 'all_visits', 'referers', 'top_pages', 'top_downloads', 'referers_diff', 'ip_to_geo', 'operating_systems', 'browsers', 'feeds', 'hours_stats', 'top_downloads_diff']
|
||||||
|
|
||||||
# Reverse DNS timeout
|
# Reverse DNS timeout
|
||||||
reverse_dns_timeout = 0.2
|
reverse_dns_timeout = 0.2
|
||||||
|
|
|
@ -0,0 +1,120 @@
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
#
|
||||||
|
# Copyright Grégory Soutadé 2015
|
||||||
|
|
||||||
|
# This file is part of iwla
|
||||||
|
|
||||||
|
# iwla is free software: you can redistribute it and/or modify
|
||||||
|
# it under the terms of the GNU General Public License as published by
|
||||||
|
# the Free Software Foundation, either version 3 of the License, or
|
||||||
|
# (at your option) any later version.
|
||||||
|
#
|
||||||
|
# iwla is distributed in the hope that it will be useful,
|
||||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
# GNU General Public License for more details.
|
||||||
|
#
|
||||||
|
# You should have received a copy of the GNU General Public License
|
||||||
|
# along with iwla. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
#
|
||||||
|
|
||||||
|
import re
|
||||||
|
|
||||||
|
from iwla import IWLA
|
||||||
|
from iplugin import IPlugin
|
||||||
|
from display import *
|
||||||
|
|
||||||
|
import awstats_data
|
||||||
|
|
||||||
|
"""
|
||||||
|
Display hook
|
||||||
|
|
||||||
|
Add geo statistics
|
||||||
|
|
||||||
|
Plugin requirements :
|
||||||
|
post_analysis/ip_to_geo
|
||||||
|
|
||||||
|
Conf values needed :
|
||||||
|
create_geo_page*
|
||||||
|
|
||||||
|
Output files :
|
||||||
|
OUTPUT_ROOT/year/month/index.html
|
||||||
|
|
||||||
|
Statistics creation :
|
||||||
|
None
|
||||||
|
|
||||||
|
Statistics update :
|
||||||
|
None
|
||||||
|
|
||||||
|
Statistics deletion :
|
||||||
|
None
|
||||||
|
"""
|
||||||
|
|
||||||
|
class IWLADisplayTopGeo(IPlugin):
|
||||||
|
def __init__(self, iwla):
|
||||||
|
super(IWLADisplayTopGeo, self).__init__(iwla)
|
||||||
|
self.API_VERSION = 1
|
||||||
|
self.requires = ['IWLAPostAnalysisIPToGeo']
|
||||||
|
|
||||||
|
def load(self):
|
||||||
|
self.icon_path = self.iwla.getConfValue('icon_path', '/')
|
||||||
|
self.create_geo_page = self.iwla.getConfValue('create_geo_page_page', True)
|
||||||
|
|
||||||
|
display = self.iwla.getDisplay()
|
||||||
|
display.addColumnFilter(self.iwla._(u'Host'), self.FlagFilter, {'self':self})
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
@staticmethod # Needed to have unbound methd
|
||||||
|
def FlagFilter(host, self):
|
||||||
|
cc = None
|
||||||
|
host_name = host.split(' ')[0] # hostname or ip
|
||||||
|
if host_name in self.valid_visitors.keys():
|
||||||
|
cc = self.valid_visitors[host_name]['country_code']
|
||||||
|
else:
|
||||||
|
for visitor in self.valid_visitors.values():
|
||||||
|
if visitor['remote_addr'] == host_name:
|
||||||
|
cc = visitor['country_code']
|
||||||
|
break
|
||||||
|
if not cc or cc == 'ip': return None
|
||||||
|
icon = '<img src="/%s/flags/%s.png"/>' % (self.icon_path, cc)
|
||||||
|
return '%s %s' % (icon ,host)
|
||||||
|
|
||||||
|
def hook(self):
|
||||||
|
display = self.iwla.getDisplay()
|
||||||
|
geo = self.iwla.getMonthStats()['geo']
|
||||||
|
geo = sorted(geo.items(), key=lambda t: t[1], reverse=True)
|
||||||
|
self.valid_visitors = self.iwla.getValidVisitors()
|
||||||
|
|
||||||
|
# All in a page
|
||||||
|
if self.create_geo_page:
|
||||||
|
title = createCurTitle(self.iwla, u'All Coutries')
|
||||||
|
filename = 'geo.html'
|
||||||
|
path = self.iwla.getCurDisplayPath(filename)
|
||||||
|
|
||||||
|
page = display.createPage(title, path, self.iwla.getConfValue('css_path', []))
|
||||||
|
table = display.createBlock(DisplayHTMLBlockTable, self.iwla._(u'Countries'), ['', self.iwla._(u'Country'), self.iwla._(u'Visitors')])
|
||||||
|
table.setColsCSSClass(['', '', 'iwla_hit'])
|
||||||
|
for (cc, visitors) in geo:
|
||||||
|
icon = '<img src="/%s/flags/%s.png"/>' % (self.icon_path, cc)
|
||||||
|
table.appendRow([icon, cc, visitors])
|
||||||
|
table.computeRatio(2)
|
||||||
|
page.appendBlock(table)
|
||||||
|
|
||||||
|
display.addPage(page)
|
||||||
|
|
||||||
|
# Countries in index
|
||||||
|
title = self.iwla._(u'Countries')
|
||||||
|
if self.create_geo_page:
|
||||||
|
link = '<a href=\'%s\'>%s</a>' % (filename, self.iwla._(u'Details'))
|
||||||
|
title = '%s - %s' % (title, link)
|
||||||
|
|
||||||
|
index = self.iwla.getDisplayIndex()
|
||||||
|
|
||||||
|
table = display.createBlock(DisplayHTMLBlockTable, title, ['', self.iwla._(u'Countries'), self.iwla._(u'Visitors')])
|
||||||
|
table.setColsCSSClass(['', '', 'iwla_hit'])
|
||||||
|
for (cc, visitors) in geo[:10]:
|
||||||
|
icon = '<img src="/%s/flags/%s.png"/>' % (self.icon_path, cc)
|
||||||
|
table.appendRow([icon, cc, visitors])
|
||||||
|
table.computeRatio(2)
|
||||||
|
index.appendBlock(table)
|
|
@ -0,0 +1,92 @@
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
#
|
||||||
|
# Copyright Grégory Soutadé 2016
|
||||||
|
|
||||||
|
# This file is part of iwla
|
||||||
|
|
||||||
|
# iwla is free software: you can redistribute it and/or modify
|
||||||
|
# it under the terms of the GNU General Public License as published by
|
||||||
|
# the Free Software Foundation, either version 3 of the License, or
|
||||||
|
# (at your option) any later version.
|
||||||
|
#
|
||||||
|
# iwla is distributed in the hope that it will be useful,
|
||||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
# GNU General Public License for more details.
|
||||||
|
#
|
||||||
|
# You should have received a copy of the GNU General Public License
|
||||||
|
# along with iwla. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
#
|
||||||
|
|
||||||
|
from iwla import IWLA
|
||||||
|
from iplugin import IPlugin
|
||||||
|
|
||||||
|
from iptogeo import IPToGeo
|
||||||
|
|
||||||
|
"""
|
||||||
|
Post analysis hook
|
||||||
|
|
||||||
|
Get country code from IP address
|
||||||
|
|
||||||
|
Plugin requirements :
|
||||||
|
None
|
||||||
|
|
||||||
|
Conf values needed :
|
||||||
|
iptogeo_remote_addr*
|
||||||
|
iptogeo_remote_port*
|
||||||
|
|
||||||
|
Output files :
|
||||||
|
None
|
||||||
|
|
||||||
|
Statistics creation :
|
||||||
|
geo =>
|
||||||
|
country_code => count
|
||||||
|
None
|
||||||
|
|
||||||
|
Statistics update :
|
||||||
|
valid_visitors:
|
||||||
|
country_code
|
||||||
|
|
||||||
|
Statistics deletion :
|
||||||
|
None
|
||||||
|
"""
|
||||||
|
|
||||||
|
class IWLAPostAnalysisIPToGeo(IPlugin):
|
||||||
|
|
||||||
|
def __init__(self, iwla):
|
||||||
|
super(IWLAPostAnalysisIPToGeo, self).__init__(iwla)
|
||||||
|
self.API_VERSION = 1
|
||||||
|
|
||||||
|
def load(self):
|
||||||
|
remote_addr = self.iwla.getConfValue('iptogeo_remote_addr',
|
||||||
|
None)
|
||||||
|
remote_port = self.iwla.getConfValue('iptogeo_remote_port',
|
||||||
|
None)
|
||||||
|
|
||||||
|
args = {}
|
||||||
|
if remote_addr: args['remote_addr'] = remote_addr
|
||||||
|
if remote_port: args['remote_port'] = remote_port
|
||||||
|
self.iptogeo = IPToGeo(**args)
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
def hook(self):
|
||||||
|
visitors = self.iwla.getValidVisitors()
|
||||||
|
month_stats = self.iwla.getMonthStats()
|
||||||
|
|
||||||
|
geo = month_stats.get('geo', {})
|
||||||
|
|
||||||
|
for (ip, visitor) in visitors.items():
|
||||||
|
if visitor.get('country_code', False): continue
|
||||||
|
try:
|
||||||
|
(_, cc) = self.iptogeo.ip_to_geo(ip)
|
||||||
|
cc = cc and cc or 'ip'
|
||||||
|
visitor['country_code'] = cc
|
||||||
|
if cc in geo.keys():
|
||||||
|
geo[cc] += 1
|
||||||
|
else:
|
||||||
|
geo[cc] = 1
|
||||||
|
except Exception, e:
|
||||||
|
print e
|
||||||
|
|
||||||
|
month_stats['geo'] = geo
|
|
@ -0,0 +1,89 @@
|
||||||
|
#!/usr/bin/env python
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
import socket
|
||||||
|
import struct
|
||||||
|
|
||||||
|
class IPToGeoException(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
class IPToGeo(object):
|
||||||
|
|
||||||
|
MAGIC = 0x179E08EF
|
||||||
|
VERSION = 1
|
||||||
|
REQ = 1
|
||||||
|
RESP = 0
|
||||||
|
IPV4 = 4
|
||||||
|
IPV6 = 16
|
||||||
|
|
||||||
|
IP_NOT_FOUND = 6
|
||||||
|
|
||||||
|
PACKET_SIZE = 32
|
||||||
|
|
||||||
|
ERRORS = {1 : 'Bad magic',
|
||||||
|
2 : 'Bad version',
|
||||||
|
3 : 'Bad request field' ,
|
||||||
|
4 : 'Bad IP version',
|
||||||
|
5 : 'Unsupported IP version',
|
||||||
|
6 : 'IP not found'}
|
||||||
|
|
||||||
|
def __init__(self, remote_addr='127.0.0.1', remote_port=53333, timeout=None):
|
||||||
|
self._remote_addr = remote_addr
|
||||||
|
self._remote_port = remote_port
|
||||||
|
self._timeout = timeout
|
||||||
|
|
||||||
|
self._create_socket()
|
||||||
|
|
||||||
|
def _create_socket(self):
|
||||||
|
self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||||
|
if not self._timeout is None:
|
||||||
|
self._socket.settimeout(self._timeout)
|
||||||
|
self._socket.connect((self._remote_addr, self._remote_port))
|
||||||
|
|
||||||
|
def _create_request(self, ip):
|
||||||
|
packet = ''
|
||||||
|
packet += struct.pack('<IBBBBI', IPToGeo.MAGIC, IPToGeo.VERSION, IPToGeo.REQ,
|
||||||
|
0, #err
|
||||||
|
IPToGeo.IPV4, # ip type
|
||||||
|
0) # flags
|
||||||
|
packet += struct.pack('<BBBB', ip[0], ip[1], ip[2], ip[3]) # ipv4
|
||||||
|
packet += struct.pack('<III', 0, 0, 0) # ipv6
|
||||||
|
packet += struct.pack('<I', 0) # country code
|
||||||
|
|
||||||
|
return packet
|
||||||
|
|
||||||
|
def _check_request(self, packet):
|
||||||
|
(magic, version, req, err, ip_type, flags, ipv4, ipv6b, ipv6c, ipv6d) = struct.unpack_from('<IBBBBIIIII', packet, 0)
|
||||||
|
|
||||||
|
if magic != IPToGeo.MAGIC:
|
||||||
|
raise IPToGeoException('Invalid magic %08x' % (magic))
|
||||||
|
|
||||||
|
if err == IPToGeo.IP_NOT_FOUND: return (ipv4, None) # IP not found
|
||||||
|
if err != 0:
|
||||||
|
raise IPToGeoException(IPToGeo.ERRORS[err])
|
||||||
|
|
||||||
|
(cc0, cc1, cc2, cc3) = struct.unpack_from('BBBB', packet, 7*4)
|
||||||
|
|
||||||
|
return (ipv4, '%c%c%c%c' % (cc0, cc1, cc2, cc3))
|
||||||
|
|
||||||
|
def ip_to_geo(self, ip):
|
||||||
|
splitted_ip = [int(a) for a in ip.split('.')]
|
||||||
|
|
||||||
|
packet = self._create_request(splitted_ip)
|
||||||
|
try:
|
||||||
|
self._socket.send(packet)
|
||||||
|
except IOError, e:
|
||||||
|
# Give another chance (we may have been disconnected due to timeout)
|
||||||
|
self._create_socket()
|
||||||
|
self._socket.send(packet)
|
||||||
|
packet = self._socket.recv(IPToGeo.PACKET_SIZE)
|
||||||
|
if not packet:
|
||||||
|
raise IPToGeoException('Error, empty packet')
|
||||||
|
(ip, country_code) = self._check_request(packet)
|
||||||
|
if country_code:
|
||||||
|
# convert to string
|
||||||
|
country_code = '%c%c' % (country_code[0], country_code[1])
|
||||||
|
return (ip, country_code)
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
self._socket.close()
|
After Width: | Height: | Size: 381 B |
After Width: | Height: | Size: 389 B |
After Width: | Height: | Size: 255 B |
After Width: | Height: | Size: 224 B |
After Width: | Height: | Size: 381 B |
After Width: | Height: | Size: 240 B |
After Width: | Height: | Size: 255 B |
After Width: | Height: | Size: 271 B |
After Width: | Height: | Size: 247 B |
After Width: | Height: | Size: 219 B |
After Width: | Height: | Size: 241 B |
After Width: | Height: | Size: 251 B |
After Width: | Height: | Size: 255 B |
After Width: | Height: | Size: 231 B |
After Width: | Height: | Size: 226 B |
After Width: | Height: | Size: 248 B |
After Width: | Height: | Size: 225 B |
After Width: | Height: | Size: 260 B |
After Width: | Height: | Size: 222 B |
After Width: | Height: | Size: 484 B |
After Width: | Height: | Size: 260 B |
After Width: | Height: | Size: 234 B |
After Width: | Height: | Size: 252 B |
After Width: | Height: | Size: 248 B |
After Width: | Height: | Size: 246 B |
After Width: | Height: | Size: 258 B |
After Width: | Height: | Size: 226 B |
After Width: | Height: | Size: 229 B |
After Width: | Height: | Size: 244 B |
After Width: | Height: | Size: 381 B |
After Width: | Height: | Size: 182 B |
After Width: | Height: | Size: 298 B |
After Width: | Height: | Size: 286 B |
After Width: | Height: | Size: 224 B |
After Width: | Height: | Size: 279 B |
After Width: | Height: | Size: 242 B |
After Width: | Height: | Size: 294 B |
After Width: | Height: | Size: 243 B |
After Width: | Height: | Size: 232 B |
After Width: | Height: | Size: 198 B |
After Width: | Height: | Size: 253 B |
After Width: | Height: | Size: 258 B |
After Width: | Height: | Size: 260 B |
After Width: | Height: | Size: 268 B |
After Width: | Height: | Size: 281 B |
After Width: | Height: | Size: 251 B |
After Width: | Height: | Size: 256 B |
After Width: | Height: | Size: 200 B |
After Width: | Height: | Size: 285 B |
After Width: | Height: | Size: 237 B |
After Width: | Height: | Size: 226 B |
After Width: | Height: | Size: 253 B |
After Width: | Height: | Size: 229 B |
After Width: | Height: | Size: 381 B |
After Width: | Height: | Size: 381 B |
After Width: | Height: | Size: 241 B |
After Width: | Height: | Size: 239 B |
After Width: | Height: | Size: 249 B |
After Width: | Height: | Size: 247 B |
After Width: | Height: | Size: 280 B |
After Width: | Height: | Size: 256 B |
After Width: | Height: | Size: 239 B |
After Width: | Height: | Size: 192 B |
After Width: | Height: | Size: 244 B |
After Width: | Height: | Size: 223 B |
After Width: | Height: | Size: 267 B |
After Width: | Height: | Size: 226 B |
After Width: | Height: | Size: 253 B |
After Width: | Height: | Size: 245 B |
After Width: | Height: | Size: 256 B |
After Width: | Height: | Size: 206 B |
After Width: | Height: | Size: 236 B |
After Width: | Height: | Size: 262 B |
After Width: | Height: | Size: 302 B |
After Width: | Height: | Size: 250 B |
After Width: | Height: | Size: 231 B |
After Width: | Height: | Size: 229 B |
After Width: | Height: | Size: 309 B |
After Width: | Height: | Size: 259 B |
After Width: | Height: | Size: 269 B |
After Width: | Height: | Size: 239 B |
After Width: | Height: | Size: 299 B |
After Width: | Height: | Size: 310 B |
After Width: | Height: | Size: 263 B |
After Width: | Height: | Size: 234 B |
After Width: | Height: | Size: 191 B |
After Width: | Height: | Size: 191 B |
After Width: | Height: | Size: 199 B |
After Width: | Height: | Size: 280 B |
After Width: | Height: | Size: 298 B |
After Width: | Height: | Size: 223 B |
After Width: | Height: | Size: 264 B |
After Width: | Height: | Size: 605 B |
After Width: | Height: | Size: 237 B |
After Width: | Height: | Size: 289 B |
After Width: | Height: | Size: 240 B |