Add IPToGeo plugin

conf.py

@@ -10,8 +10,8 @@ display_visitor_ip = True
 
 # Hooks used
 pre_analysis_hooks = ['page_to_hit', 'robots']
-post_analysis_hooks = ['referers', 'top_pages', 'top_downloads', 'operating_systems', 'browsers', 'feeds', 'hours_stats', 'reverse_dns']
-display_hooks = ['track_users', 'top_visitors', 'all_visits', 'referers', 'top_pages', 'top_downloads', 'referers_diff', 'operating_systems', 'browsers', 'feeds', 'hours_stats', 'top_downloads_diff']
+post_analysis_hooks = ['referers', 'top_pages', 'top_downloads', 'operating_systems', 'browsers', 'feeds', 'hours_stats', 'reverse_dns', 'ip_to_geo']
+display_hooks = ['track_users', 'top_visitors', 'all_visits', 'referers', 'top_pages', 'top_downloads', 'referers_diff', 'ip_to_geo', 'operating_systems', 'browsers', 'feeds', 'hours_stats', 'top_downloads_diff']
 
 # Reverse DNS timeout
 reverse_dns_timeout = 0.2

plugins/display/ip_to_geo.py

@@ -0,0 +1,120 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright Grégory Soutadé 2015
+
+# This file is part of iwla
+
+# iwla is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# iwla is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with iwla.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import re
+
+from iwla import IWLA
+from iplugin import IPlugin
+from display import *
+
+import awstats_data
+
+"""
+Display hook
+
+Add geo statistics
+
+Plugin requirements :
+    post_analysis/ip_to_geo
+
+Conf values needed :
+    create_geo_page*
+
+Output files :
+    OUTPUT_ROOT/year/month/index.html
+
+Statistics creation :
+    None
+
+Statistics update :
+    None
+
+Statistics deletion :
+    None
+"""
+
+class IWLADisplayTopGeo(IPlugin):
+    def __init__(self, iwla):
+        super(IWLADisplayTopGeo, self).__init__(iwla)
+        self.API_VERSION = 1
+        self.requires = ['IWLAPostAnalysisIPToGeo']
+
+    def load(self):
+        self.icon_path = self.iwla.getConfValue('icon_path', '/')
+        self.create_geo_page = self.iwla.getConfValue('create_geo_page_page', True)
+
+        display = self.iwla.getDisplay()
+        display.addColumnFilter(self.iwla._(u'Host'), self.FlagFilter, {'self':self})
+
+        return True
+
+    @staticmethod # Needed to have unbound methd
+    def FlagFilter(host, self):
+        cc = None
+        host_name = host.split(' ')[0] # hostname or ip
+        if host_name in self.valid_visitors.keys():
+            cc = self.valid_visitors[host_name]['country_code']
+        else:
+            for visitor in self.valid_visitors.values():
+                if visitor['remote_addr'] == host_name:
+                    cc = visitor['country_code']
+                    break
+        if not cc or cc == 'ip': return None
+        icon = '<img src="/%s/flags/%s.png"/>' % (self.icon_path, cc)
+        return '%s %s' % (icon ,host)
+    
+    def hook(self):
+        display = self.iwla.getDisplay()
+        geo = self.iwla.getMonthStats()['geo']
+        geo = sorted(geo.items(), key=lambda t: t[1], reverse=True)
+        self.valid_visitors = self.iwla.getValidVisitors()
+
+        # All in a page
+        if self.create_geo_page:
+            title = createCurTitle(self.iwla, u'All Coutries')
+            filename = 'geo.html'
+            path = self.iwla.getCurDisplayPath(filename)
+
+            page = display.createPage(title, path, self.iwla.getConfValue('css_path', []))
+            table = display.createBlock(DisplayHTMLBlockTable, self.iwla._(u'Countries'), ['', self.iwla._(u'Country'), self.iwla._(u'Visitors')])
+            table.setColsCSSClass(['', '', 'iwla_hit'])
+            for (cc, visitors) in geo:
+                icon = '<img src="/%s/flags/%s.png"/>' % (self.icon_path, cc)
+                table.appendRow([icon, cc, visitors])
+            table.computeRatio(2)
+            page.appendBlock(table)
+
+            display.addPage(page)
+
+        # Countries in index
+        title = self.iwla._(u'Countries')
+        if self.create_geo_page:
+            link = '<a href=\'%s\'>%s</a>' % (filename, self.iwla._(u'Details'))
+            title = '%s - %s' % (title, link)
+
+        index = self.iwla.getDisplayIndex()
+        
+        table = display.createBlock(DisplayHTMLBlockTable, title, ['', self.iwla._(u'Countries'), self.iwla._(u'Visitors')])        
+        table.setColsCSSClass(['', '', 'iwla_hit'])
+        for (cc, visitors) in geo[:10]:
+            icon = '<img src="/%s/flags/%s.png"/>' % (self.icon_path, cc)
+            table.appendRow([icon, cc, visitors])
+        table.computeRatio(2)
+        index.appendBlock(table)

plugins/post_analysis/ip_to_geo.py

@@ -0,0 +1,92 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright Grégory Soutadé 2016
+
+# This file is part of iwla
+
+# iwla is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# iwla is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with iwla.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from iwla import IWLA
+from iplugin import IPlugin
+
+from iptogeo import IPToGeo
+
+"""
+Post analysis hook
+
+Get country code from IP address
+
+Plugin requirements :
+    None
+
+Conf values needed :
+    iptogeo_remote_addr*
+    iptogeo_remote_port*
+
+Output files :
+    None
+
+Statistics creation :
+    geo =>
+        country_code => count
+    None
+
+Statistics update :
+valid_visitors:
+   country_code
+
+Statistics deletion :
+    None
+"""
+
+class IWLAPostAnalysisIPToGeo(IPlugin):
+
+    def __init__(self, iwla):
+        super(IWLAPostAnalysisIPToGeo, self).__init__(iwla)
+        self.API_VERSION = 1
+        
+    def load(self):
+        remote_addr = self.iwla.getConfValue('iptogeo_remote_addr',
+                                             None)
+        remote_port = self.iwla.getConfValue('iptogeo_remote_port',
+                                             None)
+
+        args = {}
+        if remote_addr: args['remote_addr'] = remote_addr
+        if remote_port: args['remote_port'] = remote_port
+        self.iptogeo = IPToGeo(**args)
+
+        return True
+        
+    def hook(self):
+        visitors = self.iwla.getValidVisitors()
+        month_stats = self.iwla.getMonthStats()
+
+        geo = month_stats.get('geo', {})
+
+        for (ip, visitor) in visitors.items():
+            if visitor.get('country_code', False): continue
+            try:
+                (_, cc) = self.iptogeo.ip_to_geo(ip)
+                cc = cc and cc or 'ip'
+                visitor['country_code'] = cc
+                if cc in geo.keys():
+                    geo[cc] += 1
+                else:
+                    geo[cc] = 1
+            except Exception, e:
+                print e
+
+        month_stats['geo'] = geo

plugins/post_analysis/iptogeo.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import socket
+import struct
+
+class IPToGeoException(Exception):
+    pass
+
+class IPToGeo(object):
+
+    MAGIC = 0x179E08EF
+    VERSION = 1
+    REQ = 1
+    RESP = 0
+    IPV4 = 4
+    IPV6 = 16
+
+    IP_NOT_FOUND = 6
+
+    PACKET_SIZE = 32
+    
+    ERRORS = {1 : 'Bad magic',
+              2 : 'Bad version',
+              3 : 'Bad request field' ,
+              4 : 'Bad IP version',
+              5 : 'Unsupported IP version',
+              6 : 'IP not found'}
+    
+    def __init__(self, remote_addr='127.0.0.1', remote_port=53333, timeout=None):
+        self._remote_addr = remote_addr
+        self._remote_port = remote_port
+        self._timeout = timeout
+        
+        self._create_socket()
+
+    def _create_socket(self):
+        self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        if not self._timeout is None:
+            self._socket.settimeout(self._timeout)
+        self._socket.connect((self._remote_addr, self._remote_port))
+
+    def _create_request(self, ip):
+        packet = ''
+        packet += struct.pack('<IBBBBI', IPToGeo.MAGIC, IPToGeo.VERSION, IPToGeo.REQ,
+                              0, #err
+                              IPToGeo.IPV4, # ip type
+                              0) # flags
+        packet += struct.pack('<BBBB', ip[0], ip[1], ip[2], ip[3]) # ipv4
+        packet += struct.pack('<III', 0, 0, 0) # ipv6
+        packet += struct.pack('<I', 0) # country code
+
+        return packet
+
+    def _check_request(self, packet):
+        (magic, version, req, err, ip_type, flags, ipv4, ipv6b, ipv6c, ipv6d) = struct.unpack_from('<IBBBBIIIII', packet, 0)
+
+        if magic != IPToGeo.MAGIC:
+            raise IPToGeoException('Invalid magic %08x' % (magic))
+        
+        if err == IPToGeo.IP_NOT_FOUND: return (ipv4, None) # IP not found
+        if err != 0:
+            raise IPToGeoException(IPToGeo.ERRORS[err])
+
+        (cc0, cc1, cc2, cc3) = struct.unpack_from('BBBB', packet, 7*4)
+
+        return (ipv4, '%c%c%c%c' % (cc0, cc1, cc2, cc3))
+        
+    def ip_to_geo(self, ip):
+        splitted_ip = [int(a) for a in ip.split('.')]
+
+        packet = self._create_request(splitted_ip)
+        try:
+            self._socket.send(packet)
+        except IOError, e:
+            # Give another chance (we may have been disconnected due to timeout)
+            self._create_socket()
+            self._socket.send(packet)
+        packet = self._socket.recv(IPToGeo.PACKET_SIZE)
+        if not packet:
+            raise IPToGeoException('Error, empty packet')
+        (ip, country_code) = self._check_request(packet)
+        if country_code:
+            # convert to string
+            country_code = '%c%c' % (country_code[0], country_code[1])
+        return (ip, country_code)
+
+    def close(self):
+        self._socket.close()