iwla

iwla Commit Details

Date:2015-02-17 19:11:04 (6 years 5 months ago)
Author:Grégory Soutadé
Branch:dev, master
Commit:e0f9260802db8835239efaa73b72f75118a79937
Parents: bcdb68b0d410dab5add4128b04150af27f03f45e
Message:Add Feeds plugin

Changes:
Cplugins/post_analysis/reverse_dns.py → plugins/post_analysis/feeds.py
Aplugins/display/feeds.py (full)
Miplugin.py (1 diff)
Mplugins/post_analysis/top_downloads.py (1 diff)

File differences

iplugin.py
103103
104104
105105
106
106
107107
108108
109109
requirement_validated = False
for r in requirements:
for (_,p) in cache_plugins.items():
for p in cache_plugins.values():
if p.__class__.__name__ == r:
requirement_validated = True
break
plugins/display/feeds.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
# -*- coding: utf-8 -*-
#
# Copyright Grégory Soutadé 2015
# This file is part of iwla
# iwla is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# iwla is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with iwla. If not, see <http://www.gnu.org/licenses/>.
#
from iwla import IWLA
from iplugin import IPlugin
from display import *
import awstats_data
"""
Display hook
Display feeds parsers
Plugin requirements :
None
Conf values needed :
create_all_feeds_page*
Output files :
OUTPUT_ROOT/year/month/index.html
OUTPUT_ROOT/year/month/all_feeds.html
Statistics creation :
None
Statistics update :
None
Statistics deletion :
None
"""
class IWLADisplayFeeds(IPlugin):
def __init__(self, iwla):
super(IWLADisplayFeeds, self).__init__(iwla)
self.API_VERSION = 1
self.requires = ['IWLAPostAnalysisFeeds']
def load(self):
self.create_all_feeds_page = self.iwla.getConfValue('create_all_feeds_page', True)
return True
def hook(self):
display = self.iwla.getDisplay()
hits = self.iwla.getCurrentVisists()
nb_feeds_parsers = 0
# All in a page
if self.create_all_feeds_page:
title = createCurTitle(self.iwla, u'All Feeds parsers')
filename = 'all_feeds.html'
path = self.iwla.getCurDisplayPath(filename)
display_visitor_ip = self.iwla.getConfValue('display_visitor_ip', False)
page = display.createPage(title, path, self.iwla.getConfValue('css_path', []))
table = display.createBlock(DisplayHTMLBlockTable, self.iwla._(u'All feeds parsers'), [self.iwla._(u'Host'), self.iwla._(u'Pages'), self.iwla._(u'Hits')])
for super_hit in hits.values():
if not super_hit['feed_parser']: continue
nb_feeds_parsers += 1
address = super_hit['remote_addr']
if display_visitor_ip and\
super_hit.get('dns_name_replaced', False):
address = '%s [%s]' % (address, super_hit['remote_ip'])
table.appendRow([address, super_hit['viewed_pages'], super_hit['viewed_hits']])
page.appendBlock(table)
display.addPage(page)
# Found in index
title = self.iwla._(u'Feeds parsers')
if self.create_all_feeds_page:
link = '<a href=\'%s\'>%s</a>' % (filename, self.iwla._(u'Details'))
title = '%s - %s' % (title, link)
index = self.iwla.getDisplayIndex()
table = display.createBlock(DisplayHTMLBlockTable, title, [self.iwla._(u'Found')])
table.appendRow([nb_feeds_parsers])
index.appendBlock(table)
plugins/post_analysis/feeds.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
# -*- coding: utf-8 -*-
#
# Copyright Grégory Soutadé 2015
# This file is part of iwla
# iwla is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# iwla is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with iwla. If not, see <http://www.gnu.org/licenses/>.
#
import re
from iwla import IWLA
from iplugin import IPlugin
"""
Post analysis hook
Find feeds parsers (first hit in feeds conf value and no viewed pages if it's a robot)
Plugin requirements :
None
Conf values needed :
feeds
Output files :
None
Statistics creation :
remote_addr =>
feed_parser
Statistics update :
None
Statistics deletion :
None
"""
class IWLAPostAnalysisFeeds(IPlugin):
def __init__(self, iwla):
super(IWLAPostAnalysisFeeds, self).__init__(iwla)
self.API_VERSION = 1
def load(self):
feeds = self.iwla.getConfValue('feeds', None)
if feeds is None: return False
self.feeds_re = []
for f in feeds:
self.feeds_re.append(re.compile(r'.*%s.*' % (f)))
return True
def hook(self):
hits = self.iwla.getCurrentVisists()
for hit in hits.values():
if not hit.get('feed_parser', None) is None: continue
isFeedParser = False
uri = hit['requests'][0]['extract_request']['extract_uri'].lower()
for regexp in self.feeds_re:
if regexp.match(uri):
# Robot that views pages -> bot
if hit['robot']:
if hit['viewed_pages']: continue
isFeedParser = True
break
hit['feed_parser'] = isFeedParser
plugins/post_analysis/top_downloads.py
1818
1919
2020
21
22
2321
2422
2523
# along with iwla. If not, see <http://www.gnu.org/licenses/>.
#
import re
from iwla import IWLA
from iplugin import IPlugin

Archive Download the corresponding diff file

Branches

Tags