iwla

iwla Commit Details

Date:2014-12-19 11:34:25 (6 years 7 months ago)
Author:Grégory Soutadé
Branch:dev, master
Commit:a35d462cb773a135cd09e6e9e12bcac1a041c7bc
Parents: 2f05a70ee505456e11c6fa3fa84857a019ebc51e
Message:Replace # for module description by """ (help auto extraction)

Changes:
Riwla_convert.pl → tools/iwla_convert.pl
Mdefault_conf.py (2 diffs)
Miwla.py (1 diff)
Mplugins/display/all_visits.py (1 diff)
Mplugins/display/referers.py (1 diff)
Mplugins/display/top_downloads.py (1 diff)
Mplugins/display/top_hits.py (1 diff)
Mplugins/display/top_pages.py (1 diff)
Mplugins/display/top_visitors.py (1 diff)
Mplugins/post_analysis/referers.py (1 diff)
Mplugins/post_analysis/reverse_dns.py (1 diff)
Mplugins/post_analysis/top_downloads.py (1 diff)
Mplugins/post_analysis/top_hits.py (1 diff)
Mplugins/post_analysis/top_pages.py (1 diff)
Mplugins/pre_analysis/page_to_hit.py (1 diff)
Mplugins/pre_analysis/robots.py (1 diff)

File differences

default_conf.py
22
33
44
5
5
66
7
7
88
99
1010
......
4949
5050
5151
52
52
5353
5454
5555
# Default configuration
# Default directory where to store database information
# Default database directory
DB_ROOT = './output_db'
# Default directory where to create html files
# Default HTML output directory
DISPLAY_ROOT = './output'
# Hooks directories (don't edit)
# CSS path (you can add yours)
css_path = ['%s/%s/%s' % (os.path.basename(resources_path[0]), 'css', 'iwla.css')]
# Extensions to compress in gzip during display build
# Extensions to compress in gzip during display build
compress_output_files = []
# Path to locales files
iwla.py
4141
4242
4343
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
126127
127128
128129
from iplugin import *
from display import *
#
# Main class IWLA
# Parse Log, compute them, call plugins and produce output
# For now, only HTTP log are valid
#
# Plugin requirements :
# None
#
# Conf values needed :
# analyzed_filename
# domain_name
# locales_path
# compress_output_files*
#
# Output files :
# DB_ROOT/meta.db
# DB_ROOT/year/month/iwla.db
# OUTPUT_ROOT/index.html
# OUTPUT_ROOT/year/month/index.html
#
# Statistics creation :
#
# meta :
# last_time
# start_analysis_time
# stats =>
# year =>
# month =>
# viewed_bandwidth
# not_viewed_bandwidth
# viewed_pages
# viewed_hits
# nb_visits
# nb_visitors
#
# month_stats :
# viewed_bandwidth
# not_viewed_bandwidth
# viewed_pages
# viewed_hits
# nb_visits
#
# days_stats :
# day =>
# viewed_bandwidth
# not_viewed_bandwidth
# viewed_pages
# viewed_hits
# nb_visits
# nb_visitors
#
# visits :
# remote_addr =>
# remote_addr
# remote_ip
# viewed_pages
# viewed_hits
# not_viewed_pages
# not_viewed_hits
# bandwidth
# last_access
# requests =>
# [fields_from_format_log]
# extract_request =>
# extract_uri
# extract_parameters*
# extract_referer* =>
# extract_uri
# extract_parameters*
# robot
# hit_only
# is_page
#
# valid_visitors:
# month_stats without robot and hit only visitors (if not conf.count_hit_only_visitors)
#
# Statistics update :
# None
#
# Statistics deletion :
# None
#
"""
Main class IWLA
Parse Log, compute them, call plugins and produce output
For now, only HTTP log are valid
Plugin requirements :
None
Conf values needed :
analyzed_filename
domain_name
locales_path
compress_output_files*
Output files :
DB_ROOT/meta.db
DB_ROOT/year/month/iwla.db
OUTPUT_ROOT/index.html
OUTPUT_ROOT/year/month/index.html
Statistics creation :
meta :
last_time
start_analysis_time
stats =>
year =>
month =>
viewed_bandwidth
not_viewed_bandwidth
viewed_pages
viewed_hits
nb_visits
nb_visitors
month_stats :
viewed_bandwidth
not_viewed_bandwidth
viewed_pages
viewed_hits
nb_visits
days_stats :
day =>
viewed_bandwidth
not_viewed_bandwidth
viewed_pages
viewed_hits
nb_visits
nb_visitors
visits :
remote_addr =>
remote_addr
remote_ip
viewed_pages
viewed_hits
not_viewed_pages
not_viewed_hits
bandwidth
last_access
requests =>
[fields_from_format_log]
extract_request =>
extract_uri
extract_parameters*
extract_referer* =>
extract_uri
extract_parameters*
robot
hit_only
is_page
valid_visitors:
month_stats without robot and hit only visitors (if not conf.count_hit_only_visitors)
Statistics update :
None
Statistics deletion :
None
"""
class IWLA(object):
plugins/display/all_visits.py
2424
2525
2626
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
5151
5252
5353
from iplugin import IPlugin
from display import *
#
# Display hook
#
# Create All visits page
#
# Plugin requirements :
# None
#
# Conf values needed :
# display_visitor_ip*
#
# Output files :
# OUTPUT_ROOT/year/month/all_visits.html
# OUTPUT_ROOT/year/month/index.html
#
# Statistics creation :
# None
#
# Statistics update :
# None
#
# Statistics deletion :
# None
#
"""
Display hook
Create All visits page
Plugin requirements :
None
Conf values needed :
display_visitor_ip*
Output files :
OUTPUT_ROOT/year/month/all_visits.html
OUTPUT_ROOT/year/month/index.html
Statistics creation :
None
Statistics update :
None
Statistics deletion :
None
"""
class IWLADisplayAllVisits(IPlugin):
def __init__(self, iwla):
plugins/display/referers.py
2222
2323
2424
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
5353
5454
5555
from iplugin import IPlugin
from display import *
#
# Display hook
#
# Create Referers page
#
# Plugin requirements :
# post_analysis/referers
#
# Conf values needed :
# max_referers_displayed*
# create_all_referers_page*
# max_key_phrases_displayed*
# create_all_key_phrases_page*
#
# Output files :
# OUTPUT_ROOT/year/month/referers.html
# OUTPUT_ROOT/year/month/key_phrases.html
# OUTPUT_ROOT/year/month/index.html
#
# Statistics creation :
# None
#
# Statistics update :
# None
#
# Statistics deletion :
# None
#
"""
Display hook
Create Referers page
Plugin requirements :
post_analysis/referers
Conf values needed :
max_referers_displayed*
create_all_referers_page*
max_key_phrases_displayed*
create_all_key_phrases_page*
Output files :
OUTPUT_ROOT/year/month/referers.html
OUTPUT_ROOT/year/month/key_phrases.html
OUTPUT_ROOT/year/month/index.html
Statistics creation :
None
Statistics update :
None
Statistics deletion :
None
"""
class IWLADisplayReferers(IPlugin):
def __init__(self, iwla):
plugins/display/top_downloads.py
2222
2323
2424
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
5050
5151
5252
from iplugin import IPlugin
from display import *
#
# Display hook
#
# Create TOP downloads page
#
# Plugin requirements :
# post_analysis/top_downloads
#
# Conf values needed :
# max_downloads_displayed*
# create_all_downloads_page*
#
# Output files :
# OUTPUT_ROOT/year/month/top_downloads.html
# OUTPUT_ROOT/year/month/index.html
#
# Statistics creation :
# None
#
# Statistics update :
# None
#
# Statistics deletion :
# None
#
"""
Display hook
Create TOP downloads page
Plugin requirements :
post_analysis/top_downloads
Conf values needed :
max_downloads_displayed*
create_all_downloads_page*
Output files :
OUTPUT_ROOT/year/month/top_downloads.html
OUTPUT_ROOT/year/month/index.html
Statistics creation :
None
Statistics update :
None
Statistics deletion :
None
"""
class IWLADisplayTopDownloads(IPlugin):
def __init__(self, iwla):
plugins/display/top_hits.py
2222
2323
2424
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
5050
5151
5252
from iplugin import IPlugin
from display import *
#
# Display hook
#
# Create TOP hits page
#
# Plugin requirements :
# post_analysis/top_hits
#
# Conf values needed :
# max_hits_displayed*
# create_all_hits_page*
#
# Output files :
# OUTPUT_ROOT/year/month/top_hits.html
# OUTPUT_ROOT/year/month/index.html
#
# Statistics creation :
# None
#
# Statistics update :
# None
#
# Statistics deletion :
# None
#
"""
Display hook
Create TOP hits page
Plugin requirements :
post_analysis/top_hits
Conf values needed :
max_hits_displayed*
create_all_hits_page*
Output files :
OUTPUT_ROOT/year/month/top_hits.html
OUTPUT_ROOT/year/month/index.html
Statistics creation :
None
Statistics update :
None
Statistics deletion :
None
"""
class IWLADisplayTopHits(IPlugin):
def __init__(self, iwla):
plugins/display/top_pages.py
2222
2323
2424
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
5050
5151
5252
from iplugin import IPlugin
from display import *
#
# Display hook
#
# Create TOP pages page
#
# Plugin requirements :
# post_analysis/top_pages
#
# Conf values needed :
# max_pages_displayed*
# create_all_pages_page*
#
# Output files :
# OUTPUT_ROOT/year/month/top_pages.html
# OUTPUT_ROOT/year/month/index.html
#
# Statistics creation :
# None
#
# Statistics update :
# None
#
# Statistics deletion :
# None
#
"""
Display hook
Create TOP pages page
Plugin requirements :
post_analysis/top_pages
Conf values needed :
max_pages_displayed*
create_all_pages_page*
Output files :
OUTPUT_ROOT/year/month/top_pages.html
OUTPUT_ROOT/year/month/index.html
Statistics creation :
None
Statistics update :
None
Statistics deletion :
None
"""
class IWLADisplayTopPages(IPlugin):
def __init__(self, iwla):
plugins/display/top_visitors.py
2424
2525
2626
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
5050
5151
5252
from iplugin import IPlugin
from display import *
#
# Display hook
#
# Create TOP visitors block
#
# Plugin requirements :
# None
#
# Conf values needed :
# display_visitor_ip*
#
# Output files :
# OUTPUT_ROOT/year/month/index.html
#
# Statistics creation :
# None
#
# Statistics update :
# None
#
# Statistics deletion :
# None
#
"""
Display hook
Create TOP visitors block
Plugin requirements :
None
Conf values needed :
display_visitor_ip*
Output files :
OUTPUT_ROOT/year/month/index.html
Statistics creation :
None
Statistics update :
None
Statistics deletion :
None
"""
class IWLADisplayTopVisitors(IPlugin):
def __init__(self, iwla):
plugins/post_analysis/referers.py
2626
2727
2828
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
6363
6464
6565
import awstats_data
#
# Post analysis hook
#
# Extract referers and key phrases from requests
#
# Plugin requirements :
# None
#
# Conf values needed :
# domain_name
#
# Output files :
# None
#
# Statistics creation :
# None
#
# Statistics update :
# month_stats :
# referers =>
# pages
# hits
# robots_referers =>
# pages
# hits
# search_engine_referers =>
# pages
# hits
# key_phrases =>
# phrase
#
# Statistics deletion :
# None
#
"""
Post analysis hook
Extract referers and key phrases from requests
Plugin requirements :
None
Conf values needed :
domain_name
Output files :
None
Statistics creation :
None
Statistics update :
month_stats :
referers =>
pages
hits
robots_referers =>
pages
hits
search_engine_referers =>
pages
hits
key_phrases =>
phrase
Statistics deletion :
None
"""
class IWLAPostAnalysisReferers(IPlugin):
def __init__(self, iwla):
plugins/post_analysis/reverse_dns.py
2323
2424
2525
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
5252
5353
5454
from iwla import IWLA
from iplugin import IPlugin
#
# Post analysis hook
#
# Replace IP by reverse DNS names
#
# Plugin requirements :
# None
#
# Conf values needed :
# reverse_dns_timeout*
#
# Output files :
# None
#
# Statistics creation :
# None
#
# Statistics update :
# valid_visitors:
# remote_addr
# dns_name_replaced
# dns_analyzed
#
# Statistics deletion :
# None
#
"""
Post analysis hook
Replace IP by reverse DNS names
Plugin requirements :
None
Conf values needed :
reverse_dns_timeout*
Output files :
None
Statistics creation :
None
Statistics update :
valid_visitors:
remote_addr
dns_name_replaced
dns_analyzed
Statistics deletion :
None
"""
class IWLAPostAnalysisReverseDNS(IPlugin):
DEFAULT_DNS_TIMEOUT = 0.5
plugins/post_analysis/top_downloads.py
2323
2424
2525
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
5151
5252
5353
from iwla import IWLA
from iplugin import IPlugin
#
# Post analysis hook
#
# Count TOP downloads
#
# Plugin requirements :
# None
#
# Conf values needed :
# None
#
# Output files :
# None
#
# Statistics creation :
# None
#
# Statistics update :
# month_stats:
# top_downloads =>
# uri
#
# Statistics deletion :
# None
#
"""
Post analysis hook
Count TOP downloads
Plugin requirements :
None
Conf values needed :
None
Output files :
None
Statistics creation :
None
Statistics update :
month_stats:
top_downloads =>
uri
Statistics deletion :
None
"""
class IWLAPostAnalysisTopDownloads(IPlugin):
def __init__(self, iwla):
plugins/post_analysis/top_hits.py
2121
2222
2323
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
4949
5050
5151
from iwla import IWLA
from iplugin import IPlugin
#
# Post analysis hook
#
# Count TOP hits
#
# Plugin requirements :
# None
#
# Conf values needed :
# None
#
# Output files :
# None
#
# Statistics creation :
# None
#
# Statistics update :
# month_stats:
# top_hits =>
# uri
#
# Statistics deletion :
# None
#
"""
Post analysis hook
Count TOP hits
Plugin requirements :
None
Conf values needed :
None
Output files :
None
Statistics creation :
None
Statistics update :
month_stats:
top_hits =>
uri
Statistics deletion :
None
"""
class IWLAPostAnalysisTopHits(IPlugin):
def __init__(self, iwla):
plugins/post_analysis/top_pages.py
2323
2424
2525
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
5151
5252
5353
from iwla import IWLA
from iplugin import IPlugin
#
# Post analysis hook
#
# Count TOP pages
#
# Plugin requirements :
# None
#
# Conf values needed :
# None
#
# Output files :
# None
#
# Statistics creation :
# None
#
# Statistics update :
# month_stats:
# top_pages =>
# uri
#
# Statistics deletion :
# None
#
"""
Post analysis hook
Count TOP pages
Plugin requirements :
None
Conf values needed :
None
Output files :
None
Statistics creation :
None
Statistics update :
month_stats:
top_pages =>
uri
Statistics deletion :
None
"""
class IWLAPostAnalysisTopPages(IPlugin):
def __init__(self, iwla):
plugins/pre_analysis/page_to_hit.py
2323
2424
2525
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
5151
5252
5353
from iwla import IWLA
from iplugin import IPlugin
#
# Pre analysis hook
# Change page into hit and hit into page into statistics
#
# Plugin requirements :
# None
#
# Conf values needed :
# page_to_hit_conf*
# hit_to_page_conf*
#
# Output files :
# None
#
# Statistics creation :
# None
#
# Statistics update :
# visits :
# remote_addr =>
# is_page
#
# Statistics deletion :
# None
#
"""
Pre analysis hook
Change page into hit and hit into page into statistics
Plugin requirements :
None
Conf values needed :
page_to_hit_conf*
hit_to_page_conf*
Output files :
None
Statistics creation :
None
Statistics update :
visits :
remote_addr =>
is_page
Statistics deletion :
None
"""
class IWLAPreAnalysisPageToHit(IPlugin):
plugins/pre_analysis/robots.py
2525
2626
2727
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
5454
5555
5656
import awstats_data
#
# Pre analysis hook
#
# Filter robots
#
# Plugin requirements :
# None
#
# Conf values needed :
# page_to_hit_conf*
# hit_to_page_conf*
#
# Output files :
# None
#
# Statistics creation :
# None
#
# Statistics update :
# visits :
# remote_addr =>
# robot
#
# Statistics deletion :
# None
#
"""
Pre analysis hook
Filter robots
Plugin requirements :
None
Conf values needed :
page_to_hit_conf*
hit_to_page_conf*
Output files :
None
Statistics creation :
None
Statistics update :
visits :
remote_addr =>
robot
Statistics deletion :
None
"""
class IWLAPreAnalysisRobots(IPlugin):
def __init__(self, iwla):

Archive Download the corresponding diff file

Branches

Tags