iwla

iwla Git Source Tree

Root/docs/modules.md

1 * iwla.py
2 * plugins/display/all_visits.py
3 * plugins/display/browsers.py
4 * plugins/display/feeds.py
5 * plugins/display/hours_stats.py
6 * plugins/display/ip_to_geo.py
7 * plugins/display/istats_diff.py
8 * plugins/display/operating_systems.py
9 * plugins/display/referers_diff.py
10 * plugins/display/referers.py
11 * plugins/display/robot_bandwidth.py
12 * plugins/display/top_downloads_diff.py
13 * plugins/display/top_downloads.py
14 * plugins/display/top_hits.py
15 * plugins/display/top_pages_diff.py
16 * plugins/display/top_pages.py
17 * plugins/display/top_visitors.py
18 * plugins/display/track_users.py
19 * plugins/post_analysis/browsers.py
20 * plugins/post_analysis/feeds.py
21 * plugins/post_analysis/hours_stats.py
22 * plugins/post_analysis/ip_to_geo.py
23 * plugins/post_analysis/iptogeo.py
24 * plugins/post_analysis/operating_systems.py
25 * plugins/post_analysis/referers.py
26 * plugins/post_analysis/reverse_dns.py
27 * plugins/post_analysis/top_downloads.py
28 * plugins/post_analysis/top_hits.py
29 * plugins/post_analysis/top_pages.py
30 * plugins/pre_analysis/page_to_hit.py
31 * plugins/pre_analysis/robots.py
32
33
34iwla
35----
36
37 Main class IWLA
38 Parse Log, compute them, call plugins and produce output
39 For now, only HTTP log are valid
40
41 Plugin requirements :
42 None
43
44 Conf values needed :
45 analyzed_filename
46 domain_name
47 locales_path
48 compress_output_files
49 excluded_ip
50
51 Output files :
52 DB_ROOT/meta.db
53 DB_ROOT/year/month/iwla.db
54 OUTPUT_ROOT/index.html
55 OUTPUT_ROOT/year/_stats.html
56 OUTPUT_ROOT/year/month/index.html
57
58 Statistics creation :
59
60 meta :
61 last_time
62 start_analysis_time
63 stats =>
64 year =>
65 month =>
66 viewed_bandwidth
67 not_viewed_bandwidth
68 viewed_pages
69 viewed_hits
70 nb_visits
71 nb_visitors
72
73 month_stats :
74 viewed_bandwidth
75 not_viewed_bandwidth
76 viewed_pages
77 viewed_hits
78 nb_visits
79
80 days_stats :
81 day =>
82 viewed_bandwidth
83 not_viewed_bandwidth
84 viewed_pages
85 viewed_hits
86 nb_visits
87 nb_visitors
88
89 visits :
90 remote_addr =>
91 remote_addr
92 remote_ip
93 viewed_pages{0..31} # 0 contains total
94 viewed_hits{0..31} # 0 contains total
95 not_viewed_pages{0..31}
96 not_viewed_hits{0..31}
97 bandwidth{0..31}
98 last_access
99 requests =>
100 [fields_from_format_log]
101 extract_request =>
102 http_method
103 http_uri
104 http_version
105 extract_uri
106 extract_parameters*
107 extract_referer* =>
108 extract_uri
109 extract_parameters*
110 robot
111 hit_only
112 is_page
113
114 valid_visitors:
115 month_stats without robot and hit only visitors (if not conf.count_hit_only_visitors)
116
117 Statistics update :
118 None
119
120 Statistics deletion :
121 None
122
123
124plugins.display.all_visits
125--------------------------
126
127 Display hook
128
129 Create All visits page
130
131 Plugin requirements :
132 None
133
134 Conf values needed :
135 display_visitor_ip*
136
137 Output files :
138 OUTPUT_ROOT/year/month/all_visits.html
139 OUTPUT_ROOT/year/month/index.html
140
141 Statistics creation :
142 None
143
144 Statistics update :
145 None
146
147 Statistics deletion :
148 None
149
150
151plugins.display.browsers
152------------------------
153
154 Display hook
155
156 Create browsers page
157
158 Plugin requirements :
159 post_analysis/browsers
160
161 Conf values needed :
162 max_browsers_displayed*
163 create_browsers_page*
164
165 Output files :
166 OUTPUT_ROOT/year/month/browsers.html
167 OUTPUT_ROOT/year/month/index.html
168
169 Statistics creation :
170 None
171
172 Statistics update :
173 None
174
175 Statistics deletion :
176 None
177
178
179plugins.display.feeds
180---------------------
181
182 Display hook
183
184 Display feeds parsers
185
186 Plugin requirements :
187 post_analysis/feeds
188
189 Conf values needed :
190 create_all_feeds_page*
191
192 Output files :
193 OUTPUT_ROOT/year/month/index.html
194 OUTPUT_ROOT/year/month/all_feeds.html
195
196 Statistics creation :
197 None
198
199 Statistics update :
200 None
201
202 Statistics deletion :
203 None
204
205
206plugins.display.hours_stats
207---------------------------
208
209 Display hook
210
211 Display statistics by hour/week day
212
213 Plugin requirements :
214 post_analysis/hours_stats
215
216 Conf values needed :
217 None
218
219 Output files :
220 OUTPUT_ROOT/year/month/index.html
221
222 Statistics creation :
223 None
224
225 Statistics update :
226 None
227
228 Statistics deletion :
229 None
230
231
232plugins.display.ip_to_geo
233-------------------------
234
235 Display hook
236
237 Add geo statistics
238
239 Plugin requirements :
240 post_analysis/ip_to_geo
241
242 Conf values needed :
243 create_geo_page*
244
245 Output files :
246 OUTPUT_ROOT/year/month/index.html
247
248 Statistics creation :
249 None
250
251 Statistics update :
252 None
253
254 Statistics deletion :
255 None
256
257
258plugins.display.istats_diff
259---------------------------
260
261 Display hook interface
262
263 Enlight new and updated statistics
264
265 Plugin requirements :
266 None
267
268 Conf values needed :
269 None
270
271 Output files :
272 None
273
274 Statistics creation :
275 None
276
277 Statistics update :
278 None
279
280 Statistics deletion :
281 None
282
283
284plugins.display.operating_systems
285---------------------------------
286
287 Display hook
288
289 Add operating systems statistics
290
291 Plugin requirements :
292 post_analysis/operating_systems
293
294 Conf values needed :
295 create_families_page*
296
297 Output files :
298 OUTPUT_ROOT/year/month/index.html
299
300 Statistics creation :
301 None
302
303 Statistics update :
304 None
305
306 Statistics deletion :
307 None
308
309
310plugins.display.referers_diff
311-----------------------------
312
313 Display hook
314
315 Enlight new and updated key phrases in in all_key_phrases.html
316
317 Plugin requirements :
318 display/referers
319
320 Conf values needed :
321 None
322
323 Output files :
324 None
325
326 Statistics creation :
327 None
328
329 Statistics update :
330 None
331
332 Statistics deletion :
333 None
334
335
336plugins.display.referers
337------------------------
338
339 Display hook
340
341 Create Referers page
342
343 Plugin requirements :
344 post_analysis/referers
345
346 Conf values needed :
347 max_referers_displayed*
348 create_all_referers_page*
349 max_key_phrases_displayed*
350 create_all_key_phrases_page*
351
352 Output files :
353 OUTPUT_ROOT/year/month/referers.html
354 OUTPUT_ROOT/year/month/key_phrases.html
355 OUTPUT_ROOT/year/month/index.html
356
357 Statistics creation :
358 None
359
360 Statistics update :
361 None
362
363 Statistics deletion :
364 None
365
366
367plugins.display.robot_bandwidth
368-------------------------------
369
370 Display hook
371
372 Display top 10 robot bandwidth use
373
374 Plugin requirements :
375 None
376
377 Conf values needed :
378 display_visitor_ip*
379 create_all_robot_bandwidth_page*
380
381 Output files :
382 OUTPUT_ROOT/year/month/top_robots_bandwidth.html
383 OUTPUT_ROOT/year/month/index.html
384
385 Statistics creation :
386 None
387
388 Statistics update :
389 None
390
391 Statistics deletion :
392 None
393
394
395plugins.display.top_downloads_diff
396----------------------------------
397
398 Display hook
399
400 Enlight new and updated downloads in in top_downloads.html
401
402 Plugin requirements :
403 display/top_downloads
404
405 Conf values needed :
406 None
407
408 Output files :
409 None
410
411 Statistics creation :
412 None
413
414 Statistics update :
415 None
416
417 Statistics deletion :
418 None
419
420
421plugins.display.top_downloads
422-----------------------------
423
424 Display hook
425
426 Create TOP downloads page
427
428 Plugin requirements :
429 post_analysis/top_downloads
430
431 Conf values needed :
432 max_downloads_displayed*
433 create_all_downloads_page*
434
435 Output files :
436 OUTPUT_ROOT/year/month/top_downloads.html
437 OUTPUT_ROOT/year/month/index.html
438
439 Statistics creation :
440 None
441
442 Statistics update :
443 None
444
445 Statistics deletion :
446 None
447
448
449plugins.display.top_hits
450------------------------
451
452 Display hook
453
454 Create TOP hits page
455
456 Plugin requirements :
457 post_analysis/top_hits
458
459 Conf values needed :
460 max_hits_displayed*
461 create_all_hits_page*
462
463 Output files :
464 OUTPUT_ROOT/year/month/top_hits.html
465 OUTPUT_ROOT/year/month/index.html
466
467 Statistics creation :
468 None
469
470 Statistics update :
471 None
472
473 Statistics deletion :
474 None
475
476
477plugins.display.top_pages_diff
478------------------------------
479
480 Display hook
481
482 Enlight new and updated pages in in top_pages.html
483
484 Plugin requirements :
485 display/top_pages
486
487 Conf values needed :
488 None
489
490 Output files :
491 None
492
493 Statistics creation :
494 None
495
496 Statistics update :
497 None
498
499 Statistics deletion :
500 None
501
502
503plugins.display.top_pages
504-------------------------
505
506 Display hook
507
508 Create TOP pages page
509
510 Plugin requirements :
511 post_analysis/top_pages
512
513 Conf values needed :
514 max_pages_displayed*
515 create_all_pages_page*
516
517 Output files :
518 OUTPUT_ROOT/year/month/top_pages.html
519 OUTPUT_ROOT/year/month/index.html
520
521 Statistics creation :
522 None
523
524 Statistics update :
525 None
526
527 Statistics deletion :
528 None
529
530
531plugins.display.top_visitors
532----------------------------
533
534 Display hook
535
536 Create TOP visitors block
537
538 Plugin requirements :
539 None
540
541 Conf values needed :
542 display_visitor_ip*
543
544 Output files :
545 OUTPUT_ROOT/year/month/index.html
546
547 Statistics creation :
548 None
549
550 Statistics update :
551 None
552
553 Statistics deletion :
554 None
555
556
557plugins.display.track_users
558---------------------------
559
560 Display hook
561
562 Track users
563
564 Plugin requirements :
565 None
566
567 Conf values needed :
568 tracked_ip
569 create_tracked_page*
570
571 Output files :
572 OUTPUT_ROOT/year/month/index.html
573 OUTPUT_ROOT/year/month/tracked_users.html
574
575 Statistics creation :
576 None
577
578 Statistics update :
579 None
580
581 Statistics deletion :
582 None
583
584
585plugins.post_analysis.browsers
586------------------------------
587
588 Post analysis hook
589
590 Detect browser information from requests
591
592 Plugin requirements :
593 None
594
595 Conf values needed :
596 None
597
598 Output files :
599 None
600
601 Statistics creation :
602 visits :
603 remote_addr =>
604 browser
605
606 month_stats :
607 browsers =>
608 browser => count
609
610 Statistics update :
611 None
612
613 Statistics deletion :
614 None
615
616
617plugins.post_analysis.feeds
618---------------------------
619
620 Post analysis hook
621
622 Find feeds parsers (first hit in feeds conf value and no viewed pages if it's a robot)
623 If there is ony one hit per day to a feed, merge feeds parsers with the same user agent
624 as it must be the same person with a different IP address.
625
626 Plugin requirements :
627 None
628
629 Conf values needed :
630 feeds
631 merge_one_hit_only_feeds_parsers*
632
633 Output files :
634 None
635
636 Statistics creation :
637 remote_addr =>
638 feed_parser
639
640 Statistics update :
641 None
642
643 Statistics deletion :
644 None
645
646
647plugins.post_analysis.hours_stats
648---------------------------------
649
650 Post analysis hook
651
652 Count pages, hits and bandwidth by hour/week day
653
654 Plugin requirements :
655 None
656
657 Conf values needed :
658 None
659
660 Output files :
661 None
662
663 Statistics creation :
664 month_stats:
665 hours_stats =>
666 00 .. 23 =>
667 pages
668 hits
669 bandwidth
670
671 days_stats =>
672 0 .. 6 =>
673 pages
674 hits
675 bandwidth
676
677 Statistics update :
678 None
679
680 Statistics deletion :
681 None
682
683
684plugins.post_analysis.ip_to_geo
685-------------------------------
686
687 Post analysis hook
688
689 Get country code from IP address
690
691 Plugin requirements :
692 None
693
694 Conf values needed :
695 iptogeo_remote_addr*
696 iptogeo_remote_port*
697
698 Output files :
699 None
700
701 Statistics creation :
702 geo =>
703 country_code => count
704 None
705
706 Statistics update :
707 valid_visitors:
708 country_code
709
710 Statistics deletion :
711 None
712
713
714plugins.post_analysis.iptogeo
715-----------------------------
716
717
718
719plugins.post_analysis.operating_systems
720---------------------------------------
721
722 Post analysis hook
723
724 Detect operating systems from requests
725
726 Plugin requirements :
727 None
728
729 Conf values needed :
730 None
731
732 Output files :
733 None
734
735 Statistics creation :
736 visits :
737 remote_addr =>
738 operating_system
739
740 month_stats :
741 operating_systems =>
742 operating_system => count
743
744 os_families =>
745 family => count
746
747 Statistics update :
748 None
749
750 Statistics deletion :
751 None
752
753
754plugins.post_analysis.referers
755------------------------------
756
757 Post analysis hook
758
759 Extract referers and key phrases from requests
760
761 Plugin requirements :
762 None
763
764 Conf values needed :
765 domain_name
766
767 Output files :
768 None
769
770 Statistics creation :
771 None
772
773 Statistics update :
774 month_stats :
775 referers =>
776 pages => count
777 hits => count
778 robots_referers =>
779 pages => count
780 hits => count
781 search_engine_referers =>
782 pages => count
783 hits => count
784 key_phrases =>
785 phrase => count
786
787 Statistics deletion :
788 None
789
790
791plugins.post_analysis.reverse_dns
792---------------------------------
793
794 Post analysis hook
795
796 Replace IP by reverse DNS names
797
798 Plugin requirements :
799 None
800
801 Conf values needed :
802 reverse_dns_timeout*
803
804 Output files :
805 None
806
807 Statistics creation :
808 None
809
810 Statistics update :
811 valid_visitors:
812 remote_addr
813 dns_name_replaced
814 dns_analyzed
815
816 Statistics deletion :
817 None
818
819
820plugins.post_analysis.top_downloads
821-----------------------------------
822
823 Post analysis hook
824
825 Count TOP downloads
826
827 Plugin requirements :
828 None
829
830 Conf values needed :
831 None
832
833 Output files :
834 None
835
836 Statistics creation :
837 None
838
839 Statistics update :
840 month_stats:
841 top_downloads =>
842 uri => count
843
844 Statistics deletion :
845 None
846
847
848plugins.post_analysis.top_hits
849------------------------------
850
851 Post analysis hook
852
853 Count TOP hits
854
855 Plugin requirements :
856 None
857
858 Conf values needed :
859 None
860
861 Output files :
862 None
863
864 Statistics creation :
865 None
866
867 Statistics update :
868 month_stats:
869 top_hits =>
870 uri => count
871
872 Statistics deletion :
873 None
874
875
876plugins.post_analysis.top_pages
877-------------------------------
878
879 Post analysis hook
880
881 Count TOP pages
882
883 Plugin requirements :
884 None
885
886 Conf values needed :
887 None
888
889 Output files :
890 None
891
892 Statistics creation :
893 None
894
895 Statistics update :
896 month_stats:
897 top_pages =>
898 uri => count
899
900 Statistics deletion :
901 None
902
903
904plugins.pre_analysis.page_to_hit
905--------------------------------
906
907 Pre analysis hook
908 Change page into hit and hit into page into statistics
909
910 Plugin requirements :
911 None
912
913 Conf values needed :
914 page_to_hit_conf*
915 hit_to_page_conf*
916
917 Output files :
918 None
919
920 Statistics creation :
921 None
922
923 Statistics update :
924 visits :
925 remote_addr =>
926 is_page
927
928 Statistics deletion :
929 None
930
931
932plugins.pre_analysis.robots
933---------------------------
934
935 Pre analysis hook
936
937 Filter robots
938
939 Plugin requirements :
940 None
941
942 Conf values needed :
943 page_to_hit_conf*
944 hit_to_page_conf*
945
946 Output files :
947 None
948
949 Statistics creation :
950 None
951
952 Statistics update :
953 visits :
954 remote_addr =>
955 robot
956
957 Statistics deletion :
958 None
959
960

Archive Download this file

Branches

Tags