iwla

iwla Git Source Tree

Root/docs/modules.md

1 * iwla.py
2 * plugins/display/all_visits.py
3 * plugins/display/browsers.py
4 * plugins/display/feeds.py
5 * plugins/display/filter_users.py
6 * plugins/display/hours_stats.py
7 * plugins/display/ip_to_geo.py
8 * plugins/display/istats_diff.py
9 * plugins/display/operating_systems.py
10 * plugins/display/referers_diff.py
11 * plugins/display/referers.py
12 * plugins/display/robot_bandwidth.py
13 * plugins/display/top_downloads_diff.py
14 * plugins/display/top_downloads.py
15 * plugins/display/top_hits.py
16 * plugins/display/top_pages_diff.py
17 * plugins/display/top_pages.py
18 * plugins/display/top_visitors.py
19 * plugins/display/track_users.py
20 * plugins/post_analysis/browsers.py
21 * plugins/post_analysis/feeds.py
22 * plugins/post_analysis/filter_users.py
23 * plugins/post_analysis/hours_stats.py
24 * plugins/post_analysis/ip_to_geo.py
25 * plugins/post_analysis/iptogeo.py
26 * plugins/post_analysis/iptogeo.reset.py
27 * plugins/post_analysis/operating_systems.py
28 * plugins/post_analysis/referers.py
29 * plugins/post_analysis/reverse_dns.py
30 * plugins/post_analysis/top_downloads.py
31 * plugins/post_analysis/top_hits.py
32 * plugins/post_analysis/top_pages.py
33 * plugins/pre_analysis/page_to_hit.py
34 * plugins/pre_analysis/robots.py
35
36
37iwla
38----
39
40 Main class IWLA
41 Parse Log, compute them, call plugins and produce output
42 For now, only HTTP log are valid
43
44 Plugin requirements :
45 None
46
47 Conf values needed :
48 analyzed_filename
49 domain_name
50 locales_path
51 compress_output_files
52 excluded_ip
53
54 Output files :
55 DB_ROOT/meta.db
56 DB_ROOT/year/month/iwla.db
57 OUTPUT_ROOT/index.html
58 OUTPUT_ROOT/year/_stats.html
59 OUTPUT_ROOT/year/month/index.html
60
61 Statistics creation :
62
63 meta :
64 last_time
65 start_analysis_time
66 stats =>
67 year =>
68 month =>
69 viewed_bandwidth
70 not_viewed_bandwidth
71 viewed_pages
72 viewed_hits
73 nb_visits
74 nb_visitors
75
76 month_stats :
77 viewed_bandwidth
78 not_viewed_bandwidth
79 viewed_pages
80 viewed_hits
81 nb_visits
82
83 days_stats :
84 day =>
85 viewed_bandwidth
86 not_viewed_bandwidth
87 viewed_pages
88 viewed_hits
89 nb_visits
90 nb_visitors
91
92 visits :
93 remote_addr =>
94 remote_addr
95 remote_ip
96 viewed_pages{0..31} # 0 contains total
97 viewed_hits{0..31} # 0 contains total
98 not_viewed_pages{0..31}
99 not_viewed_hits{0..31}
100 bandwidth{0..31}
101 last_access
102 requests =>
103 [fields_from_format_log]
104 extract_request =>
105 http_method
106 http_uri
107 http_version
108 extract_uri
109 extract_parameters*
110 extract_referer* =>
111 extract_uri
112 extract_parameters*
113 robot
114 hit_only
115 is_page
116 keep_requests
117
118 valid_visitors:
119 month_stats without robot and hit only visitors (if not conf.count_hit_only_visitors)
120
121 Statistics update :
122 None
123
124 Statistics deletion :
125 None
126
127
128plugins.display.all_visits
129--------------------------
130
131 Display hook
132
133 Create All visits page
134
135 Plugin requirements :
136 None
137
138 Conf values needed :
139 display_visitor_ip*
140
141 Output files :
142 OUTPUT_ROOT/year/month/all_visits.html
143 OUTPUT_ROOT/year/month/index.html
144
145 Statistics creation :
146 None
147
148 Statistics update :
149 None
150
151 Statistics deletion :
152 None
153
154
155plugins.display.browsers
156------------------------
157
158 Display hook
159
160 Create browsers page
161
162 Plugin requirements :
163 post_analysis/browsers
164
165 Conf values needed :
166 max_browsers_displayed*
167 create_browsers_page*
168
169 Output files :
170 OUTPUT_ROOT/year/month/browsers.html
171 OUTPUT_ROOT/year/month/index.html
172
173 Statistics creation :
174 None
175
176 Statistics update :
177 None
178
179 Statistics deletion :
180 None
181
182
183plugins.display.feeds
184---------------------
185
186 Display hook
187
188 Display feeds parsers
189
190 Plugin requirements :
191 post_analysis/feeds
192
193 Conf values needed :
194 create_all_feeds_page*
195
196 Output files :
197 OUTPUT_ROOT/year/month/index.html
198 OUTPUT_ROOT/year/month/all_feeds.html
199
200 Statistics creation :
201 None
202
203 Statistics update :
204 None
205
206 Statistics deletion :
207 None
208
209
210plugins.display.filter_users
211----------------------------
212
213 Display hook
214
215 Filter users
216
217 Plugin requirements :
218 None
219
220 Conf values needed :
221 create_filtered_page*
222
223 Output files :
224 OUTPUT_ROOT/year/month/index.html
225 OUTPUT_ROOT/year/month/filtered_users.html
226
227 Statistics creation :
228 None
229
230 Statistics update :
231 None
232
233 Statistics deletion :
234 None
235
236
237plugins.display.hours_stats
238---------------------------
239
240 Display hook
241
242 Display statistics by hour/week day
243
244 Plugin requirements :
245 post_analysis/hours_stats
246
247 Conf values needed :
248 None
249
250 Output files :
251 OUTPUT_ROOT/year/month/index.html
252
253 Statistics creation :
254 None
255
256 Statistics update :
257 None
258
259 Statistics deletion :
260 None
261
262
263plugins.display.ip_to_geo
264-------------------------
265
266 Display hook
267
268 Add geo statistics
269
270 Plugin requirements :
271 post_analysis/ip_to_geo
272
273 Conf values needed :
274 create_geo_page*
275
276 Output files :
277 OUTPUT_ROOT/year/month/index.html
278
279 Statistics creation :
280 None
281
282 Statistics update :
283 None
284
285 Statistics deletion :
286 None
287
288
289plugins.display.istats_diff
290---------------------------
291
292 Display hook interface
293
294 Enlight new and updated statistics
295
296 Plugin requirements :
297 None
298
299 Conf values needed :
300 None
301
302 Output files :
303 None
304
305 Statistics creation :
306 None
307
308 Statistics update :
309 None
310
311 Statistics deletion :
312 None
313
314
315plugins.display.operating_systems
316---------------------------------
317
318 Display hook
319
320 Add operating systems statistics
321
322 Plugin requirements :
323 post_analysis/operating_systems
324
325 Conf values needed :
326 create_families_page*
327
328 Output files :
329 OUTPUT_ROOT/year/month/index.html
330
331 Statistics creation :
332 None
333
334 Statistics update :
335 None
336
337 Statistics deletion :
338 None
339
340
341plugins.display.referers_diff
342-----------------------------
343
344 Display hook
345
346 Enlight new and updated key phrases in in all_key_phrases.html
347
348 Plugin requirements :
349 display/referers
350
351 Conf values needed :
352 None
353
354 Output files :
355 None
356
357 Statistics creation :
358 None
359
360 Statistics update :
361 None
362
363 Statistics deletion :
364 None
365
366
367plugins.display.referers
368------------------------
369
370 Display hook
371
372 Create Referers page
373
374 Plugin requirements :
375 post_analysis/referers
376
377 Conf values needed :
378 max_referers_displayed*
379 create_all_referers_page*
380 max_key_phrases_displayed*
381 create_all_key_phrases_page*
382
383 Output files :
384 OUTPUT_ROOT/year/month/referers.html
385 OUTPUT_ROOT/year/month/key_phrases.html
386 OUTPUT_ROOT/year/month/index.html
387
388 Statistics creation :
389 None
390
391 Statistics update :
392 None
393
394 Statistics deletion :
395 None
396
397
398plugins.display.robot_bandwidth
399-------------------------------
400
401 Display hook
402
403 Display top 10 robot bandwidth use
404
405 Plugin requirements :
406 None
407
408 Conf values needed :
409 display_visitor_ip*
410 create_all_robot_bandwidth_page*
411
412 Output files :
413 OUTPUT_ROOT/year/month/top_robots_bandwidth.html
414 OUTPUT_ROOT/year/month/index.html
415
416 Statistics creation :
417 None
418
419 Statistics update :
420 None
421
422 Statistics deletion :
423 None
424
425
426plugins.display.top_downloads_diff
427----------------------------------
428
429 Display hook
430
431 Enlight new and updated downloads in in top_downloads.html
432
433 Plugin requirements :
434 display/top_downloads
435
436 Conf values needed :
437 None
438
439 Output files :
440 None
441
442 Statistics creation :
443 None
444
445 Statistics update :
446 None
447
448 Statistics deletion :
449 None
450
451
452plugins.display.top_downloads
453-----------------------------
454
455 Display hook
456
457 Create TOP downloads page
458
459 Plugin requirements :
460 post_analysis/top_downloads
461
462 Conf values needed :
463 max_downloads_displayed*
464 create_all_downloads_page*
465
466 Output files :
467 OUTPUT_ROOT/year/month/top_downloads.html
468 OUTPUT_ROOT/year/month/index.html
469
470 Statistics creation :
471 None
472
473 Statistics update :
474 None
475
476 Statistics deletion :
477 None
478
479
480plugins.display.top_hits
481------------------------
482
483 Display hook
484
485 Create TOP hits page
486
487 Plugin requirements :
488 post_analysis/top_hits
489
490 Conf values needed :
491 max_hits_displayed*
492 create_all_hits_page*
493
494 Output files :
495 OUTPUT_ROOT/year/month/top_hits.html
496 OUTPUT_ROOT/year/month/index.html
497
498 Statistics creation :
499 None
500
501 Statistics update :
502 None
503
504 Statistics deletion :
505 None
506
507
508plugins.display.top_pages_diff
509------------------------------
510
511 Display hook
512
513 Enlight new and updated pages in in top_pages.html
514
515 Plugin requirements :
516 display/top_pages
517
518 Conf values needed :
519 None
520
521 Output files :
522 None
523
524 Statistics creation :
525 None
526
527 Statistics update :
528 None
529
530 Statistics deletion :
531 None
532
533
534plugins.display.top_pages
535-------------------------
536
537 Display hook
538
539 Create TOP pages page
540
541 Plugin requirements :
542 post_analysis/top_pages
543
544 Conf values needed :
545 max_pages_displayed*
546 create_all_pages_page*
547
548 Output files :
549 OUTPUT_ROOT/year/month/top_pages.html
550 OUTPUT_ROOT/year/month/index.html
551
552 Statistics creation :
553 None
554
555 Statistics update :
556 None
557
558 Statistics deletion :
559 None
560
561
562plugins.display.top_visitors
563----------------------------
564
565 Display hook
566
567 Create TOP visitors block
568
569 Plugin requirements :
570 None
571
572 Conf values needed :
573 display_visitor_ip*
574
575 Output files :
576 OUTPUT_ROOT/year/month/index.html
577
578 Statistics creation :
579 None
580
581 Statistics update :
582 None
583
584 Statistics deletion :
585 None
586
587
588plugins.display.track_users
589---------------------------
590
591 Display hook
592
593 Track users
594
595 Plugin requirements :
596 None
597
598 Conf values needed :
599 tracked_ip
600 create_tracked_page*
601
602 Output files :
603 OUTPUT_ROOT/year/month/index.html
604 OUTPUT_ROOT/year/month/tracked_users.html
605
606 Statistics creation :
607 None
608
609 Statistics update :
610 None
611
612 Statistics deletion :
613 None
614
615
616plugins.post_analysis.browsers
617------------------------------
618
619 Post analysis hook
620
621 Detect browser information from requests
622
623 Plugin requirements :
624 None
625
626 Conf values needed :
627 None
628
629 Output files :
630 None
631
632 Statistics creation :
633 visits :
634 remote_addr =>
635 browser
636
637 month_stats :
638 browsers =>
639 browser => count
640
641 Statistics update :
642 None
643
644 Statistics deletion :
645 None
646
647
648plugins.post_analysis.feeds
649---------------------------
650
651 Post analysis hook
652
653 Find feeds parsers (first hit in feeds conf value and no viewed pages if it's a robot)
654 If there is ony one hit per day to a feed, merge feeds parsers with the same user agent
655 as it must be the same person with a different IP address.
656
657 Plugin requirements :
658 None
659
660 Conf values needed :
661 feeds
662 feeds_referers*
663 merge_one_hit_only_feeds_parsers*
664
665 Output files :
666 None
667
668 Statistics creation :
669 remote_addr =>
670 feed_parser
671
672 Statistics update :
673 None
674
675 Statistics deletion :
676 None
677
678
679plugins.post_analysis.filter_users
680----------------------------------
681
682 Post analysis hook
683
684 Filter users with given user conditions
685
686 Plugin requirements :
687 None
688
689 Conf values needed :
690 filtered_users : list of filters
691 filtered_ip : list of ip (string)
692 create_filtered_page*
693
694 Filter is a list of filter description combined by AND operator
695 Filter description is a list of 3 elements :
696
697 * Field to match in visits
698 * Operator '=', '==', '!=', '>', '>=', '<', '<=' for int value
699 * Operator '=', '==', '!=', 'in', 'match' for str value
700 * Target value
701
702 For easiest config, you can indicate both 'remote_addr' or 'ip' in field element
703
704 Output files :
705 None
706
707 Statistics creation :
708 visits :
709 remote_addr =>
710 filtered
711
712 Statistics update :
713 visits :
714 remote_addr =>
715 keep_requests
716
717 Statistics deletion :
718 None
719
720
721plugins.post_analysis.hours_stats
722---------------------------------
723
724 Post analysis hook
725
726 Count pages, hits and bandwidth by hour/week day
727
728 Plugin requirements :
729 None
730
731 Conf values needed :
732 None
733
734 Output files :
735 None
736
737 Statistics creation :
738 month_stats:
739 hours_stats =>
740 00 .. 23 =>
741 pages
742 hits
743 bandwidth
744
745 days_stats =>
746 0 .. 6 =>
747 pages
748 hits
749 bandwidth
750
751 Statistics update :
752 None
753
754 Statistics deletion :
755 None
756
757
758plugins.post_analysis.ip_to_geo
759-------------------------------
760
761 Post analysis hook
762
763 Get country code from IP address
764
765 Plugin requirements :
766 None
767
768 Conf values needed :
769 iptogeo_remote_addr*
770 iptogeo_remote_port*
771
772 Output files :
773 None
774
775 Statistics creation :
776 geo =>
777 country_code => count
778 None
779
780 Statistics update :
781 valid_visitors:
782 country_code
783
784 Statistics deletion :
785 None
786
787
788plugins.post_analysis.iptogeo
789-----------------------------
790
791
792
793plugins.post_analysis.iptogeo.reset
794-----------------------------------
795
796
797
798plugins.post_analysis.operating_systems
799---------------------------------------
800
801 Post analysis hook
802
803 Detect operating systems from requests
804
805 Plugin requirements :
806 None
807
808 Conf values needed :
809 None
810
811 Output files :
812 None
813
814 Statistics creation :
815 visits :
816 remote_addr =>
817 operating_system
818
819 month_stats :
820 operating_systems =>
821 operating_system => count
822
823 os_families =>
824 family => count
825
826 Statistics update :
827 None
828
829 Statistics deletion :
830 None
831
832
833plugins.post_analysis.referers
834------------------------------
835
836 Post analysis hook
837
838 Extract referers and key phrases from requests
839
840 Plugin requirements :
841 None
842
843 Conf values needed :
844 domain_name
845
846 Output files :
847 None
848
849 Statistics creation :
850 None
851
852 Statistics update :
853 month_stats :
854 referers =>
855 pages => count
856 hits => count
857 robots_referers =>
858 pages => count
859 hits => count
860 search_engine_referers =>
861 pages => count
862 hits => count
863 key_phrases =>
864 phrase => count
865
866 Statistics deletion :
867 None
868
869
870plugins.post_analysis.reverse_dns
871---------------------------------
872
873 Post analysis hook
874
875 Replace IP by reverse DNS names
876
877 Plugin requirements :
878 None
879
880 Conf values needed :
881 reverse_dns_timeout*
882
883 Output files :
884 None
885
886 Statistics creation :
887 None
888
889 Statistics update :
890 valid_visitors:
891 remote_addr
892 dns_name_replaced
893 dns_analyzed
894
895 Statistics deletion :
896 None
897
898
899plugins.post_analysis.top_downloads
900-----------------------------------
901
902 Post analysis hook
903
904 Count TOP downloads
905
906 Plugin requirements :
907 None
908
909 Conf values needed :
910 None
911
912 Output files :
913 None
914
915 Statistics creation :
916 None
917
918 Statistics update :
919 month_stats:
920 top_downloads =>
921 uri => count
922
923 Statistics deletion :
924 None
925
926
927plugins.post_analysis.top_hits
928------------------------------
929
930 Post analysis hook
931
932 Count TOP hits
933
934 Plugin requirements :
935 None
936
937 Conf values needed :
938 None
939
940 Output files :
941 None
942
943 Statistics creation :
944 None
945
946 Statistics update :
947 month_stats:
948 top_hits =>
949 uri => count
950
951 Statistics deletion :
952 None
953
954
955plugins.post_analysis.top_pages
956-------------------------------
957
958 Post analysis hook
959
960 Count TOP pages
961
962 Plugin requirements :
963 None
964
965 Conf values needed :
966 None
967
968 Output files :
969 None
970
971 Statistics creation :
972 None
973
974 Statistics update :
975 month_stats:
976 top_pages =>
977 uri => count
978
979 Statistics deletion :
980 None
981
982
983plugins.pre_analysis.page_to_hit
984--------------------------------
985
986 Pre analysis hook
987 Change page into hit and hit into page into statistics
988
989 Plugin requirements :
990 None
991
992 Conf values needed :
993 page_to_hit_conf*
994 hit_to_page_conf*
995
996 Output files :
997 None
998
999 Statistics creation :
1000 None
1001
1002 Statistics update :
1003 visits :
1004 remote_addr =>
1005 is_page
1006
1007 Statistics deletion :
1008 None
1009
1010
1011plugins.pre_analysis.robots
1012---------------------------
1013
1014 Pre analysis hook
1015
1016 Filter robots
1017
1018 Plugin requirements :
1019 None
1020
1021 Conf values needed :
1022 None
1023
1024 Output files :
1025 None
1026
1027 Statistics creation :
1028 None
1029
1030 Statistics update :
1031 visits :
1032 remote_addr =>
1033 robot
1034 keep_requests
1035
1036 Statistics deletion :
1037 None
1038
1039

Archive Download this file

Branches

Tags