Update documentation

This commit is contained in:
Grégory Soutadé 2022-11-08 20:42:57 +01:00
parent 928e4eea5a
commit 2e955e5fe2
5 changed files with 95 additions and 364 deletions

View File

@ -8,6 +8,11 @@ iwla (Intelligent Web Log Analyzer) is basically a clone of [awstats](http://www
Nevertheless, iwla is only focused on HTTP logs. It uses data (robots definitions, search engines definitions) and design from awstats. Moreover, it's not dynamic, but only generates static HTML page (with gzip compression option).
Demo
----
A demonstration instance is available [here](https://iwla-demo.soutade.fr)
Usage
-----
@ -19,7 +24,7 @@ Usage
-f : Analyse this log file, multiple files can be specified (comma separated). gz files are acceptedRead data from FILE instead of conf.analyzed_filename
-d : Loglevel in ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
-r : Reset analysis to a specific date (month/year)
-z : Don't compress databases (bigger but faster, not compatible with compressed databases)
-z : Don't compress databases (bigger but faster, not compatible with compressed databases)
-p : Only generate display
-d : Dry run (don't write/update files to disk)
@ -36,6 +41,8 @@ Main values to edit are :
* **post_analysis_hooks** : List of post analysis hooks
* **display_hooks** : List of display hooks
* **locale** : Displayed locale (_en_ or _fr_)
* **feeds** : Address of your feeds files
* **count_hit_only_visitors** true/false (don't) count visitors that only do one hit (for a picture, ...)
You can also append an element to an existing default configuration list by using "_append" suffix. Example :
multimedia_files_append = ['xml']
@ -82,7 +89,7 @@ Statistics are stored in dictionaries :
* **month_stats** : Statistics of current analysed month
* **valid_visitor** : A subset of month_stats without robots
* **days_stats** : Statistics of current analysed day
* **visits** : All visitors with all of its requests
* **visits** : All visitors with all of its requests (only if 'keep_requests' is true or filtered)
* **meta** : Final result of month statistics (by year)
Create a Plugins
@ -119,17 +126,15 @@ Optional configuration values ends with *.
* plugins/display/top_pages_diff.py
* plugins/display/top_pages.py
* plugins/display/top_visitors.py
* plugins/post_analysis/anonymize_ip.py
* plugins/post_analysis/browsers.py
* plugins/post_analysis/feeds.py
* plugins/post_analysis/filter_users.py
* plugins/post_analysis/google_console_api.py
* plugins/post_analysis/hours_stats.py
* plugins/post_analysis/ip_to_geo.py
* plugins/post_analysis/iptogeo.py
* plugins/post_analysis/operating_systems.py
* plugins/post_analysis/referers.py
* plugins/post_analysis/reverse_dns.py
* plugins/post_analysis/search_analytics_api_sample.py
* plugins/post_analysis/top_downloads.py
* plugins/post_analysis/top_hits.py
* plugins/post_analysis/top_pages.py
@ -688,6 +693,33 @@ plugins.display.top_visitors
None
plugins.post_analysis.anonymize_ip
----------------------------------
Post analysis hook
Replace remote_addr by a SHA1
Plugin requirements :
None
Conf values needed :
None
Output files :
None
Statistics creation :
None
Statistics update :
valid_visitors:
remote_addr
Statistics deletion :
None
plugins.post_analysis.browsers
------------------------------
@ -807,34 +839,6 @@ plugins.post_analysis.filter_users
None
plugins.post_analysis.google_console_api
----------------------------------------
Post analysis hook
Extract key phrases from Google console API
Plugin requirements :
None
Conf values needed :
domain_name
Output files :
None
Statistics creation :
None
Statistics update :
month_stats :
key_phrases =>
phrase => count
Statistics deletion :
None
plugins.post_analysis.hours_stats
---------------------------------
@ -902,11 +906,6 @@ plugins.post_analysis.ip_to_geo
None
plugins.post_analysis.iptogeo
-----------------------------
plugins.post_analysis.operating_systems
---------------------------------------
@ -1008,141 +1007,6 @@ plugins.post_analysis.reverse_dns
None
plugins.post_analysis.search_analytics_api_sample
-------------------------------------------------
from __future__ import print_function
import argparse
import sys
from googleapiclient import sample_tools
# Declare command-line flags.
argparser = argparse.ArgumentParser(add_help=False)
argparser.add_argument('property_uri', type=str,
help=('Site or app URI to query data for (including '
'trailing slash).'))
argparser.add_argument('start_date', type=str,
help=('Start date of the requested date range in '
'YYYY-MM-DD format.'))
argparser.add_argument('end_date', type=str,
help=('End date of the requested date range in '
'YYYY-MM-DD format.'))
def main(argv):
service, flags = sample_tools.init(
argv, 'webmasters', 'v3', __doc__, __file__, parents=[argparser],
scope='https://www.googleapis.com/auth/webmasters.readonly')
# First run a query to learn which dates we have data for. You should always
# check which days in a date range have data before running your main query.
# This query shows data for the entire range, grouped and sorted by day,
# descending; any days without data will be missing from the results.
request = {
'startDate': flags.start_date,
'endDate': flags.end_date,
'dimensions': ['query']
}
response = execute_request(service, flags.property_uri, request)
print_table(response, 'Available dates')
return
# Get totals for the date range.
request = {
'startDate': flags.start_date,
'endDate': flags.end_date
}
response = execute_request(service, flags.property_uri, request)
print_table(response, 'Totals')
# Get top 10 queries for the date range, sorted by click count, descending.
request = {
'startDate': flags.start_date,
'endDate': flags.end_date,
'dimensions': ['query'],
'rowLimit': 10
}
response = execute_request(service, flags.property_uri, request)
print_table(response, 'Top Queries')
# Get top 11-20 mobile queries for the date range, sorted by click count, descending.
request = {
'startDate': flags.start_date,
'endDate': flags.end_date,
'dimensions': ['query'],
'dimensionFilterGroups': [{
'filters': [{
'dimension': 'device',
'expression': 'mobile'
}]
}],
'rowLimit': 10,
'startRow': 10
}
response = execute_request(service, flags.property_uri, request)
print_table(response, 'Top 11-20 Mobile Queries')
# Get top 10 pages for the date range, sorted by click count, descending.
request = {
'startDate': flags.start_date,
'endDate': flags.end_date,
'dimensions': ['page'],
'rowLimit': 10
}
response = execute_request(service, flags.property_uri, request)
print_table(response, 'Top Pages')
# Get the top 10 queries in India, sorted by click count, descending.
request = {
'startDate': flags.start_date,
'endDate': flags.end_date,
'dimensions': ['query'],
'dimensionFilterGroups': [{
'filters': [{
'dimension': 'country',
'expression': 'ind'
}]
}],
'rowLimit': 10
}
response = execute_request(service, flags.property_uri, request)
print_table(response, 'Top queries in India')
# Group by both country and device.
request = {
'startDate': flags.start_date,
'endDate': flags.end_date,
'dimensions': ['country', 'device'],
'rowLimit': 10
}
response = execute_request(service, flags.property_uri, request)
print_table(response, 'Group by country and device')
# Group by total number of Search Appearance count.
# Note: It is not possible to use searchAppearance with other
# dimensions.
request = {
'startDate': flags.start_date,
'endDate': flags.end_date,
'dimensions': ['searchAppearance'],
'rowLimit': 10
}
response = execute_request(service, flags.property_uri, request)
print_table(response, 'Search Appearance Features')
def execute_request(service, property_uri, request):
"""Executes a searchAnalytics.query request.
Args:
service: The webmasters service to use when executing the query.
property_uri: The site or app URI to request data for.
request: The request to be executed.
Returns:
An array of response rows.
plugins.post_analysis.top_downloads
-----------------------------------

View File

@ -8,6 +8,11 @@ iwla (Intelligent Web Log Analyzer) is basically a clone of [awstats](http://www
Nevertheless, iwla is only focused on HTTP logs. It uses data (robots definitions, search engines definitions) and design from awstats. Moreover, it's not dynamic, but only generates static HTML page (with gzip compression option).
Demo
----
A demonstration instance is available [here](https://iwla-demo.soutade.fr)
Usage
-----
@ -19,7 +24,7 @@ Usage
-f : Analyse this log file, multiple files can be specified (comma separated). gz files are acceptedRead data from FILE instead of conf.analyzed_filename
-d : Loglevel in ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
-r : Reset analysis to a specific date (month/year)
-z : Don't compress databases (bigger but faster, not compatible with compressed databases)
-z : Don't compress databases (bigger but faster, not compatible with compressed databases)
-p : Only generate display
-d : Dry run (don't write/update files to disk)
@ -36,6 +41,8 @@ Main values to edit are :
* **post_analysis_hooks** : List of post analysis hooks
* **display_hooks** : List of display hooks
* **locale** : Displayed locale (_en_ or _fr_)
* **feeds** : Address of your feeds files
* **count_hit_only_visitors** true/false (don't) count visitors that only do one hit (for a picture, ...)
You can also append an element to an existing default configuration list by using "_append" suffix. Example :
multimedia_files_append = ['xml']
@ -82,7 +89,7 @@ Statistics are stored in dictionaries :
* **month_stats** : Statistics of current analysed month
* **valid_visitor** : A subset of month_stats without robots
* **days_stats** : Statistics of current analysed day
* **visits** : All visitors with all of its requests
* **visits** : All visitors with all of its requests (only if 'keep_requests' is true or filtered)
* **meta** : Final result of month statistics (by year)
Create a Plugins

View File

@ -16,17 +16,15 @@
* plugins/display/top_pages_diff.py
* plugins/display/top_pages.py
* plugins/display/top_visitors.py
* plugins/post_analysis/anonymize_ip.py
* plugins/post_analysis/browsers.py
* plugins/post_analysis/feeds.py
* plugins/post_analysis/filter_users.py
* plugins/post_analysis/google_console_api.py
* plugins/post_analysis/hours_stats.py
* plugins/post_analysis/ip_to_geo.py
* plugins/post_analysis/iptogeo.py
* plugins/post_analysis/operating_systems.py
* plugins/post_analysis/referers.py
* plugins/post_analysis/reverse_dns.py
* plugins/post_analysis/search_analytics_api_sample.py
* plugins/post_analysis/top_downloads.py
* plugins/post_analysis/top_hits.py
* plugins/post_analysis/top_pages.py
@ -585,6 +583,33 @@ plugins.display.top_visitors
None
plugins.post_analysis.anonymize_ip
----------------------------------
Post analysis hook
Replace remote_addr by a SHA1
Plugin requirements :
None
Conf values needed :
None
Output files :
None
Statistics creation :
None
Statistics update :
valid_visitors:
remote_addr
Statistics deletion :
None
plugins.post_analysis.browsers
------------------------------
@ -704,34 +729,6 @@ plugins.post_analysis.filter_users
None
plugins.post_analysis.google_console_api
----------------------------------------
Post analysis hook
Extract key phrases from Google console API
Plugin requirements :
None
Conf values needed :
domain_name
Output files :
None
Statistics creation :
None
Statistics update :
month_stats :
key_phrases =>
phrase => count
Statistics deletion :
None
plugins.post_analysis.hours_stats
---------------------------------
@ -799,11 +796,6 @@ plugins.post_analysis.ip_to_geo
None
plugins.post_analysis.iptogeo
-----------------------------
plugins.post_analysis.operating_systems
---------------------------------------
@ -905,141 +897,6 @@ plugins.post_analysis.reverse_dns
None
plugins.post_analysis.search_analytics_api_sample
-------------------------------------------------
from __future__ import print_function
import argparse
import sys
from googleapiclient import sample_tools
# Declare command-line flags.
argparser = argparse.ArgumentParser(add_help=False)
argparser.add_argument('property_uri', type=str,
help=('Site or app URI to query data for (including '
'trailing slash).'))
argparser.add_argument('start_date', type=str,
help=('Start date of the requested date range in '
'YYYY-MM-DD format.'))
argparser.add_argument('end_date', type=str,
help=('End date of the requested date range in '
'YYYY-MM-DD format.'))
def main(argv):
service, flags = sample_tools.init(
argv, 'webmasters', 'v3', __doc__, __file__, parents=[argparser],
scope='https://www.googleapis.com/auth/webmasters.readonly')
# First run a query to learn which dates we have data for. You should always
# check which days in a date range have data before running your main query.
# This query shows data for the entire range, grouped and sorted by day,
# descending; any days without data will be missing from the results.
request = {
'startDate': flags.start_date,
'endDate': flags.end_date,
'dimensions': ['query']
}
response = execute_request(service, flags.property_uri, request)
print_table(response, 'Available dates')
return
# Get totals for the date range.
request = {
'startDate': flags.start_date,
'endDate': flags.end_date
}
response = execute_request(service, flags.property_uri, request)
print_table(response, 'Totals')
# Get top 10 queries for the date range, sorted by click count, descending.
request = {
'startDate': flags.start_date,
'endDate': flags.end_date,
'dimensions': ['query'],
'rowLimit': 10
}
response = execute_request(service, flags.property_uri, request)
print_table(response, 'Top Queries')
# Get top 11-20 mobile queries for the date range, sorted by click count, descending.
request = {
'startDate': flags.start_date,
'endDate': flags.end_date,
'dimensions': ['query'],
'dimensionFilterGroups': [{
'filters': [{
'dimension': 'device',
'expression': 'mobile'
}]
}],
'rowLimit': 10,
'startRow': 10
}
response = execute_request(service, flags.property_uri, request)
print_table(response, 'Top 11-20 Mobile Queries')
# Get top 10 pages for the date range, sorted by click count, descending.
request = {
'startDate': flags.start_date,
'endDate': flags.end_date,
'dimensions': ['page'],
'rowLimit': 10
}
response = execute_request(service, flags.property_uri, request)
print_table(response, 'Top Pages')
# Get the top 10 queries in India, sorted by click count, descending.
request = {
'startDate': flags.start_date,
'endDate': flags.end_date,
'dimensions': ['query'],
'dimensionFilterGroups': [{
'filters': [{
'dimension': 'country',
'expression': 'ind'
}]
}],
'rowLimit': 10
}
response = execute_request(service, flags.property_uri, request)
print_table(response, 'Top queries in India')
# Group by both country and device.
request = {
'startDate': flags.start_date,
'endDate': flags.end_date,
'dimensions': ['country', 'device'],
'rowLimit': 10
}
response = execute_request(service, flags.property_uri, request)
print_table(response, 'Group by country and device')
# Group by total number of Search Appearance count.
# Note: It is not possible to use searchAppearance with other
# dimensions.
request = {
'startDate': flags.start_date,
'endDate': flags.end_date,
'dimensions': ['searchAppearance'],
'rowLimit': 10
}
response = execute_request(service, flags.property_uri, request)
print_table(response, 'Search Appearance Features')
def execute_request(service, property_uri, request):
"""Executes a searchAnalytics.query request.
Args:
service: The webmasters service to use when executing the query.
property_uri: The site or app URI to request data for.
request: The request to be executed.
Returns:
An array of response rows.
plugins.post_analysis.top_downloads
-----------------------------------

Binary file not shown.

View File

@ -5,8 +5,8 @@
msgid ""
msgstr ""
"Project-Id-Version: iwla\n"
"POT-Creation-Date: 2022-11-07 21:14+0100\n"
"PO-Revision-Date: 2022-11-07 21:14+0100\n"
"POT-Creation-Date: 2022-11-10 20:07+0100\n"
"PO-Revision-Date: 2022-11-10 20:08+0100\n"
"Last-Translator: Soutadé <soutade@gmail.com>\n"
"Language-Team: iwla\n"
"Language: fr\n"
@ -84,14 +84,14 @@ msgstr "Visites"
#: iwla.py:454 iwla.py:505 plugins/display/all_visits.py:70
#: plugins/display/feeds.py:76 plugins/display/filter_users.py:77
#: plugins/display/filter_users.py:119 plugins/display/hours_stats.py:73
#: plugins/display/filter_users.py:118 plugins/display/hours_stats.py:73
#: plugins/display/hours_stats.py:83 plugins/display/referers.py:95
#: plugins/display/referers.py:153 plugins/display/top_visitors.py:72
msgid "Pages"
msgstr "Pages"
#: iwla.py:454 iwla.py:505 plugins/display/all_visits.py:70
#: plugins/display/feeds.py:76 plugins/display/filter_users.py:119
#: plugins/display/feeds.py:76 plugins/display/filter_users.py:118
#: plugins/display/hours_stats.py:73 plugins/display/hours_stats.py:83
#: plugins/display/referers.py:95 plugins/display/referers.py:153
#: plugins/display/top_downloads.py:97 plugins/display/top_visitors.py:72
@ -170,12 +170,11 @@ msgid "Month"
msgstr "Mois"
#: iwla.py:505 iwla.py:517 plugins/display/feeds.py:101
#: plugins/display/filter_users.py:114 plugins/display/ip_to_geo.py:109
#: plugins/display/operating_systems.py:90
#: plugins/display/filter_users.py:113 plugins/display/operating_systems.py:90
msgid "Details"
msgstr "Détails"
#: iwla.py:505 plugins/display/ip_to_geo.py:96 plugins/display/ip_to_geo.py:114
#: iwla.py:505 plugins/display/ip_to_geo.py:94 plugins/display/ip_to_geo.py:112
msgid "Visitors"
msgstr "Visiteurs"
@ -203,8 +202,12 @@ msgstr "minutes"
msgid "seconds"
msgstr "secondes"
#: plugins/display/all_visits.py:70 plugins/display/all_visits.py:92
msgid "All visits"
msgstr "Toutes les visites"
#: plugins/display/all_visits.py:70 plugins/display/feeds.py:76
#: plugins/display/filter_users.py:119 plugins/display/ip_to_geo.py:64
#: plugins/display/filter_users.py:118 plugins/display/ip_to_geo.py:62
#: plugins/display/robot_bandwidth.py:81 plugins/display/robot_bandwidth.py:106
#: plugins/display/top_visitors.py:72
msgid "Host"
@ -215,10 +218,6 @@ msgstr "Hôte"
msgid "Last seen"
msgstr "Dernière visite"
#: plugins/display/all_visits.py:92
msgid "All visits"
msgstr "Toutes les visites"
#: plugins/display/all_visits.py:93 plugins/display/top_visitors.py:72
msgid "Top visitors"
msgstr "Top visiteurs"
@ -240,7 +239,7 @@ msgid "Entrance"
msgstr "Entrées"
#: plugins/display/browsers.py:99 plugins/display/browsers.py:130
#: plugins/display/filter_users.py:129 plugins/display/referers.py:110
#: plugins/display/filter_users.py:128 plugins/display/referers.py:110
#: plugins/display/referers.py:125 plugins/display/referers.py:140
#: plugins/display/referers.py:163 plugins/display/referers.py:174
#: plugins/display/referers.py:185 plugins/display/referers.py:222
@ -291,11 +290,11 @@ msgstr "Origine"
msgid "User Agent"
msgstr "Navigateur"
#: plugins/display/filter_users.py:77 plugins/display/filter_users.py:112
#: plugins/display/filter_users.py:77 plugins/display/filter_users.py:111
msgid "Filtered users"
msgstr "Utilisateurs filtrés"
#: plugins/display/filter_users.py:77 plugins/display/filter_users.py:119
#: plugins/display/filter_users.py:77 plugins/display/filter_users.py:118
msgid "Last Access"
msgstr "Dernière visite"
@ -335,15 +334,19 @@ msgstr "Par heures"
msgid "Hours"
msgstr "Heures"
#: plugins/display/ip_to_geo.py:96
#: plugins/display/ip_to_geo.py:94
msgid "Country"
msgstr "Pays"
#: plugins/display/ip_to_geo.py:96 plugins/display/ip_to_geo.py:107
#: plugins/display/ip_to_geo.py:114
#: plugins/display/ip_to_geo.py:94 plugins/display/ip_to_geo.py:105
#: plugins/display/ip_to_geo.py:112
msgid "Countries"
msgstr "Pays"
#: plugins/display/ip_to_geo.py:107
msgid "All countries"
msgstr "Tous les pays"
#: plugins/display/operating_systems.py:78
#: plugins/display/operating_systems.py:88
msgid "Operating Systems"