Merge branch 'master' of soutade.fr:iwla

TODO

@@ -4,7 +4,7 @@ doc auto generation
 doc enhancement
 Limit hits/pages/downloads by rate
 Automatic tests
-quiet mode
 Add Licence
 Free memory as soon as possible
-gzip output files
+gzip output files
+different debug output levels

conf.py

@@ -24,3 +24,5 @@ hit_to_page_conf = [r'^.+/category/.+$', r'^.+/tag/.+$', r'^.+/archive/.+$', r'^
 # Because it's too long to build HTML when there is too much entries
 max_hits_displayed = 100
 max_downloads_displayed = 100
+
+compress_output_files = ['html', 'css', 'js']

default_conf.py

@@ -48,3 +48,6 @@ resources_path = ['resources']
 icon_path = '%s/%s' % (os.path.basename(resources_path[0]), 'icon')
 # CSS path (you can add yours)
 css_path = ['%s/%s/%s' % (os.path.basename(resources_path[0]), 'css', 'iwla.css')]
+
+# Extensions to compress in gzip during display build 
+compress_output_files = []

iwla.py

@@ -31,6 +31,7 @@ from display import *
 # Conf values needed :
 #     analyzed_filename
 #     domain_name
+#     compress_output_files*
 #
 # Output files :
 #     DB_ROOT/meta.db
@@ -40,7 +41,7 @@ from display import *
 #
 # Statistics creation :
 #
-# meta =>
+# meta :
 #     last_time
 #     start_analysis_time
 #     stats =>
@@ -50,6 +51,7 @@ from display import *
 #                 not_viewed_bandwidth
 #                 viewed_pages
 #                 viewed_hits
+#                 nb_visits
 #                 nb_visitors
 #
 # month_stats :
@@ -57,7 +59,7 @@ from display import *
 #     not_viewed_bandwidth
 #     viewed_pages
 #     viewed_hits
-#     nb_visitors
+#     nb_visits
 #
 # days_stats :
 #     day =>
@@ -65,6 +67,7 @@ from display import *
 #         not_viewed_bandwidth
 #         viewed_pages
 #         viewed_hits
+#         nb_visits
 #         nb_visitors
 #
 # visits :
@@ -338,18 +341,18 @@ class IWLA(object):
         page = self.display.createPage(title, filename, conf.css_path)
 
         _, nb_month_days = monthrange(cur_time.tm_year, cur_time.tm_mon)
-        days = self.display.createBlock(DisplayHTMLBlockTableWithGraph, 'By day', ['Day', 'Visitors', 'Pages', 'Hits', 'Bandwidth', 'Not viewed Bandwidth'], None, nb_month_days, range(1,6))
+        days = self.display.createBlock(DisplayHTMLBlockTableWithGraph, 'By day', ['Day', 'Visits', 'Pages', 'Hits', 'Bandwidth', 'Not viewed Bandwidth'], None, nb_month_days, range(1,6))
-        days.setColsCSSClass(['', 'iwla_visitor', 'iwla_page', 'iwla_hit', 'iwla_bandwidth', 'iwla_bandwidth'])
+        days.setColsCSSClass(['', 'iwla_visit', 'iwla_page', 'iwla_hit', 'iwla_bandwidth', 'iwla_bandwidth'])
         nb_visits = 0
         nb_days = 0
         for i in range(1, nb_month_days+1):
             day = '%d<br/>%s' % (i, time.strftime('%b', cur_time))
-            full_day = '%d %s %d' % (i, time.strftime('%b', cur_time), cur_time.tm_year)
+            full_day = '%02d %s %d' % (i, time.strftime('%b', cur_time), cur_time.tm_year)
             if i in self.current_analysis['days_stats'].keys():
                 stats = self.current_analysis['days_stats'][i]
-                row = [full_day, stats['nb_visitors'], stats['viewed_pages'], stats['viewed_hits'],
+                row = [full_day, stats['nb_visits'], stats['viewed_pages'], stats['viewed_hits'],
                        stats['viewed_bandwidth'], stats['not_viewed_bandwidth']]
-                nb_visits += stats['nb_visitors']
+                nb_visits += stats['nb_visits']
                 nb_days += 1
             else:
                 row = [full_day, 0, 0, 0, 0, 0]
@@ -391,10 +394,10 @@ class IWLA(object):
         cur_time = time.localtime()
         months_name = ['', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'June', 'July', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
         title = 'Summary %d' % (year)
-        cols = ['Month', 'Visitors', 'Pages', 'Hits', 'Bandwidth', 'Not viewed Bandwidth', 'Details']
+        cols = ['Month', 'Visitors', 'Visits', 'Pages', 'Hits', 'Bandwidth', 'Not viewed Bandwidth', 'Details']
-        graph_cols=range(1,6)
+        graph_cols=range(1,7)
         months = self.display.createBlock(DisplayHTMLBlockTableWithGraph, title, cols, None, 12, graph_cols)
-        months.setColsCSSClass(['', 'iwla_visitor', 'iwla_page', 'iwla_hit', 'iwla_bandwidth', 'iwla_bandwidth', ''])
+        months.setColsCSSClass(['', 'iwla_visitor', 'iwla_visit', 'iwla_page', 'iwla_hit', 'iwla_bandwidth', 'iwla_bandwidth', ''])
         total = [0] * len(cols)
         for i in range(1, 13):
             month = '%s<br/>%d' % (months_name[i], year)
@@ -402,15 +405,15 @@ class IWLA(object):
             if i in month_stats.keys():
                 stats = month_stats[i]
                 link = '<a href="%d/%02d/index.html">Details</a>' % (year, i)
-                row = [full_month, stats['nb_visitors'], stats['viewed_pages'], stats['viewed_hits'],
+                row = [full_month, stats['nb_visitors'], stats['nb_visits'], stats['viewed_pages'], stats['viewed_hits'],
                        stats['viewed_bandwidth'], stats['not_viewed_bandwidth'], link]
                 for j in graph_cols:
                     total[j] += row[j]
             else:
-                row = [full_month, 0, 0, 0, 0, 0, '']
+                row = [full_month, 0, 0, 0, 0, 0, 0, '']
             months.appendRow(row)
-            months.setCellValue(i-1, 4, bytesToStr(row[4]))
             months.setCellValue(i-1, 5, bytesToStr(row[5]))
+            months.setCellValue(i-1, 6, bytesToStr(row[6]))
             months.appendShortTitle(month)
             if year == cur_time.tm_year and i == cur_time.tm_mon:
                 css = months.getCellCSSClass(i-1, 0)
@@ -419,8 +422,8 @@ class IWLA(object):
                 months.setCellCSSClass(i-1, 0, css)
 
         total[0] = 'Total'
-        total[4] = bytesToStr(total[4])
         total[5] = bytesToStr(total[5])
+        total[6] = bytesToStr(total[6])
         months.appendRow(total)
         page.appendBlock(months)
 
@@ -431,19 +434,40 @@ class IWLA(object):
 
         page = self.display.createPage(title, filename, conf.css_path)
 
-        last_update = '<b>Last update</b> %s<br />' % (time.strftime('%d %b %Y %H:%M', time.localtime()))
+        last_update = '<b>Last update</b> %s<br />' % (time.strftime('%02d %b %Y %H:%M', time.localtime()))
         page.appendBlock(self.display.createBlock(DisplayHTMLRaw, last_update))
 
-        for year in self.meta_infos['stats'].keys():
+        for year in sorted(self.meta_infos['stats'].keys(), reverse=True):
             self._generateDisplayMonthStats(page, year, self.meta_infos['stats'][year])
 
         self.display.addPage(page)
 
+    def _compressFile(self, build_time, root, filename):
+        path = os.path.join(root, filename)
+        gz_path = path + '.gz'
+        #print 'Compress %s => %s' % (path, gz_path)
+        if not os.path.exists(gz_path) or\
+           os.stat(path).st_mtime > build_time:
+            with open(path, 'rb') as f_in:
+                with gzip.open(gz_path, 'wb') as f_out:
+                    f_out.write(f_in.read())
+
+    def _compressFiles(self, build_time, root):
+        if not conf.compress_output_files: return
+        for rootdir, subdirs, files in os.walk(root, followlinks=True):
+            for f in files:
+                for ext in conf.compress_output_files:
+                    if f.endswith(ext):
+                        self._compressFile(build_time, rootdir, f)
+                        break
+                        
     def _generateDisplay(self):
         self._generateDisplayDaysStats()
         self._callPlugins(conf.DISPLAY_HOOK_DIRECTORY)
         self._generateDisplayWholeMonthStats()
+        build_time = time.localtime()
         self.display.build(conf.DISPLAY_ROOT)
+        self._compressFiles(build_time, conf.DISPLAY_ROOT)
 
     def _createEmptyStats(self):
         stats = {}
@@ -451,7 +475,7 @@ class IWLA(object):
         stats['not_viewed_bandwidth'] = 0
         stats['viewed_pages'] = 0
         stats['viewed_hits'] = 0
-        stats['nb_visitors'] = 0
+        stats['nb_visits'] = 0
 
         return stats
 
@@ -464,7 +488,7 @@ class IWLA(object):
         for (day, stat) in self.current_analysis['days_stats'].items():
             for k in stats.keys():
                 stats[k] += stat[k]
-            
+
         duplicated_stats = {k:v for (k,v) in stats.items()}
 
         cur_time = self.meta_infos['last_time']
@@ -485,7 +509,7 @@ class IWLA(object):
                 continue
             self.valid_visitors[k] = v
 
-        duplicated_stats['visitors'] = stats['visitors'] = len(self.valid_visitors.keys())
+        duplicated_stats['nb_visitors'] = stats['nb_visitors'] = len(self.valid_visitors.keys())
 
         self._callPlugins(conf.POST_HOOK_DIRECTORY)
 
@@ -535,9 +559,9 @@ class IWLA(object):
             if (conf.count_hit_only_visitors or\
                     viewed_pages) and\
                     not super_hit['robot']:
-                stats['nb_visitors'] += 1
+                stats['nb_visits'] += 1
 
-        print "== Stats for %d/%02d/%d ==" % (cur_time.tm_year, cur_time.tm_mon, cur_time.tm_mday)
+        print "== Stats for %d/%02d/%02d ==" % (cur_time.tm_year, cur_time.tm_mon, cur_time.tm_mday)
 
         print stats