iwla

iwla Commit Details

Date:2015-01-08 20:59:11 (6 years 6 months ago)
Author:Grégory Soutadé
Branch:dev, master
Commit:a40f116c71df4135730f7fa9665c708a91bab30f
Parents: 9a1c23ec78a1ebd8b9fcccc7cc6d2f002623d6b6
Message:Add Operating Systems post analysis and display

Changes:
Aplugins/display/operating_systems.py (full)
Aplugins/post_analysis/operating_systems.py (full)
Mawstats_data.py (2 diffs)
Mconf.py (1 diff)
Mtools/iwla_convert.pl (3 diffs)

File differences

awstats_data.py
1
2
13
24
35
......
1012
1113
1214
15
16
17
18
19
20
21
22
23
24
25
26
#This file was automatically generated by iwla_convert.pl. Do not edit manually.
robots = ['appie', 'architext', 'jeeves', 'bjaaland', 'contentmatch', 'ferret', 'googlebot', 'google\-sitemaps', 'gulliver', 'virus[_+ ]detector', 'harvest', 'htdig', 'linkwalker', 'lilina', 'lycos[_+ ]', 'moget', 'muscatferret', 'myweb', 'nomad', 'scooter', 'slurp', '^voyager\/', 'weblayers', 'antibot', 'bruinbot', 'digout4u', 'echo!', 'fast\-webcrawler', 'ia_archiver\-web\.archive\.org', 'ia_archiver', 'jennybot', 'mercator', 'netcraft', 'msnbot\-media', 'msnbot', 'petersnews', 'relevantnoise\.com', 'unlost_web_crawler', 'voila', 'webbase', 'webcollage', 'cfetch', 'zyborg', 'wisenutbot', '[^a]fish', 'abcdatos', 'acme\.spider', 'ahoythehomepagefinder', 'alkaline', 'anthill', 'arachnophilia', 'arale', 'araneo', 'aretha', 'ariadne', 'powermarks', 'arks', 'aspider', 'atn\.txt', 'atomz', 'auresys', 'backrub', 'bbot', 'bigbrother', 'blackwidow', 'blindekuh', 'bloodhound', 'borg\-bot', 'brightnet', 'bspider', 'cactvschemistryspider', 'calif[^r]', 'cassandra', 'cgireader', 'checkbot', 'christcrawler', 'churl', 'cienciaficcion', 'collective', 'combine', 'conceptbot', 'coolbot', 'core', 'cosmos', 'cruiser', 'cusco', 'cyberspyder', 'desertrealm', 'deweb', 'dienstspider', 'digger', 'diibot', 'direct_hit', 'dnabot', 'download_express', 'dragonbot', 'dwcp', 'e\-collector', 'ebiness', 'elfinbot', 'emacs', 'emcspider', 'esther', 'evliyacelebi', 'fastcrawler', 'feedcrawl', 'fdse', 'felix', 'fetchrover', 'fido', 'finnish', 'fireball', 'fouineur', 'francoroute', 'freecrawl', 'funnelweb', 'gama', 'gazz', 'gcreep', 'getbot', 'geturl', 'golem', 'gougou', 'grapnel', 'griffon', 'gromit', 'gulperbot', 'hambot', 'havindex', 'hometown', 'htmlgobble', 'hyperdecontextualizer', 'iajabot', 'iaskspider', 'hl_ftien_spider', 'sogou', 'iconoclast', 'ilse', 'imagelock', 'incywincy', 'informant', 'infoseek', 'infoseeksidewinder', 'infospider', 'inspectorwww', 'intelliagent', 'irobot', 'iron33', 'israelisearch', 'javabee', 'jbot', 'jcrawler', 'jobo', 'jobot', 'joebot', 'jubii', 'jumpstation', 'kapsi', 'katipo', 'kilroy', 'ko[_+ ]yappo[_+ ]robot', 'kummhttp', 'labelgrabber\.txt', 'larbin', 'legs', 'linkidator', 'linkscan', 'lockon', 'logo_gif', 'macworm', 'magpie', 'marvin', 'mattie', 'mediafox', 'merzscope', 'meshexplorer', 'mindcrawler', 'mnogosearch', 'momspider', 'monster', 'motor', 'muncher', 'mwdsearch', 'ndspider', 'nederland\.zoek', 'netcarta', 'netmechanic', 'netscoop', 'newscan\-online', 'nhse', 'northstar', 'nzexplorer', 'objectssearch', 'occam', 'octopus', 'openfind', 'orb_search', 'packrat', 'pageboy', 'parasite', 'patric', 'pegasus', 'perignator', 'perlcrawler', 'phantom', 'phpdig', 'piltdownman', 'pimptrain', 'pioneer', 'pitkow', 'pjspider', 'plumtreewebaccessor', 'poppi', 'portalb', 'psbot', 'python', 'raven', 'rbse', 'resumerobot', 'rhcs', 'road_runner', 'robbie', 'robi', 'robocrawl', 'robofox', 'robozilla', 'roverbot', 'rules', 'safetynetrobot', 'search\-info', 'search_au', 'searchprocess', 'senrigan', 'sgscout', 'shaggy', 'shaihulud', 'sift', 'simbot', 'site\-valet', 'sitetech', 'skymob', 'slcrawler', 'smartspider', 'snooper', 'solbot', 'speedy', 'spider[_+ ]monkey', 'spiderbot', 'spiderline', 'spiderman', 'spiderview', 'spry', 'sqworm', 'ssearcher', 'suke', 'sunrise', 'suntek', 'sven', 'tach_bw', 'tagyu_agent', 'tailrank', 'tarantula', 'tarspider', 'techbot', 'templeton', 'titan', 'titin', 'tkwww', 'tlspider', 'ucsd', 'udmsearch', 'universalfeedparser', 'urlck', 'valkyrie', 'verticrawl', 'victoria', 'visionsearch', 'voidbot', 'vwbot', 'w3index', 'w3m2', 'wallpaper', 'wanderer', 'wapspIRLider', 'webbandit', 'webcatcher', 'webcopy', 'webfetcher', 'webfoot', 'webinator', 'weblinker', 'webmirror', 'webmoose', 'webquest', 'webreader', 'webreaper', 'websnarf', 'webspider', 'webvac', 'webwalk', 'webwalker', 'webwatch', 'whatuseek', 'whowhere', 'wired\-digital', 'wmir', 'wolp', 'wombat', 'wordpress', 'worm', 'woozweb', 'wwwc', 'wz101', 'xget', '1\-more_scanner', 'accoona\-ai\-agent', 'activebookmark', 'adamm_bot', 'almaden', 'aipbot', 'aleadsoftbot', 'alpha_search_agent', 'allrati', 'aport', 'archive\.org_bot', 'argus', 'arianna\.libero\.it', 'aspseek', 'asterias', 'awbot', 'baiduspider', 'becomebot', 'bender', 'betabot', 'biglotron', 'bittorrent_bot', 'biz360[_+ ]spider', 'blogbridge[_+ ]service', 'bloglines', 'blogpulse', 'blogsearch', 'blogshares', 'blogslive', 'blogssay', 'bncf\.firenze\.sbn\.it\/raccolta\.txt', 'bobby', 'boitho\.com\-dc', 'bookmark\-manager', 'boris', 'bumblebee', 'candlelight[_+ ]favorites[_+ ]inspector', 'cbn00glebot', 'cerberian_drtrs', 'cfnetwork', 'cipinetbot', 'checkweb_link_validator', 'commons\-httpclient', 'computer_and_automation_research_institute_crawler', 'converamultimediacrawler', 'converacrawler', 'cscrawler', 'cse_html_validator_lite_online', 'cuasarbot', 'cursor', 'custo', 'datafountains\/dmoz_downloader', 'daviesbot', 'daypopbot', 'deepindex', 'dipsie\.bot', 'dnsgroup', 'domainchecker', 'domainsdb\.net', 'dulance', 'dumbot', 'dumm\.de\-bot', 'earthcom\.info', 'easydl', 'edgeio\-retriever', 'ets_v', 'exactseek', 'extreme[_+ ]picture[_+ ]finder', 'eventax', 'everbeecrawler', 'everest\-vulcan', 'ezresult', 'enteprise', 'facebook', 'fast_enterprise_crawler.*crawleradmin\.t\-info@telekom\.de', 'fast_enterprise_crawler.*t\-info_bi_cluster_crawleradmin\.t\-info@telekom\.de', 'matrix_s\.p\.a\._\-_fast_enterprise_crawler', 'fast_enterprise_crawler', 'fast\-search\-engine', 'favicon', 'favorg', 'favorites_sweeper', 'feedburner', 'feedfetcher\-google', 'feedflow', 'feedster', 'feedsky', 'feedvalidator', 'filmkamerabot', 'findlinks', 'findexa_crawler', 'fooky\.com\/ScorpionBot', 'g2crawler', 'gaisbot', 'geniebot', 'gigabot', 'girafabot', 'global_fetch', 'gnodspider', 'goforit\.com', 'goforitbot', 'gonzo', 'grub', 'gpu_p2p_crawler', 'henrythemiragorobot', 'heritrix', 'holmes', 'hoowwwer', 'hpprint', 'htmlparser', 'html[_+ ]link[_+ ]validator', 'httrack', 'hundesuche\.com\-bot', 'ichiro', 'iltrovatore\-setaccio', 'infobot', 'infociousbot', 'infomine', 'insurancobot', 'internet[_+ ]ninja', 'internetarchive', 'internetseer', 'internetsupervision', 'irlbot', 'isearch2006', 'iupui_research_bot', 'jrtwine[_+ ]software[_+ ]check[_+ ]favorites[_+ ]utility', 'justview', 'kalambot', 'kamano\.de_newsfeedverzeichnis', 'kazoombot', 'kevin', 'keyoshid', 'kinjabot', 'kinja\-imagebot', 'knowitall', 'knowledge\.com', 'kouaa_krawler', 'krugle', 'ksibot', 'kurzor', 'lanshanbot', 'letscrawl\.com', 'libcrawl', 'linkbot', 'link_valet_online', 'metager\-linkchecker', 'linkchecker', 'livejournal\.com', 'lmspider', 'lwp\-request', 'lwp\-trivial', 'magpierss', 'mail\.ru', 'mapoftheinternet\.com', 'mediapartners\-google', 'megite', 'metaspinner', 'microsoft[_+ ]url[_+ ]control', 'mini\-reptile', 'minirank', 'missigua_locator', 'misterbot', 'miva', 'mizzu_labs', 'mj12bot', 'mojeekbot', 'msiecrawler', 'ms_search_4\.0_robot', 'msrabot', 'msrbot', 'mt::telegraph::agent', 'nagios', 'nasa_search', 'mydoyouhike', 'netluchs', 'netsprint', 'newsgatoronline', 'nicebot', 'nimblecrawler', 'noxtrumbot', 'npbot', 'nutchcvs', 'nutchosu\-vlib', 'nutch', 'ocelli', 'octora_beta_bot', 'omniexplorer[_+ ]bot', 'onet\.pl[_+ ]sa', 'onfolio', 'opentaggerbot', 'openwebspider', 'oracle_ultra_search', 'orbiter', 'yodaobot', 'qihoobot', 'passwordmaker\.org', 'pear_http_request_class', 'peerbot', 'perman', 'php[_+ ]version[_+ ]tracker', 'pictureofinternet', 'ping\.blo\.gs', 'plinki', 'pluckfeedcrawler', 'pogodak', 'pompos', 'popdexter', 'port_huron_labs', 'postfavorites', 'projectwf\-java\-test\-crawler', 'proodlebot', 'pyquery', 'rambler', 'redalert', 'rojo', 'rssimagesbot', 'ruffle', 'rufusbot', 'sandcrawler', 'sbider', 'schizozilla', 'scumbot', 'searchguild[_+ ]dmoz[_+ ]experiment', 'seekbot', 'sensis_web_crawler', 'seznambot', 'shim\-crawler', 'shoutcast', 'slysearch', 'snap\.com_beta_crawler', 'sohu\-search', 'sohu', 'snappy', 'sphere_scout', 'spip', 'sproose_crawler', 'steeler', 'steroid__download', 'suchfin\-bot', 'superbot', 'surveybot', 'susie', 'syndic8', 'syndicapi', 'synoobot', 'tcl_http_client_package', 'technoratibot', 'teragramcrawlersurf', 'test_crawler', 'testbot', 't\-h\-u\-n\-d\-e\-r\-s\-t\-o\-n\-e', 'topicblogs', 'turnitinbot', 'turtlescanner', 'turtle', 'tutorgigbot', 'twiceler', 'ubicrawler', 'ultraseek', 'unchaos_bot_hybrid_web_search_engine', 'unido\-bot', 'updated', 'ustc\-semantic\-group', 'vagabondo\-wap', 'vagabondo', 'vermut', 'versus_crawler_from_eda\.baykan@epfl\.ch', 'vespa_crawler', 'vortex', 'vse\/', 'w3c\-checklink', 'w3c[_+ ]css[_+ ]validator[_+ ]jfouffa', 'w3c_validator', 'watchmouse', 'wavefire', 'webclipping\.com', 'webcompass', 'webcrawl\.net', 'web_downloader', 'webdup', 'webfilter', 'webindexer', 'webminer', 'website[_+ ]monitoring[_+ ]bot', 'webvulncrawl', 'wells_search', 'wonderer', 'wume_crawler', 'wwweasel', 'xenu\'s_link_sleuth', 'xenu_link_sleuth', 'xirq', 'y!j', 'yacy', 'yahoo\-blogs', 'yahoo\-verticalcrawler', 'yahoofeedseeker', 'yahooseeker\-testing', 'yahooseeker', 'yahoo\-mmcrawler', 'yahoo!_mindset', 'yandex', 'flexum', 'yanga', 'yooglifetchagent', 'z\-add_link_checker', 'zealbot', 'zhuaxia', 'zspider', 'zeus', 'ng\/1\.', 'ng\/2\.', 'exabot', 'wget', 'libwww', 'java\/[0-9]']
search_engines = ['google\.[\w.]+/products', 'base\.google\.', 'froogle\.google\.', 'groups\.google\.', 'images\.google\.', 'google\.', 'googlee\.', 'googlecom\.com', 'goggle\.co\.hu', '216\.239\.(35|37|39|51)\.100', '216\.239\.(35|37|39|51)\.101', '216\.239\.5[0-9]\.104', '64\.233\.1[0-9]{2}\.104', '66\.102\.[1-9]\.104', '66\.249\.93\.104', '72\.14\.2[0-9]{2}\.104', 'msn\.', 'live\.com', 'bing\.', 'voila\.', 'mindset\.research\.yahoo', 'yahoo\.', '(66\.218\.71\.225|216\.109\.117\.135|216\.109\.125\.130|66\.218\.69\.11)', 'search\.aol\.co', 'tiscali\.', 'lycos\.', 'alexa\.com', 'alltheweb\.com', 'altavista\.', 'a9\.com', 'dmoz\.org', 'netscape\.', 'search\.terra\.', 'www\.search\.com', 'search\.sli\.sympatico\.ca', 'excite\.']
search_engines_knwown_url = {'dmoz' : 'search=', 'google' : '(p|q|as_p|as_q)=', 'searchalot' : 'q=', 'teoma' : 'q=', 'looksmartuk' : 'key=', 'polymeta_hu' : '', 'google_groups' : 'group\/', 'iune' : '(keywords|q)=', 'chellosk' : 'q1=', 'eniro' : 'q=', 'msn' : 'q=', 'webcrawler' : 'searchText=', 'mirago' : '(txtsearch|qry)=', 'enirose' : 'q=', 'miragobe' : '(txtsearch|qry)=', 'netease' : 'q=', 'netluchs' : 'query=', 'google_products' : '(p|q|as_p|as_q)=', 'jyxo' : '(s|q)=', 'origo' : '(q|search)=', 'ilse' : 'search_for=', 'chellocom' : 'q1=', 'goodsearch' : 'Keywords=', 'ledix' : 'q=', 'mozbot' : 'q=', 'chellocz' : 'q1=', 'webde' : 'su=', 'biglotron' : 'question=', 'metacrawler_de' : 'qry=', 'finddk' : 'words=', 'start' : 'q=', 'sagool' : 'q=', 'miragoch' : '(txtsearch|qry)=', 'google_base' : '(p|q|as_p|as_q)=', 'aliceit' : 'qs=', 'shinyseek\.it' : 'KEY=', 'onetpl' : 'qt=', 'clusty' : 'query=', 'chellonl' : 'q1=', 'miragode' : '(txtsearch|qry)=', 'miragose' : '(txtsearch|qry)=', 'o2pl' : 'qt=', 'goliat' : 'KERESES=', 'kvasir' : 'q=', 'askfr' : '(ask|q)=', 'infoseek' : 'qt=', 'yahoo_mindset' : 'p=', 'comettoolbar' : 'qry=', 'alltheweb' : 'q(|uery)=', 'miner' : 'q=', 'aol' : 'query=', 'rambler' : 'words=', 'scroogle' : 'Gw=', 'chellose' : 'q1=', 'ineffabile' : '', 'miragoit' : '(txtsearch|qry)=', 'yandex' : 'text=', 'segnalo' : '', 'dodajpl' : 'keyword=', 'avantfind' : 'keywords=', 'nusearch' : 'nusearch_terms=', 'bbc' : 'q=', 'supereva' : 'q=', 'atomz' : 'sp-q=', 'searchy' : 'search_term=', 'dogpile' : 'q(|kw)=', 'chellohu' : 'q1=', 'vnet' : 'kw=', '1klik' : 'query=', 't-online' : 'q=', 'hogapl' : 'qt=', 'stumbleupon' : '', 'soso' : 'q=', 'zhongsou' : '(word|w)=', 'a9' : 'a9\.com\/', 'centraldatabase' : 'query=', 'mamma' : 'query=', 'icerocket' : 'q=', 'ask' : '(ask|q)=', 'chellobe' : 'q1=', 'altavista' : 'q=', 'vindex' : 'in=', 'miragodk' : '(txtsearch|qry)=', 'chelloat' : 'q1=', 'digg' : 's=', 'metacrawler' : 'general=', 'nbci' : 'keyword=', 'chellono' : 'q1=', 'icq' : 'q=', 'arianna' : 'query=', 'miragocouk' : '(txtsearch|qry)=', '3721' : '(p|name)=', 'pogodak' : 'q=', 'ukdirectory' : 'k=', 'overture' : 'keywords=', 'heureka' : 'heureka=', 'teecnoit' : 'q=', 'miragoes' : '(txtsearch|qry)=', 'haku' : 'w=', 'go' : 'qt=', 'fireball' : 'q=', 'wisenut' : 'query=', 'sify' : 'keyword=', 'ixquick' : 'query=', 'anzwers' : 'search=', 'quick' : 'query=', 'jubii' : 'soegeord=', 'questionanswering' : '', 'asknl' : '(ask|q)=', 'askde' : '(ask|q)=', 'att' : 'qry=', 'terra' : 'query=', 'bing' : 'q=', 'wowpl' : 'q=', 'freeserve' : 'q=', 'atlas' : '(searchtext|q)=', 'askuk' : '(ask|q)=', 'godado' : 'Keywords=', 'northernlight' : 'qr=', 'answerbus' : '', 'search.com' : 'q=', 'google_image' : '(p|q|as_p|as_q)=', 'jumpy\.it' : 'searchWord=', 'gazetapl' : 'slowo=', 'yahoo' : 'p=', 'hotbot' : 'mt=', 'metabot' : 'st=', 'copernic' : 'web\/', 'kartoo' : '', 'metaspinner' : 'qry=', 'toile' : 'q=', 'aolde' : 'q=', 'blingo' : 'q=', 'askit' : '(ask|q)=', 'netscape' : 'search=', 'splut' : 'pattern=', 'looksmart' : 'key=', 'sphere' : 'q=', 'sol' : 'q=', 'miragono' : '(txtsearch|qry)=', 'kataweb' : 'q=', 'ofir' : 'querytext=', 'aliceitmaster' : 'qs=', 'miragofr' : '(txtsearch|qry)=', 'spray' : 'string=', 'seznam' : '(w|q)=', 'interiapl' : 'q=', 'euroseek' : 'query=', 'schoenerbrausen' : 'q=', 'centrum' : 'q=', 'netsprintpl' : 'q=', 'go2net' : 'general=', 'katalogonetpl' : 'qt=', 'ukindex' : 'stext=', 'shawca' : 'q=', 'szukaczpl' : 'q=', 'accoona' : 'qt=', 'live' : 'q=', 'google4counter' : '(p|q|as_p|as_q)=', 'iask' : '(w|k)=', 'earthlink' : 'q=', 'tiscali' : 'key=', 'askes' : '(ask|q)=', 'gotuneed' : '', 'clubinternet' : 'q=', 'redbox' : 'srch=', 'delicious' : 'all=', 'chellofr' : 'q1=', 'lycos' : 'query=', 'sympatico' : 'query=', 'vivisimo' : 'query=', 'bluewin' : 'qry=', 'mysearch' : 'searchfor=', 'google_cache' : '(p|q|as_p|as_q)=cache:[0-9A-Za-z]{12}:', 'ukplus' : 'search=', 'gerypl' : 'q=', 'keresolap_hu' : 'q=', 'abacho' : 'q=', 'engine' : 'p1=', 'opasia' : 'q=', 'wp' : 'szukaj=', 'steadysearch' : 'w=', 'chellopl' : 'q1=', 'voila' : '(kw|rdata)=', 'aport' : 'r=', 'internetto' : 'searchstr=', 'passagen' : 'q=', 'wwweasel' : 'q=', 'najdi' : 'dotaz=', 'alexa' : 'q=', 'baidu' : '(wd|word)=', 'spotjockey' : 'Search_Keyword=', 'virgilio' : 'qs=', 'orbis' : 'search_field=', 'tango_hu' : 'q=', 'askjp' : '(ask|q)=', 'bungeebonesdotcom' : 'query=', 'francite' : 'name=', 'searchch' : 'q=', 'google_froogle' : '(p|q|as_p|as_q)=', 'excite' : 'search=', 'infospace' : 'qkw=', 'polskapl' : 'qt=', 'swik' : 'swik\.net/', 'edderkoppen' : 'query=', 'mywebsearch' : 'searchfor=', 'danielsen' : 'q=', 'wahoo' : 'q=', 'sogou' : 'query=', 'miragonl' : '(txtsearch|qry)=', 'findarticles' : 'key='}
operating_systems = ['windows[_+ ]?2005', 'windows[_+ ]nt[_+ ]6\.0', 'windows[_+ ]?2008', 'windows[_+ ]nt[_+ ]6\.1', 'windows[_+ ]?vista', 'windows[_+ ]nt[_+ ]6', 'windows[_+ ]?2003', 'windows[_+ ]nt[_+ ]5\.2', 'windows[_+ ]xp', 'windows[_+ ]nt[_+ ]5\.1', 'windows[_+ ]me', 'win[_+ ]9x', 'windows[_+ ]?2000', 'windows[_+ ]nt[_+ ]5', 'winnt', 'windows[_+ \-]?nt', 'win32', 'win(.*)98', 'win(.*)95', 'win(.*)16', 'windows[_+ ]3', 'win(.*)ce', 'mac[_+ ]os[_+ ]x', 'mac[_+ ]?p', 'mac[_+ ]68', 'macweb', 'macintosh', 'linux(.*)android', 'linux(.*)asplinux', 'linux(.*)centos', 'linux(.*)debian', 'linux(.*)fedora', 'linux(.*)gentoo', 'linux(.*)mandr', 'linux(.*)momonga', 'linux(.*)pclinuxos', 'linux(.*)red[_+ ]hat', 'linux(.*)suse', 'linux(.*)ubuntu', 'linux(.*)vector', 'linux(.*)vine', 'linux(.*)white\sbox', 'linux(.*)zenwalk', 'linux', 'gnu.hurd', 'bsdi', 'gnu.kfreebsd', 'freebsd', 'openbsd', 'netbsd', 'dragonfly', 'aix', 'sunos', 'irix', 'osf', 'hp\-ux', 'unix', 'x11', 'gnome\-vfs', 'beos', 'os/2', 'amiga', 'atari', 'vms', 'commodore', 'qnx', 'inferno', 'palmos', 'syllable', 'blackberry', 'cp/m', 'crayos', 'dreamcast', 'iphone[_+ ]os', 'risc[_+ ]?os', 'symbian', 'webtv', 'playstation', 'xbox', 'wii', 'vienna', 'newsfire', 'applesyndication', 'akregator', 'plagger', 'syndirella', 'j2me', 'java', 'microsoft', 'msie[_+ ]', 'ms[_+ ]frontpage', 'windows']
operating_systems_hashid = {'crayos' : 'crayos', 'linux(.*)white\sbox' : 'linuxwhitebox', 'windows[_+ \-]?nt' : 'winnt', 'windows[_+ ]?2003' : 'win2003', 'mac[_+ ]?p' : 'macintosh', 'netbsd' : 'bsdnetbsd', 'win[_+ ]9x' : 'winme', 'vms' : 'vms', 'gnome\-vfs' : 'unix', 'windows[_+ ]nt[_+ ]5' : 'win2000', 'windows[_+ ]nt[_+ ]6\.0' : 'winlong', 'dragonflybsd' : 'bsddflybsd', 'wii' : 'wii', 'linux(.*)vector' : 'linuxvector', 'microsoft' : 'winunknown', 'plagger' : 'unix', 'amiga' : 'amigaos', 'windows[_+ ]me' : 'winme', 'irix' : 'irix', 'linux(.*)android' : 'linuxandroid', 'linux(.*)suse' : 'linuxsuse', 'java' : 'java', 'win(.*)ce' : 'wince', 'cp/m' : 'cp/m', 'windows[_+ ]3' : 'win16', 'win(.*)98' : 'win98', 'windows' : 'winunknown', 'os/2' : 'os/2', 'syndirella' : 'winxp', 'osf' : 'osf', 'macweb' : 'macintosh', 'linux(.*)centos' : 'linuxcentos', 'gnu.hurd' : 'gnu', 'dreamcast' : 'dreamcast', 'linux' : 'linux', 'win(.*)16' : 'win16', 'freebsd' : 'bsdfreebsd', 'windows[_+ ]xp' : 'winxp', 'blackberry' : 'blackberry', 'macintosh' : 'macintosh', 'symbian' : 'symbian', 'linux(.*)debian' : 'linuxdebian', 'windows[_+ ]?2005' : 'winlong', 'linux(.*)red[_+ ]hat' : 'linuxredhat', 'x11' : 'unix', 'windows[_+ ]nt[_+ ]5\.2' : 'win2003', 'j2me' : 'j2me', 'sunos' : 'sunos', 'linux(.*)vine' : 'linuxvine', 'mac[_+ ]os[_+ ]x' : 'macosx', 'unix' : 'unix', 'windows[_+ ]?2000' : 'win2000', 'inferno' : 'inferno', 'aix' : 'aix', 'akregator' : 'linux', 'atari' : 'atari', 'linux(.*)asplinux' : 'linuxasplinux', 'linux(.*)ubuntu' : 'linuxubuntu', 'win(.*)95' : 'win95', 'xbox' : 'winxbox', 'applesyndication' : 'macosx', 'risc[_+ ]?os' : 'riscos', 'playstation' : 'psp', 'winnt' : 'winnt', 'windows[_+ ]nt[_+ ]5\.1' : 'winxp', 'palmos' : 'palmos', 'windows[_+ ]nt[_+ ]6\.1' : 'win7', 'syllable' : 'syllable', 'commodore' : 'commodore', 'vienna' : 'macosx', 'linux(.*)gentoo' : 'linuxgentoo', 'hp\-ux' : 'hp\-ux', 'linux(.*)fedora' : 'linuxfedora', 'linux(.*)pclinuxos' : 'linuxpclinuxos', 'openbsd' : 'bsdopenbsd', 'windows[_+ ]nt[_+ ]6' : 'winvista', 'mac[_+ ]68' : 'macintosh', 'windows[_+ ]?vista' : 'winvista', 'newsfire' : 'macosx', 'windows[_+ ]?2008' : 'win2008', 'linux(.*)mandr' : 'linuxmandr', 'gnu.kfreebsd' : 'bsdkfreebsd', 'bsdi' : 'bsdi', 'win32' : 'winnt', 'webtv' : 'webtv', 'linux(.*)momonga' : 'linuxmomonga', 'msie[_+ ]' : 'winunknown', 'qnx' : 'qnx', 'iphone[_+ ]os' : 'ios', 'linux(.*)zenwalk' : 'linuxzenwalk', 'beos' : 'beos', 'ms[_+ ]frontpage' : 'winunknown'}
operating_systems_family = {'mac' : 'Macintosh', 'linux' : 'Linux', 'bsd' : 'BSD', 'win' : 'Windows'}
browsers = ['elinks', 'firebird', 'go!zilla', 'icab', 'links', 'lynx', 'omniweb', '22acidownload', 'abrowse', 'aol\-iweng', 'amaya', 'amigavoyager', 'arora', 'aweb', 'charon', 'donzilla', 'seamonkey', 'flock', 'minefield', 'bonecho', 'granparadiso', 'songbird', 'strata', 'sylera', 'kazehakase', 'prism', 'icecat', 'iceape', 'iceweasel', 'w3clinemode', 'bpftp', 'camino', 'chimera', 'cyberdog', 'dillo', 'xchaos_arachne', 'doris', 'dreamcast', 'xbox', 'downloadagent', 'ecatch', 'emailsiphon', 'encompass', 'epiphany', 'friendlyspider', 'fresco', 'galeon', 'flashget', 'freshdownload', 'getright', 'leechget', 'netants', 'headdump', 'hotjava', 'ibrowse', 'intergo', 'k\-meleon', 'k\-ninja', 'linemodebrowser', 'lotus\-notes', 'macweb', 'multizilla', 'ncsa_mosaic', 'netcaptor', 'netpositive', 'nutscrape', 'msfrontpageexpress', 'contiki', 'emacs\-w3', 'phoenix', 'shiira', 'tzgeturl', 'viking', 'webfetcher', 'webexplorer', 'webmirror', 'webvcr', 'qnx\svoyager', 'teleport', 'webcapture', 'webcopier', 'real', 'winamp', 'windows\-media\-player', 'audion', 'freeamp', 'itunes', 'jetaudio', 'mint_audio', 'mpg123', 'mplayer', 'nsplayer', 'qts', 'quicktime', 'sonique', 'uplayer', 'xaudio', 'xine', 'xmms', 'gstreamer', 'abilon', 'aggrevator', 'aiderss', 'akregator', 'applesyndication', 'betanews_reader', 'blogbridge', 'cyndicate', 'feeddemon', 'feedreader', 'feedtools', 'greatnews', 'gregarius', 'hatena_rss', 'jetbrains_omea', 'liferea', 'netnewswire', 'newsfire', 'newsgator', 'newzcrawler', 'plagger', 'pluck', 'potu', 'pubsub\-rss\-reader', 'pulpfiction', 'rssbandit', 'rssreader', 'rssowl', 'rss\sxpress', 'rssxpress', 'sage', 'sharpreader', 'shrook', 'straw', 'syndirella', 'vienna', 'wizz\srss\snews\sreader', 'alcatel', 'lg\-', 'mot\-', 'nokia', 'panasonic', 'philips', 'sagem', 'samsung', 'sie\-', 'sec\-', 'sonyericsson', 'ericsson', 'mmef', 'mspie', 'vodafone', 'wapalizer', 'wapsilon', 'wap', 'webcollage', 'up\.', 'android', 'blackberry', 'cnf2', 'docomo', 'ipcheck', 'iphone', 'portalmmm', 'webtv', 'democracy', 'cjb\.net', 'ossproxy', 'smallproxy', 'adobeair', 'apt', 'analogx_proxy', 'gnome\-vfs', 'neon', 'curl', 'csscheck', 'httrack', 'fdm', 'javaws', 'wget', 'fget', 'chilkat', 'webdownloader\sfor\sx', 'w3m', 'wdg_validator', 'w3c_validator', 'jigsaw', 'webreaper', 'webzip', 'staroffice', 'gnus', 'nikto', 'download\smaster', 'microsoft\-webdav\-miniredir', 'microsoft\sdata\saccess\sinternet\spublishing\sprovider\scache\smanager', 'microsoft\sdata\saccess\sinternet\spublishing\sprovider\sdav', 'POE\-Component\-Client\-HTTP', 'mozilla', 'libwww', 'lwp', 'WebSec']
browsers_hashid = {'jetbrains_omea' : 'Omea (RSS Reader)', 'aol\-iweng' : 'AOL-Iweng', 'webcapture' : 'Acrobat Webcapture', 'winamp' : 'WinAmp (media player)', 'chrome' : 'Google Chrome', 'analogx_proxy' : 'AnalogX Proxy', 'sylera' : 'Sylera', 'rss\sxpress' : 'RSS Xpress (RSS Reader)', 'xchaos_arachne' : 'Arachne', 'mspie' : 'MS Pocket Internet Explorer (PDA/Phone browser)', 'lynx' : 'Lynx', 'alcatel' : 'Alcatel Browser (PDA/Phone browser)', 'emailsiphon' : 'EmailSiphon', 'POE\-Component\-Client\-HTTP' : 'HTTP user-agent for POE (portable networking framework for Perl)', 'javaws' : 'Java Web Start', 'ecatch' : 'eCatch', 'aggrevator' : 'Aggrevator (RSS Reader)', 'mmef' : 'Microsoft Mobile Explorer (PDA/Phone browser)', 'qts' : 'QuickTime (media player)', 'linemodebrowser' : 'W3C Line Mode Browser', 'wizz\srss\snews\sreader' : 'Wizz RSS News Reader (RSS Reader)', 'xine' : 'Xine, a free multimedia player (media player)', 'gnome\-vfs' : 'Gnome FileSystem Abstraction library', 'flock' : 'Flock', 'audion' : 'Audion (media player)', 'icecat' : 'GNU IceCat', 'webfetcher' : 'WebFetcher', 'flashget' : 'FlashGet', 'docomo' : 'I-Mode phone (PDA/Phone browser)', 'friendlyspider' : 'FriendlySpider', 'wapalizer' : 'WAPalizer (PDA/Phone browser)', 'ipcheck' : 'Supervision IP Check (phone)', 'wapsilon' : 'WAPsilon (PDA/Phone browser)', 'svn' : 'Subversion client', 'lwp' : 'LibWWW-perl', 'plagger' : 'Plagger (RSS Reader)', 'shrook' : 'Shrook (RSS Reader)', 'mplayer' : 'The Movie Player (media player)', 'nsplayer' : 'NetShow Player (media player)', 'mint_audio' : 'Mint Audio (media player)', 'fget' : 'FGet', 'panasonic' : 'Panasonic Browser (PDA/Phone browser)', 'rssreader' : 'RssReader (RSS Reader)', 'download\smaster' : 'Download Master', 'itunes' : 'Apple iTunes (media player)', 'arora' : 'Arora', 'contiki' : 'Contiki', 'mot\-' : 'Motorola Browser (PDA/Phone browser)', 'nutscrape' : 'Nutscrape', 'fdm' : '<a href="http://www.freedownloadmanager.org/" title="Browser home page [new window]" target="_blank">FDM Free Download Manager</a>', 'prism' : 'Prism', 'safari' : 'Safari', 'encompass' : 'Encompass', 'feedreader' : 'FeedReader (RSS Reader)', 'newsgator' : 'NewsGator (RSS Reader)', 'microsoft\-webdav\-miniredir' : 'Microsoft Data Access Component Internet Publishing Provider', 'android' : 'Android browser (PDA/Phone browser)', 'strata' : 'Strata', 'teleport' : 'TelePort Pro', 'songbird' : 'Songbird', 'syndirella' : 'Syndirella (RSS Reader)', 'epiphany' : 'Epiphany', 'minefield' : 'Minefield (Firefox 3.0 development)', 'ncsa_mosaic' : 'NCSA Mosaic', 'links' : 'Links', 'macweb' : 'MacWeb', 'iphone' : 'IPhone (PDA/Phone browser)', 'cjb\.net' : '<a href="http://proxy.cjb.net/" title="Browser home page [new window]" target="_blank">CJB.NET Proxy</a>', 'bpftp' : 'BPFTP', 'netcaptor' : 'NetCaptor', 'microsoft\sdata\saccess\sinternet\spublishing\sprovider\sdav' : 'Microsoft Data Access Component Internet Publishing Provider DAV', 'dreamcast' : 'Dreamcast', 'straw' : 'Straw (RSS Reader)', 'windows\-media\-player' : 'Windows Media Player (media player)', 'philips' : 'Philips Browser (PDA/Phone browser)', 'netpositive' : 'NetPositive', 'doris' : 'Doris (for Symbian)', 'gstreamer' : 'GStreamer (media library)', 'intergo' : 'InterGO', 'shiira' : 'Shiira', 'gregarius' : '<a href="http://devlog.gregarius.net/docs/ua" title="Browser home page [new window]" target="_blank">Gregarius (RSS Reader)</a>', 'potu' : '<a href="http://www.potu.com/" title="Potu Rss-Reader home page [new window]" target="_blank">Potu (RSS Reader)</a>', 'blackberry' : 'BlackBerry (PDA/Phone browser)', 'smallproxy' : '<a href="http://www.smallproxy.ru/" title="SmallProxy home page [new window]" target="_blank">SmallProxy</a>', 'galeon' : 'Galeon', 'iceweasel' : 'Iceweasel', 'leechget' : 'LeechGet', 'opera' : 'Opera', 'pubsub\-rss\-reader' : '<a href="http://www.pubsub.com/" title="Browser home page [new window]" target="_blank">PubSub (RSS Reader)</a>', 'vodafone' : 'Vodaphone browser (PDA/Phone browser)', 'rssbandit' : 'RSS Bandit (RSS Reader)', 'samsung' : 'Samsung (PDA/Phone browser)', 'charon' : 'Charon', 'democracy' : 'Democracy', 'freshdownload' : 'FreshDownload', 'freeamp' : 'FreeAmp (media player)', 'nokia' : 'Nokia Browser (PDA/Phone browser)', 'elinks' : 'ELinks', 'multizilla' : 'MultiZilla', 'ericsson' : 'Ericsson Browser (PDA/Phone browser)', 'nikto' : 'Nikto Web Scanner', 'mpg123' : 'mpg123 (media player)', 'gnus' : 'Gnus Network User Services', 'firefox' : 'Firefox', 'msie' : 'MS Internet Explorer', 'betanews_reader' : 'Betanews Reader (RSS Reader)', 'akregator' : '<a href="http://akregator.sourceforge.net/" title="Browser home page [new window]" target="_blank">Akregator (RSS Reader)</a>', 'hatena_rss' : '<a href="http://r.hatena.ne.jp/" title="Browser home page [new window]" target="_blank">Hatena (RSS Reader)</a>', 'iceape' : 'GNU IceApe', 'viking' : 'Viking', 'k\-ninja' : 'K-Ninja', 'ibrowse' : 'iBrowse', 'sonyericsson' : 'Sony/Ericsson Browser (PDA/Phone browser)', 'portalmmm' : 'I-Mode phone (PDA/Phone browser)', 'apt' : 'Debian APT', 'curl' : 'Curl', 'xbox' : 'XBoX', 'aweb' : 'AWeb', 'WebSec' : '<a href="http://baruch.ev-en.org/proj/websec/" title="Browser home page [new window]" target="_blank">Web Secretary</a>', 'applesyndication' : '<a href="http://www.apple.com/macosx/features/safari/" title="Browser home page [new window]" target="_blank">AppleSyndication (RSS Reader)</a>', 'qnx\svoyager' : 'QNX Voyager', 'netnewswire' : 'NetNewsWire (RSS Reader)', 'cnf2' : 'Supervision I-Mode ByTel (phone)', 'microsoft\sdata\saccess\sinternet\spublishing\sprovider\scache\smanager' : 'Microsoft Data Access Component Internet Publishing Provider Cache Manager', 'go!zilla' : 'Go!Zilla', 'cyndicate' : 'Cyndicate (RSS Reader)', 'wget' : 'Wget', 'jetaudio' : 'JetAudio (media player)', 'sharpreader' : 'SharpReader (RSS Reader)', 'w3c_validator' : 'W3C Validator', 'netscape' : 'Netscape', 'webcollage' : 'WebCollage (PDA/Phone browser)', 'feeddemon' : 'FeedDemon (RSS Reader)', 'wap' : 'Unknown WAP browser (PDA/Phone browser)', 'aiderss' : 'AideRSS (RSS Reader)', 'lg\-' : 'LG (PDA/Phone browser)', 'webzip' : 'WebZIP', 'pulpfiction' : 'PulpFiction (RSS Reader)', 'webreaper' : 'WebReaper', 'k\-meleon' : 'K-Meleon', 'pluck' : 'Pluck (RSS Reader)', 'msfrontpageexpress' : 'MS FrontPage Express', 'fresco' : 'ANT Fresco', 'httrack' : 'HTTrack', 'real' : 'Real player or compatible (media player)', 'quicktime' : 'QuickTime (media player)', 'konqueror' : 'Konqueror', 'jigsaw' : 'W3C Validator', 'sie\-' : 'SIE (PDA/Phone browser)', 'mozilla' : 'Mozilla', '22acidownload' : '22AciDownload', 'netants' : 'NetAnts', 'csscheck' : 'WDG CSS Validator', 'newzcrawler' : 'NewzCrawler (RSS Reader)', 'sagem' : 'Sagem (PDA/Phone browser)', 'xmms' : 'XMMS (media player)', 'rssxpress' : 'RSSXpress (RSS Reader)', 'wdg_validator' : 'WDG HTML Validator', 'amigavoyager' : 'AmigaVoyager', 'vienna' : '<a href="http://www.vienna-rss.org/" title="Vienna RSS-Reader [new window]" target="_blank">Vienna (RSS Reader)</a>', 'feedtools' : '<a href="http://sporkmonger.com/projects/feedtools/" title="Browser home page [new window]" target="_blank">FeedTools (RSS Reader)</a>', 'camino' : 'Camino', 'blogbridge' : '<a href="http://www.blogbridge.com/" title="Browser home page [new window]" target="_blank">BlogBridge (RSS Reader)</a>', 'bonecho' : 'BonEcho (Firefox 2.0 development)', 'granparadiso' : 'GranParadiso (Firefox 3.0 development)', 'hotjava' : 'Sun HotJava', 'up\.' : 'UP.Browser (PDA/Phone browser)', 'w3m' : 'w3m', 'dillo' : 'Dillo', 'liferea' : '<a href="http://liferea.sourceforge.net/" title="Browser home page [new window]" target="_blank">Liferea (RSS Reader)</a>', 'getright' : 'GetRight', 'kazehakase' : 'Kazehakase', 'lotus\-notes' : 'Lotus Notes web client', 'tzgeturl' : 'TzGetURL', 'sage' : 'Sage (RSS Reader)', 'webcopier' : 'WebCopier', 'phoenix' : 'Phoenix', 'abrowse' : 'ABrowse', 'xaudio' : 'Some XAudio Engine based MPEG player (media player)', 'sec\-' : 'Sony/Ericsson (PDA/Phone browser)', 'w3clinemode' : 'W3CLineMode', 'chimera' : 'Chimera (Old Camino)', 'headdump' : 'HeadDump', 'abilon' : 'Abilon (RSS Reader)', 'downloadagent' : 'DownloadAgent', 'cyberdog' : 'Cyberdog', 'rssowl' : 'RSSOwl (RSS Reader)', 'newsfire' : 'NewsFire (RSS Reader)', 'webdownloader\sfor\sx' : 'Downloader for X', 'sonique' : 'Sonique (media player)', 'webmirror' : 'WebMirror', 'webexplorer' : 'IBM-WebExplorer', 'chilkat' : 'Chilkat', 'ossproxy' : '<a href="http://www.marketscore.com/FAQ.Aspx" title="OSSProxy home page [new window]" target="_blank">OSSProxy</a>', 'libwww' : 'LibWWW', 'adobeair' : 'AdobeAir', 'uplayer' : 'Ultra Player (media player)', 'amaya' : 'Amaya', 'webtv' : 'WebTV browser', 'neon' : 'Neon HTTP and WebDAV client library', 'greatnews' : '<a href="http://www.curiostudio.com/" title="Browser home page [new window]" target="_blank">GreatNews (RSS Reader)</a>', 'seamonkey' : 'SeaMonkey', 'omniweb' : 'OmniWeb', 'donzilla' : 'Donzilla', 'webvcr' : 'WebVCR', 'icab' : 'iCab', 'firebird' : 'Firebird (Old Firefox)', 'staroffice' : 'StarOffice', 'emacs\-w3' : 'Emacs/w3s'}
browsers_icons = {'jetbrains_omea' : 'rss', 'webcapture' : 'adobe', 'winamp' : 'mediaplayer', 'chrome' : 'chrome', 'analogx_proxy' : 'analogx', 'sylera' : 'mozilla', 'rss\sxpress' : 'rss', 'mspie' : 'pdaphone', 'lynx' : 'lynx', 'alcatel' : 'pdaphone', 'javaws' : 'java', 'ecatch' : 'ecatch', 'aggrevator' : 'rss', 'mmef' : 'pdaphone', 'qts' : 'mediaplayer', 'wizz\srss\snews\sreader' : 'wizz', 'xine' : 'mediaplayer', 'gnome\-vfs' : 'gnome', 'flock' : 'flock', 'audion' : 'mediaplayer', 'icecat' : 'icecat', 'flashget' : 'flashget', 'docomo' : 'pdaphone', 'avantbrowser' : 'avant', 'wapalizer' : 'pdaphone', 'wapsilon' : 'pdaphone', 'svn' : 'subversion', 'plagger' : 'rss', 'shrook' : 'rss', 'mplayer' : 'mediaplayer', 'nsplayer' : 'netshow', 'mint_audio' : 'mediaplayer', 'panasonic' : 'pdaphone', 'rssreader' : 'rss', 'itunes' : 'mediaplayer', 'microsoft\soffice\sprotocol\sdiscovery' : 'frontpage', 'mot\-' : 'pdaphone', 'prism' : 'mozilla', 'safari' : 'safari', 'encompass' : 'encompass', 'feedreader' : 'rss', 'newsgator' : 'rss', 'microsoft\-webdav\-miniredir' : 'frontpage', 'android' : 'android', 'teleport' : 'teleport', 'strata' : 'mozilla', 'songbird' : 'mozilla', 'syndirella' : 'rss', 'epiphany' : 'epiphany', 'minefield' : 'firefox', 'ncsa_mosaic' : 'ncsa_mosaic', 'macweb' : 'macweb', 'iphone' : 'pdaphone', 'cjb\.net' : 'cjbnet', 'bpftp' : 'bpftp', 'microsoft\sdata\saccess\sinternet\spublishing\sprovider\sdav' : 'frontpage', 'dreamcast' : 'dreamcast', 'straw' : 'rss', 'windows\-media\-player' : 'mplayer', 'philips' : 'pdaphone', 'netpositive' : 'netpositive', 'doris' : 'doris', 'gregarius' : 'rss', 'potu' : 'rss', 'blackberry' : 'pdaphone', 'galeon' : 'galeon', 'iceweasel' : 'iceweasel', 'leechget' : 'leechget', 'opera' : 'opera', 'pubsub\-rss\-reader' : 'rss', 'vodafone' : 'pdaphone', 'rssbandit' : 'rss', 'samsung' : 'pdaphone', 'freshdownload' : 'freshdownload', 'freeamp' : 'mediaplayer', 'nokia' : 'pdaphone', 'multizilla' : 'multizilla', 'ericsson' : 'pdaphone', 'mpg123' : 'mediaplayer', 'gnus' : 'gnus', 'firefox' : 'firefox', 'msie' : 'msie', 'betanews_reader' : 'rss', 'akregator' : 'rss', 'hatena_rss' : 'rss', 'iceape' : 'mozilla', 'ibrowse' : 'ibrowse', 'sonyericsson' : 'pdaphone', 'portalmmm' : 'pdaphone', 'apt' : 'apt', 'xbox' : 'winxbox', 'aweb' : 'aweb', 'applesyndication' : 'rss', 'netnewswire' : 'rss', 'microsoft\sdata\saccess\sinternet\spublishing\sprovider\scache\smanager' : 'frontpage', 'go!zilla' : 'gozilla', 'microsoft\sdata\saccess\sinternet\spublishing\sprovider\sprotocol\sdiscovery' : 'frontpage', 'jetaudio' : 'mediaplayer', 'sharpreader' : 'rss', 'netscape' : 'netscape', 'webcollage' : 'pdaphone', 'feeddemon' : 'rss', 'wap' : 'pdaphone', 'aiderss' : 'rss', 'lg\-' : 'pdaphone', 'webzip' : 'webzip', 'pulpfiction' : 'rss', 'webreaper' : 'webreaper', 'pluck' : 'rss', 'k\-meleon' : 'kmeleon', 'msfrontpageexpress' : 'fpexpress', 'fresco' : 'fresco', 'httrack' : 'httrack', 'real' : 'real', 'konqueror' : 'konqueror', 'sie\-' : 'pdaphone', 'mozilla' : 'mozilla', 'sagem' : 'pdaphone', 'newzcrawler' : 'rss', 'rssxpress' : 'rss', 'xmms' : 'mediaplayer', 'vienna' : 'rss', 'amigavoyager' : 'amigavoyager', 'feedtools' : 'rss', 'camino' : 'chimera', 'blogbridge' : 'rss', 'bonecho' : 'firefox', 'granparadiso' : 'firefox', 'hotjava' : 'hotjava', 'up\.' : 'pdaphone', 'dillo' : 'dillo', 'liferea' : 'rss', 'getright' : 'getright', 'kazehakase' : 'mozilla', 'lotus\-notes' : 'lotusnotes', 'sage' : 'rss', 'webcopier' : 'webcopier', 'phoenix' : 'phoenix', 'sec\-' : 'pdaphone', 'xaudio' : 'mediaplayer', 'microsoft\soffice\sexistence\sdiscovery' : 'frontpage', 'chimera' : 'chimera', 'abilon' : 'abilon', 'rssowl' : 'rss', 'cyberdog' : 'cyberdog', 'newsfire' : 'rss', 'sonique' : 'mediaplayer', 'adobeair' : 'adobe', 'uplayer' : 'mediaplayer', 'amaya' : 'amaya', 'webtv' : 'webtv', 'neon' : 'neon', 'greatnews' : 'rss', 'seamonkey' : 'seamonkey', 'omniweb' : 'omniweb', 'donzilla' : 'mozilla', 'icab' : 'icab', 'firebird' : 'phoenix', 'staroffice' : 'staroffice'}
conf.py
1010
1111
1212
13
14
13
14
1515
1616
1717
# Hooks used
pre_analysis_hooks = ['page_to_hit', 'robots']
post_analysis_hooks = ['referers', 'top_pages', 'top_downloads', 'reverse_dns']
display_hooks = ['top_visitors', 'all_visits', 'referers', 'top_pages', 'top_downloads', 'referers_diff']
post_analysis_hooks = ['referers', 'top_pages', 'top_downloads', 'operating_systems', 'browsers'] #, 'reverse_dns']
display_hooks = ['top_visitors', 'all_visits', 'referers', 'top_pages', 'top_downloads', 'referers_diff', 'operating_systems', 'browsers']
# Reverse DNS timeout
reverse_dns_timeout = 0.2
plugins/display/operating_systems.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
# -*- coding: utf-8 -*-
#
# Copyright Grégory Soutadé 2015
# This file is part of iwla
# iwla is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# iwla is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with iwla. If not, see <http://www.gnu.org/licenses/>.
#
from iwla import IWLA
from iplugin import IPlugin
from display import *
import awstats_data
"""
Display hook
Add operating systems statistics
Plugin requirements :
post_analysis/operating_systems
Conf values needed :
create_families_page*
Output files :
OUTPUT_ROOT/year/month/index.html
Statistics creation :
None
Statistics update :
None
Statistics deletion :
None
"""
class IWLADisplayTopOperatingSystems(IPlugin):
def __init__(self, iwla):
super(IWLADisplayTopOperatingSystems, self).__init__(iwla)
self.API_VERSION = 1
self.requires = ['IWLAPostAnalysisOperatingSystems']
def load(self):
self.icon_path = self.iwla.getConfValue('icon_path', '/')
self.create_families_pages = self.iwla.getConfValue('create_families_pages_page', True)
self.icon_names = {v:k for (k, v) in awstats_data.operating_systems_family.items()}
return True
def hook(self):
display = self.iwla.getDisplay()
os_families = self.iwla.getMonthStats()['os_families']
os_families = sorted(os_families.items(), key=lambda t: t[1], reverse=True)
operating_systems = self.iwla.getMonthStats()['operating_systems']
operating_systems = sorted(operating_systems.items(), key=lambda t: t[1], reverse=True)
# All in a page
if self.create_families_pages:
title = createCurTitle(self.iwla, u'All Operating Systems')
filename = 'operating_systems.html'
path = self.iwla.getCurDisplayPath(filename)
page = display.createPage(title, path, self.iwla.getConfValue('css_path', []))
table = display.createBlock(DisplayHTMLBlockTable, self.iwla._(u'Operating Systems'), ['', self.iwla._(u'Operating System'), self.iwla._(u'Entrance')])
table.setColsCSSClass(['', '', 'iwla_hit'])
for (os_name, entrance) in operating_systems:
icon = '<img src="/%s/os/%s.png"/>' % (self.icon_path, os_name)
table.appendRow([icon, os_name, entrance])
page.appendBlock(table)
display.addPage(page)
# Families in index
title = self.iwla._(u'Operating Systems')
if self.create_families_pages:
link = '<a href=\'%s\'>%s</a>' % (filename, self.iwla._(u'Details'))
title = '%s - %s' % (title, link)
index = self.iwla.getDisplayIndex()
table = display.createBlock(DisplayHTMLBlockTable, title, ['', self.iwla._(u'Operating System'), self.iwla._(u'Entrance')])
table.setColsCSSClass(['', '', 'iwla_hit'])
for (family, entrance) in os_families:
icon = '<img src="/%s/os/%s.png"/>' % (self.icon_path, self.icon_names[family])
table.appendRow([icon, family, entrance])
index.appendBlock(table)
plugins/post_analysis/operating_systems.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
# -*- coding: utf-8 -*-
#
# Copyright Grégory Soutadé 2015
# This file is part of iwla
# iwla is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# iwla is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with iwla. If not, see <http://www.gnu.org/licenses/>.
#
import re
from iwla import IWLA
from iplugin import IPlugin
import awstats_data
"""
Post analysis hook
Detect operating systems from requests
Plugin requirements :
None
Conf values needed :
None
Output files :
None
Statistics creation :
visits :
remote_addr =>
operating_system
month_stats :
operating_systems =>
operating_system => count
os_families =>
family => count
Statistics update :
None
Statistics deletion :
None
"""
class IWLAPostAnalysisOperatingSystems(IPlugin):
def __init__(self, iwla):
super(IWLAPostAnalysisOperatingSystems, self).__init__(iwla)
self.API_VERSION = 1
def load(self):
self.operating_systems = []
self.os_family = {}
for hashid in awstats_data.operating_systems:
hashid_re = re.compile(r'.*%s.*' % (hashid), re.IGNORECASE)
if hashid in awstats_data.operating_systems_hashid.keys():
self.operating_systems.append((hashid_re, awstats_data.operating_systems_hashid[hashid]))
for (name, family) in awstats_data.operating_systems_family.items():
name_re = re.compile(r'.*%s.*' % (name))
self.os_family[name_re] = family
return True
def hook(self):
stats = self.iwla.getValidVisitors()
month_stats = self.iwla.getMonthStats()
operating_systems = month_stats.get('operating_systems', {})
os_stats = {}
family_stats = {}
for (k, super_hit) in stats.items():
if not 'operating_system' in super_hit:
for r in super_hit['requests'][::-1]:
user_agent = r['http_user_agent']
if not user_agent: continue
os_name = 'unknown'
for (hashid_re, operating_system) in self.operating_systems:
if hashid_re.match(user_agent):
os_name = operating_system
break
super_hit['operating_system'] = os_name
break
else:
os_name = super_hit['operating_system']
os_family = ''
if os_name != 'unknown':
for (name_re, family) in self.os_family.items():
if name_re.match(os_name):
os_family = family
break
if not os_name in os_stats.keys():
os_stats[os_name] = 1
else:
os_stats[os_name] += 1
if os_family:
if not os_family in family_stats.keys():
family_stats[os_family] = 1
else:
family_stats[os_family] += 1
month_stats['operating_systems'] = os_stats
month_stats['os_families'] = family_stats
tools/iwla_convert.pl
11
22
3
4
3
4
5
56
6
77
88
99
......
5151
5252
5353
54
55
5456
5557
5658
......
7678
7779
7880
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
79105
#!/usr/bin/perl
my $awstats_lib_root = './';
my @awstats_libs = ('search_engines.pm', 'robots.pm');
my $awstats_lib_root = '/usr/share/awstats/lib/';
# my $awstats_lib_root = './';
my @awstats_libs = ('search_engines.pm', 'robots.pm', 'operating_systems.pm', 'browsers.pm');
# my $awstats_lib_root = '/usr/share/awstats/lib/';
# my @awstats_libs = ('browsers.pm', 'browsers_phone.pm', 'mime.pm', 'referer_spam.pm', 'search_engines.pm', 'operating_systems.pm', 'robots.pm', 'worms.pm');
foreach $lib (@awstats_libs) {require $awstats_lib_root . $lib;}
# Robots
open($FIC,">", "awstats_data.py") or die $!;
print $FIC "#This file was automatically generated by iwla_convert.pl. Do not edit manually.\n\n";
print $FIC "robots = [";
dumpList(\@RobotsSearchIDOrder_list1, $FIC, 1);
dumpList(\@RobotsSearchIDOrder_list2, $FIC, 0);
dumpHash(\%SearchEnginesKnownUrl, $FIC, 1);
print $FIC "}\n\n";
print $FIC "operating_systems = [";
dumpList(\@OSSearchIDOrder, $FIC, 1);
print $FIC "]\n\n";
print $FIC "operating_systems_hashid = {";
dumpHash(\%OSHashID, $FIC, 1);
print $FIC "}\n\n";
print $FIC "operating_systems_family = {";
dumpHash(\%OSFamily, $FIC, 1);
print $FIC "}\n\n";
print $FIC "browsers = [";
dumpList(\@BrowsersSearchIDOrder, $FIC, 1);
print $FIC "]\n\n";
print $FIC "browsers_hashid = {";
dumpHash(\%BrowsersHashIDLib, $FIC, 1);
print $FIC "}\n\n";
print $FIC "browsers_icons = {";
dumpHash(\%BrowsersHashIcon, $FIC, 1);
print $FIC "}\n\n";
close($FIC);

Archive Download the corresponding diff file

Branches

Tags