soutade
/
iwla
1
0
Fork 0
Code Issues Pull Requests Releases 2 Activity

Add "compatible" as a criteria for robot

This commit is contained in:
Gregory Soutade 2023-02-18 08:49:14 +01:00
parent a5bef4ece6
commit c8dfdd17f7
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
plugins/pre_analysis/robots.py
View File

@ -100,7 +100,8 @@ class IWLAPreAnalysisRobots(IPlugin):
first_page = super_hit['requests'][0]
if self.robot_re.match(first_page['http_user_agent']) or\
self.crawl_re.match(first_page['http_user_agent']):
self.crawl_re.match(first_page['http_user_agent']) or\
self.compatible_re.match(first_page['http_user_agent']):
self.logger.debug(first_page['http_user_agent'])
self._setRobot(k, super_hit)
continue