crawler.py 791 B

1234567891011121314151617181920212223
  1. from database import CRAWLERS_NAMES, CRAWLERS_AGENTS, CRAWLERS_HOSTS
  2. class Crawler(object):
  3. crawler = False
  4. host = None
  5. username = None
  6. def __init__(self, agent = None, ip = None):
  7. if agent is not None:
  8. for item in CRAWLERS_AGENTS.keys():
  9. if agent.find(item) != -1:
  10. self.crawler = True
  11. self.username = CRAWLERS_AGENTS[item]
  12. if ip is not None:
  13. for item in CRAWLERS_HOSTS.keys():
  14. if ip == item:
  15. self.crawler = True
  16. self.username = CRAWLERS_HOSTS[item]
  17. if self.crawler:
  18. self.username = CRAWLERS_NAMES[self.username]
  19. self.host = ip