stuartl/hackaday.io-spambot-hunter
Archived
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

crawler: Periodically refresh admin group members.

Browse Source
This commit is contained in:
Stuart Longland 2018-01-07 16:44:37 +10:00
parent 9ad7b7dd86
commit 6b5c4e8b6a
Signed by: stuartl
GPG Key ID: 6AA32EFB18079BAA
1 changed files with 11 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
hadsh/crawler/crawler.py
View File

@ -39,7 +39,8 @@ class Crawler(object):
self._manual_suspect = self._get_or_make_group('suspect') self._manual_suspect = self._get_or_make_group('suspect')
self._manual_legit = self._get_or_make_group('legit') self._manual_legit = self._get_or_make_group('legit')
self._io_loop.add_callback(self._refresh_admin_group) self._io_loop.add_callback(self.refresh_admin_group)
self._refresh_admin_group_timeout = None
def _get_or_make_group(self, name): def _get_or_make_group(self, name):
group = self._db.query(Group).filter( group = self._db.query(Group).filter(
@ -52,10 +53,13 @@ class Crawler(object):
return group return group
@coroutine @coroutine
def _refresh_admin_group(self): def refresh_admin_group(self):
""" """
Refresh the membership of the admin group. Refresh the membership of the admin group.
""" """
if self._refresh_admin_group_timeout is not None:
self._io_loop.remove_timeout(self._refresh_admin_group_timeout)
members_data = [] members_data = []
page = 1 page = 1
page_cnt = 1 page_cnt = 1
@ -91,6 +95,11 @@ class Crawler(object):
self._db.commit() self._db.commit()
# Schedule this to run again tomorrow.
self._refresh_admin_group_timeout = self._io_loop.add_timeout(
self._io_loop.time() + 86400.0,
self.refresh_admin_group)
def get_avatar(self, avatar_url): def get_avatar(self, avatar_url):
avatar = self._db.query(Avatar).filter( avatar = self._db.query(Avatar).filter(
Avatar.url==avatar_url).first() Avatar.url==avatar_url).first()