stuartl/hackaday.io-spambot-hunter
Archived
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

crawler: Skip deferred user scanning if none due.

Browse Source
This commit is contained in:
Stuart Longland 2018-03-03 11:47:55 +10:00
parent 4e42937224
commit cc0eef4f93
Signed by: stuartl
GPG Key ID: 6AA32EFB18079BAA
1 changed files with 16 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
hadsh/crawler/crawler.py
View File

@ -596,21 +596,23 @@ class Crawler(object):
DeferredUser.inspect_time <
datetime.datetime.now(tz=pytz.utc)).order_by(
DeferredUser.inspect_time).limit(50).all()]
self._log.debug('Scanning %s', ids)
user_data = yield self._api.get_users(ids=ids, per_page=50)
self._log.debug('Received: %s', user_data)
if isinstance(user_data['users'], list):
for this_user_data in user_data['users']:
while True:
try:
yield self.update_user_from_data(
this_user_data, inspect_all=True)
break
except InvalidUser:
pass
except InvalidRequestError:
self._db.rollback()
if ids:
self._log.debug('Scanning %s', ids)
user_data = yield self._api.get_users(ids=ids, per_page=50)
self._log.debug('Received: %s', user_data)
if isinstance(user_data['users'], list):
for this_user_data in user_data['users']:
while True:
try:
yield self.update_user_from_data(
this_user_data, inspect_all=True)
break
except InvalidUser:
pass
except InvalidRequestError:
self._db.rollback()
except:
self._log.exception('Failed to retrieve deferred users')