stuartl/hackaday.io-spambot-hunter
Archived
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

crawler: Report current page, move to next if no users.

Browse Source
This commit is contained in:
Stuart Longland 2018-01-07 11:35:29 +10:00
parent 83aa52a117
commit b09494b726
Signed by: stuartl
GPG Key ID: 6AA32EFB18079BAA
1 changed files with 9 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
hadsh/crawler/crawler.py
View File

@ -185,10 +185,13 @@ class Crawler(object):
page=page, per_page=50)
users = []
now = datetime.datetime.now(tz=pytz.utc)
for user_data in new_user_data['users']:
user = yield self.update_user_from_data(user_data, inspect_all)
if (now - user.last_update).total_seconds() > 300.0:
continue
users.append(user)
raise Return((users, new_user_data.get('last_page')))
while len(users) < 10:
for user_data in new_user_data['users']:
user = yield self.update_user_from_data(user_data, inspect_all)
if (now - user.last_update).total_seconds() > 300.0:
continue
users.append(user)
page += 1
raise Return((users, page, new_user_data.get('last_page')))