stuartl/hackaday.io-spambot-hunter
Archived
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

crawler: Fix retrieval when existing users seen.

Browse Source
This commit is contained in:
Stuart Longland 2018-01-07 11:39:31 +10:00
parent 6be5e9a297
commit 786cc1339a
Signed by: stuartl
GPG Key ID: 6AA32EFB18079BAA
1 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hadsh/crawler/crawler.py
View File

@ -181,12 +181,13 @@ class Crawler(object):
Retrieve new users from the Hackaday.io API and inspect the new arrivals.
Returns the list of users on the given page and the total number of pages.
"""
new_user_data = yield self._api.get_users(sortby=UserSortBy.newest,
page=page, per_page=50)
users = []
now = datetime.datetime.now(tz=pytz.utc)
while len(users) < 10:
new_user_data = yield self._api.get_users(sortby=UserSortBy.newest,
page=page, per_page=50)
for user_data in new_user_data['users']:
user = yield self.update_user_from_data(user_data, inspect_all)
if (now - user.last_update).total_seconds() > 300.0: