From 971c10b3dbb7d8fe2634339edd370f102b614d30 Mon Sep 17 00:00:00 2001 From: Rishabh Singh Date: Thu, 21 May 2020 19:09:29 +0530 Subject: [PATCH 1/7] add user agent --- hsc/crawler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hsc/crawler.py b/hsc/crawler.py index a07f6f6d..73f6b0c7 100755 --- a/hsc/crawler.py +++ b/hsc/crawler.py @@ -9,6 +9,7 @@ class Crawler: base_url = 'https://www.hackerrank.com/' + user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X x.y; rv:42.0) Gecko/20100101 Firefox/42.0' login_url = base_url + 'auth/login' submissions_url = base_url + 'rest/contests/master/submissions/?offset={}&limit={}' challenge_url = base_url + 'rest/contests/master/challenges/{}/submissions/{}' @@ -40,7 +41,7 @@ def __init__(self): self.options = {} def login(self, username, password): - resp = self.session.get(self.login_url, auth=(username, password)) + resp = self.session.get(self.login_url, auth=(username, password), headers={'user-agent': self.user_agent}) self.cookies = self.session.cookies.get_dict() self.headers = resp.request.headers self.get_number_of_submissions() From 030540d5ca4a0ae0c317b4a03d5b0094167671ad Mon Sep 17 00:00:00 2001 From: Rishabh Singh Date: Thu, 21 May 2020 19:11:04 +0530 Subject: [PATCH 2/7] remove extra whitespace --- hsc/crawler.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hsc/crawler.py b/hsc/crawler.py index 73f6b0c7..acb75418 100755 --- a/hsc/crawler.py +++ b/hsc/crawler.py @@ -129,7 +129,7 @@ def update_readmes(self, domain_name, subdomain_name, domain_url, subdomain_url, subdomain_readme_path, domain_readme_path, root_readme_path) problem_url = self.problem_url.format(challenge_slug) - + file_path_relative_to_subdomain = './' + file_name_with_extension file_path_relative_to_domain = '{}/{}'.format(subdomain_name, file_name_with_extension) file_path_relative_to_root = '{}/{}/{}'.format(domain_name, subdomain_name, file_name_with_extension) @@ -174,7 +174,7 @@ def get_submissions(self, submissions): data = resp.json()['model'] code = data['code'] track = data['track'] - + # Default should be empty file_extension = '' file_name = challenge_slug @@ -190,7 +190,7 @@ def get_submissions(self, submissions): subdomain_name = track['name'].strip().replace(' ', '') domain_slug = track['track_slug'] subdomain_slug = track['slug'] - + domain_url = self.domain_url.format(domain_slug) subdomain_url = self.subdomain_url.format(domain_slug, subdomain_slug) @@ -207,7 +207,7 @@ def get_submissions(self, submissions): if self.make_language_folder: file_path = os.path.join(self.base_folder_name, domain_name, subdomain_name, language, file_name_with_extension) self.store_submission(file_path, code) - + self.update_readmes(domain_name, subdomain_name, domain_url, subdomain_url, challenge_name, challenge_slug, language, file_name_with_extension) From 745ba33671ad08c4daf9d98c518b10a7cb669917 Mon Sep 17 00:00:00 2001 From: Rishabh Singh Date: Thu, 21 May 2020 19:16:33 +0530 Subject: [PATCH 3/7] use post request --- hsc/crawler.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/hsc/crawler.py b/hsc/crawler.py index acb75418..e20d9eac 100755 --- a/hsc/crawler.py +++ b/hsc/crawler.py @@ -2,9 +2,9 @@ import requests import getpass import configargparse -from .progress_bar import CustomProgress -from .metadata import Metadata -from .constants import extensions +from hsc.progress_bar import CustomProgress +from hsc.metadata import Metadata +from hsc.constants import extensions class Crawler: @@ -41,11 +41,13 @@ def __init__(self): self.options = {} def login(self, username, password): - resp = self.session.get(self.login_url, auth=(username, password), headers={'user-agent': self.user_agent}) - self.cookies = self.session.cookies.get_dict() - self.headers = resp.request.headers - self.get_number_of_submissions() - return self.total_submissions != 0 + resp = self.session.post(self.login_url, auth=(username, password), headers={'user-agent': self.user_agent}) + data = resp.json() + if data['status']: + self.cookies = self.session.cookies.get_dict() + self.headers = resp.request.headers + self.get_number_of_submissions() + return data['status'] def parse_script(self): p = configargparse.ArgParser(default_config_files=['./user.yaml']) From 78773dfbe7c361642c7c9d7128b9088af02206a8 Mon Sep 17 00:00:00 2001 From: Rishabh Singh Date: Thu, 21 May 2020 21:14:19 +0530 Subject: [PATCH 4/7] use main --- hsc/__init__.py | 1 - hsc/__main__.py | 4 ++++ hsc/crawler.py | 6 +++--- setup.py | 5 ----- 4 files changed, 7 insertions(+), 9 deletions(-) create mode 100644 hsc/__main__.py diff --git a/hsc/__init__.py b/hsc/__init__.py index 287696aa..e69de29b 100644 --- a/hsc/__init__.py +++ b/hsc/__init__.py @@ -1 +0,0 @@ -from .crawler import Crawler diff --git a/hsc/__main__.py b/hsc/__main__.py new file mode 100644 index 00000000..cb023657 --- /dev/null +++ b/hsc/__main__.py @@ -0,0 +1,4 @@ +from hsc.crawler import main + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/hsc/crawler.py b/hsc/crawler.py index e20d9eac..369fb7bd 100755 --- a/hsc/crawler.py +++ b/hsc/crawler.py @@ -2,9 +2,9 @@ import requests import getpass import configargparse -from hsc.progress_bar import CustomProgress -from hsc.metadata import Metadata -from hsc.constants import extensions +from .progress_bar import CustomProgress +from .metadata import Metadata +from .constants import extensions class Crawler: diff --git a/setup.py b/setup.py index c144e9a4..843d2407 100644 --- a/setup.py +++ b/setup.py @@ -19,10 +19,5 @@ "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", ], - entry_points={ - 'console_scripts': [ - 'hsc=hsc.crawler:main', - ], - }, install_requires=['progress', 'requests', 'configargparse'] ) From 0c046215d1b7f4ac4015df9ba4febabce7677bf8 Mon Sep 17 00:00:00 2001 From: rishabhsingh971 Date: Thu, 21 May 2020 21:42:40 +0530 Subject: [PATCH 5/7] reexpose crawler class --- hsc/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hsc/__init__.py b/hsc/__init__.py index e69de29b..59728e39 100644 --- a/hsc/__init__.py +++ b/hsc/__init__.py @@ -0,0 +1 @@ +from .crawler import Crawler \ No newline at end of file From b3b429e108f4db70934019305179e2c2ebbc1622 Mon Sep 17 00:00:00 2001 From: rishabhsingh971 Date: Wed, 27 May 2020 17:53:28 +0530 Subject: [PATCH 6/7] add entry point --- setup.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/setup.py b/setup.py index 843d2407..eb7580c1 100644 --- a/setup.py +++ b/setup.py @@ -19,5 +19,10 @@ "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", ], + entry_points={ + 'console_scripts': [ + 'hsc=hsc.__main__:main', + ], + }, install_requires=['progress', 'requests', 'configargparse'] ) From 74a5de277a8a512611e7185c52aac6e123a87310 Mon Sep 17 00:00:00 2001 From: rishabhsingh971 Date: Wed, 27 May 2020 18:20:32 +0530 Subject: [PATCH 7/7] update version to 1.2.2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index eb7580c1..08834d4c 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='hsc', - version='1.2.1', + version='1.2.2', author='Nullifiers', author_email='nullifiersorg@gmail.com', description='Hackerrank Solution Crawler',