From 763fdc53738492f2f69f879aba3047965c53f35c Mon Sep 17 00:00:00 2001 From: Rajat Srivastava Date: Thu, 10 Oct 2019 00:41:10 +0530 Subject: [PATCH 1/5] script parameters --- README.md | 24 ++++++++++++++++++++++++ hsc/crawler.py | 21 ++++++++++++++++----- setup.py | 2 +- 3 files changed, 41 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index d11da1b3..c626f1fd 100644 --- a/README.md +++ b/README.md @@ -9,3 +9,27 @@ Crawls solutions of hackerrank and stores as local files. - Login with your Hackerrank Credentials - Enter the limit of successful solutions you want to be crawled - A new folder with name **Hackerrank** would be created with all your solutions in it + +## Options to use while running script +Script `hsc` supports following options +- help: -h or --help +- username: -u or --username -> username of hackerrank profile +- password: -p or --password -> password of hackerrank profile +- limit: -l or --limit -> no. of solutions to be downloaded +- config: -c or --config -> path of config file + +Usage: +We can use above script helpers as +```bash +hsc -l 34 -p testpassword -u testuser +``` + +We can also use config file to download solutions +Let config file be /etc/user.yaml +```yaml +username: testuser +``` + +```bash +hsc -c /etc/user.yaml -l 34 -p testpassword +``` diff --git a/hsc/crawler.py b/hsc/crawler.py index 57623946..f434dcd6 100755 --- a/hsc/crawler.py +++ b/hsc/crawler.py @@ -2,6 +2,7 @@ import json import requests import getpass +import configargparse from progress.bar import ChargingBar @@ -105,6 +106,7 @@ class Crawler: def __init__(self): self.session = requests.Session() self.total_submissions = 0 + self.options = {} def login(self, username, password): resp = self.session.get(self.login_url, auth=(username, password)) @@ -113,9 +115,18 @@ def login(self, username, password): self.get_number_of_submissions() return self.total_submissions != 0 + def parse_script(self): + p = configargparse.ArgParser(default_config_files=['./user.yaml']) + p.add('-c', '--config', is_config_file=True, help='config file path') + p.add('-l', '--limit', help='limit to no. of solutions to be crawled') + p.add('-u', '--username', help='username') + p.add('-p', '--password', help='password') + + self.options = p.parse_args() + def authenticate(self): - username = input('Hackerrank Username: ') - password = getpass.getpass('Hackerrank Password: ') + username = self.options.username or input('Hackerrank Username: ') + password = self.options.password or getpass.getpass('Hackerrank Password: ') return self.login(username, password) def get_number_of_submissions(self): @@ -224,14 +235,14 @@ def get_submissions(self, submissions): def main(): offset = 0 - limit = 10 # you should change this crawler = Crawler() - + crawler.parse_script() while(not crawler.authenticate()): print('Auth was unsuccessful') - limit = input('Enter limit needed to crawl: ') + limit = crawler.options.limit or crawler.get_number_of_submissions() + print('Start crawling {} solutions.....'.format(limit)) all_submissions_url = crawler.get_all_submissions_url(offset, limit) resp = crawler.session.get(all_submissions_url, headers=crawler.headers) diff --git a/setup.py b/setup.py index d6f8a7e4..cd712f41 100644 --- a/setup.py +++ b/setup.py @@ -24,5 +24,5 @@ 'hsc=hsc.crawler:main', ], }, - install_requires=['progress', 'requests'] + install_requires=['progress', 'requests', 'configargparse'] ) From d58d3e375fa61daadab1c4ecac44d00b990252cf Mon Sep 17 00:00:00 2001 From: Rajat Srivastava Date: Thu, 10 Oct 2019 00:46:10 +0530 Subject: [PATCH 2/5] Update version to 1.2 as major update in usage --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0549099d..34f26485 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='hsc', - version='1.1.3', + version='1.2.0', author='Nullifiers', author_email='nullifiersorg@gmail.com', description='Hackerrank Solution Crawler', From ad9302180288f3f93b48e8abe9dd453da397e21d Mon Sep 17 00:00:00 2001 From: Rajat Goyal Date: Sun, 13 Oct 2019 15:09:19 +0530 Subject: [PATCH 3/5] Add argument for offset --- hsc/crawler.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/hsc/crawler.py b/hsc/crawler.py index f434dcd6..bd3c4396 100755 --- a/hsc/crawler.py +++ b/hsc/crawler.py @@ -119,8 +119,9 @@ def parse_script(self): p = configargparse.ArgParser(default_config_files=['./user.yaml']) p.add('-c', '--config', is_config_file=True, help='config file path') p.add('-l', '--limit', help='limit to no. of solutions to be crawled') - p.add('-u', '--username', help='username') - p.add('-p', '--password', help='password') + p.add('-o', '--offset', help='crawl solutions starting from this number') + p.add('-u', '--username', help='hackerrank account username') + p.add('-p', '--password', help='hackerrank account password') self.options = p.parse_args() @@ -234,15 +235,15 @@ def get_submissions(self, submissions): print('All Solutions Crawled') def main(): - offset = 0 crawler = Crawler() crawler.parse_script() while(not crawler.authenticate()): print('Auth was unsuccessful') - limit = crawler.options.limit or crawler.get_number_of_submissions() - print('Start crawling {} solutions.....'.format(limit)) + limit = crawler.options.limit or crawler.total_submissions + offset = crawler.options.offset or 0 + print('Start crawling {} solutions starting from {}'.format(limit, offset)) all_submissions_url = crawler.get_all_submissions_url(offset, limit) resp = crawler.session.get(all_submissions_url, headers=crawler.headers) From 67c1250215e86c19337c63bc6f03d0816189934b Mon Sep 17 00:00:00 2001 From: Rajat Goyal Date: Sun, 13 Oct 2019 15:32:44 +0530 Subject: [PATCH 4/5] Exit the program if auth is unsuccessful --- hsc/crawler.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hsc/crawler.py b/hsc/crawler.py index bd3c4396..5b174a2c 100755 --- a/hsc/crawler.py +++ b/hsc/crawler.py @@ -238,8 +238,9 @@ def main(): crawler = Crawler() crawler.parse_script() - while(not crawler.authenticate()): - print('Auth was unsuccessful') + if not crawler.authenticate(): + print('Auth was unsuccessful. Exiting the program') + exit(1) limit = crawler.options.limit or crawler.total_submissions offset = crawler.options.offset or 0 From 2af0e58149dbb96d3bd35fba349db5f258db621c Mon Sep 17 00:00:00 2001 From: Rajat Goyal Date: Sun, 13 Oct 2019 15:52:20 +0530 Subject: [PATCH 5/5] Add offset in README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c626f1fd..f4d4a4a4 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ Script `hsc` supports following options - username: -u or --username -> username of hackerrank profile - password: -p or --password -> password of hackerrank profile - limit: -l or --limit -> no. of solutions to be downloaded +- offset: -o or --offset -> crawl solutions starting from this number - config: -c or --config -> path of config file Usage: