From 333e79eeaee1c19ac3ae0b28ef9b74eed929432c Mon Sep 17 00:00:00 2001 From: aaradhyaberi07 <64523034+aaradhyaberi07@users.noreply.github.com> Date: Thu, 23 Dec 2021 20:24:13 +0530 Subject: [PATCH 1/3] added new endpoint alog with function for cron --- codedigger/lists/cron.py | 6 +++++- codedigger/lists/serializers.py | 7 +++++++ codedigger/lists/solved_update.py | 31 +++++++++++++++++++++++++++++++ codedigger/lists/urls.py | 2 ++ codedigger/lists/views.py | 23 ++++++++++++++++++++--- 5 files changed, 65 insertions(+), 4 deletions(-) diff --git a/codedigger/lists/cron.py b/codedigger/lists/cron.py index 18da564..26caaa4 100644 --- a/codedigger/lists/cron.py +++ b/codedigger/lists/cron.py @@ -12,7 +12,7 @@ from codedigger.settings import EMAIL_HOST_USER from codeforces.api import user_status from user.exception import ValidationException - +from .solved_update import UpdateforUserCodeforces def cron_codeforces(user): if user is None: @@ -43,6 +43,10 @@ def cron_codeforces(user): continue +def cron_codeforces_all_users(limit): + for ele in User.objects.all(): + UpdateforUserCodeforces(ele,limit) + def cron_uva(user): if user is None: return diff --git a/codedigger/lists/serializers.py b/codedigger/lists/serializers.py index 7f31866..180b1d8 100644 --- a/codedigger/lists/serializers.py +++ b/codedigger/lists/serializers.py @@ -206,3 +206,10 @@ class EnrollInListSerializer(serializers.Serializer): class Meta: fields = ('slug', ) + + +class UpdateCodeforcesForUserSerializer(serializers.Serializer): + limit = serializers.IntegerField(required=False) + + class Meta: + fields = ('limit', ) \ No newline at end of file diff --git a/codedigger/lists/solved_update.py b/codedigger/lists/solved_update.py index c88ae09..958ffd4 100644 --- a/codedigger/lists/solved_update.py +++ b/codedigger/lists/solved_update.py @@ -149,3 +149,34 @@ def atcoder_scraper_check(user, prob): check = soup.find_all("span", {"class": "label label-success"}) if check: Solved.objects.create(user=user, problem=prob) + +def UpdateforUserCodeforces(user, limit): + # limit should either be None, or be an integer greater than or equal to 1. + if user is None: + return (False,"Given User object cannot be None") + cf_handle = Profile.objects.get(owner=user).codeforces + if cf_handle == None: + return (False,"cf_handle cannot be None.") + try: + submission_user = user_status(cf_handle) + except ValidationException: + return (False, "Not able to fetch submission data from codeforces.") + for ele in submission_user: + if 'verdict' not in ele or 'contestId' not in ele or ele[ + 'verdict'] != 'OK': + continue + prob_id = str(ele['problem']['contestId']) + str( + ele['problem']['index']) + prob = Problem.objects.filter(prob_id=prob_id, platform='F') + if not prob.exists(): + continue + solve, created = Solved.objects.get_or_create(user=user, + problem=prob[0]) + if not created and limit != None: + limit -= 1 + if limit <= 0: + break + continue + return (True, "Submission data for the given user has been saved successfully.") + + diff --git a/codedigger/lists/urls.py b/codedigger/lists/urls.py index 6166e17..ff5ff83 100644 --- a/codedigger/lists/urls.py +++ b/codedigger/lists/urls.py @@ -19,6 +19,7 @@ SearchUserlistView, ListGetView, EnrollListView, + UpdateCodeforcesForUserView, testing) urlpatterns = [ @@ -64,5 +65,6 @@ path('enroll-list/', EnrollListView.as_view(), name='enroll-list'), path('userlists', SearchUserlistView.as_view(), name='userlist-search'), path('user/', ListGetView.as_view(), name='user-list'), + path('update/codeforces/', UpdateCodeforcesForUserView.as_view(), name='update-codeforces'), path('testing', testing), ] diff --git a/codedigger/lists/views.py b/codedigger/lists/views.py index 9c87f9a..24d4f1a 100644 --- a/codedigger/lists/views.py +++ b/codedigger/lists/views.py @@ -7,14 +7,14 @@ GetUserlistSerializer, EditUserlistSerializer, CreateUserlistSerializer, ProblemSerializer, UserlistAddSerializer, AddProblemsAdminSerializer, - EnrollInListSerializer) + EnrollInListSerializer,UpdateCodeforcesForUserSerializer) from django.db.models import Q from user.permissions import * from user.exception import * from .utils import * from user.models import User - - +from codeforces.api import user_status +from .solved_update import UpdateforUserCodeforces class TopicwiseGetListView(generics.ListAPIView): serializer_class = GetSerializer permission_classes = [AuthenticatedOrReadOnly] @@ -765,6 +765,23 @@ def post(self, request, *args, **kwargs): status=status.HTTP_201_CREATED) +class UpdateCodeforcesForUserView(generics.GenericAPIView): + permission_classes = [AuthenticatedActivated] + serializer_class = UpdateCodeforcesForUserSerializer + + def post(self,request,*args, **kwargs): + curr_user = self.request.user + data = request.data + limit = data.get("limit",None) + returned_status,returned_response = UpdateforUserCodeforces(curr_user,limit); + if returned_status: + return response.Response({'status': 'OK', 'result': returned_response}, status = status.HTTP_200_OK) + else: + return ValidationException(returned_response) + + + + def testing(request): updater() return JsonResponse({'status': 'OK'}) From b1207c0c288a225168459ffbb57f88e6e41ed552 Mon Sep 17 00:00:00 2001 From: aaradhyaberi07 <64523034+aaradhyaberi07@users.noreply.github.com> Date: Sat, 22 Jan 2022 18:37:43 +0530 Subject: [PATCH 2/3] updater for all functions has been added --- codedigger/lists/solved_update.py | 110 ++++++++++++++++++++++++++++++ codedigger/lists/urls.py | 4 +- codedigger/lists/views.py | 31 ++++++--- 3 files changed, 135 insertions(+), 10 deletions(-) diff --git a/codedigger/lists/solved_update.py b/codedigger/lists/solved_update.py index 958ffd4..2c0024d 100644 --- a/codedigger/lists/solved_update.py +++ b/codedigger/lists/solved_update.py @@ -180,3 +180,113 @@ def UpdateforUserCodeforces(user, limit): return (True, "Submission data for the given user has been saved successfully.") +def UpdateforUserCodechef(user,limit): + if user is None: + return (False,"Given User object cannot be None") + codechef_handle = Profile.objects.get(owner=user).codechef + if codechef_handle is None: + return (False,"codechef_handle cannot be None.") + url = 'https://www.codechef.com/users/' + str(codechef_handle) + res = requests.get(url) + soup = bs4.BeautifulSoup(res.content, 'html.parser') + problems_solved = soup.find( + 'section', {'class': 'rating-data-section problems-solved'}) + if problems_solved is None: + return (True,"Submission data for the given user has been saved successfully.") + if problems_solved.find('h5').text == 'Fully Solved (0)': + return (True,"Submission data for the given user has been saved successfully.") + for ele in problems_solved.find('article').find_all('a'): + prob = Problem.objects.filter(prob_id=ele.text, platform='C') + if not prob.exists(): + continue + solve, created = Solved.objects.get_or_create(problem=prob[0], + user=user) + if not created and limit != None: + limit -= 1 + if limit <= 0: + break + continue + return (True,"Submission data for the given user has been saved successfully.") + + +def UpdateforUserAtcoder(user,limit): + if user is None: + return (False,"Given User object cannot be None") + atcoder_handle = Profile.objects.get(owner=user).atcoder + if atcoder_handle is None: + return (False,"atcoder_handle cannot be None.") + url = 'https://kenkoooo.com/atcoder/atcoder-api/results?user=' + atcoder_handle + req = requests.get(url) + if req.status_code != 200: + return (False,"User with the given handle does not exist.") + req = req.json() + sorted_req = sorted(req, key=lambda k: k['epoch_second'], reverse=True) + for ele in sorted_req: + if ele['result'] != "AC": + continue + prob = Problem.objects.filter(prob_id=ele['problem_id'], platform='A') + if not prob.exists(): + continue + solve, created = Solved.objects.get_or_create(user=user, + problem=prob[0]) + if not created and limit != None: + limit -= 1 + if limit <= 0: + break + continue + return (True,"Submission data for the given user has been saved successfully.") + +def UpdateforUserSpoj(user,limit): + if user is None: + return (False,"Given User object cannot be None") + spoj_handle = Profile.objects.get(owner=user).spoj + if spoj_handle == None: + return (False,"spoj_handle cannot be None.") + url = 'https://www.spoj.com/users/' + spoj_handle + res = requests.get(url) + soup = bs4.BeautifulSoup(res.content, 'html.parser') + problems = soup.find('table', {'class': 'table table-condensed'}) + if problems is None: + return (True,"Submission data for the given user has been saved successfully.") + for ele in problems.find_all('td'): + if ele.text == "": + continue + prob = Problem.objects.filter(prob_id=ele.text, platform='S') + if not prob.exists(): + continue + solve, created = Solved.objects.get_or_create(problem=prob[0], + user=user) + if not created and limit != None: + limit -= 1 + if limit <= 0: + break + continue + return (True,"Submission data for the given user has been saved successfully.") + + +def UpdateforUserUva(user,limit): + if user is None: + return (False,"Given User object cannot be None") + uva_id = Profile.objects.get(owner=user).uva_id + if uva_id is None: + return (False,"uva_id cannot be None.") + url1 = "https://uhunt.onlinejudge.org/api/subs-user/" + str(uva_id) + req1 = requests.get(url1) + if req1.status_code != 200: + return (False,"User with the given UVA-ID does not exist") + req1 = req1.json() + sorted_req1 = sorted(req1['subs'], key=lambda k: k[4], reverse=True) + for ele in sorted_req1: + if str(ele[2]) != '90': + continue + prob = Problem.objects.filter(prob_id=str(ele[1]), platform='U') + if not prob.exists(): + continue + solve, created = Solved.objects.get_or_create(user=user, + problem=prob[0]) + if not created and limit != None: + limit -= 1 + if limit <= 0: + break + continue + return (True,"Submission data for the given user has been saved successfully.") diff --git a/codedigger/lists/urls.py b/codedigger/lists/urls.py index ae6c207..bd12e8b 100644 --- a/codedigger/lists/urls.py +++ b/codedigger/lists/urls.py @@ -20,7 +20,7 @@ SearchUserlistView, ListGetView, EnrollListView, - UpdateCodeforcesForUserView, + UpdateForUserView, testing) urlpatterns = [ @@ -66,7 +66,7 @@ path('enroll-list/', EnrollListView.as_view(), name='enroll-list'), path('userlists', SearchUserlistView.as_view(), name='userlist-search'), path('user/', ListGetView.as_view(), name='user-list'), - path('update/codeforces/', UpdateCodeforcesForUserView.as_view(), name='update-codeforces'), + path('update//', UpdateForUserView.as_view(), name='update-codeforces'), # path('/stats', ListStats.as_view(), name='list-stats'), path('/stats/standing', UserStandingStats.as_view(), diff --git a/codedigger/lists/views.py b/codedigger/lists/views.py index 99a9c7d..41dc4f7 100644 --- a/codedigger/lists/views.py +++ b/codedigger/lists/views.py @@ -8,13 +8,15 @@ CreateUserlistSerializer, ProblemSerializer, UserlistAddSerializer, AddProblemsAdminSerializer, EnrollInListSerializer,UpdateCodeforcesForUserSerializer) -from django.db.models import Q -from user.permissions import * -from user.exception import * -from .utils import * -from user.models import User from codeforces.api import user_status -from .solved_update import UpdateforUserCodeforces,EnrollInListSerializer +from .solved_update import ( + UpdateforUserCodeforces, + UpdateforUserAtcoder, + UpdateforUserCodechef, + UpdateforUserSpoj, + UpdateforUserUva, + EnrollInListSerializer +) from django.db.models import Q, Subquery, Count from user.permissions import * from user.exception import * @@ -825,15 +827,28 @@ def post(self, request, *args, **kwargs): status=status.HTTP_201_CREATED) -class UpdateCodeforcesForUserView(generics.GenericAPIView): +class UpdatesForUserView(generics.GenericAPIView): permission_classes = [AuthenticatedActivated] serializer_class = UpdateCodeforcesForUserSerializer def post(self,request,*args, **kwargs): curr_user = self.request.user data = request.data + platform = self.kwargs['platform'] + username = data.get("username", None) limit = data.get("limit",None) - returned_status,returned_response = UpdateforUserCodeforces(curr_user,limit); + if curr_user and curr_user.is_staff and username: + curr_user = User.objects.get(username = username) + if platform == 'F': + returned_status,returned_response = UpdateforUserCodeforces(curr_user,limit); + if platform == 'C': + returned_status,returned_response = UpdateforUserCodechef(curr_user,limit); + if platform == 'A': + returned_status,returned_response = UpdateforUserAtcoder(curr_user,limit); + if platform == 'S': + returned_status,returned_response = UpdateforUserSpoj(curr_user,limit); + if platform == 'U': + returned_status,returned_response = UpdateforUserUva(curr_user,limit); if returned_status: return response.Response({'status': 'OK', 'result': returned_response}, status = status.HTTP_200_OK) else: From 9063ae325800bf2f0291fc4df4ea3f464390ca10 Mon Sep 17 00:00:00 2001 From: github-actions Date: Sat, 22 Jan 2022 15:08:00 +0000 Subject: [PATCH 3/3] refactor: automatic code reformatting to PEP8 by yapf --- codedigger/lists/cron.py | 4 +- codedigger/lists/solved_update.py | 65 ++++++++++++++++++------------- codedigger/lists/urls.py | 4 +- codedigger/lists/views.py | 48 ++++++++++++----------- 4 files changed, 71 insertions(+), 50 deletions(-) diff --git a/codedigger/lists/cron.py b/codedigger/lists/cron.py index 26caaa4..9666841 100644 --- a/codedigger/lists/cron.py +++ b/codedigger/lists/cron.py @@ -14,6 +14,7 @@ from user.exception import ValidationException from .solved_update import UpdateforUserCodeforces + def cron_codeforces(user): if user is None: return @@ -45,7 +46,8 @@ def cron_codeforces(user): def cron_codeforces_all_users(limit): for ele in User.objects.all(): - UpdateforUserCodeforces(ele,limit) + UpdateforUserCodeforces(ele, limit) + def cron_uva(user): if user is None: diff --git a/codedigger/lists/solved_update.py b/codedigger/lists/solved_update.py index 2c0024d..d8d9237 100644 --- a/codedigger/lists/solved_update.py +++ b/codedigger/lists/solved_update.py @@ -150,13 +150,14 @@ def atcoder_scraper_check(user, prob): if check: Solved.objects.create(user=user, problem=prob) + def UpdateforUserCodeforces(user, limit): # limit should either be None, or be an integer greater than or equal to 1. if user is None: - return (False,"Given User object cannot be None") + return (False, "Given User object cannot be None") cf_handle = Profile.objects.get(owner=user).codeforces if cf_handle == None: - return (False,"cf_handle cannot be None.") + return (False, "cf_handle cannot be None.") try: submission_user = user_status(cf_handle) except ValidationException: @@ -177,24 +178,29 @@ def UpdateforUserCodeforces(user, limit): if limit <= 0: break continue - return (True, "Submission data for the given user has been saved successfully.") + return (True, + "Submission data for the given user has been saved successfully.") -def UpdateforUserCodechef(user,limit): +def UpdateforUserCodechef(user, limit): if user is None: - return (False,"Given User object cannot be None") + return (False, "Given User object cannot be None") codechef_handle = Profile.objects.get(owner=user).codechef if codechef_handle is None: - return (False,"codechef_handle cannot be None.") + return (False, "codechef_handle cannot be None.") url = 'https://www.codechef.com/users/' + str(codechef_handle) res = requests.get(url) soup = bs4.BeautifulSoup(res.content, 'html.parser') problems_solved = soup.find( 'section', {'class': 'rating-data-section problems-solved'}) if problems_solved is None: - return (True,"Submission data for the given user has been saved successfully.") + return ( + True, + "Submission data for the given user has been saved successfully.") if problems_solved.find('h5').text == 'Fully Solved (0)': - return (True,"Submission data for the given user has been saved successfully.") + return ( + True, + "Submission data for the given user has been saved successfully.") for ele in problems_solved.find('article').find_all('a'): prob = Problem.objects.filter(prob_id=ele.text, platform='C') if not prob.exists(): @@ -206,19 +212,20 @@ def UpdateforUserCodechef(user,limit): if limit <= 0: break continue - return (True,"Submission data for the given user has been saved successfully.") + return (True, + "Submission data for the given user has been saved successfully.") -def UpdateforUserAtcoder(user,limit): +def UpdateforUserAtcoder(user, limit): if user is None: - return (False,"Given User object cannot be None") + return (False, "Given User object cannot be None") atcoder_handle = Profile.objects.get(owner=user).atcoder if atcoder_handle is None: - return (False,"atcoder_handle cannot be None.") + return (False, "atcoder_handle cannot be None.") url = 'https://kenkoooo.com/atcoder/atcoder-api/results?user=' + atcoder_handle req = requests.get(url) if req.status_code != 200: - return (False,"User with the given handle does not exist.") + return (False, "User with the given handle does not exist.") req = req.json() sorted_req = sorted(req, key=lambda k: k['epoch_second'], reverse=True) for ele in sorted_req: @@ -234,20 +241,24 @@ def UpdateforUserAtcoder(user,limit): if limit <= 0: break continue - return (True,"Submission data for the given user has been saved successfully.") + return (True, + "Submission data for the given user has been saved successfully.") + -def UpdateforUserSpoj(user,limit): +def UpdateforUserSpoj(user, limit): if user is None: - return (False,"Given User object cannot be None") + return (False, "Given User object cannot be None") spoj_handle = Profile.objects.get(owner=user).spoj if spoj_handle == None: - return (False,"spoj_handle cannot be None.") + return (False, "spoj_handle cannot be None.") url = 'https://www.spoj.com/users/' + spoj_handle res = requests.get(url) soup = bs4.BeautifulSoup(res.content, 'html.parser') problems = soup.find('table', {'class': 'table table-condensed'}) if problems is None: - return (True,"Submission data for the given user has been saved successfully.") + return ( + True, + "Submission data for the given user has been saved successfully.") for ele in problems.find_all('td'): if ele.text == "": continue @@ -260,20 +271,21 @@ def UpdateforUserSpoj(user,limit): limit -= 1 if limit <= 0: break - continue - return (True,"Submission data for the given user has been saved successfully.") + continue + return (True, + "Submission data for the given user has been saved successfully.") -def UpdateforUserUva(user,limit): +def UpdateforUserUva(user, limit): if user is None: - return (False,"Given User object cannot be None") + return (False, "Given User object cannot be None") uva_id = Profile.objects.get(owner=user).uva_id if uva_id is None: - return (False,"uva_id cannot be None.") + return (False, "uva_id cannot be None.") url1 = "https://uhunt.onlinejudge.org/api/subs-user/" + str(uva_id) req1 = requests.get(url1) if req1.status_code != 200: - return (False,"User with the given UVA-ID does not exist") + return (False, "User with the given UVA-ID does not exist") req1 = req1.json() sorted_req1 = sorted(req1['subs'], key=lambda k: k[4], reverse=True) for ele in sorted_req1: @@ -288,5 +300,6 @@ def UpdateforUserUva(user,limit): limit -= 1 if limit <= 0: break - continue - return (True,"Submission data for the given user has been saved successfully.") + continue + return (True, + "Submission data for the given user has been saved successfully.") diff --git a/codedigger/lists/urls.py b/codedigger/lists/urls.py index bd12e8b..176ec70 100644 --- a/codedigger/lists/urls.py +++ b/codedigger/lists/urls.py @@ -66,7 +66,9 @@ path('enroll-list/', EnrollListView.as_view(), name='enroll-list'), path('userlists', SearchUserlistView.as_view(), name='userlist-search'), path('user/', ListGetView.as_view(), name='user-list'), - path('update//', UpdateForUserView.as_view(), name='update-codeforces'), + path('update//', + UpdateForUserView.as_view(), + name='update-codeforces'), # path('/stats', ListStats.as_view(), name='list-stats'), path('/stats/standing', UserStandingStats.as_view(), diff --git a/codedigger/lists/views.py b/codedigger/lists/views.py index 41dc4f7..1d4c4a6 100644 --- a/codedigger/lists/views.py +++ b/codedigger/lists/views.py @@ -7,16 +7,12 @@ GetUserlistSerializer, EditUserlistSerializer, CreateUserlistSerializer, ProblemSerializer, UserlistAddSerializer, AddProblemsAdminSerializer, - EnrollInListSerializer,UpdateCodeforcesForUserSerializer) + EnrollInListSerializer, + UpdateCodeforcesForUserSerializer) from codeforces.api import user_status -from .solved_update import ( - UpdateforUserCodeforces, - UpdateforUserAtcoder, - UpdateforUserCodechef, - UpdateforUserSpoj, - UpdateforUserUva, - EnrollInListSerializer -) +from .solved_update import (UpdateforUserCodeforces, UpdateforUserAtcoder, + UpdateforUserCodechef, UpdateforUserSpoj, + UpdateforUserUva, EnrollInListSerializer) from django.db.models import Q, Subquery, Count from user.permissions import * from user.exception import * @@ -830,32 +826,40 @@ def post(self, request, *args, **kwargs): class UpdatesForUserView(generics.GenericAPIView): permission_classes = [AuthenticatedActivated] serializer_class = UpdateCodeforcesForUserSerializer - - def post(self,request,*args, **kwargs): + + def post(self, request, *args, **kwargs): curr_user = self.request.user data = request.data platform = self.kwargs['platform'] username = data.get("username", None) - limit = data.get("limit",None) + limit = data.get("limit", None) if curr_user and curr_user.is_staff and username: - curr_user = User.objects.get(username = username) + curr_user = User.objects.get(username=username) if platform == 'F': - returned_status,returned_response = UpdateforUserCodeforces(curr_user,limit); + returned_status, returned_response = UpdateforUserCodeforces( + curr_user, limit) if platform == 'C': - returned_status,returned_response = UpdateforUserCodechef(curr_user,limit); + returned_status, returned_response = UpdateforUserCodechef( + curr_user, limit) if platform == 'A': - returned_status,returned_response = UpdateforUserAtcoder(curr_user,limit); + returned_status, returned_response = UpdateforUserAtcoder( + curr_user, limit) if platform == 'S': - returned_status,returned_response = UpdateforUserSpoj(curr_user,limit); + returned_status, returned_response = UpdateforUserSpoj( + curr_user, limit) if platform == 'U': - returned_status,returned_response = UpdateforUserUva(curr_user,limit); + returned_status, returned_response = UpdateforUserUva( + curr_user, limit) if returned_status: - return response.Response({'status': 'OK', 'result': returned_response}, status = status.HTTP_200_OK) + return response.Response( + { + 'status': 'OK', + 'result': returned_response + }, + status=status.HTTP_200_OK) else: return ValidationException(returned_response) - - - + def testing(request): updater()