Skip to content

Commit b9a3599

Browse files
author
Rishabh Agrahari
committed
Bugfix in scheduler
1 parent 8c90f51 commit b9a3599

File tree

3 files changed

+118
-213
lines changed

3 files changed

+118
-213
lines changed

‎server/requirements.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ chardet==3.0.4
99
Django==2.0.2
1010
idna==2.6
1111
kombu==3.0.37
12-
lxml==4.1.1
12+
lxml
1313
mysqlclient==1.3.12
1414
pytz==2018.3
1515
redis==2.10.3

‎server/swag/tasks.py

+79-168
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
from .models import CSV
12
from celery.task.schedules import crontab
23
from celery.decorators import periodic_task
34

@@ -12,179 +13,89 @@
1213
import os
1314
from bs4 import BeautifulSoup
1415

15-
import sys,os
16-
djangoproject_home="website"
16+
import sys
17+
import os
18+
djangoproject_home = "website"
1719
sys.path.append(djangoproject_home)
1820
os.environ['DJANGO_SETTINGS_MODULE'] = 'website.settings'
19-
from .models import CSV
20-
import csv,datetime
21-
22-
# from .views import forecasted_plot
2321

24-
# @periodic_task(run_every=(crontab(minute='*/1')), name="update_database", ignore_result=True
25-
26-
# size=0
2722

2823
@periodic_task(run_every=(crontab(minute=55, hour='0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23')), name="reload_task", ignore_result=True)
2924
def reload_task():
30-
# global size
31-
print("date dekh le bhai",datetime.date.today().day)
32-
url = 'http://www.delhisldc.org/Loaddata.aspx?mode='
33-
day_range = list(range(datetime.date.today().day,datetime.date.today().day+1)) # days, 1 to 31
34-
# months, Aug to Dec for 2017, and Jan for 2018
35-
month_range = {
36-
# 2017: list(range(12, 13)),
37-
2018: [datetime.date.today().month]
38-
}
39-
40-
year_range = [datetime.date.today().year]
41-
42-
if not os.path.exists('static/SLDC_Data'):
43-
os.makedirs('static/SLDC_Data')
44-
45-
for year in year_range:
46-
for month in month_range[year]:
47-
month_dir = 'static/SLDC_Data/%d/%02d/' %(year, month)
48-
if not os.path.exists(month_dir): os.makedirs(month_dir)
49-
try:
50-
for day in day_range:
51-
date = '%02d/%02d/%d' %(day, month, year)
52-
print('Scraping', date)
53-
resp = requests.get(url+date) # send a get request to the url, get response
54-
soup = BeautifulSoup(resp.text, 'lxml') # Yummy HTML soup
55-
table = soup.find('table', {'id':'ContentPlaceHolder3_DGGridAv'}) # get the table from html
56-
trs = table.findAll('tr') # extract all rows of the table
57-
if len(trs[1:])!=0: # no need to create csv file, if there's no data, for Aug month of 2017
58-
csv_filename = month_dir + '%s.csv' % date.replace('/', '-')
59-
if os.path.exists(csv_filename): os.remove(csv_filename) # remove the file it already exists, can result in data duplicacy
60-
with open(csv_filename, 'a') as f:
61-
writer = csv.writer(f)
62-
# writer.writerow(['x', 'y'])
63-
for tr in trs[1:]:
64-
time, delhi = tr.findChildren('font')[:2]
65-
t = (time.text).split(':')
66-
TimE = datetime.time(int(t[0]),int(t[1]))
67-
writer.writerow([str(TimE), delhi.text])
68-
print(str(TimE))
69-
results=[]
70-
results.append(CSV.objects.filter(date = datetime.date.today(), timestamp = str(TimE)))
71-
# print("hhhhhhhhhhheeeee", len(results[0]))
72-
if (len(results[0]) == 0):
73-
# for z in results[0]:
74-
# print (z.timestamp, type(z.timestamp), len(results))
75-
print(TimE, delhi.text)
76-
data = CSV()
77-
data.timestamp = str(TimE)
78-
data.load_value = delhi.text
79-
data.date = datetime.date.today()
80-
data.save()
81-
82-
if(datetime.datetime.time(datetime.datetime.now()).hour == 23 and datetime.datetime.time(datetime.datetime.now()).minute >= 55):
83-
print('............................................')
84-
# x = '00:00:00'
85-
start = datetime.time(0,0)
86-
end = datetime.time(23,55)
87-
# size_reduction = 0
88-
while(start != end):
89-
# tym = x.split(':')
90-
# Time = datetime.time(int(tym[0]),int(tym[1]))
91-
# print((datetime.datetime.combine(datetime.date(1,1,1),Time)+timedelta(minutes=5)).time())
92-
blanck_val=[]
93-
blanck_val.append(CSV.objects.filter(date = datetime.date.today(), timestamp = str(start)))
94-
# print("hhhhhhhhhhheeeee", len(results[0]))
95-
if (len(blanck_val[0]) == 0):
96-
data = CSV()
97-
data.timestamp = str(start)
98-
data.load_value = None
99-
data.date = datetime.date.today()
100-
data.save()
101-
writer.writerow([str(start), None])
102-
# CSV.objects.filter(date = datetime.date.today(), timestamp = str(start)).update(load_value = None)
103-
start = (datetime.datetime.combine(datetime.date(1,1,1),start)+timedelta(minutes=5)).time()
104-
105-
except Exception as e:
106-
print(e)
107-
108-
# link = "static/SLDC_Data/"+ str(year) + "/"+ str(month).zfill(2) + "/" + str(day).zfill(2) + "-" + str(month).zfill(2) + "-" + str(year) +".csv"
109-
# # link = "static/SLDC_DataBase/2017/11/"+str(a).zfill(2)+"-11-2017.csv"
110-
# dataReader = csv.reader(open(link), delimiter=',', quotechar='"')
111-
# rows = list(dataReader)
112-
# print(len(rows))
113-
# print(rows)
114-
# print("global k baad",size)
115-
# # if(len(rows) > size):
116-
# print(len(rows))
117-
# print(size_reduction)
118-
# for row in rows[size:len(rows)+1]:
119-
# print("inside 2")
120-
# data = CSV()
121-
# if(row[1]=='None'):
122-
# break
123-
# print('row[0]',row[0])
124-
# data.timestamp = row[0]
125-
# data.load_value = row[1]
126-
# data.date = datetime.date(datetime.date.today().year,datetime.date.today().month,datetime.date.today().day)
127-
# data.save()
128-
# size = len(rows)-size_reduction
129-
# if(datetime.datetime.time(datetime.datetime.now()).hour == 23 and datetime.datetime.time(datetime.datetime.now()).minute >= 40):
130-
# size = 0
131-
# print("final wali........end",size)
132-
# print("update_done...............yupppp")
133-
134-
135-
136-
# def update_database():
137-
# print("qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq")
138-
# for i in range(30):
139-
# a = 1+i;
140-
# print(a);
141-
# print("000000000000000000000000mmmmmmmmmmm")
142-
# print("adscvadsfvadsfvadsfa")
143-
# print("date dekh le bhai",datetime.date.today().day)
144-
# url = 'https://www.delhisldc.org/Loaddata.aspx?mode='
145-
# day_range = list(range(datetime.date.today().day,datetime.date.today().day+1)) # days, 1 to 31
146-
# # months, Aug to Dec for 2017, and Jan for 2018
147-
# month_range = {
148-
# # 2017: list(range(12, 13)),
149-
# 2018: [datetime.date.today().month]
150-
# }
151-
152-
# year_range = [datetime.date.today().year]
25+
# global size
26+
print("date:", datetime.date.today().day)
27+
url = 'http://www.delhisldc.org/Loaddata.aspx?mode='
28+
day_range = list(range(datetime.date.today().day,
29+
datetime.date.today().day+1)) # days, 1 to 31
30+
# months, Aug to Dec for 2017, and Jan for 2018
31+
32+
month = datetime.date.today().month
33+
year = datetime.date.today().year
34+
35+
if not os.path.exists('static/SLDC_Data'):
36+
os.makedirs('static/SLDC_Data')
37+
38+
month_dir = 'static/SLDC_Data/%d/%02d/' % (year, month)
39+
if not os.path.exists(month_dir):
40+
os.makedirs(month_dir)
41+
try:
42+
for day in day_range:
43+
date = '%02d/%02d/%d' % (day, month, year)
44+
print('Scraping', date)
45+
# send a get request to the url, get response
46+
resp = requests.get(url+date)
47+
soup = BeautifulSoup(resp.text, 'lxml') # Yummy HTML soup
48+
# get the table from html
49+
table = soup.find(
50+
'table', {'id': 'ContentPlaceHolder3_DGGridAv'})
51+
trs = table.findAll('tr') # extract all rows of the table
52+
# no need to create csv file, if there's no data, for Aug month of 2017
53+
if len(trs[1:]) != 0:
54+
csv_filename = month_dir + \
55+
'%s.csv' % date.replace('/', '-')
56+
if os.path.exists(csv_filename):
57+
# remove the file it already exists, can result in data duplicacy
58+
os.remove(csv_filename)
59+
with open(csv_filename, 'a') as f:
60+
writer = csv.writer(f)
61+
for tr in trs[1:]:
62+
time, delhi = tr.findChildren('font')[:2]
63+
t = (time.text).split(':')
64+
TimE = datetime.time(int(t[0]), int(t[1]))
65+
writer.writerow([str(TimE), delhi.text])
66+
print(str(TimE))
67+
results = []
68+
results.append(CSV.objects.filter(
69+
date=datetime.date.today(), timestamp=str(TimE)))
70+
if (len(results[0]) == 0):
71+
print(TimE, delhi.text)
72+
data = CSV()
73+
data.timestamp = str(TimE)
74+
data.load_value = delhi.text
75+
data.date = datetime.date.today()
76+
data.save()
77+
78+
if(datetime.datetime.time(datetime.datetime.now()).hour == 23 and datetime.datetime.time(datetime.datetime.now()).minute >= 55):
79+
print(
80+
'............................................')
81+
start = datetime.time(0, 0)
82+
end = datetime.time(23, 55)
83+
while(start != end):
84+
blanck_val = []
85+
blanck_val.append(CSV.objects.filter(
86+
date=datetime.date.today(), timestamp=str(start)))
87+
if (len(blanck_val[0]) == 0):
88+
data = CSV()
89+
data.timestamp = str(start)
90+
data.load_value = None
91+
data.date = datetime.date.today()
92+
data.save()
93+
writer.writerow([str(start), None])
94+
# CSV.objects.filter(date = datetime.date.today(), timestamp = str(start)).update(load_value = None)
95+
start = (datetime.datetime.combine(datetime.date(
96+
1, 1, 1), start)+timedelta(minutes=5)).time()
97+
98+
except Exception as e:
99+
print(e)
153100

154-
# if not os.path.exists('static/SLDC_DataBase'):
155-
# os.makedirs('static/SLDC_DataBase')
156101

157-
# for year in year_range:
158-
# for month in month_range[year]:
159-
# month_dir = 'static/SLDC_DataBase/%d/%02d/' %(year, month)
160-
# if not os.path.exists(month_dir): os.makedirs(month_dir)
161-
# try:
162-
# for day in day_range:
163-
# date = '%02d/%02d/%d' %(day, month, year)
164-
# print('Scraping', date)
165-
# resp = requests.get(url+date) # send a get request to the url, get response
166-
# soup = BeautifulSoup(resp.text, 'lxml') # Yummy HTML soup
167-
# table = soup.find('table', {'id':'ContentPlaceHolder3_DGGridAv'}) # get the table from html
168-
# trs = table.findAll('tr') # extract all rows of the table
169-
# if len(trs[1:])!=0: # no need to create csv file, if there's no data, for Aug month of 2017
170-
# csv_filename = month_dir + '%s.csv' % date.replace('/', '-')
171-
# if os.path.exists(csv_filename): os.remove(csv_filename) # remove the file it already exists, can result in data duplicacy
172-
# with open(csv_filename, 'a') as f:
173-
# writer = csv.writer(f)
174-
# # writer.writerow(['x', 'y'])
175-
# for tr in trs[1:]:
176-
# time, delhi = tr.findChildren('font')[:2]
177-
# writer.writerow([time.text, delhi.text])
178-
# except Exception as e:
179-
# print(e)
180-
# print("done updating")
181-
# link = "/static/SLDC_DataBase/"+ year + "/"+ month + "/" + day + "-" + month + "-" + year +".csv"
182-
# # link = "static/SLDC_DataBase/2017/11/"+str(a).zfill(2)+"-11-2017.csv"
183-
# dataReader = csv.reader(open(link), delimiter=',', quotechar='"')
184-
# for row in dataReader:
185-
# data = CSV()
186-
# data.timestamp = row[0]
187-
# data.load_value = row[1]
188-
# data.date = datetime.date(datetime.date.today().year,datetime.date.today().month,datetime.date.today().day)
189-
# data.save()
190-
# print("update_done...............yupppp")

0 commit comments

Comments
 (0)