dsyoon/DeepStock
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

timeout 5초 추가

Browse Source
This commit is contained in:
dosang.yoon
2022-06-03 12:28:58 +09:00
parent c5848ebadc
commit 5774a6eaf0
8 changed files with 87 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
stockpredictor/crawler/sQLite/MetaCrawler.py
View File

@@ -48,7 +48,14 @@ class MetaCrawler:
finish = False
for i in range(1, self.limit_page_count):
#html = pd.read_html(input['URL'] + '&page=%s' % i, header=0)
html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header).text)
html = None
while True:
try:
html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header, timeout=5).text)
break
except:
print(input['URL'] + '&page=%s' % i)
continue
# 마지막 페이지 까지 받기
if len(html[0].날짜.values) <= 1:
@@ -124,7 +131,14 @@ class MetaCrawler:
finish = False
for i in range(1, self.limit_page_count):
#html = pd.read_html(url + str(i), header=0)
html = pd.read_html(requests.get(url + str(i), headers=self.header).text)
html = None
while True:
try:
html = pd.read_html(requests.get(url + str(i), headers=self.header, timeout=5).text)
break
except:
print(url + str(i))
continue
# 마지막 페이지 까지 받기
if len(html[0].날짜.values) <= 2:
@@ -194,7 +208,14 @@ class MetaCrawler:
finish = False
for i in range(1, self.limit_page_count):
#html = pd.read_html(url + str(i), header=0, encoding='euc-kr')
html = pd.read_html(requests.get(url + str(i), headers=self.header).text, encoding='euc-kr')
html = None
while True:
try:
html = pd.read_html(requests.get(url + str(i), headers=self.header, timeout=5).text, encoding='euc-kr')
break
except:
print(url + str(i))
continue
# 마지막 페이지 까지 받기
if len(html[0].날짜.values) <= 10:
@@ -273,7 +294,14 @@ class MetaCrawler:
finish = False
for i in range(1, self.limit_page_count):
#html = pd.read_html(input['URL'] + '&page=%s' % i, header=0)
html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header).text)
html = None
while True:
try:
html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header, timeout=5).text)
break
except:
print (input['URL'] + '&page=%s' % i)
continue
ymd, close, diff, rate = "", 0.0, 0.0, 0.0
# 마지막 페이지 까지 받기
@@ -347,7 +375,14 @@ class MetaCrawler:
finish = False
for i in range(1, self.limit_page_count):
#html = pd.read_html(input['URL'] + '&page=%s' % i, header=0)
html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header).text)
html = None
while True:
try:
html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header, timeout=5).text)
break
except:
print(input['URL'] + '&page=%s' % i)
continue
# 마지막 페이지 까지 받기
if len(html[0].날짜.values) <= 1: