init

stockpredictor/crawler/toSQLite/StockCrawler.py

@@ -7,6 +7,7 @@ import sqlite3
 import requests
 import math
 import time
+from time import sleep
 
 class Queue(object):
 	def __init__(self, max):
@@ -228,8 +229,17 @@ class StockCrawler:
 			# 최근 상장 기업의 마지막 반복되는 페이지를 제외시킨다.
 			pg_url = '{url}&page={page}'.format(url=url, page=page)
 			#html = pd.read_html(pg_url, header=0)
-			html = pd.read_html(requests.get(pg_url, headers=self.header, timeout=30).text)
-			sleep(0.5)
+			html = None
+			while True:
+				try:
+					html = pd.read_html(requests.get(pg_url, headers=self.header, timeout=30).text)
+					sleep(0.5)
+					break
+				except:
+					print(pg_url)
+					if page > 200:
+						break
+					continue
 
 			for date in html[0].날짜.values:
 				if type(date) is str: