init

stockpredictor/crawler/toJsonFile/StockCrawler.py

@@ -5,6 +5,7 @@ import re
 import json
 import os
 import requests
+from time import sleep
 
 class Queue(object):
 	def __init__(self, max):
@@ -255,8 +256,17 @@ class StockCrawler:
 			# 최근 상장 기업의 마지막 반복되는 페이지를 제외시킨다.
 			pg_url = '{url}&page={page}'.format(url=url, page=page)
 			#html = pd.read_html(pg_url, header=0)
-			html = pd.read_html(requests.get(pg_url, headers=self.header, timeout=30).text)
-			sleep(0.5)
+			html = None
+			while True:
+				try:
+					html = pd.read_html(requests.get(pg_url, headers=self.header, timeout=30).text)
+					sleep(0.5)
+					break
+				except:
+					print(pg_url)
+					if page > 200:
+						break
+					continue
 
 			count = 0
 			for date in html[0].날짜.values:
@@ -379,8 +389,17 @@ class StockCrawler:
 				# 최근 상장 기업의 마지막 반복되는 페이지를 제외시킨다.
 				pg_url = '{url}&page={page}'.format(url=url, page=page)
 				#html = pd.read_html(pg_url, header=0)
-				html = pd.read_html(requests.get(pg_url, headers=self.header, timeout=30).text)
-				sleep(0.5)
+				html = None
+				while True:
+					try:
+						html = pd.read_html(requests.get(pg_url, headers=self.header, timeout=30).text)
+						sleep(0.5)
+						break
+					except:
+						print(pg_url)
+						if page > 200:
+							break
+						continue
 
 				count = 0
 				for date in html[0].날짜.values:
@@ -603,8 +622,17 @@ class StockCrawler:
 			# 최근 상장 기업의 마지막 반복되는 페이지를 제외시킨다.
 			pg_url = '{url}&page={page}'.format(url=url, page=page)
 			#html = pd.read_html(pg_url, header=0)
-			html = pd.read_html(requests.get(pg_url, headers=self.header, timeout=30).text)
-			sleep(0.5)
+			html = None
+			while True:
+				try:
+					html = pd.read_html(requests.get(pg_url, headers=self.header, timeout=30).text)
+					sleep(0.5)
+					break
+				except:
+					print(pg_url)
+					if page > 200:
+						break
+					continue
 
 			count = 0
 			for date in html[0].날짜.values: