timeout 5초 추가

stockpredictor/crawler/sQLite/Crawler.py

@@ -44,9 +44,9 @@ metaCrawler.crawl_meterials(stockFileName)
 
 print("\n[종목 다운로드]")
 stockCrawler = StockCrawler(START_DATE)
-#stockCrawler.crawl_etf_stocks(stockFileName)
-#stockCrawler.crawl_stocks(stockFileName)
-stockCrawler.crawl_us_stocks(stockFileName)
+stockCrawler.crawl_etf_stocks(stockFileName)
+stockCrawler.crawl_stocks(stockFileName)
+stockCrawler.crawl_special_stocks(stockFileName)
 
 
 print("\n[종목 분석]")

stockpredictor/crawler/sQLite/MetaCrawler.py

@@ -48,7 +48,14 @@ class MetaCrawler:
             finish = False
             for i in range(1, self.limit_page_count):
                 #html = pd.read_html(input['URL'] + '&page=%s' % i, header=0)
-                html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header).text)
+                html = None
+                while True:
+                    try:
+                        html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header, timeout=5).text)
+                        break
+                    except:
+                        print(input['URL'] + '&page=%s' % i)
+                        continue
 
                 # 마지막 페이지 까지 받기
                 if len(html[0].날짜.values) <= 1:
@@ -124,7 +131,14 @@ class MetaCrawler:
         finish = False
         for i in range(1, self.limit_page_count):
             #html = pd.read_html(url + str(i), header=0)
-            html = pd.read_html(requests.get(url + str(i), headers=self.header).text)
+            html = None
+            while True:
+                try:
+                    html = pd.read_html(requests.get(url + str(i), headers=self.header, timeout=5).text)
+                    break
+                except:
+                    print(url + str(i))
+                    continue
 
             # 마지막 페이지 까지 받기
             if len(html[0].날짜.values) <= 2:
@@ -194,7 +208,14 @@ class MetaCrawler:
         finish = False
         for i in range(1, self.limit_page_count):
             #html = pd.read_html(url + str(i), header=0, encoding='euc-kr')
-            html = pd.read_html(requests.get(url + str(i), headers=self.header).text, encoding='euc-kr')
+            html = None
+            while True:
+                try:
+                    html = pd.read_html(requests.get(url + str(i), headers=self.header, timeout=5).text, encoding='euc-kr')
+                    break
+                except:
+                    print(url + str(i))
+                    continue
 
             # 마지막 페이지 까지 받기
             if len(html[0].날짜.values) <= 10:
@@ -273,7 +294,14 @@ class MetaCrawler:
             finish = False
             for i in range(1, self.limit_page_count):
                 #html = pd.read_html(input['URL'] + '&page=%s' % i, header=0)
-                html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header).text)
+                html = None
+                while True:
+                    try:
+                        html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header, timeout=5).text)
+                        break
+                    except:
+                        print (input['URL'] + '&page=%s' % i)
+                        continue
 
                 ymd, close, diff, rate = "", 0.0, 0.0, 0.0
                 # 마지막 페이지 까지 받기
@@ -347,7 +375,14 @@ class MetaCrawler:
             finish = False
             for i in range(1, self.limit_page_count):
                 #html = pd.read_html(input['URL'] + '&page=%s' % i, header=0)
-                html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header).text)
+                html = None
+                while True:
+                    try:
+                        html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header, timeout=5).text)
+                        break
+                    except:
+                        print(input['URL'] + '&page=%s' % i)
+                        continue
 
                 # 마지막 페이지 까지 받기
                 if len(html[0].날짜.values) <= 1:

stockpredictor/crawler/sQLite/StockCrawler.py

@@ -45,7 +45,7 @@ class StockCrawler:
 
 	def getStockInfo(self):
 		#code_df = pd.read_html('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', header=0)[0]
-		code_df = pd.read_html(requests.get('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', headers=self.header).text)[0]
+		code_df = pd.read_html(requests.get('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', headers=self.header, timeout=5).text)[0]
 		# code_df = pd.read_excel('../resources/stock/상장법인목록.xls')
 
 		# 종목코드가 6자리이기 때문에 6자리를 맞춰주기 위해 설정해줌
@@ -207,7 +207,7 @@ class StockCrawler:
 
 		return
 
-	def crawl_us_stocks(self, inFileName):
+	def crawl_special_stocks(self, inFileName):
 		tableName = 'stock'
 		conn = sqlite3.connect(inFileName)
 		cursor = conn.cursor()
@@ -228,7 +228,10 @@ class StockCrawler:
 		pd.options.display.float_format = '{:.4f}'.format
 		pd.set_option('display.max_columns', None)
 
-		us_stocks = {
+		special_stocks = {
+			'^KS11': 'Kospi',
+			'^KQ11': 'Kosdak',
+
 			'SQQQ': 'ProShares UltraPro Short QQQ',
 			'TQQQ': 'ProShares UltraPro QQQ',
 			'SCO': 'ProShares UltraShort Bloomberg Crude Oil',
@@ -286,21 +289,28 @@ class StockCrawler:
 		cursor = conn.cursor()
 
 		us_sotck_data = {}
-		for ticker in us_stocks:
+		for ticker in special_stocks:
 
 			cursor.execute('SELECT * FROM ' + tableName + ' WHERE CODE=?', (ticker,))
 
 			result = cursor.fetchone()
 			if result == None:
-				#start = pd.to_datetime('2017-01-01')
-				start = pd.to_datetime(self.START_DATE.replace(".", "-"))
-				end = pd.to_datetime(datetime.today().strftime('%Y-%m-%d'))
+				#start_day = pd.to_datetime('2017-01-01')
+				start_day = pd.to_datetime(self.START_DATE.replace(".", "-"))
+				end_day = pd.to_datetime(datetime.today().strftime('%Y-%m-%d'))
 			else:
 				#start = (datetime.today() - timedelta(days=300)).strftime('%Y-%m-%d')
-				start = pd.to_datetime(self.START_DATE.replace(".", "-"))
-				end = pd.to_datetime(datetime.today().strftime('%Y-%m-%d'))
+				start_day = pd.to_datetime(self.START_DATE.replace(".", "-"))
+				end_day = pd.to_datetime(datetime.today().strftime('%Y-%m-%d'))
 
-			data = pdr.get_data_yahoo(ticker, start, end)
+			data = None
+			while True:
+				try:
+					data = pdr.get_data_yahoo(ticker, start_day, end_day)
+					break
+				except:
+					print(ticker)
+					continue
 			us_sotck_data[ticker] = {
 				'close': data['Close'].to_dict(),
 				'open': data['Open'].to_dict(),
@@ -369,7 +379,15 @@ class StockCrawler:
 			# 최근 상장 기업의 마지막 반복되는 페이지를 제외시킨다.
 			pg_url = '{url}&page={page}'.format(url=url, page=page)
 			#html = pd.read_html(pg_url, header=0)
-			html = pd.read_html(requests.get(pg_url, headers=self.header).text)
+			html = None
+			while True:
+				try:
+					html = pd.read_html(requests.get(pg_url, headers=self.header, timeout=5).text)
+					break
+				except:
+					print(pg_url)
+					continue
+
 			for date in html[0].날짜.values:
 				if type(date) is str:
 					if date in date_set:

stockpredictor/crawler/toJsonFile/FnGuideCrawler.py

@@ -10,7 +10,7 @@ class FnGuideCrawler:
 
     def getStockInfo(self):
         #code_df = pd.read_html('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', header=0)[0]
-        code_df = pd.read_html(requests.get('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', headers=self.header).text)[0]
+        code_df = pd.read_html(requests.get('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', headers=self.header, timeout=5).text)[0]
         # code_df = pd.read_excel('../resources/stock/상장법인목록.xls')
 
         # 종목코드가 6자리이기 때문에 6자리를 맞춰주기 위해 설정해줌

stockpredictor/crawler/toJsonFile/MetaCrawler.py

@@ -46,7 +46,7 @@ class MetaCrawler:
             finish = False
             for i in range(1, self.limit_page_count):
                 #html = pd.read_html(input['URL'] + '&page=%s' % i, header=0)
-                html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header).text)
+                html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header, timeout=5).text)
 
                 # 마지막 페이지 까지 받기
                 if len(html[0].날짜.values) <= 1:
@@ -119,7 +119,7 @@ class MetaCrawler:
         finish = False
         for i in range(1, self.limit_page_count):
             #html = pd.read_html(url + str(i), header=0)
-            html = pd.read_html(requests.get(url + str(i), headers=self.header).text)
+            html = pd.read_html(requests.get(url + str(i), headers=self.header, timeout=5).text)
 
             # 마지막 페이지 까지 받기
             if len(html[0].날짜.values) <= 2:
@@ -179,7 +179,7 @@ class MetaCrawler:
         finish = False
         for i in range(1, self.limit_page_count):
             #html = pd.read_html(url + str(i), header=0, encoding='euc-kr')
-            html = pd.read_html(requests.get(url + str(i), headers=self.header).text, encoding='euc-kr')
+            html = pd.read_html(requests.get(url + str(i), headers=self.header, timeout=5).text, encoding='euc-kr')
 
             # 마지막 페이지 까지 받기
             if len(html[0].날짜.values) <= 10:
@@ -250,7 +250,7 @@ class MetaCrawler:
             finish = False
             for i in range(1, self.limit_page_count):
                 #html = pd.read_html(input['URL'] + '&page=%s' % i, header=0)
-                html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header).text)
+                html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header, timeout=5).text)
 
                 # 마지막 페이지 까지 받기
                 if len(html[0].날짜.values) <= 1:

stockpredictor/crawler/toJsonFile/StockCrawler.py

@@ -74,7 +74,7 @@ class StockCrawler:
 
 	def getStockInfo(self):
 		#code_df = pd.read_html('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', header=0)[0]
-		code_df = pd.read_html(requests.get('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', headers=self.header).text)[0]
+		code_df = pd.read_html(requests.get('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', headers=self.header, timeout=5).text)[0]
 		# code_df = pd.read_excel('../resources/stock/상장법인목록.xls')
 
 		# 종목코드가 6자리이기 때문에 6자리를 맞춰주기 위해 설정해줌
@@ -255,7 +255,7 @@ class StockCrawler:
 			# 최근 상장 기업의 마지막 반복되는 페이지를 제외시킨다.
 			pg_url = '{url}&page={page}'.format(url=url, page=page)
 			#html = pd.read_html(pg_url, header=0)
-			html = pd.read_html(requests.get(pg_url, headers=self.header).text)
+			html = pd.read_html(requests.get(pg_url, headers=self.header, timeout=5).text)
 
 			count = 0
 			for date in html[0].날짜.values:
@@ -378,7 +378,7 @@ class StockCrawler:
 				# 최근 상장 기업의 마지막 반복되는 페이지를 제외시킨다.
 				pg_url = '{url}&page={page}'.format(url=url, page=page)
 				#html = pd.read_html(pg_url, header=0)
-				html = pd.read_html(requests.get(pg_url, headers=self.header).text)
+				html = pd.read_html(requests.get(pg_url, headers=self.header, timeout=5).text)
 				count = 0
 				for date in html[0].날짜.values:
 					if type(date) is str:
@@ -600,7 +600,7 @@ class StockCrawler:
 			# 최근 상장 기업의 마지막 반복되는 페이지를 제외시킨다.
 			pg_url = '{url}&page={page}'.format(url=url, page=page)
 			#html = pd.read_html(pg_url, header=0)
-			html = pd.read_html(requests.get(pg_url, headers=self.header).text)
+			html = pd.read_html(requests.get(pg_url, headers=self.header, timeout=5).text)
 			count = 0
 			for date in html[0].날짜.values:
 				if type(date) is str:

stockpredictor/crawler/toSQLite/MetaCrawler.py

@@ -46,7 +46,7 @@ class MetaCrawler:
             finish = False
             for i in range(1, self.limit_page_count):
                 #html = pd.read_html(input['URL'] + '&page=%s' % i, header=0)
-                html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header).text)
+                html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header, timeout=5).text)
 
                 # 마지막 페이지 까지 받기
                 if len(html[0].날짜.values) <= 1:
@@ -125,7 +125,7 @@ class MetaCrawler:
         finish = False
         for i in range(1, self.limit_page_count):
             #html = pd.read_html(url + str(i), header=0)
-            html = pd.read_html(requests.get(url + str(i), headers=self.header).text)
+            html = pd.read_html(requests.get(url + str(i), headers=self.header, timeout=5).text)
 
             # 마지막 페이지 까지 받기
             if len(html[0].날짜.values) <= 2:
@@ -190,7 +190,7 @@ class MetaCrawler:
         finish = False
         for i in range(1, self.limit_page_count):
             #html = pd.read_html(url + str(i), header=0, encoding='euc-kr')
-            html = pd.read_html(requests.get(url + str(i), headers=self.header).text, encoding='euc-kr')
+            html = pd.read_html(requests.get(url + str(i), headers=self.header, timeout=5).text, encoding='euc-kr')
 
             # 마지막 페이지 까지 받기
             if len(html[0].날짜.values) <= 10:
@@ -264,7 +264,7 @@ class MetaCrawler:
             finish = False
             for i in range(1, self.limit_page_count):
                 #html = pd.read_html(input['URL'] + '&page=%s' % i, header=0)
-                html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header).text)
+                html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header, timeout=5).text)
 
                 # 마지막 페이지 까지 받기
                 if len(html[0].날짜.values) <= 1:

stockpredictor/crawler/toSQLite/StockCrawler.py

@@ -75,7 +75,7 @@ class StockCrawler:
 
 	def getStockInfo(self):
 		#code_df = pd.read_html('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', header=0)[0]
-		code_df = pd.read_html(requests.get('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', headers=self.header).text)[0]
+		code_df = pd.read_html(requests.get('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', headers=self.header, timeout=5).text)[0]
 		# code_df = pd.read_excel('../resources/stock/상장법인목록.xls')
 
 		# 종목코드가 6자리이기 때문에 6자리를 맞춰주기 위해 설정해줌
@@ -228,7 +228,7 @@ class StockCrawler:
 			# 최근 상장 기업의 마지막 반복되는 페이지를 제외시킨다.
 			pg_url = '{url}&page={page}'.format(url=url, page=page)
 			#html = pd.read_html(pg_url, header=0)
-			html = pd.read_html(requests.get(pg_url, headers=self.header).text)
+			html = pd.read_html(requests.get(pg_url, headers=self.header, timeout=5).text)
 			for date in html[0].날짜.values:
 				if type(date) is str:
 					if date in date_set: