init

stockpredictor/crawler/sQLite/StockCrawler.py

@@ -5,7 +5,7 @@ import re
 import json
 import sqlite3
 import requests
-import math
+from time import sleep
 import time
 
 # 닐짜 형식으로 바뀐 this_date값을 확인 가능
@@ -126,7 +126,7 @@ class StockCrawler:
 			cursor.execute('SELECT ymd FROM ' + tableName + ' WHERE CODE=? order by ymd desc', (stock["CODE"],))
 			result = cursor.fetchone()
 			ymd = "2019.01.01"
-			if result != None:
+			if result is not None:
 				ymd = result[0]
 
 			stock_data = self.crawl_specific_stock(stock["CODE"], ymd)
@@ -139,6 +139,7 @@ class StockCrawler:
 				#else:
 				#	cursor.execute("UPDATE " + tableName + " SET close=?, diff=?, open=?, high=?, low=?, volume=? WHERE CODE=? and ymd=?", (item['close'], item['diff'], item['open'], item['high'], item['low'], item['volume'], stock["CODE"], item['ymd']))
 
+			sleep(0.05)
 		conn.commit()
 		cursor.close()
 		conn.close()
@@ -166,19 +167,20 @@ class StockCrawler:
 		start_time = time.time()
 		idx = 0
 		for item in items:
+			idx += 1
+
 			conn = sqlite3.connect(inFileName)
 			cursor = conn.cursor()
 
-			idx += 1
-
 			item_name = item[0]
 			item_code = item[1]
 
 			cursor.execute('SELECT ymd FROM ' + tableName + ' WHERE CODE=? order by ymd desc', (item_code,))
 			result = cursor.fetchone()
 			stock = {"CODE": item_code, "NAME": item_name}
+
 			ymd = "2020.01.01"
-			if result != None:
+			if result is not None:
 				ymd = result[0]
 
 			stock_data = self.crawl_specific_stock(item_code, ymd)
@@ -197,6 +199,7 @@ class StockCrawler:
 			print(idx, item_name, item_code, (time.time() - start_time), "s")
 
 			start_time = time.time()
+			sleep(0.05)
 
 		return
 
@@ -228,7 +231,8 @@ class StockCrawler:
 						df = df.append(html[0], ignore_index=True)
 						break
 			df = df.append(html[0], ignore_index=True)
-			if lastPage:
+			df = df.dropna()
+			if lastPage or len(df) < 1 or df.날짜[0]=='':
 				print("\t- lastpage:", page)
 				break
 
@@ -319,7 +323,7 @@ class StockCrawler:
 		lastDay = ""
 		cursor.execute('SELECT * FROM ' + tableName + ' WHERE CODE=?', (stock["CODE"],))
 		result = cursor.fetchone()
-		if result != None:
+		if result is not None:
 			stock["PRICE"] = json.loads(result[2])
 			lastDay = stock["PRICE"][len(stock["PRICE"]) - 1]["DATE"]