init

stockpredictor/crawler/sQLite/StockCrawler.py

@@ -21,14 +21,15 @@ class StockCrawler:
 	special_pattern = None
 	fnGuideCrawler = None
 
-	limit_page_count = 10000
+	limit_page_count = 100000
+	START_DATE = None
 
-	def __init__(self):
+	def __init__(self, START_DATE):
 		self.historical_prices = dict()
 		self.special_pattern = (
 		'[', '!', '@', '#', '$', '%', '^', '&', '*', '(', ')', ',', '.', '?', '"', ':', ';', '{', '}', '|', '<', '>',
 		']', '+', '-', '/', '=', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9')
-
+		self.START_DATE = START_DATE
 		return
 
 	def clean_str(self, string):
@@ -127,7 +128,7 @@ class StockCrawler:
 			start_time = time.time()
 			cursor.execute('SELECT ymd FROM ' + tableName + ' WHERE CODE=? order by ymd desc', (stock["CODE"],))
 			result = cursor.fetchone()
-			ymd = "2019.01.01"
+			ymd = self.START_DATE
 			if result is not None:
 				ymd = result[0]
 
@@ -181,7 +182,8 @@ class StockCrawler:
 			result = cursor.fetchone()
 			stock = {"CODE": item_code, "NAME": item_name}
 
-			ymd = (datetime.today() - timedelta(days=300)).strftime('%Y-%m-%d')
+			#ymd = (datetime.today() - timedelta(days=300)).strftime('%Y-%m-%d')
+			ymd = self.START_DATE.replace(".", "-")
 			if result is not None:
 				ymd = result[0]
 
@@ -248,10 +250,12 @@ class StockCrawler:
 
 			result = cursor.fetchone()
 			if result == None:
-				start = pd.to_datetime('2017-01-01')
+				#start = pd.to_datetime('2017-01-01')
+				start = pd.to_datetime(self.START_DATE.replace(".", "-"))
 				end = pd.to_datetime(datetime.today().strftime('%Y-%m-%d'))
 			else:
-				start = (datetime.today() - timedelta(days=300)).strftime('%Y-%m-%d')
+				#start = (datetime.today() - timedelta(days=300)).strftime('%Y-%m-%d')
+				start = pd.to_datetime(self.START_DATE.replace(".", "-"))
 				end = pd.to_datetime(datetime.today().strftime('%Y-%m-%d'))
 
 			data = pdr.get_data_yahoo(ticker, start, end)