init

stockpredictor/crawler/toSQLite/Crawler.py

@@ -2,7 +2,7 @@ import os
 import shutil
 import datetime
 import time
-import sys
+from stockpredictor.crawler.toSQLite.FnGuideCrawler import FnGuideCrawler
 from stockpredictor.crawler.toSQLite.MetaCrawler import MetaCrawler
 from stockpredictor.crawler.toSQLite.StockCrawler import StockCrawler
 from stockpredictor.analysis.Analyzer import Analyzer
@@ -16,7 +16,9 @@ PROJECT_HOME = os.path.join(os.path.dirname(os.path.join(os.path.dirname(os.path
 
 start = time.time()
 inFnguideFileName = PROJECT_HOME + '/resources/fnguide.db'
+
 """
+# 재무제표는 3개월마다 다운로드를 한다.
 crawler = FnGuideCrawler()
 print("[KOSPI 상장기업 재무제표 다운로드]")
 crawler.crawl_fnguide(inFnguideFileName)
@@ -27,6 +29,7 @@ print("\n[환율 (USD, JPY, EUR, CNY), 원유 (WTI), 국제금]")
 inFileName = PROJECT_HOME + '/resources/meta_1.db'
 crawler.crawl_stocks(inFileName)
 
+"""
 print("\n[투자자별 매매동향(Trading_Trend)]")
 inFileName = PROJECT_HOME + '/resources/meta_2.db'
 crawler.crawl_trading_trend(inFileName)
@@ -38,6 +41,7 @@ crawler.crawl_money_trend(inFileName)
 print("\n[국내 시장금리]")
 inFileName = PROJECT_HOME  + '/resources/meta_4.db'
 crawler.crawl_interest_rates(inFileName)
+"""
 
 print("\n[종목 다운로드]")
 inFileName = PROJECT_HOME + '/resources/stock.db'
@@ -45,6 +49,7 @@ crawler = StockCrawler()
 crawler.crawl_etf_stocks(inFileName)
 crawler.crawl_stocks(inFileName)
 
+
 print("\n[지수 저장]")
 kospiFileName = PROJECT_HOME + '/resources/kospi.tsv'
 kosdakFileName = PROJECT_HOME + '/resources/kosdak.tsv'

stockpredictor/crawler/toSQLite/StockCrawler.py

@@ -6,6 +6,7 @@ import json
 import sqlite3
 import requests
 import math
+import time
 
 class Queue(object):
 	def __init__(self, max):
@@ -145,13 +146,14 @@ class StockCrawler:
 		stocks.append({"NAME": 'KODEX 은행', "CODE": "091170", "PRICE": []})
 		stocks.append({"NAME": 'TIGER 탄소효율그린뉴딜', "CODE": "376410", "PRICE": []})
 
+		start_time = time.time()
 		for i, stock in enumerate(stocks):
-			print (i, stock["NAME"], stock["CODE"])
+			print (i, stock["NAME"], stock["CODE"], (time.time()-start_time), "s")
+			start_time = time.time()
 			cursor.execute('SELECT * FROM ' + tableName + ' WHERE CODE=?', (stock["CODE"],))
 			result = cursor.fetchone()
 			if result != None:
 				stock["PRICE"] = json.loads(result[2])
-
 			self.crawl_specific_stock(stock)
 			text = json.dumps(stock['PRICE'], ensure_ascii=False)
 
@@ -176,13 +178,15 @@ class StockCrawler:
 		code_df = self.getStockInfo()
 		items = code_df.values
 
+		start_time = time.time()
 		idx = 0
 		for item in items:
 			idx += 1
 
 			item_name = item[0]
 			item_code = item[1]
-			print(idx, item_name, item_code)
+			print(idx, item_name, item_code, (time.time()-start_time),"s")
+			start_time = time.time()
 
 			cursor.execute('SELECT * FROM ' + tableName + ' WHERE CODE=?', (item_code,))
 			result = cursor.fetchone()