diff --git a/stockpredictor/crawler/toSQLite/Crawler.py b/stockpredictor/crawler/toSQLite/Crawler.py index 2bfd210..4c1ef1e 100644 --- a/stockpredictor/crawler/toSQLite/Crawler.py +++ b/stockpredictor/crawler/toSQLite/Crawler.py @@ -2,7 +2,7 @@ import os import shutil import datetime import time -import sys +from stockpredictor.crawler.toSQLite.FnGuideCrawler import FnGuideCrawler from stockpredictor.crawler.toSQLite.MetaCrawler import MetaCrawler from stockpredictor.crawler.toSQLite.StockCrawler import StockCrawler from stockpredictor.analysis.Analyzer import Analyzer @@ -16,7 +16,9 @@ PROJECT_HOME = os.path.join(os.path.dirname(os.path.join(os.path.dirname(os.path start = time.time() inFnguideFileName = PROJECT_HOME + '/resources/fnguide.db' + """ +# 재무제표는 3개월마다 다운로드를 한다. crawler = FnGuideCrawler() print("[KOSPI 상장기업 재무제표 다운로드]") crawler.crawl_fnguide(inFnguideFileName) @@ -27,6 +29,7 @@ print("\n[환율 (USD, JPY, EUR, CNY), 원유 (WTI), 국제금]") inFileName = PROJECT_HOME + '/resources/meta_1.db' crawler.crawl_stocks(inFileName) +""" print("\n[투자자별 매매동향(Trading_Trend)]") inFileName = PROJECT_HOME + '/resources/meta_2.db' crawler.crawl_trading_trend(inFileName) @@ -38,6 +41,7 @@ crawler.crawl_money_trend(inFileName) print("\n[국내 시장금리]") inFileName = PROJECT_HOME + '/resources/meta_4.db' crawler.crawl_interest_rates(inFileName) +""" print("\n[종목 다운로드]") inFileName = PROJECT_HOME + '/resources/stock.db' @@ -45,6 +49,7 @@ crawler = StockCrawler() crawler.crawl_etf_stocks(inFileName) crawler.crawl_stocks(inFileName) + print("\n[지수 저장]") kospiFileName = PROJECT_HOME + '/resources/kospi.tsv' kosdakFileName = PROJECT_HOME + '/resources/kosdak.tsv' diff --git a/stockpredictor/crawler/toSQLite/StockCrawler.py b/stockpredictor/crawler/toSQLite/StockCrawler.py index 286808c..9b4f4ca 100644 --- a/stockpredictor/crawler/toSQLite/StockCrawler.py +++ b/stockpredictor/crawler/toSQLite/StockCrawler.py @@ -6,6 +6,7 @@ import json import sqlite3 import requests import math +import time class Queue(object): def __init__(self, max): @@ -145,13 +146,14 @@ class StockCrawler: stocks.append({"NAME": 'KODEX 은행', "CODE": "091170", "PRICE": []}) stocks.append({"NAME": 'TIGER 탄소효율그린뉴딜', "CODE": "376410", "PRICE": []}) + start_time = time.time() for i, stock in enumerate(stocks): - print (i, stock["NAME"], stock["CODE"]) + print (i, stock["NAME"], stock["CODE"], (time.time()-start_time), "s") + start_time = time.time() cursor.execute('SELECT * FROM ' + tableName + ' WHERE CODE=?', (stock["CODE"],)) result = cursor.fetchone() if result != None: stock["PRICE"] = json.loads(result[2]) - self.crawl_specific_stock(stock) text = json.dumps(stock['PRICE'], ensure_ascii=False) @@ -176,13 +178,15 @@ class StockCrawler: code_df = self.getStockInfo() items = code_df.values + start_time = time.time() idx = 0 for item in items: idx += 1 item_name = item[0] item_code = item[1] - print(idx, item_name, item_code) + print(idx, item_name, item_code, (time.time()-start_time),"s") + start_time = time.time() cursor.execute('SELECT * FROM ' + tableName + ' WHERE CODE=?', (item_code,)) result = cursor.fetchone()