DeepStock/StockCrawler.py

import os
import sys
import time
import shutil
from datetime import datetime
from dateutil.relativedelta import relativedelta

from stock.crawler.FnGuideCrawler import FnGuideCrawler
from stock.crawler.MetaCrawler import MetaCrawler
from stock.crawler.StockCrawler import StockCrawler
from AnalyzerSqlite import AnalyzerSqlite

class StockCrawlerDaily:

    PROJECT_HOME = None
    def __init__(self, PROJECT_HOME):
        self.PROJECT_HOME = PROJECT_HOME

        return

    def findCandidates(sefl, PROJECT_HOME):
        RESOURCE_PATH = os.path.join(PROJECT_HOME, 'resources')
        stockFileName = os.path.join(RESOURCE_PATH, 'stock.db')
        analyzer = AnalyzerSqlite(stockFileName)

        analyzer.analyzeDaily()
        analyzer.analyzeGrouping("weekly")
        analyzer.analyzeGrouping("monthly")

        # HTML 출력
        outPath = os.path.join(PROJECT_HOME, "resources", "analysis")
        if not os.path.isdir(outPath):
            os.mkdir(outPath)
        day = datetime.today().strftime("%Y%m%d")
        before_7_day = datetime.today() + relativedelta(days=-7)
        dayList = os.listdir(outPath)
        for dayDir in dayList:
            if dayDir[0] != '.' and dayDir < before_7_day.strftime("%Y%m%d"):
                if os.path.exists(os.path.join(outPath, dayDir)) and os.path.isdir(os.path.join(outPath, dayDir)):
                    shutil.rmtree(os.path.join(outPath, dayDir))

        outPath = os.path.join(outPath, day)

        if os.path.isdir(outPath):
            shutil.rmtree(outPath)
        os.mkdir(outPath)
        print("print to Html...")
        analyzer.findCandidates(outPath)

        return

    def crawl(self):

        START_DATE = "2000.01.01"
        RESOURCE_PATH = os.path.join(self.PROJECT_HOME, 'resources')
        stockFileName = os.path.join(RESOURCE_PATH, 'stock.db')

        ERROR_COUNT = 0
        while ERROR_COUNT < 3:
            try:
                # 재무제표는 3개월마다 다운로드를 한다.
                fnGuideCrawler = FnGuideCrawler(START_DATE)
                print("[KOSPI 상장기업 재무제표 다운로드]")
                fnGuideCrawler.crawl_fnguide(stockFileName)
                break
            except:
                ERROR_COUNT += 1
                continue
        if ERROR_COUNT >= 3:
            exit()

        metaCrawler = MetaCrawler(START_DATE)

        ERROR_COUNT = 0
        while ERROR_COUNT < 3:
            try:
                print("\n[증시자금동향 (신용잔고, 펀드자금 잔고)]")
                metaCrawler.crawl_money_trend(stockFileName)
                break
            except:
                ERROR_COUNT += 1
                continue
        if ERROR_COUNT >= 3:
            exit()

        ERROR_COUNT = 0
        while ERROR_COUNT < 3:
            try:
                print("\n[국내 시장금리]")
                metaCrawler.crawl_interest_rates(stockFileName)
                break
            except:
                ERROR_COUNT += 1
                continue
        if ERROR_COUNT >= 3:
            exit()

        ERROR_COUNT = 0
        while ERROR_COUNT < 3:
            try:
                print("\n[투자자별 매매동향(Trading_Trend)]")
                metaCrawler.crawl_trading_trend(stockFileName)
                break
            except:
                ERROR_COUNT += 1
                continue
        if ERROR_COUNT >= 3:
            exit()

        ERROR_COUNT = 0
        while ERROR_COUNT < 3:
            try:
                print("\n[환율 (USD, JPY, EUR, CNY, HKD, GBP, CAD, CHF, AUD, THB, INR, PHP, BRL, VND, RUB, TWD')]")
                metaCrawler.crawl_exchange(stockFileName)
                break
            except:
                ERROR_COUNT += 1
                continue
        if ERROR_COUNT >= 3:
            exit()

        ERROR_COUNT = 0
        while ERROR_COUNT < 3:
            try:
                print("\n[원유 (WTI), 국제금, COPPER, NATURALGAS, CORN, SOYBEAN 등]")
                metaCrawler.crawl_meterials(stockFileName)
                break
            except:
                ERROR_COUNT += 1
                continue
        if ERROR_COUNT >= 3:
            exit()

        stockCrawler = StockCrawler(START_DATE)

        print("\n[국내 종목 수집]")
        stockCrawler.crawl_stocks(stockFileName)

        print("\n[국내 ETF 수집]")
        stockCrawler.crawl_etf_stocks(stockFileName)

        print("\n[US 종목 수집]")
        stockCrawler.crawl_special_stocks(stockFileName)

        return


if __name__ == "__main__":
    # DB Browser for SQLite: http://hleecaster.com/python-sqlite3/

    if len(sys.argv) == 2:
        PROJECT_HOME = sys.argv[1]
    else:
        PROJECT_HOME = os.getcwd()

    START_DATE = "2000.01.01"
    stockCrawlerDaily = StockCrawlerDaily(PROJECT_HOME)

    week = datetime.today().weekday()
    if week in (0, 1, 2, 3, 4):  # 0:월, 1:화, 2:수, 3:목, 4:금, 5:토, 6:일
        start = time.time()
        stockCrawlerDaily.crawl()

        print("\n[후보 결정]")
        stockCrawlerDaily.findCandidates(PROJECT_HOME)

        today = datetime.now().strftime("%Y-%m-%d")
        print("time : %6.2f 초", (time.time() - start))
        print("done...")