DeepStock/StockCrawler.py

import os
import re
import sys
import shutil
import time
import sqlite3
from datetime import datetime
from dateutil.relativedelta import relativedelta
from stock.crawler.FnGuideCrawler import FnGuideCrawler
from stock.crawler.MetaCrawler import MetaCrawler
from stock.crawler.StockCrawler import StockCrawler
from stock.analysis.AnalyzerSqlite import AnalyzerSqlite
from stock.analysis.StockStatus import StockStatus
from stock.util.SlackBot import SlackBot


today = datetime.now().strftime("%Y-%m-%d")

# DB Browser for SQLite: http://hleecaster.com/python-sqlite3/

if len(sys.argv) == 2:
        PROJECT_HOME = sys.argv[1]
else:
        PROJECT_HOME = os.getcwd()

START_DATE = "2000.01.01"
start = time.time()
RESOURCE_PATH = os.path.join(PROJECT_HOME, 'resources')
stockFileName = os.path.join(RESOURCE_PATH, 'stock.db')

week = datetime.today().weekday()

slackBot = SlackBot()

if week in (0, 1, 2, 3, 4): # 0:월, 1:화, 2:수, 3:목, 4:금, 5:토, 6:일

        slackBot.sendMsg("1. start to crawl...")



        ERROR_COUNT = 0
        while ERROR_COUNT < 3:
                try:
                        # 재무제표는 3개월마다 다운로드를 한다.
                        fnGuideCrawler = FnGuideCrawler(START_DATE)
                        print("[KOSPI 상장기업 재무제표 다운로드]")
                        fnGuideCrawler.crawl_fnguide(stockFileName)
                        break
                except:
                        ERROR_COUNT += 1
                        continue
        if ERROR_COUNT >= 3:
                exit()


        metaCrawler = MetaCrawler(START_DATE)


        ERROR_COUNT = 0
        while ERROR_COUNT < 3:
                try:
                        print("\n[증시자금동향 (신용잔고, 펀드자금 잔고)]")
                        metaCrawler.crawl_money_trend(stockFileName)
                        slackBot.sendMsg("2. done meta data...")
                        break
                except:
                        ERROR_COUNT += 1
                        continue
        if ERROR_COUNT >= 3:
                exit()

        ERROR_COUNT = 0
        while ERROR_COUNT < 3:
                try:
                        print("\n[국내 시장금리]")
                        metaCrawler.crawl_interest_rates(stockFileName)
                        slackBot.sendMsg("3. done interest rates...")
                        break
                except:
                        ERROR_COUNT += 1
                        continue
        if ERROR_COUNT >= 3:
                exit()
        
        ERROR_COUNT = 0
        while ERROR_COUNT < 3:
                try:
                        print("\n[투자자별 매매동향(Trading_Trend)]")
                        metaCrawler.crawl_trading_trend(stockFileName)
                        slackBot.sendMsg("4. done trade trend...")
                        break
                except:
                        ERROR_COUNT += 1
                        continue
        if ERROR_COUNT >= 3:
                exit()



        ERROR_COUNT = 0
        while ERROR_COUNT < 3:
                try:
                        print("\n[환율 (USD, JPY, EUR, CNY, HKD, GBP, CAD, CHF, AUD, THB, INR, PHP, BRL, VND, RUB, TWD')]")
                        metaCrawler.crawl_exchange(stockFileName)
                        slackBot.sendMsg("5. done exchange data...")
                        break
                except:
                        ERROR_COUNT += 1
                        continue
        if ERROR_COUNT >= 3:
                exit()

        ERROR_COUNT = 0
        while ERROR_COUNT < 3:
                try:
                        print("\n[원유 (WTI), 국제금, COPPER, NATURALGAS, CORN, SOYBEAN 등]")
                        metaCrawler.crawl_meterials(stockFileName)
                        slackBot.sendMsg("6. done additional data...")
                        break
                except:
                        ERROR_COUNT += 1
                        continue
        if ERROR_COUNT >= 3:
                exit()



        print("\n[종목 다운로드]")
        stockCrawler = StockCrawler(START_DATE)
        ERROR_COUNT = 0
        while ERROR_COUNT < 3:
                try:
                        stockCrawler.crawl_etf_stocks(stockFileName)
                        slackBot.sendMsg("7. done etf stocks...")
                        break
                except:
                        ERROR_COUNT += 1
                        continue
        if ERROR_COUNT >= 3:
                exit()

        ERROR_COUNT = 0
        while ERROR_COUNT < 3:
                try:
                        stockCrawler.crawl_stocks(stockFileName)
                        slackBot.sendMsg("8. done stocks...")
                        break
                except:
                        ERROR_COUNT += 1
                        continue
        if ERROR_COUNT >= 3:
                exit()

        ERROR_COUNT = 0
        while ERROR_COUNT < 3:
                try:
                        stockCrawler.crawl_special_stocks(stockFileName)
                        slackBot.sendMsg("9. done US stocks...")
                        break
                except:
                        ERROR_COUNT += 1
                        continue
        if ERROR_COUNT >= 3:
                exit()

        inFileName = PROJECT_HOME + '/resources/stock.db'
        analyzerSqlite = AnalyzerSqlite(stockFileName)

        ERROR_COUNT = 0
        while ERROR_COUNT < 3:
                try:
                        print("\n[종목 분석]")
                        # S: 분석까지 진행
                        analyzerSqlite.analyzeDaily()
                        analyzerSqlite.analyzeGrouping("weekly")
                        analyzerSqlite.analyzeGrouping("monthly")
                        slackBot.sendMsg("10. analyze...")
                        break
                except:
                        ERROR_COUNT += 1
                        continue
        if ERROR_COUNT >= 3:
                exit()

        ERROR_COUNT = 0
        while ERROR_COUNT < 3:
                try:
                        print("\n[종목 결정]")
                        # HTML 출력
                        outPath = os.path.join(PROJECT_HOME, "resources", "analysis")
                        if not os.path.isdir(outPath):
                            os.mkdir(outPath)
                        day = datetime.today().strftime("%Y%m%d")
                        before_7_day = datetime.today() + relativedelta(days=-7)
                        dayList = os.listdir(outPath)
                        for dayDir in dayList:
                            if dayDir[0] != '.' and dayDir < before_7_day.strftime("%Y%m%d"):
                                if os.path.exists(os.path.join(outPath, dayDir)) and os.path.isdir(os.path.join(outPath, dayDir)):
                                    shutil.rmtree(os.path.join(outPath, dayDir))
                        outPath = os.path.join(outPath, day)
                        if os.path.isdir(outPath):
                            shutil.rmtree(outPath)
                        os.mkdir(outPath)

                        analyzerSqlite.findCandidates(outPath)
                        slackBot.sendMsg("11. done decision...")
                        break
                except:
                        ERROR_COUNT += 1
                        continue
        if ERROR_COUNT >= 3:
                exit()



print("time : %6.2f 초", (time.time() - start))
print ("done...")