init
This commit is contained in:
dsyoon
206
StockCrawler.py
206
StockCrawler.py
@@ -10,153 +10,123 @@ from stock.crawler.FnGuideCrawler import FnGuideCrawler
|
|||||||
from stock.crawler.MetaCrawler import MetaCrawler
|
from stock.crawler.MetaCrawler import MetaCrawler
|
||||||
from stock.crawler.StockCrawler import StockCrawler
|
from stock.crawler.StockCrawler import StockCrawler
|
||||||
from stock.analysis.AnalyzerSqlite import AnalyzerSqlite
|
from stock.analysis.AnalyzerSqlite import AnalyzerSqlite
|
||||||
from stock.analysis.StockStatus import StockStatus
|
|
||||||
from stock.util.TelegramBot import TelegramBot
|
from stock.util.TelegramBot import TelegramBot
|
||||||
|
|
||||||
|
|
||||||
today = datetime.now().strftime("%Y-%m-%d")
|
class StockCrawlerDaily:
|
||||||
|
|
||||||
# DB Browser for SQLite: http://hleecaster.com/python-sqlite3/
|
PROJECT_HOME = None
|
||||||
|
bot = None
|
||||||
|
|
||||||
if len(sys.argv) == 2:
|
def __init__(self, PROJECT_HOME):
|
||||||
PROJECT_HOME = sys.argv[1]
|
self.PROJECT_HOME = PROJECT_HOME
|
||||||
else:
|
self.bot = TelegramBot()
|
||||||
PROJECT_HOME = os.getcwd()
|
|
||||||
|
|
||||||
START_DATE = "2000.01.01"
|
return
|
||||||
start = time.time()
|
|
||||||
RESOURCE_PATH = os.path.join(PROJECT_HOME, 'resources')
|
|
||||||
stockFileName = os.path.join(RESOURCE_PATH, 'stock.db')
|
|
||||||
|
|
||||||
week = datetime.today().weekday()
|
def crawl(self):
|
||||||
|
|
||||||
bot = TelegramBot()
|
START_DATE = "2000.01.01"
|
||||||
|
RESOURCE_PATH = os.path.join(self.PROJECT_HOME, 'resources')
|
||||||
|
stockFileName = os.path.join(RESOURCE_PATH, 'stock.db')
|
||||||
|
|
||||||
if week in (0, 1, 2, 3, 4): # 0:월, 1:화, 2:수, 3:목, 4:금, 5:토, 6:일
|
self.bot.sendMsg("1. start to crawl...")
|
||||||
|
|
||||||
bot.sendMsg("1. start to crawl...")
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
ERROR_COUNT = 0
|
ERROR_COUNT = 0
|
||||||
while ERROR_COUNT < 3:
|
while ERROR_COUNT < 3:
|
||||||
try:
|
try:
|
||||||
# 재무제표는 3개월마다 다운로드를 한다.
|
# 재무제표는 3개월마다 다운로드를 한다.
|
||||||
fnGuideCrawler = FnGuideCrawler(START_DATE)
|
fnGuideCrawler = FnGuideCrawler(START_DATE)
|
||||||
print("[KOSPI 상장기업 재무제표 다운로드]")
|
print("[KOSPI 상장기업 재무제표 다운로드]")
|
||||||
fnGuideCrawler.crawl_fnguide(stockFileName)
|
fnGuideCrawler.crawl_fnguide(stockFileName)
|
||||||
break
|
break
|
||||||
except:
|
except:
|
||||||
ERROR_COUNT += 1
|
ERROR_COUNT += 1
|
||||||
continue
|
continue
|
||||||
if ERROR_COUNT >= 3:
|
if ERROR_COUNT >= 3:
|
||||||
exit()
|
exit()
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
metaCrawler = MetaCrawler(START_DATE)
|
metaCrawler = MetaCrawler(START_DATE)
|
||||||
|
|
||||||
|
ERROR_COUNT = 0
|
||||||
|
while ERROR_COUNT < 3:
|
||||||
|
try:
|
||||||
|
print("\n[증시자금동향 (신용잔고, 펀드자금 잔고)]")
|
||||||
|
metaCrawler.crawl_money_trend(stockFileName)
|
||||||
|
self.bot.sendMsg("2. done meta data...")
|
||||||
|
break
|
||||||
|
except:
|
||||||
|
ERROR_COUNT += 1
|
||||||
|
continue
|
||||||
|
if ERROR_COUNT >= 3:
|
||||||
|
exit()
|
||||||
|
|
||||||
ERROR_COUNT = 0
|
ERROR_COUNT = 0
|
||||||
while ERROR_COUNT < 3:
|
while ERROR_COUNT < 3:
|
||||||
try:
|
try:
|
||||||
print("\n[증시자금동향 (신용잔고, 펀드자금 잔고)]")
|
print("\n[국내 시장금리]")
|
||||||
metaCrawler.crawl_money_trend(stockFileName)
|
metaCrawler.crawl_interest_rates(stockFileName)
|
||||||
bot.sendMsg("2. done meta data...")
|
self.bot.sendMsg("3. done interest rates...")
|
||||||
break
|
break
|
||||||
except:
|
except:
|
||||||
ERROR_COUNT += 1
|
ERROR_COUNT += 1
|
||||||
continue
|
continue
|
||||||
if ERROR_COUNT >= 3:
|
if ERROR_COUNT >= 3:
|
||||||
exit()
|
exit()
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
ERROR_COUNT = 0
|
ERROR_COUNT = 0
|
||||||
while ERROR_COUNT < 3:
|
while ERROR_COUNT < 3:
|
||||||
try:
|
try:
|
||||||
print("\n[국내 시장금리]")
|
print("\n[투자자별 매매동향(Trading_Trend)]")
|
||||||
metaCrawler.crawl_interest_rates(stockFileName)
|
metaCrawler.crawl_trading_trend(stockFileName)
|
||||||
bot.sendMsg("3. done interest rates...")
|
self.bot.sendMsg("4. done trade trend...")
|
||||||
break
|
break
|
||||||
except:
|
except:
|
||||||
ERROR_COUNT += 1
|
ERROR_COUNT += 1
|
||||||
continue
|
continue
|
||||||
if ERROR_COUNT >= 3:
|
if ERROR_COUNT >= 3:
|
||||||
exit()
|
exit()
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
ERROR_COUNT = 0
|
ERROR_COUNT = 0
|
||||||
while ERROR_COUNT < 3:
|
while ERROR_COUNT < 3:
|
||||||
try:
|
try:
|
||||||
print("\n[투자자별 매매동향(Trading_Trend)]")
|
print("\n[환율 (USD, JPY, EUR, CNY, HKD, GBP, CAD, CHF, AUD, THB, INR, PHP, BRL, VND, RUB, TWD')]")
|
||||||
metaCrawler.crawl_trading_trend(stockFileName)
|
metaCrawler.crawl_exchange(stockFileName)
|
||||||
bot.sendMsg("4. done trade trend...")
|
self.bot.sendMsg("5. done exchange data...")
|
||||||
break
|
break
|
||||||
except:
|
except:
|
||||||
ERROR_COUNT += 1
|
ERROR_COUNT += 1
|
||||||
continue
|
continue
|
||||||
if ERROR_COUNT >= 3:
|
if ERROR_COUNT >= 3:
|
||||||
exit()
|
exit()
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
ERROR_COUNT = 0
|
ERROR_COUNT = 0
|
||||||
while ERROR_COUNT < 3:
|
while ERROR_COUNT < 3:
|
||||||
try:
|
try:
|
||||||
print("\n[환율 (USD, JPY, EUR, CNY, HKD, GBP, CAD, CHF, AUD, THB, INR, PHP, BRL, VND, RUB, TWD')]")
|
print("\n[원유 (WTI), 국제금, COPPER, NATURALGAS, CORN, SOYBEAN 등]")
|
||||||
metaCrawler.crawl_exchange(stockFileName)
|
metaCrawler.crawl_meterials(stockFileName)
|
||||||
bot.sendMsg("5. done exchange data...")
|
self.bot.sendMsg("6. done additional data...")
|
||||||
break
|
break
|
||||||
except:
|
except:
|
||||||
ERROR_COUNT += 1
|
ERROR_COUNT += 1
|
||||||
continue
|
continue
|
||||||
if ERROR_COUNT >= 3:
|
if ERROR_COUNT >= 3:
|
||||||
exit()
|
exit()
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
ERROR_COUNT = 0
|
|
||||||
while ERROR_COUNT < 3:
|
|
||||||
try:
|
|
||||||
print("\n[원유 (WTI), 국제금, COPPER, NATURALGAS, CORN, SOYBEAN 등]")
|
|
||||||
metaCrawler.crawl_meterials(stockFileName)
|
|
||||||
bot.sendMsg("6. done additional data...")
|
|
||||||
break
|
|
||||||
except:
|
|
||||||
ERROR_COUNT += 1
|
|
||||||
continue
|
|
||||||
if ERROR_COUNT >= 3:
|
|
||||||
exit()
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
stockCrawler = StockCrawler(START_DATE)
|
stockCrawler = StockCrawler(START_DATE)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
print("\n[국내 ETF 수집]")
|
print("\n[국내 ETF 수집]")
|
||||||
stockCrawler.crawl_etf_stocks(stockFileName)
|
stockCrawler.crawl_etf_stocks(stockFileName)
|
||||||
bot.sendMsg("7. done etf stocks...")
|
self.bot.sendMsg("7. done etf stocks...")
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
print("\n[국내 종목 수집]")
|
print("\n[국내 종목 수집]")
|
||||||
stockCrawler.crawl_stocks(stockFileName)
|
stockCrawler.crawl_stocks(stockFileName)
|
||||||
bot.sendMsg("8. done stocks...")
|
self.bot.sendMsg("8. done stocks...")
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
print("\n[US 종목 수집]")
|
print("\n[US 종목 수집]")
|
||||||
stockCrawler.crawl_special_stocks(stockFileName)
|
stockCrawler.crawl_special_stocks(stockFileName)
|
||||||
bot.sendMsg("9. done US stocks...")
|
self.bot.sendMsg("9. done US stocks...")
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
inFileName = PROJECT_HOME + '/resources/stock.db'
|
|
||||||
analyzerSqlite = AnalyzerSqlite(stockFileName)
|
analyzerSqlite = AnalyzerSqlite(stockFileName)
|
||||||
|
|
||||||
print("\n[종목 분석]")
|
print("\n[종목 분석]")
|
||||||
@@ -164,13 +134,11 @@ if week in (0, 1, 2, 3, 4): # 0:월, 1:화, 2:수, 3:목, 4:금, 5:토, 6:일
|
|||||||
analyzerSqlite.analyzeDaily()
|
analyzerSqlite.analyzeDaily()
|
||||||
analyzerSqlite.analyzeGrouping("weekly")
|
analyzerSqlite.analyzeGrouping("weekly")
|
||||||
analyzerSqlite.analyzeGrouping("monthly")
|
analyzerSqlite.analyzeGrouping("monthly")
|
||||||
bot.sendMsg("10. analyze...")
|
self.bot.sendMsg("10. analyze...")
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
print("\n[종목 결정]")
|
print("\n[종목 결정]")
|
||||||
# HTML 출력
|
# HTML 출력
|
||||||
outPath = os.path.join(PROJECT_HOME, "resources", "analysis")
|
outPath = os.path.join(self.PROJECT_HOME, "resources", "analysis")
|
||||||
if not os.path.isdir(outPath):
|
if not os.path.isdir(outPath):
|
||||||
os.mkdir(outPath)
|
os.mkdir(outPath)
|
||||||
day = datetime.today().strftime("%Y%m%d")
|
day = datetime.today().strftime("%Y%m%d")
|
||||||
@@ -186,8 +154,26 @@ if week in (0, 1, 2, 3, 4): # 0:월, 1:화, 2:수, 3:목, 4:금, 5:토, 6:일
|
|||||||
os.mkdir(outPath)
|
os.mkdir(outPath)
|
||||||
|
|
||||||
analyzerSqlite.findCandidates(outPath)
|
analyzerSqlite.findCandidates(outPath)
|
||||||
bot.sendMsg("11. done decision...")
|
self.bot.sendMsg("11. done decision...")
|
||||||
|
|
||||||
|
return
|
||||||
|
|
||||||
|
|
||||||
print("time : %6.2f 초", (time.time() - start))
|
if __name__ == "__main__":
|
||||||
print ("done...")
|
# DB Browser for SQLite: http://hleecaster.com/python-sqlite3/
|
||||||
|
|
||||||
|
if len(sys.argv) == 2:
|
||||||
|
PROJECT_HOME = sys.argv[1]
|
||||||
|
else:
|
||||||
|
PROJECT_HOME = os.getcwd()
|
||||||
|
|
||||||
|
START_DATE = "2000.01.01"
|
||||||
|
stockCrawlerDaily = StockCrawlerDaily(PROJECT_HOME)
|
||||||
|
|
||||||
|
week = datetime.today().weekday()
|
||||||
|
if week in (0, 1, 2, 3, 4, 5): # 0:월, 1:화, 2:수, 3:목, 4:금, 5:토, 6:일
|
||||||
|
start = time.time()
|
||||||
|
stockCrawlerDaily.crawl()
|
||||||
|
today = datetime.now().strftime("%Y-%m-%d")
|
||||||
|
print("time : %6.2f 초", (time.time() - start))
|
||||||
|
print("done...")
|
||||||
|
|||||||
@@ -23,9 +23,9 @@ class TelegramBot:
|
|||||||
username for the bot: ncue_stock_bot
|
username for the bot: ncue_stock_bot
|
||||||
token to access the HTTP API: 6874078562:AAEHxGDavfc0ssAXPQIaW8JGYmTR7LNUJOw
|
token to access the HTTP API: 6874078562:AAEHxGDavfc0ssAXPQIaW8JGYmTR7LNUJOw
|
||||||
"""
|
"""
|
||||||
self.botname = "coinbot"
|
self.botname = "stockbot"
|
||||||
self.username = "ncue_coin_bot"
|
self.username = "ncue_stock_bot"
|
||||||
self.token = "6435061393:AAHOh9wB5yGNGUdb3SfCYJrrWTBe7wgConM"
|
self.token = "6874078562:AAEHxGDavfc0ssAXPQIaW8JGYmTR7LNUJOw"
|
||||||
self.chat_id = '574661323'
|
self.chat_id = '574661323'
|
||||||
self.client = telegram.Bot(token=self.token)
|
self.client = telegram.Bot(token=self.token)
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user