From d2edee26b423da5ae59bda080a76f2683c5b7256 Mon Sep 17 00:00:00 2001 From: dsyoon Date: Tue, 11 Jul 2023 12:34:18 +0900 Subject: [PATCH] init --- StockCrawler.py | 10 ++++++--- stock/crawler/MetaCrawler.py | 42 ++++++++++++++++++++++++++---------- 2 files changed, 38 insertions(+), 14 deletions(-) diff --git a/StockCrawler.py b/StockCrawler.py index 45a1e7e..a95e533 100644 --- a/StockCrawler.py +++ b/StockCrawler.py @@ -36,6 +36,7 @@ if week in (0, 1, 2, 3, 4): # 0:월, 1:화, 2:수, 3:목, 4:금, 5:토, 6:일 slackBot.sendMsg("1. start to crawl...") + ERROR_COUNT = 0 while ERROR_COUNT < 3: try: @@ -51,6 +52,7 @@ if week in (0, 1, 2, 3, 4): # 0:월, 1:화, 2:수, 3:목, 4:금, 5:토, 6:일 exit() metaCrawler = MetaCrawler(START_DATE) + ERROR_COUNT = 0 while ERROR_COUNT < 3: try: @@ -76,7 +78,7 @@ if week in (0, 1, 2, 3, 4): # 0:월, 1:화, 2:수, 3:목, 4:금, 5:토, 6:일 continue if ERROR_COUNT >= 3: exit() - + ERROR_COUNT = 0 while ERROR_COUNT < 3: try: @@ -90,10 +92,11 @@ if week in (0, 1, 2, 3, 4): # 0:월, 1:화, 2:수, 3:목, 4:금, 5:토, 6:일 if ERROR_COUNT >= 3: exit() + ERROR_COUNT = 0 while ERROR_COUNT < 3: try: - print("\n[환율 (USD, JPY, EUR, CNY)]") + print("\n[환율 (USD, JPY, EUR, CNY, HKD, GBP, CAD, CHF, AUD, THB, INR, PHP, BRL, VND, RUB, TWD')]") metaCrawler.crawl_exchange(stockFileName) slackBot.sendMsg("5. done exchange data...") break @@ -106,7 +109,7 @@ if week in (0, 1, 2, 3, 4): # 0:월, 1:화, 2:수, 3:목, 4:금, 5:토, 6:일 ERROR_COUNT = 0 while ERROR_COUNT < 3: try: - print("\n[원유 (WTI), 국제금, COPPER, NATURALGAS, CORN, SOYBEAN]") + print("\n[원유 (WTI), 국제금, COPPER, NATURALGAS, CORN, SOYBEAN 등]") metaCrawler.crawl_meterials(stockFileName) slackBot.sendMsg("6. done additional data...") break @@ -204,5 +207,6 @@ if week in (0, 1, 2, 3, 4): # 0:월, 1:화, 2:수, 3:목, 4:금, 5:토, 6:일 if ERROR_COUNT >= 3: exit() + print("time : %6.2f 초", (time.time() - start)) print ("done...") diff --git a/stock/crawler/MetaCrawler.py b/stock/crawler/MetaCrawler.py index cffd83b..98c304c 100644 --- a/stock/crawler/MetaCrawler.py +++ b/stock/crawler/MetaCrawler.py @@ -29,10 +29,22 @@ class MetaCrawler: cursor.execute(create_key) inputs = [] - inputs.append( {'NAME':'USD', 'CODE':'FX_USDKRW', 'URL':'http://finance.naver.com/marketindex/exchangeDailyQuote.nhn?marketindexCd=FX_USDKRW'} ) # 미국 USD - inputs.append( {'NAME':'JPY', 'CODE':'FX_JPYKRW', 'URL':'http://finance.naver.com/marketindex/exchangeDailyQuote.nhn?marketindexCd=FX_JPYKRW'} ) # 일본 JPY - inputs.append( {'NAME':'EUR', 'CODE':'FX_EURKRW', 'URL':'http://finance.naver.com/marketindex/exchangeDailyQuote.nhn?marketindexCd=FX_EURKRW'} ) # 유럽연합 EUR' - inputs.append( {'NAME':'CNY', 'CODE':'FX_CNYKRW', 'URL':'http://finance.naver.com/marketindex/exchangeDailyQuote.nhn?marketindexCd=FX_CNYKRW'} ) # 중국 CNY + inputs.append( {'NAME':'USD', 'CODE': 'FX_USDKRW', 'URL':'http://finance.naver.com/marketindex/exchangeDailyQuote.nhn?marketindexCd=FX_USDKRW'} ) # 미국 USD + inputs.append( {'NAME':'JPY', 'CODE': 'FX_JPYKRW', 'URL':'http://finance.naver.com/marketindex/exchangeDailyQuote.nhn?marketindexCd=FX_JPYKRW'} ) # 일본 JPY + inputs.append( {'NAME':'EUR', 'CODE': 'FX_EURKRW', 'URL':'http://finance.naver.com/marketindex/exchangeDailyQuote.nhn?marketindexCd=FX_EURKRW'} ) # 유럽연합 EUR' + inputs.append( {'NAME':'CNY', 'CODE': 'FX_CNYKRW', 'URL':'http://finance.naver.com/marketindex/exchangeDailyQuote.nhn?marketindexCd=FX_CNYKRW'} ) # 중국 CNY + inputs.append({'NAME': 'HKD', 'CODE': 'FX_HKDKRW', 'URL': 'http://finance.naver.com/marketindex/exchangeDailyQuote.nhn?marketindexCd=FX_HKDKRW'}) # 홍콩 HKD + inputs.append({'NAME': 'GBP', 'CODE': 'FX_GBPKRW', 'URL': 'http://finance.naver.com/marketindex/exchangeDailyQuote.nhn?marketindexCd=FX_GBPKRW'}) # 영국 GRP + inputs.append({'NAME': 'CAD', 'CODE': 'FX_CADKRW', 'URL': 'http://finance.naver.com/marketindex/exchangeDailyQuote.nhn?marketindexCd=FX_CADKRW'}) # 캐나다CAD + inputs.append({'NAME': 'CHF', 'CODE': 'FX_CHFKRW', 'URL': 'http://finance.naver.com/marketindex/exchangeDailyQuote.nhn?marketindexCd=FX_CHFKRW'}) # 스위스 CHF + inputs.append({'NAME': 'AUD', 'CODE': 'FX_AUDKRW', 'URL': 'http://finance.naver.com/marketindex/exchangeDailyQuote.nhn?marketindexCd=FX_AUDKRW'}) # 호주 AUD + inputs.append({'NAME': 'THB', 'CODE': 'FX_THBKRW', 'URL': 'http://finance.naver.com/marketindex/exchangeDailyQuote.nhn?marketindexCd=FX_THBKRW'}) # 태국 THB + inputs.append({'NAME': 'INR', 'CODE': 'FX_INRKRW', 'URL': 'http://finance.naver.com/marketindex/exchangeDailyQuote.nhn?marketindexCd=FX_INRKRW'}) # 인도 INR + inputs.append({'NAME': 'PHP', 'CODE': 'FX_PHPKRW', 'URL': 'http://finance.naver.com/marketindex/exchangeDailyQuote.nhn?marketindexCd=FX_PHPKRW'}) # 필리핀 PHP + inputs.append({'NAME': 'BRL', 'CODE': 'FX_BRLKRW', 'URL': 'http://finance.naver.com/marketindex/exchangeDailyQuote.nhn?marketindexCd=FX_BRLKRW'}) # 브라질 BRL + inputs.append({'NAME': 'VND', 'CODE': 'FX_VNDKRW', 'URL': 'http://finance.naver.com/marketindex/exchangeDailyQuote.nhn?marketindexCd=FX_VNDKRW'}) # 베트남 VND + inputs.append({'NAME': 'RUB', 'CODE': 'FX_RUBKRW', 'URL': 'http://finance.naver.com/marketindex/exchangeDailyQuote.nhn?marketindexCd=FX_RUBKRW'}) # 러시아 RUB + inputs.append({'NAME': 'TWD', 'CODE': 'FX_TWDKRW', 'URL': 'http://finance.naver.com/marketindex/exchangeDailyQuote.nhn?marketindexCd=FX_TWDKRW'}) # 대만 TWD for i in range(len(inputs)): input = inputs[i] @@ -57,7 +69,7 @@ class MetaCrawler: break except: print(input['URL'] + '&page=%s' % i) - if i>200: + if i > 200: break continue @@ -366,12 +378,20 @@ class MetaCrawler: cursor.execute(create_key) inputs = [] - inputs.append( {'NAME':'WTI', 'CODE':'OIL_CL', 'URL':'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=OIL_CL&fdtc=2'} ) # WTI - inputs.append( {'NAME':'GOLD', 'CODE':'CMDT_GC', 'URL':'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=CMDT_GC&fdtc=2'} ) # 국제 금 - inputs.append({'NAME': 'COPPER', 'CODE': 'CMDT_CDY','URL': 'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=CMDT_CDY&fdtc=2'}) # 구리 - inputs.append({'NAME': 'NATURALGAS', 'CODE': 'CMDT_NG','URL': 'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=CMDT_NG&fdtc=2'}) # 천연가스 - inputs.append({'NAME': 'CORN', 'CODE': 'CMDT_C','URL': 'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=CMDT_C&fdtc=2'}) # 국제 옥수수 - inputs.append({'NAME': 'SOYBEAN', 'CODE': 'CMDT_S','URL': 'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=CMDT_S&fdtc=2'}) # 국제 대두 + inputs.append( {'NAME':'WTI', 'CODE':'OIL_CL', 'URL':'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=OIL_CL&fdtc=2'} ) # WTI + inputs.append({'NAME': 'COPPER', 'CODE': 'CMDT_CDY','URL': 'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=CMDT_CDY&fdtc=2'}) # 구리 + inputs.append({'NAME': 'NATURALGAS', 'CODE': 'CMDT_NG','URL': 'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=CMDT_NG&fdtc=2'}) # 천연가스 + inputs.append({'NAME': 'CORN', 'CODE': 'CMDT_C','URL': 'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=CMDT_C&fdtc=2'}) # 국제 옥수수 + inputs.append({'NAME': 'SOYBEAN', 'CODE': 'CMDT_S','URL': 'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=CMDT_S&fdtc=2'}) # 국제 대두 + inputs.append({'NAME': 'LEAD', 'CODE': 'CMDT_PDY','URL': 'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=CMDT_PDY&fdtc=2'}) # 납 + inputs.append({'NAME': 'NICKEL', 'CODE': 'CMDT_NDY','URL': 'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=CMDT_NDY&fdtc=2'}) # 니켈 + inputs.append({'NAME': 'SUGAR', 'CODE': 'CMDT_SB','URL': 'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=CMDT_SB&fdtc=2'}) # 설탕 + inputs.append({'NAME': 'COFFEE', 'CODE': 'CMDT_KC','URL': 'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=CMDT_KC&fdtc=2'}) # 커피 + inputs.append({'NAME': 'COCOA', 'CODE': 'CMDT_CC','URL': 'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=CMDT_CC&fdtc=2'}) # 코코아 + inputs.append( {'NAME':'GOLD', 'CODE':'CMDT_GC', 'URL':'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=CMDT_GC&fdtc=2'} ) # 국제 금 + inputs.append({'NAME': 'SILVER', 'CODE': 'CMDT_SI','URL': 'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=CMDT_SI&fdtc=2'}) # 국제은 + inputs.append({'NAME': 'PLATINUM', 'CODE': 'CMDT_PL','URL': 'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=CMDT_PL&fdtc=2'}) # 국제 백금 + inputs.append({'NAME': 'PALADIUM', 'CODE': 'CMDT_PA','URL': 'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=CMDT_PA&fdtc=2'}) # 국제 팔라듐 for i in range(len(inputs)): input = inputs[i]