dsyoon/DeepStock
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

init

Browse Source
This commit is contained in:
dosangyoon
2022-04-17 23:06:39 +09:00
parent 505e32eba5
commit c52da139e6
2 changed files with 68 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
stockpredictor/crawler/sQLite/Crawler.py
View File

@@ -19,10 +19,13 @@ stockFileName = PROJECT_HOME + '/resources/stock.db'
# 재무제표는 3개월마다 다운로드를 한다.
"""
fnGuideCrawler = FnGuideCrawler(START_DATE)
print("[KOSPI 상장기업 재무제표 다운로드]")
fnGuideCrawler.crawl_fnguide(stockFileName)
"""
"""
metaCrawler = MetaCrawler(START_DATE)
print("\n[증시자금동향 (신용잔고, 펀드자금 잔고)]")
metaCrawler.crawl_money_trend(stockFileName)
@@ -38,11 +41,11 @@ metaCrawler.crawl_exchange(stockFileName)
print("\n[원유 (WTI), 국제금, COPPER, NATURALGAS, CORN, SOYBEAN]")
metaCrawler.crawl_meterials(stockFileName)
"""
print("\n[종목 다운로드]")
stockCrawler = StockCrawler(START_DATE)
stockCrawler.crawl_usa_stocks(stockFileName)
stockCrawler.crawl_us_stocks(stockFileName)
stockCrawler.crawl_etf_stocks(stockFileName)
stockCrawler.crawl_stocks(stockFileName)

84
stockpredictor/crawler/sQLite/StockCrawler.py
View File

@@ -207,7 +207,7 @@ class StockCrawler:
return
def crawl_usa_stocks(self, inFileName):
def crawl_us_stocks(self, inFileName):
tableName = 'stock'
conn = sqlite3.connect(inFileName)
cursor = conn.cursor()
@@ -228,23 +228,65 @@ class StockCrawler:
pd.options.display.float_format = '{:.4f}'.format
pd.set_option('display.max_columns', None)
usa_stock_code = ['SQQQ','TQQQ','SCO','UCO','GLL','UGL','DQ','EBAY','NTAP','VZ','AAPL','MSFT','GOOG','NVDA',
'FB','ASML','AVGO','COST','INTC','AMD','QCOM','HON','VZ','EBAY','TSM','V','WMT','BABA','KO',
'DIS','T']
item_name = {'SQQQ': 'ProShares UltraPro Short QQQ','TQQQ': 'ProShares UltraPro QQQ','SCO': 'ProShares UltraShort Bloomberg Crude Oil',
'UCO': 'ProShares Ultra Bloomberg Crude Oil','GLL': 'ProShares UltraShort Gold','UGL': 'ProShares Ultra Gold',
'DQ': 'DAQO New Energy Corp.','EBAY': 'eBay Inc.','NTAP': 'NetApp, Inc.','VZ': 'Verizon Communications Inc.',
'AAPL': 'Apple Inc.','MSFT': 'Microsoft Corporation','GOOG': 'Alphabet Inc.','NVDA': 'NVIDIA Corporation',
'FB': 'Meta Platforms, Inc.', 'ASML': 'ASML Holding','AVGO': 'Broadcom Inc.','COST': 'Costco Wholesale Corporation',
'INTC': 'Intel Corporation','AMD': 'Advanced Micro Devices, Inc.','QCOM': 'QUALCOMM Incorporated',
'HON': 'Honeywell International Inc.','TSM': 'Taiwan Semiconductor Manufacturing','V': 'Visa Inc.',
'WMT': 'Walmart Inc.','BABA': 'Alibaba Group Holding','KO': 'Coca-Cola','DIS': 'Walt Disney','T': 'AT&T Inc.'}
us_stocks = {
'SQQQ': 'ProShares UltraPro Short QQQ',
'TQQQ': 'ProShares UltraPro QQQ',
'SCO': 'ProShares UltraShort Bloomberg Crude Oil',
'UCO': 'ProShares Ultra Bloomberg Crude Oil',
'GLL': 'ProShares UltraShort Gold',
'UGL': 'ProShares Ultra Gold',
'SOXS': 'Direxion Daily Semiconductor Bear -3X Shares',
'SOXL': 'Direxion Daily Semiconductor Bull 3X Shares',
'FNGD': 'MicroSectors™ FANG+™ Index -3X Inverse Leveraged ETN',
'FNGU': 'MicroSectors™ FANG+™ Index 3X Leveraged ETN',
'AAPL': 'Apple',
'MSFT': 'Microsoft',
'GOOG': 'Alphabet C',
'AMZN': 'Amazon.com',
'AVGO': 'Broadcom',
'FB': 'Meta Platforms',
'NVDA': 'NVIDIA',
'UNH': 'UnitedHealth',
'TSM': 'Taiwan Semiconductor',
'JNJ': 'Johnson & Johnson (JNJ)',
'TCTZF': 'Tencent Holdings',
'V': 'Visa A',
'WMT': 'Walmart',
'XOM': 'Exxon Mobil',
'JPM': 'JPMorgan',
'MA': 'Mastercard',
'CVX': 'Chevron Corp',
'HD': 'Home Depot',
'BAC': 'Bank of America',
'KO': 'Coca-Cola',
'COST': 'Costco',
'DIS': 'Walt Disney',
'VZ': 'Verizon',
'CSCO': 'Cisco',
'ORCL': 'Oracle',
'NKE': 'Nike',
'ACN': 'Accenture',
'ADBE': 'Adobe',
'CRM': 'Salesforce.com',
'INTC': 'Intel',
'QCOM': 'Qualcomm',
'AMD': 'AMD',
'MS': 'Morgan Stanley',
'T': 'AT&T',
'HON': 'Honeywell',
'IBM': 'IBM',
'DQ': 'Daqo New Energy Corp ADR',
'EBAY': 'eBay Inc',
'NTAP': 'NetApp Inc',
'ASML': 'ASML Holding NV ADR',
'BABA': 'Alibaba Group Holdings Ltd ADR'}
conn = sqlite3.connect(inFileName)
cursor = conn.cursor()
usa_stock = {}
for ticker in usa_stock_code:
us_sotck_data = {}
for ticker in us_stocks:
cursor.execute('SELECT * FROM ' + tableName + ' WHERE CODE=?', (ticker,))
@@ -259,7 +301,7 @@ class StockCrawler:
end = pd.to_datetime(datetime.today().strftime('%Y-%m-%d'))
data = pdr.get_data_yahoo(ticker, start, end)
usa_stock[ticker] = {
us_sotck_data[ticker] = {
'close': data['Close'].to_dict(),
'open': data['Open'].to_dict(),
'high': data['High'].to_dict(),
@@ -269,10 +311,10 @@ class StockCrawler:
cursor.close()
conn.close()
dateList = list(usa_stock['SQQQ']['close'])
for idx, item_code in enumerate(usa_stock):
dateList = list(us_sotck_data['SQQQ']['close'])
for idx, item_code in enumerate(us_sotck_data):
stock_data = []
stock = usa_stock[item_code]
stock = us_sotck_data[item_code]
for i, ymd in enumerate(dateList):
if i > 0:
if ymd in stock['close'] and dateList[i-1] in stock['close']:
@@ -286,7 +328,7 @@ class StockCrawler:
continue
stock_data.append({
'CODE':item_code, 'NAME':item_name[item_code], 'ymd': ymd.strftime('%Y.%m.%d'),
'CODE':item_code, 'NAME':us_stocks[item_code], 'ymd': ymd.strftime('%Y.%m.%d'),
'close': round(stock['close'][ymd], 2), 'diff': round(diff, 2), 'open': round(stock['open'][ymd], 2),
'high': round(stock['high'][ymd], 2), 'low': round(stock['low'][ymd], 2), 'volume': stock['volume'][ymd]
})
@@ -306,7 +348,7 @@ class StockCrawler:
conn.commit()
cursor.close()
conn.close()
print(idx, item_code, item_name[item_code], (time.time() - start_time), "s")
print(idx, item_code, us_stocks[item_code], (time.time() - start_time), "s")
start_time = time.time()
sleep(0.05)
@@ -485,4 +527,4 @@ if __name__ == "__main__":
PROJECT_HOME = os.path.join(os.path.dirname(os.path.join(os.path.dirname(os.path.join(os.path.dirname(os.path.join(os.path.dirname(__file__))))))))
stockFileName = PROJECT_HOME + '/resources/stock.db'
stockCrawler.crawl_usa_stocks(stockFileName)
stockCrawler.crawl_us_sotck_datas(stockFileName)