dsyoon/DeepStock
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

timeout 5초 추가

Browse Source
This commit is contained in:
dosang.yoon
2022-06-03 12:28:58 +09:00
parent c5848ebadc
commit 5774a6eaf0
8 changed files with 87 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
stockpredictor/crawler/sQLite/StockCrawler.py
View File

@@ -45,7 +45,7 @@ class StockCrawler:
def getStockInfo(self):
#code_df = pd.read_html('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', header=0)[0]
code_df = pd.read_html(requests.get('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', headers=self.header).text)[0]
code_df = pd.read_html(requests.get('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', headers=self.header, timeout=5).text)[0]
# code_df = pd.read_excel('../resources/stock/상장법인목록.xls')
# 종목코드가 6자리이기 때문에 6자리를 맞춰주기 위해 설정해줌
@@ -207,7 +207,7 @@ class StockCrawler:
return
def crawl_us_stocks(self, inFileName):
def crawl_special_stocks(self, inFileName):
tableName = 'stock'
conn = sqlite3.connect(inFileName)
cursor = conn.cursor()
@@ -228,7 +228,10 @@ class StockCrawler:
pd.options.display.float_format = '{:.4f}'.format
pd.set_option('display.max_columns', None)
us_stocks = {
special_stocks = {
'^KS11': 'Kospi',
'^KQ11': 'Kosdak',
'SQQQ': 'ProShares UltraPro Short QQQ',
'TQQQ': 'ProShares UltraPro QQQ',
'SCO': 'ProShares UltraShort Bloomberg Crude Oil',
@@ -286,21 +289,28 @@ class StockCrawler:
cursor = conn.cursor()
us_sotck_data = {}
for ticker in us_stocks:
for ticker in special_stocks:
cursor.execute('SELECT * FROM ' + tableName + ' WHERE CODE=?', (ticker,))
result = cursor.fetchone()
if result == None:
#start = pd.to_datetime('2017-01-01')
start = pd.to_datetime(self.START_DATE.replace(".", "-"))
end = pd.to_datetime(datetime.today().strftime('%Y-%m-%d'))
#start_day = pd.to_datetime('2017-01-01')
start_day = pd.to_datetime(self.START_DATE.replace(".", "-"))
end_day = pd.to_datetime(datetime.today().strftime('%Y-%m-%d'))
else:
#start = (datetime.today() - timedelta(days=300)).strftime('%Y-%m-%d')
start = pd.to_datetime(self.START_DATE.replace(".", "-"))
end = pd.to_datetime(datetime.today().strftime('%Y-%m-%d'))
start_day = pd.to_datetime(self.START_DATE.replace(".", "-"))
end_day = pd.to_datetime(datetime.today().strftime('%Y-%m-%d'))
data = pdr.get_data_yahoo(ticker, start, end)
data = None
while True:
try:
data = pdr.get_data_yahoo(ticker, start_day, end_day)
break
except:
print(ticker)
continue
us_sotck_data[ticker] = {
'close': data['Close'].to_dict(),
'open': data['Open'].to_dict(),
@@ -369,7 +379,15 @@ class StockCrawler:
# 최근 상장 기업의 마지막 반복되는 페이지를 제외시킨다.
pg_url = '{url}&page={page}'.format(url=url, page=page)
#html = pd.read_html(pg_url, header=0)
html = pd.read_html(requests.get(pg_url, headers=self.header).text)
html = None
while True:
try:
html = pd.read_html(requests.get(pg_url, headers=self.header, timeout=5).text)
break
except:
print(pg_url)
continue
for date in html[0].날짜.values:
if type(date) is str:
if date in date_set: