dsyoon/DeepStock
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

timeout 5초 추가

Browse Source
This commit is contained in:
dosang.yoon
2022-06-03 12:28:58 +09:00
parent c5848ebadc
commit 5774a6eaf0
8 changed files with 87 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
stockpredictor/crawler/sQLite/Crawler.py
View File

@@ -44,9 +44,9 @@ metaCrawler.crawl_meterials(stockFileName)
print("\n[종목 다운로드]") print("\n[종목 다운로드]")
stockCrawler = StockCrawler(START_DATE) stockCrawler = StockCrawler(START_DATE)
#stockCrawler.crawl_etf_stocks(stockFileName) stockCrawler.crawl_etf_stocks(stockFileName)
#stockCrawler.crawl_stocks(stockFileName) stockCrawler.crawl_stocks(stockFileName)
stockCrawler.crawl_us_stocks(stockFileName) stockCrawler.crawl_special_stocks(stockFileName)
print("\n[종목 분석]") print("\n[종목 분석]")

45
stockpredictor/crawler/sQLite/MetaCrawler.py
View File

@@ -48,7 +48,14 @@ class MetaCrawler:
finish = False finish = False
for i in range(1, self.limit_page_count): for i in range(1, self.limit_page_count):
#html = pd.read_html(input['URL'] + '&page=%s' % i, header=0) #html = pd.read_html(input['URL'] + '&page=%s' % i, header=0)
html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header).text) html = None
while True:
try:
html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header, timeout=5).text)
break
except:
print(input['URL'] + '&page=%s' % i)
continue
# 마지막 페이지 까지 받기 # 마지막 페이지 까지 받기
if len(html[0].날짜.values) <= 1: if len(html[0].날짜.values) <= 1:
@@ -124,7 +131,14 @@ class MetaCrawler:
finish = False finish = False
for i in range(1, self.limit_page_count): for i in range(1, self.limit_page_count):
#html = pd.read_html(url + str(i), header=0) #html = pd.read_html(url + str(i), header=0)
html = pd.read_html(requests.get(url + str(i), headers=self.header).text) html = None
while True:
try:
html = pd.read_html(requests.get(url + str(i), headers=self.header, timeout=5).text)
break
except:
print(url + str(i))
continue
# 마지막 페이지 까지 받기 # 마지막 페이지 까지 받기
if len(html[0].날짜.values) <= 2: if len(html[0].날짜.values) <= 2:
@@ -194,7 +208,14 @@ class MetaCrawler:
finish = False finish = False
for i in range(1, self.limit_page_count): for i in range(1, self.limit_page_count):
#html = pd.read_html(url + str(i), header=0, encoding='euc-kr') #html = pd.read_html(url + str(i), header=0, encoding='euc-kr')
html = pd.read_html(requests.get(url + str(i), headers=self.header).text, encoding='euc-kr') html = None
while True:
try:
html = pd.read_html(requests.get(url + str(i), headers=self.header, timeout=5).text, encoding='euc-kr')
break
except:
print(url + str(i))
continue
# 마지막 페이지 까지 받기 # 마지막 페이지 까지 받기
if len(html[0].날짜.values) <= 10: if len(html[0].날짜.values) <= 10:
@@ -273,7 +294,14 @@ class MetaCrawler:
finish = False finish = False
for i in range(1, self.limit_page_count): for i in range(1, self.limit_page_count):
#html = pd.read_html(input['URL'] + '&page=%s' % i, header=0) #html = pd.read_html(input['URL'] + '&page=%s' % i, header=0)
html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header).text) html = None
while True:
try:
html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header, timeout=5).text)
break
except:
print (input['URL'] + '&page=%s' % i)
continue
ymd, close, diff, rate = "", 0.0, 0.0, 0.0 ymd, close, diff, rate = "", 0.0, 0.0, 0.0
# 마지막 페이지 까지 받기 # 마지막 페이지 까지 받기
@@ -347,7 +375,14 @@ class MetaCrawler:
finish = False finish = False
for i in range(1, self.limit_page_count): for i in range(1, self.limit_page_count):
#html = pd.read_html(input['URL'] + '&page=%s' % i, header=0) #html = pd.read_html(input['URL'] + '&page=%s' % i, header=0)
html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header).text) html = None
while True:
try:
html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header, timeout=5).text)
break
except:
print(input['URL'] + '&page=%s' % i)
continue
# 마지막 페이지 까지 받기 # 마지막 페이지 까지 받기
if len(html[0].날짜.values) <= 1: if len(html[0].날짜.values) <= 1:

40
stockpredictor/crawler/sQLite/StockCrawler.py
View File

@@ -45,7 +45,7 @@ class StockCrawler:
def getStockInfo(self): def getStockInfo(self):
#code_df = pd.read_html('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', header=0)[0] #code_df = pd.read_html('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', header=0)[0]
code_df = pd.read_html(requests.get('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', headers=self.header).text)[0] code_df = pd.read_html(requests.get('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', headers=self.header, timeout=5).text)[0]
# code_df = pd.read_excel('../resources/stock/상장법인목록.xls') # code_df = pd.read_excel('../resources/stock/상장법인목록.xls')
# 종목코드가 6자리이기 때문에 6자리를 맞춰주기 위해 설정해줌 # 종목코드가 6자리이기 때문에 6자리를 맞춰주기 위해 설정해줌
@@ -207,7 +207,7 @@ class StockCrawler:
return return
def crawl_us_stocks(self, inFileName): def crawl_special_stocks(self, inFileName):
tableName = 'stock' tableName = 'stock'
conn = sqlite3.connect(inFileName) conn = sqlite3.connect(inFileName)
cursor = conn.cursor() cursor = conn.cursor()
@@ -228,7 +228,10 @@ class StockCrawler:
pd.options.display.float_format = '{:.4f}'.format pd.options.display.float_format = '{:.4f}'.format
pd.set_option('display.max_columns', None) pd.set_option('display.max_columns', None)
us_stocks = { special_stocks = {
'^KS11': 'Kospi',
'^KQ11': 'Kosdak',
'SQQQ': 'ProShares UltraPro Short QQQ', 'SQQQ': 'ProShares UltraPro Short QQQ',
'TQQQ': 'ProShares UltraPro QQQ', 'TQQQ': 'ProShares UltraPro QQQ',
'SCO': 'ProShares UltraShort Bloomberg Crude Oil', 'SCO': 'ProShares UltraShort Bloomberg Crude Oil',
@@ -286,21 +289,28 @@ class StockCrawler:
cursor = conn.cursor() cursor = conn.cursor()
us_sotck_data = {} us_sotck_data = {}
for ticker in us_stocks: for ticker in special_stocks:
cursor.execute('SELECT * FROM ' + tableName + ' WHERE CODE=?', (ticker,)) cursor.execute('SELECT * FROM ' + tableName + ' WHERE CODE=?', (ticker,))
result = cursor.fetchone() result = cursor.fetchone()
if result == None: if result == None:
#start = pd.to_datetime('2017-01-01') #start_day = pd.to_datetime('2017-01-01')
start = pd.to_datetime(self.START_DATE.replace(".", "-")) start_day = pd.to_datetime(self.START_DATE.replace(".", "-"))
end = pd.to_datetime(datetime.today().strftime('%Y-%m-%d')) end_day = pd.to_datetime(datetime.today().strftime('%Y-%m-%d'))
else: else:
#start = (datetime.today() - timedelta(days=300)).strftime('%Y-%m-%d') #start = (datetime.today() - timedelta(days=300)).strftime('%Y-%m-%d')
start = pd.to_datetime(self.START_DATE.replace(".", "-")) start_day = pd.to_datetime(self.START_DATE.replace(".", "-"))
end = pd.to_datetime(datetime.today().strftime('%Y-%m-%d')) end_day = pd.to_datetime(datetime.today().strftime('%Y-%m-%d'))
data = pdr.get_data_yahoo(ticker, start, end) data = None
while True:
try:
data = pdr.get_data_yahoo(ticker, start_day, end_day)
break
except:
print(ticker)
continue
us_sotck_data[ticker] = { us_sotck_data[ticker] = {
'close': data['Close'].to_dict(), 'close': data['Close'].to_dict(),
'open': data['Open'].to_dict(), 'open': data['Open'].to_dict(),
@@ -369,7 +379,15 @@ class StockCrawler:
# 최근 상장 기업의 마지막 반복되는 페이지를 제외시킨다. # 최근 상장 기업의 마지막 반복되는 페이지를 제외시킨다.
pg_url = '{url}&page={page}'.format(url=url, page=page) pg_url = '{url}&page={page}'.format(url=url, page=page)
#html = pd.read_html(pg_url, header=0) #html = pd.read_html(pg_url, header=0)
html = pd.read_html(requests.get(pg_url, headers=self.header).text) html = None
while True:
try:
html = pd.read_html(requests.get(pg_url, headers=self.header, timeout=5).text)
break
except:
print(pg_url)
continue
for date in html[0].날짜.values: for date in html[0].날짜.values:
if type(date) is str: if type(date) is str:
if date in date_set: if date in date_set:

2
stockpredictor/crawler/toJsonFile/FnGuideCrawler.py
View File

@@ -10,7 +10,7 @@ class FnGuideCrawler:
def getStockInfo(self): def getStockInfo(self):
#code_df = pd.read_html('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', header=0)[0] #code_df = pd.read_html('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', header=0)[0]
code_df = pd.read_html(requests.get('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', headers=self.header).text)[0] code_df = pd.read_html(requests.get('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', headers=self.header, timeout=5).text)[0]
# code_df = pd.read_excel('../resources/stock/상장법인목록.xls') # code_df = pd.read_excel('../resources/stock/상장법인목록.xls')
# 종목코드가 6자리이기 때문에 6자리를 맞춰주기 위해 설정해줌 # 종목코드가 6자리이기 때문에 6자리를 맞춰주기 위해 설정해줌

8
stockpredictor/crawler/toJsonFile/MetaCrawler.py
View File

@@ -46,7 +46,7 @@ class MetaCrawler:
finish = False finish = False
for i in range(1, self.limit_page_count): for i in range(1, self.limit_page_count):
#html = pd.read_html(input['URL'] + '&page=%s' % i, header=0) #html = pd.read_html(input['URL'] + '&page=%s' % i, header=0)
html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header).text) html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header, timeout=5).text)
# 마지막 페이지 까지 받기 # 마지막 페이지 까지 받기
if len(html[0].날짜.values) <= 1: if len(html[0].날짜.values) <= 1:
@@ -119,7 +119,7 @@ class MetaCrawler:
finish = False finish = False
for i in range(1, self.limit_page_count): for i in range(1, self.limit_page_count):
#html = pd.read_html(url + str(i), header=0) #html = pd.read_html(url + str(i), header=0)
html = pd.read_html(requests.get(url + str(i), headers=self.header).text) html = pd.read_html(requests.get(url + str(i), headers=self.header, timeout=5).text)
# 마지막 페이지 까지 받기 # 마지막 페이지 까지 받기
if len(html[0].날짜.values) <= 2: if len(html[0].날짜.values) <= 2:
@@ -179,7 +179,7 @@ class MetaCrawler:
finish = False finish = False
for i in range(1, self.limit_page_count): for i in range(1, self.limit_page_count):
#html = pd.read_html(url + str(i), header=0, encoding='euc-kr') #html = pd.read_html(url + str(i), header=0, encoding='euc-kr')
html = pd.read_html(requests.get(url + str(i), headers=self.header).text, encoding='euc-kr') html = pd.read_html(requests.get(url + str(i), headers=self.header, timeout=5).text, encoding='euc-kr')
# 마지막 페이지 까지 받기 # 마지막 페이지 까지 받기
if len(html[0].날짜.values) <= 10: if len(html[0].날짜.values) <= 10:
@@ -250,7 +250,7 @@ class MetaCrawler:
finish = False finish = False
for i in range(1, self.limit_page_count): for i in range(1, self.limit_page_count):
#html = pd.read_html(input['URL'] + '&page=%s' % i, header=0) #html = pd.read_html(input['URL'] + '&page=%s' % i, header=0)
html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header).text) html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header, timeout=5).text)
# 마지막 페이지 까지 받기 # 마지막 페이지 까지 받기
if len(html[0].날짜.values) <= 1: if len(html[0].날짜.values) <= 1:

8
stockpredictor/crawler/toJsonFile/StockCrawler.py
View File

@@ -74,7 +74,7 @@ class StockCrawler:
def getStockInfo(self): def getStockInfo(self):
#code_df = pd.read_html('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', header=0)[0] #code_df = pd.read_html('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', header=0)[0]
code_df = pd.read_html(requests.get('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', headers=self.header).text)[0] code_df = pd.read_html(requests.get('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', headers=self.header, timeout=5).text)[0]
# code_df = pd.read_excel('../resources/stock/상장법인목록.xls') # code_df = pd.read_excel('../resources/stock/상장법인목록.xls')
# 종목코드가 6자리이기 때문에 6자리를 맞춰주기 위해 설정해줌 # 종목코드가 6자리이기 때문에 6자리를 맞춰주기 위해 설정해줌
@@ -255,7 +255,7 @@ class StockCrawler:
# 최근 상장 기업의 마지막 반복되는 페이지를 제외시킨다. # 최근 상장 기업의 마지막 반복되는 페이지를 제외시킨다.
pg_url = '{url}&page={page}'.format(url=url, page=page) pg_url = '{url}&page={page}'.format(url=url, page=page)
#html = pd.read_html(pg_url, header=0) #html = pd.read_html(pg_url, header=0)
html = pd.read_html(requests.get(pg_url, headers=self.header).text) html = pd.read_html(requests.get(pg_url, headers=self.header, timeout=5).text)
count = 0 count = 0
for date in html[0].날짜.values: for date in html[0].날짜.values:
@@ -378,7 +378,7 @@ class StockCrawler:
# 최근 상장 기업의 마지막 반복되는 페이지를 제외시킨다. # 최근 상장 기업의 마지막 반복되는 페이지를 제외시킨다.
pg_url = '{url}&page={page}'.format(url=url, page=page) pg_url = '{url}&page={page}'.format(url=url, page=page)
#html = pd.read_html(pg_url, header=0) #html = pd.read_html(pg_url, header=0)
html = pd.read_html(requests.get(pg_url, headers=self.header).text) html = pd.read_html(requests.get(pg_url, headers=self.header, timeout=5).text)
count = 0 count = 0
for date in html[0].날짜.values: for date in html[0].날짜.values:
if type(date) is str: if type(date) is str:
@@ -600,7 +600,7 @@ class StockCrawler:
# 최근 상장 기업의 마지막 반복되는 페이지를 제외시킨다. # 최근 상장 기업의 마지막 반복되는 페이지를 제외시킨다.
pg_url = '{url}&page={page}'.format(url=url, page=page) pg_url = '{url}&page={page}'.format(url=url, page=page)
#html = pd.read_html(pg_url, header=0) #html = pd.read_html(pg_url, header=0)
html = pd.read_html(requests.get(pg_url, headers=self.header).text) html = pd.read_html(requests.get(pg_url, headers=self.header, timeout=5).text)
count = 0 count = 0
for date in html[0].날짜.values: for date in html[0].날짜.values:
if type(date) is str: if type(date) is str:

8
stockpredictor/crawler/toSQLite/MetaCrawler.py
View File

@@ -46,7 +46,7 @@ class MetaCrawler:
finish = False finish = False
for i in range(1, self.limit_page_count): for i in range(1, self.limit_page_count):
#html = pd.read_html(input['URL'] + '&page=%s' % i, header=0) #html = pd.read_html(input['URL'] + '&page=%s' % i, header=0)
html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header).text) html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header, timeout=5).text)
# 마지막 페이지 까지 받기 # 마지막 페이지 까지 받기
if len(html[0].날짜.values) <= 1: if len(html[0].날짜.values) <= 1:
@@ -125,7 +125,7 @@ class MetaCrawler:
finish = False finish = False
for i in range(1, self.limit_page_count): for i in range(1, self.limit_page_count):
#html = pd.read_html(url + str(i), header=0) #html = pd.read_html(url + str(i), header=0)
html = pd.read_html(requests.get(url + str(i), headers=self.header).text) html = pd.read_html(requests.get(url + str(i), headers=self.header, timeout=5).text)
# 마지막 페이지 까지 받기 # 마지막 페이지 까지 받기
if len(html[0].날짜.values) <= 2: if len(html[0].날짜.values) <= 2:
@@ -190,7 +190,7 @@ class MetaCrawler:
finish = False finish = False
for i in range(1, self.limit_page_count): for i in range(1, self.limit_page_count):
#html = pd.read_html(url + str(i), header=0, encoding='euc-kr') #html = pd.read_html(url + str(i), header=0, encoding='euc-kr')
html = pd.read_html(requests.get(url + str(i), headers=self.header).text, encoding='euc-kr') html = pd.read_html(requests.get(url + str(i), headers=self.header, timeout=5).text, encoding='euc-kr')
# 마지막 페이지 까지 받기 # 마지막 페이지 까지 받기
if len(html[0].날짜.values) <= 10: if len(html[0].날짜.values) <= 10:
@@ -264,7 +264,7 @@ class MetaCrawler:
finish = False finish = False
for i in range(1, self.limit_page_count): for i in range(1, self.limit_page_count):
#html = pd.read_html(input['URL'] + '&page=%s' % i, header=0) #html = pd.read_html(input['URL'] + '&page=%s' % i, header=0)
html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header).text) html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header, timeout=5).text)
# 마지막 페이지 까지 받기 # 마지막 페이지 까지 받기
if len(html[0].날짜.values) <= 1: if len(html[0].날짜.values) <= 1:

4
stockpredictor/crawler/toSQLite/StockCrawler.py
View File

@@ -75,7 +75,7 @@ class StockCrawler:
def getStockInfo(self): def getStockInfo(self):
#code_df = pd.read_html('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', header=0)[0] #code_df = pd.read_html('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', header=0)[0]
code_df = pd.read_html(requests.get('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', headers=self.header).text)[0] code_df = pd.read_html(requests.get('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', headers=self.header, timeout=5).text)[0]
# code_df = pd.read_excel('../resources/stock/상장법인목록.xls') # code_df = pd.read_excel('../resources/stock/상장법인목록.xls')
# 종목코드가 6자리이기 때문에 6자리를 맞춰주기 위해 설정해줌 # 종목코드가 6자리이기 때문에 6자리를 맞춰주기 위해 설정해줌
@@ -228,7 +228,7 @@ class StockCrawler:
# 최근 상장 기업의 마지막 반복되는 페이지를 제외시킨다. # 최근 상장 기업의 마지막 반복되는 페이지를 제외시킨다.
pg_url = '{url}&page={page}'.format(url=url, page=page) pg_url = '{url}&page={page}'.format(url=url, page=page)
#html = pd.read_html(pg_url, header=0) #html = pd.read_html(pg_url, header=0)
html = pd.read_html(requests.get(pg_url, headers=self.header).text) html = pd.read_html(requests.get(pg_url, headers=self.header, timeout=5).text)
for date in html[0].날짜.values: for date in html[0].날짜.values:
if type(date) is str: if type(date) is str:
if date in date_set: if date in date_set: