dsyoon/DeepStock
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

init

Browse Source
This commit is contained in:
dsyoon
2021-02-16 04:29:48 +09:00
parent e78aad9259
commit 890418a3ae
28 changed files with 54202 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

534
stockpredictor/analysis/Analyzer.py Normal file
View File

@@ -0,0 +1,534 @@
import json
import os
import shutil
from stockpredictor.analysis.Common import Common
from stockpredictor.analysis.MACD import MACD
from stockpredictor.analysis.RSI import RSI
from stockpredictor.analysis.Stochastic import Stochastic
from stockpredictor.analysis.IchimokuCloud import IchimokuCloud
import matplotlib.pyplot as plt
import datetime
import sqlite3
from datetime import datetime
from matplotlib import rc
rc('font', family='AppleGothic')
plt.rcParams['axes.unicode_minus'] = False
import pandas as pd
import plotly.graph_objs as go
from plotly import tools, subplots
import plotly.io as po
class Analyzer:
tableName = 'stock'
PROJECT_HOME = None
stocks = None
candidate = None
macd = None
rsi = None
stochastic = None
ichimokuCloud = None
common = None
inFileName = None
fnguideFileName = None
fnguide = {}
def __init__(self, PROJECT_HOME, inFileName, fnguideFileName):
self.PROJECT_HOME = PROJECT_HOME
self.inFileName = inFileName
self.fnguideFileName = fnguideFileName
self.stocks = []
self.candidate = []
self.common = Common()
self.macd = MACD()
self.rsi = RSI()
self.stochastic = Stochastic()
self.ichimokuCloud = IchimokuCloud()
self.readFnguide()
return
def readFnguide(self):
conn = sqlite3.connect(self.fnguideFileName)
cursor = conn.cursor()
today = datetime.today()
year1 = str(today.year - 1) + ".12.01"
year2 = str(today.year - 2) + ".12.01"
year3 = str(today.year - 3) + ".12.01"
rowid = 1
cursor.execute('SELECT * FROM fnguide WHERE rowid=?', (rowid,))
result = cursor.fetchone()
while result != None:
data = json.loads(result[2])
self.fnguide[result[0]] = True
if (year1 in data):
if (data[year1]['영업이익'] > 0 and data[year1]['당기순이익'] > 0):
self.fnguide[result[0]] = True
if (year2 in data):
if (data[year2]['영업이익'] > 0 and data[year2]['당기순이익'] > 0):
self.fnguide[result[0]] = True
if (year3 in data):
if (data[year3]['영업이익'] > 0 and data[year3]['당기순이익'] > 0):
self.fnguide[result[0]] = True
else:
self.fnguide[result[0]] = False
else:
if (data[year1]['영업이익'] > data[year2]['영업이익']):
self.fnguide[result[0]] = True
else:
self.fnguide[result[0]] = False
else:
self.fnguide[result[0]] = False
else:
self.fnguide[result[0]] = False
rowid += 1
cursor.execute('SELECT * FROM fnguide WHERE rowid=?', (rowid,))
result = cursor.fetchone()
cursor.close()
conn.close()
return
def draw(self, stock):
last_index = self.get_last_index(stock)
if last_index > 300:
index = 300 # 최대 300일치 그래프 확인
df_stock = pd.DataFrame(stock["PRICE"][len(stock["PRICE"]) - index:])
df_macd = pd.DataFrame(stock["MACD"][len(stock["MACD"]) - index:last_index+1])
df_stochastic = pd.DataFrame(stock["STOCHASTIC"][len(stock["STOCHASTIC"]) - index:last_index+1])
df_rsi = pd.DataFrame(stock["RSI"][len(stock["RSI"]) - index:last_index+1])
df_ichimoku = pd.DataFrame(stock["ICHIMOKU"][len(stock["ICHIMOKU"]) - index:])
else:
index = last_index
df_stock = pd.DataFrame(stock["PRICE"][:index+1])
df_macd = pd.DataFrame(stock["MACD"][:index+1])
df_stochastic = pd.DataFrame(stock["STOCHASTIC"][:index+1])
df_ichimoku = pd.DataFrame(stock["ICHIMOKU"][:index+1])
df_rsi = pd.DataFrame(stock["RSI"][:index+1])
# general
volume = go.Bar(x=df_stock.DATE, y=df_stock['volume'], name="volume")
volume_data = [volume]
leadingSpan1 = go.Scatter(x=df_ichimoku.DATE, y=df_ichimoku['leadingSpan1'], name="선행스팬", line_color='#8B4513')
leadingSpan2 = go.Scatter(x=df_ichimoku.DATE, y=df_ichimoku['leadingSpan2'], name="후행스팬", line_color='#4169E1')
cnadle = go.Candlestick(x=df_stock.DATE, open=df_stock.open, high=df_stock.high, low=df_stock.low, close=df_stock.close, increasing_line_color= 'red', decreasing_line_color= 'blue')
ichimokuCloud_data = [leadingSpan1, leadingSpan2, cnadle]
# macd
macd = go.Scatter(x=df_macd.DATE, y=df_macd['macd'], name="MACD", line_color='#8B4513')
macd_signal = go.Scatter(x=df_macd.DATE, y=df_macd['macds'], name="MACD Signal", line_color='#4169E1')
oscillator = go.Bar(x=df_macd.DATE, y=df_macd['macdo'], name="oscillator")
macd_data = [macd, macd_signal, oscillator]
# stochastic
slow_k = go.Scatter(x=df_stochastic.DATE, y=df_stochastic['slow_k'], name="Slow%K", line_color='#8B4513')
slow_d = go.Scatter(x=df_stochastic.DATE, y=df_stochastic['slow_d'], name="Slow%D", line_color='#4169E1')
stochastic_data = [slow_k, slow_d]
# rsi
rsi = go.Scatter(x=df_macd.DATE, y=df_rsi['rsi'], name="RSI", line_color='#8B4513')
rsi_signal = go.Scatter(x=df_macd.DATE, y=df_rsi['rsis'], name="RSI Signal", line_color='#4169E1')
rsi_data = [rsi, rsi_signal]
fig = subplots.make_subplots(rows=5, cols=1, subplot_titles=('MACD', 'Stochastic', 'RSI', '거래량', '일목균형표'))
for trace in macd_data:
fig.append_trace(trace, 1, 1)
for trace in stochastic_data:
fig.append_trace(trace, 2, 1)
for trace in rsi_data:
fig.append_trace(trace, 3, 1)
for trace in volume_data:
fig.append_trace(trace, 4, 1)
for trace in ichimokuCloud_data:
fig.append_trace(trace, 5, 1)
fig.update_layout(height=2000)
return fig
def get_last_index(self, stock):
for i in range(0, len(stock['PRICE'])):
if (stock['PRICE'][i]['close'] == 0 and stock['PRICE'][i]['open'] == 0 and stock['PRICE'][i]['volume'] == 0):
return i-1
return len(stock['PRICE']) - 1
def analyzeMACD(self):
conn = sqlite3.connect(self.inFileName)
cursor = conn.cursor()
# 기존 분석 데이터를 모두 지움
cursor.execute('update ' + self.tableName + ' set MACD = ""')
rowid = 1
cursor.execute('SELECT * FROM ' + self.tableName + ' WHERE rowid=?', (rowid,))
result = cursor.fetchone()
while result != None:
stock = {"CODE": result[0], "NAME": result[1], "PRICE": json.loads(result[2])}
results = self.macd.analyze(stock)
text = json.dumps(results, ensure_ascii=False)
cursor.execute("UPDATE " + self.tableName + " SET MACD=? WHERE CODE=?", (text, stock["CODE"]))
print("#analyzeMACD", rowid, stock['NAME'])
rowid += 1
cursor.execute('SELECT * FROM ' + self.tableName + ' WHERE rowid=?', (rowid,))
result = cursor.fetchone()
conn.commit()
cursor.close()
conn.close()
return
def analyzeRSI(self):
conn = sqlite3.connect(self.inFileName)
cursor = conn.cursor()
# 기존 분석 데이터를 모두 지움
cursor.execute('update ' + self.tableName + ' set RSI = ""')
rowid = 1
cursor.execute('SELECT * FROM ' + self.tableName + ' WHERE rowid=?', (rowid,))
result = cursor.fetchone()
while result != None:
prices = json.loads(result[2])
for price in prices:
del price['diff']
stock = {"CODE": result[0], "NAME": result[1], "PRICE": prices}
results = self.rsi.analyze(stock)
text = json.dumps(results, ensure_ascii=False)
cursor.execute("UPDATE " + self.tableName + " SET RSI=? WHERE CODE=?", (text, stock["CODE"]))
print("#analyzeRSI", rowid, stock['NAME'])
rowid += 1
cursor.execute('SELECT * FROM ' + self.tableName + ' WHERE rowid=?', (rowid,))
result = cursor.fetchone()
conn.commit()
cursor.close()
conn.close()
return
def analyzeStochastic(self):
conn = sqlite3.connect(self.inFileName)
cursor = conn.cursor()
# 기존 분석 데이터를 모두 지움
cursor.execute('update ' + self.tableName + ' set STOCHASTIC = ""')
rowid = 1
cursor.execute('SELECT * FROM ' + self.tableName + ' WHERE rowid=?', (rowid,))
result = cursor.fetchone()
while result != None:
stock = {"CODE": result[0], "NAME": result[1], "PRICE": json.loads(result[2])}
results = self.stochastic.analyze(stock)
text = json.dumps(results, ensure_ascii=False)
cursor.execute("UPDATE " + self.tableName + " SET STOCHASTIC=? WHERE CODE=?", (text, stock["CODE"]))
print("#analyzeStochastic", rowid, stock['NAME'])
rowid += 1
cursor.execute('SELECT * FROM ' + self.tableName + ' WHERE rowid=?', (rowid,))
result = cursor.fetchone()
conn.commit()
cursor.close()
conn.close()
return
def analyzeIchimokuCloud(self):
conn = sqlite3.connect(self.inFileName)
cursor = conn.cursor()
# 기존 분석 데이터를 모두 지움
cursor.execute('update ' + self.tableName + ' set ICHIMOKU = ""')
rowid = 1
cursor.execute('SELECT * FROM ' + self.tableName + ' WHERE rowid=?', (rowid,))
result = cursor.fetchone()
while result != None:
stock = {"CODE": result[0], "NAME": result[1], "PRICE": json.loads(result[2])}
results = self.ichimokuCloud.analyze(stock)
text = json.dumps(results, ensure_ascii=False)
cursor.execute("UPDATE " + self.tableName + " SET ICHIMOKU=? WHERE CODE=?", (text, stock["CODE"]))
print("#analyzeIchimokuCloud", rowid, stock['NAME'])
rowid += 1
cursor.execute('SELECT * FROM ' + self.tableName + ' WHERE rowid=?', (rowid,))
result = cursor.fetchone()
conn.commit()
cursor.close()
conn.close()
return
def analyzeFinalScore(self, last_index, STOCK, MACD, STOCHASTIC, ICHIMOKU, RSI):
"""
[매도]
1. MACD
1) MACD가 시그널설을 하향 돌파하면 매도한다.
2) MACD가 기준선 (0) 아래에 있는 한 주가는 하향추세이거나 또는 상승하지 않는다.
3) 시그널 추세가 하향인 한 매수 신호가 나올 때까지 주식을 보유하지 않는다.
4) 하락형 다이버전스가 발생하면 적극 매도를 검토한다.
2. Stochasic: %K선이 %D선을 하향 돌파하면 매도 신호로 보되 다음 사항들이 일치하면 매도한다.
1) 스토캐스틱 지표가 하락추세를 보이고, 50 이하에 있어야 한다.
2) 스토캐스틱 지표가 고점이 낮아지는 하락파동에 있어야 한다.
3) 주가가 5일 또는 20일 이동 평균선 아래에 있어야 한다.
4) MACD 지표가 하락으로 전환하거나 또는 최소한 상승을 멈추어야 한다.
5) 하락형 다이버전스가 발생하면 매도를 검토한다.
3. rsi
1) rsi가 하향이고 70이하로 떨어지면 매도,
2) rsi가 하향이고 50이하로 떨어지면 매도,
3) rsi가 하향이고 30이하로 떨어지면 단기매도,
[매수]
1. MACD
1) MACD가 시그널설을 상향 돌파하면 매수한다.
2) MACD가 기준선 (0) 위에 있는 한 주가는 상승추세이거나 또는 하락하지 않는다.
3) 시그널 추세가 상승하는 한 매도 신호가 나올 때까지 주식을 보유한다.
4) 상승형 다이버전스가 발생하면 적극 매수를 검토한다.
2. Stochasic: %K선이 %D선을 상향 돌파하면 매수 신호로 보되 다음 사항들이 일치하면 매수한다.
1) 스토캐스틱 지표가 상승추세를 보이고, 50 이상에 있어야 한다.
2) 스토캐스틱 지표가 저점을 높이는 상승파동에 있어야 한다.
3) 주가가 5일 또는 20일 이동 평균선 위에 있어야 한다.
4) MACD 지표가 상승으로 전환하거나 또는 최소한 하락을 멈추어야 한다.
5) 상승형 다이버전스가 발생하면 매수를 검토한다.
3. rsi
1) 상향이고 30을 돌파하면 매수,
2) rsi가 상향이고 40을 돌파하면 매수,
3) rsi가 상향이고 70을 돌파하면 단기매수,
"""
i = last_index
# 매수금액을 구
# 이전 3일 동안의 어제종가-오늘저가의 평균을 구함 --> (종가-시가)/3
# 그래서 오늘 종가에 구한 평균값을 더해서 내일 종목을 매수함
buy_price = 0
count = 0
for idx in range(i, i-5, -1):
if idx-1 < 0:
break
buy_price += STOCK[idx-1]['close'] - STOCK[idx]['low']
count += 1
if count == 0:
buy_price = STOCK[i]['close']
else:
buy_price = round(STOCK[i]['close'] - (buy_price/count))
stochastic_score = self.common.getStochasticScore(STOCHASTIC, i)
"""
if STOCK[i]['volume'] > 10000:
if MACD[i - 1]['macd'] < MACD[i]['macd']:
if MACD[i - 1]['macd'] < MACD[i - 1]['macds'] and MACD[i]['macd'] > MACD[i]['macds']:
return True,buy_price, stochastic_score
if stochastic_score > 0:
return True, buy_price, stochastic_score
"""
if STOCHASTIC[i]['slow_k'] < 10 and self.common.checkLongYangBongAfterUmBong(STOCK, i):
return 'STOCHASTIC_YANGBONG', buy_price
if STOCHASTIC[i]['slow_k'] < 10:
return 'STOCHASTIC', buy_price
if self.common.checkLongYangBongAfterUmBong(STOCK, i):
return 'YANGBONG', buy_price
return "", buy_price
def analyzeToFile(self, outFileName):
conn = sqlite3.connect(self.inFileName)
cursor = conn.cursor()
outfp = open(outFileName, "w", encoding="utf-8")
rowid = 1
cursor.execute('SELECT * FROM ' + self.tableName + ' WHERE rowid=?', (rowid,))
result = cursor.fetchone()
while result != None:
stock = {"CODE": result[0], "NAME": result[1], "PRICE": json.loads(result[2])}
macd = json.loads(result[3])
stochastic = json.loads(result[4])
ichimokuCloud = json.loads(result[5])
rsi = json.loads(result[6])
last_index = self.get_last_index(stock)
lastStock = stock['PRICE']
state, buy_price = self.analyzeFinalScore(last_index, lastStock, macd, stochastic, ichimokuCloud, rsi)
if state != "":
# self.macd.draw(stock)
print(stock['CODE'], stock['NAME'], str(buy_price), stochastic[last_index]['slow_k'], macd[last_index]['macd'], rsi[last_index]['rsi_buy'], ichimokuCloud[last_index]['ichimoku_buy'])
outfp.write("%s\t%s\t%s\t%d\t%s\t%s\t%s\n"%(stock['CODE'], stock['NAME'], str(buy_price), stochastic[last_index]['slow_k'], rsi[last_index]['rsi_buy'], macd[last_index]['macd'], ichimokuCloud[last_index]['ichimoku_buy']))
print("#file", rowid, stock['NAME'])
rowid += 1
cursor.execute('SELECT * FROM ' + self.tableName + ' WHERE rowid=?', (rowid,))
result = cursor.fetchone()
cursor.close()
conn.close()
outfp.close()
return
# 그래프 출력
def analyzeToHtml(self, outPath):
tmp_path = outPath + "/tmp"
if os.path.isdir(tmp_path):
os.rmdir(tmp_path)
os.mkdir(tmp_path)
conn = sqlite3.connect(self.inFileName)
cursor = conn.cursor()
rowid = 1
cursor.execute('SELECT * FROM ' + self.tableName + ' WHERE rowid=?', (rowid,))
result = cursor.fetchone()
while result != None:
item_code = result[0]
item_name = result[1]
"""
if (item_code in self.fnguide and not self.fnguide[item_code]):
rowid += 1
cursor.execute('SELECT * FROM ' + self.tableName + ' WHERE rowid=?', (rowid,))
result = cursor.fetchone()
continue
"""
stock = {"CODE": result[0], "NAME": result[1], "PRICE": json.loads(result[2]), "MACD": json.loads(result[3]), "STOCHASTIC": json.loads(result[4]), "ICHIMOKU": json.loads(result[5]), "RSI": json.loads(result[6])}
last_index = self.get_last_index(stock)
STOCK = stock['PRICE']
MACD = stock['MACD']
STOCHASTIC = stock['STOCHASTIC']
ICHIMOKU = stock['ICHIMOKU']
RSI = stock['RSI']
state, buy_price = self.analyzeFinalScore(last_index, STOCK, MACD, STOCHASTIC, ICHIMOKU, RSI)
stochastic_score = STOCHASTIC[last_index]['slow_k']
macd_score = MACD[last_index]['macd']
rsi_score = RSI[last_index]['rsi']
ichimoku_score = ICHIMOKU[last_index]['ichimoku_buy']
if state != "":
fig = self.draw(stock)
title = "%s (%s), %s, buy_price (%d), stochastic(%.3f), rsi(%.3f), macd(%.3f), ichimoku(%d)) 차트" % (item_name, item_code, state, buy_price, stochastic_score, rsi_score, macd_score, ichimoku_score)
fig['layout'].update(title=title)
fileName = "%s/%s_%.3f_%.3f_%.3f_%d_%s.html" % (outPath, state, stochastic_score, rsi_score, macd_score, ichimoku_score, item_name.replace(" ", ""))
po.write_html(fig, file=fileName, auto_open=False)
else:
if RSI[last_index]['rsi_buy'] == 1 and STOCK[last_index]['volume'] > 10000:
fig = self.draw(stock)
title = "%s (%s) buy_price (%d), stochastic(%.3f), rsi(%.3f), macd(%.3f), ichimoku(%d)) 차트"%(item_name, item_code, buy_price, stochastic_score, rsi_score, macd_score, ichimoku_score)
fig['layout'].update(title=title)
fileName = "%s/%.3f_%.3f_%.3f_%d_%s.html"%(tmp_path, stochastic_score, rsi_score, macd_score, ichimoku_score, item_name.replace(" ", ""))
po.write_html(fig, file=fileName, auto_open=False)
print ("#html", rowid, stock['NAME'])
rowid += 1
cursor.execute('SELECT * FROM ' + self.tableName + ' WHERE rowid=?', (rowid,))
result = cursor.fetchone()
cursor.close()
conn.close()
return
def analyze(self):
conn = sqlite3.connect(self.inFileName)
cursor = conn.cursor()
# 기존 분석 데이터를 모두 지움
cursor.execute('update ' + self.tableName + ' set ICHIMOKU = "", MACD = "", STOCHASTIC = "", RSI = ""')
rowid = 1
cursor.execute('SELECT * FROM ' + self.tableName + ' WHERE rowid=?', (rowid,))
result = cursor.fetchone()
while result != None:
stock = {"CODE": result[0], "NAME": result[1], "PRICE": json.loads(result[2])}
try:
results_ICHIMOKU = self.ichimokuCloud.analyze(stock)
text_ICHIMOKU = json.dumps(results_ICHIMOKU, ensure_ascii=False)
results_MACD = self.macd.analyze(stock)
text_MACD = json.dumps(results_MACD, ensure_ascii=False)
results_STOCHASTIC = self.stochastic.analyze(stock)
text_STOCHASTIC = json.dumps(results_STOCHASTIC, ensure_ascii=False)
results_RSI = self.rsi.analyze(stock)
text_RSI = json.dumps(results_RSI, ensure_ascii=False)
except:
print("#", rowid, stock['NAME'])
rowid += 1
cursor.execute('SELECT * FROM ' + self.tableName + ' WHERE rowid=?', (rowid,))
result = cursor.fetchone()
continue
cursor.execute("UPDATE " + self.tableName + " SET ICHIMOKU=?, MACD=?, STOCHASTIC=?, RSI=? WHERE CODE=?", (text_ICHIMOKU,text_MACD,text_STOCHASTIC,text_RSI, stock["CODE"]))
print("#", rowid, stock['NAME'])
rowid += 1
cursor.execute('SELECT * FROM ' + self.tableName + ' WHERE rowid=?', (rowid,))
result = cursor.fetchone()
conn.commit()
cursor.close()
conn.close()
return
if __name__ == "__main__":
PROJECT_HOME = "../.."
inFileName = PROJECT_HOME + '/resources/stock.db'
inFnguideFileName = PROJECT_HOME + '/resources/fnguide.db'
analyzer = Analyzer(PROJECT_HOME, inFileName, inFnguideFileName)
# 분석 & update DB
"""
#print ("analyze IchimokuCloud...")
analyzer.analyzeIchimokuCloud()
#print ("analyze MACD...")
analyzer.analyzeMACD()
#print ("analyze Stochastic...")
analyzer.analyzeStochastic()
#print ("analyze RSI...")
analyzer.analyzeRSI()
"""
###analyzer.analyze()
day = datetime.today().strftime("%Y%m%d")
# HTML 출력
outPath = PROJECT_HOME + "/resources/analysis/"+day
if os.path.isdir(outPath):
shutil.rmtree(outPath)
os.mkdir(outPath)
print("print to Html...")
analyzer.analyzeToHtml(outPath)
# 파일 출력
#print("print to File...")
#outFileName = PROJECT_HOME + '/resources/analysis/'+day+'.json'
#analyzer.analyzeToFile(outFileName)
print("done...")

134
stockpredictor/analysis/Common.py Normal file
View File

@@ -0,0 +1,134 @@
class Common:
# 상향
def checkUpward(self, type, data):
check = True
if type != None:
for i in range(len(data)-1):
# 만약 이전이 이후보다 크다면, 상승이 아님
if data[i][type] > data[i+1][type]:
check = False
break
else:
for i in range(len(data)-1):
# 만약 이전이 이후보다 크다면, 상승이 아님
if data[i] > data[i+1]:
check = False
break
return check
# 하향
def checkDownward(self, type, data):
check = True
for i in range(len(data)-1):
# 만약 이전이 이후보다 작다면, 하락이 아님
if data[i][type] < data[i+1][type]:
check = False
break
return check
# 상향 돌파
def checkUpwardBreakthrough(self, type1, type2, data):
if (type1 in data[0] and type1 in data[1] and type1 in data[2] and
type2 in data[0] and type2 in data[1] and type2 in data[2]):
if ((data[0][type1] < data[1][type1] < data[2][type1]) and
(data[0][type1] < data[0][type2] and data[2][type1] > data[2][type2])):
return True
return False
# 하향 돌파
def checkDownwardBreakthrough(self, type1, type2, data):
if (type1 in data[0] and type1 in data[1] and type1 in data[2] and
type2 in data[0] and type2 in data[1] and type2 in data[2]):
if ((data[0][type1] > data[1][type1] > data[2][type1]) and
(data[0][type1] > data[0][type2] and data[2][type1] < data[2][type2])):
return True
return False
def getStochasticScore(self, stock, i):
score = 0
if (stock[i - 1]['slow_k'] < stock[i]['slow_k'] and
stock[i]['slow_d'] < stock[i]['slow_k']):
if stock[i]['slow_k'] < 5:
score = 8
elif 5 <= stock[i]['slow_k'] < 10:
score = 7
elif 10 <= stock[i]['slow_k'] < 15:
score = 6
elif 15 <= stock[i]['slow_k'] < 20:
score = 5
elif 20 <= stock[i]['slow_k'] < 30:
score = 4
elif 30 <= stock[i]['slow_k'] < 40:
score = 3
elif 40 <= stock[i]['slow_k'] < 50:
score = 2
else:
score = 1
if (stock[i - 1]['slow_k'] > stock[i]['slow_k'] and
stock[i - 1]['slow_k'] > stock[i - 1]['slow_d'] and
stock[i]['slow_k'] < stock[i]['slow_d']):
if stock[i]['slow_k'] > 90:
score = -6
elif 90 >= stock[i]['slow_k'] > 80:
score = -5
elif 80 >= stock[i]['slow_k'] > 70:
score = -4
elif 70 >= stock[i]['slow_k'] > 60:
score = -3
elif 60 >= stock[i]['slow_k'] > 50:
score = -2
else:
score = -1
return score
def getIchimokuCloudScore(self, stock, i):
score = 0
if stock[i - 1]['leadingSpan1'] != 0 and stock[i - 1]['leadingSpan2'] != 0:
# 후행스팬 > 선행스펜 일때, 후행스펜 > 어제 주가 > 선행스팬 이고, 오늘 주가 > 후행스팬 < 선행스팬 이라면, 매수 2점
if (stock[i - 1]['leadingSpan2'] > stock[i - 1]['leadingSpan1'] and stock[i]['leadingSpan2'] > stock[i]['leadingSpan1']):
if (stock[i - 1]['leadingSpan2'] > stock[i - 1]['close'] > stock[i - 1]['leadingSpan1'] and
stock[i]['close'] > stock[i]['leadingSpan2'] > stock[i - 1]['leadingSpan1']):
score = 2
# 후행스팬 > 선행스펜 일때, 후행스펜 > 선행스팬 > 어제 주가 이고, 오늘 주가 > 후행스팬 < 선행스팬 이라면, 매수 4점
if (stock[i - 1]['leadingSpan2'] > stock[i - 1]['leadingSpan1'] and stock[i]['leadingSpan2'] > stock[i]['leadingSpan1']):
if (stock[i - 1]['leadingSpan2'] > stock[i - 1]['leadingSpan1'] > stock[i - 1]['close'] and
stock[i]['close'] > stock[i]['leadingSpan2'] > stock[i - 1]['leadingSpan1']):
score = 4
# 선행스팬 > 후행스팬 일때, 선행스팬 > 어제 주가 > 후행스팬 이고, 오늘 주가 > 선행스팬 < 후행스팬 이라면, 매수 1점
if (stock[i - 1]['leadingSpan1'] > stock[i - 1]['leadingSpan2'] and stock[i]['leadingSpan1'] > stock[i]['leadingSpan2']):
if (stock[i - 1]['leadingSpan1'] > stock[i - 1]['close'] > stock[i - 1]['leadingSpan2'] and
stock[i]['close'] > stock[i]['leadingSpan1'] > stock[i - 1]['leadingSpan2']):
score = 1
# 선행스팬 > 후행스팬 일때, 선행스팬 > 후행스팬 > 어제 주가 이고, 오늘 주가 > 선행스팬 < 후행스팬 이라면, 매수 3점
if (stock[i - 1]['leadingSpan1'] > stock[i - 1]['leadingSpan2'] and stock[i]['leadingSpan1'] > stock[i]['leadingSpan2']):
if (stock[i - 1]['leadingSpan1'] > stock[i - 1]['leadingSpan2'] > stock[i - 1]['close'] and
stock[i]['close'] > stock[i]['leadingSpan1'] > stock[i - 1]['leadingSpan2']):
score = 3
# 어제는 주가가 선행이나 후행스팬 위에 있었지만, 오늘은 두 스팬 모두 아래로 내려왔을 때 매도
if (stock[i - 1]['close'] > stock[i - 1]['leadingSpan1'] or stock[i - 1]['close'] > stock[i - 1]['leadingSpan2']):
if (stock[i]['close'] < stock[i]['leadingSpan1'] and stock[i]['close'] < stock[i]['leadingSpan2']):
score = -1
return score
def checkLongYangBongAfterUmBong(self, stock, i):
if i > 0:
if stock[i-1]['open'] > stock[i-1]['close']: # 어제가 음봉인지 체크
if stock[i]['open'] < stock[i]['close'] and stock[i]['close'] == stock[i]['high']: # 오늘 장대양봉인지 체크
if stock[i-1]['volume']*2 < stock[i]['volume']: # 어제 거래량 보다 두배 이상일 때
return True
return False

127
stockpredictor/analysis/IchimokuCloud.py Normal file
View File

@@ -0,0 +1,127 @@
import pandas as pd
import datetime
from plotly import tools, subplots
import plotly.offline as offline
import plotly.graph_objs as go
import plotly.io as po
from stockpredictor.analysis.Common import Common
# graph: https://plotly.com/python/candlestick-charts/
# https://www.nanumtrading.com/fx-%EB%B0%B0%EC%9A%B0%EA%B8%B0/%EC%B0%A8%ED%8A%B8-%EB%B3%B4%EC%A1%B0%EC%A7%80%ED%91%9C-%EC%9D%B4%ED%95%B4/06-%EC%9D%BC%EB%AA%A9%EA%B7%A0%ED%98%95%ED%91%9C/
# 일목균형표 - 매매기법 알아보기 !: https://layhope.tistory.com/222
class IchimokuCloud:
common = None
def __init__(self):
self.common = Common()
return
def draw(self, stock):
item_name = stock["NAME"]
item_code = stock["CODE"]
df = pd.DataFrame(stock["PRICE"])
leadingSpan1 = go.Scatter(x=df.DATE, y=df['leadingSpan1'], name="선행스팬", line_color='#8B4513')
leadingSpan2 = go.Scatter(x=df.DATE, y=df['leadingSpan2'], name="후행스팬", line_color='#4169E1')
candle = go.Candlestick(x=df.DATE, open=df.open, high=df.high, low=df.low, close=df.close,
increasing_line_color= 'red', decreasing_line_color= 'blue')
data = [leadingSpan1, leadingSpan2, candle]
layout = go.Layout(title='{} MACD 그래프'.format(item_name))
fig = subplots.make_subplots(rows=1, cols=1, shared_xaxes=True)
for trace in data:
fig.append_trace(trace, 1,1)
fig = go.Figure(data=data, layout=layout)
path = "/Users/dsyoon/workspace/StockPredictor/resources/analysis/html"
po.write_html(fig, file=path + "/ichimokuCloud_" + item_code+'.html', auto_open=False)
return fig
# c=9, b=26, l=52
def apply(self, df, c=9, b=26, l=52):
# 입력받은 값이 dataframe이라는 것을 정의해줌
df = pd.DataFrame(df)
# 1. 전환선 = (과거 9일 동안 최고가 + 최저가) / 2
# 당일을 포함한 9일 동안의 최고가와 최저가의 중간 값을 평균으로 나타낸다.
changeLine = (df.high.rolling(c).max() + df.low.rolling(c).min()) / 2
# 2. 기준선 = 과거 26일 동안 최고가 + 최저가) / 2
# 당일을 포함한 26일 동안의 최고가와 최저가의 중간 값을 평균으로 나타낸다.
baseLine = (df.high.rolling(b).max() + df.low.rolling(b).min()) / 2
# 3. 선행스팬 1 = ((기준선 + 전환선) / 2)를 26일 선행하여 배치
# 전환선과 기준선의 평균값을 구해 당일 포함 26일 앞으로 이동시킨 선 (중-단기 구간의 힘을 보여줌)
leadingSpan1 = (changeLine + baseLine) / 2
move_LeadingSpan1 = list(leadingSpan1.values)
for i in range(b - 1):
move_LeadingSpan1.insert(0, None)
# 4. 선행스팬 2 = ((최근 52일 동안 최고가 + 최저가) / 2)를 26일 선행하여 배치
# 당일을 포함한 52일 동안의 최고가와 최저가의 평균을 26일 앞으로 이동시킨 선 (장기으로 형성된 선이기 때문에 가장 느리게 변함)
leadingSpan2 = (df.high.rolling(l).max() + df.low.rolling(l).min()) / 2
move_LeadingSpan2 = list(leadingSpan2.values)
for i in range(l - 1):
move_LeadingSpan2.insert(0, None)
# leadingSpan2에 맞추어 뒤로 빈 row를 채운다.
for i in range(len(move_LeadingSpan2) - len(df)):
df = df.append({"DATE": None, "close": None, "diff": None, "open": None, "high": None, "low": None, "volume": None, "avg3": None, "avg5": None, "avg7": None, "avg10": None, "avg20": None, "avg30": None, "avg60": None, "avg90": None, "avg100": None, "avg120": None, "avg150": None, "avg180": None, "avg200": None, "avg240": None}, ignore_index=True)
move_changeLine = list(changeLine.values)
for i in range(len(move_LeadingSpan2) - len(move_changeLine)):
move_changeLine.append(None)
move_baseLine = list(baseLine.values)
for i in range(len(move_LeadingSpan2) - len(move_baseLine)):
move_baseLine.append(None)
for i in range(len(move_LeadingSpan2) - len(move_baseLine)):
move_LeadingSpan1.append(None)
# dataframe에 컬럼 추가
df = df.assign(changeLine=pd.Series(move_changeLine), baseLine=pd.Series(move_baseLine), leadingSpan1=pd.Series(move_LeadingSpan1), leadingSpan2=pd.Series(move_LeadingSpan2))
return df
# 일목균형표의 구성을 훑어보면 주가를 선행과 후행으로 과거의 주가를 통해 미래 혹은 현재의 주식의 가격의 추세를 예측해보려는 지표라는 것을 이해할 수 있다.
# 또한 구름층의 색을 통해서 주식의 추세를 손쉽게 확인할 수 있을 것 같다는 것도 이해할 수 있다면 끝 !
def analyze(self, stock):
df = pd.DataFrame()
df = df.from_dict(stock['PRICE'])
df = self.apply(df)
diff = len(df.changeLine) - len(stock['PRICE'])
lastDay = stock['PRICE'][len(stock['PRICE']) - 1]['DATE']
tmpLastDay = datetime.datetime.strptime(lastDay, "%Y-%m-%d")
for i in range(diff):
nextDay = tmpLastDay + datetime.timedelta(days=(i + 1))
stock['PRICE'].append(
{"DATE": nextDay.strftime("%Y-%m-%d"), "close": 0, "diff": 0, "open": 0, "high": 0, "low": 0, "volume": 0,
"avg3": 0, "avg5": 0, "avg7": 0, "avg10": 0, "avg20": 0, "avg30": 0, "avg60": 0, "avg90": 0, "avg100": 0,
"avg120": 0, "avg150": 0, "avg180": 0, "avg200": 0, "avg240": 0})
for i in range(len(df.changeLine)):
stock['PRICE'][i]['ichimoku_buy'] = 0
stock['PRICE'][i]['changeLine'] = df.changeLine.values[i]
stock['PRICE'][i]['baseLine'] = df.baseLine.values[i]
stock['PRICE'][i]['leadingSpan1'] = df.leadingSpan1.values[i]
stock['PRICE'][i]['leadingSpan2'] = df.leadingSpan2.values[i]
for i in range(len(df.changeLine)):
stock['PRICE'][i]['ichimoku_buy'] = self.common.getIchimokuCloudScore(stock['PRICE'], i)
results = []
for day in stock['PRICE']:
results.append({'DATE': day['DATE'],
'changeLine': day['changeLine'],
'baseLine': day['baseLine'],
'leadingSpan1': day['leadingSpan1'],
'leadingSpan2': day['leadingSpan2'],
'ichimoku_buy': day['ichimoku_buy']})
return results
if __name__ == "__main__":
ichimokuCloud = IchimokuCloud()

110
stockpredictor/analysis/MACD.py Normal file
View File

@@ -0,0 +1,110 @@
import pandas as pd
from stockpredictor.analysis.Common import Common
from plotly import tools, subplots
import plotly.offline as offline
import plotly.graph_objs as go
import plotly.io as po
# [청송촌놈] 파생을 알아야 시장이 보인다. 청송이 종목 고르는법! https://www.youtube.com/watch?v=weABtgZDeGg
# 6. Pandas와 Plotly를 이용한 MACD 차트 그리기 https://excelsior-cjh.tistory.com/110
# 첫번째. MACD 지표를 이용한 차트분석: https://post.naver.com/viewer/postView.nhn?volumeNo=7435935&memberNo=32471429
# MACD (Moving Average Conver gence Divergence)
# 빨간 네모박스권으로 MACD가 MACD-Sign 을 골든크로스하며, 상승하였을때, 주가는 상승추세를 유지하며, MACD가 MACD-Sign(분홍색)을 데드크로스 할때 주가는 하락의 추세를 보이게 됩니다.
# 즉, MSCD가 0이상에서 MACD-Sign 위에서 상승하는 그림이어야
class MACD:
common = None
def __init__(self):
self.common = Common()
return
def draw(self, stock):
item_name = stock["NAME"]
item_code = stock["CODE"]
df = pd.DataFrame(stock["PRICE"])
macd = go.Scatter(x=df.DATE, y=df['macd'], name="MACD")
signal = go.Scatter(x=df.DATE, y=df['macds'], name="Signal")
oscillator = go.Bar(x=df.DATE, y=df['macdo'], name="oscillator")
trade_volume = go.Bar(x=df.DATE, y=df['volume'], name="volume")
data = [macd, signal, oscillator]
layout = go.Layout(title='{} MACD 그래프'.format(item_name))
fig = subplots.make_subplots(rows=2, cols=1, shared_xaxes=True)
for trace in data:
fig.append_trace(trace, 1,1)
fig.append_trace(trade_volume, 2,1)
fig = go.Figure(data=data, layout=layout)
path = "/Users/dsyoon/workspace/StockPredictor/resources/analysis/html"
po.write_html(fig, file=path + "/macd_" + item_code+'.html', auto_open=False)
return fig
# macd 0선 위에서 매수를 한다. 0이하는 절대 처다보지 않는다.
def apply(self, df, short=12, long=26, t=9):
# 입력받은 값이 dataframe이라는 것을 정의해줌
df = pd.DataFrame(df)
# MACD 관련 수식
ma_12 = df.close.ewm(span=short).mean() # 단기(12) EMA(지수이동평균)
ma_26 = df.close.ewm(span=long).mean() # 장기(26) EMA
macd = ma_12 - ma_26 # MACD
macds = macd.ewm(span=t).mean() # Signal
macdo = macd - macds # Oscillator
#df = df.assign(macd=macd, macds=macds, macdo=macdo).dropna()
df = df.assign(macd=macd, macds=macds, macdo=macdo)
return df
"""
# 기존 stock에 삽입
# macd 0선 위에서 매수를 한다. 0이하는 절대 처다보지 않는다.
def analyze(self, stock):
df = pd.DataFrame()
df = df.from_dict(stock['PRICE'])
df = self.apply(df)
for i in range(len(df.macd)):
stock['PRICE'][i]['macd_buy'] = 0
stock['PRICE'][i]['macd'] = df.macd.values[i]
stock['PRICE'][i]['macds'] = df.macds.values[i]
stock['PRICE'][i]['macdo'] = df.macdo.values[i]
if df.macd.values[i] > 0 and self.common.checkUpward('close', stock['PRICE'][i - 4: i + 1]):
stock['PRICE'][i]['macd_buy'] = df.macd.values[i]
return
"""
def analyze(self, stock):
results = []
df = pd.DataFrame()
df = df.from_dict(stock['PRICE'])
df = self.apply(df)
for i in range(len(df.macd)):
result = {'DATE':stock['PRICE'][i]['DATE'],
'macd': df.macd.values[i],
'macds': df.macds.values[i],
'macdo': df.macdo.values[i],
'macd_buy': 0}
# MACD가 3일 전부터 상승이라면 매수
if df.macd.values[i - 1] < df.macd.values[i]:
# 어제는 MACD-Sign이 MACD 위에 있지만, 오늘은 MACD가 MACD-Sign 위로 올라오면 매수
if df.macd.values[i-1] < df.macds.values[i-1] and df.macd.values[i] > df.macds.values[i]:
result['macd_buy'] = 1
# MACD가 3일 전부터 하락이라면 매도
if df.macd.values[i - 1] > df.macd.values[i]:
# 어제는 MACD가 MACD-Sign 위에 있지만, 오늘은 MACD-Sign이 MACD 위로 올라오면 매도
if df.macd.values[i - 1] > df.macds.values[i - 1] and df.macd.values[i] < df.macds.values[i]:
result['macd_buy'] = -1
results.append(result)
return results

113
stockpredictor/analysis/RSI.py Normal file
View File

@@ -0,0 +1,113 @@
import pandas as pd
from stockpredictor.analysis.Common import Common
from plotly import tools, subplots
import numpy as np
import plotly.graph_objs as go
import plotly.io as po
# [청송촌놈] 파생을 알아야 시장이 보인다. 청송이 종목 고르는법! https://www.youtube.com/watch?v=weABtgZDeGg
# 6. Pandas와 Plotly를 이용한 MACD 차트 그리기 https://excelsior-cjh.tistory.com/110
# 첫번째. MACD 지표를 이용한 차트분석: https://post.naver.com/viewer/postView.nhn?volumeNo=7435935&memberNo=32471429
# MACD (Moving Average Conver gence Divergence)
# 빨간 네모박스권으로 MACD가 MACD-Sign 을 골든크로스하며, 상승하였을때, 주가는 상승추세를 유지하며, MACD가 MACD-Sign(분홍색)을 데드크로스 할때 주가는 하락의 추세를 보이게 됩니다.
# 즉, MSCD가 0이상에서 MACD-Sign 위에서 상승하는 그림이어야
class RSI:
common = None
def __init__(self):
self.common = Common()
return
def draw(self, stock):
item_name = stock["NAME"]
item_code = stock["CODE"]
df = pd.DataFrame(stock["PRICE"])
rsi = go.Scatter(x=df.DATE, y=df['rsi'], name="RSI")
signal = go.Scatter(x=df.DATE, y=df['rsis'], name="RSI Signal")
data = [rsi, signal]
layout = go.Layout(title='{} RSI 그래프'.format(item_name))
fig = subplots.make_subplots(rows=2, cols=1, shared_xaxes=True)
for trace in data:
fig.append_trace(trace, 1,1)
fig = go.Figure(data=data, layout=layout)
path = "/Users/dsyoon/workspace/StockPredictor/resources/analysis/html"
po.write_html(fig, file=path + "/rsi" + item_code+'.html', auto_open=False)
return fig
def apply(sefl, df, period=14):
# df.diff를 통해 (기준일 종가 - 기준일 전일 종가)를 계산하여 0보다 크면 증가분을 감소했으면 0을 넣어줌
U = np.where(df.close.diff(1) > 0, df.close.diff(1), 0)
# df.diff를 통해 (기준일 종가 - 기준일 전일 종가)를 계산하여 0보다 작으면 감소분을 증가했으면 0을 넣어줌
D = np.where(df.close.diff(1) < 0, df.close.diff(1) * (-1), 0)
# AU, period=14일 동안의 U의 평균
AU = pd.DataFrame(U).rolling(window=period, min_periods=period).mean()
# AD, period=14일 동안의 D의 평균
AD = pd.DataFrame(D).rolling(window=period, min_periods=period).mean()
rsi = AU.div(AD + AU) * 100
rsis = rsi.rolling(window=9).mean()
df = df.assign(rsi=rsi, rsis=rsis)
return df
def analyze(self, stock):
"""
RSI 값이 100에 접근하면 ㄷ 이상의 주가 상승을 기대하기 어렵고, 0에 접근하면 더 이상 하락을 기대하기 어렵다.
70이상이면 과매수 구간이라 할 수 있고, 30 이하면 과매도 구간이라 볼 수 있다.
따라서 과매수 구간에서는 매도 준비를, 과매도 구간에서는 매수 준비를 해야 한다.
"""
results = []
df = pd.DataFrame()
df = df.from_dict(stock['PRICE'])
df = self.apply(df)
for i in range(len(df.rsi)):
result = {'DATE':stock['PRICE'][i]['DATE'],
'rsi': df.rsi.values[i],
'rsis': df.rsis.values[i],
'rsi_buy': 0}
# rsi가 상향이고 30을 돌파하면 매수,
if df.rsi.values[i - 1] < df.rsi.values[i]:
if df.rsi.values[i-1] <= 30 and df.rsi.values[i] > 30:
result['rsi_buy'] = 1
# rsi가 상향이고 40을 돌파하면 매수,
if df.rsi.values[i - 1] < df.rsi.values[i]:
if df.rsi.values[i-1] <= 50 and df.rsi.values[i] > 50:
result['rsi_buy'] = 1
# rsi가 상향이고 70을 돌파하면 단기매수,
if df.rsi.values[i - 1] < df.rsi.values[i]:
if df.rsi.values[i-1] <= 70 and df.rsi.values[i] > 70:
result['rsi_buy'] = 1
# rsi가 하향이고 70이하로 떨어지면 매도,
if df.rsi.values[i - 1] > df.rsi.values[i]:
if df.rsi.values[i-1] > 70 and df.rsi.values[i] <= 70:
result['rsi_buy'] = -1
# rsi가 하향이고 50이하로 떨어지면 매도,
if df.rsi.values[i - 1] > df.rsi.values[i]:
if df.rsi.values[i-1] > 50 and df.rsi.values[i] <= 50:
result['rsi_buy'] = -1
# rsi가 하향이고 30이하로 떨어지면 단기매도,
if df.rsi.values[i - 1] > df.rsi.values[i]:
if df.rsi.values[i-1] > 30 and df.rsi.values[i] <= 30:
result['rsi_buy'] = -1
results.append(result)
return results

188
stockpredictor/analysis/Stochastic.py Normal file
View File

@@ -0,0 +1,188 @@
import pandas as pd
from stockpredictor.analysis.Common import Common
import plotly.graph_objs as go
from plotly import tools, subplots
import plotly.io as po
# 6. Pandas와 Plotly를 이용한 MACD 차트 그리기: https://excelsior-cjh.tistory.com/111
# 스토캐스틱 슬로우(Stochastics Slow)를 이용한 간단한 매매기법: https://bagal.tistory.com/124
# Stochastic Slow, 스토캐스틱을 통한 주식 매매법 정리: https://m.blog.naver.com/PostView.nhn?blogId=kangyh427&logNo=220957146041&proxyReferer=https:%2F%2Fwww.google.com%2F
# 차트분석 2편 스토캐스틱을 활용한 매수타이밍 포착!: https://m.post.naver.com/viewer/postView.nhn?volumeNo=7446693&memberNo=32471429
class Stochastic:
common = None
def __init__(self):
self.common = Common()
return
def draw(self, stock):
item_name = stock["NAME"]
item_code = stock["CODE"]
df = pd.DataFrame(stock["PRICE"])
slow_k = go.Scatter(x=df.DATE, y=df['slow_k'], name="Slow%K")
slow_d = go.Scatter(x=df.DATE, y=df['slow_d'], name="Slow%D")
trade_volume = go.Bar(x=df.DATE, y=df['volume'], name="volume")
data1 = [slow_k, slow_d]
data2 = [trade_volume]
fig = subplots.make_subplots(rows=2, cols=1, shared_xaxes=True)
for trace in data1:
fig.append_trace(trace, 1, 1)
for trace in data2:
fig.append_trace(trace, 2, 1)
fig['layout'].update(title='{} MACD 그래프'.format(item_name))
path = "/Users/dsyoon/workspace/StockPredictor/resources/analysis/html"
po.write_html(fig, file=path + "/stochastic_" + item_code+'.html', auto_open=False)
return fig
# 일자(n,m,t)에 따른 Stochastic(KDJ)의 값을 구하기 위해 함수형태로 만듬
# n=15 (%k), m=5 (%d), t=3
def apply(self, df, n=10, m=6, t=6):
# 입력받은 값이 dataframe이라는 것을 정의해줌
df = pd.DataFrame(df)
# n일중 최고가
ndays_high = df.high.rolling(n).max()
# n일중 최저가
ndays_low = df.low.rolling(n).min()
# Fast%K 계산
# n(15)일 동안의 최고가(high)와 최저가(low) 사이 중 현재 종가(close)의 상대적 위치를 판단하는 값
# 가격이 지속적으로 상승하고 있다면 Stochastic 값은 100에 가까워 지며, 반대로 지속적으로 하락하고 있다면 Stochastic 값은 0에 가까워 지는 경향을 나타낸다.
fast_k = ((df.close - ndays_low) / (ndays_high - ndays_low)) * 100
# Fast%D (=Slow%K) 계산
# m(5)일간의 Fast%K의 이동평균 값
# 기본값으로 5일을 설정하며 Fast%K 값을 일반화하는 역할을 한다.
#slow_k = fast_k.ewm(span=m).mean()
slow_k = fast_k.rolling(m).mean()
# Slow%D 계산
# t(3)일간의 Slow%K의 이동평균 값
# 기본값으로 3일을 설정하며 Slow%K 값을 일반화 하는 역할을 한다.
#slow_d = slow_k.ewm(span=t).mean()
slow_d = slow_k.rolling(t).mean()
# dataframe에 컬럼 추가
#df = df.assign(fast_k=fast_k, slow_k=slow_k, slow_d=slow_d).dropna()
df = df.assign(fast_k=fast_k, slow_k=slow_k, slow_d=slow_d)
return df
"""
def analyze(self, stock):
df = pd.DataFrame()
df = df.from_dict(stock['PRICE'])
df = self.apply(df)
for i in range(len(df.fast_k)):
stock['PRICE'][i]['stochastic_buy'] = 0
if "fast_k" not in stock['PRICE'][i]:
stock['PRICE'][i]['fast_k'] = -1
stock['PRICE'][i]['slow_k'] = -1
stock['PRICE'][i]['slow_d'] = -1
i_ = i + len(stock['PRICE']) - len(df.fast_k)
stock['PRICE'][i_]['fast_k'] = df.fast_k.values[i]
stock['PRICE'][i_]['slow_k'] = df.slow_k.values[i]
stock['PRICE'][i_]['slow_d'] = df.slow_d.values[i]
# 0: 중립, 1: 매수, -1: 매도
stock['PRICE'][i_]['stochastic_buy'] = 0
stock['PRICE'][i_]['direction'] = ''
if i_ > 2:
if ('slow_k' in stock['PRICE'][i_-1] and 'slow_k' in stock['PRICE'][i_]):
# %k선이 %d선을 상향 돌파하면 매수 신호
if (self.common.checkUpward('slow_k', stock['PRICE'][i_-2:i_+1]) and
stock['PRICE'][i_-1]['slow_k'] < stock['PRICE'][i_-1]['slow_d'] and
stock['PRICE'][i_]['slow_k'] > stock['PRICE'][i_]['slow_d']):
if stock['PRICE'][i_]['slow_k'] < 5:
stock['PRICE'][i_]['stochastic_buy'] = 8
elif 5 <= stock['PRICE'][i_]['slow_k'] < 10:
stock['PRICE'][i_]['stochastic_buy'] = 7
elif 10 <= stock['PRICE'][i_]['slow_k'] < 15:
stock['PRICE'][i_]['stochastic_buy'] = 6
elif 15 <= stock['PRICE'][i_]['slow_k'] < 20:
stock['PRICE'][i_]['stochastic_buy'] = 5
elif 20 <= stock['PRICE'][i_]['slow_k'] < 30:
stock['PRICE'][i_]['stochastic_buy'] = 4
elif 30 <= stock['PRICE'][i_]['slow_k'] < 40:
stock['PRICE'][i_]['stochastic_buy'] = 3
elif 40 <= stock['PRICE'][i_]['slow_k'] < 50:
stock['PRICE'][i_]['stochastic_buy'] = 2
else:
stock['PRICE'][i_]['stochastic_buy'] = 1
if self.common.checkUpward('slow_k', stock['PRICE'][i_ - 1:i_ + 1]):
stock['PRICE'][i_]['direction'] = 'up'
if (self.common.checkDownward('slow_k', stock['PRICE'][i_ - 2:i_ + 1]) and
stock['PRICE'][i_-1]['slow_k'] > stock['PRICE'][i_-1]['slow_d'] and
stock['PRICE'][i_]['slow_k'] < stock['PRICE'][i_]['slow_d']):
if stock['PRICE'][i_]['slow_k'] > 90:
stock['PRICE'][i_]['stochastic_buy'] = -6
elif 90 >= stock['PRICE'][i_]['slow_k'] > 80:
stock['PRICE'][i_]['stochastic_buy'] = -5
elif 80 >= stock['PRICE'][i_]['slow_k'] > 70:
stock['PRICE'][i_]['stochastic_buy'] = -4
elif 70 >= stock['PRICE'][i_]['slow_k'] > 60:
stock['PRICE'][i_]['stochastic_buy'] = -3
elif 60 >= stock['PRICE'][i_]['slow_k'] > 50:
stock['PRICE'][i_]['stochastic_buy'] = -2
else:
stock['PRICE'][i_]['stochastic_buy'] = -1
if self.common.checkDownward('slow_k', stock['PRICE'][i_ - 1:i_ + 1]):
stock['PRICE'][i_]['direction'] = 'down'
results = []
for day in stock['PRICE']:
results.append({'DATE': day['DATE'],
'fast_k': day['fast_k'],
'slow_k': day['slow_k'],
'slow_d': day['slow_d'],
'stochastic_buy': day['stochastic_buy']})
return results
"""
def analyze(self, stock):
df = pd.DataFrame()
df = df.from_dict(stock['PRICE'])
df = self.apply(df)
for i in range(len(df.fast_k)):
stock['PRICE'][i]['stochastic_buy'] = 0
if "fast_k" not in stock['PRICE'][i]:
stock['PRICE'][i]['fast_k'] = -1
stock['PRICE'][i]['slow_k'] = -1
stock['PRICE'][i]['slow_d'] = -1
stock['PRICE'][i]['fast_k'] = df.fast_k.values[i]
stock['PRICE'][i]['slow_k'] = df.slow_k.values[i]
stock['PRICE'][i]['slow_d'] = df.slow_d.values[i]
# 0: 중립, 1: 매수, -1: 매도
stock['PRICE'][i]['stochastic_buy'] = 0
if i > 0:
if ('slow_k' in stock['PRICE'][i-1] and 'slow_k' in stock['PRICE'][i]):
stock['PRICE'][i]['stochastic_buy'] = self.common.getStochasticScore(stock['PRICE'], i)
results = []
for day in stock['PRICE']:
results.append({'DATE': day['DATE'],
'fast_k': day['fast_k'],
'slow_k': day['slow_k'],
'slow_d': day['slow_d'],
'stochastic_buy': day['stochastic_buy']})
return results
if __name__ == "__main__":
stochastic = Stochastic()

77
stockpredictor/crawler/toJsonFile/Crawler.py Normal file
View File

@@ -0,0 +1,77 @@
import os
import shutil
import datetime
from stockpredictor.crawler.toJsonFile.FnGuideCrawler import FnGuideCrawler
from stockpredictor.crawler.toJsonFile.MetaCrawler import MetaCrawler
from stockpredictor.crawler.toJsonFile.StockCrawler import StockCrawler
today = datetime.datetime.now().strftime("%Y-%m-%d")
PROJECT_HOME = "../../.."
crawler = FnGuideCrawler()
print("[KOSPI 상장기업 재무제표 다운로드]")
inFileName = PROJECT_HOME + '/resources/fnguide.json'
outFileName = PROJECT_HOME + '/resources/fnguide.temp.json'
if os.path.isfile(inFileName):
shutil.copy(inFileName, outFileName)
crawler.crawl_fnguide(outFileName)
if os.path.isfile(inFileName):
os.remove(inFileName)
shutil.move(outFileName, inFileName)
crawler = MetaCrawler()
print("[환율 (USD, JPY, EUR, CNY), 원유 (WTI), 국제금]")
inFileName = PROJECT_HOME + '/resources/meta_1.json'
outFileName = PROJECT_HOME + '/resources/meta_1.temp.json'
if os.path.isfile(inFileName):
shutil.copy(inFileName, outFileName)
crawler.crawl_stocks(outFileName)
if os.path.isfile(inFileName):
os.remove(inFileName)
shutil.move(outFileName, inFileName)
print("[투자자별 매매동향(Trading_Trend)]")
inFileName = PROJECT_HOME + '/resources/meta_2.json'
outFileName = PROJECT_HOME + '/resources/meta_2.temp.json'
if os.path.isfile(inFileName):
shutil.copy(inFileName, outFileName)
crawler.crawl_trading_trend(outFileName)
if os.path.isfile(inFileName):
os.remove(inFileName)
shutil.move(outFileName, inFileName)
print("[증시자금동향 (신용잔고, 펀드자금 잔고)]")
inFileName = PROJECT_HOME + '/resources/meta_3.json'
outFileName = PROJECT_HOME + '/resources/meta_3.temp.json'
if os.path.isfile(inFileName):
shutil.copy(inFileName, outFileName)
crawler.crawl_money_trend(outFileName)
if os.path.isfile(inFileName):
os.remove(inFileName)
shutil.move(outFileName, inFileName)
print("[국내 시장금리]")
inFileName = PROJECT_HOME + '/resources/meta_4.json'
outFileName = PROJECT_HOME + '/resources/meta_4.temp.json'
if os.path.isfile(inFileName):
shutil.copy(inFileName, outFileName)
crawler.crawl_interest_rates(outFileName)
if os.path.isfile(inFileName):
os.remove(inFileName)
shutil.move(outFileName, inFileName)
print("[종목 다운로드]")
inFileName = PROJECT_HOME + '/resources/stock.json'
outFileName = PROJECT_HOME + '/resources/stock.temp.json'
if os.path.isfile(inFileName):
shutil.copy(inFileName, outFileName)
crawler = StockCrawler()
crawler.crawl_stocks(outFileName)
crawler.get_stocks_avg(outFileName, inFileName)
if os.path.isfile(outFileName):
os.remove(outFileName)
print ("done...")

113
stockpredictor/crawler/toJsonFile/FnGuideCrawler.py Normal file
View File

@@ -0,0 +1,113 @@
from bs4 import BeautifulSoup
from pandas import DataFrame, Series
import requests as re
import pandas as pd
import json
import requests
class FnGuideCrawler:
header = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36'}
def getStockInfo(self):
#code_df = pd.read_html('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', header=0)[0]
code_df = pd.read_html(requests.get('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', headers=self.header).text)[0]
# code_df = pd.read_excel('../resources/stock/상장법인목록.xls')
# 종목코드가 6자리이기 때문에 6자리를 맞춰주기 위해 설정해줌
code_df.종목코드 = code_df.종목코드.map('{:06d}'.format)
# 우리가 필요한 것은 회사명과 종목코드이기 때문에 필요없는 column들은 제외해준다.
code_df = code_df[['회사명', '종목코드']]
# 한글로된 컬럼명을 영어로 바꿔준다.
code_df = code_df.rename(columns={'회사명': 'name', '종목코드': 'code'})
###print (code_df.head())
return code_df
# FnGuide에서 크롤링한 KOSPI 상장기업의 재무제표
# http://blog.naver.com/PostView.nhn?blogId=koko8624&logNo=221294884955&parentCategoryNo=&categoryNo=&viewDate=&isShowPopularPosts=false&from=postView
def get_fnguide_table(self, code):
url = re.get('http://comp.fnguide.com/SVO2/ASP/SVD_main.asp?pGB=1&gicode=A%s'%(code.strip()))
url = url.content
html = BeautifulSoup(url,'html.parser')
body = html.find('body')
try:
fn_body = body.find('div', {'class': 'fng_body asp_body'})
ur_table = fn_body.find('div', {'id': 'div15'})
table = ur_table.find('div', {'id': 'highlight_D_Y'})
tbody = table.find('tbody')
tr = tbody.find_all('tr')
Table = DataFrame()
except:
return {}
for i in tr:
''' 자료 항목 가져오기'''
category = i.find('span', {'class': 'txt_acd'})
if category == None:
category = i.find('th')
category = category.text.strip()
'''값 가져오기'''
value_list = []
j = i.find_all('td', {'class': 'r'})
for value in j:
temp = value.text.replace(',', '').strip()
try:
temp = float(temp)
value_list.append(temp)
except:
value_list.append(0)
Table['%s' % (category)] = value_list
''' 기간 가져오기 '''
thead = table.find('thead')
tr_2 = thead.find('tr', {'class': 'td_gapcolor2'}).find_all('th')
year_list = []
for i in tr_2:
try:
temp_year = i.find('span', {'class': 'txt_acd'}).text
except:
temp_year = i.text
temp_year = temp_year.replace("/",".")+".01"
year_list.append(temp_year)
Table.index = year_list
return Table.T.to_dict()
def crawl_fnguide(self, inFileName):
code_df = self.getStockInfo()
outFp = open(inFileName, 'w', encoding='utf-8')
idx = 0
for item in code_df.values:
item_name = item[0]
item_code = item[1]
idx += 1
print(idx, item_name)
fnGuideData = self.get_fnguide_table(item_code)
stock = {"NAME": item_name, "CODE": item_code, "PRICE": fnGuideData}
outFp.write(json.dumps(stock, ensure_ascii=False) + "\n")
outFp.close()
return
if __name__ == "__main__":
crawler = FnGuideCrawler()
crawler.get_fnguide_table('155660')

278
stockpredictor/crawler/toJsonFile/MetaCrawler.py Normal file
View File

@@ -0,0 +1,278 @@
import json
import datetime
import requests
import pandas as pd
import os
class MetaCrawler:
header = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36'}
limit_page_count = 10000
def __init__(self):
return
# 참고) http://blog.naver.com/PostView.nhn?blogId=koko8624&logNo=221288761509
def crawl_stocks(self, inFileName):
stocks = []
if os.path.isfile(inFileName):
inFp = open(inFileName, 'r', encoding='utf-8')
for line in inFp.readlines():
line = line.strip()
if line:
stocks.append(json.loads(line))
inFp.close()
outFp = open(inFileName, 'w', encoding='utf-8')
inputs = []
inputs.append( {'NAME':'USD', 'CODE':'FX_USDKRW', 'URL':'http://finance.naver.com/marketindex/exchangeDailyQuote.nhn?marketindexCd=FX_USDKRW'} ) # 미국 USD
inputs.append( {'NAME':'JPY', 'CODE':'FX_JPYKRW', 'URL':'http://finance.naver.com/marketindex/exchangeDailyQuote.nhn?marketindexCd=FX_JPYKRW'} ) # 일본 JPY
inputs.append( {'NAME':'EUR', 'CODE':'FX_EURKRW', 'URL':'http://finance.naver.com/marketindex/exchangeDailyQuote.nhn?marketindexCd=FX_EURKRW'} ) # 유럽연합 EUR'
inputs.append( {'NAME':'CNY', 'CODE':'FX_CNYKRW', 'URL':'http://finance.naver.com/marketindex/exchangeDailyQuote.nhn?marketindexCd=FX_CNYKRW'} ) # 중국 CNY
inputs.append( {'NAME':'WTI', 'CODE':'OIL_CL', 'URL':'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=OIL_CL&fdtc=2'} ) # WTI
inputs.append( {'NAME':'GOLD', 'CODE':'CMDT_GC', 'URL':'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=CMDT_GC&fdtc=2'} ) # 국제 금
for i in range(len(inputs)):
input = inputs[i]
if len(stocks) == 0:
meta = {}
meta["NAME"] = input['NAME']
meta["CODE"] = input['CODE']
meta["PRICE"] = []
lastDay = "1900.01.01"
else:
meta = stocks[i]
lastDay = meta['PRICE'][0]['DATE']
finish = False
for i in range(1, self.limit_page_count):
#html = pd.read_html(input['URL'] + '&page=%s' % i, header=0)
html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header).text)
# 마지막 페이지 까지 받기
if len(html[0].날짜.values) <= 1:
break
for j in range(0, len(html[0].values)):
item = html[0].values[j]
if input['NAME'] in ('USD', 'JPY', 'EUR', 'CNY'):
if j == 0:
continue
if item[0] <= lastDay:
finish = True
break
meta["PRICE"].append({
"DATE": item[0], # 날짜
"close": item[1], # 매매기준율
"diff": item[2] # 전일대비
})
elif input['NAME'] in ('WTI', 'GOLD'):
if item[0] <= lastDay:
finish = True
break
meta["PRICE"].append({
"DATE": item[0], # 날짜
"close": item[1], # 종가
"diff": item[2], # 전일대비
"rate": item[3] # 등락율
})
if finish:
break
meta["PRICE"] = sorted(meta["PRICE"], key=lambda x: x['DATE'], reverse=True)
outFp.write(json.dumps(meta, ensure_ascii=False) + "\n")
outFp.close()
return
# 투자자별 매매동향 (Trading_Trend) 크롤링
# (pri, 개인)
# (for, 외국인)
# (ins, 기관합)
# (ins0, 금융투자)
# (ins1, 보험)
# (ins2, 투신 (사모))
# (ins3, 은행)
# (ins4, 기타금융기관)
# (ins5, 연기금 등)
# (cor, 기타법인)
# 참고) http://blog.naver.com/PostView.nhn?blogId=koko8624&logNo=221289696771&parentCategoryNo=&categoryNo=&viewDate=&isShowPopularPosts=false&from=postView
def crawl_trading_trend(self, inFileName):
stocks = []
if os.path.isfile(inFileName):
inFp = open(inFileName, 'r', encoding='utf-8')
for line in inFp.readlines():
line = line.strip()
if line:
stocks.append(json.loads(line))
inFp.close()
outFp = open(inFileName, 'w', encoding='utf-8')
today = datetime.datetime.now().strftime("%Y%m%d")
url = 'http://finance.naver.com/sise/investorDealTrendDay.nhn?bizdate='+today+'&sosok=&page='
if len(stocks) == 0:
lastDay = "1900.01.01"
else:
lastDay = stocks[0]['DATE']
previousDay = ""
finish = False
for i in range(1, self.limit_page_count):
#html = pd.read_html(url + str(i), header=0)
html = pd.read_html(requests.get(url + str(i), headers=self.header).text)
# 마지막 페이지 까지 받기
if len(html[0].날짜.values) <= 2:
break
for j in range(1, len(html[0].values)):
item = html[0].values[j]
if str(item[0]) == "nan":
continue
if "20" + item[0] <= lastDay or item[0] == previousDay:
finish = True
break
meta = {
"DATE": "20"+item[0],
"pri": item[1], # 개인
"for": item[2], # 외국인
"ins": item[3], # 기관합
"ins0": item[4], # 금융투자
"ins1": item[5], # 보험
"ins2": item[6], # 투신 (사모)
"ins3": item[7], # 은행
"ins4": item[8], # 기타금융기관
"ins5": item[9], # 연기금 등
"cor": item[10]} # 기타법인
outFp.write(json.dumps(meta, ensure_ascii=False) + "\n")
print ("20"+item[0])
previousDay = html[0].values[2][0]
if finish:
break
if len(stocks) > 0:
for stock in stocks:
outFp.write(json.dumps(stock, ensure_ascii=False) + "\n")
outFp.close()
return
# 증시자금동향 (신용잔고, 펀드자금 잔고) 크롤링
# 참고) http://blog.naver.com/PostView.nhn?blogId=koko8624&logNo=221290138187&parentCategoryNo=&categoryNo=&viewDate=&isShowPopularPosts=false&from=postView
def crawl_money_trend(self, inFileName):
stocks = []
if os.path.isfile(inFileName):
inFp = open(inFileName, 'r', encoding='utf-8')
for line in inFp.readlines():
line = line.strip()
if line:
stocks.append(json.loads(line))
inFp.close()
outFp = open(inFileName, 'w', encoding='utf-8')
url = 'http://finance.naver.com/sise/sise_deposit.nhn?&page='
if len(stocks) == 0:
lastDay = "1900.01.01"
else:
lastDay = stocks[0]['DATE']
previousDay = ""
finish = False
for i in range(1, self.limit_page_count):
#html = pd.read_html(url + str(i), header=0, encoding='euc-kr')
html = pd.read_html(requests.get(url + str(i), headers=self.header).text, encoding='euc-kr')
# 마지막 페이지 까지 받기
if len(html[0].날짜.values) <= 10:
break
for j in range(1, len(html[0].values)):
item = html[0].values[j]
if str(item[0]) == "nan":
continue
if "20"+item[0] <= lastDay or item[0] == previousDay:
finish = True
break
meta = {
"DATE": "20"+item[0],
"dep1_1": item[1], # 고객예탁금 누적
"dep1_2": item[2], # 고객예탁금 당일
"dep2_1": item[3], # 신용잔고 누적
"dep2_2": item[4], # 신용잔고 당일
"dep3_1": item[5], # 주식형펀드 누적
"dep3_2": item[6], # 주식형펀드 당일
"dep4_1": item[7], # 혼합형펀드 누적
"dep4_2": item[8], # 혼합형펀드 당일
"dep5_1": item[9], # 채권형펀드 누적
"dep5_2": item[10]} # 채권형펀드 당일
outFp.write(json.dumps(meta, ensure_ascii=False) + "\n")
print("20"+item[0])
if finish:
break
previousDay = html[0].values[2][0]
if len(stocks) > 0:
for stock in stocks:
outFp.write(json.dumps(stock, ensure_ascii=False) + "\n")
outFp.close()
return
# 국내 시장금리 크롤링
# 참고) http://blog.naver.com/PostView.nhn?blogId=koko8624&logNo=221292348073&parentCategoryNo=&categoryNo=&viewDate=&isShowPopularPosts=false&from=postView
def crawl_interest_rates(self, inFileName):
stocks = []
if os.path.isfile(inFileName):
inFp = open(inFileName, 'r', encoding='utf-8')
for line in inFp.readlines():
line = line.strip()
if line:
stocks.append(json.loads(line))
inFp.close()
outFp = open(inFileName, 'w', encoding='utf-8')
inputs = []
inputs.append({'NAME': '91일 CD금리', 'CODE': 'IRR_CD91', 'URL': 'http://finance.naver.com/marketindex/interestDailyQuote.nhn?marketindexCd=IRR_CD91'})
inputs.append({'NAME': '콜금리', 'CODE': 'IRR_CALL', 'URL': 'http://finance.naver.com/marketindex/interestDailyQuote.nhn?marketindexCd=IRR_CALL'})
inputs.append({'NAME': '국고채(3년)', 'CODE': 'IRR_GOVT03Y', 'URL': 'http://finance.naver.com/marketindex/interestDailyQuote.nhn?marketindexCd=IRR_GOVT03Y'})
inputs.append({'NAME': '회사채(3년)', 'CODE': 'IRR_CORP03Y', 'URL': 'http://finance.naver.com/marketindex/interestDailyQuote.nhn?marketindexCd=IRR_CORP03Y'})
for i in range(len(inputs)):
input = inputs[i]
if len(stocks) == 0:
meta = {}
meta["NAME"] = input['NAME']
meta["CODE"] = input['CODE']
meta["PRICE"] = []
lastDay = "1900.01.01"
else:
meta = stocks[i]
lastDay = meta['PRICE'][0]['DATE']
finish = False
for i in range(1, self.limit_page_count):
#html = pd.read_html(input['URL'] + '&page=%s' % i, header=0)
html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header).text)
# 마지막 페이지 까지 받기
if len(html[0].날짜.values) <= 1:
break
for j in range(len(html[0].values)):
item = html[0].values[j]
if str(item[0]) == "nan":
continue
if item[0] <= lastDay:
finish = True
break
meta["PRICE"].append({
"DATE": item[0],
"close": item[1], # 종가
"diff": item[2], # 전일대비
"rate": item[3]}) # 등락율
if finish:
break
print(meta["NAME"] + " / " + item[0])
meta["PRICE"] = sorted(meta["PRICE"], key=lambda x: x['DATE'], reverse=True)
outFp.write(json.dumps(meta, ensure_ascii=False) + "\n")
outFp.close()
return

695
stockpredictor/crawler/toJsonFile/StockCrawler.py Normal file
View File

@@ -0,0 +1,695 @@
# https://bigdata-sk.tistory.com/10
import pandas as pd
import re
import json
import os
import requests
class Queue(object):
def __init__(self, max):
self.queue = []
self.max = max
def dequeue(self):
length = len(self.queue)
if length == 0 or length < self.max:
return -1
return self.queue.pop(0)
def enqueue(self, n):
length = len(self.queue)
if length == self.max:
self.dequeue()
self.queue.append(n)
pass
def sum(self):
sum = 0
for item in self.queue:
sum += item
return sum
def avg(self):
length = len(self.queue)
total = self.sum()
return round(total / length)
def print(self):
print(self.sum(), self.queue)
# 닐짜 형식으로 바뀐 this_date값을 확인 가능
# 읽어온 날짜 정보를 date형식으로 바꿀 일이 계속 생기므로 이 기능을 함수로 정의해줌.
# 함수명은 date_format()
class StockCrawler:
header = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36'}
historical_prices = None
special_pattern = None
fnGuideCrawler = None
limit_page_count = 40
def __init__(self):
self.historical_prices = dict()
self.special_pattern = (
'[', '!', '@', '#', '$', '%', '^', '&', '*', '(', ')', ',', '.', '?', '"', ':', ';', '{', '}', '|', '<', '>',
']', '+', '-', '/', '=', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9')
return
def clean_str(self, string):
string = re.sub(r"\\", " ", string)
string = re.sub(r"\'", " ", string)
string = re.sub(r"\"", " ", string)
string = re.sub(r"`", " ", string)
string = re.sub(r"-", " ", string)
string = re.sub(r"\(.*?\)", " ", string)
string = re.sub(r" ", " ", string)
return string.strip().lower()
def getStockInfo(self):
#code_df = pd.read_html('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', header=0)[0]
code_df = pd.read_html(requests.get('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', headers=self.header).text)[0]
# code_df = pd.read_excel('../resources/stock/상장법인목록.xls')
# 종목코드가 6자리이기 때문에 6자리를 맞춰주기 위해 설정해줌
code_df.종목코드 = code_df.종목코드.map('{:06d}'.format)
# 우리가 필요한 것은 회사명과 종목코드이기 때문에 필요없는 column들은 제외해준다.
code_df = code_df[['회사명', '종목코드']]
# 한글로된 컬럼명을 영어로 바꿔준다.
code_df = code_df.rename(columns={'회사명': 'name', '종목코드': 'code'})
###print (code_df.head())
return code_df
# 종목 이름을 입력하면 종목에 해당하는 코드를 불러와
# 네이버 금융(http://finance.naver.com)에 넣어줌
def get_url(self, item_name, code_df):
code = code_df.query("name=='{}'".format(item_name))['code'].to_string(index=False).strip()
url = 'http://finance.naver.com/item/sise_day.nhn?code={code}'.format(code=code.strip())
return code, url
def date_format(slef, d):
d = str(d).replace('-', '.')
#yyyy = int(d.split('.')[0])
#mm = int(d.split('.')[1])
#dd = int(d.split('.')[2])
#this_date = dt.date(yyyy, mm, dd)
return d
def getCodeIndex(self, stocks, item_code):
for i, stock in enumerate(stocks):
if item_code == stock['CODE']:
return i
return -1
def crawl_stocks(self, inFileName):
stocks = []
if os.path.isfile(inFileName):
inFp = open(inFileName, 'r', encoding='utf-8')
for line in inFp.readlines():
line = line.strip()
if line:
stocks.append(json.loads(line))
inFp.close()
if len(stocks)>0:
stock_cosdak_inverse = {"NAME": 'KODEX 코스닥150선물인버스', "CODE": "251340", "PRICE": stocks[0]["PRICE"]}
stock_cosdak_reverage = {"NAME": 'KODEX 코스닥150 레버리지', "CODE": "233740", "PRICE": stocks[1]["PRICE"]}
stock_inverse = {"NAME": 'KODEX 200선물인버스2X', "CODE": "252670", "PRICE": stocks[2]["PRICE"]}
stock_reverage = {"NAME": 'KODEX 레버리지', "CODE": "122630", "PRICE": stocks[3]["PRICE"]}
stock_gold = {"NAME": 'KODEX 골드선물(H)', "CODE": "132030", "PRICE": stocks[4]["PRICE"]}
else:
stock_cosdak_inverse = {"NAME": 'KODEX 코스닥150선물인버스', "CODE": "251340", "PRICE": []}
stock_cosdak_reverage = {"NAME": 'KODEX 코스닥150 레버리지', "CODE": "233740", "PRICE": []}
stock_inverse = {"NAME": 'KODEX 200선물인버스2X', "CODE": "252670", "PRICE": []}
stock_reverage = {"NAME": 'KODEX 레버리지', "CODE": "122630", "PRICE": []}
stock_gold = {"NAME": 'KODEX 골드선물(H)', "CODE": "132030", "PRICE": []}
outFp = open(inFileName, "w", encoding="utf-8")
kodex_cosdak_inverse = self.crawl_specific_stock('KODEX 코스닥150선물인버스', '251340', stock_cosdak_inverse)
outFp.write(json.dumps(kodex_cosdak_inverse, ensure_ascii=False) + "\n")
kodex_cosdak_reverage = self.crawl_specific_stock('KODEX 코스닥150 레버리지', '233740', stock_cosdak_reverage)
outFp.write(json.dumps(kodex_cosdak_reverage, ensure_ascii=False) + "\n")
kodex_inverse = self.crawl_specific_stock('KODEX 200선물인버스2X', '252670', stock_inverse)
outFp.write(json.dumps(kodex_inverse, ensure_ascii=False) + "\n")
kodex_reverage = self.crawl_specific_stock('KODEX 레버리지', '122630', stock_reverage)
outFp.write(json.dumps(kodex_reverage, ensure_ascii=False) + "\n")
kodex_gold = self.crawl_specific_stock('KODEX 골드선물(H)', '132030', stock_gold)
outFp.write(json.dumps(kodex_gold, ensure_ascii=False) + "\n")
code_df = self.getStockInfo()
items = code_df.values
idx = 0
for item in items:
idx += 1
item_name = item[0]
item_code = item[1]
print(idx, item_name, item_code)
if len(stocks) > 0:
index = self.getCodeIndex(stocks, item_code)
if index < 0:
stock = {"NAME": item_name, "CODE": item_code, "PRICE": []}
else:
stock = {"NAME": item_name, "CODE": item_code, "PRICE": stocks[index]["PRICE"]}
else:
stock = {"NAME": item_name, "CODE": item_code, "PRICE": []}
stock = self.crawl_specific_stock(item_name, item_code, stock)
outFp.write(json.dumps(stock, ensure_ascii=False) + "\n")
outFp.close()
return
def get_stocks_avg(self, inFileName, outFileName):
outFp = open(outFileName, 'w', encoding='utf-8')
inFp = open(inFileName, 'r', encoding='utf-8')
idx = 0
for line in inFp.readlines():
idx += 1
line = line.strip()
if line:
jsonData = json.loads(line)
q_3 = Queue(3)
q_5 = Queue(5)
q_7 = Queue(7)
q_10 = Queue(10)
q_20 = Queue(20)
q_30 = Queue(30)
q_60 = Queue(60)
q_90 = Queue(90)
q_100 = Queue(100)
q_120 = Queue(120)
q_150 = Queue(150)
q_180 = Queue(180)
q_200 = Queue(200)
q_240 = Queue(240)
for item in jsonData["PRICE"]:
q_3.enqueue(item['close'])
q_5.enqueue(item['close'])
q_7.enqueue(item['close'])
q_10.enqueue(item['close'])
q_20.enqueue(item['close'])
q_30.enqueue(item['close'])
q_60.enqueue(item['close'])
q_90.enqueue(item['close'])
q_100.enqueue(item['close'])
q_120.enqueue(item['close'])
q_150.enqueue(item['close'])
q_180.enqueue(item['close'])
q_200.enqueue(item['close'])
q_240.enqueue(item['close'])
item['avg3'] = q_3.avg()
item['avg5'] = q_5.avg()
item['avg7'] = q_7.avg()
item['avg10'] = q_10.avg()
item['avg20'] = q_20.avg()
item['avg30'] = q_30.avg()
item['avg60'] = q_60.avg()
item['avg90'] = q_90.avg()
item['avg100'] = q_100.avg()
item['avg120'] = q_120.avg()
item['avg150'] = q_150.avg()
item['avg180'] = q_180.avg()
item['avg200'] = q_200.avg()
item['avg240'] = q_240.avg()
outFp.write(json.dumps(jsonData, ensure_ascii=False) + "\n")
inFp.close()
outFp.close()
return
def crawl_specific_stock(self, code_name, code, stock):
item_name = code_name
item_code = code
url = 'http://finance.naver.com/item/sise_day.nhn?code={code}'.format(code=item_code.strip())
# 일자 데이터를 담을 df라는 DataFrame 정의
df = pd.DataFrame()
lastDay = ""
if len(stock) > 0 and len(stock["PRICE"])-1 > 0:
lastDay = stock["PRICE"][len(stock["PRICE"])-1]["DATE"].replace("-",".")
lastPage = False
# 1페이지에서 1000페이지의 데이터만 가져오기
for page in range(1, self.limit_page_count):
# 최근 상장 기업의 마지막 반복되는 페이지를 제외시킨다.
pg_url = '{url}&page={page}'.format(url=url, page=page)
#html = pd.read_html(pg_url, header=0)
html = pd.read_html(requests.get(pg_url, headers=self.header).text)
count = 0
for date in html[0].날짜.values:
if type(date) is str:
count += 1
if date == lastDay:
lastPage = True
df = df.append(html[0], ignore_index=True)
break
if count == 10:
df = df.append(html[0], ignore_index=True)
else:
if lastPage == False:
df = df.append(html[0], ignore_index=True)
lastPage = True
else:
break
# df.dropna()를 이용해 결측값 있는 행 제거
df = df.dropna()
# 상위 5개 데이터 확인하기
###print (df.head())
# 한글로 된 컬럼명을 영어로 바꿔줌
df = df.rename(columns={'날짜': 'date', '종가': 'close', '전일비': 'diff', '시가': 'open', '고가': 'high', '저가': 'low', '거래량': 'volume'})
# 데이터의 타입을 int형으로 바꿔줌
df[['close', 'diff', 'open', 'high', 'low', 'volume']] = df[['close', 'diff', 'open', 'high', 'low', 'volume']].astype(int)
# 컬럼명 'date'의 타입을 date로 바꿔줌
df['date'] = pd.to_datetime(df['date'])
# 일자(date)를 기준으로 오름차순 정렬
#df = df.sort_values(by=['date'], ascending=True)
# 상위 5개 데이터 확인
###print (df.head())
if len(stock) > 0 and len(stock["PRICE"]) - 1 > 0:
lastDay = stock["PRICE"][len(stock["PRICE"])-1]["DATE"]
for values in df.values:
day = str(values[0]).split(' ')[0]
if lastDay == day:
break
stock["PRICE"].append({
"DATE": day,
df.columns[1]: values[1],
df.columns[2]: values[2],
df.columns[3]: values[3],
df.columns[4]: values[4],
df.columns[5]: values[5],
df.columns[6]: values[6],
})
#stock["PRICE"] = sorted(stock["PRICE"], key=lambda x: x['DATE'], reverse=True)
stock["PRICE"] = sorted(stock["PRICE"], key=lambda x: x['DATE'])
return stock
def update_stocks(self, inFileName):
stock_inverse = {"NAME": 'KODEX 200선물인버스2X', "CODE": "252670", "PRICE": []}
stock_reverage = {"NAME": 'KODEX 레버리지', "CODE": "122630", "PRICE": []}
stock_gold = {"NAME": 'KODEX 골드선물(H)', "CODE": "132030", "PRICE": []}
stocks = []
if os.path.isfile(inFileName):
inFp = open(inFileName, 'r', encoding='utf-8')
for line in inFp.readlines():
line = line.strip()
if line:
jsonData = json.loads(line)
jsonData["PRICE"] = sorted(jsonData["PRICE"], key=lambda x: x['DATE'], reverse=True)
if jsonData['CODE'] == "252670":
stock_inverse = jsonData
elif jsonData['CODE'] == "122630":
stock_reverage = jsonData
elif jsonData['CODE'] == "132030":
stock_gold = jsonData
else:
stocks.append(jsonData)
inFp.close()
outFp = open(inFileName, 'w', encoding='utf-8')
if len(stocks) == 0:
limit_page_count = 1000
code_df = self.getStockInfo()
stocks = code_df.values
else:
limit_page_count = 2
code_df = None
idx = 0
for item in stocks:
idx += 1
if limit_page_count == 1000:
item_name = item[0]
item_code = item[1]
print(idx, item_name)
stock = {"NAME": item_name, "CODE": item_code, "PRICE": []}
code, url = self.get_url(item_name, code_df)
else:
item_name = item['NAME']
item_code = item['CODE']
print(idx, item_name)
stock = {"NAME": item_name, "CODE": item_code, "PRICE": []}
url = 'http://finance.naver.com/item/sise_day.nhn?code={code}'.format(code=item_code.strip())
# 일자 데이터를 담을 df라는 DataFrame 정의
df = pd.DataFrame()
lastPage = False
# 1페이지에서 1000페이지의 데이터만 가져오기
for page in range(1, limit_page_count):
# 최근 상장 기업의 마지막 반복되는 페이지를 제외시킨다.
pg_url = '{url}&page={page}'.format(url=url, page=page)
#html = pd.read_html(pg_url, header=0)
html = pd.read_html(requests.get(pg_url, headers=self.header).text)
count = 0
for date in html[0].날짜.values:
if type(date) is str:
count += 1
if count == 10:
df = df.append(html[0], ignore_index=True)
else:
if lastPage == False:
df = df.append(html[0], ignore_index=True)
lastPage = True
else:
break
# df.dropna()를 이용해 결측값 있는 행 제거
df = df.dropna()
# 상위 5개 데이터 확인하기
###print (df.head())
# 한글로 된 컬럼명을 영어로 바꿔줌
df = df.rename(columns={'날짜': 'date', '종가': 'close', '전일비': 'diff', '시가': 'open', '고가': 'high', '저가': 'low', '거래량': 'volume'})
# 데이터의 타입을 int형으로 바꿔줌
df[['close', 'diff', 'open', 'high', 'low', 'volume']] = df[['close', 'diff', 'open', 'high', 'low', 'volume']].astype(int)
# 컬럼명 'date'의 타입을 date로 바꿔줌
df['date'] = pd.to_datetime(df['date'])
# 일자(date)를 기준으로 오름차순 정렬
# df = df.sort_values(by=['date'], ascending=False)
# 상위 5개 데이터 확인
###print (df.head())
q_3 = Queue(3)
q_5 = Queue(5)
q_7 = Queue(7)
q_10 = Queue(10)
q_20 = Queue(20)
q_30 = Queue(30)
q_60 = Queue(60)
q_90 = Queue(90)
q_100 = Queue(100)
q_120 = Queue(120)
q_150 = Queue(150)
q_180 = Queue(180)
q_200 = Queue(200)
q_240 = Queue(240)
if limit_page_count == 1000:
for values in df.values:
q_3.enqueue(values[1])
q_5.enqueue(values[1])
q_7.enqueue(values[1])
q_10.enqueue(values[1])
q_20.enqueue(values[1])
q_30.enqueue(values[1])
q_60.enqueue(values[1])
q_90.enqueue(values[1])
q_100.enqueue(values[1])
q_120.enqueue(values[1])
q_150.enqueue(values[1])
q_180.enqueue(values[1])
q_200.enqueue(values[1])
q_240.enqueue(values[1])
stock["PRICE"].append({
"DATE": str(values[0]).split(' ')[0],
df.columns[1]: values[1],
df.columns[2]: values[2],
df.columns[3]: values[3],
df.columns[4]: values[4],
df.columns[5]: values[5],
df.columns[6]: values[6],
'avg3': q_3.avg(),
'avg5': q_5.avg(),
'avg7': q_7.avg(),
'avg10': q_10.avg(),
'avg20': q_20.avg(),
'avg30': q_30.avg(),
'avg60': q_60.avg(),
'avg90': q_90.avg(),
'avg100': q_100.avg(),
'avg120': q_120.avg(),
'avg150': q_150.avg(),
'avg180': q_180.avg(),
'avg200': q_200.avg(),
'avg240': q_240.avg()
})
else:
for values in item["PRICE"]:
q_3.enqueue(values["close"])
q_5.enqueue(values["close"])
q_7.enqueue(values["close"])
q_10.enqueue(values["close"])
q_20.enqueue(values["close"])
q_30.enqueue(values["close"])
q_60.enqueue(values["close"])
q_90.enqueue(values["close"])
q_100.enqueue(values["close"])
q_120.enqueue(values["close"])
q_150.enqueue(values["close"])
q_180.enqueue(values["close"])
q_200.enqueue(values["close"])
q_240.enqueue(values["close"])
# 기존 파일에서 읽은 것
stock["PRICE"].append({
"DATE": str(values["DATE"]).split(' ')[0],
df.columns[1]: values["close"],
df.columns[2]: values["diff"],
df.columns[3]: values["open"],
df.columns[4]: values["high"],
df.columns[5]: values["low"],
df.columns[6]: values["volume"],
'avg3': q_5.avg(),
'avg5': q_5.avg(),
'avg7': q_5.avg(),
'avg10': q_10.avg(),
'avg20': q_20.avg(),
'avg30': q_30.avg(),
'avg60': q_60.avg(),
'avg90': q_90.avg(),
'avg100': q_100.avg(),
'avg120': q_120.avg(),
'avg150': q_150.avg(),
'avg180': q_180.avg(),
'avg200': q_200.avg(),
'avg240': q_240.avg()
})
if limit_page_count != 1000:
# 새로 웹에서 수집한 것
for values in df.values:
date = str(values[0]).split(' ')[0]
isExist = False
for i in range(len(stock["PRICE"])):
if (stock["PRICE"][i]['DATE'] == date):
stock["PRICE"][i][df.columns[1]] = values[1]
stock["PRICE"][i][df.columns[2]] = values[2]
stock["PRICE"][i][df.columns[3]] = values[3]
stock["PRICE"][i][df.columns[4]] = values[4]
stock["PRICE"][i][df.columns[5]] = values[5]
stock["PRICE"][i][df.columns[6]] = values[6]
isExist = True
break
# 새로운 데이터나 오늘 날짜의 데이터
if not isExist:
q_3.enqueue(values[1])
q_5.enqueue(values[1])
q_7.enqueue(values[1])
q_10.enqueue(values[1])
q_20.enqueue(values[1])
q_30.enqueue(values[1])
q_60.enqueue(values[1])
q_90.enqueue(values[1])
q_100.enqueue(values[1])
q_120.enqueue(values[1])
q_150.enqueue(values[1])
q_180.enqueue(values[1])
q_200.enqueue(values[1])
q_240.enqueue(values[1])
stock["PRICE"].append({
"DATE": str(values[0]).split(' ')[0],
df.columns[1]: values[1],
df.columns[2]: values[2],
df.columns[3]: values[3],
df.columns[4]: values[4],
df.columns[5]: values[5],
df.columns[6]: values[6],
'avg3': q_3.avg(),
'avg5': q_5.avg(),
'avg7': q_7.avg(),
'avg10': q_10.avg(),
'avg20': q_20.avg(),
'avg30': q_30.avg(),
'avg60': q_60.avg(),
'avg90': q_90.avg(),
'avg100': q_100.avg(),
'avg120': q_120.avg(),
'avg150': q_150.avg(),
'avg180': q_180.avg(),
'avg200': q_200.avg(),
'avg240': q_240.avg()
})
stock["PRICE"] = sorted(stock["PRICE"], key=lambda x: x['DATE'], reverse=True)
outFp.write(json.dumps(stock, ensure_ascii=False)+"\n")
kodex_inverse = self.crawl_specific_stock('KODEX 200선물인버스2X', '252670', stock_inverse)
outFp.write(json.dumps(kodex_inverse, ensure_ascii=False) + "\n")
kodex_reverage = self.crawl_specific_stock('KODEX 레버리지', '122630', stock_reverage)
outFp.write(json.dumps(kodex_reverage, ensure_ascii=False) + "\n")
kodex_gold = self.crawl_specific_stock('KODEX 골드선물(H)', '132030', stock_gold)
outFp.write(json.dumps(kodex_gold, ensure_ascii=False) + "\n")
outFp.close()
return
def update_specific_stock(self, code_name, code, stock):
item_name = code_name
item_code = code
print(item_name)
if len(stock["PRICE"]) == 0:
limit_page_count = 1000
else:
limit_page_count = 2
url = 'http://finance.naver.com/item/sise_day.nhn?code={code}'.format(code=item_code.strip())
# 일자 데이터를 담을 df라는 DataFrame 정의
df = pd.DataFrame()
lastPage = False
# 1페이지에서 1000페이지의 데이터만 가져오기
for page in range(1, limit_page_count):
# 최근 상장 기업의 마지막 반복되는 페이지를 제외시킨다.
pg_url = '{url}&page={page}'.format(url=url, page=page)
#html = pd.read_html(pg_url, header=0)
html = pd.read_html(requests.get(pg_url, headers=self.header).text)
count = 0
for date in html[0].날짜.values:
if type(date) is str:
count += 1
if count == 10:
df = df.append(html[0], ignore_index=True)
else:
if lastPage == False:
df = df.append(html[0], ignore_index=True)
lastPage = True
else:
break
# df.dropna()를 이용해 결측값 있는 행 제거
df = df.dropna()
# 상위 5개 데이터 확인하기
###print (df.head())
# 한글로 된 컬럼명을 영어로 바꿔줌
df = df.rename(columns={'날짜': 'date', '종가': 'close', '전일비': 'diff', '시가': 'open', '고가': 'high', '저가': 'low', '거래량': 'volume'})
# 데이터의 타입을 int형으로 바꿔줌
df[['close', 'diff', 'open', 'high', 'low', 'volume']] = df[['close', 'diff', 'open', 'high', 'low', 'volume']].astype(int)
# 컬럼명 'date'의 타입을 date로 바꿔줌
df['date'] = pd.to_datetime(df['date'])
# 일자(date)를 기준으로 오름차순 정렬
#df = df.sort_values(by=['date'], ascending=True)
# 상위 5개 데이터 확인
###print (df.head())
q_3 = Queue(3)
q_5 = Queue(5)
q_7 = Queue(7)
q_10 = Queue(10)
q_20 = Queue(20)
q_30 = Queue(30)
q_60 = Queue(60)
q_90 = Queue(90)
q_100 = Queue(100)
q_120 = Queue(120)
q_150 = Queue(150)
q_180 = Queue(180)
q_200 = Queue(200)
q_240 = Queue(240)
for values in df.values:
q_3.enqueue(values[1])
q_5.enqueue(values[1])
q_7.enqueue(values[1])
q_10.enqueue(values[1])
q_20.enqueue(values[1])
q_30.enqueue(values[1])
q_60.enqueue(values[1])
q_90.enqueue(values[1])
q_100.enqueue(values[1])
q_120.enqueue(values[1])
q_150.enqueue(values[1])
q_180.enqueue(values[1])
q_200.enqueue(values[1])
q_240.enqueue(values[1])
stock["PRICE"].append({
"DATE": str(values[0]).split(' ')[0],
df.columns[1]: values[1],
df.columns[2]: values[2],
df.columns[3]: values[3],
df.columns[4]: values[4],
df.columns[5]: values[5],
df.columns[6]: values[6],
'avg3': q_3.avg(),
'avg5': q_5.avg(),
'avg7': q_7.avg(),
'avg10': q_10.avg(),
'avg20': q_20.avg(),
'avg30': q_30.avg(),
'avg60': q_60.avg(),
'avg90': q_90.avg(),
'avg100': q_100.avg(),
'avg120': q_120.avg(),
'avg150': q_150.avg(),
'avg180': q_180.avg(),
'avg200': q_200.avg(),
'avg240': q_240.avg()
})
stock["PRICE"] = sorted(stock["PRICE"], key=lambda x: x['DATE'], reverse=True)
return stock

63
stockpredictor/crawler/toSQLite/Crawler.py Normal file
View File

@@ -0,0 +1,63 @@
import os
import shutil
import datetime
from stockpredictor.crawler.toSQLite.MetaCrawler import MetaCrawler
from stockpredictor.crawler.toSQLite.StockCrawler import StockCrawler
from stockpredictor.analysis.Analyzer import Analyzer
today = datetime.datetime.now().strftime("%Y-%m-%d")
# DB Browser for SQLite: http://hleecaster.com/python-sqlite3/
PROJECT_HOME = "../../.."
inFnguideFileName = PROJECT_HOME + '/resources/fnguide.db'
"""
crawler = FnGuideCrawler()
print("[KOSPI 상장기업 재무제표 다운로드]")
crawler.crawl_fnguide(inFnguideFileName)
"""
crawler = MetaCrawler()
print("\n[환율 (USD, JPY, EUR, CNY), 원유 (WTI), 국제금]")
inFileName = PROJECT_HOME + '/resources/meta_1.db'
crawler.crawl_stocks(inFileName)
print("\n[투자자별 매매동향(Trading_Trend)]")
inFileName = PROJECT_HOME + '/resources/meta_2.db'
crawler.crawl_trading_trend(inFileName)
print("\n[증시자금동향 (신용잔고, 펀드자금 잔고)]")
inFileName = PROJECT_HOME + '/resources/meta_3.db'
crawler.crawl_money_trend(inFileName)
print("\n[국내 시장금리]")
inFileName = PROJECT_HOME + '/resources/meta_4.db'
crawler.crawl_interest_rates(inFileName)
print("\n[종목 다운로드]")
inFileName = PROJECT_HOME + '/resources/stock.db'
crawler = StockCrawler()
crawler.crawl_etf_stocks(inFileName)
crawler.crawl_stocks(inFileName)
print("\n[종목 분석]")
# S: 분석까지 진행
inFileName = PROJECT_HOME + '/resources/stock.db'
analyzer = Analyzer(PROJECT_HOME, inFileName, inFnguideFileName)
analyzer.analyze()
print("\n[종목 결정]")
day = datetime.datetime.today().strftime("%Y%m%d")
outPath = PROJECT_HOME + "/resources/analysis/" + day
if os.path.isdir(outPath):
shutil.rmtree(outPath)
os.mkdir(outPath)
print("print to Html...")
analyzer.analyzeToHtml(outPath)
# E: 분석까지 진행
print ("done...")

124
stockpredictor/crawler/toSQLite/FnGuideCrawler.py Normal file
View File

@@ -0,0 +1,124 @@
from bs4 import BeautifulSoup
from pandas import DataFrame, Series
import requests as re
import pandas as pd
import json
import sqlite3
import requests
class FnGuideCrawler:
header = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36'}
def getStockInfo(self):
#code_df = pd.read_html('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', header=0)[0]
code_df = pd.read_html(requests.get('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', headers=self.header).text)
# code_df = pd.read_excel('../resources/stock/상장법인목록.xls')
# 종목코드가 6자리이기 때문에 6자리를 맞춰주기 위해 설정해줌
code_df.종목코드 = code_df.종목코드.map('{:06d}'.format)
# 우리가 필요한 것은 회사명과 종목코드이기 때문에 필요없는 column들은 제외해준다.
code_df = code_df[['회사명', '종목코드']]
# 한글로된 컬럼명을 영어로 바꿔준다.
code_df = code_df.rename(columns={'회사명': 'name', '종목코드': 'code'})
###print (code_df.head())
return code_df
# FnGuide에서 크롤링한 KOSPI 상장기업의 재무제표
# http://blog.naver.com/PostView.nhn?blogId=koko8624&logNo=221294884955&parentCategoryNo=&categoryNo=&viewDate=&isShowPopularPosts=false&from=postView
def get_fnguide_table(self, code):
url = re.get('http://comp.fnguide.com/SVO2/ASP/SVD_main.asp?pGB=1&gicode=A%s'%(code.strip()))
url = url.content
html = BeautifulSoup(url,'html.parser')
body = html.find('body')
try:
fn_body = body.find('div', {'class': 'fng_body asp_body'})
ur_table = fn_body.find('div', {'id': 'div15'})
table = ur_table.find('div', {'id': 'highlight_D_Y'})
tbody = table.find('tbody')
tr = tbody.find_all('tr')
Table = DataFrame()
except:
return {}
for i in tr:
''' 자료 항목 가져오기'''
category = i.find('span', {'class': 'txt_acd'})
if category == None:
category = i.find('th')
category = category.text.strip()
'''값 가져오기'''
value_list = []
j = i.find_all('td', {'class': 'r'})
for value in j:
temp = value.text.replace(',', '').strip()
try:
temp = float(temp)
value_list.append(temp)
except:
value_list.append(0)
Table['%s' % (category)] = value_list
''' 기간 가져오기 '''
thead = table.find('thead')
tr_2 = thead.find('tr', {'class': 'td_gapcolor2'}).find_all('th')
year_list = []
for i in tr_2:
try:
temp_year = i.find('span', {'class': 'txt_acd'}).text
except:
temp_year = i.text
temp_year = temp_year.replace("/",".")+".01"
year_list.append(temp_year)
Table.index = year_list
return Table.T.to_dict()
def crawl_fnguide(self, inFileName):
tableName = 'fnguide'
conn = sqlite3.connect(inFileName, isolation_level=None)
cursor = conn.cursor()
cursor.execute("CREATE TABLE IF NOT EXISTS "+tableName+" (CODE text PRIMARY KEY, NAME text, PRICE text)")
code_df = self.getStockInfo()
idx = 0
for item in code_df.values:
item_name = item[0]
item_code = item[1]
idx += 1
print(idx, item_name)
fnGuideData = self.get_fnguide_table(item_code)
text = json.dumps(fnGuideData, ensure_ascii=False)
cursor.execute('SELECT * FROM '+tableName+' WHERE CODE=?', (item_code, ))
result = cursor.fetchone()
if result == None:
cursor.execute("INSERT INTO "+tableName+"(CODE, NAME, PRICE) VALUES(?, ?, ?)", (item_code, item_name, text))
else:
cursor.execute("UPDATE "+tableName+" SET PRICE=? WHERE CODE=?", (text, item_code))
cursor.close()
conn.close()
return
if __name__ == "__main__":
crawler = FnGuideCrawler()
crawler.get_fnguide_table('155660')

301
stockpredictor/crawler/toSQLite/MetaCrawler.py Normal file
View File

@@ -0,0 +1,301 @@
import json
import datetime
import requests
import sqlite3
import pandas as pd
import os
class MetaCrawler:
header = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36'}
limit_page_count = 10000
def __init__(self):
return
# 참고) http://blog.naver.com/PostView.nhn?blogId=koko8624&logNo=221288761509
def crawl_stocks(self, inFileName):
tableName = 'meta_1'
conn = sqlite3.connect(inFileName)
cursor = conn.cursor()
cursor.execute("CREATE TABLE IF NOT EXISTS "+tableName+" (CODE text PRIMARY KEY, NAME text, PRICE text)")
inputs = []
inputs.append( {'NAME':'USD', 'CODE':'FX_USDKRW', 'URL':'http://finance.naver.com/marketindex/exchangeDailyQuote.nhn?marketindexCd=FX_USDKRW'} ) # 미국 USD
inputs.append( {'NAME':'JPY', 'CODE':'FX_JPYKRW', 'URL':'http://finance.naver.com/marketindex/exchangeDailyQuote.nhn?marketindexCd=FX_JPYKRW'} ) # 일본 JPY
inputs.append( {'NAME':'EUR', 'CODE':'FX_EURKRW', 'URL':'http://finance.naver.com/marketindex/exchangeDailyQuote.nhn?marketindexCd=FX_EURKRW'} ) # 유럽연합 EUR'
inputs.append( {'NAME':'CNY', 'CODE':'FX_CNYKRW', 'URL':'http://finance.naver.com/marketindex/exchangeDailyQuote.nhn?marketindexCd=FX_CNYKRW'} ) # 중국 CNY
inputs.append( {'NAME':'WTI', 'CODE':'OIL_CL', 'URL':'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=OIL_CL&fdtc=2'} ) # WTI
inputs.append( {'NAME':'GOLD', 'CODE':'CMDT_GC', 'URL':'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=CMDT_GC&fdtc=2'} ) # 국제 금
for i in range(len(inputs)):
input = inputs[i]
meta = {}
meta["NAME"] = input['NAME']
meta["CODE"] = input['CODE']
meta["PRICE"] = []
cursor.execute('SELECT * FROM ' + tableName + ' WHERE CODE=?', (meta["CODE"],))
result = cursor.fetchone()
if result == None:
lastDay = "1900.01.01"
else:
meta["PRICE"] = json.loads(result[2])
lastDay = meta['PRICE'][0]['DATE']
finish = False
for i in range(1, self.limit_page_count):
#html = pd.read_html(input['URL'] + '&page=%s' % i, header=0)
html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header).text)
# 마지막 페이지 까지 받기
if len(html[0].날짜.values) <= 1:
break
for j in range(0, len(html[0].values)):
item = html[0].values[j]
if input['NAME'] in ('USD', 'JPY', 'EUR', 'CNY'):
if j == 0:
continue
if item[0] <= lastDay:
finish = True
break
meta["PRICE"].append({
"DATE": item[0], # 날짜
"close": item[1], # 매매기준율
"diff": item[2] # 전일대비
})
elif input['NAME'] in ('WTI', 'GOLD'):
if item[0] <= lastDay:
finish = True
break
meta["PRICE"].append({
"DATE": item[0], # 날짜
"close": item[1], # 종가
"diff": item[2], # 전일대비
"rate": item[3] # 등락율
})
if finish:
break
meta["PRICE"] = sorted(meta["PRICE"], key=lambda x: x['DATE'], reverse=True)
text = json.dumps(meta["PRICE"], ensure_ascii=False)
cursor.execute('SELECT * FROM '+tableName+' WHERE CODE=?', (meta["CODE"], ))
result = cursor.fetchone()
if result == None:
cursor.execute("INSERT INTO "+tableName+"(CODE, NAME, PRICE) VALUES(?, ?, ?)", (meta["CODE"], meta["NAME"], text))
else:
cursor.execute("UPDATE "+tableName+" SET PRICE=? WHERE CODE=?", (text, meta["CODE"]))
conn.commit()
cursor.close()
conn.close()
return
# 투자자별 매매동향 (Trading_Trend) 크롤링
# (pri, 개인)
# (for, 외국인)
# (ins, 기관합)
# (ins0, 금융투자)
# (ins1, 보험)
# (ins2, 투신 (사모))
# (ins3, 은행)
# (ins4, 기타금융기관)
# (ins5, 연기금 등)
# (cor, 기타법인)
# 참고) http://blog.naver.com/PostView.nhn?blogId=koko8624&logNo=221289696771&parentCategoryNo=&categoryNo=&viewDate=&isShowPopularPosts=false&from=postView
def crawl_trading_trend(self, inFileName):
tableName = 'meta_2'
conn = sqlite3.connect(inFileName)
cursor = conn.cursor()
cursor.execute("CREATE TABLE IF NOT EXISTS "+tableName+" (DATE text PRIMARY KEY, pri integer, for integer, ins integer, ins0 integer, ins1 integer, ins2 integer, ins3 integer, ins4 integer, ins5 integer, cor integer)")
cursor.execute('SELECT * FROM ' + tableName + ' order by DATE desc')
result = cursor.fetchone()
if result == None:
lastDay = "1900.01.01"
else:
lastDay = result[0]
today = datetime.datetime.now().strftime("%Y%m%d")
url = 'http://finance.naver.com/sise/investorDealTrendDay.nhn?bizdate='+today+'&sosok=&page='
previousDay = ""
finish = False
for i in range(1, self.limit_page_count):
#html = pd.read_html(url + str(i), header=0)
html = pd.read_html(requests.get(url + str(i), headers=self.header).text)
# 마지막 페이지 까지 받기
if len(html[0].날짜.values) <= 2:
break
for j in range(1, len(html[0].values)):
item = html[0].values[j]
if str(item[0]) == "nan":
continue
if "20" + item[0] <= lastDay or item[0] == previousDay:
finish = True
break
meta = {
"DATE": "20"+item[0],
"pri": item[1], # 개인
"for": item[2], # 외국인
"ins": item[3], # 기관합
"ins0": item[4], # 금융투자
"ins1": item[5], # 보험
"ins2": item[6], # 투신 (사모)
"ins3": item[7], # 은행
"ins4": item[8], # 기타금융기관
"ins5": item[9], # 연기금 등
"cor": item[10]} # 기타법인
cursor.execute('SELECT * FROM ' + tableName + ' WHERE DATE=?', (meta["DATE"],))
result = cursor.fetchone()
if result == None:
cursor.execute("INSERT INTO " + tableName + "(DATE, pri, for, ins, ins0, ins1, ins2, ins3, ins4, ins5, cor) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (meta["DATE"], meta["pri"], meta["for"], meta["ins"], meta["ins0"], meta["ins1"], meta["ins2"], meta["ins3"], meta["ins4"], meta["ins5"], meta["cor"]))
else:
cursor.execute("UPDATE " + tableName + " SET pri=?, for=?, ins=?, ins0=?, ins1=?, ins2=?, ins3=?, ins4=?, ins5=?, cor=? WHERE DATE=?", (meta["pri"], meta["for"], meta["ins"], meta["ins0"], meta["ins1"], meta["ins2"], meta["ins3"], meta["ins4"], meta["ins5"], meta["cor"], meta["DATE"]))
print ("20"+item[0])
previousDay = html[0].values[2][0]
if finish:
break
conn.commit()
cursor.close()
conn.close()
return
# 증시자금동향 (신용잔고, 펀드자금 잔고) 크롤링
# 참고) http://blog.naver.com/PostView.nhn?blogId=koko8624&logNo=221290138187&parentCategoryNo=&categoryNo=&viewDate=&isShowPopularPosts=false&from=postView
def crawl_money_trend(self, inFileName):
tableName = 'meta_3'
conn = sqlite3.connect(inFileName)
cursor = conn.cursor()
cursor.execute("CREATE TABLE IF NOT EXISTS "+tableName+" (DATE text PRIMARY KEY, dep1_1 integer, dep1_2 integer, dep2_1 integer, dep2_2 integer, dep3_1 integer, dep3_2 integer, dep4_1 integer, dep4_2 integer, dep5_1 integer, dep5_2 integer)")
cursor.execute('SELECT * FROM ' + tableName + ' order by DATE desc')
result = cursor.fetchone()
if result == None:
lastDay = "1900.01.01"
else:
lastDay = result[0]
previousDay = ""
url = 'http://finance.naver.com/sise/sise_deposit.nhn?&page='
finish = False
for i in range(1, self.limit_page_count):
#html = pd.read_html(url + str(i), header=0, encoding='euc-kr')
html = pd.read_html(requests.get(url + str(i), headers=self.header).text, encoding='euc-kr')
# 마지막 페이지 까지 받기
if len(html[0].날짜.values) <= 10:
break
for j in range(1, len(html[0].values)):
item = html[0].values[j]
if str(item[0]) == "nan":
continue
if "20"+item[0] <= lastDay or item[0] == previousDay:
finish = True
break
meta = {
"DATE": "20"+item[0],
"dep1_1": item[1], # 고객예탁금 누적
"dep1_2": item[2], # 고객예탁금 당일
"dep2_1": item[3], # 신용잔고 누적
"dep2_2": item[4], # 신용잔고 당일
"dep3_1": item[5], # 주식형펀드 누적
"dep3_2": item[6], # 주식형펀드 당일
"dep4_1": item[7], # 혼합형펀드 누적
"dep4_2": item[8], # 혼합형펀드 당일
"dep5_1": item[9], # 채권형펀드 누적
"dep5_2": item[10]} # 채권형펀드 당일
cursor.execute('SELECT * FROM ' + tableName + ' WHERE DATE=?', (meta["DATE"],))
result = cursor.fetchone()
if result == None:
cursor.execute("INSERT INTO " + tableName + "(DATE, dep1_1, dep1_2, dep2_1, dep2_2, dep3_1, dep3_2, dep4_1, dep4_2, dep5_1, dep5_2) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (meta["DATE"], meta["dep1_1"], meta["dep1_2"], meta["dep2_1"], meta["dep2_2"], meta["dep3_1"], meta["dep3_2"], meta["dep4_1"], meta["dep4_2"], meta["dep5_1"], meta["dep5_2"]))
else:
cursor.execute("UPDATE " + tableName + " SET dep1_1=?, dep1_2=?, dep2_1=?, dep2_2=?, dep3_1=?, dep3_2=?, dep4_1=?, dep4_2=?, dep5_1=?, dep5_2=? WHERE DATE=?", (meta["dep1_1"], meta["dep1_2"], meta["dep2_1"], meta["dep2_2"], meta["dep3_1"], meta["dep3_2"], meta["dep4_1"], meta["dep4_2"], meta["dep5_1"], meta["dep5_2"], meta["DATE"]))
print("20"+item[0])
if finish:
break
previousDay = html[0].values[2][0]
conn.commit()
cursor.close()
conn.close()
return
# 국내 시장금리 크롤링
# 참고) http://blog.naver.com/PostView.nhn?blogId=koko8624&logNo=221292348073&parentCategoryNo=&categoryNo=&viewDate=&isShowPopularPosts=false&from=postView
def crawl_interest_rates(self, inFileName):
tableName = 'meta_4'
conn = sqlite3.connect(inFileName)
cursor = conn.cursor()
cursor.execute("CREATE TABLE IF NOT EXISTS " + tableName + " (CODE text PRIMARY KEY, NAME text, PRICE text)")
inputs = []
inputs.append({'NAME': '91일 CD금리', 'CODE': 'IRR_CD91', 'URL': 'http://finance.naver.com/marketindex/interestDailyQuote.nhn?marketindexCd=IRR_CD91'})
inputs.append({'NAME': '콜금리', 'CODE': 'IRR_CALL', 'URL': 'http://finance.naver.com/marketindex/interestDailyQuote.nhn?marketindexCd=IRR_CALL'})
inputs.append({'NAME': '국고채(3년)', 'CODE': 'IRR_GOVT03Y', 'URL': 'http://finance.naver.com/marketindex/interestDailyQuote.nhn?marketindexCd=IRR_GOVT03Y'})
inputs.append({'NAME': '회사채(3년)', 'CODE': 'IRR_CORP03Y', 'URL': 'http://finance.naver.com/marketindex/interestDailyQuote.nhn?marketindexCd=IRR_CORP03Y'})
for i in range(len(inputs)):
input = inputs[i]
meta = {}
meta["NAME"] = input['NAME']
meta["CODE"] = input['CODE']
meta["PRICE"] = []
cursor.execute('SELECT * FROM ' + tableName + ' WHERE CODE=?', (meta["CODE"],))
result = cursor.fetchone()
if result == None:
lastDay = "1900.01.01"
else:
meta["PRICE"] = json.loads(result[2])
lastDay = meta['PRICE'][0]['DATE']
finish = False
for i in range(1, self.limit_page_count):
#html = pd.read_html(input['URL'] + '&page=%s' % i, header=0)
html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header).text)
# 마지막 페이지 까지 받기
if len(html[0].날짜.values) <= 1:
break
for j in range(len(html[0].values)):
item = html[0].values[j]
if str(item[0]) == "nan":
continue
if item[0] <= lastDay:
finish = True
break
meta["PRICE"].append({
"DATE": item[0],
"close": item[1], # 종가
"diff": item[2], # 전일대비
"rate": item[3]}) # 등락율
if finish:
break
print(meta["NAME"] + " / " + item[0])
meta["PRICE"] = sorted(meta["PRICE"], key=lambda x: x['DATE'], reverse=True)
text = json.dumps(meta["PRICE"], ensure_ascii=False)
cursor.execute('SELECT * FROM ' + tableName + ' WHERE CODE=?', (meta["CODE"],))
result = cursor.fetchone()
if result == None:
cursor.execute("INSERT INTO " + tableName + "(CODE, NAME, PRICE) VALUES(?, ?, ?)", (meta["CODE"], meta["NAME"], text))
else:
cursor.execute("UPDATE " + tableName + " SET PRICE=? WHERE CODE=?", (text, meta["CODE"]))
conn.commit()
cursor.close()
conn.close()
return

352
stockpredictor/crawler/toSQLite/StockCrawler.py Normal file
View File

@@ -0,0 +1,352 @@
# https://bigdata-sk.tistory.com/10
import pandas as pd
import re
import json
import sqlite3
import requests
class Queue(object):
def __init__(self, max):
self.queue = []
self.max = max
def dequeue(self):
length = len(self.queue)
if length == 0 or length < self.max:
return -1
return self.queue.pop(0)
def enqueue(self, n):
length = len(self.queue)
if length == self.max:
self.dequeue()
self.queue.append(n)
pass
def sum(self):
sum = 0
for item in self.queue:
sum += item
return sum
def avg(self):
length = len(self.queue)
total = self.sum()
return round(total / length)
def print(self):
print(self.sum(), self.queue)
# 닐짜 형식으로 바뀐 this_date값을 확인 가능
# 읽어온 날짜 정보를 date형식으로 바꿀 일이 계속 생기므로 이 기능을 함수로 정의해줌.
# 함수명은 date_format()
class StockCrawler:
header = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36'}
historical_prices = None
special_pattern = None
fnGuideCrawler = None
limit_page_count = 10000
def __init__(self):
self.historical_prices = dict()
self.special_pattern = (
'[', '!', '@', '#', '$', '%', '^', '&', '*', '(', ')', ',', '.', '?', '"', ':', ';', '{', '}', '|', '<', '>',
']', '+', '-', '/', '=', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9')
return
def clean_str(self, string):
string = re.sub(r"\\", " ", string)
string = re.sub(r"\'", " ", string)
string = re.sub(r"\"", " ", string)
string = re.sub(r"`", " ", string)
string = re.sub(r"-", " ", string)
string = re.sub(r"\(.*?\)", " ", string)
string = re.sub(r" ", " ", string)
return string.strip().lower()
def getStockInfo(self):
#code_df = pd.read_html('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', header=0)[0]
code_df = pd.read_html(requests.get('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', headers=self.header).text)[0]
# code_df = pd.read_excel('../resources/stock/상장법인목록.xls')
# 종목코드가 6자리이기 때문에 6자리를 맞춰주기 위해 설정해줌
code_df.종목코드 = code_df.종목코드.map('{:06d}'.format)
# 우리가 필요한 것은 회사명과 종목코드이기 때문에 필요없는 column들은 제외해준다.
code_df = code_df[['회사명', '종목코드']]
# 한글로된 컬럼명을 영어로 바꿔준다.
code_df = code_df.rename(columns={'회사명': 'name', '종목코드': 'code'})
###print (code_df.head())
return code_df
# 종목 이름을 입력하면 종목에 해당하는 코드를 불러와
# 네이버 금융(http://finance.naver.com)에 넣어줌
def get_url(self, item_name, code_df):
code = code_df.query("name=='{}'".format(item_name))['code'].to_string(index=False).strip()
url = 'http://finance.naver.com/item/sise_day.nhn?code={code}'.format(code=code.strip())
return code, url
def date_format(slef, d):
d = str(d).replace('-', '.')
#yyyy = int(d.split('.')[0])
#mm = int(d.split('.')[1])
#dd = int(d.split('.')[2])
#this_date = dt.date(yyyy, mm, dd)
return d
def getCodeIndex(self, stocks, item_code):
for i, stock in enumerate(stocks):
if item_code == stock['CODE']:
return i
return -1
def crawl_etf_stocks(self, inFileName):
tableName = 'stock'
conn = sqlite3.connect(inFileName)
cursor = conn.cursor()
cursor.execute("CREATE TABLE IF NOT EXISTS " + tableName + " (CODE text PRIMARY KEY, NAME text, PRICE text)")
stocks = []
stocks.append({"NAME": 'KODEX 코스닥150선물인버스', "CODE": "251340", "PRICE": []})
stocks.append({"NAME": 'KODEX 코스닥150 레버리지', "CODE": "233740", "PRICE": []})
stocks.append({"NAME": 'KODEX 200선물인버스2X', "CODE": "252670", "PRICE": []})
stocks.append({"NAME": 'KODEX 레버리지', "CODE": "122630", "PRICE": []})
stocks.append({"NAME": 'KODEX 골드선물(H)', "CODE": "132030", "PRICE": []})
for stock in stocks:
cursor.execute('SELECT * FROM ' + tableName + ' WHERE CODE=?', (stock["CODE"],))
result = cursor.fetchone()
if result != None:
stock["PRICE"] = json.loads(result[2])
self.crawl_specific_stock(stock)
text = json.dumps(stock['PRICE'], ensure_ascii=False)
cursor.execute('SELECT * FROM ' + tableName + ' WHERE CODE=?', (stock["CODE"],))
result = cursor.fetchone()
if result == None:
cursor.execute("INSERT INTO " + tableName + "(CODE, NAME, PRICE) VALUES(?, ?, ?)", (stock["CODE"], stock["NAME"], text))
else:
cursor.execute("UPDATE " + tableName + " SET PRICE=? WHERE CODE=?", (text, stock["CODE"]))
conn.commit()
cursor.close()
conn.close()
return
def crawl_stocks(self, inFileName):
tableName = 'stock'
conn = sqlite3.connect(inFileName)
cursor = conn.cursor()
cursor.execute("CREATE TABLE IF NOT EXISTS " + tableName + " (CODE text PRIMARY KEY, NAME text, PRICE text)")
code_df = self.getStockInfo()
items = code_df.values
idx = 0
for item in items:
idx += 1
item_name = item[0]
item_code = item[1]
print(idx, item_name, item_code)
cursor.execute('SELECT * FROM ' + tableName + ' WHERE CODE=?', (item_code,))
result = cursor.fetchone()
stock = {"CODE": item_code, "NAME": item_name, "PRICE": []}
if result != None:
stock["PRICE"] = json.loads(result[2])
self.crawl_specific_stock(stock)
text = json.dumps(stock['PRICE'], ensure_ascii=False)
cursor.execute('SELECT * FROM ' + tableName + ' WHERE CODE=?', (stock["CODE"],))
result = cursor.fetchone()
if result == None:
cursor.execute("INSERT INTO " + tableName + "(CODE, NAME, PRICE) VALUES(?, ?, ?)", (stock["CODE"], stock["NAME"], text))
else:
cursor.execute("UPDATE " + tableName + " SET PRICE=? WHERE CODE=?", (text, stock["CODE"]))
conn.commit()
cursor.close()
conn.close()
return
def get_data(self, stock):
url = 'http://finance.naver.com/item/sise_day.nhn?code={code}'.format(code=stock['CODE'].strip())
# 일자 데이터를 담을 df라는 DataFrame 정의
df = pd.DataFrame()
lastDay = ""
if len(stock) > 0 and len(stock["PRICE"]) - 1 > 0:
lastDay = stock["PRICE"][len(stock["PRICE"]) - 1]["DATE"].replace("-", ".")
lastPage = False
# 1페이지에서 1000페이지의 데이터만 가져오기
for page in range(1, self.limit_page_count):
# 최근 상장 기업의 마지막 반복되는 페이지를 제외시킨다.
pg_url = '{url}&page={page}'.format(url=url, page=page)
#html = pd.read_html(pg_url, header=0)
html = pd.read_html(requests.get(pg_url, headers=self.header).text)
count = 0
for date in html[0].날짜.values:
if type(date) is str:
count += 1
if date == lastDay:
lastPage = True
df = df.append(html[0], ignore_index=True)
break
if count == 10:
df = df.append(html[0], ignore_index=True)
else:
if lastPage == False:
df = df.append(html[0], ignore_index=True)
lastPage = True
else:
break
# df.dropna()를 이용해 결측값 있는 행 제거
df = df.dropna()
# 상위 5개 데이터 확인하기
###print (df.head())
# 한글로 된 컬럼명을 영어로 바꿔줌
df = df.rename(columns={'날짜': 'date', '종가': 'close', '전일비': 'diff', '시가': 'open', '고가': 'high', '저가': 'low',
'거래량': 'volume'})
# 데이터의 타입을 int형으로 바꿔줌
df[['close', 'diff', 'open', 'high', 'low', 'volume']] = df[
['close', 'diff', 'open', 'high', 'low', 'volume']].astype(int)
# 컬럼명 'date'의 타입을 date로 바꿔줌
df['date'] = pd.to_datetime(df['date'])
# 일자(date)를 기준으로 오름차순 정렬
# df = df.sort_values(by=['date'], ascending=True)
# 상위 5개 데이터 확인
###print (df.head())
if len(stock) > 0 and len(stock["PRICE"]) - 1 > 0:
lastDay = stock["PRICE"][len(stock["PRICE"]) - 1]["DATE"]
for values in df.values:
day = str(values[0]).split(' ')[0]
if lastDay == day:
break
stock["PRICE"].append({
"DATE": day,
df.columns[1]: values[1],
df.columns[2]: values[2],
df.columns[3]: values[3],
df.columns[4]: values[4],
df.columns[5]: values[5],
df.columns[6]: values[6],
})
# stock["PRICE"] = sorted(stock["PRICE"], key=lambda x: x['DATE'], reverse=True)
stock["PRICE"] = sorted(stock["PRICE"], key=lambda x: x['DATE'])
return
def get_moving_avg(self, stock):
q_3 = Queue(3)
q_5 = Queue(5)
q_7 = Queue(7)
q_10 = Queue(10)
q_20 = Queue(20)
q_30 = Queue(30)
q_60 = Queue(60)
q_90 = Queue(90)
q_100 = Queue(100)
q_120 = Queue(120)
q_150 = Queue(150)
q_180 = Queue(180)
q_200 = Queue(200)
q_240 = Queue(240)
for i in range(len(stock['PRICE'])):
q_3.enqueue(stock['PRICE'][i]['close'])
q_5.enqueue(stock['PRICE'][i]['close'])
q_7.enqueue(stock['PRICE'][i]['close'])
q_10.enqueue(stock['PRICE'][i]['close'])
q_20.enqueue(stock['PRICE'][i]['close'])
q_30.enqueue(stock['PRICE'][i]['close'])
q_60.enqueue(stock['PRICE'][i]['close'])
q_90.enqueue(stock['PRICE'][i]['close'])
q_100.enqueue(stock['PRICE'][i]['close'])
q_120.enqueue(stock['PRICE'][i]['close'])
q_150.enqueue(stock['PRICE'][i]['close'])
q_180.enqueue(stock['PRICE'][i]['close'])
q_200.enqueue(stock['PRICE'][i]['close'])
q_240.enqueue(stock['PRICE'][i]['close'])
stock['PRICE'][i]['avg3'] = q_3.avg()
stock['PRICE'][i]['avg5'] = q_5.avg()
stock['PRICE'][i]['avg7'] = q_7.avg()
stock['PRICE'][i]['avg10'] = q_10.avg()
stock['PRICE'][i]['avg20'] = q_20.avg()
stock['PRICE'][i]['avg30'] = q_30.avg()
stock['PRICE'][i]['avg60'] = q_60.avg()
stock['PRICE'][i]['avg90'] = q_90.avg()
stock['PRICE'][i]['avg100'] = q_100.avg()
stock['PRICE'][i]['avg120'] = q_120.avg()
stock['PRICE'][i]['avg150'] = q_150.avg()
stock['PRICE'][i]['avg180'] = q_180.avg()
stock['PRICE'][i]['avg200'] = q_200.avg()
stock['PRICE'][i]['avg240'] = q_240.avg()
return
def crawl_specific_stock(self, stock):
# 데이터 수집
self.get_data(stock)
# 이동 평균 계산
self.get_moving_avg(stock)
return
def update(self, inFileName, outFileName):
"""
Full json 데이터를 db에 import 시킴
inFileName = PROJECT_HOME + '/resources/stock.json.full'
outFileName = PROJECT_HOME + '/resources/stock.db'
crawler = StockCrawler()
crawler.update(inFileName, outFileName)
:param inFileName:
:param outFileName:
:return:
"""
tableName = 'stock'
conn = sqlite3.connect(outFileName, isolation_level=None)
cursor = conn.cursor()
cursor.execute("CREATE TABLE IF NOT EXISTS " + tableName + " (CODE text PRIMARY KEY, NAME text, PRICE text)")
idx = 0
inFp = open(inFileName, 'r')
for line in inFp.readlines():
if line:
idx += 1
stock = json.loads(line)
print(idx, stock["CODE"], stock["NAME"])
text = json.dumps(stock["PRICE"], ensure_ascii=False)
cursor.execute('SELECT * FROM ' + tableName + ' WHERE CODE=?', (stock["CODE"],))
result = cursor.fetchone()
if result == None:
cursor.execute("INSERT INTO " + tableName + "(CODE, NAME, PRICE) VALUES(?, ?, ?)", (stock["CODE"], stock["NAME"], text))
else:
cursor.execute("UPDATE " + tableName + " SET PRICE=? WHERE CODE=?", (text, stock["CODE"]))
return

157
stockpredictor/train/PricePredictor.py Normal file
View File

@@ -0,0 +1,157 @@
import os
import json
import sqlite3
import numpy as np
from keras.models import Sequential
from keras.layers import Dense
from keras.callbacks import EarlyStopping, ModelCheckpoint
from keras.layers import LSTM
class PricePredictor:
trainingRate = 0.7
TYPE = 0 # OPEN = 0, CLOSE = 1, HIGH = 2, LOW = 3, VOLUME = 4
MAX_VALUE = 500000
MAX_VOLUME_VALUE = 300000000
def getNormalPrice(self, value):
value = value / self.MAX_VALUE
if value>1:
value = 1
return value
def getNormalVolume(self, value):
value = value / self.MAX_VOLUME_VALUE
if value > 1:
value = 1
return value
def read (self, inFileName):
conn = sqlite3.connect(inFileName)
cursor = conn.cursor()
stocks = []
rowid = 1
cursor.execute('SELECT * FROM stock WHERE rowid=?', (rowid,))
result = cursor.fetchone()
while result != None:
prices = json.loads(result[2])
stock = []
for price in prices:
stock.append([self.getNormalPrice(price['open']), self.getNormalPrice(price['close']), self.getNormalPrice(price['high']), self.getNormalPrice(price['low']), self.getNormalVolume(price['volume'])])
stocks.append(stock)
rowid += 1
cursor.execute('SELECT * FROM stock WHERE rowid=?', (rowid,))
result = cursor.fetchone()
cursor.close()
conn.close()
return stocks
def make_dataset(self, stocks, window_size=5):
feature = []
label = []
test_feature = []
test_label = []
for i in range(len(stocks)):
stock = stocks[i]
for j in range(len(stock) - window_size):
feature.append(stock[j:j+window_size])
label.append([stock[j+window_size][self.TYPE]])
test_feature.append(stock[len(stock)-window_size-1:len(stock)-1])
test_label.append(stock[len(stock)-1][self.TYPE])
index = int(len(feature) * self.trainingRate)
train_feature = feature[:index]
train_label = label[:index]
valid_feature = feature[index:]
valid_label = label[index:]
test_feature = feature[len(feature) - 1]
test_label = label[len(feature) - 1]
return np.array(train_feature), np.array(train_label), np.array(valid_feature), np.array(valid_label), np.array(test_feature), np.array(test_label)
def train(self, inFileName, model_path):
stocks = self.read(inFileName)
train_feature, train_label, valid_feature, valid_label, test_feature, test_label = self.make_dataset(stocks)
print (train_feature.shape)
print (train_label.shape)
model = Sequential()
model.add(LSTM(32,
input_shape=(train_feature.shape[1], train_feature.shape[2]),
activation='relu',
return_sequences=False))
model.add(Dense(1))
model.compile(loss='mean_squared_error', optimizer='adam')
early_stop = EarlyStopping(monitor='val_loss', patience=5)
filename = ''
if self.TYPE==0 :
filename = os.path.join(model_path, 'open_checkpoint.h5')
elif self.TYPE==1 :
filename = os.path.join(model_path, 'close_checkpoint.h5')
elif self.TYPE==2 :
filename = os.path.join(model_path, 'high_checkpoint.h5')
elif self.TYPE==3 :
filename = os.path.join(model_path, 'low_checkpoint.h5')
checkpoint = ModelCheckpoint(filename, monitor='val_loss', verbose=1, save_best_only=True, mode='auto')
history = model.fit(train_feature, train_label,
epochs=1000,
batch_size=100000,
validation_data=(valid_feature, valid_label),
callbacks=[early_stop, checkpoint])
test_feature = np.array([test_feature])
predition = model.predict(test_feature)
print(predition[0], test_label)
print(predition[0] * self.MAX_VALUE, test_label * self.MAX_VALUE)
return
def predict(self, inFileName, model_path):
stocks = self.read(inFileName)
train_feature, train_label, valid_feature, valid_label, test_feature, test_label = self.make_dataset(stocks)
filename = ''
if self.TYPE==0 :
filename = os.path.join(model_path, 'open_checkpoint.h5')
elif self.TYPE==1 :
filename = os.path.join(model_path, 'close_checkpoint.h5')
elif self.TYPE==2 :
filename = os.path.join(model_path, 'high_checkpoint.h5')
elif self.TYPE==3 :
filename = os.path.join(model_path, 'low_checkpoint.h5')
model = Sequential()
model.add(LSTM(32,
input_shape=(train_feature.shape[1], train_feature.shape[2]),
activation='relu',
return_sequences=False))
model.add(Dense(1))
model.compile(loss='mean_squared_error', optimizer='adam')
model.load_weights(filename)
test_feature = np.array([test_feature])
predition = model.predict(test_feature)
print(predition[0], test_label)
print(predition[0] * self.MAX_VALUE, test_label * self.MAX_VALUE)
return
if __name__ == "__main__":
PROJECT_HOME = "../.."
inFileName = PROJECT_HOME + '/resources/stock.db'
model_path = PROJECT_HOME + '/resources/model'
pricePredictor = PricePredictor()
pricePredictor.train(inFileName, model_path)
#pricePredictor.predict(inFileName, model_path)