dsyoon/DeepStock
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

init

Browse Source
This commit is contained in:
dosangyoon
2022-07-29 14:54:45 +09:00
parent 932a3e5303
commit 17be8fd147
25 changed files with 0 additions and 7136 deletions
Download Patch File Download Diff File Expand all files Collapse all files

651
stockpredictor_back/analysis/Analyzer.py
View File

@@ -1,651 +0,0 @@
import json
import os
import time
import shutil
from stockpredictor.analysis.Common import Common
from stockpredictor.analysis.Stochastic import Stochastic
from stockpredictor.analysis.BolingerBand import BolingerBand
import matplotlib.pyplot as plt
import datetime
import sqlite3
from datetime import datetime
from matplotlib import rc
rc('font', family='AppleGothic')
plt.rcParams['axes.unicode_minus'] = False
import pandas as pd
import plotly.graph_objs as go
from plotly import tools, subplots
import plotly.io as po
class Analyzer:
tableName = 'stock'
PROJECT_HOME = None
stocks = None
candidate = None
stochastic = None
common = None
inFileName = None
fnguideFileName = None
fnguide = {}
def __init__(self, PROJECT_HOME, inFileName, fnguideFileName):
self.PROJECT_HOME = PROJECT_HOME
self.inFileName = inFileName
self.fnguideFileName = fnguideFileName
self.stocks = []
self.candidate = []
self.common = Common()
self.stochastic = Stochastic()
self.bolingerBand = BolingerBand()
self.readFnguide()
return
def readFnguide(self):
conn = sqlite3.connect(self.fnguideFileName)
cursor = conn.cursor()
today = datetime.today()
year1 = str(today.year - 1) + ".12.01"
year2 = str(today.year - 2) + ".12.01"
year3 = str(today.year - 3) + ".12.01"
rowid = 1
cursor.execute('SELECT * FROM fnguide WHERE rowid=?', (rowid,))
result = cursor.fetchone()
while result is not None:
if result[2] == "227950":
print (1)
data = json.loads(result[2])
self.fnguide[result[0]] = True
if (year1 in data and year2 in data and year3 in data):
if (data[year1]['영업이익'] < 0 or data[year2]['영업이익'] < 0 or data[year3]['영업이익'] < 0):
# and 3년 연속 영업이익이 적자이면 매수하지 않는다.
# or: 3년 중 1번이라도 영업이익이 적자이면 매수하지 않는다.
self.fnguide[result[0]] = False
if (data[year1]['영업이익'] < -100):
# 전년 영억적자가 100억 이상이면 매수하지 않는다.
self.fnguide[result[0]] = False
rowid += 1
cursor.execute('SELECT * FROM fnguide WHERE rowid=?', (rowid,))
result = cursor.fetchone()
cursor.close()
conn.close()
return
def draw(self, stock):
last_index = self.get_last_index(stock)
if last_index > 540:
index = 540 # 최대 540일치 그래프 확인
df_stock = pd.DataFrame(stock["PRICE"][len(stock["PRICE"]) - index:])
df_stochastic = pd.DataFrame(stock["STOCHASTIC"][len(stock["STOCHASTIC"]) - index:last_index+1])
else:
index = last_index
df_stock = pd.DataFrame(stock["PRICE"][:index+1])
df_stochastic = pd.DataFrame(stock["STOCHASTIC"][:index+1])
# general
candle_stick = go.Candlestick(x=df_stock.DATE, open=df_stock.open, high=df_stock.high, low=df_stock.low, close=df_stock.close, increasing_line_color='red', decreasing_line_color='blue')
avg5 = go.Scatter(x=df_stock.DATE, y=df_stock.avg5, name="avg5", line_color='#000000')
avg20 = go.Scatter(x=df_stock.DATE, y=df_stock.avg20, name="avg20", line_color='#f84c43')
avg60 = go.Scatter(x=df_stock.DATE, y=df_stock.avg60, name="avg60", line_color='#f89543')
avg120 = go.Scatter(x=df_stock.DATE, y=df_stock.avg120, name="avg120", line_color='#0ed604')
candle_data = [candle_stick, avg5, avg20, avg60, avg120]
volume = go.Bar(x=df_stock.DATE, y=df_stock['volume'], name="volume")
volume_data = [volume]
# stochastic
slow_k = go.Scatter(x=df_stochastic.DATE, y=df_stochastic['slow_k'], name="Slow%K", line_color='#8B4513')
slow_d = go.Scatter(x=df_stochastic.DATE, y=df_stochastic['slow_d'], name="Slow%D", line_color='#4169E1')
stochastic_data = [slow_k, slow_d]
fig = subplots.make_subplots(rows=3, cols=1, subplot_titles=('차트', '거래량', 'Stochastic'))
for trace in candle_data:
fig.append_trace(trace, 1, 1)
for trace in volume_data:
fig.append_trace(trace, 2, 1)
for trace in stochastic_data:
fig.append_trace(trace, 3, 1)
fig.update_layout(height=1200, xaxis_rangeslider_visible=False)
return fig
def get_last_index(self, stock):
for i in range(0, len(stock['PRICE'])):
if (stock['PRICE'][i]['close'] == 0 and stock['PRICE'][i]['open'] == 0 and stock['PRICE'][i]['volume'] == 0):
return i-1
return len(stock['PRICE']) - 1
def analyzeStochastic(self):
conn = sqlite3.connect(self.inFileName)
cursor = conn.cursor()
# 기존 분석 데이터를 모두 지움
cursor.execute('update ' + self.tableName + ' set STOCHASTIC = ""')
rowid = 1
cursor.execute('SELECT * FROM ' + self.tableName + ' WHERE rowid=?', (rowid,))
result = cursor.fetchone()
while result is not None:
stock = {"CODE": result[0], "NAME": result[1], "PRICE": json.loads(result[2])}
results = self.stochastic.analyze(stock)
text = json.dumps(results, ensure_ascii=False)
cursor.execute("UPDATE " + self.tableName + " SET STOCHASTIC=? WHERE CODE=?", (text, stock["CODE"]))
print("#analyzeStochastic", rowid, stock['NAME'])
rowid += 1
cursor.execute('SELECT * FROM ' + self.tableName + ' WHERE rowid=?', (rowid,))
result = cursor.fetchone()
conn.commit()
cursor.close()
conn.close()
return
def analyzeFinalScore(self, last_index, STOCK, STOCHASTIC):
"""
매수 조건
#0. 최소 매수 조건은 거래량은 20만건, 종가는 2천원 이상인 종목이어야 한다.
#1. 골든크로스: 5일선, 20일선, 60일선, 120일선이 순서대로 나열되는 순간
"""
i = last_index
buy_price = 0
count = 0
for idx in range(i, i-5, -1):
if idx-1 < 0:
break
buy_price += STOCK[idx-1]['close'] - STOCK[idx]['low']
count += 1
if count == 0:
buy_price = STOCK[i]['close']
else:
# 종가 - 최저가의 최근 3일 평균 가격을 산정한다.
buy_price = round(STOCK[i]['close'] - (buy_price/count))
status = ""
if STOCK[i]['volume'] > 100000 and STOCK[i]['close'] > 2000:
# 거래량이 100만 이상이고, 종가가 1천원 이상인지 체크 (https://happpy-rich.tistory.com/94)
# 정배열 체크
temp_status = self.common.check_RightArrange(STOCK, i)
if temp_status != "":
status += temp_status
# 20일선 돌파
temp_status = self.common.check_Dolpa_Jiji(STOCK, i, '20')
if temp_status != "":
status += temp_status
# 60일선 돌파
temp_status = self.common.check_Dolpa_Jiji(STOCK, i, '60')
if temp_status != "":
status += temp_status
# 120일선 돌파
temp_status = self.common.check_Dolpa_Jiji(STOCK, i, '120')
if temp_status != "":
status += temp_status
# 240일선 돌파
temp_status = self.common.check_Dolpa_Jiji(STOCK, i, '240')
if temp_status != "":
status += temp_status
# 20일선 지지 매수가 추천지
temp_status = self.common.check_Dolpa_Jiji_20(STOCK, i)
if temp_status != "":
status += temp_status
# 음봉인데 어제보다 종가가 더 높은 경우
# 이 경우 정배열 상태인지도 함께 체크를 한다.
higher_umbong_status = self.common.checkHigherUmbong(STOCK, i)
if higher_umbong_status != "":
status += higher_umbong_status
"""
# 단타 #1
temp_status = self.common.check_Danta1(STOCK, i)
if temp_status != "":
status += temp_status
# 단타 #2
temp_status = self.common.check_Danta2(STOCK, i)
if temp_status != "":
status += temp_status
all_upper_cross_status = self.common.checkAllUpperCross(STOCK, i)
if all_upper_cross_status != "":
status += all_upper_cross_status
# 1주일 동안 몇 10% 이상 오른 종목
W1Rise = self.common.check_W1Rise(STOCK, i, 0.1)
if W1Rise != "":
status += W1Rise
# 1일 동안 몇 10% 이상 내린 종목
W1Fall = self.common.check_D1Fall(STOCK, i, -0.1)
if W1Fall != "":
status += W1Fall
"""
# GOLDENCROSS#1은 바로 매수하지 않고, 이 시점 이후로 5일선이 20일선을 하방으로 뚫었다가 다시 20일선을 상방으로 뚫는 순간 매수를 시도한다.
# GOLDENCROSS#2은 바로 매수 가능
# GOLDENCROSS#3은 바로 매수 가능
golden_cross_status = self.common.check_golded_cross(STOCK, i)
if golden_cross_status != "":
status += golden_cross_status
"""
# BUYINGBEARMARKET#1은 바로 매수 가능
# BUYINGBEARMARKET#2은 바로 매수 가능
bearmarket_buying_status = self.common.check_bearmarket_buying(STOCK, STOCHASTIC, i)
if bearmarket_buying_status != "":
status += bearmarket_buying_status
"""
# STOCHASTIC
stochastic_status = self.common.check_stochastic(STOCK, STOCHASTIC, i)
if stochastic_status != "":
status += stochastic_status
# YANGBONG
"""
longYangBongAfterUmBong_status = self.common.checkLongYangBongAfterUmBong(STOCK, i)
# 어제 음봉 이후 장대양봉이었다면,
if longYangBongAfterUmBong_status != "":
status += longYangBongAfterUmBong_status
"""
# Doji
doji_status = self.common.checkDoji(STOCK, i)
# 하락 추세에서 도지가 나오면 매수
if doji_status != "":
status += doji_status
"""---------------------------------
# Gravestone
gravestone_status = self.common.checkGravestone(STOCK, i)
# 상승 추세에서 그레이브스톤이 나오면 매도
if gravestone_status != "":
status += gravestone_status
---------------------------------"""
"""
# Dragonfly
dragonfly_status = self.common.checkDragonfly(STOCK, i)
# 하락 추세에서 드레곤플라이가 나오면 매수
if dragonfly_status != "":
status += dragonfly_status
# Hammer
hammer_status = self.common.checkHammer(STOCK, i)
# 하락 추세에서 해머가 나오면 매수
if hammer_status != "":
status += hammer_status
"""
"""---------------------------------
# Hangingman
hangingman_status = self.common.checkHangingman(STOCK, i)
# 상승 추세에서 행잉맨이 나오면 매도
if hangingman_status != "":
status += hangingman_status
---------------------------------"""
"""
# 상승장악형 (Engulfing) - 다음 날도 양봉이라면 매수
engulfing_status = self.common.checkEngulfingHigh(STOCK, i)
# 하락 추세에서 상승장악형이 나오면 매수
if engulfing_status != "":
status += engulfing_status
"""
"""---------------------------------
# 하락장악형 (Engulfing)
engulfing_status = self.common.checkEngulfingLow(STOCK, i)
# 상승 추세에서 하락장악형이 나오면 매도
if engulfing_status != "":
status += engulfing_status
---------------------------------"""
"""
# 상승 포아형 (Harami)
harami_status = self.common.checkHaramiHigh(STOCK, i)
# 하락 추세에서 상승포아형이 나오면 매수
if harami_status != "":
status += harami_status
"""
"""---------------------------------
# 하락 포아형 (Harami)
harami_status = self.common.checkHaramiLow(STOCK, i)
# 상승 추세에서 하락포아형이 나오면 매도
if harami_status != "":
status += harami_status
---------------------------------"""
"""
# 관통형 (piercing)
piercing_status = self.common.checkPiercing(STOCK, i)
# 하락 추세에서 관통형이 나오면 매수
if piercing_status != "":
status += piercing_status
"""
"""---------------------------------
# 흑운형 (Dark-cloud)
darkcloud_status = self.common.checkDarkCloud(STOCK, i)
# 상승 추세에서 흑운형이 나오면 매도
if darkcloud_status != "":
status += darkcloud_status
---------------------------------"""
"""
# 샛별 (Morning start)
morningstar_status = self.common.checkMorningstar(STOCK, i)
# 하락 추세에서 샛별형이 나오면 매수
if morningstar_status != "":
status += morningstar_status
"""
"""---------------------------------
# 저녁별 (Evening start)
eveningstar_status = self.common.checkEveningstar(STOCK, i)
# 상승 추세에서 저녁별형이 나오면 매도
if eveningstar_status != "":
status += eveningstar_status
---------------------------------"""
return status, buy_price
def getPositionalEnergy(self, stock, last_index):
# 260 (= 52 * 5)일 중 가장 찾은 금액과 가장 높았던 금액 중 현재가의 위치 계산
close = stock[last_index]['close']
top = close
bottom = close
for i in range(1, 260):
if i > len(stock) or not stock[-i]:
break
if top < stock[-i]["close"]:
top = stock[-i]["close"]
if bottom > stock[-i]["close"]:
bottom = stock[-i]["close"]
if top-close == 0:
energy = 100.0
else:
energy = round((close-bottom) / (top-close), 2)
return energy
def writeFile(self, fig, state, buy_price, bolingerband_score, stochastic_score, positionalEnergy, item_name, item_code):
if state != "":
fileName = self.status_path
fileName = "%s/%s__p(%.2f)__b(%.2f)__s(%.2f)__%d__%s_%s.html" % (fileName, state, positionalEnergy, bolingerband_score, stochastic_score, buy_price, item_name.replace(" ", ""), item_code)
po.write_html(fig, file=fileName, auto_open=False)
if bolingerband_score < 0.15:
fileName = self.bolingerband_path
fileName = "%s/b(%.2f)__p(%.2f)__s(%.2f)__%s__%d__%s_%s.html" % (fileName, bolingerband_score, positionalEnergy, stochastic_score, state, buy_price, item_name.replace(" ", ""), item_code)
po.write_html(fig, file=fileName, auto_open=False)
if stochastic_score < 15:
fileName = self.stochastic_path
fileName = "%s/s(%.2f)__b(%.2f)__p(%.2f)__%s__%d__%s_%s.html" % (fileName, stochastic_score, bolingerband_score, positionalEnergy, state, buy_price, item_name.replace(" ", ""), item_code)
po.write_html(fig, file=fileName, auto_open=False)
if positionalEnergy < 0.5:
fileName = self.positionalEnergy_path
fileName = "%s/p(%.2f)__b(%.2f)__s(%.2f)__%s__%d__%s_%s.html" % (fileName, positionalEnergy, bolingerband_score, stochastic_score, state, buy_price, item_name.replace(" ", ""), item_code)
po.write_html(fig, file=fileName, auto_open=False)
return
def makeDirectory(self, outPath):
self.outPath = outPath
if os.path.isdir(outPath):
shutil.rmtree(outPath)
os.mkdir(outPath)
self.stochastic_path = outPath + "/stochastic"
if os.path.isdir(self.stochastic_path):
os.rmdir(self.stochastic_path)
os.mkdir(self.stochastic_path)
self.bolingerband_path = outPath + "/bolingerband"
if os.path.isdir(self.bolingerband_path):
os.rmdir(self.bolingerband_path)
os.mkdir(self.bolingerband_path)
self.positionalEnergy_path = outPath + "/positionalEnergy"
if os.path.isdir(self.positionalEnergy_path):
os.rmdir(self.positionalEnergy_path)
os.mkdir(self.positionalEnergy_path)
self.status_path = outPath + "/status"
if os.path.isdir(self.status_path):
os.rmdir(self.status_path)
os.mkdir(self.status_path)
return
# 그래프 출력
def analyzeToHtml(self, outPath):
self.makeDirectory(outPath)
conn = sqlite3.connect(self.inFileName)
cursor = conn.cursor()
rowid = 1
cursor.execute('SELECT CODE, NAME, PRICE, STOCHASTIC, BOLINGERBAND FROM ' + self.tableName + ' WHERE rowid=?', (rowid,))
result = cursor.fetchone()
# 최근 20일간 +-종목 개수 체크
inde_check = []
for check_index in range(365):
inde_check.append([0,0])
while result is not None:
item_code = result[0]
item_name = result[1]
print("#html", rowid, item_name)
# 부실 기업은 매수하지 않고 그냥 넘긴다.
# kospi 지수와 kosdak 지수도 그냥 넘긴다.
#if ((item_code in self.fnguide and not self.fnguide[item_code]) or (item_code == "KOSPI" or item_code == "KOSDAK") or result[3] == ''):
if ((item_code in self.fnguide and not self.fnguide[item_code]) or (item_code == "KOSPI" or item_code == "KOSDAK")):
rowid += 1
# 다음 종목을 가져옴
cursor.execute('SELECT CODE, NAME, PRICE, STOCHASTIC, BOLINGERBAND FROM ' + self.tableName + ' WHERE rowid=?', (rowid,))
result = cursor.fetchone()
continue
result_3 = result[3]
if result[3] != result[3]:
result_3 = result[3].replace("NaN", "0")
if result[3]==None or (result[2]=="[]" or result[3]=='' or result[4] == ''):
rowid += 1
cursor.execute('SELECT CODE, NAME, PRICE, STOCHASTIC, BOLINGERBAND FROM ' + self.tableName + ' WHERE rowid=?', (rowid,))
result = cursor.fetchone()
continue
stock = {"CODE": result[0], "NAME": result[1], "PRICE": json.loads(result[2]), "STOCHASTIC": json.loads(result_3), "BOLINGERBAND": json.loads(result[4])}
last_index = self.get_last_index(stock)
STOCK = stock['PRICE']
STOCHASTIC = stock['STOCHASTIC']
BOLINGERBAND = stock['BOLINGERBAND']
stochastic_score = STOCHASTIC[last_index]['slow_k']
# upper: 20, lower: 10
# 14 = (14-10) / (20-10) = 0.4
# 18 = (18-10) / (20-10) = 0.8
# 5 = (5-10) / (20-10) = -0.5
if BOLINGERBAND[last_index]['upper'] == BOLINGERBAND[last_index]['lower']:
bolingerband_score = 0
else:
bolingerband_score = round(((STOCK[last_index]['close']-BOLINGERBAND[last_index]['lower'])/(BOLINGERBAND[last_index]['upper']-BOLINGERBAND[last_index]['lower'])), 2)
# 위치 에너지
positionalEnergy = self.getPositionalEnergy(STOCK, last_index)
if STOCK[last_index]['volume'] > 100000 and STOCK[last_index]['close'] > 1000 and positionalEnergy < 0.3:
# 종목 상태 체크 분석
state, buy_price = self.analyzeFinalScore(last_index, STOCK, STOCHASTIC)
# 스토케스틱이 20이하이어야 하며, 볼린저밴드 0.3 보다 작으며, 위치에너지도 0.3보다 낮다면,
if stochastic_score < 20 and bolingerband_score < 0.3 and positionalEnergy < 0.3 and len(STOCK) > 5:
state = '1_' + state
# 종가가 240일선 아래라면 매수한다.
if STOCK[last_index]['close'] < STOCK[last_index]['avg240']:
state = '2_' + state
# 볼린저밴드 하단에 부딪혔다면,
if (STOCK[last_index - 2]['low'] <= BOLINGERBAND[last_index - 2]['lower'] <= STOCK[last_index - 2]['high'] or
STOCK[last_index - 3]['low'] <= BOLINGERBAND[last_index - 3]['lower'] <= STOCK[last_index - 3]['high']):
# 어제 양봉이거나
# 음봉이라면 그저깨 종가보다 어제 시가가 높거나 같고 그저깨 시가보다 어제 종가가 높다.
# 음봉이라면 그저깨 시가보다 어제 시가가 높거나 같고 그저깨 종가보다 어제 종가가 높다.
if STOCK[last_index-1]['open'] < STOCK[last_index-1]['close'] or (
(STOCK[last_index-2]['close'] <= STOCK[last_index-1]['open'] and STOCK[last_index-2]['open'] < STOCK[last_index-1]['close']) or
(STOCK[last_index-2]['open'] <= STOCK[last_index-1]['open'] and STOCK[last_index-2]['close'] < STOCK[last_index-1]['close'])):
# (KOSPI: 2011년 8월 11일)
# 오늘 양봉이어야 한다.
if STOCK[last_index]['open'] < STOCK[last_index]['close']:
state = '3_' + state
# (KOSPI: 2011년 9월 26일)
# 오늘 음봉이라면, 오늘 시가는 어제 종가보다 높아야 하고, 오늘 종가는 어제 시가보다 높아야 한다.
if (STOCK[last_index]['close'] < STOCK[last_index]['open']) and (STOCK[last_index-1]['close'] < STOCK[last_index]['open'] and STOCK[last_index-1]['open'] < STOCK[last_index]['close']):
state = '4_' + state
if (STOCHASTIC[last_index]['slow_k'] < 15 and
STOCHASTIC[last_index-1]['slow_k'] < STOCHASTIC[last_index-1]['slow_d'] and
STOCHASTIC[last_index]['slow_d'] < STOCHASTIC[last_index]['slow_k']):
state = '5_' + state
if (STOCHASTIC[last_index]['slow_k'] < 15 and
STOCHASTIC[last_index - 1]['slow_k'] < STOCHASTIC[last_index - 1]['slow_d'] and
STOCHASTIC[last_index]['slow_d'] < STOCHASTIC[last_index]['slow_k']):
state = '6_' + state
fig = self.draw(stock)
title = "%s (%s), %s, buy_price (%d), stochastic(%.2f), bolingerband(%.2f), positionalEnergy(%.2f) 차트" % (item_name, item_code, state, buy_price, stochastic_score, bolingerband_score, positionalEnergy)
fig['layout'].update(title=title)
self.writeFile(fig, state, buy_price, bolingerband_score, stochastic_score, positionalEnergy, item_name, item_code)
for check_index in range(365):
if len(STOCK) > check_index and STOCK[last_index-check_index]['open'] < STOCK[last_index-check_index]['close']:
inde_check[check_index][0] += 1
else:
inde_check[check_index][1] += 1
rowid += 1
cursor.execute('SELECT CODE, NAME, PRICE, STOCHASTIC, BOLINGERBAND FROM ' + self.tableName + ' WHERE rowid=?', (rowid,))
result = cursor.fetchone()
cursor.close()
conn.close()
return
def analyze(self):
conn = sqlite3.connect(self.inFileName)
cursor = conn.cursor()
# 기존 분석 데이터를 모두 지움
cursor.execute('update ' + self.tableName + ' set STOCHASTIC = "", BOLINGERBAND = ""')
rowid = 1
cursor.execute('SELECT * FROM ' + self.tableName + ' WHERE rowid=?', (rowid,))
result = cursor.fetchone()
while result is not None:
stock = {"CODE": result[0], "NAME": result[1], "PRICE": json.loads(result[2])}
try:
results_STOCHASTIC = self.stochastic.analyze(stock)
text_STOCHASTIC = json.dumps(results_STOCHASTIC, ensure_ascii=False)
results_BolingerBand = self.bolingerBand.analyze(stock)
text_BOLINGERBAND = json.dumps(results_BolingerBand, ensure_ascii=False)
except:
print("#", rowid, stock['NAME'])
rowid += 1
cursor.execute('SELECT * FROM ' + self.tableName + ' WHERE rowid=?', (rowid,))
result = cursor.fetchone()
continue
cursor.execute("UPDATE " + self.tableName + " SET STOCHASTIC=?, BOLINGERBAND=? WHERE CODE=?", (text_STOCHASTIC, text_BOLINGERBAND, stock["CODE"]))
print("#", rowid, stock['NAME'])
rowid += 1
cursor.execute('SELECT * FROM ' + self.tableName + ' WHERE rowid=?', (rowid,))
result = cursor.fetchone()
conn.commit()
cursor.close()
conn.close()
return
if __name__ == "__main__":
start = time.time()
PROJECT_HOME = os.path.join(os.path.dirname(os.path.join(os.path.dirname(os.path.join(os.path.dirname(__file__))))))
inFileName = PROJECT_HOME + '/resources/stock.db'
inFnguideFileName = PROJECT_HOME + '/resources/fnguide.db'
analyzer = Analyzer(PROJECT_HOME, inFileName, inFnguideFileName)
# 분석 & update DB
"""
#print ("analyze Stochastic...")
analyzer.analyzeStochastic()
"""
#######
# analyzer.analyze()
#######
day = datetime.today().strftime("%Y%m%d")
# HTML 출력
outPath = PROJECT_HOME + "/resources/analysis/"+day
if os.path.isdir(outPath):
shutil.rmtree(outPath)
os.mkdir(outPath)
print("print to Html...")
analyzer.analyzeToHtml(outPath)
# 파일 출력
#print("print to File...")
#outFileName = PROJECT_HOME + '/resources/analysis/'+day+'.json'
#analyzer.analyzeToFile(outFileName)
print("time : %6.2f 초" % (time.time() - start))
print("done...")

898
stockpredictor_back/analysis/AnalyzerSqlite.py
View File

@@ -1,898 +0,0 @@
import os
import time
import shutil
import matplotlib.pyplot as plt
import datetime
import sqlite3
from datetime import datetime
from matplotlib import rc
import pandas as pd
import copy
rc('font', family='AppleGothic')
plt.rcParams['axes.unicode_minus'] = False
import plotly.graph_objs as go
from plotly import tools, subplots
import plotly.io as po
from stockpredictor.analysis.Common import Common
from stockpredictor.analysis.Stochastic import Stochastic
from stockpredictor.analysis.BolingerBand import BolingerBand
from stockpredictor.analysis.IchimokuCloud import IchimokuCloud
from stockpredictor.analysis.RSI import RSI
from stockpredictor.crawler.sQLite.MovingAverage import MovingAverage
class AnalyzerSqlite:
PROJECT_HOME = None
stochastic = None
bolingerBand = None
ichimokuCloud = None
rsi = None
topCompany = None
fnguide = None
common = None
stockFileName = None
analyzedFileName = None
moving_avg = None
def __init__(self, PROJECT_HOME, stockFileName):
self.PROJECT_HOME = PROJECT_HOME
self.stockFileName = stockFileName
self.common = Common()
self.stochastic = Stochastic()
self.bolingerBand = BolingerBand()
self.ichimokuCloud = IchimokuCloud()
self.rsi = RSI()
self.topCompany = self.getTopCompany(stockFileName, 2000)
self.fnguide = self.readFnguide(stockFileName)
return
def getTopCompany(self, fnguideFileName, top):
conn = sqlite3.connect(fnguideFileName)
cursor = conn.cursor()
sql = "select DISTINCT CODE, NAME from fnguide order by total_ownership_interest desc limit " + str(top)
cursor.execute(sql)
result = cursor.fetchall()
top_company = {}
for idx, item in enumerate(result):
top_company[item[0]] = (idx+1, item[1])
cursor.close()
conn.close()
return top_company
def readFnguide(self, fnguideFileName):
conn = sqlite3.connect(fnguideFileName)
cursor = conn.cursor()
today = datetime.today()
year1 = str(today.year - 1) + ".12.01"
year2 = str(today.year - 2) + ".12.01"
year3 = str(today.year - 3) + ".12.01"
sql = "SELECT CODE, NAME, ymd, business_profits, business_profits_ratio, debt_ratio, ROA, ROE, EPS, BPS, DPS, PER, PBR FROM fnguide "
sql += " WHERE (ymd=? or ymd=? or ymd=?) and type=''"
sql += " order by code, ymd desc"
cursor.execute(sql, (year1,year2,year3))
result = cursor.fetchall()
fnguide = {}
for item in result:
if item[0] not in fnguide:
fnguide[item[0]] = []
fnguide[item[0]].append(
{'NAME': item[1],
'ymd': item[2],
'business_profits': item[3],
'business_profits_ratio': item[4],
'debt_ratio': item[5],
'ROA': item[6],
'ROE': item[7],
'EPS': item[8],
'BPS': item[9],
'DPS': item[10],
'PER': item[11],
'PBR': item[12]})
cursor.close()
conn.close()
return fnguide
def draw(self, stock):
ymd = list(reversed(stock['ymd']))
open = list(reversed(stock['open']))
close = list(reversed(stock['close']))
high = list(reversed(stock['high']))
low = list(reversed(stock['low']))
volume = list(reversed(stock['volume']))
avg4 = list(reversed(stock['avg4']))
avg5 = list(reversed(stock['avg5']))
avg6 = list(reversed(stock['avg6']))
avg10 = list(reversed(stock['avg10']))
avg12 = list(reversed(stock['avg12']))
avg20 = list(reversed(stock['avg20']))
avg36 = list(reversed(stock['avg36']))
avg40 = list(reversed(stock['avg40']))
avg48 = list(reversed(stock['avg48']))
avg60 = list(reversed(stock['avg60']))
avg120 = list(reversed(stock['avg120']))
avg240 = list(reversed(stock['avg240']))
avg300 = list(reversed(stock['avg300']))
rsi = list(reversed(stock['rsi']))
rsis = list(reversed(stock['rsis']))
stochastic_slow_k = list(reversed(stock['stochastic_slow_k']))
stochastic_slow_d = list(reversed(stock['stochastic_slow_d']))
bolingerband_upper = list(reversed(stock['bolingerband_upper']))
bolingerband_lower = list(reversed(stock['bolingerband_lower']))
ichimokucloud_changeLine = list(reversed(stock['ichimokucloud_changeLine']))
ichimokucloud_baseLine = list(reversed(stock['ichimokucloud_baseLine']))
# general
candle_stick = go.Candlestick(x=ymd, open=open, high=high, low=low, close=close, increasing_line_color='red', decreasing_line_color='blue')
avg4 = go.Scatter(x=ymd, y=avg4, name="avg4", line_color='#085F1B')
avg5 = go.Scatter(x=ymd, y=avg5, name="avg5", line_color='#000000')
avg6 = go.Scatter(x=ymd, y=avg6, name="avg6", line_color='#698D09')
avg10 = go.Scatter(x=ymd, y=avg10, name="avg10", line_color='#041366')
avg12 = go.Scatter(x=ymd, y=avg12, name="avg12", line_color='#6C2507')
avg20 = go.Scatter(x=ymd, y=avg20, name="avg20", line_color='#f84c43')
avg36 = go.Scatter(x=ymd, y=avg36, name="avg36", line_color='#370557')
avg40 = go.Scatter(x=ymd, y=avg40, name="avg40", line_color='#8013ED')
avg48 = go.Scatter(x=ymd, y=avg48, name="avg48", line_color='#7A1E66')
avg60 = go.Scatter(x=ymd, y=avg60, name="avg60", line_color='#f89543')
avg120 = go.Scatter(x=ymd, y=avg120, name="avg120", line_color='#0ed604')
avg240 = go.Scatter(x=ymd, y=avg240, name="avg240", line_color='#FF00F7')
avg300 = go.Scatter(x=ymd, y=avg300, name="avg300", line_color='#00FF49')
bolinger_upper = go.Scatter(x=ymd, y=bolingerband_upper, name="upper", line_color='#8B4513')
bolinger_lower = go.Scatter(x=ymd, y=bolingerband_lower, name="lower", line_color='#8B4513')
changeLine = go.Scatter(x=ymd, y=ichimokucloud_changeLine, name="changeLine", line_color='#000000')
baseLine = go.Scatter(x=ymd, y=ichimokucloud_baseLine, name="baseLine", line_color='#FF0000')
candle_data = [candle_stick, avg4, avg5, avg6, avg10, avg12, avg20, avg36, avg40, avg48, avg60, avg120, avg240, avg300, bolinger_upper, bolinger_lower, changeLine, baseLine]
#candle_data = [candle_stick, bolinger_upper, bolinger_lower, changeLine, baseLine]
volume = go.Bar(x=ymd, y=volume, name="volume")
volume_data = [volume]
# stochastic
rsi = go.Scatter(x=ymd, y=rsi, name="rsi", line_color='#8B4513')
rsis = go.Scatter(x=ymd, y=rsis, name="rsis", line_color='#4169E1')
rsi_data = [rsi, rsis]
# stochastic
slow_k = go.Scatter(x=ymd, y=stochastic_slow_k, name="Slow%K", line_color='#8B4513')
slow_d = go.Scatter(x=ymd, y=stochastic_slow_d, name="Slow%D", line_color='#4169E1')
stochastic_data = [slow_k, slow_d]
fig = subplots.make_subplots(rows=4, cols=1,
subplot_titles=('차트', '거래량', 'rsi', 'Stochastic'),
row_heights=[1200, 500, 500, 500])
for trace in candle_data:
fig.append_trace(trace, 1, 1)
for trace in volume_data:
fig.append_trace(trace, 2, 1)
for trace in rsi_data:
fig.append_trace(trace, 3, 1)
for trace in stochastic_data:
fig.append_trace(trace, 4, 1)
fig.update_layout(height=2200, xaxis_rangeslider_visible=False)
return fig
def getPositionalEnergy(self, close):
# 260 (= 52 * 5)일 중 가장 찾은 금액과 가장 높았던 금액 중 현재가의 위치 계산
top = close[0]
bottom = close[0]
for i in range(1, 260):
if i >= len(close):
break
if top < close[i]:
top = close[i]
if bottom > close[i]:
bottom = close[i]
if top-close[0] == 0:
energy1 = 100.0
else:
energy1 = round((close[0]-bottom) / (top-close[0]), 2)
energy2 = round((close[0] / top), 2)
return energy1, energy2
def makeDir(self, type):
if os.path.isdir(self.outPath + "/" + type):
os.rmdir(self.outPath + "/" + type)
os.mkdir(self.outPath + "/" + type)
return
def makeDirectory(self, outPath):
self.outPath = outPath
if os.path.isdir(outPath):
shutil.rmtree(outPath)
os.mkdir(outPath)
self.makeDir("final")
self.makeDir("weekly_4주선_48주선_상향돌파")
self.makeDir("weekly_종가_12주선_상향돌파")
self.makeDir("weekly_rsi_20이하")
self.makeDir("weekly_rsi_rsis_위로_올라옴")
self.makeDir("weekly_BB하단_내려옴")
self.makeDir("monthly_6월선_36월선_상향돌파")
self.makeDir("monthly_종가_12월선_상향돌파")
self.makeDir("monthly_rsi_30이하")
self.makeDir("monthly_rsi_rsis_위로_올라옴")
self.makeDir("monthly_BB하단_내려옴")
self.makeDir("daily_rsi_10이하")
self.makeDir("daily_거래량_7배_이상_상승")
self.makeDir("daily_BB하단_내려옴")
self.makeDir("daily_종가_60일선_상향돌파")
return
def writeFile(self, type, CODE, NAME, top, stock, state, final_status_count=-1):
# 3년 이내 한번이라도 영업이익이 났는지 체크를 함
fnguide = None
if CODE in self.fnguide:
fnguide = self.fnguide[CODE]
check = True
if fnguide:
check = False
for item in fnguide:
if item['business_profits'] > 0:
check = True
if check:
fig = self.draw(stock)
title = "%s (%s), %d, %s, %s 차트 (<a href=\"https://alphasquare.co.kr/home/stock/financial-information?code=%s\">URL1</a>, <a href=\"https://www.tradingview.com/chart/jJ8zOXz0/?symbol=KRX:%s\">URL2</a>)" % (NAME, CODE, stock['close'][0], state, type, CODE, CODE)
fig['layout'].update(title=title)
fileName = self.outPath + "/" + str(type)
if final_status_count == -1:
fileName = "%s/%s_%s_%s_%s.html" % (fileName, top, NAME.replace(" ", ""), CODE, state)
else:
fileName = "%s/%s_%s_%s_%s_%s.html" % (fileName, str(final_status_count), top, NAME.replace(" ", ""), CODE, state)
po.write_html(fig, file=fileName, auto_open=False)
return
def checkVolume(self, p_volume, volume):
if 0 < p_volume <= 10000 and p_volume * 700 < volume:
return True
if 10000 < p_volume <= 50000 and p_volume * 40 < volume:
return True
if 50000 < p_volume <= 100000 and p_volume * 25 < volume:
return True
if 100000 < p_volume <= 200000 and p_volume * 15 < volume:
return True
if 200000 < p_volume <= 700000 and p_volume * 13 < volume:
return True
if 700000 < p_volume <= 1000000 and p_volume * 10 < volume:
return True
if 5000000 < p_volume <= 5000000 and p_volume * 5 < volume:
return True
if 5000000 < p_volume and p_volume * 4 < volume:
return True
return False
def getStockData(self, TableName, CODE):
conn = sqlite3.connect(self.stockFileName)
cursor = conn.cursor()
sql = 'SELECT ymd, close, open, high, low, volume, '
sql += ' avg4, avg5, avg6, avg10, avg12, avg20, avg36, avg40, avg48, avg60, avg120, avg200, avg240, avg300, '
sql += ' bolingerband_upper, bolingerband_lower, bolingerband_middle, '
sql += ' ichimokucloud_changeLine, ichimokucloud_baseLine, ichimokucloud_leadingSpan1, ichimokucloud_leadingSpan2, '
sql += ' stochastic_fast_k, stochastic_slow_k, stochastic_slow_d, '
sql += ' rsi, rsis '
sql += ' FROM ' + TableName + ' where CODE=? order by ymd desc limit 512'
cursor.execute(sql, (CODE,))
prices = cursor.fetchall()
cursor.close()
conn.close()
ymd = []
close, open, high, low, volume = [], [], [], [], []
avg4, avg5, avg6, avg10, avg12, avg20, avg36, avg40, avg48, avg60, avg120, avg200, avg240, avg300 = [], [], [], [], [], [], [], [], [], [], [], [], [], []
bolingerband_upper, bolingerband_lower, bolingerband_middle = [], [], []
ichimokucloud_changeLine, ichimokucloud_baseLine, ichimokucloud_leadingSpan1, ichimokucloud_leadingSpan2 = [], [], [], []
stochastic_fast_k, stochastic_slow_k, stochastic_slow_d = [], [], []
rsi, rsis = [], []
for price in prices:
ymd.append(price[0])
close.append(price[1])
open.append(price[2])
high.append(price[3])
low.append(price[4])
volume.append(price[5])
avg4.append(price[6])
avg5.append(price[7])
avg6.append(price[8])
avg10.append(price[9])
avg12.append(price[10])
avg20.append(price[11])
avg36.append(price[12])
avg40.append(price[13])
avg48.append(price[14])
avg60.append(price[15])
avg120.append(price[16])
avg200.append(price[17])
avg240.append(price[18])
avg300.append(price[19])
bolingerband_upper.append(price[20])
bolingerband_lower.append(price[21])
bolingerband_middle.append(price[22])
ichimokucloud_changeLine.append(price[23])
ichimokucloud_baseLine.append(price[24])
ichimokucloud_leadingSpan1.append(price[25])
ichimokucloud_leadingSpan2.append(price[26])
stochastic_fast_k.append(price[27])
stochastic_slow_k.append(price[28])
stochastic_slow_d.append(price[29])
rsi.append(price[30])
rsis.append(price[31])
stock = {
"ymd": ymd,
"close": close, "open": open, "high": high, "low": low, "volume": volume,
"avg4": avg4, "avg5": avg5, "avg6": avg6, "avg10": avg10, "avg12": avg12, "avg20": avg20, "avg36": avg36, "avg40": avg40, "avg48": avg48, "avg60": avg60, "avg120": avg120, "avg200": avg200, "avg300": avg300,
"avg240": avg240,
"bolingerband_upper": bolingerband_upper, "bolingerband_lower": bolingerband_lower,
"bolingerband_middle": bolingerband_middle,
"ichimokucloud_changeLine": ichimokucloud_changeLine, "ichimokucloud_baseLine": ichimokucloud_baseLine,
"ichimokucloud_leadingSpan1": ichimokucloud_leadingSpan1,
"ichimokucloud_leadingSpan2": ichimokucloud_leadingSpan2,
"stochastic_fast_k": stochastic_fast_k, "stochastic_slow_k": stochastic_slow_k, "stochastic_slow_d": stochastic_slow_d,
"rsi": rsi, "rsis": rsis
}
return stock
# 후보 찾기
def findCandidate(self, outPath):
self.makeDirectory(outPath)
stockTableName = 'stock'
stockAnalysisTableName = 'stock_analysis'
stockAnalysisWeeklyTableName = 'stock_analysis_weekly'
stockAnalysisMonthlyTableName = 'stock_analysis_monthly'
conn = sqlite3.connect(self.stockFileName)
cursor = conn.cursor()
cursor.execute('SELECT distinct code, name FROM ' + stockTableName + ' order by code')
items = cursor.fetchall()
cursor.close()
conn.close()
for idx, item in enumerate(items):
CODE = item[0]
NAME = item[1]
print (idx, CODE, NAME)
print("Analysis # :", idx, ", CODE: ", CODE, ", NAME: ", NAME)
top = "0"
if CODE in self.topCompany:
top = str(self.topCompany[CODE][0])
stock_daily = self.getStockData(stockAnalysisTableName, CODE)
stock_weekly = self.getStockData(stockAnalysisWeeklyTableName, CODE)
stock_monthly = self.getStockData(stockAnalysisMonthlyTableName, CODE)
status = ""
final_status = ""
final_status_count = 0
# 거래량이 100만 이상이고, 종가가 1천원 이상인지 체크 (https://happpy-rich.tistory.com/94)
if stock_weekly['volume'][0] > 100000 and stock_weekly['close'][0] > 1000:
# 종목 상태 체크 분석
# [Weekly]
# 정배열 체크
temp_status = self.common.check_RightArrange(stock_weekly)
if temp_status != "":
status += temp_status
# 4주선이 48주 선을 상향 돌파함
if len(stock_weekly['close']) > 40:
if (stock_weekly['avg4'][1] is not None and stock_weekly['avg48'][1] is not None and
stock_weekly['avg4'][0] is not None and stock_weekly['avg48'][0] is not None):
if stock_weekly['avg4'][1] <= stock_weekly['avg48'][1] and stock_weekly['avg4'][0] > stock_weekly['avg48'][0]:
type = "weekly_4주선_48주선_상향돌파"
final_status += " " + type
final_status_count += 1
self.writeFile(type, CODE, NAME, top, stock_weekly, status)
# 종가가 20주선을 상향 돌파함
if len(stock_weekly['close']) > 2:
if stock_weekly['close'][1] is not None and stock_weekly['avg12'][1] is not None and stock_weekly['close'][0] is not None and stock_weekly['avg12'][0] is not None:
if stock_weekly['close'][1] <= stock_weekly['avg12'][1] and stock_weekly['close'][0] > stock_weekly['avg12'][0]:
type = "weekly_종가_12주선_상향돌파"
final_status += " " + type
final_status_count += 1
self.writeFile(type, CODE, NAME, top, stock_weekly, status)
# RSI가 32 이하인 경우 (30이 이하로 떨어지는 것 미리 확인)
if len(stock_monthly['close']) > 1:
if stock_weekly['rsi'][0] is not None:
if stock_weekly['rsi'][0] <= 20:
type = "weekly_rsi_20이하"
final_status += " " + type
final_status_count += 1
self.writeFile(type, CODE, NAME, top, stock_weekly, status)
# rsi가 40 이하이고, rsis 아래에서 위로 올라올 때
if len(stock_weekly['close']) > 60:
if stock_weekly['rsi'][0] is not None and stock_weekly['rsis'][0] is not None and stock_weekly['rsi'][1] is not None and stock_weekly['rsis'][1] is not None:
if stock_weekly['rsi'][0] < 40:
if stock_weekly['rsi'][0] > stock_weekly['rsis'][0] and stock_weekly['rsi'][1] <= stock_weekly['rsis'][1]:
type = "weekly_rsi_rsis_위로_올라옴"
#final_status += " " + type
#final_status_count += 1
self.writeFile(type, CODE, NAME, top, stock_weekly, status)
if len(stock_weekly['volume']) > 6:
# BB 하단에 부딪힘
for c_index in range(1, 5):
if stock_weekly['bolingerband_lower'][c_index+1] is not None and stock_weekly['close'][c_index] <= (stock_weekly['bolingerband_lower'][c_index+1]):
type = "weekly_BB하단_내려옴"
final_status += " " + type
final_status_count += 1
self.writeFile(type, CODE, NAME, top, stock_weekly, status)
break
# [Monthly]
# 20주선이 40주 선을 상향 돌파함
if len(stock_monthly['close']) > 40:
if (stock_monthly['avg6'][1] is not None and stock_monthly['avg36'][1] is not None and
stock_monthly['avg6'][0] is not None and stock_monthly['avg36'][0] is not None):
if stock_monthly['avg6'][1] <= stock_monthly['avg36'][1] and stock_monthly['avg6'][0] > stock_monthly['avg36'][0]:
type = "monthly_6월선_36월선_상향돌파"
final_status += " " + type
final_status_count += 1
self.writeFile(type, CODE, NAME, top, stock_weekly, status)
# 종가가 20주선을 상향 돌파함
if len(stock_monthly['close']) > 2:
if stock_monthly['close'][1] is not None and stock_monthly['avg12'][1] is not None and stock_monthly['close'][0] is not None and stock_monthly['avg12'][0] is not None:
if stock_monthly['close'][1] <= stock_monthly['avg12'][1] and stock_monthly['close'][0] > stock_monthly['avg12'][0]:
type = "monthly_종가_12월선_상향돌파"
final_status += " " + type
final_status_count += 1
self.writeFile(type, CODE, NAME, top, stock_weekly, status)
# RSI가 32 이하인 경우 (30이 이하로 떨어지는 것 미리 확인)
if len(stock_monthly['close']) > 1:
if stock_monthly['rsi'][0] is not None:
if stock_monthly['rsi'][0] <= 30:
type = "monthly_rsi_30이하"
final_status += " " + type
final_status_count += 1
self.writeFile(type, CODE, NAME, top, stock_weekly, status)
# rsi가 rsis 아래에서 위로 올라올 때
if len(stock_monthly['close']) > 60:
if stock_monthly['rsi'][0] is not None and stock_monthly['rsis'][0] is not None and stock_monthly['rsi'][1] is not None and stock_monthly['rsis'][1] is not None:
if stock_monthly['rsi'][0] < 40:
if stock_monthly['rsi'][0] > stock_monthly['rsis'][0] and stock_monthly['rsi'][1] <= stock_monthly['rsis'][1]:
type = "monthly_rsi_rsis_위로_올라옴"
#final_status += " " + type
#final_status_count += 1
self.writeFile(type, CODE, NAME, top, stock_monthly, status)
if len(stock_monthly['volume']) > 5:
# BB 하단에 부딪힘
for c_index in range(1, 5):
if stock_monthly['bolingerband_lower'][c_index+1] is None:
break
if stock_monthly['close'][c_index] <= (stock_monthly['bolingerband_lower'][c_index+1]):
type = "monthly_BB하단_내려옴"
final_status += " " + type
final_status_count += 1
self.writeFile(type, CODE, NAME, top, stock_monthly, status)
break
# 3) daily
if len(stock_daily['volume']) > 5:
# RSI가 10 이하인 경우 (10이 이하로 떨어지는 것 미리 확인)
if stock_daily['rsi'][0] is not None:
if stock_daily['rsi'][0] <= 10:
type = "daily_rsi_10이하"
final_status += " " + type
final_status_count += 1
self.writeFile(type, CODE, NAME, top, stock_weekly, status)
# 거래량이 7배 이상 상승한 경우
for c_index in range(1, 5):
if stock_daily['volume'][c_index] > (stock_daily['volume'][c_index+1]) * 5:
type = "daily_거래량_7배_이상_상승"
final_status += " " + type
final_status_count += 1
self.writeFile(type, CODE, NAME, top, stock_daily, status)
break
# BB 하단에 내려옴
for c_index in range(1, 5):
if stock_daily['bolingerband_lower'][c_index+1] is None:
break
if stock_daily['close'][c_index] <= (stock_daily['bolingerband_lower'][c_index + 1]):
type = "daily_BB하단_내려옴"
final_status += " " + type
final_status_count += 1
self.writeFile(type, CODE, NAME, top, stock_daily, status)
break
# 종가가 60일선을 상향 돌파함
if len(stock_daily['close']) > 60:
if stock_daily['close'][1] is not None and stock_daily['avg60'][1] is not None and stock_daily['close'][0] is not None and stock_daily['avg60'][0] is not None:
if stock_daily['close'][1] <= stock_daily['avg60'][1] and stock_daily['close'][0] > stock_daily['avg60'][0]:
type = "daily_종가_60일선_상향돌파"
final_status += " " + type
final_status_count += 1
self.writeFile(type, CODE, NAME, top, stock_daily, status)
if final_status_count > 0:
type = "final"
self.writeFile(type, CODE, NAME, top, stock_daily, final_status, final_status_count)
return
def get_moving_average(self, stock):
q_4 = MovingAverage(4)
q_5 = MovingAverage(5)
q_6 = MovingAverage(6)
q_10 = MovingAverage(10)
q_12 = MovingAverage(12)
q_20 = MovingAverage(20)
q_36 = MovingAverage(36)
q_40 = MovingAverage(40)
q_48 = MovingAverage(48)
q_60 = MovingAverage(60)
q_120 = MovingAverage(120)
q_200 = MovingAverage(200)
q_240 = MovingAverage(240)
q_300 = MovingAverage(300)
for i in range(len(stock)):
q_4.enqueue(stock[i]['close'])
q_5.enqueue(stock[i]['close'])
q_6.enqueue(stock[i]['close'])
q_10.enqueue(stock[i]['close'])
q_12.enqueue(stock[i]['close'])
q_20.enqueue(stock[i]['close'])
q_36.enqueue(stock[i]['close'])
q_40.enqueue(stock[i]['close'])
q_48.enqueue(stock[i]['close'])
q_60.enqueue(stock[i]['close'])
q_120.enqueue(stock[i]['close'])
q_200.enqueue(stock[i]['close'])
q_240.enqueue(stock[i]['close'])
q_300.enqueue(stock[i]['close'])
stock[i]['avg4'] = q_4.avg()
stock[i]['avg5'] = q_5.avg()
stock[i]['avg6'] = q_6.avg()
stock[i]['avg10'] = q_10.avg()
stock[i]['avg12'] = q_12.avg()
stock[i]['avg20'] = q_20.avg()
stock[i]['avg36'] = q_36.avg()
stock[i]['avg40'] = q_40.avg()
stock[i]['avg48'] = q_48.avg()
stock[i]['avg60'] = q_60.avg()
stock[i]['avg120'] = q_120.avg()
stock[i]['avg200'] = q_200.avg()
stock[i]['avg240'] = q_240.avg()
stock[i]['avg300'] = q_300.avg()
return
def convertFormat(self, weekDict):
previous_close = 0
stock_price = []
for ts in weekDict['open']:
stock_price.append(
{
"ymd": ts.strftime("%Y.%m.%d"),
"close": weekDict['close'][ts],
"diff": weekDict['close'][ts] - previous_close,
"open": weekDict['open'][ts],
"high": weekDict['high'][ts],
"low": weekDict['low'][ts],
"volume": weekDict['volume'][ts],
"avg4": -1,
"avg5": -1,
"avg6": -1,
"avg10": -1,
"avg12": -1,
"avg20": -1,
"avg36": -1,
"avg40": -1,
"avg48": -1,
"avg60": -1,
"avg120": -1,
"avg200": -1,
"avg240": -1,
"avg300": -1,
"bolingerband_upper": -1,
"bolingerband_lower": -1,
"bolingerband_middle": -1,
"ichimokucloud_changeLine": -1,
"ichimokucloud_baseLine": -1,
"ichimokucloud_leadingSpan1": -1,
"ichimokucloud_leadingSpan2": -1,
"stochastic_fast_k": -1,
"stochastic_slow_k": -1,
"stochastic_slow_d": -1,
"rsi": -1,
"rsis": -1
}
)
previous_close = weekDict['close'][ts]
return stock_price
def analyzeAdditionalInfo(self, stock, cursor, type=None):
if type==None:
stockAnalysisTableName = 'stock_analysis'
else:
stockAnalysisTableName = 'stock_analysis_' + type
# 테이블 생성
cursor.execute("CREATE TABLE IF NOT EXISTS " + stockAnalysisTableName + " (CODE text, NAME text, ymd text, close REAL, diff REAL, open REAL, high REAL, low REAL, volume REAL, avg4 REAL, avg5 REAL, avg6 REAL, avg10 REAL, avg12 REAL, avg20 REAL, avg36 REAL, avg40 REAL, avg48 REAL, avg60 REAL, avg120 REAL, avg200 REAL, avg240 REAL, avg300 REAL, bolingerband_upper REAL, bolingerband_lower REAL, bolingerband_middle REAL, ichimokucloud_changeLine REAL, ichimokucloud_baseLine REAL, ichimokucloud_leadingSpan1 REAL, ichimokucloud_leadingSpan2 REAL, stochastic_fast_k REAL, stochastic_slow_k REAL, stochastic_slow_d REAL, rsi REAL, rsis REAL)")
# 키 생성
create_key = "CREATE INDEX IF NOT EXISTS " + stockAnalysisTableName + "_idx on " + stockAnalysisTableName + " (CODE, ymd) "
cursor.execute(create_key)
# 이동 평균 계산
stock["PRICE"] = sorted(stock["PRICE"], key=lambda x: x['ymd'])
self.get_moving_average(stock["PRICE"])
self.ichimokuCloud.analyze(stock)
self.stochastic.analyze(stock)
self.bolingerBand.analyze(stock)
self.rsi.analyze(stock)
sorted_stock = sorted(stock["PRICE"], key=lambda x: x['ymd'], reverse=True)
for price in sorted_stock:
cursor.execute('SELECT * FROM ' + stockAnalysisTableName + ' WHERE CODE=? and ymd=?', (stock['CODE'], price['ymd'],))
result = cursor.fetchone()
if result == None:
sql = "INSERT INTO " + stockAnalysisTableName + "(CODE, NAME, ymd, close, diff, open, high, low, volume, "
sql += " avg4, avg5, avg6, avg10, avg12, avg20, avg36, avg40, avg48, avg60, avg120, avg200, avg240, avg300, "
sql += " bolingerband_upper, bolingerband_lower, bolingerband_middle, "
sql += " ichimokucloud_changeLine, ichimokucloud_baseLine, ichimokucloud_leadingSpan1, ichimokucloud_leadingSpan2, "
sql += " stochastic_fast_k, stochastic_slow_k, stochastic_slow_d, "
sql += " rsi, rsis) "
sql += " VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
cursor.execute(sql, (
stock["CODE"], stock["NAME"], price['ymd'], price['close'], price['diff'], price['open'], price['high'], price['low'], price['volume'],
price['avg4'], price['avg5'], price['avg6'], price['avg10'], price['avg12'], price['avg20'], price['avg36'], price['avg40'], price['avg48'], price['avg60'], price['avg120'], price['avg200'], price['avg240'], price['avg300'],
price['bolingerband_upper'], price['bolingerband_lower'], price['bolingerband_middle'],
price['ichimokucloud_changeLine'], price['ichimokucloud_baseLine'], price['ichimokucloud_leadingSpan1'],
price['ichimokucloud_leadingSpan2'],
price['stochastic_fast_k'], price['stochastic_slow_k'], price['stochastic_slow_d'],
price['rsi'], price['rsis'],))
else:
sql = "UPDATE " + stockAnalysisTableName + " SET close=?, diff=?, open=?, high=?, low=?, volume=?, "
sql += " avg4=?, avg5=?, avg6=?, avg10=?, avg12=?, avg20=?, avg36=?, avg40=?, avg48=?, avg60=?, avg120=?, avg200=?, avg240=?, avg300=?, "
sql += " bolingerband_upper=?, bolingerband_lower=?, bolingerband_middle=?, "
sql += " ichimokucloud_changeLine=?, ichimokucloud_baseLine=?, ichimokucloud_leadingSpan1=?, ichimokucloud_leadingSpan2=?, "
sql += " stochastic_fast_k=?, stochastic_slow_k=?, stochastic_slow_d=?, "
sql += " rsi=?, rsis=? "
sql += " WHERE CODE=? and ymd=?"
cursor.execute(sql,
(price['close'], price['diff'], price['open'], price['high'], price['low'], price['volume'],
price['avg4'], price['avg5'], price['avg6'], price['avg10'], price['avg12'], price['avg20'], price['avg36'], price['avg40'], price['avg48'], price['avg60'], price['avg120'], price['avg200'], price['avg240'], price['avg300'],
price['bolingerband_upper'], price['bolingerband_lower'], price['bolingerband_middle'], price['ichimokucloud_changeLine'], price['ichimokucloud_baseLine'],
price['ichimokucloud_leadingSpan1'], price['ichimokucloud_leadingSpan2'],
price['stochastic_fast_k'], price['stochastic_slow_k'], price['stochastic_slow_d'],
price['rsi'], price['rsis'],
stock["CODE"], price['ymd'],))
break
return
def analyzeDaily(self):
stockTableName = 'stock'
conn = sqlite3.connect(self.stockFileName)
cursor = conn.cursor()
cursor.execute('SELECT distinct code, name FROM ' + stockTableName + ' order by code')
items = cursor.fetchall()
for rowid, item in enumerate(items):
stock = {"CODE": item[0], "NAME": item[1], "PRICE":[]}
print("Daily # :", rowid, ", CODE: ", stock['CODE'], ", NAME: ", stock['NAME'])
sql = 'SELECT ymd, close, diff, open, high, low, volume FROM ' + stockTableName + ' where CODE=? order by ymd desc '
#if result is not None:
# sql += ' limit 300'
cursor.execute(sql, (stock['CODE'],))
items = cursor.fetchall()
items_reverse = reversed(items)
for item in items_reverse:
stock['PRICE'].append(
{"ymd": item[0],
"close": item[1],
"diff": item[2],
"open": item[3],
"high": item[4],
"low": item[5],
"volume": item[6],
"avg4": -1,
"avg5": -1,
"avg6": -1,
"avg10": -1,
"avg12": -1,
"avg20": -1,
"avg36": -1,
"avg40": -1,
"avg48": -1,
"avg60": -1,
"avg120": -1,
"avg200": -1,
"avg240": -1,
"avg300": -1,
"bolingerband_upper": -1,
"bolingerband_lower": -1,
"bolingerband_middle": -1,
"ichimokucloud_changeLine": -1,
"ichimokucloud_baseLine": -1,
"ichimokucloud_leadingSpan1": -1,
"ichimokucloud_leadingSpan2": -1,
"stochastic_fast_k": -1,
"stochastic_slow_k": -1,
"stochastic_slow_d": -1,
"rsi": -1,
"rsis": -1})
self.analyzeAdditionalInfo(stock, cursor)
conn.commit()
cursor.close()
conn.close()
return
def analyzeGrouping(self, type):
stockTableName = 'stock'
conn = sqlite3.connect(self.stockFileName)
cursor = conn.cursor()
cursor.execute('SELECT distinct code, name FROM ' + stockTableName + ' order by code')
items = cursor.fetchall()
for rowid, item in enumerate(items):
stock = {"CODE": item[0], "NAME": item[1], "PRICE": []}
print(type, "# :", rowid, ", CODE: ", stock['CODE'], ", NAME: ", stock['NAME'])
sql = 'SELECT ymd, close, diff, open, high, low, volume FROM ' + stockTableName + ' where CODE=? order by ymd desc '
cursor.execute(sql, (stock['CODE'],))
items = cursor.fetchall()
items_reverse = reversed(items)
for item in items_reverse:
stock['PRICE'].append(
{"ymd": item[0],
"close": item[1],
"diff": item[2],
"open": item[3],
"high": item[4],
"low": item[5],
"volume": item[6],
"avg4": -1,
"avg5": -1,
"avg6": -1,
"avg10": -1,
"avg12": -1,
"avg20": -1,
"avg36": -1,
"avg40": -1,
"avg48": -1,
"avg60": -1,
"avg120": -1,
"avg200": -1,
"avg240": -1,
"avg300": -1,
"bolingerband_upper": -1,
"bolingerband_lower": -1,
"bolingerband_middle": -1,
"ichimokucloud_changeLine": -1,
"ichimokucloud_baseLine": -1,
"ichimokucloud_leadingSpan1": -1,
"ichimokucloud_leadingSpan2": -1,
"stochastic_fast_k": -1,
"stochastic_slow_k": -1,
"stochastic_slow_d": -1,
"rsi": -1,
"rsis": -1})
agg_dict = {'open': 'first',
'high': 'max',
'low': 'min',
'close': 'last',
'volume': 'sum'}
df = pd.DataFrame(stock['PRICE'])
df['ymd'] = pd.to_datetime(df['ymd'])
df.set_index('ymd', inplace=True)
if type == "weekly":
condition="W"
else:
condition='M'
df_group = df.resample(condition).agg(agg_dict)
df_group = df_group.dropna()
df_group.merge(df, )
stock['PRICE'] = self.convertFormat(df_group.to_dict())
self.analyzeAdditionalInfo(stock, cursor, type)
conn.commit()
cursor.close()
conn.close()
return
if __name__ == "__main__":
start = time.time()
PROJECT_HOME = os.path.join(os.path.dirname(os.path.join(os.path.dirname(os.path.join(os.path.dirname(__file__))))))
stockFileName = PROJECT_HOME + '/resources/stock.db'
analyzer = AnalyzerSqlite(PROJECT_HOME, stockFileName)
#analyzer.analyzeDaily()
#analyzer.analyzeGrouping("weekly")
#analyzer.analyzeGrouping("monthly")
day = datetime.today().strftime("%Y%m%d")
# HTML 출력
outPath = PROJECT_HOME + "/resources/analysis/"+day
if os.path.isdir(outPath):
shutil.rmtree(outPath)
os.mkdir(outPath)
print("print to Html...")
analyzer.findCandidate(outPath)
print("time : %6.2f 초" % (time.time() - start))
print("done...")

40
stockpredictor_back/analysis/BolingerBand.py
View File

@@ -1,40 +0,0 @@
import pandas as pd
from stockpredictor.analysis.Common import Common
import plotly.graph_objs as go
from plotly import tools, subplots
import plotly.io as po
class BolingerBand:
common = None
def __init__(self):
self.common = Common()
return
def apply(self, df, n=13, t=2):
# 입력받은 값이 dataframe이라는 것을 정의해줌
df = pd.DataFrame(df)
max20 = df["close"].rolling(window=n).mean()
stddev = df["close"].rolling(window=n).std()
upper = max20 + (stddev * t) # 상단 볼리저 밴드
lower = max20 - (stddev * t) # 하단 볼리저 밴드
middle = (upper + lower ) / 2
# dataframe에 컬럼 추가
df = df.assign(upper=upper, middle=middle, lower=lower)
return df
def analyze(self, stock):
df = pd.DataFrame()
df = df.from_dict(stock['PRICE'])
df = self.apply(df)
for i in range(len(df.upper)):
stock['PRICE'][i]['bolingerband_upper'] = df.upper.values[i]
stock['PRICE'][i]['bolingerband_middle'] = df.middle.values[i]
stock['PRICE'][i]['bolingerband_lower'] = df.lower.values[i]
return

560
stockpredictor_back/analysis/Common.old.py
View File

@@ -1,560 +0,0 @@
class Common:
# 상향
def checkUpward(self, type, data):
check = True
if type is not None:
for i in range(len(data)-1):
# 만약 이전이 이후보다 크다면, 상승이 아님
if data[i][type] > data[i+1][type]:
check = False
break
else:
for i in range(len(data)-1):
# 만약 이전이 이후보다 크다면, 상승이 아님
if data[i] > data[i+1]:
check = False
break
return check
# 하향
def checkDownward(self, type, data):
check = True
for i in range(len(data)-1):
# 만약 이전이 이후보다 작다면, 하락이 아님
if data[i][type] < data[i+1][type]:
check = False
break
return check
# 상향 돌파
def checkUpwardBreakthrough(self, type1, type2, data):
if (type1 in data[0] and type1 in data[1] and type1 in data[2] and
type2 in data[0] and type2 in data[1] and type2 in data[2]):
if ((data[0][type1] < data[1][type1] < data[2][type1]) and
(data[0][type1] < data[0][type2] and data[2][type1] > data[2][type2])):
return True
return False
# 하향 돌파
def checkDownwardBreakthrough(self, type1, type2, data):
if (type1 in data[0] and type1 in data[1] and type1 in data[2] and
type2 in data[0] and type2 in data[1] and type2 in data[2]):
if ((data[0][type1] > data[1][type1] > data[2][type1]) and
(data[0][type1] > data[0][type2] and data[2][type1] < data[2][type2])):
return True
return False
def getStochasticScore(self, stock, i):
score = 0
if (stock[i - 1]['slow_k'] < stock[i]['slow_k'] and
stock[i]['slow_d'] < stock[i]['slow_k']):
if stock[i]['slow_k'] < 5:
score = 8
elif 5 <= stock[i]['slow_k'] < 10:
score = 7
elif 10 <= stock[i]['slow_k'] < 15:
score = 6
elif 15 <= stock[i]['slow_k'] < 20:
score = 5
elif 20 <= stock[i]['slow_k'] < 30:
score = 4
elif 30 <= stock[i]['slow_k'] < 40:
score = 3
elif 40 <= stock[i]['slow_k'] < 50:
score = 2
else:
score = 1
if (stock[i - 1]['slow_k'] > stock[i]['slow_k'] and
stock[i - 1]['slow_k'] > stock[i - 1]['slow_d'] and
stock[i]['slow_k'] < stock[i]['slow_d']):
if stock[i]['slow_k'] > 90:
score = -6
elif 90 >= stock[i]['slow_k'] > 80:
score = -5
elif 80 >= stock[i]['slow_k'] > 70:
score = -4
elif 70 >= stock[i]['slow_k'] > 60:
score = -3
elif 60 >= stock[i]['slow_k'] > 50:
score = -2
else:
score = -1
return score
def getBolingerBandScore(self, stock, i):
return 0
def getIchimokuCloudScore(self, stock, i):
score = 0
if stock[i - 1]['leadingSpan1'] != 0 and stock[i - 1]['leadingSpan2'] != 0:
# 후행스팬 > 선행스펜 일때, 후행스펜 > 어제 주가 > 선행스팬 이고, 오늘 주가 > 후행스팬 < 선행스팬 이라면, 매수 2점
if (stock[i - 1]['leadingSpan2'] > stock[i - 1]['leadingSpan1'] and stock[i]['leadingSpan2'] > stock[i]['leadingSpan1']):
if (stock[i - 1]['leadingSpan2'] > stock[i - 1]['close'] > stock[i - 1]['leadingSpan1'] and
stock[i]['close'] > stock[i]['leadingSpan2'] > stock[i - 1]['leadingSpan1']):
score = 2
# 후행스팬 > 선행스펜 일때, 후행스펜 > 선행스팬 > 어제 주가 이고, 오늘 주가 > 후행스팬 < 선행스팬 이라면, 매수 4점
if (stock[i - 1]['leadingSpan2'] > stock[i - 1]['leadingSpan1'] and stock[i]['leadingSpan2'] > stock[i]['leadingSpan1']):
if (stock[i - 1]['leadingSpan2'] > stock[i - 1]['leadingSpan1'] > stock[i - 1]['close'] and
stock[i]['close'] > stock[i]['leadingSpan2'] > stock[i - 1]['leadingSpan1']):
score = 4
# 선행스팬 > 후행스팬 일때, 선행스팬 > 어제 주가 > 후행스팬 이고, 오늘 주가 > 선행스팬 < 후행스팬 이라면, 매수 1점
if (stock[i - 1]['leadingSpan1'] > stock[i - 1]['leadingSpan2'] and stock[i]['leadingSpan1'] > stock[i]['leadingSpan2']):
if (stock[i - 1]['leadingSpan1'] > stock[i - 1]['close'] > stock[i - 1]['leadingSpan2'] and
stock[i]['close'] > stock[i]['leadingSpan1'] > stock[i - 1]['leadingSpan2']):
score = 1
# 선행스팬 > 후행스팬 일때, 선행스팬 > 후행스팬 > 어제 주가 이고, 오늘 주가 > 선행스팬 < 후행스팬 이라면, 매수 3점
if (stock[i - 1]['leadingSpan1'] > stock[i - 1]['leadingSpan2'] and stock[i]['leadingSpan1'] > stock[i]['leadingSpan2']):
if (stock[i - 1]['leadingSpan1'] > stock[i - 1]['leadingSpan2'] > stock[i - 1]['close'] and
stock[i]['close'] > stock[i]['leadingSpan1'] > stock[i - 1]['leadingSpan2']):
score = 3
# 어제는 주가가 선행이나 후행스팬 위에 있었지만, 오늘은 두 스팬 모두 아래로 내려왔을 때 매도
if (stock[i - 1]['close'] > stock[i - 1]['leadingSpan1'] or stock[i - 1]['close'] > stock[i - 1]['leadingSpan2']):
if (stock[i]['close'] < stock[i]['leadingSpan1'] and stock[i]['close'] < stock[i]['leadingSpan2']):
score = -1
return score
# YANGBONG
# 어제 음봉 이후 장대양봉이었다면, 매수
def checkLongYangBongAfterUmBong(self, stock, i):
if i > 0:
if stock[i-1]['close'] < stock[i-1]['open']: # 어제가 음봉인지 체크
if stock[i]['open'] < stock[i]['close'] and stock[i]['close'] == stock[i]['high']: # 오늘 장대양봉인지 체크
if stock[i-1]['volume']*2 < stock[i]['volume']: # 어제 거래량 보다 두배 이상일 때
return "UMYANG_"
return ""
def checkDoji(self, stock, i):
# 하락 추세이고, 그저께, 어제 음봉이고, 오늘 도지인지 체크한다
if i > 2:
# 하락 추세이고
if stock[i - 1]['close'] < stock[i - 2]['close']:
# 그저께와 어제가 음봉인지 체크
if stock[i-2]['close'] < stock[i-2]['open'] and stock[i-1]['close'] < stock[i-1]['open']:
# 도지 체크
if stock[i]['open'] == stock[i]['close'] and stock[i]['low'] < stock[i]['close'] < stock[i]['high']:
return "DOJI_"
return ""
def checkGravestone(self, stock, i):
# 상승 추세이고, 어제 양봉이고, 오늘 그레이브스톤인지 체크한다
if i > 2:
# 상승 추세이고
if stock[i-2]['close'] < stock[i - 1]['close']:
# 어제 양봉인지 체크
if stock[i-1]['open'] < stock[i-1]['close']:
# 오늘 그레이브스톤인지 체크한다
if stock[i]['open'] == stock[i]['close'] == stock[i]['low'] and stock[i]['low'] < stock[i]['high']:
return "GRAVESTONE_"
return ""
# 하락 추세에서 드레곤플라이가 나오면 매수
def checkDragonfly(self, stock, i):
# 하락 추세이고, 그저께, 어제 음봉이고, 오늘 드레곤플라이인지 체크한다
if i > 1:
# 하락 추세이고
if stock[i - 1]['close'] < stock[i - 2]['close']:
# 그저께와 어제가 음봉인지 체크
if stock[i - 2]['close'] < stock[i - 2]['open'] and stock[i - 1]['close'] < stock[i - 1]['open']:
# 오늘 드레곤플라이인지 체크한다
if stock[i]['open'] == stock[i]['close'] == stock[i]['high'] and stock[i]['low'] < stock[i]['high']:
return "DRAGONEFLY_"
return ""
def checkHammer(self, stock, i):
# 하락 추세이고, 그저께, 어제 음봉이고, 오늘 해머인지 체크한다
if i > 1:
# 하락 추세이고
if stock[i - 1]['close'] < stock[i - 2]['close']:
# 그저께와 어제가 음봉인지 체크
if stock[i - 2]['close'] < stock[i - 2]['open'] and stock[i - 1]['close'] < stock[i - 1]['open']:
# 오늘 해머인지 체크한다
if stock[i]['open'] < stock[i]['close']:
if (stock[i]['close'] - stock[i]['open']) * 2 < stock[i]['open'] - stock[i]['low']:
# 윗꼬리가 몸통보다 짧아야 한다.
if stock[i]['high'] - stock[i]['close'] < stock[i]['close'] - stock[i]['open']:
return "HAMMER_"
if stock[i]['close'] < stock[i]['open']:
if (stock[i]['open'] - stock[i]['close']) * 2 < stock[i]['close'] - stock[i]['low']:
# 윗꼬리가 몸통보다 짧아야 한다.
if stock[i]['high'] - stock[i]['open'] < stock[i]['open'] - stock[i]['close']:
return "HAMMER_"
return ""
def checkHangingman(self, stock, i):
# 상승 추세이고, 어제 양봉이고, 오늘 행잉맨인지 체크한다
if i > 2:
# 상승 추세이고
if stock[i-2]['close'] < stock[i - 1]['close']:
# 어제 양봉인지 체크
if stock[i-1]['open'] < stock[i-1]['close']:
# 오늘 해머인지 체크한다
if stock[i]['open'] < stock[i]['close']:
if (stock[i]['close'] - stock[i]['open']) * 2 < stock[i]['open'] - stock[i]['low']:
# 윗꼬리가 몸통보다 짧아야 한다.
if stock[i]['high'] - stock[i]['close'] < stock[i]['close'] - stock[i]['open']:
return "HANGINGMAN_"
if stock[i]['close'] < stock[i]['open']:
if (stock[i]['open'] - stock[i]['close']) * 2 < stock[i]['close'] - stock[i]['low']:
# 윗꼬리가 몸통보다 짧아야 한다.
if stock[i]['high'] - stock[i]['open'] < stock[i]['open'] - stock[i]['close']:
return "HANGINGMAN_"
return ""
def checkEngulfingHigh(self, stock, i):
# 하락 추세에서 상승 장악형인지 체크
if i > 2:
# 하락 추세이고
if stock[i - 1]['close'] < stock[i - 2]['close']:
# 그저께와 어제가 음봉인지 체크
if stock[i - 2]['close'] < stock[i - 2]['open'] and stock[i - 1]['close'] < stock[i - 1]['open']:
# 오늘 상승장악형인지 체크
if stock[i]['open'] < stock[i]['close']:
if stock[i-1]['open'] < stock[i]['close'] and stock[i]['open'] < stock[i-1]['close']:
return "ENHIGH_"
return ""
def checkEngulfingLow(self, stock, i):
# 상승 추세에서 하락 장악형인지 체크
if i > 2:
# 상승 추세이고
if stock[i - 2]['close'] < stock[i - 1]['close']:
# 어제 양봉인지 체크
if stock[i - 1]['open'] < stock[i - 1]['close']:
# 오늘 하락장악형인지 체크
if stock[i]['close'] < stock[i]['open']:
if stock[i-1]['close'] < stock[i]['open'] and stock[i]['close'] < stock[i-1]['open']:
return "ENLOW_"
return ""
def checkHaramiHigh(self, stock, i):
# # 하락 추세에서 상승포아형인지 체크
if i > 2:
# 하락 추세이고
if stock[i - 1]['close'] < stock[i - 2]['close']:
# 그저께와 어제가 음봉인지 체크
if stock[i - 2]['close'] < stock[i - 2]['open'] and stock[i - 1]['close'] < stock[i - 1]['open']:
# 오늘 상승포아형인지 체크
if stock[i]['open'] < stock[i]['close']:
if stock[i-1]['close'] < stock[i]['low'] and stock[i]['high'] < stock[i-1]['open']:
return "HAHIGH_"
return ""
def checkHaramiLow(self, stock, i):
# 상승 추세에서 하락 포아형인지 체크
if i > 2:
# 상승 추세이고
if stock[i - 2]['close'] < stock[i - 1]['close']:
# 어제 양봉인지 체크
if stock[i - 1]['open'] < stock[i - 1]['close']:
# 오늘 하락포아형인지 체크
if stock[i]['close'] < stock[i]['open']:
if stock[i-1]['open'] < stock[i]['low'] and stock[i]['high'] < stock[i-1]['close']:
return "HALOW_"
return ""
def checkPiercing(self, stock, i):
# 하락 추세에서 관통형인지 체크
if i > 2:
# 하락 추세이고
if stock[i - 1]['close'] < stock[i - 2]['close']:
# 그저께와 어제가 음봉인지 체크
if stock[i - 2]['close'] < stock[i - 2]['open'] and stock[i - 1]['close'] < stock[i - 1]['open']:
# 오늘 관통형인지 체크
if stock[i]['open'] < stock[i]['close']:
if stock[i]['open'] < stock[i-1]['low'] and (stock[i-1]['close'] + stock[i-1]['open'])/2 < stock[i]['close'] < stock[i-1]['close']:
return "PIERCING_"
return ""
def checkDarkCloud(self, stock, i):
# 상승 추세에서 흑운형인지 체크
if i > 2:
# 상승 추세이고
if stock[i - 2]['close'] < stock[i - 1]['close']:
# 어제 양봉인지 체크
if stock[i - 1]['open'] < stock[i - 1]['close']:
# 오늘 흑운형인지 체크
if stock[i]['close'] < stock[i]['open']:
if stock[i-1]['high'] < stock[i]['open'] and stock[i-1]['open'] < stock[i]['close'] < (stock[i-1]['open'] + stock[i-1]['close'])/2:
return "DARKCLOUD_"
return ""
def checkMorningstar(self, stock, i):
# 하락 추세에서 샛별인지 체크
if i > 3:
# 하락 추세이고
if stock[i-1]['close'] < stock[i-2]['close']:
# 그저께와 어제가 음봉인지 체크
if stock[i - 2]['close'] < stock[i - 2]['open'] and stock[i - 1]['close'] < stock[i - 1]['open']:
# 오늘 샛별인지 체크
# 어제 갭 체크
if stock[i-1]['open'] < stock[i - 2]['close'] and stock[i-1]['close'] < stock[i - 2]['close']:
# 오늘 시가가 어제 종가보다 높으며 양봉
if stock[i-1]['close'] < stock[i]['open'] < stock[i]['close']:
return "MORNINGSTAR_"
return ""
def checkEveningstar(self, stock, i):
# 상승 추세에서 저녁별형인지 체크
if i > 3:
# 상승 추세이고
if stock[i-2]['close'] < stock[i-1]['close']:
# 어제 양봉인지 체크
if stock[i-1]['open'] < stock[i-1]['close']:
# 오늘 저녁별형인지 체크
# 어제 갭 체크
if stock[i-2]['close'] < stock[i-1]['open'] and stock[i-2]['close'] < stock[i-1]['close']:
# 오늘 시가가 어제 종가보다 낮으며 음봉
if stock[i]['close'] < stock[i-1]['open'] < stock[i-1]['close']:
return "EVENINGSTAR_"
return ""
def checkAllUpperCross(self, stock, i):
if i > 10:
if stock[i]['avg5'] < stock[i]['close'] and stock[i]['avg20'] < stock[i]['close'] and stock[i]['avg60'] < stock[i]['close'] and stock[i]['avg120'] < stock[i]['close']:
for j in range(1, 6):
if stock[i-j]['close'] < stock[i-j]['avg5'] and stock[i-j]['close'] < stock[i-j]['avg20'] and stock[i-j]['close'] < stock[i-j]['avg60'] and stock[i-j]['close'] < stock[i-j]['avg120']:
return "ALLUPPER_"
return ""
def check_golded_cross(self, stock, i):
if i > 1:
# 60 -> 120
# 오늘 지수는 120 < 60 < 20 < 5
# 어제 지수는 60 < 120 이었다.
# 60, 20, 5일선 모두 어제 보다 오늘이 더 높다 (상승중)
# 5일과 20일선은 상승 중이며, 60일선이 120일 선을 뚫고 올라온 순간인지 체크함 (삼성전자 2021-07-29)
# 이때 바로 매수하지 않는다.
# 이 시점 이후로 5일선이 20일선을 하방으로 뚫었다가 다시 20일선을 상방으로 뚫는 순간 매수를 시도한다.
if stock[i]['avg120'] < stock[i]['avg60'] < stock[i]['avg20'] < stock[i]['avg5']:
if stock[i-1]['avg120'] > stock[i-1]['avg60']:
if (stock[i-1]['avg60'] < stock[i]['avg60'] and stock[i-1]['avg20'] < stock[i]['avg20'] and stock[i-1]['avg5'] < stock[i]['avg5']):
return "GOLDEN#1_"
# 20 -> 120: 5일과 20일, 60일선은 상승 중이며, 20일선이 120일 선을 뚫고 올라온 순간인지 체크 (SK 2021-12-09, 나노스 2021-02-04)
# 어제는 60일선 < 20일선 < 120일선 < 5일선이지만, 오늘은 60일선 < 120일선 < 20일선 < 5일선
# 이때 바로 매수하지 않는다.
# 이 시점 이후로 5일선이 20일선을 하방으로 뚫었다가 다시 20일선을 상방으로 뚫는 순간 매수를 시도한다.
if stock[i]['avg60'] < stock[i]['avg120'] < stock[i]['avg20'] < stock[i]['avg5']:
if stock[i-1]['avg60'] < stock[i-1]['avg20'] < stock[i]['avg120'] < stock[i]['avg5']:
if (stock[i-1]['avg60'] < stock[i]['avg60'] and stock[i-1]['avg20'] < stock[i]['avg20'] and stock[i-1]['avg5'] < stock[i]['avg5']):
return "GOLDEN#2_"
# 20 -> 120: 5일과 20일, 60일선은 상승 중이며, 20일선이 120일 선을 뚫고 올라온 순간인지 체크 (갤럭시아머니트리 2021-02-08)
# 어제는 60일선 < 120일선 < 5일선 < 20일선이지만, 오늘은 60일선 < 120일선 < 20일선 < 5일선
if stock[i]['avg60'] < stock[i]['avg120'] < stock[i]['avg20'] < stock[i]['avg5']:
if stock[i-1]['avg60'] < stock[i-1]['avg20'] < stock[i]['avg120'] < stock[i]['avg5']:
if (stock[i-1]['avg60'] < stock[i]['avg60'] and stock[i-1]['avg20'] < stock[i]['avg20'] and stock[i-1]['avg5'] < stock[i]['avg5']):
return "GOLDEN#3_"
return ""
def check_bearmarket_buying(self, stock, stochastic, i):
if i > 1:
# 5일선 상승 시점 확인 (SK 2020년 3월 24일)
# 어제는 5일선 < 20일선 < 120일선 < 60일선이며, 오늘은 20일선 < 120일선 < 60일선
# 어제와 오늘 모두 20일, 60일, 120일선은 모두 하락이다.
# 어제 종가보다 오늘 종가가 높고, 종가는 5일선 위에 올라왔다.
# 오늘 slow_k는 30 이하이며, 어제는 slow_d가 높았지만, 오늘은 slow_k가 더 높음
if (stock[i-1]['avg5'] < stock[i-1]['avg20'] < stock[i-1]['avg120'] < stock[i]['avg60']) and (stock[i]['avg20'] < stock[i-1]['avg120'] < stock[i]['avg60']):
if stock[i]['avg120'] < stock[i-1]['avg120'] and stock[i]['avg60'] < stock[i-1]['avg60'] and stock[i]['avg20'] < stock[i-1]['avg20']:
if stock[i-1]['close'] <= stock[i]['close'] and stock[i]['avg5'] <= stock[i]['close']:
if (stochastic[i]['slow_k'] < 30 and (stochastic[i-1]['slow_k'] < stochastic[i-1]['slow_d'] and stochastic[i]['slow_d'] < stochastic[i]['slow_k'])):
return "BEARMARKET#1_"
# 5일선 상승 시점 확인 (원풍물산 2020년 3월 24일, NHN한국사이버결제 2018년 11월 2일)
# 어제는 5일선 < 20일선 < 60일선 < 120일선이며, 오늘은 20일선 < 60일선 < 120일선
# 어제와 오늘 모두 20일, 60일, 120일선은 모두 하락이다.
# 어제 종가보다 오늘 종가가 높고, 종가는 5일선 위에 올라왔다.
# 오늘 slow_k는 30 이하이며, 어제는 slow_d가 높았지만, 오늘은 slow_k가 더 높음
if (stock[i-1]['avg5'] < stock[i-1]['avg20'] < stock[i-1]['avg60'] < stock[i]['avg120']) and (stock[i]['avg20'] < stock[i-1]['avg60'] < stock[i]['avg120']):
if stock[i]['avg120'] < stock[i-1]['avg120'] and stock[i]['avg60'] < stock[i-1]['avg60'] and stock[i]['avg20'] < stock[i-1]['avg20']:
if stock[i-1]['close'] <= stock[i]['close'] and stock[i]['avg5'] <= stock[i]['close']:
if (stochastic[i]['slow_k'] < 30 and (stochastic[i-1]['slow_k'] < stochastic[i-1]['slow_d'] and stochastic[i]['slow_d'] < stochastic[i]['slow_k'])):
return "BEARMARKET#2_"
return ""
def check_stochastic(self, stock, stochastic, i):
if i > 2:
# 스토케스틱이 15 이하인 경우
# 어제보다 slow_k가 상승했고, 오늘 slow_k가 slow_d 위에 있는 경우,
if stochastic[i]['slow_k'] < 15:
if stochastic[i-1]['slow_k'] < stochastic[i]['slow_k'] and stochastic[i]['slow_d'] < stochastic[i]['slow_k']:
return "STOCHASTIC_"
return ""
def check_stochastic_buying(self, stock, stochastic, ichimoku, i):
if i > 3:
# 삼성전자 2020년 11월 4일
# 어제는 slow_K가 Slow_d 아래였지만, 오늘은 slow_K가 Slow_d 보다 높다.
# 에제의 slow_k는 20보다 작고, 오늘의 slow_K는 30보다 작다
# 1일전이나, 2, 3일전의 종가가 일목균형표 내의 선행스팬1 아래 존재하며,오늘 고가는 선행스팬1 위에 존재한다.
# 그저께 시가보다 어제의 시가가, 어제의 시가보다는 오늘의 시가가 높다.
if (stochastic[i-1]['slow_k'] < stochastic[i-1]['slow_d'] and stochastic[i]['slow_d'] < stochastic[i]['slow_k']):
if (stochastic[i - 1]['slow_k'] < 20 and stochastic[i]['slow_k'] < 30):
if ((stock[i-3]['close'] < ichimoku[i-3]['leadingSpan1'] or stock[i-2]['close'] < ichimoku[i-2]['leadingSpan1'] or stock[i-1]['close'] < ichimoku[i-1]['leadingSpan1']) and ichimoku[i-1]['leadingSpan1'] < stock[i-1]['high']):
if stock[i-2]['open'] < stock[i-1]['open'] < stock[i]['open']:
return "STOCHASTIC#1_"
# 스토케스틱이 15 이하인 경우
# 어제보다 slow_k가 상승했고, 오늘 slow_k가 slow_d 위에 있는 경우,
# 오늘 종가가 5일선 위에 있는 경우
if stochastic[i]['slow_k'] < 15:
if stochastic[i - 1]['slow_k'] < stochastic[i]['slow_k'] and stochastic[i]['slow_d'] < stochastic[i]['slow_k']:
if stock[i]['avg5'] < stock[i]['close']:
return "STOCHASTIC#2_"
return ""
def check_Dolpa(self, stock, i, avg1, avg2):
upper_index = 0
if len(stock) > 2:
if stock[i-1]["avg"+avg1] < stock[i-1]["avg"+avg2] and stock[i]["avg"+avg1] > stock[i]["avg"+avg2]:
return avg1+"_"+avg2+"_"
return ""
def check_Dolpa_Jiji(self, stock, i, day='20'):
upper_index = 0
if len(stock) > 5:
for idx in range(1, 5):
# day선을 돌파하는 양봉이고, 종가가 최고가 보다 100 이내이어야 한다.
if stock[-idx]['open'] < stock[-idx]["avg"+day] < stock[-idx]['close'] and stock[-idx]['high'] - 100 <= stock[-idx]['close']:
upper_index = idx
break
if upper_index != 0:
for cidx in range(1, upper_index):
# 해당일의 종가보다 현재의 시가가 높거나 같아야 하며, 현재가는 양봉이어야 한다.
if stock[-upper_index]['close'] <= stock[-cidx]['open'] and stock[-cidx]['open'] < stock[-cidx]['close']:
# 해당 기준일 선은 상승이어야 한다.
if stock[-upper_index]['avg'+day] < stock[-cidx]['avg'+day]:
return day + "_"
return ""
def check_Dolpa_Jiji_20(self, stock, i):
"""
top: 이전 5일선이 20일선 위에 있을 때 최고가
top일 체크 사항 (20일 < 5일선)
5일선이 20일 선으로 내려왔다가 다시 20일선 위로 올라왔고, top < 오늘 시가 + 100
top < 시가 < 종가 라면 다음날 매수한다.
# https://docs.google.com/presentation/d/1MVuaeRNljqLCdn4dPZmvVdtl2Ab09Zwg/edit#slide=id.gc7b796e645_0_80
"""
if len(stock) > 61:
if stock[i]['avg20'] < stock[i]['close'] and stock[i]['avg20'] < stock[i]['open']:
if stock[i]['avg5'] < stock[i]['avg20']:
index1 = -1
for j in range(1, 61):
if stock[i-j]['avg20'] < stock[i-j]['avg5']:
index1 = j
break
top = 0
for j in range(index1+1, 61):
if stock[i - j]['open'] < stock[i - j]['close']:
if top < stock[i - j]['close']:
top = stock[i - j]['close']
else:
if top < stock[i - j]['open']:
top = stock[i - j]['open']
if stock[i-j]['avg5'] < stock[i-j]['avg20']:
break
return "5-20_"
return ""
def check_Danta1(self, stock, i):
"""
어제 상한가 혹은 상승양봉이 나온다.
오늘 상승 출발을 해야 하며 상승 음봉이 나온다
- 어제 종가 = 어제 상한가 < 종가 < 시가 < 상한가
https://docs.google.com/presentation/d/1MVuaeRNljqLCdn4dPZmvVdtl2Ab09Zwg/edit#slide=id.gc7b796e645_0_109
만약 다음날 시작초가가 오늘 종가보다 높게 상승으로 출발한다면 매수를 한다.
손절가는 오늘 최저가이다.
"""
if stock[i-1]['open'] < stock[i-1]['close'] == stock[i-1]['high']:
if stock[i-1]['close'] < stock[i]['close'] < stock[i]['open'] < stock[i]['high']:
return "danta1_"
return ""
def check_Danta2(self, stock, i):
"""
쐐기, 수렴, 깃대 패턴 확인
# https://docs.google.com/presentation/d/1MVuaeRNljqLCdn4dPZmvVdtl2Ab09Zwg/edit#slide=id.gc7b796e645_0_144
상단 추세선을 돌파하면 매수를 한다.
"""
price_10 = round(stock[i]["close"] / 10)
if stock[-i]["open"] < stock[-i]["close"]:
top = stock[-i]["close"]
bottom = stock[-i]["open"]
else:
top = stock[-i]["open"]
bottom = stock[-i]["close"]
if len(stock) > 21:
for i in range(2, 21):
if stock[-i]["open"] < stock[-i]["close"]:
if top < stock[-i]["close"]:
top = stock[-i]["close"]
if stock[-i]["open"] < bottom:
bottom = stock[-i]["open"]
else:
if top < stock[-i]["open"]:
top = stock[-i]["open"]
if stock[-i]["close"] < bottom:
bottom = stock[-i]["close"]
if top - bottom < price_10:
return "danta2_"
return ""
def check_RightArrange(self, stock, i):
"""
어제는 정배열이 아니었는데, 오늘은 정배열인 경우
"""
if len(stock) > 2:
if not (stock[i - 1]["avg120"] < stock[i - 1]["avg60"] < stock[i - 1]["avg20"] < stock[i - 1]["avg5"]) and (stock[i]["avg120"] < stock[i]["avg60"] < stock[i]["avg20"] < stock[i]["avg5"]):
return "arrange_"
return ""
def checkHigherUmbong(self, stock, i):
# 음봉인데 어제보다 종가가 더 높은 경우
# 이 경우 정배열 상태인지도 함께 체크를 한다.
if len(stock) > 3:
# 어제는 거래량이 터진 양봉이다.
if stock[i-1]['open'] < stock[i-1]['close'] and 5*stock[i-2]['volume'] < stock[i-1]['volume']:
# 오늘은 음봉인데, 오늘 종가는 어제 시가보다는 높다
if stock[i]['close'] < stock[i]['open'] and stock[i-1]['open'] < stock[i]['close']:
return "HIGHERUMBONG_"
return ""
def check_W1Rise(self, stock, i, limit):
if len(stock) > 5:
rate = round((stock[i]["close"] - stock[i-4]["close"]) / stock[i-4]["close"],2)
if rate >= limit:
return "1w("+str(rate)+")_"
return ""
def check_D1Fall(self, stock, i, limit):
if len(stock) > 2:
# 1000, 900, (900 - 1000) / 900 = -0.111
# 1000, 800, (800 - 1000) / 800 = -0.25
rate = round((stock[i]["close"] - stock[i-1]["close"]) / stock[i-1]["close"], 2)
if rate <= limit:
return "1d("+str(rate)+")_"
return ""

482
stockpredictor_back/analysis/Common.py
View File

@@ -1,482 +0,0 @@
class Common:
# 상향
def checkUpward(self, type, data):
check = True
if type is not None:
for i in range(len(data)-1):
# 만약 이전이 이후보다 크다면, 상승이 아님
if data[i][type] > data[i+1][type]:
check = False
break
else:
for i in range(len(data)-1):
# 만약 이전이 이후보다 크다면, 상승이 아님
if data[i] > data[i+1]:
check = False
break
return check
# 하향
def checkDownward(self, type, data):
check = True
for i in range(len(data)-1):
# 만약 이전이 이후보다 작다면, 하락이 아님
if data[i][type] < data[i+1][type]:
check = False
break
return check
# 상향 돌파
def checkUpwardBreakthrough(self, type1, type2, data):
if (type1 in data[0] and type1 in data[1] and type1 in data[2] and
type2 in data[0] and type2 in data[1] and type2 in data[2]):
if ((data[0][type1] < data[1][type1] < data[2][type1]) and
(data[0][type1] < data[0][type2] and data[2][type1] > data[2][type2])):
return True
return False
# 하향 돌파
def checkDownwardBreakthrough(self, type1, type2, data):
if (type1 in data[0] and type1 in data[1] and type1 in data[2] and
type2 in data[0] and type2 in data[1] and type2 in data[2]):
if ((data[0][type1] > data[1][type1] > data[2][type1]) and
(data[0][type1] > data[0][type2] and data[2][type1] < data[2][type2])):
return True
return False
# YANGBONG
# 어제 음봉 이후 장대양봉이었다면, 매수
def checkLongYangBongAfterUmBong(self, stock):
if len(stock['close']) > 2:
if stock['close'][1] < stock['open'][1]: # 어제가 음봉인지 체크
if stock['open'][0] < stock['close'][0] and stock['close'][0] == stock['high'][0]: # 오늘 장대양봉인지 체크
if stock['volume'][1]*2 < stock['volume'][0]: # 어제 거래량 보다 두배 이상일 때
return "UMYANG_"
return ""
def checkDoji(self, stock):
# 하락 추세이고, 그저께, 어제 음봉이고, 오늘 도지인지 체크한다
if len(stock['close']) > 2:
# 하락 추세이고
if stock['close'][1] < stock['close'][2]:
# 그저께와 어제가 음봉인지 체크
if stock['close'][2] < stock['open'][2] and stock['close'][1] < stock['open'][1]:
# 도지 체크
if stock['open'][0] == stock['close'][0] and stock['low'][0] < stock['close'][0] < stock['high'][0]:
return "DOJI_"
return ""
def checkGravestone(self, stock):
# 상승 추세이고, 어제 양봉이고, 오늘 그레이브스톤인지 체크한다
if len(stock['close']) > 2:
# 상승 추세이고
if stock['close'][2] < stock['close'][1]:
# 어제 양봉인지 체크
if stock['open'][1] < stock['close'][1]:
# 오늘 그레이브스톤인지 체크한다
if stock['open'][0] == stock['close'][0] == stock['low'][0] and stock['low'][0] < stock['high'][0]:
return "GRAVESTONE_"
return ""
# 하락 추세에서 드레곤플라이가 나오면 매수
def checkDragonfly(self, stock):
# 하락 추세이고, 그저께, 어제 음봉이고, 오늘 드레곤플라이인지 체크한다
if len(stock['close']) > 2:
# 하락 추세이고
if stock['close'][1] < stock['close'][1]:
# 그저께와 어제가 음봉인지 체크
if stock['close'][2] < stock['open'][2] and stock['close'][1] < stock['open'][1]:
# 오늘 드레곤플라이인지 체크한다
if stock['open'][0] == stock['close'][0] == stock['high'][0] and stock['low'][0] < stock['high'][0]:
return "DRAGONEFLY_"
return ""
def checkHammer(self, stock):
# 하락 추세이고, 그저께, 어제 음봉이고, 오늘 해머인지 체크한다
if len(stock['close']) > 2:
# 하락 추세이고
if stock['close'][1] < stock['close'][1]:
# 그저께와 어제가 음봉인지 체크
if stock['close'][2] < stock['open'][2] and stock[1]['close'] < stock[1]['open']:
# 오늘 해머인지 체크한다
if stock['open'][0] < stock['close'][0]:
if (stock['close'][0] - stock['open'][0]) * 2 < stock['open'][0] - stock['low'][0]:
# 윗꼬리가 몸통보다 짧아야 한다.
if stock['high'][0] - stock['close'][0] < stock['close'][0] - stock['open'][0]:
return "HAMMER_"
if stock['close'][0] < stock['open'][0]:
if (stock['open'][0] - stock['close'][0]) * 2 < stock['close'][0] - stock['low'][0]:
# 윗꼬리가 몸통보다 짧아야 한다.
if stock['high'][0] - stock['open'][0] < stock['open'][0] - stock['close'][0]:
return "HAMMER_"
return ""
def checkHangingman(self, stock):
# 상승 추세이고, 어제 양봉이고, 오늘 행잉맨인지 체크한다
if len(stock['close']) > 2:
# 상승 추세이고
if stock['close'][2] < stock['close'][1]:
# 어제 양봉인지 체크
if stock['open'][1] < stock['close'][1]:
# 오늘 해머인지 체크한다
if stock['open'][0] < stock['close'][0]:
if (stock['close'][0] - stock['open'][0]) * 2 < stock['open'][0] - stock['low'][0]:
# 윗꼬리가 몸통보다 짧아야 한다.
if stock['high'][0] - stock['close'][0] < stock['close'][0] - stock['open'][0]:
return "HANGINGMAN_"
if stock['close'][0] < stock['open'][0]:
if (stock['open'][0] - stock['close'][0]) * 2 < stock['close'][0] - stock['low'][0]:
# 윗꼬리가 몸통보다 짧아야 한다.
if stock['high'][0] - stock['open'][0] < stock['open'][0] - stock['close'][0]:
return "HANGINGMAN_"
return ""
def checkEngulfingHigh(self, stock):
# 하락 추세에서 상승 장악형인지 체크
if len(stock['close']) > 2:
# 하락 추세이고
if stock['close'][1] < stock['close'][2]:
# 그저께와 어제가 음봉인지 체크
if stock['close'][2] < stock['open'][2] and stock['close'][1] < stock['open'][1]:
# 오늘 상승장악형인지 체크
if stock['open'][0] < stock['close'][0]:
if stock['open'][1] < stock['close'][0] and stock['open'][0] < stock['close'][1]:
return "ENHIGH_"
return ""
def checkEngulfingLow(self, stock):
# 상승 추세에서 하락 장악형인지 체크
if len(stock['close']) > 2:
# 상승 추세이고
if stock['close'][2] < stock['close'][1]:
# 어제 양봉인지 체크
if stock['open'][1] < stock['close'][1]:
# 오늘 하락장악형인지 체크
if stock['close'][0] < stock['open'][0]:
if stock['close'][1] < stock['open'][0] and stock['close'][0] < stock['open'][1]:
return "ENLOW_"
return ""
def checkHaramiHigh(self, stock):
# # 하락 추세에서 상승포아형인지 체크
if len(stock['close']) > 2:
# 하락 추세이고
if stock['close'][1] < stock['close'][2]:
# 그저께와 어제가 음봉인지 체크
if stock['close'][2] < stock['open'][2] and stock['close'][1] < stock['open'][1]:
# 오늘 상승포아형인지 체크
if stock['open'][0] < stock['close'][0]:
if stock['close'][1] < stock['low'][0] and stock['high'][0] < stock['open'][1]:
return "HAHIGH_"
return ""
def checkHaramiLow(self, stock):
# 상승 추세에서 하락 포아형인지 체크
if len(stock['close']) > 2:
# 상승 추세이고
if stock['close'][2] < stock['close'][1]:
# 어제 양봉인지 체크
if stock['open'][1] < stock['close'][1]:
# 오늘 하락포아형인지 체크
if stock['close'][0] < stock['open'][0]:
if stock['open'][1] < stock['low'][0] and stock['high'][0] < stock['close'][1]:
return "HALOW_"
return ""
def checkPiercing(self, stock):
# 하락 추세에서 관통형인지 체크
if len(stock['close']) > 2:
# 하락 추세이고
if stock['close'][1] < stock['close'][2]:
# 그저께와 어제가 음봉인지 체크
if stock['close'][2] < stock['open'][2] and stock['close'][1] < stock['open'][1]:
# 오늘 관통형인지 체크
if stock['open'][0] < stock['close'][0]:
if stock['open'][0] < stock['low'][1] and (stock['close'][1] + stock['open'][1])/2 < stock['close'][0] < stock['close'][1]:
return "PIERCING_"
return ""
def checkDarkCloud(self, stock):
# 상승 추세에서 흑운형인지 체크
if len(stock['close']) > 2:
# 상승 추세이고
if stock['close'][2] < stock['close'][1]:
# 어제 양봉인지 체크
if stock['open'][1] < stock['close'][1]:
# 오늘 흑운형인지 체크
if stock['close'][0] < stock['open'][0]:
if stock['high'][1] < stock['open'][0] and stock['open'][1] < stock['close'][0] < (stock['open'][1] + stock['close'][1])/2:
return "DARKCLOUD_"
return ""
def checkMorningstar(self, stock):
# 하락 추세에서 샛별인지 체크
if len(stock['close']) > 2:
# 하락 추세이고
if stock['close'][1] < stock['close'][2]:
# 그저께와 어제가 음봉인지 체크
if stock['close'][2] < stock['open'][2] and stock['close'][1] < stock['open'][1]:
# 오늘 샛별인지 체크
# 어제 갭 체크
if stock['open'][1] < stock['close'][2] and stock['close'][1] < stock['close'][2]:
# 오늘 시가가 어제 종가보다 높으며 양봉
if stock['close'][1] < stock['open'][0] < stock['close'][0]:
return "MORNINGSTAR_"
return ""
def checkEveningstar(self, stock):
# 상승 추세에서 저녁별형인지 체크
if len(stock['close']) > 2:
# 상승 추세이고
if stock['close'][2] < stock['close'][1]:
# 어제 양봉인지 체크
if stock['open'][1] < stock['close'][1]:
# 오늘 저녁별형인지 체크
# 어제 갭 체크
if stock['close'][2] < stock['open'][1] and stock['close'][2] < stock['close'][1]:
# 오늘 시가가 어제 종가보다 낮으며 음봉
if stock['close'][0] < stock['open'][1] < stock['close'][1]:
return "EVENINGSTAR_"
return ""
def checkAllUpperCross(self, stock):
if len(stock['close']) > 10:
if stock['avg5'][0] < stock['close'][0] and stock['avg20'][0] < stock['close'][0] and stock['avg60'][0] < stock['close'][0] and stock['avg120'][0] < stock['close'][0]:
for j in range(1, 6):
if stock['close'][j] < stock['avg5'][j] and stock['close'][j] < stock['avg20'][j] and stock['close'][j] < stock['avg60'][j] and stock['close'][j] < stock['avg120'][j]:
return "ALLUPPER_"
return ""
def check_golded_cross(self, stock):
if len(stock['close']) > 1:
# 60 -> 120
# 오늘 지수는 120 < 60 < 20 < 5
# 어제 지수는 60 < 120 이었다.
# 60, 20, 5일선 모두 어제 보다 오늘이 더 높다 (상승중)
# 5일과 20일선은 상승 중이며, 60일선이 120일 선을 뚫고 올라온 순간인지 체크함 (삼성전자 2021-07-29)
# 이때 바로 매수하지 않는다.
# 이 시점 이후로 5일선이 20일선을 하방으로 뚫었다가 다시 20일선을 상방으로 뚫는 순간 매수를 시도한다.
if stock['avg120'][0] < stock['avg60'][0] < stock['avg20'][0] < stock['avg5'][0]:
if stock['avg120'][1] > stock['avg60'][1]:
if (stock['avg60'][1] < stock['avg60'][0] and stock['avg20'][1] < stock['avg20'][0] and stock['avg5'][1] < stock['avg5'][0]):
return "GOLDEN#1_"
# 20 -> 120: 5일과 20일, 60일선은 상승 중이며, 20일선이 120일 선을 뚫고 올라온 순간인지 체크 (SK 2021-12-09, 나노스 2021-02-04)
# 어제는 60일선 < 20일선 < 120일선 < 5일선이지만, 오늘은 60일선 < 120일선 < 20일선 < 5일선
# 이때 바로 매수하지 않는다.
# 이 시점 이후로 5일선이 20일선을 하방으로 뚫었다가 다시 20일선을 상방으로 뚫는 순간 매수를 시도한다.
if stock['avg60'][0] < stock['avg120'][0] < stock['avg20'][0] < stock['avg5'][0]:
if stock['avg60'][1] < stock['avg20'][1] < stock['avg120'][0] < stock['avg5'][0]:
if (stock['avg60'][1] < stock['avg60'][0] and stock['avg20'][1] < stock['avg20'][0] and stock['avg5'][1] < stock['avg5'][0]):
return "GOLDEN#2_"
# 20 -> 120: 5일과 20일, 60일선은 상승 중이며, 20일선이 120일 선을 뚫고 올라온 순간인지 체크 (갤럭시아머니트리 2021-02-08)
# 어제는 60일선 < 120일선 < 5일선 < 20일선이지만, 오늘은 60일선 < 120일선 < 20일선 < 5일선
if stock['avg60'][0] < stock['avg120'][0] < stock['avg20'][0] < stock['avg5'][0]:
if stock['avg60'][1] < stock['avg20'][1] < stock['avg120'][0] < stock['avg5'][0]:
if (stock['avg60'][1] < stock['avg60'][0] and stock['avg20'][1] < stock['avg20'][0] and stock['avg5'][1] < stock['avg5'][0]):
return "GOLDEN#3_"
return ""
def check_bearmarket_buying(self, stock):
if len(stock['close']) > 1:
# 5일선 상승 시점 확인 (SK 2020년 3월 24일)
# 어제는 5일선 < 20일선 < 120일선 < 60일선이며, 오늘은 20일선 < 120일선 < 60일선
# 어제와 오늘 모두 20일, 60일, 120일선은 모두 하락이다.
# 어제 종가보다 오늘 종가가 높고, 종가는 5일선 위에 올라왔다.
# 오늘 slow_k는 30 이하이며, 어제는 slow_d가 높았지만, 오늘은 slow_k가 더 높음
if (stock['avg5'][1] < stock['avg20'][1] < stock['avg120'][1] < stock['avg60'][0]) and (stock['avg20'][0] < stock['avg120'][1] < stock['avg60'][0]):
if stock['avg120'][0] < stock['avg120'][1] and stock['avg60'][0] < stock['avg60'][1] and stock['avg20'][0] < stock['avg20'][1]:
if stock['close'][1] <= stock['close'][0] and stock['avg5'][0] <= stock['close'][0]:
if (stock['stochastic_slow_k'][0] < 30 and (stock['stochastic_slow_k'][1] < stock['stochastic_slow_d'][1] and stock['stochastic_slow_d'][0] < stock['stochastic_slow_k'][0])):
return "BEARMARKET#1_"
# 5일선 상승 시점 확인 (원풍물산 2020년 3월 24일, NHN한국사이버결제 2018년 11월 2일)
# 어제는 5일선 < 20일선 < 60일선 < 120일선이며, 오늘은 20일선 < 60일선 < 120일선
# 어제와 오늘 모두 20일, 60일, 120일선은 모두 하락이다.
# 어제 종가보다 오늘 종가가 높고, 종가는 5일선 위에 올라왔다.
# 오늘 slow_k는 30 이하이며, 어제는 slow_d가 높았지만, 오늘은 slow_k가 더 높음
if (stock['avg5'][1] < stock['avg20'][1] < stock['avg60'][1] < stock['avg120'][0]) and (stock['avg20'][0] < stock['avg60'][1] < stock['avg120'][0]):
if stock['avg120'][0] < stock['avg120'][1] and stock['avg60'][0] < stock['avg60'][1] and stock['avg20'][0] < stock['avg20'][1]:
if stock['close'][1] <= stock['close'][0] and stock['avg5'][0] <= stock['close'][0]:
if (stock['stochastic_slow_k'][0] < 30 and (stock['stochastic_slow_k'][1] < stock['stochastic_slow_d'][1] and stock['stochastic_slow_d'][0] < stock['stochastic_slow_k'][0])):
return "BEARMARKET#2_"
return ""
def check_stochastic(self, stock):
if len(stock['close']) > 60:
# 스토케스틱이 15 이하인 경우
# 어제보다 slow_k가 상승했고, 오늘 slow_k가 slow_d 위에 있는 경우,
if stock['stochastic_slow_k'][0] is not None and stock['stochastic_slow_k'][0] < 15:
if stock['stochastic_slow_k'][1] < stock['stochastic_slow_k'][0] and stock['stochastic_slow_d'][0] < stock['stochastic_slow_k'][0]:
return "STOCHASTIC_"
return ""
def check_stochastic_buying(self, stock, stochastic, ichimoku, i):
if len(stock['close']) > 60:
# 삼성전자 2020년 11월 4일
# 어제는 slow_K가 Slow_d 아래였지만, 오늘은 slow_K가 Slow_d 보다 높다.
# 에제의 slow_k는 20보다 작고, 오늘의 slow_K는 30보다 작다
# 1일전이나, 2, 3일전의 종가가 일목균형표 내의 선행스팬1 아래 존재하며,오늘 고가는 선행스팬1 위에 존재한다.
# 그저께 시가보다 어제의 시가가, 어제의 시가보다는 오늘의 시가가 높다.
if (stochastic[i-1]['slow_k'] < stochastic[i-1]['slow_d'] and stochastic[i]['slow_d'] < stochastic[i]['slow_k']):
if (stochastic[i - 1]['slow_k'] < 20 and stochastic[i]['slow_k'] < 30):
if ((stock[i-3]['close'] < ichimoku[i-3]['leadingSpan1'] or stock[i-2]['close'] < ichimoku[i-2]['leadingSpan1'] or stock[i-1]['close'] < ichimoku[i-1]['leadingSpan1']) and ichimoku[i-1]['leadingSpan1'] < stock[i-1]['high']):
if stock[i-2]['open'] < stock[i-1]['open'] < stock[i]['open']:
return "STOCHASTIC#1_"
# 스토케스틱이 15 이하인 경우
# 어제보다 slow_k가 상승했고, 오늘 slow_k가 slow_d 위에 있는 경우,
# 오늘 종가가 5일선 위에 있는 경우
if stochastic[i]['slow_k'] < 15:
if stochastic[i - 1]['slow_k'] < stochastic[i]['slow_k'] and stochastic[i]['slow_d'] < stochastic[i]['slow_k']:
if stock[i]['avg5'] < stock[i]['close']:
return "STOCHASTIC#2_"
return ""
def check_Dolpa(self, stock, i, avg1, avg2):
upper_index = 0
if len(stock['close']) > 2:
if stock[i-1]["avg"+avg1] < stock[i-1]["avg"+avg2] and stock[i]["avg"+avg1] > stock[i]["avg"+avg2]:
return avg1+"_"+avg2+"_"
return ""
def check_Dolpa_Jiji(self, stock, day='20'):
upper_index = 0
if len(stock['close']) > 5:
for idx in range(1, 5):
# day선을 돌파하는 양봉이고, 종가가 최고가 보다 100 이내이어야 한다.
if stock['open'][idx] < stock["avg"+day][idx] < stock['close'][idx] and stock['high'][idx] - 100 <= stock['close'][idx]:
upper_index = idx
break
if upper_index != 0:
for cidx in range(1, upper_index):
# 해당일의 종가보다 현재의 시가가 높거나 같아야 하며, 현재가는 양봉이어야 한다.
if stock['close'][upper_index] <= stock['open'][cidx] and stock['open'][cidx] < stock['close'][cidx]:
# 해당 기준일 선은 상승이어야 한다.
if stock['avg'+day][upper_index] < stock['avg'+day][cidx]:
return day + "_"
return ""
def check_Dolpa_Jiji_20(self, stock):
"""
top: 이전 5일선이 20일선 위에 있을 때 최고가
top일 체크 사항 (20일 < 5일선)
5일선이 20일 선으로 내려왔다가 다시 20일선 위로 올라왔고, top < 오늘 시가 + 100
top < 시가 < 종가 라면 다음날 매수한다.
# https://docs.google.com/presentation/d/1MVuaeRNljqLCdn4dPZmvVdtl2Ab09Zwg/edit#slide=id.gc7b796e645_0_80
"""
if len(stock['close']) > 61:
if stock['avg20'][0] < stock['close'][0] and stock['avg20'][0] < stock['open'][0]:
if stock['avg5'][0] < stock['avg20'][0]:
index1 = -1
for j in range(1, 61):
if stock['avg20'][j] < stock['avg5'][j]:
index1 = j
break
top = 0
for j in range(index1+1, 61):
if stock['open'][j] < stock['close'][j]:
if top < stock['close'][j]:
top = stock['close'][j]
else:
if top < stock['open'][j]:
top = stock['open'][j]
if stock['avg5'][j] < stock['avg20'][j]:
break
return "5-20_"
return ""
def check_Danta1(self, stock):
"""
어제 상한가 혹은 상승양봉이 나온다.
오늘 상승 출발을 해야 하며 상승 음봉이 나온다
- 어제 종가 = 어제 상한가 < 종가 < 시가 < 상한가
https://docs.google.com/presentation/d/1MVuaeRNljqLCdn4dPZmvVdtl2Ab09Zwg/edit#slide=id.gc7b796e645_0_109
만약 다음날 시작초가가 오늘 종가보다 높게 상승으로 출발한다면 매수를 한다.
손절가는 오늘 최저가이다.
"""
if stock['open'][1] < stock['close'][1] == stock['high'][1]:
if stock['close'][1] < stock['close'][0] < stock['open'][0] < stock['high'][0]:
return "danta1_"
return ""
def check_Danta2(self, stock):
"""
쐐기, 수렴, 깃대 패턴 확인
# https://docs.google.com/presentation/d/1MVuaeRNljqLCdn4dPZmvVdtl2Ab09Zwg/edit#slide=id.gc7b796e645_0_144
상단 추세선을 돌파하면 매수를 한다.
"""
price_10 = round(stock["close"][0] / 10)
if stock["open"][0] < stock["close"][0]:
top = stock["close"][0]
bottom = stock["open"][0]
else:
top = stock["open"][0]
bottom = stock["close"][0]
if len(stock) > 21:
for i in range(2, 21):
if stock["open"][i] < stock["close"][i]:
if top < stock["close"][i]:
top = stock["close"][i]
if stock["open"][i] < bottom:
bottom = stock["open"][i]
else:
if top < stock["open"][i]:
top = stock["open"][i]
if stock["close"][i] < bottom:
bottom = stock["close"][i]
if top - bottom < price_10:
return "danta2_"
return ""
def check_RightArrange(self, stock):
"""
어제는 정배열이 아니었는데, 오늘은 정배열인 경우
"""
if len(stock['close']) > 2:
if (not (stock["avg120"][1] < stock["avg60"][1] < stock["avg20"][1] < stock["avg5"][1] < stock["close"][1]) and
stock["avg120"][0] < stock["avg60"][0] < stock["avg20"][0] < stock["avg5"][0] < stock["close"][0]):
if stock["avg5"][1] < stock["avg5"][0]:
return "arrange_"
return ""
def checkHigherUmbong(self, stock):
# 음봉인데 어제보다 종가가 더 높은 경우
# 이 경우 정배열 상태인지도 함께 체크를 한다.
if len(stock['close']) > 3:
# 어제는 거래량이 터진 양봉이다.
if stock['open'][1] < stock['close'][1] and 5*stock['volume'][2] < stock['volume'][1]:
# 오늘은 음봉인데, 오늘 종가는 어제 시가보다는 높다
if stock['close'][0] < stock['open'][0] and stock['open'][1] < stock['close'][0]:
return "HIGHERUMBONG_"
return ""
def check_W1Rise(self, stock, limit):
if len(stock['close']) > 5:
rate = round((stock["close"][0] - stock["close"][4]) / stock["close"][4],2)
if rate >= limit:
return "1w("+str(rate)+")_"
return ""
def check_D1Fall(self, stock, limit):
if len(stock['close']) > 2:
# 1000, 900, (900 - 1000) / 900 = -0.111
# 1000, 800, (800 - 1000) / 800 = -0.25
rate = round((stock["close"][0] - stock["close"][1]) / stock["close"][1], 2)
if rate <= limit:
return "1d("+str(rate)+")_"
return ""

106
stockpredictor_back/analysis/IchimokuCloud.py
View File

@@ -1,106 +0,0 @@
import pandas as pd
from stockpredictor.analysis.Common import Common
# graph: https://plotly.com/python/candlestick-charts/
# https://www.nanumtrading.com/fx-%EB%B0%B0%EC%9A%B0%EA%B8%B0/%EC%B0%A8%ED%8A%B8-%EB%B3%B4%EC%A1%B0%EC%A7%80%ED%91%9C-%EC%9D%B4%ED%95%B4/06-%EC%9D%BC%EB%AA%A9%EA%B7%A0%ED%98%95%ED%91%9C/
# 일목균형표 - 매매기법 알아보기 !: https://layhope.tistory.com/222
class IchimokuCloud:
common = None
def __init__(self):
self.common = Common()
return
# c=9, b=26, l=52
def apply(self, df, c=9, b=26, l=52):
# 입력받은 값이 dataframe이라는 것을 정의해줌
df = pd.DataFrame(df)
# 1. 전환선 = (과거 9일 동안 최고가 + 최저가) / 2
# 당일을 포함한 9일 동안의 최고가와 최저가의 중간 값을 평균으로 나타낸다.
changeLine = (df.high.rolling(c).max() + df.low.rolling(c).min()) / 2
# 2. 기준선 = 과거 26일 동안 최고가 + 최저가) / 2
# 당일을 포함한 26일 동안의 최고가와 최저가의 중간 값을 평균으로 나타낸다.
baseLine = (df.high.rolling(b).max() + df.low.rolling(b).min()) / 2
# 3. 선행스팬 1 = ((기준선 + 전환선) / 2)를 26일 선행하여 배치
# 전환선과 기준선의 평균값을 구해 당일 포함 26일 앞으로 이동시킨 선 (중-단기 구간의 힘을 보여줌)
leadingSpan1 = (changeLine + baseLine) / 2
move_LeadingSpan1 = list(leadingSpan1.values)
for i in range(b - 1):
move_LeadingSpan1.insert(0, None)
# 4. 선행스팬 2 = ((최근 52일 동안 최고가 + 최저가) / 2)를 26일 선행하여 배치
# 당일을 포함한 52일 동안의 최고가와 최저가의 평균을 26일 앞으로 이동시킨 선 (장기으로 형성된 선이기 때문에 가장 느리게 변함)
leadingSpan2 = (df.high.rolling(l).max() + df.low.rolling(l).min()) / 2
move_LeadingSpan2 = list(leadingSpan2.values)
for i in range(l - 1):
move_LeadingSpan2.insert(0, None)
# leadingSpan2에 맞추어 뒤로 빈 row를 채운다.
data = {"DATE": [], "close": [], "diff": [], "open": [], "high": [], "low": [], "volume": [], "avg5": [], "avg10": [], "avg20": [], "avg60": [], "avg120": [], "avg200": [], "avg240": []}
for i in range(len(move_LeadingSpan2) - len(df)):
#df = df.append({"DATE": None, "close": None, "diff": None, "open": None, "high": None, "low": None, "volume": None,"avg3": None, "avg5": None, "avg7": None, "avg10": None, "avg20": None, "avg30": None, "avg60": None,"avg90": None, "avg100": None, "avg120": None, "avg150": None, "avg180": None, "avg200": None,"avg240": None}, ignore_index=True)
data["DATE"].append(None)
data["close"].append(None)
data["diff"].append(None)
data["open"].append(None)
data["high"].append(None)
data["low"].append(None)
data["volume"].append(None)
data["avg5"].append(None)
data["avg10"].append(None)
data["avg20"].append(None)
data["avg60"].append(None)
data["avg120"].append(None)
data["avg200"].append(None)
data["avg240"].append(None)
df_temp = pd.DataFrame(data)
df = pd.concat([df, df_temp])
move_changeLine = list(changeLine.values)
for i in range(len(move_LeadingSpan2) - len(move_changeLine)):
move_changeLine.append(None)
move_baseLine = list(baseLine.values)
for i in range(len(move_LeadingSpan2) - len(move_baseLine)):
move_baseLine.append(None)
for i in range(len(move_LeadingSpan2) - len(move_baseLine)):
move_LeadingSpan1.append(None)
# dataframe에 컬럼 추가
df = df.assign(changeLine=pd.Series(move_changeLine), baseLine=pd.Series(move_baseLine), leadingSpan1=pd.Series(move_LeadingSpan1), leadingSpan2=pd.Series(move_LeadingSpan2))
return df
# 일목균형표의 구성을 훑어보면 주가를 선행과 후행으로 과거의 주가를 통해 미래 혹은 현재의 주식의 가격의 추세를 예측해보려는 지표라는 것을 이해할 수 있다.
# 또한 구름층의 색을 통해서 주식의 추세를 손쉽게 확인할 수 있을 것 같다는 것도 이해할 수 있다면 끝 !
def analyze(self, stock):
df = pd.DataFrame()
df = df.from_dict(stock['PRICE'])
df = self.apply(df)
diff = len(df.changeLine) - len(stock['PRICE'])
"""
lastDay = stock['PRICE'][len(stock['PRICE']) - 1]['ymd']
tmpLastDay = datetime.datetime.strptime(lastDay, "%Y.%m.%d")
for i in range(diff):
nextDay = tmpLastDay + datetime.timedelta(days=(i + 1))
stock['PRICE'].append(
{"ymd": nextDay.strftime("%Y-%m-%d"), "close": 0, "diff": 0, "open": 0, "high": 0, "low": 0, "volume": 0,
"avg3": 0, "avg5": 0, "avg7": 0, "avg10": 0, "avg20": 0, "avg30": 0, "avg60": 0, "avg90": 0, "avg100": 0,
"avg120": 0, "avg150": 0, "avg180": 0, "avg200": 0, "avg240": 0})
"""
for i in range(len(df.changeLine)-diff):
stock['PRICE'][i]['ichimokucloud_changeLine'] = df.changeLine.values[i]
stock['PRICE'][i]['ichimokucloud_baseLine'] = df.baseLine.values[i]
stock['PRICE'][i]['ichimokucloud_leadingSpan1'] = df.leadingSpan1.values[i]
stock['PRICE'][i]['ichimokucloud_leadingSpan2'] = df.leadingSpan2.values[i]
return
if __name__ == "__main__":
ichimokuCloud = IchimokuCloud()

81
stockpredictor_back/analysis/MACD.py
View File

@@ -1,81 +0,0 @@
import pandas as pd
from stockpredictor.analysis.Common import Common
# [청송촌놈] 파생을 알아야 시장이 보인다. 청송이 종목 고르는법! https://www.youtube.com/watch?v=weABtgZDeGg
# 6. Pandas와 Plotly를 이용한 MACD 차트 그리기 https://excelsior-cjh.tistory.com/110
# 첫번째. MACD 지표를 이용한 차트분석: https://post.naver.com/viewer/postView.nhn?volumeNo=7435935&memberNo=32471429
# MACD (Moving Average Conver gence Divergence)
# 빨간 네모박스권으로 MACD가 MACD-Sign 을 골든크로스하며, 상승하였을때, 주가는 상승추세를 유지하며, MACD가 MACD-Sign(분홍색)을 데드크로스 할때 주가는 하락의 추세를 보이게 됩니다.
# 즉, MSCD가 0이상에서 MACD-Sign 위에서 상승하는 그림이어야
class MACD:
common = None
def __init__(self):
self.common = Common()
return
# macd 0선 위에서 매수를 한다. 0이하는 절대 처다보지 않는다.
def apply(self, df, short=12, long=26, t=9):
# 입력받은 값이 dataframe이라는 것을 정의해줌
df = pd.DataFrame(df)
# MACD 관련 수식
ma_12 = df.close.ewm(span=short).mean() # 단기(12) EMA(지수이동평균)
ma_26 = df.close.ewm(span=long).mean() # 장기(26) EMA
macd = ma_12 - ma_26 # MACD
macds = macd.ewm(span=t).mean() # Signal
macdo = macd - macds # Oscillator
#df = df.assign(macd=macd, macds=macds, macdo=macdo).dropna()
df = df.assign(macd=macd, macds=macds, macdo=macdo)
return df
"""
# 기존 stock에 삽입
# macd 0선 위에서 매수를 한다. 0이하는 절대 처다보지 않는다.
def analyze(self, stock):
df = pd.DataFrame()
df = df.from_dict(stock['PRICE'])
df = self.apply(df)
for i in range(len(df.macd)):
stock['PRICE'][i]['macd_buy'] = 0
stock['PRICE'][i]['macd'] = df.macd.values[i]
stock['PRICE'][i]['macds'] = df.macds.values[i]
stock['PRICE'][i]['macdo'] = df.macdo.values[i]
if df.macd.values[i] > 0 and self.common.checkUpward('close', stock['PRICE'][i - 4: i + 1]):
stock['PRICE'][i]['macd_buy'] = df.macd.values[i]
return
"""
def analyze(self, stock):
results = []
df = pd.DataFrame()
df = df.from_dict(stock['PRICE'])
df = self.apply(df)
for i in range(len(df.macd)):
result = {'DATE':stock['PRICE'][i]['DATE'],
'macd': df.macd.values[i],
'macds': df.macds.values[i],
'macdo': df.macdo.values[i],
'macd_buy': 0}
# MACD가 3일 전부터 상승이라면 매수
if df.macd.values[i - 1] < df.macd.values[i]:
# 어제는 MACD-Sign이 MACD 위에 있지만, 오늘은 MACD가 MACD-Sign 위로 올라오면 매수
if df.macd.values[i-1] < df.macds.values[i-1] and df.macd.values[i] > df.macds.values[i]:
result['macd_buy'] = 1
# MACD가 3일 전부터 하락이라면 매도
if df.macd.values[i - 1] > df.macd.values[i]:
# 어제는 MACD가 MACD-Sign 위에 있지만, 오늘은 MACD-Sign이 MACD 위로 올라오면 매도
if df.macd.values[i - 1] > df.macds.values[i - 1] and df.macd.values[i] < df.macds.values[i]:
result['macd_buy'] = -1
results.append(result)
return results

55
stockpredictor_back/analysis/RSI.py
View File

@@ -1,55 +0,0 @@
import pandas as pd
from stockpredictor.analysis.Common import Common
import numpy as np
# [청송촌놈] 파생을 알아야 시장이 보인다. 청송이 종목 고르는법! https://www.youtube.com/watch?v=weABtgZDeGg
# 6. Pandas와 Plotly를 이용한 MACD 차트 그리기 https://excelsior-cjh.tistory.com/110
# 첫번째. MACD 지표를 이용한 차트분석: https://post.naver.com/viewer/postView.nhn?volumeNo=7435935&memberNo=32471429
# MACD (Moving Average Conver gence Divergence)
# 빨간 네모박스권으로 MACD가 MACD-Sign 을 골든크로스하며, 상승하였을때, 주가는 상승추세를 유지하며, MACD가 MACD-Sign(분홍색)을 데드크로스 할때 주가는 하락의 추세를 보이게 됩니다.
# 즉, MSCD가 0이상에서 MACD-Sign 위에서 상승하는 그림이어야
class RSI:
common = None
def __init__(self):
self.common = Common()
return
def apply(sefl, df, period=14, window=9):
df = pd.DataFrame(df)
# df.diff를 통해 (기준일 종가 - 기준일 전일 종가)를 계산하여 0보다 크면 증가분을 감소했으면 0을 넣어줌
U = np.where(df.close.diff(1) > 0, df.close.diff(1), 0)
# df.diff를 통해 (기준일 종가 - 기준일 전일 종가)를 계산하여 0보다 작으면 감소분을 증가했으면 0을 넣어줌
D = np.where(df.close.diff(1) < 0, df.close.diff(1) * (-1), 0)
# AU, period=14일 동안의 U의 평균
AU = pd.DataFrame(U).rolling(window=period, min_periods=period).mean()
# AD, period=14일 동안의 D의 평균
AD = pd.DataFrame(D).rolling(window=period, min_periods=period).mean()
rsi = AU.div(AD + AU) * 100
rsis = rsi.rolling(window=window).mean()
df = df.assign(rsi=rsi, rsis=rsis)
return df
def analyze(self, stock):
"""
RSI 값이 100에 접근하면 ㄷ 이상의 주가 상승을 기대하기 어렵고, 0에 접근하면 더 이상 하락을 기대하기 어렵다.
70이상이면 과매수 구간이라 할 수 있고, 30 이하면 과매도 구간이라 볼 수 있다.
따라서 과매수 구간에서는 매도 준비를, 과매도 구간에서는 매수 준비를 해야 한다.
"""
df = pd.DataFrame()
df = df.from_dict(stock['PRICE'])
df = self.apply(df)
for i in range(len(df.rsi)):
stock['PRICE'][i]['rsi'] = df.rsi.values[i]
stock['PRICE'][i]['rsis'] = df.rsis.values[i]

147
stockpredictor_back/analysis/Stochastic.py
View File

@@ -1,147 +0,0 @@
import pandas as pd
from stockpredictor.analysis.Common import Common
# 6. Pandas와 Plotly를 이용한 MACD 차트 그리기: https://excelsior-cjh.tistory.com/111
# 스토캐스틱 슬로우(Stochastics Slow)를 이용한 간단한 매매기법: https://bagal.tistory.com/124
# Stochastic Slow, 스토캐스틱을 통한 주식 매매법 정리: https://m.blog.naver.com/PostView.nhn?blogId=kangyh427&logNo=220957146041&proxyReferer=https:%2F%2Fwww.google.com%2F
# 차트분석 2편 스토캐스틱을 활용한 매수타이밍 포착!: https://m.post.naver.com/viewer/postView.nhn?volumeNo=7446693&memberNo=32471429
class Stochastic:
common = None
def __init__(self):
self.common = Common()
return
# 일자(n,m,t)에 따른 Stochastic(KDJ)의 값을 구하기 위해 함수형태로 만듬
# n=15 (%k), m=5 (%d), t=3
# n=5 (%k), m=3 (%d), t=3
# n=14 (%k), m=3 (%d), t=3
def apply(self, df, n=12, m=5, t=5):
# 입력받은 값이 dataframe이라는 것을 정의해줌
df = pd.DataFrame(df)
# n일중 최고가
ndays_high = df.high.rolling(n).max()
# n일중 최저가
ndays_low = df.low.rolling(n).min()
# Fast%K 계산
# n(15)일 동안의 최고가(high)와 최저가(low) 사이 중 현재 종가(close)의 상대적 위치를 판단하는 값
# 가격이 지속적으로 상승하고 있다면 Stochastic 값은 100에 가까워 지며, 반대로 지속적으로 하락하고 있다면 Stochastic 값은 0에 가까워 지는 경향을 나타낸다.
fast_k = ((df.close - ndays_low) / (ndays_high - ndays_low)) * 100
# Fast%D (=Slow%K) 계산
# m(5)일간의 Fast%K의 이동평균 값
# 기본값으로 5일을 설정하며 Fast%K 값을 일반화하는 역할을 한다.
#slow_k = fast_k.ewm(span=m).mean()
slow_k = fast_k.rolling(m).mean()
# Slow%D 계산
# t(3)일간의 Slow%K의 이동평균 값
# 기본값으로 3일을 설정하며 Slow%K 값을 일반화 하는 역할을 한다.
#slow_d = slow_k.ewm(span=t).mean()
slow_d = slow_k.rolling(t).mean()
# dataframe에 컬럼 추가
#df = df.assign(fast_k=fast_k, slow_k=slow_k, slow_d=slow_d).dropna()
df = df.assign(fast_k=fast_k, slow_k=slow_k, slow_d=slow_d)
return df
"""
def analyze(self, stock):
df = pd.DataFrame()
df = df.from_dict(stock['PRICE'])
df = self.apply(df)
for i in range(len(df.fast_k)):
stock['PRICE'][i]['stochastic_buy'] = 0
if "fast_k" not in stock['PRICE'][i]:
stock['PRICE'][i]['fast_k'] = -1
stock['PRICE'][i]['slow_k'] = -1
stock['PRICE'][i]['slow_d'] = -1
i_ = i + len(stock['PRICE']) - len(df.fast_k)
stock['PRICE'][i_]['fast_k'] = df.fast_k.values[i]
stock['PRICE'][i_]['slow_k'] = df.slow_k.values[i]
stock['PRICE'][i_]['slow_d'] = df.slow_d.values[i]
# 0: 중립, 1: 매수, -1: 매도
stock['PRICE'][i_]['stochastic_buy'] = 0
stock['PRICE'][i_]['direction'] = ''
if i_ > 2:
if ('slow_k' in stock['PRICE'][i_-1] and 'slow_k' in stock['PRICE'][i_]):
# %k선이 %d선을 상향 돌파하면 매수 신호
if (self.common.checkUpward('slow_k', stock['PRICE'][i_-2:i_+1]) and
stock['PRICE'][i_-1]['slow_k'] < stock['PRICE'][i_-1]['slow_d'] and
stock['PRICE'][i_]['slow_k'] > stock['PRICE'][i_]['slow_d']):
if stock['PRICE'][i_]['slow_k'] < 5:
stock['PRICE'][i_]['stochastic_buy'] = 8
elif 5 <= stock['PRICE'][i_]['slow_k'] < 10:
stock['PRICE'][i_]['stochastic_buy'] = 7
elif 10 <= stock['PRICE'][i_]['slow_k'] < 15:
stock['PRICE'][i_]['stochastic_buy'] = 6
elif 15 <= stock['PRICE'][i_]['slow_k'] < 20:
stock['PRICE'][i_]['stochastic_buy'] = 5
elif 20 <= stock['PRICE'][i_]['slow_k'] < 30:
stock['PRICE'][i_]['stochastic_buy'] = 4
elif 30 <= stock['PRICE'][i_]['slow_k'] < 40:
stock['PRICE'][i_]['stochastic_buy'] = 3
elif 40 <= stock['PRICE'][i_]['slow_k'] < 50:
stock['PRICE'][i_]['stochastic_buy'] = 2
else:
stock['PRICE'][i_]['stochastic_buy'] = 1
if self.common.checkUpward('slow_k', stock['PRICE'][i_ - 1:i_ + 1]):
stock['PRICE'][i_]['direction'] = 'up'
if (self.common.checkDownward('slow_k', stock['PRICE'][i_ - 2:i_ + 1]) and
stock['PRICE'][i_-1]['slow_k'] > stock['PRICE'][i_-1]['slow_d'] and
stock['PRICE'][i_]['slow_k'] < stock['PRICE'][i_]['slow_d']):
if stock['PRICE'][i_]['slow_k'] > 90:
stock['PRICE'][i_]['stochastic_buy'] = -6
elif 90 >= stock['PRICE'][i_]['slow_k'] > 80:
stock['PRICE'][i_]['stochastic_buy'] = -5
elif 80 >= stock['PRICE'][i_]['slow_k'] > 70:
stock['PRICE'][i_]['stochastic_buy'] = -4
elif 70 >= stock['PRICE'][i_]['slow_k'] > 60:
stock['PRICE'][i_]['stochastic_buy'] = -3
elif 60 >= stock['PRICE'][i_]['slow_k'] > 50:
stock['PRICE'][i_]['stochastic_buy'] = -2
else:
stock['PRICE'][i_]['stochastic_buy'] = -1
if self.common.checkDownward('slow_k', stock['PRICE'][i_ - 1:i_ + 1]):
stock['PRICE'][i_]['direction'] = 'down'
results = []
for day in stock['PRICE']:
results.append({'DATE': day['DATE'],
'fast_k': day['fast_k'],
'slow_k': day['slow_k'],
'slow_d': day['slow_d'],
'stochastic_buy': day['stochastic_buy']})
return results
"""
def analyze(self, stock):
df = pd.DataFrame()
df = df.from_dict(stock['PRICE'])
df = self.apply(df)
for i in range(len(df.fast_k)):
if "stochastic_fast_k" not in stock['PRICE'][i]:
stock['PRICE'][i]['stochastic_fast_k'] = -1
stock['PRICE'][i]['stochastic_slow_k'] = -1
stock['PRICE'][i]['stochastic_slow_d'] = -1
stock['PRICE'][i]['stochastic_fast_k'] = df.fast_k.values[i]
stock['PRICE'][i]['stochastic_slow_k'] = df.slow_k.values[i]
stock['PRICE'][i]['stochastic_slow_d'] = df.slow_d.values[i]
return
if __name__ == "__main__":
stochastic = Stochastic()

2
stockpredictor_back/crawler/pandas/Crawler.py
View File

@@ -1,2 +0,0 @@
# https://grand-unified-engine.tistory.com/21
# https://github.com/FinanceData/FinanceDataReader

74
stockpredictor_back/crawler/sQLite/Crawler.py
View File

@@ -1,74 +0,0 @@
import os
import shutil
import datetime
import time
from stockpredictor.crawler.sQLite.FnGuideCrawler import FnGuideCrawler
from stockpredictor.crawler.sQLite.MetaCrawler import MetaCrawler
from stockpredictor.crawler.sQLite.StockCrawler import StockCrawler
from stockpredictor.analysis.AnalyzerSqlite import AnalyzerSqlite
today = datetime.datetime.now().strftime("%Y-%m-%d")
# DB Browser for SQLite: http://hleecaster.com/python-sqlite3/
PROJECT_HOME = os.path.join(os.path.dirname(os.path.join(os.path.dirname(os.path.join(os.path.dirname(os.path.join(os.path.dirname(__file__))))))))
START_DATE = "1900.01.01"
start = time.time()
stockFileName = PROJECT_HOME + '/resources/stock.db'
# 재무제표는 3개월마다 다운로드를 한다.
fnGuideCrawler = FnGuideCrawler(START_DATE)
print("[KOSPI 상장기업 재무제표 다운로드]")
fnGuideCrawler.crawl_fnguide(stockFileName)
metaCrawler = MetaCrawler(START_DATE)
print("\n[증시자금동향 (신용잔고, 펀드자금 잔고)]")
metaCrawler.crawl_money_trend(stockFileName)
print("\n[국내 시장금리]")
metaCrawler.crawl_interest_rates(stockFileName)
print("\n[투자자별 매매동향(Trading_Trend)]")
metaCrawler.crawl_trading_trend(stockFileName)
print("\n[환율 (USD, JPY, EUR, CNY)]")
metaCrawler.crawl_exchange(stockFileName)
print("\n[원유 (WTI), 국제금, COPPER, NATURALGAS, CORN, SOYBEAN]")
metaCrawler.crawl_meterials(stockFileName)
print("\n[종목 다운로드]")
stockCrawler = StockCrawler(START_DATE)
stockCrawler.crawl_etf_stocks(stockFileName)
stockCrawler.crawl_stocks(stockFileName)
#stockCrawler.crawl_special_stocks(stockFileName)
print("\n[종목 분석]")
# S: 분석까지 진행
inFileName = PROJECT_HOME + '/resources/stock.db'
analyzerSqlite = AnalyzerSqlite(PROJECT_HOME, stockFileName)
analyzerSqlite.analyzeDaily()
analyzerSqlite.analyzeGrouping("weekly")
analyzerSqlite.analyzeGrouping("monthly")
analyzerSqlite = AnalyzerSqlite(PROJECT_HOME, stockFileName)
print("\n[종목 결정]")
day = datetime.datetime.today().strftime("%Y%m%d")
outPath = PROJECT_HOME + "/resources/analysis/" + day
if os.path.isdir(outPath):
shutil.rmtree(outPath)
os.mkdir(outPath)
print("print to Html...")
analyzerSqlite.findCandidate(outPath)
# E: 분석까지 진행
print("time : %6.2f 초", (time.time() - start))
print ("done...")

223
stockpredictor_back/crawler/sQLite/FnGuideCrawler.py
View File

@@ -1,223 +0,0 @@
from bs4 import BeautifulSoup
from pandas import DataFrame, Series
import requests as re
import pandas as pd
import os
import json
import sqlite3
import requests
class FnGuideCrawler:
header = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36'}
START_DATE = None
def __init__(self, START_DATE):
self.START_DATE = START_DATE
return
def getStockInfo(self):
code_df = pd.read_html('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', header=0)[0]
#code_df = pd.read_html(requests.get('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', headers=self.header).text)
# 종목코드가 6자리이기 때문에 6자리를 맞춰주기 위해 설정해줌
code_df.종목코드 = code_df.종목코드.map('{:06d}'.format)
# 우리가 필요한 것은 회사명과 종목코드이기 때문에 필요없는 column들은 제외해준다.
code_df = code_df[['회사명', '종목코드']]
# 한글로된 컬럼명을 영어로 바꿔준다.
code_df = code_df.rename(columns={'회사명': 'name', '종목코드': 'code'})
###print (code_df.head())
return code_df
# FnGuide에서 크롤링한 KOSPI 상장기업의 재무제표
# http://blog.naver.com/PostView.nhn?blogId=koko8624&logNo=221294884955&parentCategoryNo=&categoryNo=&viewDate=&isShowPopularPosts=false&from=postView
def get_fnguide_table(self, code):
url = re.get('http://comp.fnguide.com/SVO2/ASP/SVD_main.asp?pGB=1&gicode=A%s'%(code.strip()))
url = url.content
html = BeautifulSoup(url,'html.parser')
body = html.find('body')
try:
fn_body = body.find('div', {'class': 'fng_body asp_body'})
ur_table = fn_body.find('div', {'id': 'div15'})
table = ur_table.find('div', {'id': 'highlight_D_Y'})
tbody = table.find('tbody')
tr = tbody.find_all('tr')
Table = DataFrame()
except:
return {}
for i in tr:
''' 자료 항목 가져오기'''
category = i.find('span', {'class': 'txt_acd'})
if category == None:
category = i.find('th')
category = category.text.strip()
'''값 가져오기'''
value_list = []
j = i.find_all('td', {'class': 'r'})
for value in j:
temp = value.text.replace(',', '').strip()
try:
temp = float(temp)
value_list.append(temp)
except:
value_list.append(0)
Table['%s' % (category)] = value_list
''' 기간 가져오기 '''
thead = table.find('thead')
tr_2 = thead.find('tr', {'class': 'td_gapcolor2'}).find_all('th')
year_list = []
for i in tr_2:
try:
temp_year = i.find('span', {'class': 'txt_acd'}).text
except:
temp_year = i.text
temp_year = temp_year.replace("/",".")+".01"
year_list.append(temp_year)
Table.index = year_list
return Table.T.to_dict()
def crawl_fnguide(self, inFileName):
tableName = 'fnguide'
conn = sqlite3.connect(inFileName, isolation_level=None)
cursor = conn.cursor()
# 테이블 생성
create_sql = "CREATE TABLE IF NOT EXISTS "+tableName+" (CODE text, NAME text, ymd text, "
create_sql += " type text, sales REAL, net_business_profits REAL, business_profits REAL, business_profits_release REAL, "
create_sql += " net_profit REAL, significant_shareholder_net_profit REAL, "
create_sql += " none_significant_shareholder_net_profit REAL, total_assets REAL, total_debt REAL, "
create_sql += " total_ownership_interest REAL, equity_holdings REAL, none_equity_holdings REAL, capital REAL, "
create_sql += " debt_ratio REAL, reserve_ratio REAL, business_profits_ratio REAL, "
create_sql += " significant_shareholder_profits_ratio REAL, ROA REAL, ROE REAL, EPS REAL, BPS REAL, DPS REAL, "
create_sql += " PER REAL, PBR REAL, share_outstanding REAL, dividend_rate REAL)"
cursor.execute(create_sql)
# 키 생성
create_key = "CREATE INDEX IF NOT EXISTS "+tableName+"_idx on "+tableName+" (CODE, ymd) "
cursor.execute(create_key)
code_df = self.getStockInfo()
idx = 0
for item in code_df.values:
item_name = item[0]
item_code = item[1]
idx += 1
print(idx, item_code, item_name, 'http://comp.fnguide.com/SVO2/ASP/SVD_main.asp?pGB=1&gicode=A%s'%(item_code.strip()))
fnGuideData = self.get_fnguide_table(item_code)
"""
매출액: sales
순영업수익: net_business_profits REAL,
영업이익: business_profits
영업이익(발표기준): business_profits_release
당기순이익: net_profit
지배주주순이익: significant_shareholder_net_profit
비지배주주순이익: none_significant_shareholder_net_profit
자산총계: total_assets
부채총계: total_debt
자본총계: total_ownership_interest
지배주주지분: equity_holdings
비지배주주지분: none_equity_holdings
자본금: capital
부채비율: debt_ratio
유보율: reserve_ratio
영업이익률: business_profits_ratio
지배주주순이익률: significant_shareholder_profits_ratio
ROA: ROA
ROE: ROE
EPS(원): EPS
BPS(원): BPS
DPS(원): DPS
PER: PER
PBR: PBR
발행주식수: share_outstanding
배당수익률: dividend_rate
"""
for key_ymd in fnGuideData:
ymd = key_ymd.replace('(P)', '').replace('(E)', '')
if key_ymd.find('P') > 0:
type = 'P'
elif key_ymd.find('E') > 0:
type = 'E'
else:
type = ''
if '매출액' in fnGuideData[key_ymd]:
sales = fnGuideData[key_ymd]['매출액']
else:
sales = 0
if '순영업수익' in fnGuideData[key_ymd]:
net_business_profits = fnGuideData[key_ymd]['순영업수익']
else:
net_business_profits = 0
business_profits = fnGuideData[key_ymd]['영업이익']
business_profits_release = fnGuideData[key_ymd]['영업이익(발표기준)']
net_profit = fnGuideData[key_ymd]['당기순이익']
significant_shareholder_net_profit = fnGuideData[key_ymd]['지배주주순이익']
none_significant_shareholder_net_profit = fnGuideData[key_ymd]['비지배주주순이익']
total_assets = fnGuideData[key_ymd]['자산총계']
total_debt = fnGuideData[key_ymd]['부채총계']
total_ownership_interest = fnGuideData[key_ymd]['자본총계']
equity_holdings = fnGuideData[key_ymd]['지배주주지분']
none_equity_holdings = fnGuideData[key_ymd]['비지배주주지분']
capital = fnGuideData[key_ymd]['자본금']
debt_ratio = fnGuideData[key_ymd]['부채비율']
reserve_ratio = fnGuideData[key_ymd]['유보율']
business_profits_ratio = fnGuideData[key_ymd]['영업이익률']
significant_shareholder_profits_ratio = fnGuideData[key_ymd]['지배주주순이익률']
ROA = fnGuideData[key_ymd]['ROA']
ROE = fnGuideData[key_ymd]['ROE']
if 'EPS(원)' in fnGuideData[key_ymd]:
EPS = fnGuideData[key_ymd]['EPS(원)']
else:
EPS = fnGuideData[key_ymd]['EPS']
if 'BPS(원)' in fnGuideData[key_ymd]:
BPS = fnGuideData[key_ymd]['BPS(원)']
else:
BPS = fnGuideData[key_ymd]['BPS']
if 'DPS(원)' in fnGuideData[key_ymd]:
DPS = fnGuideData[key_ymd]['DPS(원)']
else:
DPS = fnGuideData[key_ymd]['DPS']
PER = fnGuideData[key_ymd]['PER']
PBR = fnGuideData[key_ymd]['PBR']
share_outstanding = fnGuideData[key_ymd]['발행주식수']
dividend_rate = fnGuideData[key_ymd]['배당수익률']
cursor.execute('SELECT * FROM '+tableName+' WHERE CODE=? and ymd=?', (item_code, ymd))
result = cursor.fetchone()
if result == None:
cursor.execute("INSERT INTO "+tableName+"(CODE, NAME, ymd, type, sales, net_business_profits, business_profits, business_profits_release, net_profit, significant_shareholder_net_profit, none_significant_shareholder_net_profit, total_assets, total_debt, total_ownership_interest, equity_holdings, none_equity_holdings, capital, debt_ratio, reserve_ratio, business_profits_ratio, significant_shareholder_profits_ratio, ROA, ROE, EPS, BPS, DPS, PER, PBR, share_outstanding, dividend_rate) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (item_code, item_name, ymd, type, sales, net_business_profits, business_profits, business_profits_release, net_profit, significant_shareholder_net_profit, none_significant_shareholder_net_profit, total_assets, total_debt, total_ownership_interest, equity_holdings, none_equity_holdings, capital, debt_ratio, reserve_ratio, business_profits_ratio, significant_shareholder_profits_ratio, ROA, ROE, EPS, BPS, DPS, PER, PBR, share_outstanding, dividend_rate))
else:
# cursor.execute("UPDATE "+tableName+" SET ymd=?, type=?, sales=?, net_business_profits=?, business_profits=?, business_profits_release=?, net_profit=?, significant_shareholder_net_profit=?, none_significant_shareholder_net_profit=?, total_assets=?, total_debt=?, total_ownership_interest=?, equity_holdings=?, none_equity_holdings=?, capital=?, debt_ratio=?, reserve_ratio=?, business_profits_ratio=?, significant_shareholder_profits_ratio=?, ROA=?, ROE=?, EPS=?, BPS=?, DPS=?, PER=?, PBR=?, share_outstanding=?, dividend_rate=? WHERE CODE=?", (ymd, type, sales, net_business_profits, business_profits, business_profits_release, net_profit, significant_shareholder_net_profit, none_significant_shareholder_net_profit, total_assets, total_debt, total_ownership_interest, equity_holdings, none_equity_holdings, capital, debt_ratio, reserve_ratio, business_profits_ratio, significant_shareholder_profits_ratio, ROA, ROE, EPS, BPS, DPS, PER, PBR, share_outstanding, dividend_rate, item_code))
break
cursor.close()
conn.close()
return
if __name__ == "__main__":
PROJECT_HOME = os.path.join(os.path.dirname(os.path.join(os.path.dirname(os.path.join(os.path.dirname(os.path.join(os.path.dirname(__file__))))))))
inFnguideFileName = PROJECT_HOME + '/resources/stock.db'
crawler = FnGuideCrawler()
crawler.crawl_fnguide(inFnguideFileName)

460
stockpredictor_back/crawler/sQLite/MetaCrawler.py
View File

@@ -1,460 +0,0 @@
import os
import datetime
import requests
import sqlite3
from time import sleep
import pandas as pd
class MetaCrawler:
header = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36'}
limit_page_count = 1000000
START_DATE = None
def __init__(self, START_DATE):
self.START_DATE = START_DATE
return
# 참고) http://blog.naver.com/PostView.nhn?blogId=koko8624&logNo=221288761509
def crawl_exchange(self, inFileName):
tableName = 'meta_1'
conn = sqlite3.connect(inFileName)
cursor = conn.cursor()
# 테이블 생성
cursor.execute("CREATE TABLE IF NOT EXISTS "+tableName+" (CODE text, NAME text, ymd text, price REAL, diff REAL, cash_buy REAL, cash_sell REAL, transfer_buy REAL, transfer_sell REAL)")
# 키 생성
create_key = "CREATE INDEX IF NOT EXISTS " + tableName + "_idx on " + tableName + " (CODE, ymd) "
cursor.execute(create_key)
inputs = []
inputs.append( {'NAME':'USD', 'CODE':'FX_USDKRW', 'URL':'http://finance.naver.com/marketindex/exchangeDailyQuote.nhn?marketindexCd=FX_USDKRW'} ) # 미국 USD
inputs.append( {'NAME':'JPY', 'CODE':'FX_JPYKRW', 'URL':'http://finance.naver.com/marketindex/exchangeDailyQuote.nhn?marketindexCd=FX_JPYKRW'} ) # 일본 JPY
inputs.append( {'NAME':'EUR', 'CODE':'FX_EURKRW', 'URL':'http://finance.naver.com/marketindex/exchangeDailyQuote.nhn?marketindexCd=FX_EURKRW'} ) # 유럽연합 EUR'
inputs.append( {'NAME':'CNY', 'CODE':'FX_CNYKRW', 'URL':'http://finance.naver.com/marketindex/exchangeDailyQuote.nhn?marketindexCd=FX_CNYKRW'} ) # 중국 CNY
for i in range(len(inputs)):
input = inputs[i]
NAME = input['NAME']
CODE = input['CODE']
cursor.execute('SELECT ymd FROM ' + tableName + ' WHERE CODE=? order by ymd desc', (CODE,))
result = cursor.fetchone()
if result == None:
lastDay = self.START_DATE
else:
lastDay = result[0]
finish = False
for i in range(1, self.limit_page_count):
#html = pd.read_html(input['URL'] + '&page=%s' % i, header=0)
html = None
while True:
try:
html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header, timeout=30).text)
sleep(0.5)
break
except:
print(input['URL'] + '&page=%s' % i)
if i>200:
break
continue
# 마지막 페이지 까지 받기
if len(html[0].날짜.values) <= 1:
break
for j in range(0, len(html[0].values)):
item = html[0].values[j]
if input['NAME'] in ('USD', 'JPY', 'EUR', 'CNY'):
if item[0] <= lastDay:
finish = True
break
ymd = item[0] # 날짜
price = item[1] # 매매기준율
diff = item[2] # 전일대비
cash_buy = item[3] # 현찰 사실 때
cash_sell = item[4] # 현찰 파실 때
transfer_buy = item[5] # 송금 사실 때
transfer_sell = item[6] # 송금 파실 때
cursor.execute('SELECT * FROM ' + tableName + ' WHERE CODE=? and ymd=?', (CODE,ymd,))
result = cursor.fetchone()
if result == None:
cursor.execute("INSERT INTO " + tableName + "(CODE, NAME, ymd, price, diff, cash_buy, cash_sell, transfer_buy, transfer_sell) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)", (CODE, NAME, ymd, price, diff, cash_buy, cash_sell, transfer_buy, transfer_sell))
else:
# cursor.execute("UPDATE " + tableName + " SET price=?, diff=?, cash_buy=?, cash_sell=?, transfer_buy=?, transfer_sell=? WHERE CODE=? and ymd=?", (price, diff, cash_buy, cash_sell, transfer_buy, transfer_sell, CODE, ymd))
finish = True
break
print(CODE, NAME, ymd)
if finish:
break
conn.commit()
cursor.close()
conn.close()
return
# 투자자별 매매동향 (Trading_Trend) 크롤링
# (pri, 개인)
# (for, 외국인)
# (ins, 기관합)
# (ins0, 금융투자)
# (ins1, 보험)
# (ins2, 투신 (사모))
# (ins3, 은행)
# (ins4, 기타금융기관)
# (ins5, 연기금 등)
# (cor, 기타법인)
# 참고) http://blog.naver.com/PostView.nhn?blogId=koko8624&logNo=221289696771&parentCategoryNo=&categoryNo=&viewDate=&isShowPopularPosts=false&from=postView
def crawl_trading_trend(self, inFileName):
tableName = 'meta_2'
conn = sqlite3.connect(inFileName)
cursor = conn.cursor()
# 테이블 생성
cursor.execute("CREATE TABLE IF NOT EXISTS "+tableName+" (ymd text PRIMARY KEY, pri integer, fori integer, ins integer, ins0 integer, ins1 integer, ins2 integer, ins3 integer, ins4 integer, ins5 integer, cor integer)")
# 키 생성
create_key = "CREATE INDEX IF NOT EXISTS "+tableName+"_idx on "+tableName+" (ymd) "
cursor.execute(create_key)
cursor.execute('SELECT ymd FROM ' + tableName + ' order by ymd desc')
result = cursor.fetchone()
if result == None:
lastDay = self.START_DATE
else:
lastDay = result[0]
today = datetime.datetime.now().strftime("%Y%m%d")
url = 'http://finance.naver.com/sise/investorDealTrendDay.nhn?bizdate='+today+'&sosok=&page='
previousDay = ""
finish = False
for i in range(1, self.limit_page_count):
#html = pd.read_html(url + str(i), header=0)
html = None
while True:
try:
html = pd.read_html(requests.get(url + str(i), headers=self.header, timeout=30).text)
sleep(0.5)
break
except:
print(url + str(i))
if i > 200:
break
continue
# 마지막 페이지 까지 받기
if len(html[0].날짜.values) <= 2:
break
for j in range(0, len(html[0].values)):
item = html[0].values[j]
if str(item[0]) == "nan":
continue
if "20" + item[0] <= lastDay or item[0] == previousDay:
finish = True
break
ymd = "20"+item[0]
pri = item[1] # 개인
fori = item[2] # 외국인
ins = item[3] # 기관합
ins0 = item[4] # 금융투자
ins1 = item[5] # 보험
ins2 = item[6] # 투신 (사모)
ins3 = item[7] # 은행
ins4 = item[8] # 기타금융기관
ins5 = item[9] # 연기금 등
cor = item[10] # 기타법인
cursor.execute('SELECT * FROM ' + tableName + ' WHERE ymd=?', (ymd,))
result = cursor.fetchone()
if result == None:
cursor.execute("INSERT INTO " + tableName + "(ymd, pri, fori, ins, ins0, ins1, ins2, ins3, ins4, ins5, cor) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (ymd, pri, fori, ins, ins0, ins1, ins2, ins3, ins4, ins5, cor))
else:
# cursor.execute("UPDATE " + tableName + " SET pri=?, fori=?, ins=?, ins0=?, ins1=?, ins2=?, ins3=?, ins4=?, ins5=?, cor=? WHERE ymd=?", (pri, fori, ins, ins0, ins1, ins2, ins3, ins4, ins5, cor, ymd))
finish = True
break
print ("20"+item[0])
previousDay = html[0].values[2][0]
if finish:
break
conn.commit()
cursor.close()
conn.close()
return
# 증시자금동향 (신용잔고, 펀드자금 잔고) 크롤링
# 참고) http://blog.naver.com/PostView.nhn?blogId=koko8624&logNo=221290138187&parentCategoryNo=&categoryNo=&viewDate=&isShowPopularPosts=false&from=postView
def crawl_money_trend(self, inFileName):
tableName = 'meta_3'
conn = sqlite3.connect(inFileName)
cursor = conn.cursor()
cursor.execute("CREATE TABLE IF NOT EXISTS "+tableName+" (ymd text, dep1_1 integer, dep1_2 integer, dep2_1 integer, dep2_2 integer, dep3_1 integer, dep3_2 integer, dep4_1 integer, dep4_2 integer, dep5_1 integer, dep5_2 integer)")
# 키 생성
create_key = "CREATE INDEX IF NOT EXISTS "+tableName+"_idx on "+tableName+" (ymd) "
cursor.execute(create_key)
cursor.execute('SELECT ymd FROM ' + tableName + ' order by ymd desc')
result = cursor.fetchone()
if result == None:
lastDay = self.START_DATE
else:
lastDay = result[0]
previousDay = ""
url = 'http://finance.naver.com/sise/sise_deposit.nhn?&page='
finish = False
for i in range(1, self.limit_page_count):
#html = pd.read_html(url + str(i), header=0, encoding='euc-kr')
html = None
while True:
try:
html = pd.read_html(requests.get(url + str(i), headers=self.header, timeout=30).text, encoding='euc-kr')
sleep(0.5)
break
except:
print(url + str(i))
if i > 200:
break
continue
# 마지막 페이지 까지 받기
if len(html[0].날짜.values) <= 10:
break
for j in range(0, len(html[0].values)):
item = html[0].values[j]
if str(item[0]) == "nan":
continue
if "20"+item[0] <= lastDay or item[0] == previousDay:
finish = True
break
meta = {
"ymd": "20"+item[0],
"dep1_1": item[1], # 고객예탁금 누적
"dep1_2": item[2], # 고객예탁금 당일
"dep2_1": item[3], # 신용잔고 누적
"dep2_2": item[4], # 신용잔고 당일
"dep3_1": item[5], # 주식형펀드 누적
"dep3_2": item[6], # 주식형펀드 당일
"dep4_1": item[7], # 혼합형펀드 누적
"dep4_2": item[8], # 혼합형펀드 당일
"dep5_1": item[9], # 채권형펀드 누적
"dep5_2": item[10]} # 채권형펀드 당일
cursor.execute('SELECT * FROM ' + tableName + ' WHERE ymd=?', (meta["ymd"],))
result = cursor.fetchone()
if result == None:
cursor.execute("INSERT INTO " + tableName + "(ymd, dep1_1, dep1_2, dep2_1, dep2_2, dep3_1, dep3_2, dep4_1, dep4_2, dep5_1, dep5_2) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (meta["ymd"], meta["dep1_1"], meta["dep1_2"], meta["dep2_1"], meta["dep2_2"], meta["dep3_1"], meta["dep3_2"], meta["dep4_1"], meta["dep4_2"], meta["dep5_1"], meta["dep5_2"]))
else:
# cursor.execute("UPDATE " + tableName + " SET dep1_1=?, dep1_2=?, dep2_1=?, dep2_2=?, dep3_1=?, dep3_2=?, dep4_1=?, dep4_2=?, dep5_1=?, dep5_2=? WHERE ymd=?", (meta["dep1_1"], meta["dep1_2"], meta["dep2_1"], meta["dep2_2"], meta["dep3_1"], meta["dep3_2"], meta["dep4_1"], meta["dep4_2"], meta["dep5_1"], meta["dep5_2"], meta["ymd"]))
finish = True
break
print("20"+item[0])
if finish:
break
previousDay = html[0].values[2][0]
conn.commit()
cursor.close()
conn.close()
return
# 국내 시장금리 크롤링
# 참고) http://blog.naver.com/PostView.nhn?blogId=koko8624&logNo=221292348073&parentCategoryNo=&categoryNo=&viewDate=&isShowPopularPosts=false&from=postView
def crawl_interest_rates(self, inFileName):
tableName = 'meta_4'
conn = sqlite3.connect(inFileName)
cursor = conn.cursor()
# 테이블 생성
cursor.execute("CREATE TABLE IF NOT EXISTS " + tableName + " (CODE text, NAME text, ymd text, close REAL, diff REAL, rate REAL)")
# 키 생성
create_key = "CREATE INDEX IF NOT EXISTS "+tableName+"_idx on "+tableName+" (CODE, ymd) "
cursor.execute(create_key)
inputs = []
inputs.append({'NAME': '91일 CD금리', 'CODE': 'IRR_CD91', 'URL': 'http://finance.naver.com/marketindex/interestDailyQuote.nhn?marketindexCd=IRR_CD91'})
inputs.append({'NAME': '콜금리', 'CODE': 'IRR_CALL', 'URL': 'http://finance.naver.com/marketindex/interestDailyQuote.nhn?marketindexCd=IRR_CALL'})
inputs.append({'NAME': '국고채(3년)', 'CODE': 'IRR_GOVT03Y', 'URL': 'http://finance.naver.com/marketindex/interestDailyQuote.nhn?marketindexCd=IRR_GOVT03Y'})
inputs.append({'NAME': '회사채(3년)', 'CODE': 'IRR_CORP03Y', 'URL': 'http://finance.naver.com/marketindex/interestDailyQuote.nhn?marketindexCd=IRR_CORP03Y'})
for i in range(len(inputs)):
input = inputs[i]
NAME = input['NAME']
CODE = input['CODE']
cursor.execute('SELECT ymd FROM ' + tableName + ' WHERE CODE=? order by ymd desc', (CODE,))
result = cursor.fetchone()
if result == None:
lastDay = self.START_DATE
else:
lastDay = result[0]
finish = False
for i in range(1, self.limit_page_count):
#html = pd.read_html(input['URL'] + '&page=%s' % i, header=0)
html = None
while True:
try:
html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header, timeout=30).text)
sleep(0.5)
break
except:
print (input['URL'] + '&page=%s' % i)
if i>200:
break
continue
ymd, close, diff, rate = "", 0.0, 0.0, 0.0
# 마지막 페이지 까지 받기
if len(html[0].날짜.values) <= 1:
break
for j in range(len(html[0].values)):
item = html[0].values[j]
if str(item[0]) == "nan":
continue
if item[0] <= lastDay:
finish = True
break
ymd = item[0]
close = item[1] # 종가
diff = item[2] # 전일대비
rate = item[3] # 등락율
cursor.execute('SELECT * FROM ' + tableName + ' WHERE CODE=? and ymd=?', (CODE, ymd,))
result = cursor.fetchone()
if result == None:
cursor.execute("INSERT INTO " + tableName + "(CODE, NAME, ymd, close, diff, rate) VALUES(?, ?, ?, ?, ?, ?)", (CODE, NAME, ymd, close, diff, rate))
else:
# cursor.execute("UPDATE " + tableName + " SET close=?, diff=?, rate=? WHERE CODE=? and ymd=?", (close, diff, rate, CODE, ymd))
finish = True
break
if finish:
break
print(NAME + " / " + ymd)
conn.commit()
cursor.close()
conn.close()
return
# 참고) http://blog.naver.com/PostView.nhn?blogId=koko8624&logNo=221288761509
def crawl_meterials(self, inFileName):
tableName = 'meta_5'
conn = sqlite3.connect(inFileName)
cursor = conn.cursor()
# 테이블 생성
cursor.execute("CREATE TABLE IF NOT EXISTS " + tableName + " (CODE text, NAME text, ymd text, close REAL, diff REAL, rate REAL)")
# 키 생성
create_key = "CREATE INDEX IF NOT EXISTS " + tableName + "_idx on " + tableName + " (CODE, NAME, ymd) "
cursor.execute(create_key)
inputs = []
inputs.append( {'NAME':'WTI', 'CODE':'OIL_CL', 'URL':'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=OIL_CL&fdtc=2'} ) # WTI
inputs.append( {'NAME':'GOLD', 'CODE':'CMDT_GC', 'URL':'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=CMDT_GC&fdtc=2'} ) # 국제 금
inputs.append({'NAME': 'COPPER', 'CODE': 'CMDT_CDY','URL': 'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=CMDT_CDY&fdtc=2'}) # 구리
inputs.append({'NAME': 'NATURALGAS', 'CODE': 'CMDT_NG','URL': 'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=CMDT_NG&fdtc=2'}) # 천연가스
inputs.append({'NAME': 'CORN', 'CODE': 'CMDT_C','URL': 'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=CMDT_C&fdtc=2'}) # 국제 옥수수
inputs.append({'NAME': 'SOYBEAN', 'CODE': 'CMDT_S','URL': 'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=CMDT_S&fdtc=2'}) # 국제 대두
for i in range(len(inputs)):
input = inputs[i]
NAME = input['NAME']
CODE = input['CODE']
cursor.execute('SELECT ymd FROM ' + tableName + ' WHERE CODE=? order by ymd desc', (CODE,))
result = cursor.fetchone()
if result == None:
lastDay = self.START_DATE
else:
lastDay = result[0]
finish = False
for i in range(1, self.limit_page_count):
#html = pd.read_html(input['URL'] + '&page=%s' % i, header=0)
html = None
while True:
try:
html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header, timeout=30).text)
sleep(0.5)
break
except:
print(input['URL'] + '&page=%s' % i)
if i>200:
break
continue
# 마지막 페이지 까지 받기
if len(html[0].날짜.values) <= 1:
break
for j in range(0, len(html[0].values)):
item = html[0].values[j]
if item[0] <= lastDay:
finish = True
break
ymd = item[0] # 날짜
close = item[1] # 종가
diff = item[2] # 전일대비
rate = item[3] # 등락율
cursor.execute('SELECT * FROM ' + tableName + ' WHERE CODE=? and ymd=?', (CODE,ymd,))
result = cursor.fetchone()
if result == None:
cursor.execute("INSERT INTO " + tableName + "(CODE, NAME, ymd, close, diff, rate) VALUES(?, ?, ?, ?, ?, ?)", (CODE, NAME, ymd, close, diff, rate))
else:
# cursor.execute("UPDATE " + tableName + " SET close=?, diff=?, rate=? WHERE CODE=? and ymd=?", (close, diff, rate, CODE, ymd))
finish = True
break
print(CODE, NAME, ymd)
if finish:
break
conn.commit()
cursor.close()
conn.close()
return
if __name__ == "__main__":
PROJECT_HOME = os.path.join(os.path.dirname(os.path.join(os.path.dirname(os.path.join(os.path.dirname(os.path.join(os.path.dirname(__file__))))))))
metaCrawler = MetaCrawler()
print("\n[환율 (USD, JPY, EUR, CNY)]")
inFileName = PROJECT_HOME + '/resources/stock.db'
metaCrawler.crawl_exchange(inFileName)
print("\n[투자자별 매매동향(Trading_Trend)]")
inFileName = PROJECT_HOME + '/resources/stock.db'
metaCrawler.crawl_trading_trend(inFileName)
print("\n[증시자금동향 (신용잔고, 펀드자금 잔고)]")
inFileName = PROJECT_HOME + '/resources/stock.db'
metaCrawler.crawl_money_trend(inFileName)
print("\n[국내 시장금리]")
inFileName = PROJECT_HOME + '/resources/stock.db'
metaCrawler.crawl_interest_rates(inFileName)
print("\n[원유 (WTI), 국제금, COPPER, NATURALGAS, CORN, SOYBEAN]")
inFileName = PROJECT_HOME + '/resources/stock.db'
metaCrawler.crawl_meterials(inFileName)

33
stockpredictor_back/crawler/sQLite/MovingAverage.py
View File

@@ -1,33 +0,0 @@
class MovingAverage(object):
def __init__(self, max):
self.queue = []
self.max = max
def dequeue(self):
length = len(self.queue)
if length == 0 or length < self.max:
return -1
return self.queue.pop(0)
def enqueue(self, n):
length = len(self.queue)
if length == self.max:
self.dequeue()
self.queue.append(n)
pass
def sum(self):
sum = 0
for item in self.queue:
sum += item
return sum
def avg(self):
length = len(self.queue)
total = self.sum()
return round(total / length, 2)
def print(self):
print(self.sum(), self.queue)

551
stockpredictor_back/crawler/sQLite/StockCrawler.py
View File

@@ -1,551 +0,0 @@
# https://bigdata-sk.tistory.com/10
from datetime import datetime, timedelta
import os
import pandas as pd
import re
import json
import sqlite3
import requests
from time import sleep
import time
from pandas_datareader import data as pdr
# 닐짜 형식으로 바뀐 this_date값을 확인 가능
# 읽어온 날짜 정보를 date형식으로 바꿀 일이 계속 생기므로 이 기능을 함수로 정의해줌.
# 함수명은 date_format()
class StockCrawler:
header = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36'}
historical_prices = None
special_pattern = None
fnGuideCrawler = None
limit_page_count = 1000000
START_DATE = None
def __init__(self, START_DATE):
self.historical_prices = dict()
self.special_pattern = (
'[', '!', '@', '#', '$', '%', '^', '&', '*', '(', ')', ',', '.', '?', '"', ':', ';', '{', '}', '|', '<', '>',
']', '+', '-', '/', '=', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9')
self.START_DATE = START_DATE
return
def clean_str(self, string):
string = re.sub(r"\\", " ", string)
string = re.sub(r"\'", " ", string)
string = re.sub(r"\"", " ", string)
string = re.sub(r"`", " ", string)
string = re.sub(r"-", " ", string)
string = re.sub(r"\(.*?\)", " ", string)
string = re.sub(r" ", " ", string)
return string.strip().lower()
def getStockInfo(self):
#code_df = pd.read_html('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', header=0)[0]
code_df = pd.read_html(requests.get('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', headers=self.header, timeout=30).text)[0]
# code_df = pd.read_excel('../resources/stock/상장법인목록.xls')
# 종목코드가 6자리이기 때문에 6자리를 맞춰주기 위해 설정해줌
code_df.종목코드 = code_df.종목코드.map('{:06d}'.format)
# 우리가 필요한 것은 회사명과 종목코드이기 때문에 필요없는 column들은 제외해준다.
code_df = code_df[['회사명', '종목코드']]
# 한글로된 컬럼명을 영어로 바꿔준다.
code_df = code_df.rename(columns={'회사명': 'name', '종목코드': 'code'})
###print (code_df.head())
return code_df
# 종목 이름을 입력하면 종목에 해당하는 코드를 불러와
# 네이버 금융(http://finance.naver.com)에 넣어줌
def get_url(self, item_name, code_df):
code = code_df.query("name=='{}'".format(item_name))['code'].to_string(index=False).strip()
url = 'http://finance.naver.com/item/sise_day.nhn?code={code}'.format(code=code.strip())
return code, url
def date_format(slef, d):
d = str(d).replace('-', '.')
#yyyy = int(d.split('.')[0])
#mm = int(d.split('.')[1])
#dd = int(d.split('.')[2])
#this_date = dt.date(yyyy, mm, dd)
return d
def getCodeIndex(self, stocks, item_code):
for i, stock in enumerate(stocks):
if item_code == stock['CODE']:
return i
return -1
def crawl_etf_stocks(self, inFileName):
tableName = 'stock'
conn = sqlite3.connect(inFileName)
cursor = conn.cursor()
# 테이블 생성
cursor.execute("CREATE TABLE IF NOT EXISTS " + tableName + " (CODE text, NAME text, ymd text, close REAL, diff REAL, open REAL, high REAL, low REAL, volume REAL)")
# 키 생성
create_key = "CREATE INDEX IF NOT EXISTS " + tableName + "_idx on " + tableName + " (CODE, ymd) "
cursor.execute(create_key)
stocks = []
stocks.append({"NAME": 'KODEX 코스닥150선물인버스', "CODE": "251340"})
stocks.append({"NAME": 'KODEX 코스닥150 레버리지', "CODE": "233740"})
stocks.append({"NAME": 'KODEX 200선물인버스2X', "CODE": "252670"})
stocks.append({"NAME": 'KODEX 레버리지', "CODE": "122630"})
stocks.append({"NAME": 'KODEX 인버스', "CODE": "114800"})
stocks.append({"NAME": 'KODEX 중국본토CSI300', "CODE": "283580"})
stocks.append({"NAME": 'KODEX 심천ChiNext(합성)', "CODE": "256750"})
stocks.append({"NAME": 'KINDEX 블룸버그베트남VN30선물레버리지(H)', "CODE": "371130"})
stocks.append({"NAME": 'KODEX 미국S&P바이오(합성)', "CODE": "185680"})
stocks.append({"NAME": 'KODEX 미국S&P에너지(합성)', "CODE": "218420"})
stocks.append({"NAME": 'KODEX 골드선물(H)', "CODE": "132030"})
stocks.append({"NAME": 'KODEX 콩선물(H)', "CODE": "138920"})
stocks.append({"NAME": 'KODEX 3대농산물선물(H)', "CODE": "271060"})
stocks.append({"NAME": 'KODEX 건설', "CODE": "117700"})
stocks.append({"NAME": 'KODEX 헬스케어', "CODE": "266420"})
stocks.append({"NAME": 'KODEX 글로벌4차산업로보틱스(합성)', "CODE": "276990"})
stocks.append({"NAME": 'KODEX 바이오', "CODE": "244580"})
stocks.append({"NAME": 'KODEX 반도체', "CODE": "091160"})
stocks.append({"NAME": 'KODEX 보험', "CODE": "140700"})
stocks.append({"NAME": 'KODEX 필수소비재', "CODE": "266410"})
stocks.append({"NAME": 'KODEX 2차전지산업', "CODE": "305720"})
stocks.append({"NAME": 'KODEX 경기소비재', "CODE": "266390"})
stocks.append({"NAME": 'KODEX 철강', "CODE": "117680"})
stocks.append({"NAME": 'KODEX 에너지화학', "CODE": "117460"})
stocks.append({"NAME": 'KODEX 은행', "CODE": "091170"})
stocks.append({"NAME": 'TIGER 탄소효율그린뉴딜', "CODE": "376410"})
start_time = time.time()
for i, stock in enumerate(stocks):
print (i, stock["NAME"], stock["CODE"], (time.time()-start_time), "s")
start_time = time.time()
cursor.execute('SELECT ymd FROM ' + tableName + ' WHERE CODE=? order by ymd desc', (stock["CODE"],))
result = cursor.fetchone()
ymd = self.START_DATE
if result is not None:
ymd = result[0]
stock_data = self.crawl_specific_stock(stock["CODE"], ymd)
for item in stock_data:
cursor.execute('SELECT * FROM ' + tableName + ' WHERE CODE=? and ymd=?', (stock["CODE"],item['ymd'],))
result = cursor.fetchone()
if result == None:
cursor.execute("INSERT INTO " + tableName + "(CODE, NAME, ymd, close, diff, open, high, low, volume) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)", (stock["CODE"], stock["NAME"], item['ymd'], item['close'], item['diff'], item['open'], item['high'], item['low'], item['volume']))
#else:
# cursor.execute("UPDATE " + tableName + " SET close=?, diff=?, open=?, high=?, low=?, volume=? WHERE CODE=? and ymd=?", (item['close'], item['diff'], item['open'], item['high'], item['low'], item['volume'], stock["CODE"], item['ymd']))
sleep(0.05)
conn.commit()
cursor.close()
conn.close()
return
def crawl_stocks(self, inFileName):
tableName = 'stock'
conn = sqlite3.connect(inFileName)
cursor = conn.cursor()
# 테이블 생성
cursor.execute("CREATE TABLE IF NOT EXISTS " + tableName + " (CODE text, NAME text, ymd text, close REAL, diff REAL, open REAL, high REAL, low REAL, volume REAL)")
# 키 생성
create_key = "CREATE INDEX IF NOT EXISTS " + tableName + "_idx on " + tableName + " (CODE, ymd) "
cursor.execute(create_key)
conn.commit()
cursor.close()
conn.close()
code_df = self.getStockInfo()
items = code_df.values
start_time = time.time()
idx = 0
for item in items:
idx += 1
conn = sqlite3.connect(inFileName)
cursor = conn.cursor()
item_name = item[0]
item_code = item[1]
cursor.execute('SELECT ymd FROM ' + tableName + ' WHERE CODE=? order by ymd desc', (item_code,))
result = cursor.fetchone()
stock = {"CODE": item_code, "NAME": item_name}
#ymd = (datetime.today() - timedelta(days=300)).strftime('%Y-%m-%d')
ymd = self.START_DATE.replace(".", "-")
if result is not None:
ymd = result[0]
stock_data = self.crawl_specific_stock(item_code, ymd)
for item in stock_data:
cursor.execute('SELECT * FROM ' + tableName + ' WHERE CODE=? and ymd=?', (stock["CODE"],item['ymd'],))
result = cursor.fetchone()
if result == None:
cursor.execute("INSERT INTO " + tableName + "(CODE, NAME, ymd, close, diff, open, high, low, volume) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)", (stock["CODE"], stock["NAME"], item['ymd'], item['close'], item['diff'], item['open'], item['high'], item['low'], item['volume']))
#else:
# cursor.execute("UPDATE " + tableName + " SET close=?, diff=?, open=?, high=?, low=?, volume=? WHERE CODE=? and ymd=?", (item['close'], item['diff'], item['open'], item['high'], item['low'], item['volume'], stock["CODE"], item['ymd']))
conn.commit()
cursor.close()
conn.close()
print(idx, item_name, item_code, (time.time() - start_time), "s")
start_time = time.time()
sleep(0.05)
return
def crawl_special_stocks(self, inFileName):
tableName = 'stock'
conn = sqlite3.connect(inFileName)
cursor = conn.cursor()
# 테이블 생성
cursor.execute("CREATE TABLE IF NOT EXISTS " + tableName + " (CODE text, NAME text, ymd text, close REAL, diff REAL, open REAL, high REAL, low REAL, volume REAL)")
# 키 생성
create_key = "CREATE INDEX IF NOT EXISTS " + tableName + "_idx on " + tableName + " (CODE, ymd) "
cursor.execute(create_key)
conn.commit()
cursor.close()
conn.close()
start_time = time.time()
pd.options.display.float_format = '{:.4f}'.format
pd.set_option('display.max_columns', None)
special_stocks = {
'^KS11': 'Kospi',
'^KQ11': 'Kosdak',
'SQQQ': 'ProShares UltraPro Short QQQ',
'TQQQ': 'ProShares UltraPro QQQ',
'SCO': 'ProShares UltraShort Bloomberg Crude Oil',
'UCO': 'ProShares Ultra Bloomberg Crude Oil',
'GLL': 'ProShares UltraShort Gold',
'UGL': 'ProShares Ultra Gold',
'SOXS': 'Direxion Daily Semiconductor Bear -3X Shares',
'SOXL': 'Direxion Daily Semiconductor Bull 3X Shares',
'FNGD': 'MicroSectors™ FANG+™ Index -3X Inverse Leveraged ETN',
'FNGU': 'MicroSectors™ FANG+™ Index 3X Leveraged ETN',
'AAPL': 'Apple',
'MSFT': 'Microsoft',
'GOOG': 'Alphabet C',
'AMZN': 'Amazon.com',
'AVGO': 'Broadcom',
'FB': 'Meta Platforms',
'NVDA': 'NVIDIA',
'UNH': 'UnitedHealth',
'TSM': 'Taiwan Semiconductor',
'JNJ': 'Johnson & Johnson (JNJ)',
'TCTZF': 'Tencent Holdings',
'V': 'Visa A',
'WMT': 'Walmart',
'XOM': 'Exxon Mobil',
'JPM': 'JPMorgan',
'MA': 'Mastercard',
'CVX': 'Chevron Corp',
'HD': 'Home Depot',
'BAC': 'Bank of America',
'KO': 'Coca-Cola',
'COST': 'Costco',
'DIS': 'Walt Disney',
'VZ': 'Verizon',
'CSCO': 'Cisco',
'ORCL': 'Oracle',
'NKE': 'Nike',
'ACN': 'Accenture',
'ADBE': 'Adobe',
'CRM': 'Salesforce.com',
'INTC': 'Intel',
'QCOM': 'Qualcomm',
'AMD': 'AMD',
'MS': 'Morgan Stanley',
'T': 'AT&T',
'HON': 'Honeywell',
'IBM': 'IBM',
'DQ': 'Daqo New Energy Corp ADR',
'EBAY': 'eBay Inc',
'NTAP': 'NetApp Inc',
'ASML': 'ASML Holding NV ADR',
'BABA': 'Alibaba Group Holdings Ltd ADR'}
conn = sqlite3.connect(inFileName)
cursor = conn.cursor()
us_sotck_data = {}
for ticker in special_stocks:
cursor.execute('SELECT * FROM ' + tableName + ' WHERE CODE=?', (ticker,))
result = cursor.fetchone()
if result == None:
#start_day = pd.to_datetime('2017-01-01')
start_day = pd.to_datetime(self.START_DATE.replace(".", "-"))
end_day = pd.to_datetime(datetime.today().strftime('%Y-%m-%d'))
else:
#start = (datetime.today() - timedelta(days=300)).strftime('%Y-%m-%d')
start_day = pd.to_datetime(self.START_DATE.replace(".", "-"))
end_day = pd.to_datetime(datetime.today().strftime('%Y-%m-%d'))
data = None
while True:
try:
data = pdr.get_data_yahoo(ticker, start_day, end_day)
break
except:
print(ticker)
continue
us_sotck_data[ticker] = {
'close': data['Close'].to_dict(),
'open': data['Open'].to_dict(),
'high': data['High'].to_dict(),
'low': data['Low'].to_dict(),
'volume': data['Volume'].to_dict()
}
cursor.close()
conn.close()
dateList = list(us_sotck_data['SQQQ']['close'])
for idx, item_code in enumerate(us_sotck_data):
stock_data = []
stock = us_sotck_data[item_code]
for i, ymd in enumerate(dateList):
if i > 0:
if ymd in stock['close'] and dateList[i-1] in stock['close']:
diff = stock['close'][ymd] - stock['close'][dateList[i-1]]
else:
continue
else:
if ymd in stock['close']:
diff = stock['close'][ymd]
else:
continue
stock_data.append({
'CODE':item_code, 'NAME':us_stocks[item_code], 'ymd': ymd.strftime('%Y.%m.%d'),
'close': round(stock['close'][ymd], 2), 'diff': round(diff, 2), 'open': round(stock['open'][ymd], 2),
'high': round(stock['high'][ymd], 2), 'low': round(stock['low'][ymd], 2), 'volume': stock['volume'][ymd]
})
stock_data = reversed(stock_data)
conn = sqlite3.connect(inFileName)
cursor = conn.cursor()
for item in stock_data:
cursor.execute('SELECT * FROM ' + tableName + ' WHERE CODE=? and ymd=?', (item["CODE"],item['ymd'],))
result = cursor.fetchone()
if result == None:
cursor.execute("INSERT INTO " + tableName + "(CODE, NAME, ymd, close, diff, open, high, low, volume) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)", (item["CODE"], item["NAME"], item['ymd'], item['close'], item['diff'], item['open'], item['high'], item['low'], item['volume']))
#else:
# cursor.execute("UPDATE " + tableName + " SET close=?, diff=?, open=?, high=?, low=?, volume=? WHERE CODE=? and ymd=?", (item['close'], item['diff'], item['open'], item['high'], item['low'], item['volume'], stock["CODE"], item['ymd']))
conn.commit()
cursor.close()
conn.close()
print(idx, item_code, us_stocks[item_code], (time.time() - start_time), "s")
start_time = time.time()
sleep(0.05)
return
def get_data(self, code, lastDay):
url = 'http://finance.naver.com/item/sise_day.nhn?code={code}'.format(code=code.strip())
stock = []
# 일자 데이터를 담을 df라는 DataFrame 정의
df = pd.DataFrame()
date_set = set()
lastPage = False
# 1페이지에서 1000페이지의 데이터만 가져오기
for page in range(1, self.limit_page_count):
# 최근 상장 기업의 마지막 반복되는 페이지를 제외시킨다.
pg_url = '{url}&page={page}'.format(url=url, page=page)
#html = pd.read_html(pg_url, header=0)
html = None
while True:
try:
html = pd.read_html(requests.get(pg_url, headers=self.header, timeout=30).text)
sleep(0.5)
break
except:
print(pg_url)
if page > 200:
break
continue
for date in html[0].날짜.values:
if type(date) is str:
if date in date_set:
lastPage = True
break
date_set.add(date)
if date == lastDay:
lastPage = True
df = df.append(html[0], ignore_index=True)
break
df = df.append(html[0], ignore_index=True)
df = df.dropna()
if (lastPage) or (len(df) < 1) or ("날짜" not in df) or (df.날짜[1]==''):
print("\t- lastpage:", page)
break
# df.dropna()를 이용해 결측값 있는 행 제거
df = df.dropna()
# 상위 5개 데이터 확인하기
###print (df.head())
# 한글로 된 컬럼명을 영어로 바꿔줌
df = df.rename(columns={'날짜': 'date', '종가': 'close', '전일비': 'diff', '시가': 'open', '고가': 'high', '저가': 'low', '거래량': 'volume'})
# 데이터의 타입을 int형으로 바꿔줌
df[['close', 'diff', 'open', 'high', 'low', 'volume']] = df[['close', 'diff', 'open', 'high', 'low', 'volume']].astype(int)
for values in df.values:
day = str(values[0]).split(' ')[0]
if lastDay == day:
break
stock.append({
"ymd": day,
df.columns[1]: values[1],
df.columns[2]: values[2],
df.columns[3]: values[3],
df.columns[4]: values[4],
df.columns[5]: values[5],
df.columns[6]: values[6],
})
# stock = sorted(stock, key=lambda x: x['ymd'], reverse=True)
stock = sorted(stock, key=lambda x: x['ymd'])
return stock
def crawl_specific_stock(self, code, ymd):
# 데이터 수집
stock = self.get_data(code, ymd)
# 이동 평균 계산
#self.get_moving_avg(stock)
return stock
def update(self, inFileName, outFileName):
"""
Full json 데이터를 db에 import 시킴
inFileName = PROJECT_HOME + '/resources/stock.json.full'
outFileName = PROJECT_HOME + '/resources/stock.db'
crawler = StockCrawler()
crawler.update(inFileName, outFileName)
:param inFileName:
:param outFileName:
:return:
"""
tableName = 'stock'
conn = sqlite3.connect(outFileName, isolation_level=None)
cursor = conn.cursor()
cursor.execute("CREATE TABLE IF NOT EXISTS " + tableName + " (CODE text PRIMARY KEY, NAME text, PRICE text, MACD text, STOCHASTIC text, ICHIMOKU text, RSI text, BOLINGERBAND text)")
idx = 0
inFp = open(inFileName, 'r')
for line in inFp.readlines():
if line:
idx += 1
stock = json.loads(line)
print(idx, stock["CODE"], stock["NAME"])
text = json.dumps(stock["PRICE"], ensure_ascii=False)
cursor.execute('SELECT * FROM ' + tableName + ' WHERE CODE=?', (stock["CODE"],))
result = cursor.fetchone()
if result == None:
cursor.execute("INSERT INTO " + tableName + "(CODE, NAME, PRICE) VALUES(?, ?, ?)", (stock["CODE"], stock["NAME"], text))
else:
cursor.execute("UPDATE " + tableName + " SET PRICE=? WHERE CODE=?", (text, stock["CODE"]))
return
def saveIndex(self, code, inFileName, outFileName):
tableName = 'stock'
conn = sqlite3.connect(outFileName)
cursor = conn.cursor()
cursor.execute("CREATE TABLE IF NOT EXISTS " + tableName + " (CODE text PRIMARY KEY, NAME text, PRICE text, MACD text, STOCHASTIC text, ICHIMOKU text, RSI text, BOLINGERBAND text)")
stock = {"NAME": code, "CODE": code}
lastDay = ""
cursor.execute('SELECT * FROM ' + tableName + ' WHERE CODE=?', (stock["CODE"],))
result = cursor.fetchone()
if result is not None:
stock["PRICE"] = json.loads(result[2])
lastDay = stock["PRICE"][len(stock["PRICE"]) - 1]["DATE"]
with open(inFileName, "r", encoding="utf-8") as inFp:
for line in inFp:
line = line.strip()
if line[0] == "#":
continue
arr = line.split("\t")
if arr[0] == lastDay:
break
price = {"DATE": arr[0], "close": float(arr[1]), "diff": float(arr[6].replace("%", "")), "open": float(arr[2]), "high": float(arr[3]), "low": float(arr[4]), "volume": 0}
price['avg3'] = 0
price['avg5'] = 0
price['avg7'] = 0
price['avg10'] = 0
price['avg20'] = 0
price['avg30'] = 0
price['avg60'] = 0
price['avg90'] = 0
price['avg100'] = 0
price['avg120'] = 0
price['avg150'] = 0
price['avg180'] = 0
price['avg200'] = 0
price['avg240'] = 0
stock["PRICE"].append(price)
stock["PRICE"] = sorted(stock["PRICE"], key=lambda x: x['DATE'])
# self.get_moving_avg(stock)
text = json.dumps(stock['PRICE'], ensure_ascii=False)
cursor.execute('SELECT * FROM ' + tableName + ' WHERE CODE=?', (stock["CODE"],))
result = cursor.fetchone()
if result == None:
cursor.execute("INSERT INTO " + tableName + "(CODE, NAME, PRICE, MACD, STOCHASTIC, ICHIMOKU, RSI) VALUES(?, ?, ?, ?, ?, ?, ?)", (stock["CODE"], stock["NAME"], text, "[{}]", "[{}]", "[{}]", "[{}]"))
else:
cursor.execute("UPDATE " + tableName + " SET PRICE=?, MACD=?, STOCHASTIC=?, ICHIMOKU=?, RSI=? WHERE CODE=?", (text, "[{}]", "[{}]", "[{}]", "[{}]", stock["CODE"]))
conn.commit()
cursor.close()
conn.close()
return
if __name__ == "__main__":
stockCrawler = StockCrawler()
PROJECT_HOME = os.path.join(os.path.dirname(os.path.join(os.path.dirname(os.path.join(os.path.dirname(os.path.join(os.path.dirname(__file__))))))))
stockFileName = PROJECT_HOME + '/resources/stock.db'
stockCrawler.crawl_us_sotck_datas(stockFileName)

77
stockpredictor_back/crawler/toJsonFile/Crawler.py
View File

@@ -1,77 +0,0 @@
import os
import shutil
import datetime
from stockpredictor.crawler.toJsonFile.FnGuideCrawler import FnGuideCrawler
from stockpredictor.crawler.toJsonFile.MetaCrawler import MetaCrawler
from stockpredictor.crawler.toJsonFile.StockCrawler import StockCrawler
today = datetime.datetime.now().strftime("%Y-%m-%d")
PROJECT_HOME = "../../.."
crawler = FnGuideCrawler()
print("[KOSPI 상장기업 재무제표 다운로드]")
inFileName = PROJECT_HOME + '/resources/fnguide.json'
outFileName = PROJECT_HOME + '/resources/fnguide.temp.json'
if os.path.isfile(inFileName):
shutil.copy(inFileName, outFileName)
crawler.crawl_fnguide(outFileName)
if os.path.isfile(inFileName):
os.remove(inFileName)
shutil.move(outFileName, inFileName)
crawler = MetaCrawler()
print("[환율 (USD, JPY, EUR, CNY), 원유 (WTI), 국제금]")
inFileName = PROJECT_HOME + '/resources/meta_1.json'
outFileName = PROJECT_HOME + '/resources/meta_1.temp.json'
if os.path.isfile(inFileName):
shutil.copy(inFileName, outFileName)
crawler.crawl_stocks(outFileName)
if os.path.isfile(inFileName):
os.remove(inFileName)
shutil.move(outFileName, inFileName)
print("[투자자별 매매동향(Trading_Trend)]")
inFileName = PROJECT_HOME + '/resources/meta_2.json'
outFileName = PROJECT_HOME + '/resources/meta_2.temp.json'
if os.path.isfile(inFileName):
shutil.copy(inFileName, outFileName)
crawler.crawl_trading_trend(outFileName)
if os.path.isfile(inFileName):
os.remove(inFileName)
shutil.move(outFileName, inFileName)
print("[증시자금동향 (신용잔고, 펀드자금 잔고)]")
inFileName = PROJECT_HOME + '/resources/meta_3.json'
outFileName = PROJECT_HOME + '/resources/meta_3.temp.json'
if os.path.isfile(inFileName):
shutil.copy(inFileName, outFileName)
crawler.crawl_money_trend(outFileName)
if os.path.isfile(inFileName):
os.remove(inFileName)
shutil.move(outFileName, inFileName)
print("[국내 시장금리]")
inFileName = PROJECT_HOME + '/resources/meta_4.json'
outFileName = PROJECT_HOME + '/resources/meta_4.temp.json'
if os.path.isfile(inFileName):
shutil.copy(inFileName, outFileName)
crawler.crawl_interest_rates(outFileName)
if os.path.isfile(inFileName):
os.remove(inFileName)
shutil.move(outFileName, inFileName)
print("[종목 다운로드]")
inFileName = PROJECT_HOME + '/resources/stock.json'
outFileName = PROJECT_HOME + '/resources/stock.temp.json'
if os.path.isfile(inFileName):
shutil.copy(inFileName, outFileName)
crawler = StockCrawler()
crawler.crawl_stocks(outFileName)
crawler.get_stocks_avg(outFileName, inFileName)
if os.path.isfile(outFileName):
os.remove(outFileName)
print ("done...")

113
stockpredictor_back/crawler/toJsonFile/FnGuideCrawler.py
View File

@@ -1,113 +0,0 @@
from bs4 import BeautifulSoup
from pandas import DataFrame, Series
import requests as re
import pandas as pd
import json
import requests
class FnGuideCrawler:
header = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36'}
def getStockInfo(self):
#code_df = pd.read_html('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', header=0)[0]
code_df = pd.read_html(requests.get('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', headers=self.header, timeout=30).text)[0]
# code_df = pd.read_excel('../resources/stock/상장법인목록.xls')
# 종목코드가 6자리이기 때문에 6자리를 맞춰주기 위해 설정해줌
code_df.종목코드 = code_df.종목코드.map('{:06d}'.format)
# 우리가 필요한 것은 회사명과 종목코드이기 때문에 필요없는 column들은 제외해준다.
code_df = code_df[['회사명', '종목코드']]
# 한글로된 컬럼명을 영어로 바꿔준다.
code_df = code_df.rename(columns={'회사명': 'name', '종목코드': 'code'})
###print (code_df.head())
return code_df
# FnGuide에서 크롤링한 KOSPI 상장기업의 재무제표
# http://blog.naver.com/PostView.nhn?blogId=koko8624&logNo=221294884955&parentCategoryNo=&categoryNo=&viewDate=&isShowPopularPosts=false&from=postView
def get_fnguide_table(self, code):
url = re.get('http://comp.fnguide.com/SVO2/ASP/SVD_main.asp?pGB=1&gicode=A%s'%(code.strip()))
url = url.content
html = BeautifulSoup(url,'html.parser')
body = html.find('body')
try:
fn_body = body.find('div', {'class': 'fng_body asp_body'})
ur_table = fn_body.find('div', {'id': 'div15'})
table = ur_table.find('div', {'id': 'highlight_D_Y'})
tbody = table.find('tbody')
tr = tbody.find_all('tr')
Table = DataFrame()
except:
return {}
for i in tr:
''' 자료 항목 가져오기'''
category = i.find('span', {'class': 'txt_acd'})
if category == None:
category = i.find('th')
category = category.text.strip()
'''값 가져오기'''
value_list = []
j = i.find_all('td', {'class': 'r'})
for value in j:
temp = value.text.replace(',', '').strip()
try:
temp = float(temp)
value_list.append(temp)
except:
value_list.append(0)
Table['%s' % (category)] = value_list
''' 기간 가져오기 '''
thead = table.find('thead')
tr_2 = thead.find('tr', {'class': 'td_gapcolor2'}).find_all('th')
year_list = []
for i in tr_2:
try:
temp_year = i.find('span', {'class': 'txt_acd'}).text
except:
temp_year = i.text
temp_year = temp_year.replace("/",".")+".01"
year_list.append(temp_year)
Table.index = year_list
return Table.T.to_dict()
def crawl_fnguide(self, inFileName):
code_df = self.getStockInfo()
outFp = open(inFileName, 'w', encoding='utf-8')
idx = 0
for item in code_df.values:
item_name = item[0]
item_code = item[1]
idx += 1
print(idx, item_name)
fnGuideData = self.get_fnguide_table(item_code)
stock = {"NAME": item_name, "CODE": item_code, "PRICE": fnGuideData}
outFp.write(json.dumps(stock, ensure_ascii=False) + "\n")
outFp.close()
return
if __name__ == "__main__":
crawler = FnGuideCrawler()
crawler.get_fnguide_table('155660')

319
stockpredictor_back/crawler/toJsonFile/MetaCrawler.py
View File

@@ -1,319 +0,0 @@
import json
import datetime
import requests
import pandas as pd
from time import sleep
import os
class MetaCrawler:
header = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36'}
limit_page_count = 10000
def __init__(self):
return
# 참고) http://blog.naver.com/PostView.nhn?blogId=koko8624&logNo=221288761509
def crawl_stocks(self, inFileName):
stocks = []
if os.path.isfile(inFileName):
inFp = open(inFileName, 'r', encoding='utf-8')
for line in inFp.readlines():
line = line.strip()
if line:
stocks.append(json.loads(line))
inFp.close()
outFp = open(inFileName, 'w', encoding='utf-8')
inputs = []
inputs.append( {'NAME':'USD', 'CODE':'FX_USDKRW', 'URL':'http://finance.naver.com/marketindex/exchangeDailyQuote.nhn?marketindexCd=FX_USDKRW'} ) # 미국 USD
inputs.append( {'NAME':'JPY', 'CODE':'FX_JPYKRW', 'URL':'http://finance.naver.com/marketindex/exchangeDailyQuote.nhn?marketindexCd=FX_JPYKRW'} ) # 일본 JPY
inputs.append( {'NAME':'EUR', 'CODE':'FX_EURKRW', 'URL':'http://finance.naver.com/marketindex/exchangeDailyQuote.nhn?marketindexCd=FX_EURKRW'} ) # 유럽연합 EUR'
inputs.append( {'NAME':'CNY', 'CODE':'FX_CNYKRW', 'URL':'http://finance.naver.com/marketindex/exchangeDailyQuote.nhn?marketindexCd=FX_CNYKRW'} ) # 중국 CNY
inputs.append( {'NAME':'WTI', 'CODE':'OIL_CL', 'URL':'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=OIL_CL&fdtc=2'} ) # WTI
inputs.append( {'NAME':'GOLD', 'CODE':'CMDT_GC', 'URL':'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=CMDT_GC&fdtc=2'} ) # 국제 금
for i in range(len(inputs)):
input = inputs[i]
if len(stocks) == 0:
meta = {}
meta["NAME"] = input['NAME']
meta["CODE"] = input['CODE']
meta["PRICE"] = []
lastDay = "1900.01.01"
else:
meta = stocks[i]
lastDay = meta['PRICE'][0]['DATE']
finish = False
for i in range(1, self.limit_page_count):
#html = pd.read_html(input['URL'] + '&page=%s' % i, header=0)
html = None
while True:
try:
html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header, timeout=30).text)
sleep(0.5)
break
except:
print(i)
if i > 200:
break
continue
# 마지막 페이지 까지 받기
if len(html[0].날짜.values) <= 1:
break
for j in range(0, len(html[0].values)):
item = html[0].values[j]
if input['NAME'] in ('USD', 'JPY', 'EUR', 'CNY'):
if j == 0:
continue
if item[0] <= lastDay:
finish = True
break
meta["PRICE"].append({
"DATE": item[0], # 날짜
"close": item[1], # 매매기준율
"diff": item[2] # 전일대비
})
elif input['NAME'] in ('WTI', 'GOLD'):
if item[0] <= lastDay:
finish = True
break
meta["PRICE"].append({
"DATE": item[0], # 날짜
"close": item[1], # 종가
"diff": item[2], # 전일대비
"rate": item[3] # 등락율
})
if finish:
break
meta["PRICE"] = sorted(meta["PRICE"], key=lambda x: x['DATE'], reverse=True)
outFp.write(json.dumps(meta, ensure_ascii=False) + "\n")
outFp.close()
return
# 투자자별 매매동향 (Trading_Trend) 크롤링
# (pri, 개인)
# (for, 외국인)
# (ins, 기관합)
# (ins0, 금융투자)
# (ins1, 보험)
# (ins2, 투신 (사모))
# (ins3, 은행)
# (ins4, 기타금융기관)
# (ins5, 연기금 등)
# (cor, 기타법인)
# 참고) http://blog.naver.com/PostView.nhn?blogId=koko8624&logNo=221289696771&parentCategoryNo=&categoryNo=&viewDate=&isShowPopularPosts=false&from=postView
def crawl_trading_trend(self, inFileName):
stocks = []
if os.path.isfile(inFileName):
inFp = open(inFileName, 'r', encoding='utf-8')
for line in inFp.readlines():
line = line.strip()
if line:
stocks.append(json.loads(line))
inFp.close()
outFp = open(inFileName, 'w', encoding='utf-8')
today = datetime.datetime.now().strftime("%Y%m%d")
url = 'http://finance.naver.com/sise/investorDealTrendDay.nhn?bizdate='+today+'&sosok=&page='
if len(stocks) == 0:
lastDay = "1900.01.01"
else:
lastDay = stocks[0]['DATE']
previousDay = ""
finish = False
for i in range(1, self.limit_page_count):
#html = pd.read_html(url + str(i), header=0)
html = None
while True:
try:
html = pd.read_html(requests.get(url + str(i), headers=self.header, timeout=30).text)
sleep(0.5)
break
except:
print(i)
if i > 200:
break
continue
# 마지막 페이지 까지 받기
if len(html[0].날짜.values) <= 2:
break
for j in range(1, len(html[0].values)):
item = html[0].values[j]
if str(item[0]) == "nan":
continue
if "20" + item[0] <= lastDay or item[0] == previousDay:
finish = True
break
meta = {
"DATE": "20"+item[0],
"pri": item[1], # 개인
"for": item[2], # 외국인
"ins": item[3], # 기관합
"ins0": item[4], # 금융투자
"ins1": item[5], # 보험
"ins2": item[6], # 투신 (사모)
"ins3": item[7], # 은행
"ins4": item[8], # 기타금융기관
"ins5": item[9], # 연기금 등
"cor": item[10]} # 기타법인
outFp.write(json.dumps(meta, ensure_ascii=False) + "\n")
print ("20"+item[0])
previousDay = html[0].values[2][0]
if finish:
break
if len(stocks) > 0:
for stock in stocks:
outFp.write(json.dumps(stock, ensure_ascii=False) + "\n")
outFp.close()
return
# 증시자금동향 (신용잔고, 펀드자금 잔고) 크롤링
# 참고) http://blog.naver.com/PostView.nhn?blogId=koko8624&logNo=221290138187&parentCategoryNo=&categoryNo=&viewDate=&isShowPopularPosts=false&from=postView
def crawl_money_trend(self, inFileName):
stocks = []
if os.path.isfile(inFileName):
inFp = open(inFileName, 'r', encoding='utf-8')
for line in inFp.readlines():
line = line.strip()
if line:
stocks.append(json.loads(line))
inFp.close()
outFp = open(inFileName, 'w', encoding='utf-8')
url = 'http://finance.naver.com/sise/sise_deposit.nhn?&page='
if len(stocks) == 0:
lastDay = "1900.01.01"
else:
lastDay = stocks[0]['DATE']
previousDay = ""
finish = False
for i in range(1, self.limit_page_count):
#html = pd.read_html(url + str(i), header=0, encoding='euc-kr')
html = None
while True:
try:
html = pd.read_html(requests.get(url + str(i), headers=self.header, timeout=30).text, encoding='euc-kr')
sleep(0.5)
break
except:
print(url + str(i))
if i > 200:
break
continue
# 마지막 페이지 까지 받기
if len(html[0].날짜.values) <= 10:
break
for j in range(1, len(html[0].values)):
item = html[0].values[j]
if str(item[0]) == "nan":
continue
if "20"+item[0] <= lastDay or item[0] == previousDay:
finish = True
break
meta = {
"DATE": "20"+item[0],
"dep1_1": item[1], # 고객예탁금 누적
"dep1_2": item[2], # 고객예탁금 당일
"dep2_1": item[3], # 신용잔고 누적
"dep2_2": item[4], # 신용잔고 당일
"dep3_1": item[5], # 주식형펀드 누적
"dep3_2": item[6], # 주식형펀드 당일
"dep4_1": item[7], # 혼합형펀드 누적
"dep4_2": item[8], # 혼합형펀드 당일
"dep5_1": item[9], # 채권형펀드 누적
"dep5_2": item[10]} # 채권형펀드 당일
outFp.write(json.dumps(meta, ensure_ascii=False) + "\n")
print("20"+item[0])
if finish:
break
previousDay = html[0].values[2][0]
if len(stocks) > 0:
for stock in stocks:
outFp.write(json.dumps(stock, ensure_ascii=False) + "\n")
outFp.close()
return
# 국내 시장금리 크롤링
# 참고) http://blog.naver.com/PostView.nhn?blogId=koko8624&logNo=221292348073&parentCategoryNo=&categoryNo=&viewDate=&isShowPopularPosts=false&from=postView
def crawl_interest_rates(self, inFileName):
stocks = []
if os.path.isfile(inFileName):
inFp = open(inFileName, 'r', encoding='utf-8')
for line in inFp.readlines():
line = line.strip()
if line:
stocks.append(json.loads(line))
inFp.close()
outFp = open(inFileName, 'w', encoding='utf-8')
inputs = []
inputs.append({'NAME': '91일 CD금리', 'CODE': 'IRR_CD91', 'URL': 'http://finance.naver.com/marketindex/interestDailyQuote.nhn?marketindexCd=IRR_CD91'})
inputs.append({'NAME': '콜금리', 'CODE': 'IRR_CALL', 'URL': 'http://finance.naver.com/marketindex/interestDailyQuote.nhn?marketindexCd=IRR_CALL'})
inputs.append({'NAME': '국고채(3년)', 'CODE': 'IRR_GOVT03Y', 'URL': 'http://finance.naver.com/marketindex/interestDailyQuote.nhn?marketindexCd=IRR_GOVT03Y'})
inputs.append({'NAME': '회사채(3년)', 'CODE': 'IRR_CORP03Y', 'URL': 'http://finance.naver.com/marketindex/interestDailyQuote.nhn?marketindexCd=IRR_CORP03Y'})
for i in range(len(inputs)):
input = inputs[i]
if len(stocks) == 0:
meta = {}
meta["NAME"] = input['NAME']
meta["CODE"] = input['CODE']
meta["PRICE"] = []
lastDay = "1900.01.01"
else:
meta = stocks[i]
lastDay = meta['PRICE'][0]['DATE']
finish = False
for i in range(1, self.limit_page_count):
#html = pd.read_html(input['URL'] + '&page=%s' % i, header=0)
html = None
while True:
try:
html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header, timeout=30).text)
sleep(0.5)
break
except:
print(i)
if i > 200:
break
continue
# 마지막 페이지 까지 받기
if len(html[0].날짜.values) <= 1:
break
for j in range(len(html[0].values)):
item = html[0].values[j]
if str(item[0]) == "nan":
continue
if item[0] <= lastDay:
finish = True
break
meta["PRICE"].append({
"DATE": item[0],
"close": item[1], # 종가
"diff": item[2], # 전일대비
"rate": item[3]}) # 등락율
if finish:
break
print(meta["NAME"] + " / " + item[0])
meta["PRICE"] = sorted(meta["PRICE"], key=lambda x: x['DATE'], reverse=True)
outFp.write(json.dumps(meta, ensure_ascii=False) + "\n")
outFp.close()
return

728
stockpredictor_back/crawler/toJsonFile/StockCrawler.py
View File

@@ -1,728 +0,0 @@
# https://bigdata-sk.tistory.com/10
import pandas as pd
import re
import json
import os
import requests
from time import sleep
class Queue(object):
def __init__(self, max):
self.queue = []
self.max = max
def dequeue(self):
length = len(self.queue)
if length == 0 or length < self.max:
return -1
return self.queue.pop(0)
def enqueue(self, n):
length = len(self.queue)
if length == self.max:
self.dequeue()
self.queue.append(n)
pass
def sum(self):
sum = 0
for item in self.queue:
sum += item
return sum
def avg(self):
length = len(self.queue)
total = self.sum()
return round(total / length)
def print(self):
print(self.sum(), self.queue)
# 닐짜 형식으로 바뀐 this_date값을 확인 가능
# 읽어온 날짜 정보를 date형식으로 바꿀 일이 계속 생기므로 이 기능을 함수로 정의해줌.
# 함수명은 date_format()
class StockCrawler:
header = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36'}
historical_prices = None
special_pattern = None
fnGuideCrawler = None
limit_page_count = 40
def __init__(self):
self.historical_prices = dict()
self.special_pattern = (
'[', '!', '@', '#', '$', '%', '^', '&', '*', '(', ')', ',', '.', '?', '"', ':', ';', '{', '}', '|', '<', '>',
']', '+', '-', '/', '=', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9')
return
def clean_str(self, string):
string = re.sub(r"\\", " ", string)
string = re.sub(r"\'", " ", string)
string = re.sub(r"\"", " ", string)
string = re.sub(r"`", " ", string)
string = re.sub(r"-", " ", string)
string = re.sub(r"\(.*?\)", " ", string)
string = re.sub(r" ", " ", string)
return string.strip().lower()
def getStockInfo(self):
#code_df = pd.read_html('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', header=0)[0]
code_df = pd.read_html(requests.get('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', headers=self.header, timeout=30).text)[0]
# code_df = pd.read_excel('../resources/stock/상장법인목록.xls')
# 종목코드가 6자리이기 때문에 6자리를 맞춰주기 위해 설정해줌
code_df.종목코드 = code_df.종목코드.map('{:06d}'.format)
# 우리가 필요한 것은 회사명과 종목코드이기 때문에 필요없는 column들은 제외해준다.
code_df = code_df[['회사명', '종목코드']]
# 한글로된 컬럼명을 영어로 바꿔준다.
code_df = code_df.rename(columns={'회사명': 'name', '종목코드': 'code'})
###print (code_df.head())
return code_df
# 종목 이름을 입력하면 종목에 해당하는 코드를 불러와
# 네이버 금융(http://finance.naver.com)에 넣어줌
def get_url(self, item_name, code_df):
code = code_df.query("name=='{}'".format(item_name))['code'].to_string(index=False).strip()
url = 'http://finance.naver.com/item/sise_day.nhn?code={code}'.format(code=code.strip())
return code, url
def date_format(slef, d):
d = str(d).replace('-', '.')
#yyyy = int(d.split('.')[0])
#mm = int(d.split('.')[1])
#dd = int(d.split('.')[2])
#this_date = dt.date(yyyy, mm, dd)
return d
def getCodeIndex(self, stocks, item_code):
for i, stock in enumerate(stocks):
if item_code == stock['CODE']:
return i
return -1
def crawl_stocks(self, inFileName):
stocks = []
if os.path.isfile(inFileName):
inFp = open(inFileName, 'r', encoding='utf-8')
for line in inFp.readlines():
line = line.strip()
if line:
stocks.append(json.loads(line))
inFp.close()
if len(stocks)>0:
stock_cosdak_inverse = {"NAME": 'KODEX 코스닥150선물인버스', "CODE": "251340", "PRICE": stocks[0]["PRICE"]}
stock_cosdak_reverage = {"NAME": 'KODEX 코스닥150 레버리지', "CODE": "233740", "PRICE": stocks[1]["PRICE"]}
stock_inverse = {"NAME": 'KODEX 200선물인버스2X', "CODE": "252670", "PRICE": stocks[2]["PRICE"]}
stock_reverage = {"NAME": 'KODEX 레버리지', "CODE": "122630", "PRICE": stocks[3]["PRICE"]}
stock_gold = {"NAME": 'KODEX 골드선물(H)', "CODE": "132030", "PRICE": stocks[4]["PRICE"]}
else:
stock_cosdak_inverse = {"NAME": 'KODEX 코스닥150선물인버스', "CODE": "251340", "PRICE": []}
stock_cosdak_reverage = {"NAME": 'KODEX 코스닥150 레버리지', "CODE": "233740", "PRICE": []}
stock_inverse = {"NAME": 'KODEX 200선물인버스2X', "CODE": "252670", "PRICE": []}
stock_reverage = {"NAME": 'KODEX 레버리지', "CODE": "122630", "PRICE": []}
stock_gold = {"NAME": 'KODEX 골드선물(H)', "CODE": "132030", "PRICE": []}
outFp = open(inFileName, "w", encoding="utf-8")
kodex_cosdak_inverse = self.crawl_specific_stock('KODEX 코스닥150선물인버스', '251340', stock_cosdak_inverse)
outFp.write(json.dumps(kodex_cosdak_inverse, ensure_ascii=False) + "\n")
kodex_cosdak_reverage = self.crawl_specific_stock('KODEX 코스닥150 레버리지', '233740', stock_cosdak_reverage)
outFp.write(json.dumps(kodex_cosdak_reverage, ensure_ascii=False) + "\n")
kodex_inverse = self.crawl_specific_stock('KODEX 200선물인버스2X', '252670', stock_inverse)
outFp.write(json.dumps(kodex_inverse, ensure_ascii=False) + "\n")
kodex_reverage = self.crawl_specific_stock('KODEX 레버리지', '122630', stock_reverage)
outFp.write(json.dumps(kodex_reverage, ensure_ascii=False) + "\n")
kodex_gold = self.crawl_specific_stock('KODEX 골드선물(H)', '132030', stock_gold)
outFp.write(json.dumps(kodex_gold, ensure_ascii=False) + "\n")
code_df = self.getStockInfo()
items = code_df.values
idx = 0
for item in items:
idx += 1
item_name = item[0]
item_code = item[1]
print(idx, item_name, item_code)
if len(stocks) > 0:
index = self.getCodeIndex(stocks, item_code)
if index < 0:
stock = {"NAME": item_name, "CODE": item_code, "PRICE": []}
else:
stock = {"NAME": item_name, "CODE": item_code, "PRICE": stocks[index]["PRICE"]}
else:
stock = {"NAME": item_name, "CODE": item_code, "PRICE": []}
stock = self.crawl_specific_stock(item_name, item_code, stock)
outFp.write(json.dumps(stock, ensure_ascii=False) + "\n")
outFp.close()
return
def get_stocks_avg(self, inFileName, outFileName):
outFp = open(outFileName, 'w', encoding='utf-8')
inFp = open(inFileName, 'r', encoding='utf-8')
idx = 0
for line in inFp.readlines():
idx += 1
line = line.strip()
if line:
jsonData = json.loads(line)
q_3 = Queue(3)
q_5 = Queue(5)
q_7 = Queue(7)
q_10 = Queue(10)
q_20 = Queue(20)
q_30 = Queue(30)
q_60 = Queue(60)
q_90 = Queue(90)
q_100 = Queue(100)
q_120 = Queue(120)
q_150 = Queue(150)
q_180 = Queue(180)
q_200 = Queue(200)
q_240 = Queue(240)
for item in jsonData["PRICE"]:
q_3.enqueue(item['close'])
q_5.enqueue(item['close'])
q_7.enqueue(item['close'])
q_10.enqueue(item['close'])
q_20.enqueue(item['close'])
q_30.enqueue(item['close'])
q_60.enqueue(item['close'])
q_90.enqueue(item['close'])
q_100.enqueue(item['close'])
q_120.enqueue(item['close'])
q_150.enqueue(item['close'])
q_180.enqueue(item['close'])
q_200.enqueue(item['close'])
q_240.enqueue(item['close'])
item['avg3'] = q_3.avg()
item['avg5'] = q_5.avg()
item['avg7'] = q_7.avg()
item['avg10'] = q_10.avg()
item['avg20'] = q_20.avg()
item['avg30'] = q_30.avg()
item['avg60'] = q_60.avg()
item['avg90'] = q_90.avg()
item['avg100'] = q_100.avg()
item['avg120'] = q_120.avg()
item['avg150'] = q_150.avg()
item['avg180'] = q_180.avg()
item['avg200'] = q_200.avg()
item['avg240'] = q_240.avg()
outFp.write(json.dumps(jsonData, ensure_ascii=False) + "\n")
inFp.close()
outFp.close()
return
def crawl_specific_stock(self, code_name, code, stock):
item_name = code_name
item_code = code
url = 'http://finance.naver.com/item/sise_day.nhn?code={code}'.format(code=item_code.strip())
# 일자 데이터를 담을 df라는 DataFrame 정의
df = pd.DataFrame()
lastDay = ""
if len(stock) > 0 and len(stock["PRICE"])-1 > 0:
lastDay = stock["PRICE"][len(stock["PRICE"])-1]["DATE"].replace("-",".")
lastPage = False
# 1페이지에서 1000페이지의 데이터만 가져오기
for page in range(1, self.limit_page_count):
# 최근 상장 기업의 마지막 반복되는 페이지를 제외시킨다.
pg_url = '{url}&page={page}'.format(url=url, page=page)
#html = pd.read_html(pg_url, header=0)
html = None
while True:
try:
html = pd.read_html(requests.get(pg_url, headers=self.header, timeout=30).text)
sleep(0.5)
break
except:
print(pg_url)
if page > 200:
break
continue
count = 0
for date in html[0].날짜.values:
if type(date) is str:
count += 1
if date == lastDay:
lastPage = True
df = df.append(html[0], ignore_index=True)
break
if count == 10:
df = df.append(html[0], ignore_index=True)
else:
if lastPage == False:
df = df.append(html[0], ignore_index=True)
lastPage = True
else:
break
# df.dropna()를 이용해 결측값 있는 행 제거
df = df.dropna()
# 상위 5개 데이터 확인하기
###print (df.head())
# 한글로 된 컬럼명을 영어로 바꿔줌
df = df.rename(columns={'날짜': 'date', '종가': 'close', '전일비': 'diff', '시가': 'open', '고가': 'high', '저가': 'low', '거래량': 'volume'})
# 데이터의 타입을 int형으로 바꿔줌
df[['close', 'diff', 'open', 'high', 'low', 'volume']] = df[['close', 'diff', 'open', 'high', 'low', 'volume']].astype(int)
# 컬럼명 'date'의 타입을 date로 바꿔줌
df['date'] = pd.to_datetime(df['date'])
# 일자(date)를 기준으로 오름차순 정렬
#df = df.sort_values(by=['date'], ascending=True)
# 상위 5개 데이터 확인
###print (df.head())
if len(stock) > 0 and len(stock["PRICE"]) - 1 > 0:
lastDay = stock["PRICE"][len(stock["PRICE"])-1]["DATE"]
for values in df.values:
day = str(values[0]).split(' ')[0]
if lastDay == day:
break
stock["PRICE"].append({
"DATE": day,
df.columns[1]: values[1],
df.columns[2]: values[2],
df.columns[3]: values[3],
df.columns[4]: values[4],
df.columns[5]: values[5],
df.columns[6]: values[6],
})
#stock["PRICE"] = sorted(stock["PRICE"], key=lambda x: x['DATE'], reverse=True)
stock["PRICE"] = sorted(stock["PRICE"], key=lambda x: x['DATE'])
return stock
def update_stocks(self, inFileName):
stock_inverse = {"NAME": 'KODEX 200선물인버스2X', "CODE": "252670", "PRICE": []}
stock_reverage = {"NAME": 'KODEX 레버리지', "CODE": "122630", "PRICE": []}
stock_gold = {"NAME": 'KODEX 골드선물(H)', "CODE": "132030", "PRICE": []}
stocks = []
if os.path.isfile(inFileName):
inFp = open(inFileName, 'r', encoding='utf-8')
for line in inFp.readlines():
line = line.strip()
if line:
jsonData = json.loads(line)
jsonData["PRICE"] = sorted(jsonData["PRICE"], key=lambda x: x['DATE'], reverse=True)
if jsonData['CODE'] == "252670":
stock_inverse = jsonData
elif jsonData['CODE'] == "122630":
stock_reverage = jsonData
elif jsonData['CODE'] == "132030":
stock_gold = jsonData
else:
stocks.append(jsonData)
inFp.close()
outFp = open(inFileName, 'w', encoding='utf-8')
if len(stocks) == 0:
limit_page_count = 1000
code_df = self.getStockInfo()
stocks = code_df.values
else:
limit_page_count = 2
code_df = None
idx = 0
for item in stocks:
idx += 1
if limit_page_count == 1000:
item_name = item[0]
item_code = item[1]
print(idx, item_name)
stock = {"NAME": item_name, "CODE": item_code, "PRICE": []}
code, url = self.get_url(item_name, code_df)
else:
item_name = item['NAME']
item_code = item['CODE']
print(idx, item_name)
stock = {"NAME": item_name, "CODE": item_code, "PRICE": []}
url = 'http://finance.naver.com/item/sise_day.nhn?code={code}'.format(code=item_code.strip())
# 일자 데이터를 담을 df라는 DataFrame 정의
df = pd.DataFrame()
lastPage = False
# 1페이지에서 1000페이지의 데이터만 가져오기
for page in range(1, limit_page_count):
# 최근 상장 기업의 마지막 반복되는 페이지를 제외시킨다.
pg_url = '{url}&page={page}'.format(url=url, page=page)
#html = pd.read_html(pg_url, header=0)
html = None
while True:
try:
html = pd.read_html(requests.get(pg_url, headers=self.header, timeout=30).text)
sleep(0.5)
break
except:
print(pg_url)
if page > 200:
break
continue
count = 0
for date in html[0].날짜.values:
if type(date) is str:
count += 1
if count == 10:
df = df.append(html[0], ignore_index=True)
else:
if lastPage == False:
df = df.append(html[0], ignore_index=True)
lastPage = True
else:
break
# df.dropna()를 이용해 결측값 있는 행 제거
df = df.dropna()
# 상위 5개 데이터 확인하기
###print (df.head())
# 한글로 된 컬럼명을 영어로 바꿔줌
df = df.rename(columns={'날짜': 'date', '종가': 'close', '전일비': 'diff', '시가': 'open', '고가': 'high', '저가': 'low', '거래량': 'volume'})
# 데이터의 타입을 int형으로 바꿔줌
df[['close', 'diff', 'open', 'high', 'low', 'volume']] = df[['close', 'diff', 'open', 'high', 'low', 'volume']].astype(int)
# 컬럼명 'date'의 타입을 date로 바꿔줌
df['date'] = pd.to_datetime(df['date'])
# 일자(date)를 기준으로 오름차순 정렬
# df = df.sort_values(by=['date'], ascending=False)
# 상위 5개 데이터 확인
###print (df.head())
q_3 = Queue(3)
q_5 = Queue(5)
q_7 = Queue(7)
q_10 = Queue(10)
q_20 = Queue(20)
q_30 = Queue(30)
q_60 = Queue(60)
q_90 = Queue(90)
q_100 = Queue(100)
q_120 = Queue(120)
q_150 = Queue(150)
q_180 = Queue(180)
q_200 = Queue(200)
q_240 = Queue(240)
if limit_page_count == 1000:
for values in df.values:
q_3.enqueue(values[1])
q_5.enqueue(values[1])
q_7.enqueue(values[1])
q_10.enqueue(values[1])
q_20.enqueue(values[1])
q_30.enqueue(values[1])
q_60.enqueue(values[1])
q_90.enqueue(values[1])
q_100.enqueue(values[1])
q_120.enqueue(values[1])
q_150.enqueue(values[1])
q_180.enqueue(values[1])
q_200.enqueue(values[1])
q_240.enqueue(values[1])
stock["PRICE"].append({
"DATE": str(values[0]).split(' ')[0],
df.columns[1]: values[1],
df.columns[2]: values[2],
df.columns[3]: values[3],
df.columns[4]: values[4],
df.columns[5]: values[5],
df.columns[6]: values[6],
'avg3': q_3.avg(),
'avg5': q_5.avg(),
'avg7': q_7.avg(),
'avg10': q_10.avg(),
'avg20': q_20.avg(),
'avg30': q_30.avg(),
'avg60': q_60.avg(),
'avg90': q_90.avg(),
'avg100': q_100.avg(),
'avg120': q_120.avg(),
'avg150': q_150.avg(),
'avg180': q_180.avg(),
'avg200': q_200.avg(),
'avg240': q_240.avg()
})
else:
for values in item["PRICE"]:
q_3.enqueue(values["close"])
q_5.enqueue(values["close"])
q_7.enqueue(values["close"])
q_10.enqueue(values["close"])
q_20.enqueue(values["close"])
q_30.enqueue(values["close"])
q_60.enqueue(values["close"])
q_90.enqueue(values["close"])
q_100.enqueue(values["close"])
q_120.enqueue(values["close"])
q_150.enqueue(values["close"])
q_180.enqueue(values["close"])
q_200.enqueue(values["close"])
q_240.enqueue(values["close"])
# 기존 파일에서 읽은 것
stock["PRICE"].append({
"DATE": str(values["DATE"]).split(' ')[0],
df.columns[1]: values["close"],
df.columns[2]: values["diff"],
df.columns[3]: values["open"],
df.columns[4]: values["high"],
df.columns[5]: values["low"],
df.columns[6]: values["volume"],
'avg3': q_5.avg(),
'avg5': q_5.avg(),
'avg7': q_5.avg(),
'avg10': q_10.avg(),
'avg20': q_20.avg(),
'avg30': q_30.avg(),
'avg60': q_60.avg(),
'avg90': q_90.avg(),
'avg100': q_100.avg(),
'avg120': q_120.avg(),
'avg150': q_150.avg(),
'avg180': q_180.avg(),
'avg200': q_200.avg(),
'avg240': q_240.avg()
})
if limit_page_count != 1000:
# 새로 웹에서 수집한 것
for values in df.values:
date = str(values[0]).split(' ')[0]
isExist = False
for i in range(len(stock["PRICE"])):
if (stock["PRICE"][i]['DATE'] == date):
stock["PRICE"][i][df.columns[1]] = values[1]
stock["PRICE"][i][df.columns[2]] = values[2]
stock["PRICE"][i][df.columns[3]] = values[3]
stock["PRICE"][i][df.columns[4]] = values[4]
stock["PRICE"][i][df.columns[5]] = values[5]
stock["PRICE"][i][df.columns[6]] = values[6]
isExist = True
break
# 새로운 데이터나 오늘 날짜의 데이터
if not isExist:
q_3.enqueue(values[1])
q_5.enqueue(values[1])
q_7.enqueue(values[1])
q_10.enqueue(values[1])
q_20.enqueue(values[1])
q_30.enqueue(values[1])
q_60.enqueue(values[1])
q_90.enqueue(values[1])
q_100.enqueue(values[1])
q_120.enqueue(values[1])
q_150.enqueue(values[1])
q_180.enqueue(values[1])
q_200.enqueue(values[1])
q_240.enqueue(values[1])
stock["PRICE"].append({
"DATE": str(values[0]).split(' ')[0],
df.columns[1]: values[1],
df.columns[2]: values[2],
df.columns[3]: values[3],
df.columns[4]: values[4],
df.columns[5]: values[5],
df.columns[6]: values[6],
'avg3': q_3.avg(),
'avg5': q_5.avg(),
'avg7': q_7.avg(),
'avg10': q_10.avg(),
'avg20': q_20.avg(),
'avg30': q_30.avg(),
'avg60': q_60.avg(),
'avg90': q_90.avg(),
'avg100': q_100.avg(),
'avg120': q_120.avg(),
'avg150': q_150.avg(),
'avg180': q_180.avg(),
'avg200': q_200.avg(),
'avg240': q_240.avg()
})
stock["PRICE"] = sorted(stock["PRICE"], key=lambda x: x['DATE'], reverse=True)
outFp.write(json.dumps(stock, ensure_ascii=False)+"\n")
kodex_inverse = self.crawl_specific_stock('KODEX 200선물인버스2X', '252670', stock_inverse)
outFp.write(json.dumps(kodex_inverse, ensure_ascii=False) + "\n")
kodex_reverage = self.crawl_specific_stock('KODEX 레버리지', '122630', stock_reverage)
outFp.write(json.dumps(kodex_reverage, ensure_ascii=False) + "\n")
kodex_gold = self.crawl_specific_stock('KODEX 골드선물(H)', '132030', stock_gold)
outFp.write(json.dumps(kodex_gold, ensure_ascii=False) + "\n")
outFp.close()
return
def update_specific_stock(self, code_name, code, stock):
item_name = code_name
item_code = code
print(item_name)
if len(stock["PRICE"]) == 0:
limit_page_count = 1000
else:
limit_page_count = 2
url = 'http://finance.naver.com/item/sise_day.nhn?code={code}'.format(code=item_code.strip())
# 일자 데이터를 담을 df라는 DataFrame 정의
df = pd.DataFrame()
lastPage = False
# 1페이지에서 1000페이지의 데이터만 가져오기
for page in range(1, limit_page_count):
# 최근 상장 기업의 마지막 반복되는 페이지를 제외시킨다.
pg_url = '{url}&page={page}'.format(url=url, page=page)
#html = pd.read_html(pg_url, header=0)
html = None
while True:
try:
html = pd.read_html(requests.get(pg_url, headers=self.header, timeout=30).text)
sleep(0.5)
break
except:
print(pg_url)
if page > 200:
break
continue
count = 0
for date in html[0].날짜.values:
if type(date) is str:
count += 1
if count == 10:
df = df.append(html[0], ignore_index=True)
else:
if lastPage == False:
df = df.append(html[0], ignore_index=True)
lastPage = True
else:
break
# df.dropna()를 이용해 결측값 있는 행 제거
df = df.dropna()
# 상위 5개 데이터 확인하기
###print (df.head())
# 한글로 된 컬럼명을 영어로 바꿔줌
df = df.rename(columns={'날짜': 'date', '종가': 'close', '전일비': 'diff', '시가': 'open', '고가': 'high', '저가': 'low', '거래량': 'volume'})
# 데이터의 타입을 int형으로 바꿔줌
df[['close', 'diff', 'open', 'high', 'low', 'volume']] = df[['close', 'diff', 'open', 'high', 'low', 'volume']].astype(int)
# 컬럼명 'date'의 타입을 date로 바꿔줌
df['date'] = pd.to_datetime(df['date'])
# 일자(date)를 기준으로 오름차순 정렬
#df = df.sort_values(by=['date'], ascending=True)
# 상위 5개 데이터 확인
###print (df.head())
q_3 = Queue(3)
q_5 = Queue(5)
q_7 = Queue(7)
q_10 = Queue(10)
q_20 = Queue(20)
q_30 = Queue(30)
q_60 = Queue(60)
q_90 = Queue(90)
q_100 = Queue(100)
q_120 = Queue(120)
q_150 = Queue(150)
q_180 = Queue(180)
q_200 = Queue(200)
q_240 = Queue(240)
for values in df.values:
q_3.enqueue(values[1])
q_5.enqueue(values[1])
q_7.enqueue(values[1])
q_10.enqueue(values[1])
q_20.enqueue(values[1])
q_30.enqueue(values[1])
q_60.enqueue(values[1])
q_90.enqueue(values[1])
q_100.enqueue(values[1])
q_120.enqueue(values[1])
q_150.enqueue(values[1])
q_180.enqueue(values[1])
q_200.enqueue(values[1])
q_240.enqueue(values[1])
stock["PRICE"].append({
"DATE": str(values[0]).split(' ')[0],
df.columns[1]: values[1],
df.columns[2]: values[2],
df.columns[3]: values[3],
df.columns[4]: values[4],
df.columns[5]: values[5],
df.columns[6]: values[6],
'avg3': q_3.avg(),
'avg5': q_5.avg(),
'avg7': q_7.avg(),
'avg10': q_10.avg(),
'avg20': q_20.avg(),
'avg30': q_30.avg(),
'avg60': q_60.avg(),
'avg90': q_90.avg(),
'avg100': q_100.avg(),
'avg120': q_120.avg(),
'avg150': q_150.avg(),
'avg180': q_180.avg(),
'avg200': q_200.avg(),
'avg240': q_240.avg()
})
stock["PRICE"] = sorted(stock["PRICE"], key=lambda x: x['DATE'], reverse=True)
return stock

80
stockpredictor_back/crawler/toSQLite/Crawler.py
View File

@@ -1,80 +0,0 @@
import os
import shutil
import datetime
import time
from stockpredictor.crawler.toSQLite.FnGuideCrawler import FnGuideCrawler
from stockpredictor.crawler.toSQLite.MetaCrawler import MetaCrawler
from stockpredictor.crawler.toSQLite.StockCrawler import StockCrawler
from stockpredictor.analysis.Analyzer import Analyzer
today = datetime.datetime.now().strftime("%Y-%m-%d")
# DB Browser for SQLite: http://hleecaster.com/python-sqlite3/
PROJECT_HOME = os.path.join(os.path.dirname(os.path.join(os.path.dirname(os.path.join(os.path.dirname(os.path.join(os.path.dirname(__file__))))))))
start = time.time()
inFnguideFileName = PROJECT_HOME + '/resources/fnguide.db'
"""
# 재무제표는 3개월마다 다운로드를 한다.
crawler = FnGuideCrawler()
print("[KOSPI 상장기업 재무제표 다운로드]")
crawler.crawl_fnguide(inFnguideFileName)
print("\n[증시자금동향 (신용잔고, 펀드자금 잔고)]")
inFileName = PROJECT_HOME + '/resources/meta_3.db'
crawler.crawl_money_trend(inFileName)
print("\n[국내 시장금리]")
inFileName = PROJECT_HOME + '/resources/meta_4.db'
crawler.crawl_interest_rates(inFileName)
"""
crawler = MetaCrawler()
print("\n[투자자별 매매동향(Trading_Trend)]")
inFileName = PROJECT_HOME + '/resources/meta_2.db'
crawler.crawl_trading_trend(inFileName)
print("\n[환율 (USD, JPY, EUR, CNY), 원유 (WTI), 국제금]")
inFileName = PROJECT_HOME + '/resources/meta_1.db'
crawler.crawl_stocks(inFileName)
print("\n[종목 다운로드]")
inFileName = PROJECT_HOME + '/resources/stock.db'
crawler = StockCrawler()
crawler.crawl_etf_stocks(inFileName)
crawler.crawl_stocks(inFileName)
print("\n[지수 저장]")
kospiFileName = PROJECT_HOME + '/resources/kospi.tsv'
kosdakFileName = PROJECT_HOME + '/resources/kosdak.tsv'
outFileName = PROJECT_HOME + '/resources/stock.db'
crawler = StockCrawler()
crawler.saveIndex("KOSPI", kospiFileName, outFileName)
crawler.saveIndex("KOSDAK", kosdakFileName, outFileName)
print("\n[종목 분석]")
# S: 분석까지 진행
inFileName = PROJECT_HOME + '/resources/stock.db'
analyzer = Analyzer(PROJECT_HOME, inFileName, inFnguideFileName)
analyzer.analyze()
print("\n[종목 결정]")
day = datetime.datetime.today().strftime("%Y%m%d")
outPath = PROJECT_HOME + "/resources/analysis/" + day
if os.path.isdir(outPath):
shutil.rmtree(outPath)
os.mkdir(outPath)
print("print to Html...")
analyzer.analyzeToHtml(outPath)
# E: 분석까지 진행
print("time : %6.2f 초", (time.time() - start))
print ("done...")

129
stockpredictor_back/crawler/toSQLite/FnGuideCrawler.py
View File

@@ -1,129 +0,0 @@
from bs4 import BeautifulSoup
from pandas import DataFrame, Series
import requests as re
import pandas as pd
import os
import json
import sqlite3
import requests
class FnGuideCrawler:
header = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36'}
def getStockInfo(self):
code_df = pd.read_html('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', header=0)[0]
#code_df = pd.read_html(requests.get('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', headers=self.header).text)
# 종목코드가 6자리이기 때문에 6자리를 맞춰주기 위해 설정해줌
code_df.종목코드 = code_df.종목코드.map('{:06d}'.format)
# 우리가 필요한 것은 회사명과 종목코드이기 때문에 필요없는 column들은 제외해준다.
code_df = code_df[['회사명', '종목코드']]
# 한글로된 컬럼명을 영어로 바꿔준다.
code_df = code_df.rename(columns={'회사명': 'name', '종목코드': 'code'})
###print (code_df.head())
return code_df
# FnGuide에서 크롤링한 KOSPI 상장기업의 재무제표
# http://blog.naver.com/PostView.nhn?blogId=koko8624&logNo=221294884955&parentCategoryNo=&categoryNo=&viewDate=&isShowPopularPosts=false&from=postView
def get_fnguide_table(self, code):
url = re.get('http://comp.fnguide.com/SVO2/ASP/SVD_main.asp?pGB=1&gicode=A%s'%(code.strip()))
url = url.content
html = BeautifulSoup(url,'html.parser')
body = html.find('body')
try:
fn_body = body.find('div', {'class': 'fng_body asp_body'})
ur_table = fn_body.find('div', {'id': 'div15'})
table = ur_table.find('div', {'id': 'highlight_D_Y'})
tbody = table.find('tbody')
tr = tbody.find_all('tr')
Table = DataFrame()
except:
return {}
for i in tr:
''' 자료 항목 가져오기'''
category = i.find('span', {'class': 'txt_acd'})
if category == None:
category = i.find('th')
category = category.text.strip()
'''값 가져오기'''
value_list = []
j = i.find_all('td', {'class': 'r'})
for value in j:
temp = value.text.replace(',', '').strip()
try:
temp = float(temp)
value_list.append(temp)
except:
value_list.append(0)
Table['%s' % (category)] = value_list
''' 기간 가져오기 '''
thead = table.find('thead')
tr_2 = thead.find('tr', {'class': 'td_gapcolor2'}).find_all('th')
year_list = []
for i in tr_2:
try:
temp_year = i.find('span', {'class': 'txt_acd'}).text
except:
temp_year = i.text
temp_year = temp_year.replace("/",".")+".01"
year_list.append(temp_year)
Table.index = year_list
return Table.T.to_dict()
def crawl_fnguide(self, inFileName):
tableName = 'fnguide'
conn = sqlite3.connect(inFileName, isolation_level=None)
cursor = conn.cursor()
cursor.execute("CREATE TABLE IF NOT EXISTS "+tableName+" (CODE text PRIMARY KEY, NAME text, PRICE text)")
code_df = self.getStockInfo()
idx = 0
for item in code_df.values:
item_name = item[0]
item_code = item[1]
idx += 1
print(idx, item_name)
fnGuideData = self.get_fnguide_table(item_code)
text = json.dumps(fnGuideData, ensure_ascii=False)
cursor.execute('SELECT * FROM '+tableName+' WHERE CODE=?', (item_code, ))
result = cursor.fetchone()
if result == None:
cursor.execute("INSERT INTO "+tableName+"(CODE, NAME, PRICE) VALUES(?, ?, ?)", (item_code, item_name, text))
else:
cursor.execute("UPDATE "+tableName+" SET PRICE=? WHERE CODE=?", (text, item_code))
cursor.close()
conn.close()
return
if __name__ == "__main__":
crawler = FnGuideCrawler()
#crawler.get_fnguide_table('155660')
PROJECT_HOME = os.path.join(os.path.dirname(os.path.join(os.path.dirname(os.path.join(os.path.dirname(os.path.join(os.path.dirname(__file__))))))))
inFnguideFileName = PROJECT_HOME + '/resources/fnguide.db'
crawler = FnGuideCrawler()
crawler.crawl_fnguide(inFnguideFileName)

341
stockpredictor_back/crawler/toSQLite/MetaCrawler.py
View File

@@ -1,341 +0,0 @@
import json
import datetime
import requests
import sqlite3
from time import sleep
import pandas as pd
import os
class MetaCrawler:
header = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36'}
limit_page_count = 10000
def __init__(self):
return
# 참고) http://blog.naver.com/PostView.nhn?blogId=koko8624&logNo=221288761509
def crawl_stocks(self, inFileName):
tableName = 'meta_1'
conn = sqlite3.connect(inFileName)
cursor = conn.cursor()
cursor.execute("CREATE TABLE IF NOT EXISTS "+tableName+" (CODE text PRIMARY KEY, NAME text, PRICE text)")
inputs = []
inputs.append( {'NAME':'USD', 'CODE':'FX_USDKRW', 'URL':'http://finance.naver.com/marketindex/exchangeDailyQuote.nhn?marketindexCd=FX_USDKRW'} ) # 미국 USD
inputs.append( {'NAME':'JPY', 'CODE':'FX_JPYKRW', 'URL':'http://finance.naver.com/marketindex/exchangeDailyQuote.nhn?marketindexCd=FX_JPYKRW'} ) # 일본 JPY
inputs.append( {'NAME':'EUR', 'CODE':'FX_EURKRW', 'URL':'http://finance.naver.com/marketindex/exchangeDailyQuote.nhn?marketindexCd=FX_EURKRW'} ) # 유럽연합 EUR'
inputs.append( {'NAME':'CNY', 'CODE':'FX_CNYKRW', 'URL':'http://finance.naver.com/marketindex/exchangeDailyQuote.nhn?marketindexCd=FX_CNYKRW'} ) # 중국 CNY
inputs.append( {'NAME':'WTI', 'CODE':'OIL_CL', 'URL':'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=OIL_CL&fdtc=2'} ) # WTI
inputs.append( {'NAME':'GOLD', 'CODE':'CMDT_GC', 'URL':'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=CMDT_GC&fdtc=2'} ) # 국제 금
for i in range(len(inputs)):
input = inputs[i]
meta = {}
meta["NAME"] = input['NAME']
meta["CODE"] = input['CODE']
meta["PRICE"] = []
cursor.execute('SELECT * FROM ' + tableName + ' WHERE CODE=?', (meta["CODE"],))
result = cursor.fetchone()
if result == None:
lastDay = "1900.01.01"
else:
meta["PRICE"] = json.loads(result[2])
lastDay = meta['PRICE'][0]['DATE']
finish = False
for i in range(1, self.limit_page_count):
#html = pd.read_html(input['URL'] + '&page=%s' % i, header=0)
html = None
while True:
try:
html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header, timeout=30).text)
sleep(0.5)
break
except:
print(input['URL'] + '&page=%s' % i)
if i > 200:
break
continue
# 마지막 페이지 까지 받기
if len(html[0].날짜.values) <= 1:
break
for j in range(0, len(html[0].values)):
item = html[0].values[j]
if input['NAME'] in ('USD', 'JPY', 'EUR', 'CNY'):
if j == 0:
continue
if item[0] <= lastDay:
finish = True
break
meta["PRICE"].append({
"DATE": item[0], # 날짜
"close": item[1], # 매매기준율
"diff": item[2] # 전일대비
})
elif input['NAME'] in ('WTI', 'GOLD'):
if item[0] <= lastDay:
finish = True
break
meta["PRICE"].append({
"DATE": item[0], # 날짜
"close": item[1], # 종가
"diff": item[2], # 전일대비
"rate": item[3] # 등락율
})
if finish:
break
meta["PRICE"] = sorted(meta["PRICE"], key=lambda x: x['DATE'], reverse=True)
text = json.dumps(meta["PRICE"], ensure_ascii=False)
cursor.execute('SELECT * FROM '+tableName+' WHERE CODE=?', (meta["CODE"], ))
result = cursor.fetchone()
if result == None:
cursor.execute("INSERT INTO "+tableName+"(CODE, NAME, PRICE) VALUES(?, ?, ?)", (meta["CODE"], meta["NAME"], text))
else:
cursor.execute("UPDATE "+tableName+" SET PRICE=? WHERE CODE=?", (text, meta["CODE"]))
conn.commit()
cursor.close()
conn.close()
return
# 투자자별 매매동향 (Trading_Trend) 크롤링
# (pri, 개인)
# (for, 외국인)
# (ins, 기관합)
# (ins0, 금융투자)
# (ins1, 보험)
# (ins2, 투신 (사모))
# (ins3, 은행)
# (ins4, 기타금융기관)
# (ins5, 연기금 등)
# (cor, 기타법인)
# 참고) http://blog.naver.com/PostView.nhn?blogId=koko8624&logNo=221289696771&parentCategoryNo=&categoryNo=&viewDate=&isShowPopularPosts=false&from=postView
def crawl_trading_trend(self, inFileName):
tableName = 'meta_2'
conn = sqlite3.connect(inFileName)
cursor = conn.cursor()
cursor.execute("CREATE TABLE IF NOT EXISTS "+tableName+" (DATE text PRIMARY KEY, pri integer, for integer, ins integer, ins0 integer, ins1 integer, ins2 integer, ins3 integer, ins4 integer, ins5 integer, cor integer)")
cursor.execute('SELECT * FROM ' + tableName + ' order by DATE desc')
result = cursor.fetchone()
if result == None:
lastDay = "1900.01.01"
else:
lastDay = result[0]
today = datetime.datetime.now().strftime("%Y%m%d")
url = 'http://finance.naver.com/sise/investorDealTrendDay.nhn?bizdate='+today+'&sosok=&page='
previousDay = ""
finish = False
for i in range(1, self.limit_page_count):
#html = pd.read_html(url + str(i), header=0)
html = None
while True:
try:
html = pd.read_html(requests.get(url + str(i), headers=self.header, timeout=30).text)
sleep(0.5)
break
except:
print(url + str(i))
if i > 200:
break
continue
# 마지막 페이지 까지 받기
if len(html[0].날짜.values) <= 2:
break
for j in range(1, len(html[0].values)):
item = html[0].values[j]
if str(item[0]) == "nan":
continue
if "20" + item[0] <= lastDay or item[0] == previousDay:
finish = True
break
meta = {
"DATE": "20"+item[0],
"pri": item[1], # 개인
"for": item[2], # 외국인
"ins": item[3], # 기관합
"ins0": item[4], # 금융투자
"ins1": item[5], # 보험
"ins2": item[6], # 투신 (사모)
"ins3": item[7], # 은행
"ins4": item[8], # 기타금융기관
"ins5": item[9], # 연기금 등
"cor": item[10]} # 기타법인
cursor.execute('SELECT * FROM ' + tableName + ' WHERE DATE=?', (meta["DATE"],))
result = cursor.fetchone()
if result == None:
cursor.execute("INSERT INTO " + tableName + "(DATE, pri, for, ins, ins0, ins1, ins2, ins3, ins4, ins5, cor) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (meta["DATE"], meta["pri"], meta["for"], meta["ins"], meta["ins0"], meta["ins1"], meta["ins2"], meta["ins3"], meta["ins4"], meta["ins5"], meta["cor"]))
else:
cursor.execute("UPDATE " + tableName + " SET pri=?, for=?, ins=?, ins0=?, ins1=?, ins2=?, ins3=?, ins4=?, ins5=?, cor=? WHERE DATE=?", (meta["pri"], meta["for"], meta["ins"], meta["ins0"], meta["ins1"], meta["ins2"], meta["ins3"], meta["ins4"], meta["ins5"], meta["cor"], meta["DATE"]))
print ("20"+item[0])
previousDay = html[0].values[2][0]
if finish:
break
conn.commit()
cursor.close()
conn.close()
return
# 증시자금동향 (신용잔고, 펀드자금 잔고) 크롤링
# 참고) http://blog.naver.com/PostView.nhn?blogId=koko8624&logNo=221290138187&parentCategoryNo=&categoryNo=&viewDate=&isShowPopularPosts=false&from=postView
def crawl_money_trend(self, inFileName):
tableName = 'meta_3'
conn = sqlite3.connect(inFileName)
cursor = conn.cursor()
cursor.execute("CREATE TABLE IF NOT EXISTS "+tableName+" (DATE text PRIMARY KEY, dep1_1 integer, dep1_2 integer, dep2_1 integer, dep2_2 integer, dep3_1 integer, dep3_2 integer, dep4_1 integer, dep4_2 integer, dep5_1 integer, dep5_2 integer)")
cursor.execute('SELECT * FROM ' + tableName + ' order by DATE desc')
result = cursor.fetchone()
if result == None:
lastDay = "1900.01.01"
else:
lastDay = result[0]
previousDay = ""
url = 'http://finance.naver.com/sise/sise_deposit.nhn?&page='
finish = False
for i in range(1, self.limit_page_count):
#html = pd.read_html(url + str(i), header=0, encoding='euc-kr')
html = None
while True:
try:
html = pd.read_html(requests.get(url + str(i), headers=self.header, timeout=30).text, encoding='euc-kr')
sleep(0.5)
break
except:
print(url + str(i))
if i > 200:
break
continue
# 마지막 페이지 까지 받기
if len(html[0].날짜.values) <= 10:
break
for j in range(1, len(html[0].values)):
item = html[0].values[j]
if str(item[0]) == "nan":
continue
if "20"+item[0] <= lastDay or item[0] == previousDay:
finish = True
break
meta = {
"DATE": "20"+item[0],
"dep1_1": item[1], # 고객예탁금 누적
"dep1_2": item[2], # 고객예탁금 당일
"dep2_1": item[3], # 신용잔고 누적
"dep2_2": item[4], # 신용잔고 당일
"dep3_1": item[5], # 주식형펀드 누적
"dep3_2": item[6], # 주식형펀드 당일
"dep4_1": item[7], # 혼합형펀드 누적
"dep4_2": item[8], # 혼합형펀드 당일
"dep5_1": item[9], # 채권형펀드 누적
"dep5_2": item[10]} # 채권형펀드 당일
cursor.execute('SELECT * FROM ' + tableName + ' WHERE DATE=?', (meta["DATE"],))
result = cursor.fetchone()
if result == None:
cursor.execute("INSERT INTO " + tableName + "(DATE, dep1_1, dep1_2, dep2_1, dep2_2, dep3_1, dep3_2, dep4_1, dep4_2, dep5_1, dep5_2) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (meta["DATE"], meta["dep1_1"], meta["dep1_2"], meta["dep2_1"], meta["dep2_2"], meta["dep3_1"], meta["dep3_2"], meta["dep4_1"], meta["dep4_2"], meta["dep5_1"], meta["dep5_2"]))
else:
cursor.execute("UPDATE " + tableName + " SET dep1_1=?, dep1_2=?, dep2_1=?, dep2_2=?, dep3_1=?, dep3_2=?, dep4_1=?, dep4_2=?, dep5_1=?, dep5_2=? WHERE DATE=?", (meta["dep1_1"], meta["dep1_2"], meta["dep2_1"], meta["dep2_2"], meta["dep3_1"], meta["dep3_2"], meta["dep4_1"], meta["dep4_2"], meta["dep5_1"], meta["dep5_2"], meta["DATE"]))
print("20"+item[0])
if finish:
break
previousDay = html[0].values[2][0]
conn.commit()
cursor.close()
conn.close()
return
# 국내 시장금리 크롤링
# 참고) http://blog.naver.com/PostView.nhn?blogId=koko8624&logNo=221292348073&parentCategoryNo=&categoryNo=&viewDate=&isShowPopularPosts=false&from=postView
def crawl_interest_rates(self, inFileName):
tableName = 'meta_4'
conn = sqlite3.connect(inFileName)
cursor = conn.cursor()
cursor.execute("CREATE TABLE IF NOT EXISTS " + tableName + " (CODE text PRIMARY KEY, NAME text, PRICE text)")
inputs = []
inputs.append({'NAME': '91일 CD금리', 'CODE': 'IRR_CD91', 'URL': 'http://finance.naver.com/marketindex/interestDailyQuote.nhn?marketindexCd=IRR_CD91'})
inputs.append({'NAME': '콜금리', 'CODE': 'IRR_CALL', 'URL': 'http://finance.naver.com/marketindex/interestDailyQuote.nhn?marketindexCd=IRR_CALL'})
inputs.append({'NAME': '국고채(3년)', 'CODE': 'IRR_GOVT03Y', 'URL': 'http://finance.naver.com/marketindex/interestDailyQuote.nhn?marketindexCd=IRR_GOVT03Y'})
inputs.append({'NAME': '회사채(3년)', 'CODE': 'IRR_CORP03Y', 'URL': 'http://finance.naver.com/marketindex/interestDailyQuote.nhn?marketindexCd=IRR_CORP03Y'})
for i in range(len(inputs)):
input = inputs[i]
meta = {}
meta["NAME"] = input['NAME']
meta["CODE"] = input['CODE']
meta["PRICE"] = []
cursor.execute('SELECT * FROM ' + tableName + ' WHERE CODE=?', (meta["CODE"],))
result = cursor.fetchone()
if result == None:
lastDay = "1900.01.01"
else:
meta["PRICE"] = json.loads(result[2])
lastDay = meta['PRICE'][0]['DATE']
finish = False
for i in range(1, self.limit_page_count):
#html = pd.read_html(input['URL'] + '&page=%s' % i, header=0)
html = None
while True:
try:
html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header, timeout=30).text)
sleep(0.5)
break
except:
print(input['URL'] + '&page=%s' % i)
if i > 200:
break
continue
# 마지막 페이지 까지 받기
if len(html[0].날짜.values) <= 1:
break
for j in range(len(html[0].values)):
item = html[0].values[j]
if str(item[0]) == "nan":
continue
if item[0] <= lastDay:
finish = True
break
meta["PRICE"].append({
"DATE": item[0],
"close": item[1], # 종가
"diff": item[2], # 전일대비
"rate": item[3]}) # 등락율
if finish:
break
print(meta["NAME"] + " / " + item[0])
meta["PRICE"] = sorted(meta["PRICE"], key=lambda x: x['DATE'], reverse=True)
text = json.dumps(meta["PRICE"], ensure_ascii=False)
cursor.execute('SELECT * FROM ' + tableName + ' WHERE CODE=?', (meta["CODE"],))
result = cursor.fetchone()
if result == None:
cursor.execute("INSERT INTO " + tableName + "(CODE, NAME, PRICE) VALUES(?, ?, ?)", (meta["CODE"], meta["NAME"], text))
else:
cursor.execute("UPDATE " + tableName + " SET PRICE=? WHERE CODE=?", (text, meta["CODE"]))
conn.commit()
cursor.close()
conn.close()
return

464
stockpredictor_back/crawler/toSQLite/StockCrawler.py
View File

@@ -1,464 +0,0 @@
# https://bigdata-sk.tistory.com/10
import pandas as pd
import re
import json
import sqlite3
import requests
import math
import time
from time import sleep
class Queue(object):
def __init__(self, max):
self.queue = []
self.max = max
def dequeue(self):
length = len(self.queue)
if length == 0 or length < self.max:
return -1
return self.queue.pop(0)
def enqueue(self, n):
length = len(self.queue)
if length == self.max:
self.dequeue()
self.queue.append(n)
pass
def sum(self):
sum = 0
for item in self.queue:
sum += item
return sum
def avg(self):
length = len(self.queue)
total = self.sum()
return round(total / length)
def print(self):
print(self.sum(), self.queue)
# 닐짜 형식으로 바뀐 this_date값을 확인 가능
# 읽어온 날짜 정보를 date형식으로 바꿀 일이 계속 생기므로 이 기능을 함수로 정의해줌.
# 함수명은 date_format()
class StockCrawler:
header = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36'}
historical_prices = None
special_pattern = None
fnGuideCrawler = None
limit_page_count = 10000
def __init__(self):
self.historical_prices = dict()
self.special_pattern = (
'[', '!', '@', '#', '$', '%', '^', '&', '*', '(', ')', ',', '.', '?', '"', ':', ';', '{', '}', '|', '<', '>',
']', '+', '-', '/', '=', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9')
return
def clean_str(self, string):
string = re.sub(r"\\", " ", string)
string = re.sub(r"\'", " ", string)
string = re.sub(r"\"", " ", string)
string = re.sub(r"`", " ", string)
string = re.sub(r"-", " ", string)
string = re.sub(r"\(.*?\)", " ", string)
string = re.sub(r" ", " ", string)
return string.strip().lower()
def getStockInfo(self):
#code_df = pd.read_html('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', header=0)[0]
code_df = pd.read_html(requests.get('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', headers=self.header, timeout=30).text)[0]
# code_df = pd.read_excel('../resources/stock/상장법인목록.xls')
# 종목코드가 6자리이기 때문에 6자리를 맞춰주기 위해 설정해줌
code_df.종목코드 = code_df.종목코드.map('{:06d}'.format)
# 우리가 필요한 것은 회사명과 종목코드이기 때문에 필요없는 column들은 제외해준다.
code_df = code_df[['회사명', '종목코드']]
# 한글로된 컬럼명을 영어로 바꿔준다.
code_df = code_df.rename(columns={'회사명': 'name', '종목코드': 'code'})
###print (code_df.head())
return code_df
# 종목 이름을 입력하면 종목에 해당하는 코드를 불러와
# 네이버 금융(http://finance.naver.com)에 넣어줌
def get_url(self, item_name, code_df):
code = code_df.query("name=='{}'".format(item_name))['code'].to_string(index=False).strip()
url = 'http://finance.naver.com/item/sise_day.nhn?code={code}'.format(code=code.strip())
return code, url
def date_format(slef, d):
d = str(d).replace('-', '.')
#yyyy = int(d.split('.')[0])
#mm = int(d.split('.')[1])
#dd = int(d.split('.')[2])
#this_date = dt.date(yyyy, mm, dd)
return d
def getCodeIndex(self, stocks, item_code):
for i, stock in enumerate(stocks):
if item_code == stock['CODE']:
return i
return -1
def crawl_etf_stocks(self, inFileName):
tableName = 'stock'
conn = sqlite3.connect(inFileName)
cursor = conn.cursor()
cursor.execute("CREATE TABLE IF NOT EXISTS " + tableName + " (CODE text PRIMARY KEY, NAME text, PRICE text, MACD text, STOCHASTIC text, ICHIMOKU text, RSI text, BOLINGERBAND text)")
stocks = []
stocks.append({"NAME": 'KODEX 코스닥150선물인버스', "CODE": "251340", "PRICE": []})
stocks.append({"NAME": 'KODEX 코스닥150 레버리지', "CODE": "233740", "PRICE": []})
stocks.append({"NAME": 'KODEX 200선물인버스2X', "CODE": "252670", "PRICE": []})
stocks.append({"NAME": 'KODEX 레버리지', "CODE": "122630", "PRICE": []})
stocks.append({"NAME": 'KODEX 인버스', "CODE": "114800", "PRICE": []})
stocks.append({"NAME": 'KODEX 중국본토CSI300', "CODE": "283580", "PRICE": []})
stocks.append({"NAME": 'KODEX 심천ChiNext(합성)', "CODE": "256750", "PRICE": []})
stocks.append({"NAME": 'KINDEX 블룸버그베트남VN30선물레버리지(H)', "CODE": "371130", "PRICE": []})
stocks.append({"NAME": 'KODEX 미국S&P바이오(합성)', "CODE": "185680", "PRICE": []})
stocks.append({"NAME": 'KODEX 미국S&P에너지(합성)', "CODE": "218420", "PRICE": []})
stocks.append({"NAME": 'KODEX 골드선물(H)', "CODE": "132030", "PRICE": []})
stocks.append({"NAME": 'KODEX 콩선물(H)', "CODE": "138920", "PRICE": []})
stocks.append({"NAME": 'KODEX 3대농산물선물(H)', "CODE": "271060", "PRICE": []})
stocks.append({"NAME": 'KODEX 건설', "CODE": "117700", "PRICE": []})
stocks.append({"NAME": 'KODEX 헬스케어', "CODE": "266420", "PRICE": []})
stocks.append({"NAME": 'KODEX 글로벌4차산업로보틱스(합성)', "CODE": "276990", "PRICE": []})
stocks.append({"NAME": 'KODEX 바이오', "CODE": "244580", "PRICE": []})
stocks.append({"NAME": 'KODEX 반도체', "CODE": "091160", "PRICE": []})
stocks.append({"NAME": 'KODEX 보험', "CODE": "140700", "PRICE": []})
stocks.append({"NAME": 'KODEX 필수소비재', "CODE": "266410", "PRICE": []})
stocks.append({"NAME": 'KODEX 2차전지산업', "CODE": "305720", "PRICE": []})
stocks.append({"NAME": 'KODEX 경기소비재', "CODE": "266390", "PRICE": []})
stocks.append({"NAME": 'KODEX 철강', "CODE": "117680", "PRICE": []})
stocks.append({"NAME": 'KODEX 에너지화학', "CODE": "117460", "PRICE": []})
stocks.append({"NAME": 'KODEX 은행', "CODE": "091170", "PRICE": []})
stocks.append({"NAME": 'TIGER 탄소효율그린뉴딜', "CODE": "376410", "PRICE": []})
start_time = time.time()
for i, stock in enumerate(stocks):
print (i, stock["NAME"], stock["CODE"], (time.time()-start_time), "s")
start_time = time.time()
cursor.execute('SELECT * FROM ' + tableName + ' WHERE CODE=?', (stock["CODE"],))
result = cursor.fetchone()
if result is not None:
stock["PRICE"] = json.loads(result[2])
self.crawl_specific_stock(stock)
text = json.dumps(stock['PRICE'], ensure_ascii=False)
cursor.execute('SELECT * FROM ' + tableName + ' WHERE CODE=?', (stock["CODE"],))
result = cursor.fetchone()
if result == None:
cursor.execute("INSERT INTO " + tableName + "(CODE, NAME, PRICE) VALUES(?, ?, ?)", (stock["CODE"], stock["NAME"], text))
else:
cursor.execute("UPDATE " + tableName + " SET PRICE=? WHERE CODE=?", (text, stock["CODE"]))
conn.commit()
cursor.close()
conn.close()
return
def crawl_stocks(self, inFileName):
tableName = 'stock'
conn = sqlite3.connect(inFileName)
cursor = conn.cursor()
cursor.execute("CREATE TABLE IF NOT EXISTS " + tableName + " (CODE text PRIMARY KEY, NAME text, PRICE text, MACD text, STOCHASTIC text, ICHIMOKU text, RSI text, BOLINGERBAND text)")
code_df = self.getStockInfo()
items = code_df.values
start_time = time.time()
idx = 0
for item in items:
idx += 1
item_name = item[0]
item_code = item[1]
cursor.execute('SELECT * FROM ' + tableName + ' WHERE CODE=?', (item_code,))
result = cursor.fetchone()
stock = {"CODE": item_code, "NAME": item_name, "PRICE": []}
if result is not None:
stock["PRICE"] = json.loads(result[2])
self.crawl_specific_stock(stock)
text = json.dumps(stock['PRICE'], ensure_ascii=False)
print(idx, item_name, item_code, (time.time()-start_time),"s")
start_time = time.time()
cursor.execute('SELECT * FROM ' + tableName + ' WHERE CODE=?', (stock["CODE"],))
result = cursor.fetchone()
if result == None:
cursor.execute("INSERT INTO " + tableName + "(CODE, NAME, PRICE) VALUES(?, ?, ?)", (stock["CODE"], stock["NAME"], text))
else:
cursor.execute("UPDATE " + tableName + " SET PRICE=? WHERE CODE=?", (text, stock["CODE"]))
conn.commit()
cursor.close()
conn.close()
return
def get_data(self, stock):
url = 'http://finance.naver.com/item/sise_day.nhn?code={code}'.format(code=stock['CODE'].strip())
# 일자 데이터를 담을 df라는 DataFrame 정의
df = pd.DataFrame()
lastDay = ""
if len(stock) > 0 and len(stock["PRICE"]) - 1 > 0:
lastDay = stock["PRICE"][len(stock["PRICE"]) - 1]["DATE"].replace("-", ".")
date_set = set()
lastPage = False
# 1페이지에서 1000페이지의 데이터만 가져오기
for page in range(1, self.limit_page_count):
# 최근 상장 기업의 마지막 반복되는 페이지를 제외시킨다.
pg_url = '{url}&page={page}'.format(url=url, page=page)
#html = pd.read_html(pg_url, header=0)
html = None
while True:
try:
html = pd.read_html(requests.get(pg_url, headers=self.header, timeout=30).text)
sleep(0.5)
break
except:
print(pg_url)
if page > 200:
break
continue
for date in html[0].날짜.values:
if type(date) is str:
if date in date_set:
lastPage = True
break
date_set.add(date)
if date == lastDay:
lastPage = True
df = df.append(html[0], ignore_index=True)
break
df = df.append(html[0], ignore_index=True)
if lastPage:
print("\t- lastpage:", page)
break
"""
if count == 10:
df = df.append(html[0], ignore_index=True)
if lastPage:
break
else:
if lastPage == False:
df = df.append(html[0], ignore_index=True)
lastPage = True
else:
break
"""
# df.dropna()를 이용해 결측값 있는 행 제거
df = df.dropna()
# 상위 5개 데이터 확인하기
###print (df.head())
# 한글로 된 컬럼명을 영어로 바꿔줌
df = df.rename(columns={'날짜': 'date', '종가': 'close', '전일비': 'diff', '시가': 'open', '고가': 'high', '저가': 'low', '거래량': 'volume'})
# 데이터의 타입을 int형으로 바꿔줌
df[['close', 'diff', 'open', 'high', 'low', 'volume']] = df[['close', 'diff', 'open', 'high', 'low', 'volume']].astype(int)
# 컬럼명 'date'의 타입을 date로 바꿔줌
df['date'] = pd.to_datetime(df['date'])
# 일자(date)를 기준으로 오름차순 정렬
# df = df.sort_values(by=['date'], ascending=True)
# 상위 5개 데이터 확인
###print (df.head())
if len(stock) > 0 and len(stock["PRICE"]) - 1 > 0:
lastDay = stock["PRICE"][len(stock["PRICE"]) - 1]["DATE"]
for values in df.values:
day = str(values[0]).split(' ')[0]
if lastDay == day:
break
stock["PRICE"].append({
"DATE": day,
df.columns[1]: values[1],
df.columns[2]: values[2],
df.columns[3]: values[3],
df.columns[4]: values[4],
df.columns[5]: values[5],
df.columns[6]: values[6],
})
# stock["PRICE"] = sorted(stock["PRICE"], key=lambda x: x['DATE'], reverse=True)
stock["PRICE"] = sorted(stock["PRICE"], key=lambda x: x['DATE'])
return
def get_moving_avg(self, stock):
q_3 = Queue(3)
q_5 = Queue(5)
q_7 = Queue(7)
q_10 = Queue(10)
q_20 = Queue(20)
q_30 = Queue(30)
q_60 = Queue(60)
q_90 = Queue(90)
q_100 = Queue(100)
q_120 = Queue(120)
q_150 = Queue(150)
q_180 = Queue(180)
q_200 = Queue(200)
q_240 = Queue(240)
for i in range(len(stock['PRICE'])):
q_3.enqueue(stock['PRICE'][i]['close'])
q_5.enqueue(stock['PRICE'][i]['close'])
q_7.enqueue(stock['PRICE'][i]['close'])
q_10.enqueue(stock['PRICE'][i]['close'])
q_20.enqueue(stock['PRICE'][i]['close'])
q_30.enqueue(stock['PRICE'][i]['close'])
q_60.enqueue(stock['PRICE'][i]['close'])
q_90.enqueue(stock['PRICE'][i]['close'])
q_100.enqueue(stock['PRICE'][i]['close'])
q_120.enqueue(stock['PRICE'][i]['close'])
q_150.enqueue(stock['PRICE'][i]['close'])
q_180.enqueue(stock['PRICE'][i]['close'])
q_200.enqueue(stock['PRICE'][i]['close'])
q_240.enqueue(stock['PRICE'][i]['close'])
stock['PRICE'][i]['avg3'] = q_3.avg()
stock['PRICE'][i]['avg5'] = q_5.avg()
stock['PRICE'][i]['avg7'] = q_7.avg()
stock['PRICE'][i]['avg10'] = q_10.avg()
stock['PRICE'][i]['avg20'] = q_20.avg()
stock['PRICE'][i]['avg30'] = q_30.avg()
stock['PRICE'][i]['avg60'] = q_60.avg()
stock['PRICE'][i]['avg90'] = q_90.avg()
stock['PRICE'][i]['avg100'] = q_100.avg()
stock['PRICE'][i]['avg120'] = q_120.avg()
stock['PRICE'][i]['avg150'] = q_150.avg()
stock['PRICE'][i]['avg180'] = q_180.avg()
stock['PRICE'][i]['avg200'] = q_200.avg()
stock['PRICE'][i]['avg240'] = q_240.avg()
return
def crawl_specific_stock(self, stock):
# 데이터 수집
self.get_data(stock)
# 이동 평균 계산
self.get_moving_avg(stock)
return
def update(self, inFileName, outFileName):
"""
Full json 데이터를 db에 import 시킴
inFileName = PROJECT_HOME + '/resources/stock.json.full'
outFileName = PROJECT_HOME + '/resources/stock.db'
crawler = StockCrawler()
crawler.update(inFileName, outFileName)
:param inFileName:
:param outFileName:
:return:
"""
tableName = 'stock'
conn = sqlite3.connect(outFileName, isolation_level=None)
cursor = conn.cursor()
cursor.execute("CREATE TABLE IF NOT EXISTS " + tableName + " (CODE text PRIMARY KEY, NAME text, PRICE text, MACD text, STOCHASTIC text, ICHIMOKU text, RSI text, BOLINGERBAND text)")
idx = 0
inFp = open(inFileName, 'r')
for line in inFp.readlines():
if line:
idx += 1
stock = json.loads(line)
print(idx, stock["CODE"], stock["NAME"])
text = json.dumps(stock["PRICE"], ensure_ascii=False)
cursor.execute('SELECT * FROM ' + tableName + ' WHERE CODE=?', (stock["CODE"],))
result = cursor.fetchone()
if result == None:
cursor.execute("INSERT INTO " + tableName + "(CODE, NAME, PRICE) VALUES(?, ?, ?)", (stock["CODE"], stock["NAME"], text))
else:
cursor.execute("UPDATE " + tableName + " SET PRICE=? WHERE CODE=?", (text, stock["CODE"]))
return
def saveIndex(self, code, inFileName, outFileName):
tableName = 'stock'
conn = sqlite3.connect(outFileName)
cursor = conn.cursor()
cursor.execute("CREATE TABLE IF NOT EXISTS " + tableName + " (CODE text PRIMARY KEY, NAME text, PRICE text, MACD text, STOCHASTIC text, ICHIMOKU text, RSI text, BOLINGERBAND text)")
stock = {"NAME": code, "CODE": code, "PRICE": []}
lastDay = ""
cursor.execute('SELECT * FROM ' + tableName + ' WHERE CODE=?', (stock["CODE"],))
result = cursor.fetchone()
if result is not None:
stock["PRICE"] = json.loads(result[2])
lastDay = stock["PRICE"][len(stock["PRICE"]) - 1]["DATE"]
with open(inFileName, "r", encoding="utf-8") as inFp:
for line in inFp:
line = line.strip()
if line[0] == "#":
continue
arr = line.split("\t")
if arr[0] == lastDay:
break
price = {"DATE": arr[0], "close": float(arr[1]), "diff": float(arr[6].replace("%", "")), "open": float(arr[2]), "high": float(arr[3]), "low": float(arr[4]), "volume": 0}
price['avg3'] = 0
price['avg5'] = 0
price['avg7'] = 0
price['avg10'] = 0
price['avg20'] = 0
price['avg30'] = 0
price['avg60'] = 0
price['avg90'] = 0
price['avg100'] = 0
price['avg120'] = 0
price['avg150'] = 0
price['avg180'] = 0
price['avg200'] = 0
price['avg240'] = 0
stock["PRICE"].append(price)
stock["PRICE"] = sorted(stock["PRICE"], key=lambda x: x['DATE'])
self.get_moving_avg(stock)
text = json.dumps(stock['PRICE'], ensure_ascii=False)
cursor.execute('SELECT * FROM ' + tableName + ' WHERE CODE=?', (stock["CODE"],))
result = cursor.fetchone()
if result == None:
cursor.execute("INSERT INTO " + tableName + "(CODE, NAME, PRICE, MACD, STOCHASTIC, ICHIMOKU, RSI) VALUES(?, ?, ?, ?, ?, ?, ?)", (stock["CODE"], stock["NAME"], text, "[{}]", "[{}]", "[{}]", "[{}]"))
else:
cursor.execute("UPDATE " + tableName + " SET PRICE=?, MACD=?, STOCHASTIC=?, ICHIMOKU=?, RSI=? WHERE CODE=?", (text, "[{}]", "[{}]", "[{}]", "[{}]", stock["CODE"]))
conn.commit()
cursor.close()
conn.close()
return

365
stockpredictor_back/crawler/toSQLite/inout.cvs
View File

@@ -1,365 +0,0 @@
364,1186,1339,46.97
363,1086,1439,43.01
362,805,1720,31.88
361,1121,1404,44.40
360,674,1851,26.69
359,260,2265,10.30
358,1616,909,64.00
357,341,2184,13.50
356,1179,1346,46.69
355,1499,1026,59.37
354,1801,724,71.33
353,925,1600,36.63
352,532,1993,21.07
351,1248,1277,49.43
350,1092,1433,43.25
349,1292,1233,51.17
348,1253,1272,49.62
347,1070,1455,42.38
346,1966,559,77.86
345,1436,1089,56.87
344,786,1739,31.13
343,1365,1160,54.06
342,1181,1344,46.77
341,1147,1378,45.43
340,1319,1206,52.24
339,904,1621,35.80
338,748,1777,29.62
337,533,1992,21.11
336,1024,1501,40.55
335,458,2067,18.14
334,395,2130,15.64
333,923,1602,36.55
332,548,1977,21.70
331,1020,1505,40.40
330,1736,789,68.75
329,1300,1225,51.49
328,1564,961,61.94
327,999,1526,39.56
326,1642,883,65.03
325,880,1645,34.85
324,1009,1516,39.96
323,810,1715,32.08
322,624,1901,24.71
321,614,1911,24.32
320,634,1891,25.11
319,873,1652,34.57
318,1320,1205,52.28
317,1416,1109,56.08
316,710,1815,28.12
315,830,1695,32.87
314,358,2167,14.18
313,1379,1146,54.61
312,1871,654,74.10
311,1634,891,64.71
310,420,2105,16.63
309,1605,920,63.56
308,1486,1039,58.85
307,914,1611,36.20
306,1647,878,65.23
305,646,1879,25.58
304,1526,999,60.44
303,995,1530,39.41
302,1137,1388,45.03
301,1059,1466,41.94
300,991,1534,39.25
299,1205,1320,47.72
298,734,1791,29.07
297,1388,1137,54.97
296,1026,1499,40.63
295,1394,1131,55.21
294,1021,1504,40.44
293,1152,1373,45.62
292,437,2088,17.31
291,1154,1371,45.70
290,1269,1256,50.26
289,855,1670,33.86
288,1153,1372,45.66
287,1100,1425,43.56
286,858,1667,33.98
285,777,1748,30.77
284,797,1728,31.56
283,855,1670,33.86
282,1343,1182,53.19
281,1351,1174,53.50
280,1314,1211,52.04
279,735,1790,29.11
278,904,1621,35.80
277,1253,1272,49.62
276,935,1590,37.03
275,1048,1477,41.50
274,916,1609,36.28
273,359,2166,14.22
272,892,1633,35.33
271,1320,1205,52.28
270,727,1798,28.79
269,1702,823,67.41
268,1612,913,63.84
267,1182,1343,46.81
266,1470,1055,58.22
265,872,1653,34.53
264,1270,1255,50.30
263,687,1838,27.21
262,435,2090,17.23
261,1260,1265,49.90
260,1375,1150,54.46
259,1297,1228,51.37
258,653,1872,25.86
257,639,1886,25.31
256,1721,804,68.16
255,1295,1230,51.29
254,1213,1312,48.04
253,863,1662,34.18
252,1360,1165,53.86
251,656,1869,25.98
250,682,1843,27.01
249,880,1645,34.85
248,278,2247,11.01
247,1975,550,78.22
246,1364,1161,54.02
245,1142,1383,45.23
244,774,1751,30.65
243,1001,1524,39.64
242,873,1652,34.57
241,902,1623,35.72
240,1323,1202,52.40
239,1390,1135,55.05
238,1062,1463,42.06
237,1162,1363,46.02
236,717,1808,28.40
235,895,1630,35.45
234,794,1731,31.45
233,987,1538,39.09
232,297,2228,11.76
231,1538,987,60.91
230,1138,1387,45.07
229,760,1765,30.10
228,1706,819,67.56
227,1169,1356,46.30
226,1274,1251,50.46
225,617,1908,24.44
224,794,1731,31.45
223,544,1981,21.54
222,1659,866,65.70
221,1587,938,62.85
220,1173,1352,46.46
219,1278,1247,50.61
218,1043,1482,41.31
217,1141,1384,45.19
216,1728,797,68.44
215,906,1619,35.88
214,563,1962,22.30
213,1427,1098,56.51
212,1271,1254,50.34
211,1114,1411,44.12
210,945,1580,37.43
209,1358,1167,53.78
208,1041,1484,41.23
207,1150,1375,45.54
206,1035,1490,40.99
205,1195,1330,47.33
204,901,1624,35.68
203,1307,1218,51.76
202,1222,1303,48.40
201,1323,1202,52.40
200,1222,1303,48.40
199,1185,1340,46.93
198,1174,1351,46.50
197,956,1569,37.86
196,1134,1391,44.91
195,1109,1416,43.92
194,1440,1085,57.03
193,771,1754,30.53
192,1094,1431,43.33
191,1191,1334,47.17
190,1189,1336,47.09
189,947,1578,37.50
188,536,1989,21.23
187,903,1622,35.76
186,814,1711,32.24
185,927,1598,36.71
184,1178,1347,46.65
183,1632,893,64.63
182,1394,1131,55.21
181,1381,1144,54.69
180,1147,1378,45.43
179,591,1934,23.41
178,1344,1181,53.23
177,1191,1334,47.17
176,695,1830,27.52
175,1527,998,60.48
174,1239,1286,49.07
173,621,1904,24.59
172,672,1853,26.61
171,1581,944,62.61
170,1317,1208,52.16
169,1230,1295,48.71
168,1352,1173,53.54
167,1284,1241,50.85
166,1311,1214,51.92
165,1099,1426,43.52
164,1217,1308,48.20
163,1115,1410,44.16
162,867,1658,34.34
161,941,1584,37.27
160,848,1677,33.58
159,1320,1205,52.28
158,1197,1328,47.41
157,1023,1502,40.51
156,1078,1447,42.69
155,1345,1180,53.27
154,1226,1299,48.55
153,1171,1354,46.38
152,835,1690,33.07
151,1146,1379,45.39
150,793,1732,31.41
149,946,1579,37.47
148,1240,1285,49.11
147,1265,1260,50.10
146,920,1605,36.44
145,1200,1325,47.52
144,997,1528,39.49
143,987,1538,39.09
142,1116,1409,44.20
141,1218,1307,48.24
140,868,1657,34.38
139,475,2050,18.81
138,999,1526,39.56
137,1590,935,62.97
136,1458,1067,57.74
135,1001,1524,39.64
134,1221,1304,48.36
133,1020,1505,40.40
132,809,1716,32.04
131,1505,1020,59.60
130,627,1898,24.83
129,1258,1267,49.82
128,1199,1326,47.49
127,601,1924,23.80
126,1261,1264,49.94
125,827,1698,32.75
124,1270,1255,50.30
123,617,1908,24.44
122,1074,1451,42.53
121,893,1632,35.37
120,1766,759,69.94
119,1495,1030,59.21
118,914,1611,36.20
117,805,1720,31.88
116,675,1850,26.73
115,890,1635,35.25
114,1036,1489,41.03
113,514,2011,20.36
112,394,2131,15.60
111,1912,613,75.72
110,389,2136,15.41
109,320,2205,12.67
108,1889,636,74.81
107,1940,585,76.83
106,1088,1437,43.09
105,865,1660,34.26
104,1662,863,65.82
103,1132,1393,44.83
102,1238,1287,49.03
101,1597,928,63.25
100,1036,1489,41.03
99,1307,1218,51.76
98,1046,1479,41.43
97,1043,1482,41.31
96,777,1748,30.77
95,922,1603,36.51
94,1359,1166,53.82
93,663,1862,26.26
92,1542,983,61.07
91,976,1549,38.65
90,707,1818,28.00
89,1207,1318,47.80
88,819,1706,32.44
87,1026,1499,40.63
86,1335,1190,52.87
85,530,1995,20.99
84,1599,926,63.33
83,1408,1117,55.76
82,533,1992,21.11
81,691,1834,27.37
80,343,2182,13.58
79,2095,430,82.97
78,1015,1510,40.20
77,804,1721,31.84
76,1967,558,77.90
75,1895,630,75.05
74,1399,1126,55.41
73,1015,1510,40.20
72,1193,1332,47.25
71,870,1655,34.46
70,731,1794,28.95
69,1094,1431,43.33
68,1401,1124,55.49
67,1583,942,62.69
66,772,1753,30.57
65,802,1723,31.76
64,617,1908,24.44
63,1050,1475,41.58
62,1643,882,65.07
61,588,1937,23.29
60,599,1926,23.72
59,770,1755,30.50
58,1411,1114,55.88
57,997,1528,39.49
56,494,2031,19.56
55,1235,1290,48.91
54,1922,603,76.12
53,1483,1042,58.73
52,967,1558,38.30
51,691,1834,27.37
50,938,1587,37.15
49,1460,1065,57.82
48,1067,1458,42.26
47,1063,1462,42.10
46,1212,1313,48.00
45,794,1731,31.45
44,494,2031,19.56
43,1547,978,61.27
42,278,2247,11.01
41,2035,490,80.59
40,2081,444,82.42
39,1839,686,72.83
38,1541,984,61.03
37,1372,1153,54.34
36,730,1795,28.91
35,1631,894,64.59
34,1186,1339,46.97
33,980,1545,38.81
32,1117,1408,44.24
31,1078,1447,42.69
30,910,1615,36.04
29,1407,1118,55.72
28,553,1972,21.90
27,1243,1282,49.23
26,1035,1490,40.99
25,1106,1419,43.80
24,1290,1235,51.09
23,1135,1390,44.95
22,1184,1341,46.89
21,1553,972,61.50
20,1294,1231,51.25
19,1321,1204,52.32
18,1314,1211,52.04
17,787,1738,31.17
16,1121,1404,44.40
15,1690,835,66.93
14,700,1825,27.72
13,627,1898,24.83
12,1754,771,69.47
11,919,1606,36.40
10,1051,1474,41.62
9,557,1968,22.06
8,594,1931,23.52
7,1281,1244,50.73
6,2005,520,79.41
5,926,1599,36.67
4,476,2049,18.85
3,325,2200,12.87
2,1516,1009,60.04
1,309,2216,12.24
0,2134,391,84.51

157
stockpredictor_back/train/PricePredictor.py
View File

@@ -1,157 +0,0 @@
import os
import json
import sqlite3
import numpy as np
from keras.models import Sequential
from keras.layers import Dense
from keras.callbacks import EarlyStopping, ModelCheckpoint
from keras.layers import LSTM
class PricePredictor:
trainingRate = 0.7
TYPE = 0 # OPEN = 0, CLOSE = 1, HIGH = 2, LOW = 3, VOLUME = 4
MAX_VALUE = 500000
MAX_VOLUME_VALUE = 300000000
def getNormalPrice(self, value):
value = value / self.MAX_VALUE
if value>1:
value = 1
return value
def getNormalVolume(self, value):
value = value / self.MAX_VOLUME_VALUE
if value > 1:
value = 1
return value
def read (self, inFileName):
conn = sqlite3.connect(inFileName)
cursor = conn.cursor()
stocks = []
rowid = 1
cursor.execute('SELECT * FROM stock WHERE rowid=?', (rowid,))
result = cursor.fetchone()
while result is not None:
prices = json.loads(result[2])
stock = []
for price in prices:
stock.append([self.getNormalPrice(price['open']), self.getNormalPrice(price['close']), self.getNormalPrice(price['high']), self.getNormalPrice(price['low']), self.getNormalVolume(price['volume'])])
stocks.append(stock)
rowid += 1
cursor.execute('SELECT * FROM stock WHERE rowid=?', (rowid,))
result = cursor.fetchone()
cursor.close()
conn.close()
return stocks
def make_dataset(self, stocks, window_size=5):
feature = []
label = []
test_feature = []
test_label = []
for i in range(len(stocks)):
stock = stocks[i]
for j in range(len(stock) - window_size):
feature.append(stock[j:j+window_size])
label.append([stock[j+window_size][self.TYPE]])
test_feature.append(stock[len(stock)-window_size-1:len(stock)-1])
test_label.append(stock[len(stock)-1][self.TYPE])
index = int(len(feature) * self.trainingRate)
train_feature = feature[:index]
train_label = label[:index]
valid_feature = feature[index:]
valid_label = label[index:]
test_feature = feature[len(feature) - 1]
test_label = label[len(feature) - 1]
return np.array(train_feature), np.array(train_label), np.array(valid_feature), np.array(valid_label), np.array(test_feature), np.array(test_label)
def train(self, inFileName, model_path):
stocks = self.read(inFileName)
train_feature, train_label, valid_feature, valid_label, test_feature, test_label = self.make_dataset(stocks)
print (train_feature.shape)
print (train_label.shape)
model = Sequential()
model.add(LSTM(32,
input_shape=(train_feature.shape[1], train_feature.shape[2]),
activation='relu',
return_sequences=False))
model.add(Dense(1))
model.compile(loss='mean_squared_error', optimizer='adam')
early_stop = EarlyStopping(monitor='val_loss', patience=5)
filename = ''
if self.TYPE==0 :
filename = os.path.join(model_path, 'open_checkpoint.h5')
elif self.TYPE==1 :
filename = os.path.join(model_path, 'close_checkpoint.h5')
elif self.TYPE==2 :
filename = os.path.join(model_path, 'high_checkpoint.h5')
elif self.TYPE==3 :
filename = os.path.join(model_path, 'low_checkpoint.h5')
checkpoint = ModelCheckpoint(filename, monitor='val_loss', verbose=1, save_best_only=True, mode='auto')
history = model.fit(train_feature, train_label,
epochs=1000,
batch_size=100000,
validation_data=(valid_feature, valid_label),
callbacks=[early_stop, checkpoint])
test_feature = np.array([test_feature])
predition = model.predict(test_feature)
print(predition[0], test_label)
print(predition[0] * self.MAX_VALUE, test_label * self.MAX_VALUE)
return
def predict(self, inFileName, model_path):
stocks = self.read(inFileName)
train_feature, train_label, valid_feature, valid_label, test_feature, test_label = self.make_dataset(stocks)
filename = ''
if self.TYPE==0 :
filename = os.path.join(model_path, 'open_checkpoint.h5')
elif self.TYPE==1 :
filename = os.path.join(model_path, 'close_checkpoint.h5')
elif self.TYPE==2 :
filename = os.path.join(model_path, 'high_checkpoint.h5')
elif self.TYPE==3 :
filename = os.path.join(model_path, 'low_checkpoint.h5')
model = Sequential()
model.add(LSTM(32,
input_shape=(train_feature.shape[1], train_feature.shape[2]),
activation='relu',
return_sequences=False))
model.add(Dense(1))
model.compile(loss='mean_squared_error', optimizer='adam')
model.load_weights(filename)
test_feature = np.array([test_feature])
predition = model.predict(test_feature)
print(predition[0], test_label)
print(predition[0] * self.MAX_VALUE, test_label * self.MAX_VALUE)
return
if __name__ == "__main__":
PROJECT_HOME = "../.."
inFileName = PROJECT_HOME + '/resources/stock.db'
model_path = PROJECT_HOME + '/resources/model'
pricePredictor = PricePredictor()
pricePredictor.train(inFileName, model_path)
#pricePredictor.predict(inFileName, model_path)