From ce26ef5623aafb8c009353ce1e6671c0ce7360a0 Mon Sep 17 00:00:00 2001 From: dsyoon Date: Wed, 3 Aug 2022 22:40:23 +0900 Subject: [PATCH 1/9] init --- Simulation.py | 26 ++++++++-------- StockTrainer.py | 36 +++++++++++----------- hts/HTS.py | 3 +- stock/util/Stock2Vector.py | 47 ++++++++++++++++++++++++++++ stock/util/StockPredictor.py | 59 ++++++++++++++++++++++++++++++++++++ 5 files changed, 139 insertions(+), 32 deletions(-) create mode 100644 stock/util/StockPredictor.py diff --git a/Simulation.py b/Simulation.py index 36c3314..2433e0d 100644 --- a/Simulation.py +++ b/Simulation.py @@ -6,6 +6,7 @@ import os from hts.HTS import HTS from stock.util.Stock2Vector import Stock2Vector +from stock.util.StockPredictor import StockPredictor from hts.BuySellChecker import BuySellChecker class Simulation (HTS): @@ -16,6 +17,7 @@ class Simulation (HTS): super().__init__(RESOURCE_PATH) self.stock2Vector = Stock2Vector(RESOURCE_PATH) + self.stockPredictor = StockPredictor() self.buySellChecker = BuySellChecker() self.RESOURCE_PATH = RESOURCE_PATH #self.connect() @@ -126,23 +128,23 @@ class Simulation (HTS): return - def analyzeAutoMode(self, data): - return data, None - def simulate(self, stock_code, today, type="rule"): - LAST_DATA = self.stock2Vector.getLastData(stock_code, today) + def simulate(self, stock_code, today, method="rule"): - result = self.stock2Vector.getRealTime(stock_code, today, LAST_DATA) + if method == "ml": + LAST_DATA = self.stock2Vector.getLastData(stock_code, today, n=10) + result = self.stock2Vector.getRealTime(stock_code, today, LAST_DATA) + + df, minmax_df = self.stock2Vector.preprocessData(result) + bsLine, data = self.stockPredictor.predict(df, minmax_df, isRealTime=False) + else: + LAST_DATA = self.stock2Vector.getLastData(stock_code, today) + result = self.stock2Vector.getRealTime(stock_code, today, LAST_DATA) - if type == "rule": # 이동평균, RSI, MACD, 일목균형, 볼린저밴드 상/하단을 계산한다. data = self.buySellChecker.analyze(result) # 사야 할 시점과 팔아야 할 시점을 체크한다. bsLine, data = self.buySellChecker.checkTransaction(data, stock_code, isRealTime=False) - elif type == "auto": - data, bsLine = self.analyzeAutoMode(result) - else: - data, bsLine = None, None if data is not None: # 그래프를 그린다. @@ -159,13 +161,13 @@ if __name__ == "__main__": stock_codes = { # 252670 # 122630 - "122630": ['20220729'], + "122630": ['20220803'], } for stock_code in stock_codes: simulation = Simulation(RESOURCE_PATH) for given_day in stock_codes[stock_code]: - simulation.simulate(stock_code, given_day) + simulation.simulate(stock_code, given_day, method='ml') print ("done...") diff --git a/StockTrainer.py b/StockTrainer.py index a72b59f..de3abe1 100644 --- a/StockTrainer.py +++ b/StockTrainer.py @@ -1,7 +1,6 @@ import os import keras import numpy as np -from numpy import zeros, newaxis import tensorflow as tf from stock.util.Stock2Vector import Stock2Vector from classification_models.keras import Classifiers @@ -18,16 +17,17 @@ class StockTrainer: def getDataset(self, stock_code): VECTOR_SIZE = 299 - df, minmax_df = self.stock2Vector.makeTrainData(stock_code) + result = self.stock2Vector.getTrainData(stock_code) + df, minmax_df = self.stock2Vector.preprocessData(result) TOTAL_X, TOTAL_Y = [], [] - for key in df: + for key in minmax_df: if key == "date": continue elif key == "label": - TOTAL_Y.append(df[key].tolist()) + TOTAL_Y.append(minmax_df[key].tolist()) else: - TOTAL_X.append(df[key].tolist()) + TOTAL_X.append(minmax_df[key].tolist()) SIZE_WIDTH = len(TOTAL_X[0]) SIZE_HEIGHT = len(TOTAL_X) @@ -44,38 +44,37 @@ class StockTrainer: Y.append([0, 1, 0]) else: Y.append([0, 0, 1]) - if i >= VECTOR_SIZE+10: - break X = np.asarray(X) Y = np.asarray(Y) return X, Y def train(self, stock_code): - ResNet18, preprocess_input = Classifiers.get('inceptionresnetv2') - X, Y = self.getDataset(stock_code) + # build model + n_classes = 3 + Inceptionresnetv2, preprocess_input = Classifiers.get('inceptionresnetv2') X = preprocess_input(X) - n_classes = 3 - - # build model - base_model = ResNet18(input_shape=(299, 299, 3), include_top=False) + # train + checkpoint_filename = os.path.join(self.RESOURCE_PATH, "model", "stock.ckpt") + base_model = Inceptionresnetv2(input_shape=(299, 299, 3), include_top=False) x = tf.keras.layers.GlobalAveragePooling2D()(base_model.output) output = tf.keras.layers.Dense(n_classes, activation='softmax')(x) model = keras.models.Model(inputs=[base_model.input], outputs=[output]) - # train model.compile(optimizer='SGD', loss='categorical_crossentropy', metrics=['accuracy']) - - checkpoint_filename = os.path.join(self.RESOURCE_PATH, "model", "stock.ckpt") chekpoint = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_filename, save_weights_only=True, verbose=1) - earlystop = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=3, mode="auto") + earlystop = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=5, mode="auto") + + if os.path.isfile(checkpoint_filename): + model.load_weights(checkpoint_filename) model.fit(x=X, y=Y, - epochs=10, + batch_size=10000, + epochs=10000, callbacks=[chekpoint, earlystop]) return @@ -95,5 +94,4 @@ if __name__ == "__main__": stockTrainer = StockTrainer(RESOURCE_PATH) stockTrainer.train(stock_code) - print ("done...") \ No newline at end of file diff --git a/hts/HTS.py b/hts/HTS.py index 402ac56..cfd87be 100644 --- a/hts/HTS.py +++ b/hts/HTS.py @@ -634,7 +634,7 @@ class HTS: "label": []} days = [] - for i in range(1, 10): + for i in range(1, 100): last_day = (datetime.strptime(today, '%Y%m%d') - timedelta(i)).strftime('%Y%m%d') isValid = self.isValidYMD(stock_code, last_day) if isValid: @@ -655,6 +655,7 @@ class HTS: else: result = {"check": set(), "time": [], "open": [], "close": [], "high": [], "low": [], "vol": [], "label": []} + #### real time에서 아직 저장된 것이 없기 때문에 result는 아무것도 채워지지 않는다. self.getDBData(stock_code, today, result) diff --git a/stock/util/Stock2Vector.py b/stock/util/Stock2Vector.py index 55ba8bd..47f9058 100644 --- a/stock/util/Stock2Vector.py +++ b/stock/util/Stock2Vector.py @@ -153,6 +153,53 @@ class Stock2Vector(HTS): return df, minmax_df + def getTrainData(self, stock_code): + tableName = 'hts' + conn = sqlite3.connect(os.path.join(self.RESOURCE_PATH, "hts.db")) + cursor = conn.cursor() + + cursor.execute( + 'SELECT ymd, hms, open, high, low, close, volume, label FROM ' + tableName + ' WHERE CODE=? and (ymd >= ? and ymd <= ?) order by ymd desc, hms ', + (stock_code, "20220721", "20220731")) + db_result = cursor.fetchall() + temp_result = [] + for rows in db_result: + temp_result.append( + [rows[0], rows[1], rows[2], rows[3], rows[4], rows[5], rows[6], 0 if rows[7] is None else rows[7]]) + temp_result.sort(key=lambda x: (x[0], x[1])) + + result = {"check": set(), "time": [], "open": [], "close": [], "high": [], "low": [], "vol": [], "label": []} + for rows in temp_result: + ymd = rows[0] # hts.날짜 + hms = rows[1] # hts.시간 + open = rows[2] # hts.시가 + high = rows[3] # hts.고가 + low = rows[4] # hts.저가 + close = rows[5] # hts.종가 + vol = rows[6] # hts.거래량 + label = 0 if rows[7] is None else rows[7] # hts.매매구분 + + temp = datetime.strptime(str(ymd) + " " + str(hms).zfill(4) + "00", '%Y%m%d %H%M%S') + + result["time"].append(temp) + result["open"].append(int(open)) + result["close"].append(int(close)) + result["high"].append(int(high)) + result["low"].append(int(low)) + result["vol"].append(int(vol)) + result["label"].append(int(label)) + + return result + def preprocessData(self, result): + # 분석을 통해서 볼린저밴드 상/하단을 계산한다. + df = self.buySellChecker.getVectorFeature(result) + minmax_df1 = (df - df.min()) / (df.max() - df.min()) + minmax_df2 = minmax_df1.drop(["date"], axis="columns") + minmax_df = minmax_df2.join(df['date']) + + minmax_df = minmax_df.fillna(0) + return df, minmax_df + def makeTrainData(self, stock_code): result = {"check": set(), "time": [], "open": [], "close": [], "high": [], "low": [], "vol": [], "label": []} diff --git a/stock/util/StockPredictor.py b/stock/util/StockPredictor.py new file mode 100644 index 0000000..8b802c4 --- /dev/null +++ b/stock/util/StockPredictor.py @@ -0,0 +1,59 @@ +import os +import keras +import numpy as np +from keras.applications.imagenet_utils import decode_predictions +from classification_models.keras import Classifiers + +class StockPredictor: + + RESOURCE_PATH = None + stock2Vector = None + + def __init__(self): + return + + def getDataset(self, df): + VECTOR_SIZE = 299 + TOTAL_X, TOTAL_Y = [], [] + for key in df: + if key == "date": + continue + elif key == "label": + TOTAL_Y.append(df[key].tolist()) + else: + TOTAL_X.append(df[key].tolist()) + + SIZE_WIDTH = len(TOTAL_X[0]) + SIZE_HEIGHT = len(TOTAL_X) + X = [] + for i in range(VECTOR_SIZE, SIZE_WIDTH): + temp_X, temp_Y = np.zeros((VECTOR_SIZE, VECTOR_SIZE)), np.zeros(0) + for j in range(SIZE_HEIGHT): + temp_X[j][0:VECTOR_SIZE] = TOTAL_X[j][i - VECTOR_SIZE:i] + temp_X = np.stack([temp_X, temp_X, temp_X], axis=-1) + X.append(temp_X) + + X = np.asarray(X[len(X)-1]) + + return X + + def predict(self, df, minmax_df, isRealTime=False): + X = self.getDataset(df) + + # build model + n_classes = 3 + Inceptionresnetv2, preprocess_input = Classifiers.get('inceptionresnetv2') + X = preprocess_input(X) + base_model = Inceptionresnetv2(input_shape=(299, 299, 3), include_top=False) + model = keras.models.Model(inputs=[base_model.input]) + + checkpoint_filename = os.path.join(self.RESOURCE_PATH, "model", "stock.ckpt") + model.load_weights(checkpoint_filename) + + y = model.predict(X) + + # result + print(decode_predictions(y)) + + + return From 124c5de4e3ab4fc557a2e790f8a0100abf24cc79 Mon Sep 17 00:00:00 2001 From: "dosang.yoon" Date: Thu, 4 Aug 2022 13:14:33 +0900 Subject: [PATCH 2/9] init --- stock/analysis/AnalyzerSqlite.py | 81 ++++++++++++++++++-------------- 1 file changed, 45 insertions(+), 36 deletions(-) diff --git a/stock/analysis/AnalyzerSqlite.py b/stock/analysis/AnalyzerSqlite.py index 53e2d77..301a1a2 100644 --- a/stock/analysis/AnalyzerSqlite.py +++ b/stock/analysis/AnalyzerSqlite.py @@ -118,6 +118,7 @@ class AnalyzerSqlite: high = list(reversed(stock['high'])) low = list(reversed(stock['low'])) volume = list(reversed(stock['volume'])) + avg3 = list(reversed(stock['avg3'])) avg4 = list(reversed(stock['avg4'])) avg5 = list(reversed(stock['avg5'])) avg6 = list(reversed(stock['avg6'])) @@ -142,6 +143,7 @@ class AnalyzerSqlite: # general candle_stick = go.Candlestick(x=ymd, open=open, high=high, low=low, close=close, increasing_line_color='red', decreasing_line_color='blue') + avg3 = go.Scatter(x=ymd, y=avg3, name="avg3", line_color='#085F1B') avg4 = go.Scatter(x=ymd, y=avg4, name="avg4", line_color='#085F1B') avg5 = go.Scatter(x=ymd, y=avg5, name="avg5", line_color='#000000') avg6 = go.Scatter(x=ymd, y=avg6, name="avg6", line_color='#698D09') @@ -160,7 +162,7 @@ class AnalyzerSqlite: changeLine = go.Scatter(x=ymd, y=ichimokucloud_changeLine, name="changeLine", line_color='#000000') baseLine = go.Scatter(x=ymd, y=ichimokucloud_baseLine, name="baseLine", line_color='#FF0000') - candle_data = [candle_stick, avg4, avg5, avg6, avg10, avg12, avg20, avg36, avg40, avg48, avg60, avg120, avg240, avg300, bolinger_upper, bolinger_lower, changeLine, baseLine] + candle_data = [candle_stick, avg3, avg4, avg5, avg6, avg10, avg12, avg20, avg36, avg40, avg48, avg60, avg120, avg240, avg300, bolinger_upper, bolinger_lower, changeLine, baseLine] #candle_data = [candle_stick, bolinger_upper, bolinger_lower, changeLine, baseLine] volume = go.Bar(x=ymd, y=volume, name="volume") @@ -297,7 +299,7 @@ class AnalyzerSqlite: cursor = conn.cursor() sql = 'SELECT ymd, close, open, high, low, volume, ' - sql += ' avg4, avg5, avg6, avg10, avg12, avg20, avg36, avg40, avg48, avg60, avg120, avg200, avg240, avg300, ' + sql += ' avg3, avg4, avg5, avg6, avg10, avg12, avg20, avg36, avg40, avg48, avg60, avg120, avg200, avg240, avg300, ' sql += ' bolingerband_upper, bolingerband_lower, bolingerband_middle, ' sql += ' ichimokucloud_changeLine, ichimokucloud_baseLine, ichimokucloud_leadingSpan1, ichimokucloud_leadingSpan2, ' sql += ' stochastic_fast_k, stochastic_slow_k, stochastic_slow_d, ' @@ -311,7 +313,7 @@ class AnalyzerSqlite: ymd = [] close, open, high, low, volume = [], [], [], [], [] - avg4, avg5, avg6, avg10, avg12, avg20, avg36, avg40, avg48, avg60, avg120, avg200, avg240, avg300 = [], [], [], [], [], [], [], [], [], [], [], [], [], [] + avg3, avg4, avg5, avg6, avg10, avg12, avg20, avg36, avg40, avg48, avg60, avg120, avg200, avg240, avg300 = [], [], [], [], [], [], [], [], [], [], [], [], [], [], [] bolingerband_upper, bolingerband_lower, bolingerband_middle = [], [], [] ichimokucloud_changeLine, ichimokucloud_baseLine, ichimokucloud_leadingSpan1, ichimokucloud_leadingSpan2 = [], [], [], [] stochastic_fast_k, stochastic_slow_k, stochastic_slow_d = [], [], [] @@ -324,37 +326,38 @@ class AnalyzerSqlite: high.append(price[3]) low.append(price[4]) volume.append(price[5]) - avg4.append(price[6]) - avg5.append(price[7]) - avg6.append(price[8]) - avg10.append(price[9]) - avg12.append(price[10]) - avg20.append(price[11]) - avg36.append(price[12]) - avg40.append(price[13]) - avg48.append(price[14]) - avg60.append(price[15]) - avg120.append(price[16]) - avg200.append(price[17]) - avg240.append(price[18]) - avg300.append(price[19]) - bolingerband_upper.append(price[20]) - bolingerband_lower.append(price[21]) - bolingerband_middle.append(price[22]) - ichimokucloud_changeLine.append(price[23]) - ichimokucloud_baseLine.append(price[24]) - ichimokucloud_leadingSpan1.append(price[25]) - ichimokucloud_leadingSpan2.append(price[26]) - stochastic_fast_k.append(price[27]) - stochastic_slow_k.append(price[28]) - stochastic_slow_d.append(price[29]) - rsi.append(price[30]) - rsis.append(price[31]) + avg3.append(price[6]) + avg4.append(price[7]) + avg5.append(price[8]) + avg6.append(price[9]) + avg10.append(price[10]) + avg12.append(price[11]) + avg20.append(price[12]) + avg36.append(price[13]) + avg40.append(price[14]) + avg48.append(price[15]) + avg60.append(price[16]) + avg120.append(price[17]) + avg200.append(price[18]) + avg240.append(price[19]) + avg300.append(price[20]) + bolingerband_upper.append(price[21]) + bolingerband_lower.append(price[22]) + bolingerband_middle.append(price[23]) + ichimokucloud_changeLine.append(price[24]) + ichimokucloud_baseLine.append(price[25]) + ichimokucloud_leadingSpan1.append(price[26]) + ichimokucloud_leadingSpan2.append(price[27]) + stochastic_fast_k.append(price[28]) + stochastic_slow_k.append(price[29]) + stochastic_slow_d.append(price[30]) + rsi.append(price[31]) + rsis.append(price[32]) stock = { "ymd": ymd, "close": close, "open": open, "high": high, "low": low, "volume": volume, - "avg4": avg4, "avg5": avg5, "avg6": avg6, "avg10": avg10, "avg12": avg12, "avg20": avg20, "avg36": avg36, "avg40": avg40, "avg48": avg48, "avg60": avg60, "avg120": avg120, "avg200": avg200, "avg300": avg300, + "avg3": avg3, "avg4": avg4, "avg5": avg5, "avg6": avg6, "avg10": avg10, "avg12": avg12, "avg20": avg20, "avg36": avg36, "avg40": avg40, "avg48": avg48, "avg60": avg60, "avg120": avg120, "avg200": avg200, "avg300": avg300, "avg240": avg240, "bolingerband_upper": bolingerband_upper, "bolingerband_lower": bolingerband_lower, "bolingerband_middle": bolingerband_middle, @@ -564,6 +567,7 @@ class AnalyzerSqlite: return def get_moving_average(self, stock): + q_3 = MovingAverage(3) q_4 = MovingAverage(4) q_5 = MovingAverage(5) q_6 = MovingAverage(6) @@ -580,6 +584,7 @@ class AnalyzerSqlite: q_300 = MovingAverage(300) for i in range(len(stock)): + q_3.enqueue(stock[i]['close']) q_4.enqueue(stock[i]['close']) q_5.enqueue(stock[i]['close']) q_6.enqueue(stock[i]['close']) @@ -595,6 +600,7 @@ class AnalyzerSqlite: q_240.enqueue(stock[i]['close']) q_300.enqueue(stock[i]['close']) + stock[i]['avg3'] = q_3.avg() stock[i]['avg4'] = q_4.avg() stock[i]['avg5'] = q_5.avg() stock[i]['avg6'] = q_6.avg() @@ -625,6 +631,7 @@ class AnalyzerSqlite: "high": weekDict['high'][ts], "low": weekDict['low'][ts], "volume": weekDict['volume'][ts], + "avg3": -1, "avg4": -1, "avg5": -1, "avg6": -1, @@ -664,7 +671,7 @@ class AnalyzerSqlite: stockAnalysisTableName = 'stock_analysis_' + type # 테이블 생성 - cursor.execute("CREATE TABLE IF NOT EXISTS " + stockAnalysisTableName + " (CODE text, NAME text, ymd text, close REAL, diff REAL, open REAL, high REAL, low REAL, volume REAL, avg4 REAL, avg5 REAL, avg6 REAL, avg10 REAL, avg12 REAL, avg20 REAL, avg36 REAL, avg40 REAL, avg48 REAL, avg60 REAL, avg120 REAL, avg200 REAL, avg240 REAL, avg300 REAL, bolingerband_upper REAL, bolingerband_lower REAL, bolingerband_middle REAL, ichimokucloud_changeLine REAL, ichimokucloud_baseLine REAL, ichimokucloud_leadingSpan1 REAL, ichimokucloud_leadingSpan2 REAL, stochastic_fast_k REAL, stochastic_slow_k REAL, stochastic_slow_d REAL, rsi REAL, rsis REAL)") + cursor.execute("CREATE TABLE IF NOT EXISTS " + stockAnalysisTableName + " (CODE text, NAME text, ymd text, close REAL, diff REAL, open REAL, high REAL, low REAL, volume REAL, avg3 REAL, avg4 REAL, avg5 REAL, avg6 REAL, avg10 REAL, avg12 REAL, avg20 REAL, avg36 REAL, avg40 REAL, avg48 REAL, avg60 REAL, avg120 REAL, avg200 REAL, avg240 REAL, avg300 REAL, bolingerband_upper REAL, bolingerband_lower REAL, bolingerband_middle REAL, ichimokucloud_changeLine REAL, ichimokucloud_baseLine REAL, ichimokucloud_leadingSpan1 REAL, ichimokucloud_leadingSpan2 REAL, stochastic_fast_k REAL, stochastic_slow_k REAL, stochastic_slow_d REAL, rsi REAL, rsis REAL)") # 키 생성 create_key = "CREATE INDEX IF NOT EXISTS " + stockAnalysisTableName + "_idx on " + stockAnalysisTableName + " (CODE, ymd) " @@ -685,16 +692,16 @@ class AnalyzerSqlite: result = cursor.fetchone() if result == None: sql = "INSERT INTO " + stockAnalysisTableName + "(CODE, NAME, ymd, close, diff, open, high, low, volume, " - sql += " avg4, avg5, avg6, avg10, avg12, avg20, avg36, avg40, avg48, avg60, avg120, avg200, avg240, avg300, " + sql += " avg3, avg4, avg5, avg6, avg10, avg12, avg20, avg36, avg40, avg48, avg60, avg120, avg200, avg240, avg300, " sql += " bolingerband_upper, bolingerband_lower, bolingerband_middle, " sql += " ichimokucloud_changeLine, ichimokucloud_baseLine, ichimokucloud_leadingSpan1, ichimokucloud_leadingSpan2, " sql += " stochastic_fast_k, stochastic_slow_k, stochastic_slow_d, " sql += " rsi, rsis) " - sql += " VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" + sql += " VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" cursor.execute(sql, ( stock["CODE"], stock["NAME"], price['ymd'], price['close'], price['diff'], price['open'], price['high'], price['low'], price['volume'], - price['avg4'], price['avg5'], price['avg6'], price['avg10'], price['avg12'], price['avg20'], price['avg36'], price['avg40'], price['avg48'], price['avg60'], price['avg120'], price['avg200'], price['avg240'], price['avg300'], + price['avg3'], price['avg4'], price['avg5'], price['avg6'], price['avg10'], price['avg12'], price['avg20'], price['avg36'], price['avg40'], price['avg48'], price['avg60'], price['avg120'], price['avg200'], price['avg240'], price['avg300'], price['bolingerband_upper'], price['bolingerband_lower'], price['bolingerband_middle'], price['ichimokucloud_changeLine'], price['ichimokucloud_baseLine'], price['ichimokucloud_leadingSpan1'], price['ichimokucloud_leadingSpan2'], @@ -703,7 +710,7 @@ class AnalyzerSqlite: else: sql = "UPDATE " + stockAnalysisTableName + " SET close=?, diff=?, open=?, high=?, low=?, volume=?, " - sql += " avg4=?, avg5=?, avg6=?, avg10=?, avg12=?, avg20=?, avg36=?, avg40=?, avg48=?, avg60=?, avg120=?, avg200=?, avg240=?, avg300=?, " + sql += " avg3=?, avg4=?, avg5=?, avg6=?, avg10=?, avg12=?, avg20=?, avg36=?, avg40=?, avg48=?, avg60=?, avg120=?, avg200=?, avg240=?, avg300=?, " sql += " bolingerband_upper=?, bolingerband_lower=?, bolingerband_middle=?, " sql += " ichimokucloud_changeLine=?, ichimokucloud_baseLine=?, ichimokucloud_leadingSpan1=?, ichimokucloud_leadingSpan2=?, " sql += " stochastic_fast_k=?, stochastic_slow_k=?, stochastic_slow_d=?, " @@ -712,7 +719,7 @@ class AnalyzerSqlite: cursor.execute(sql, (price['close'], price['diff'], price['open'], price['high'], price['low'], price['volume'], - price['avg4'], price['avg5'], price['avg6'], price['avg10'], price['avg12'], price['avg20'], price['avg36'], price['avg40'], price['avg48'], price['avg60'], price['avg120'], price['avg200'], price['avg240'], price['avg300'], + price['avg3'], price['avg4'], price['avg5'], price['avg6'], price['avg10'], price['avg12'], price['avg20'], price['avg36'], price['avg40'], price['avg48'], price['avg60'], price['avg120'], price['avg200'], price['avg240'], price['avg300'], price['bolingerband_upper'], price['bolingerband_lower'], price['bolingerband_middle'], price['ichimokucloud_changeLine'], price['ichimokucloud_baseLine'], price['ichimokucloud_leadingSpan1'], price['ichimokucloud_leadingSpan2'], price['stochastic_fast_k'], price['stochastic_slow_k'], price['stochastic_slow_d'], @@ -752,6 +759,7 @@ class AnalyzerSqlite: "high": item[4], "low": item[5], "volume": item[6], + "avg3": -1, "avg4": -1, "avg5": -1, "avg6": -1, @@ -815,6 +823,7 @@ class AnalyzerSqlite: "high": item[4], "low": item[5], "volume": item[6], + "avg3": -1, "avg4": -1, "avg5": -1, "avg6": -1, From 90af9dcde569d29f5b6b5592dd2a77e6fb931b81 Mon Sep 17 00:00:00 2001 From: "dosang.yoon" Date: Thu, 4 Aug 2022 13:14:53 +0900 Subject: [PATCH 3/9] init --- stock/analysis/AnalyzerSqlite.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stock/analysis/AnalyzerSqlite.py b/stock/analysis/AnalyzerSqlite.py index 301a1a2..13999a1 100644 --- a/stock/analysis/AnalyzerSqlite.py +++ b/stock/analysis/AnalyzerSqlite.py @@ -887,9 +887,9 @@ if __name__ == "__main__": stockFileName = PROJECT_HOME + '/resources/stock.db' analyzer = AnalyzerSqlite(PROJECT_HOME, stockFileName) - #analyzer.analyzeDaily() - #analyzer.analyzeGrouping("weekly") - #analyzer.analyzeGrouping("monthly") + analyzer.analyzeDaily() + analyzer.analyzeGrouping("weekly") + analyzer.analyzeGrouping("monthly") day = datetime.today().strftime("%Y%m%d") From fcb3d7a6b1943b36df7c0c5809688b11b8699022 Mon Sep 17 00:00:00 2001 From: "dosang.yoon" Date: Thu, 4 Aug 2022 13:18:26 +0900 Subject: [PATCH 4/9] init --- stock/analysis/Analyzer.py | 6 +++--- stock/analysis/AnalyzerSqlite.py | 12 ++++++------ stock/analysis/BolingerBand.py | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/stock/analysis/Analyzer.py b/stock/analysis/Analyzer.py index 9172924..12841c7 100644 --- a/stock/analysis/Analyzer.py +++ b/stock/analysis/Analyzer.py @@ -2,9 +2,9 @@ import json import os import time import shutil -from stockpredictor.analysis.Common import Common -from stockpredictor.analysis.Stochastic import Stochastic -from stockpredictor.analysis.BolingerBand import BolingerBand +from stock.analysis.Common import Common +from stock.analysis.Stochastic import Stochastic +from stock.analysis.BolingerBand import BolingerBand import matplotlib.pyplot as plt import datetime import sqlite3 diff --git a/stock/analysis/AnalyzerSqlite.py b/stock/analysis/AnalyzerSqlite.py index 13999a1..5deb225 100644 --- a/stock/analysis/AnalyzerSqlite.py +++ b/stock/analysis/AnalyzerSqlite.py @@ -16,12 +16,12 @@ import plotly.graph_objs as go from plotly import tools, subplots import plotly.io as po -from stockpredictor.analysis.Common import Common -from stockpredictor.analysis.Stochastic import Stochastic -from stockpredictor.analysis.BolingerBand import BolingerBand -from stockpredictor.analysis.IchimokuCloud import IchimokuCloud -from stockpredictor.analysis.RSI import RSI -from stockpredictor.crawler.sQLite.MovingAverage import MovingAverage +from stock.analysis.Common import Common +from stock.analysis.Stochastic import Stochastic +from stock.analysis.BolingerBand import BolingerBand +from stock.analysis.IchimokuCloud import IchimokuCloud +from stock.analysis.RSI import RSI +from stock.crawler.MovingAverage import MovingAverage class AnalyzerSqlite: PROJECT_HOME = None diff --git a/stock/analysis/BolingerBand.py b/stock/analysis/BolingerBand.py index b31c74c..dd55121 100644 --- a/stock/analysis/BolingerBand.py +++ b/stock/analysis/BolingerBand.py @@ -1,5 +1,5 @@ import pandas as pd -from stockpredictor.analysis.Common import Common +from stock.analysis.Common import Common import plotly.graph_objs as go from plotly import tools, subplots import plotly.io as po From d670c4e28eb1c45b0b6bd3eda42229fd606c882e Mon Sep 17 00:00:00 2001 From: "dosang.yoon" Date: Thu, 4 Aug 2022 13:31:25 +0900 Subject: [PATCH 5/9] init --- stock/analysis/AnalyzerSqlite.py | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/stock/analysis/AnalyzerSqlite.py b/stock/analysis/AnalyzerSqlite.py index 5deb225..c0eb428 100644 --- a/stock/analysis/AnalyzerSqlite.py +++ b/stock/analysis/AnalyzerSqlite.py @@ -231,6 +231,9 @@ class AnalyzerSqlite: os.mkdir(outPath) self.makeDir("final") + self.makeDir("daily_3일선_10일선_상향돌파") + self.makeDir("daily_3일선_10일선_하향돌파") + self.makeDir("weekly_4주선_48주선_상향돌파") self.makeDir("weekly_종가_12주선_상향돌파") self.makeDir("weekly_rsi_20이하") @@ -411,6 +414,26 @@ class AnalyzerSqlite: # 종목 상태 체크 분석 + # [Dailly] + if (stock_daily['avg3'][0] >= stock_daily['avg10'][0] and + stock_daily['avg3'][1] <= stock_daily['avg10'][1] and + stock_daily['avg3'][2] <= stock_daily['avg10'][2] and + stock_daily['avg3'][3] <= stock_daily['avg10'][3]): + type = "daily_3일선_10일선_상향돌파" + final_status += " " + type + final_status_count += 1 + self.writeFile(type, CODE, NAME, top, stock_weekly, status) + + if (stock_daily['avg3'][0] <= stock_daily['avg10'][0] and + stock_daily['avg3'][1] >= stock_daily['avg10'][1] and + stock_daily['avg3'][2] >= stock_daily['avg10'][2] and + stock_daily['avg3'][3] >= stock_daily['avg10'][3]): + type = "daily_3일선_10일선_하향돌파" + final_status += " " + type + final_status_count += 1 + self.writeFile(type, CODE, NAME, top, stock_weekly, status) + + # [Weekly] # 정배열 체크 temp_status = self.common.check_RightArrange(stock_weekly) @@ -887,9 +910,9 @@ if __name__ == "__main__": stockFileName = PROJECT_HOME + '/resources/stock.db' analyzer = AnalyzerSqlite(PROJECT_HOME, stockFileName) - analyzer.analyzeDaily() - analyzer.analyzeGrouping("weekly") - analyzer.analyzeGrouping("monthly") + #analyzer.analyzeDaily() + #analyzer.analyzeGrouping("weekly") + #analyzer.analyzeGrouping("monthly") day = datetime.today().strftime("%Y%m%d") From 210537415beb1cf676ad8ba639c0e7d700d813df Mon Sep 17 00:00:00 2001 From: dsyoon Date: Thu, 4 Aug 2022 23:44:31 +0900 Subject: [PATCH 6/9] init --- StockCrawler.py | 85 +++++++++++++++++++++++++------------------------ 1 file changed, 43 insertions(+), 42 deletions(-) diff --git a/StockCrawler.py b/StockCrawler.py index 1eedfeb..7777d5e 100644 --- a/StockCrawler.py +++ b/StockCrawler.py @@ -1,73 +1,74 @@ import os import shutil -import datetime import time +from datetime import datetime + from stock.crawler.FnGuideCrawler import FnGuideCrawler from stock.crawler.MetaCrawler import MetaCrawler from stock.crawler.StockCrawler import StockCrawler from stock.analysis.AnalyzerSqlite import AnalyzerSqlite -today = datetime.datetime.now().strftime("%Y-%m-%d") +today = datetime.now().strftime("%Y-%m-%d") # DB Browser for SQLite: http://hleecaster.com/python-sqlite3/ -PROJECT_HOME = os.path.join(os.path.dirname(os.path.join(os.path.dirname(os.path.join(os.path.dirname(os.path.join(os.path.dirname(__file__)))))))) +PROJECT_HOME = os.path.join(os.path.dirname(__file__)) START_DATE = "1900.01.01" start = time.time() stockFileName = PROJECT_HOME + '/resources/stock.db' +week = datetime.today().weekday() -# 재무제표는 3개월마다 다운로드를 한다. -fnGuideCrawler = FnGuideCrawler(START_DATE) -print("[KOSPI 상장기업 재무제표 다운로드]") -fnGuideCrawler.crawl_fnguide(stockFileName) +if week in (0, 1, 2, 3, 4): + # 재무제표는 3개월마다 다운로드를 한다. + fnGuideCrawler = FnGuideCrawler(START_DATE) + print("[KOSPI 상장기업 재무제표 다운로드]") + fnGuideCrawler.crawl_fnguide(stockFileName) -metaCrawler = MetaCrawler(START_DATE) -print("\n[증시자금동향 (신용잔고, 펀드자금 잔고)]") -metaCrawler.crawl_money_trend(stockFileName) + metaCrawler = MetaCrawler(START_DATE) + print("\n[증시자금동향 (신용잔고, 펀드자금 잔고)]") + metaCrawler.crawl_money_trend(stockFileName) -print("\n[국내 시장금리]") -metaCrawler.crawl_interest_rates(stockFileName) + print("\n[국내 시장금리]") + metaCrawler.crawl_interest_rates(stockFileName) -print("\n[투자자별 매매동향(Trading_Trend)]") -metaCrawler.crawl_trading_trend(stockFileName) + print("\n[투자자별 매매동향(Trading_Trend)]") + metaCrawler.crawl_trading_trend(stockFileName) -print("\n[환율 (USD, JPY, EUR, CNY)]") -metaCrawler.crawl_exchange(stockFileName) + print("\n[환율 (USD, JPY, EUR, CNY)]") + metaCrawler.crawl_exchange(stockFileName) -print("\n[원유 (WTI), 국제금, COPPER, NATURALGAS, CORN, SOYBEAN]") -metaCrawler.crawl_meterials(stockFileName) + print("\n[원유 (WTI), 국제금, COPPER, NATURALGAS, CORN, SOYBEAN]") + metaCrawler.crawl_meterials(stockFileName) -print("\n[종목 다운로드]") -stockCrawler = StockCrawler(START_DATE) -stockCrawler.crawl_etf_stocks(stockFileName) -stockCrawler.crawl_stocks(stockFileName) -#stockCrawler.crawl_special_stocks(stockFileName) + print("\n[종목 다운로드]") + stockCrawler = StockCrawler(START_DATE) + stockCrawler.crawl_etf_stocks(stockFileName) + stockCrawler.crawl_stocks(stockFileName) + #stockCrawler.crawl_special_stocks(stockFileName) -print("\n[종목 분석]") -# S: 분석까지 진행 -inFileName = PROJECT_HOME + '/resources/stock.db' -analyzerSqlite = AnalyzerSqlite(PROJECT_HOME, stockFileName) - -analyzerSqlite.analyzeDaily() -analyzerSqlite.analyzeGrouping("weekly") -analyzerSqlite.analyzeGrouping("monthly") - -analyzerSqlite = AnalyzerSqlite(PROJECT_HOME, stockFileName) -print("\n[종목 결정]") -day = datetime.datetime.today().strftime("%Y%m%d") -outPath = PROJECT_HOME + "/resources/analysis/" + day -if os.path.isdir(outPath): - shutil.rmtree(outPath) -os.mkdir(outPath) -print("print to Html...") -analyzerSqlite.findCandidate(outPath) -# E: 분석까지 진행 + print("\n[종목 분석]") + # S: 분석까지 진행 + inFileName = PROJECT_HOME + '/resources/stock.db' + analyzerSqlite = AnalyzerSqlite(PROJECT_HOME, stockFileName) + analyzerSqlite.analyzeDaily() + analyzerSqlite.analyzeGrouping("weekly") + analyzerSqlite.analyzeGrouping("monthly") + analyzerSqlite = AnalyzerSqlite(PROJECT_HOME, stockFileName) + print("\n[종목 결정]") + day = datetime.today().strftime("%Y%m%d") + outPath = PROJECT_HOME + "/resources/analysis/" + day + if os.path.isdir(outPath): + shutil.rmtree(outPath) + os.mkdir(outPath) + print("print to Html...") + analyzerSqlite.findCandidate(outPath) + # E: 분석까지 진행 print("time : %6.2f 초", (time.time() - start)) From a561c8dfd5548b1e061a5df80648a4905d80075e Mon Sep 17 00:00:00 2001 From: dsyoon Date: Fri, 5 Aug 2022 00:06:16 +0900 Subject: [PATCH 7/9] init --- StockCrawler.py | 1 + stock/crawler/StockCrawler.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/StockCrawler.py b/StockCrawler.py index 7777d5e..25242bc 100644 --- a/StockCrawler.py +++ b/StockCrawler.py @@ -21,6 +21,7 @@ stockFileName = PROJECT_HOME + '/resources/stock.db' week = datetime.today().weekday() if week in (0, 1, 2, 3, 4): + # 재무제표는 3개월마다 다운로드를 한다. fnGuideCrawler = FnGuideCrawler(START_DATE) print("[KOSPI 상장기업 재무제표 다운로드]") diff --git a/stock/crawler/StockCrawler.py b/stock/crawler/StockCrawler.py index 6208b9c..d58f5c6 100644 --- a/stock/crawler/StockCrawler.py +++ b/stock/crawler/StockCrawler.py @@ -338,7 +338,7 @@ class StockCrawler: continue stock_data.append({ - 'CODE':item_code, 'NAME':us_stocks[item_code], 'ymd': ymd.strftime('%Y.%m.%d'), + 'CODE':item_code, 'NAME':special_stocks[item_code], 'ymd': ymd.strftime('%Y.%m.%d'), 'close': round(stock['close'][ymd], 2), 'diff': round(diff, 2), 'open': round(stock['open'][ymd], 2), 'high': round(stock['high'][ymd], 2), 'low': round(stock['low'][ymd], 2), 'volume': stock['volume'][ymd] }) @@ -358,7 +358,7 @@ class StockCrawler: conn.commit() cursor.close() conn.close() - print(idx, item_code, us_stocks[item_code], (time.time() - start_time), "s") + print(idx, item_code, special_stocks[item_code], (time.time() - start_time), "s") start_time = time.time() sleep(0.05) From cdc42fb7571cf9eca2a8933ccd36c63d458d4885 Mon Sep 17 00:00:00 2001 From: dsyoon Date: Fri, 5 Aug 2022 00:17:06 +0900 Subject: [PATCH 8/9] init --- StockCrawler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/StockCrawler.py b/StockCrawler.py index 25242bc..2a32154 100644 --- a/StockCrawler.py +++ b/StockCrawler.py @@ -50,7 +50,6 @@ if week in (0, 1, 2, 3, 4): stockCrawler.crawl_stocks(stockFileName) #stockCrawler.crawl_special_stocks(stockFileName) - print("\n[종목 분석]") # S: 분석까지 진행 inFileName = PROJECT_HOME + '/resources/stock.db' From 72053b6b6c44e2799d3600a55a205254eb909095 Mon Sep 17 00:00:00 2001 From: dosangyoon Date: Fri, 5 Aug 2022 00:27:07 +0900 Subject: [PATCH 9/9] init --- stock/crawler/StockCrawler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stock/crawler/StockCrawler.py b/stock/crawler/StockCrawler.py index d58f5c6..8d7f483 100644 --- a/stock/crawler/StockCrawler.py +++ b/stock/crawler/StockCrawler.py @@ -400,9 +400,9 @@ class StockCrawler: if date == lastDay: lastPage = True - df = df.append(html[0], ignore_index=True) + df = pd.concat((df, html[0]), ignore_index=True) break - df = df.append(html[0], ignore_index=True) + df = pd.concat((df, html[0]), ignore_index=True) df = df.dropna() if (lastPage) or (len(df) < 1) or ("날짜" not in df) or (df.날짜[1]==''): print("\t- lastpage:", page)