diff --git a/stockpredictor/analysis/AnalyzerSqlite.py b/stockpredictor/analysis/AnalyzerSqlite.py index a62d6ad..2fd26d5 100644 --- a/stockpredictor/analysis/AnalyzerSqlite.py +++ b/stockpredictor/analysis/AnalyzerSqlite.py @@ -6,6 +6,8 @@ import datetime import sqlite3 from datetime import datetime from matplotlib import rc +import pandas as pd +import copy rc('font', family='AppleGothic') plt.rcParams['axes.unicode_minus'] = False @@ -793,12 +795,47 @@ class AnalyzerSqlite: return - def analyze(self): - stockTableName = 'stock' - stockAnalysisTableName = 'stock_analysis' + def convertFormat(self, weekDict): + previous_close = 0 + stock_price = [] + for ts in weekDict['open']: + stock_price.append( + { + "ymd": ts.strftime("%Y.%m.%d"), + "close": weekDict['close'][ts], + "diff": weekDict['close'][ts] - previous_close, + "open": weekDict['open'][ts], + "high": weekDict['high'][ts], + "low": weekDict['low'][ts], + "volume": weekDict['volume'][ts], + "avg5": -1, + "avg10": -1, + "avg20": -1, + "avg60": -1, + "avg120": -1, + "avg200": -1, + "avg240": -1, + "bolingerband_upper": -1, + "bolingerband_lower": -1, + "bolingerband_middle": -1, + "ichimokucloud_changeLine": -1, + "ichimokucloud_baseLine": -1, + "ichimokucloud_leadingSpan1": -1, + "ichimokucloud_leadingSpan2": -1, + "stochastic_fast_k": -1, + "stochastic_slow_k": -1, + "stochastic_slow_d": -1 + } + ) + previous_close = weekDict['close'][ts] - conn = sqlite3.connect(self.stockFileName) - cursor = conn.cursor() + return stock_price + + def analyzeAdditionalInfo(self, stock, cursor, type=None): + if type==None: + stockAnalysisTableName = 'stock_analysis' + else: + stockAnalysisTableName = 'stock_analysis_' + type # 테이블 생성 cursor.execute("CREATE TABLE IF NOT EXISTS " + stockAnalysisTableName + " (CODE text, NAME text, ymd text, avg5 REAL, avg10 REAL, avg20 REAL, avg60 REAL, avg120 REAL, avg200 REAL, avg240 REAL, bolingerband_upper REAL, bolingerband_lower REAL, bolingerband_middle REAL, ichimokucloud_changeLine REAL, ichimokucloud_baseLine REAL, ichimokucloud_leadingSpan1 REAL, ichimokucloud_leadingSpan2 REAL, stochastic_fast_k REAL, stochastic_slow_k REAL, stochastic_slow_d REAL)") @@ -807,12 +844,48 @@ class AnalyzerSqlite: create_key = "CREATE INDEX IF NOT EXISTS " + stockAnalysisTableName + "_idx on " + stockAnalysisTableName + " (CODE, ymd) " cursor.execute(create_key) + # 이동 평균 계산 + stock["PRICE"] = sorted(stock["PRICE"], key=lambda x: x['ymd']) + self.get_moving_average(stock["PRICE"]) + + self.ichimokuCloud.analyze(stock) + self.stochastic.analyze(stock) + self.bolingerBand.analyze(stock) + + sorted_stock = sorted(stock["PRICE"], key=lambda x: x['ymd'], reverse=True) + for price in sorted_stock: + cursor.execute('SELECT * FROM ' + stockAnalysisTableName + ' WHERE CODE=? and ymd=?', + (stock['CODE'], price['ymd'],)) + result = cursor.fetchone() + if result == None: + sql = "INSERT INTO " + stockAnalysisTableName + "(CODE, NAME, ymd, avg5, avg10, avg20, avg60, avg120, avg200, avg240, " + sql += " bolingerband_upper, bolingerband_lower, bolingerband_middle, " + sql += " ichimokucloud_changeLine, ichimokucloud_baseLine, ichimokucloud_leadingSpan1, ichimokucloud_leadingSpan2, " + sql += " stochastic_fast_k, stochastic_slow_k, stochastic_slow_d) " + sql += " VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" + + cursor.execute(sql, ( + stock["CODE"], stock["NAME"], price['ymd'], price['avg5'], price['avg10'], price['avg20'], + price['avg60'], price['avg120'], price['avg200'], price['avg240'], + price['bolingerband_upper'], price['bolingerband_lower'], price['bolingerband_middle'], + price['ichimokucloud_changeLine'], price['ichimokucloud_baseLine'], price['ichimokucloud_leadingSpan1'], + price['ichimokucloud_leadingSpan2'], + price['stochastic_fast_k'], price['stochastic_slow_k'], price['stochastic_slow_d'],)) + else: + break + + return + + def analyzeDaily(self): + stockTableName = 'stock' + + conn = sqlite3.connect(self.stockFileName) + cursor = conn.cursor() + cursor.execute('SELECT distinct code, name FROM ' + stockTableName + ' order by code') items = cursor.fetchall() - cursor.close() - conn.close() - + stocks = [] for rowid, item in enumerate(items): conn = sqlite3.connect(self.stockFileName) cursor = conn.cursor() @@ -820,9 +893,6 @@ class AnalyzerSqlite: stock = {"CODE": item[0], "NAME": item[1], "PRICE":[]} print("# :", rowid, ", CODE: ", stock['CODE'], ", NAME: ", stock['NAME']) - cursor.execute('SELECT ymd FROM ' + stockAnalysisTableName + ' WHERE CODE=? order by ymd desc', (stock['CODE'],)) - result = cursor.fetchone() - sql = 'SELECT ymd, close, diff, open, high, low, volume FROM ' + stockTableName + ' where CODE=? order by ymd desc ' #if result is not None: # sql += ' limit 300' @@ -857,35 +927,79 @@ class AnalyzerSqlite: "stochastic_slow_k": -1, "stochastic_slow_d": -1}) - # 이동 평균 계산 - stock["PRICE"] = sorted(stock["PRICE"], key=lambda x: x['ymd']) - self.get_moving_average(stock["PRICE"]) + stocks.append(stock) + self.analyzeAdditionalInfo(stock, cursor) - self.ichimokuCloud.analyze(stock) - self.stochastic.analyze(stock) - self.bolingerBand.analyze(stock) + conn.commit() + cursor.close() + conn.close() + + return stocks + + def analyzeWeekly(self, stocks): + stocks_weekly = copy.deepcopy(stocks) + + conn = sqlite3.connect(self.stockFileName) + cursor = conn.cursor() + + for rowid, stock in enumerate(stocks_weekly): + print("Weekly # :", rowid, ", CODE: ", stock['CODE'], ", NAME: ", stock['NAME']) + + agg_dict = {'open': 'first', + 'high': 'max', + 'low': 'min', + 'close': 'last', + 'volume': 'sum'} + + df = pd.DataFrame(stock['PRICE']) + df['ymd'] = pd.to_datetime(df['ymd']) + df.set_index('ymd', inplace=True) + + df_week = df.resample('W-Mon').agg(agg_dict) + df_week = df_week.dropna() + df_week.merge(df, ) + stock['PRICE'] = self.convertFormat(df_week.to_dict()) + + self.analyzeAdditionalInfo(stock, cursor, "weekly") + + conn.commit() + cursor.close() + conn.close() - sorted_stock = sorted(stock["PRICE"], key=lambda x: x['ymd'], reverse=True) - for price in sorted_stock: - cursor.execute('SELECT * FROM ' + stockAnalysisTableName + ' WHERE CODE=? and ymd=?', (stock['CODE'], price['ymd'],)) - result = cursor.fetchone() - if result == None: - sql = "INSERT INTO " + stockAnalysisTableName + "(CODE, NAME, ymd, avg5, avg10, avg20, avg60, avg120, avg200, avg240, " - sql += " bolingerband_upper, bolingerband_lower, bolingerband_middle, " - sql += " ichimokucloud_changeLine, ichimokucloud_baseLine, ichimokucloud_leadingSpan1, ichimokucloud_leadingSpan2, " - sql += " stochastic_fast_k, stochastic_slow_k, stochastic_slow_d) " - sql += " VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" - cursor.execute(sql, (stock["CODE"], stock["NAME"], price['ymd'], price['avg5'], price['avg10'], price['avg20'], price['avg60'], price['avg120'], price['avg200'], price['avg240'], - price['bolingerband_upper'], price['bolingerband_lower'], price['bolingerband_middle'], - price['ichimokucloud_changeLine'], price['ichimokucloud_baseLine'], price['ichimokucloud_leadingSpan1'], price['ichimokucloud_leadingSpan2'], - price['stochastic_fast_k'], price['stochastic_slow_k'], price['stochastic_slow_d'],)) - else: - break - conn.commit() - cursor.close() - conn.close() return + def analyzeMonthly(self, stocks): + stocks_monthly = copy.deepcopy(stocks) + + conn = sqlite3.connect(self.stockFileName) + cursor = conn.cursor() + + for rowid, stock in enumerate(stocks_monthly): + + print("Monthly # :", rowid, ", CODE: ", stock['CODE'], ", NAME: ", stock['NAME']) + + agg_dict = {'open': 'first', + 'high': 'max', + 'low': 'min', + 'close': 'last', + 'volume': 'sum'} + + df = pd.DataFrame(stock['PRICE']) + df['ymd'] = pd.to_datetime(df['ymd']) + df.set_index('ymd', inplace=True) + + df_week = df.resample('M').agg(agg_dict) + df_week = df_week.dropna() + df_week.merge(df, ) + stock['PRICE'] = self.convertFormat(df_week.to_dict()) + + self.analyzeAdditionalInfo(stock, cursor, "monthly") + + conn.commit() + cursor.close() + conn.close() + + return if __name__ == "__main__": @@ -895,7 +1009,9 @@ if __name__ == "__main__": stockFileName = PROJECT_HOME + '/resources/stock.db' analyzer = AnalyzerSqlite(PROJECT_HOME, stockFileName) - #analyzer.analyze() + stocks = analyzer.analyzeDaily() + analyzer.analyzeWeekly(stocks) + analyzer.analyzeMonthly(stocks) day = datetime.today().strftime("%Y%m%d") diff --git a/stockpredictor/analysis/IchimokuCloud.py b/stockpredictor/analysis/IchimokuCloud.py index e0e85f1..02b2f72 100644 --- a/stockpredictor/analysis/IchimokuCloud.py +++ b/stockpredictor/analysis/IchimokuCloud.py @@ -68,8 +68,25 @@ class IchimokuCloud: move_LeadingSpan2.insert(0, None) # leadingSpan2에 맞추어 뒤로 빈 row를 채운다. + data = {"DATE": [], "close": [], "diff": [], "open": [], "high": [], "low": [], "volume": [], "avg5": [], "avg10": [], "avg20": [], "avg60": [], "avg120": [], "avg200": [], "avg240": []} for i in range(len(move_LeadingSpan2) - len(df)): - df = df.append({"DATE": None, "close": None, "diff": None, "open": None, "high": None, "low": None, "volume": None, "avg3": None, "avg5": None, "avg7": None, "avg10": None, "avg20": None, "avg30": None, "avg60": None, "avg90": None, "avg100": None, "avg120": None, "avg150": None, "avg180": None, "avg200": None, "avg240": None}, ignore_index=True) + #df = df.append({"DATE": None, "close": None, "diff": None, "open": None, "high": None, "low": None, "volume": None,"avg3": None, "avg5": None, "avg7": None, "avg10": None, "avg20": None, "avg30": None, "avg60": None,"avg90": None, "avg100": None, "avg120": None, "avg150": None, "avg180": None, "avg200": None,"avg240": None}, ignore_index=True) + data["DATE"].append(None) + data["close"].append(None) + data["diff"].append(None) + data["open"].append(None) + data["high"].append(None) + data["low"].append(None) + data["volume"].append(None) + data["avg5"].append(None) + data["avg10"].append(None) + data["avg20"].append(None) + data["avg60"].append(None) + data["avg120"].append(None) + data["avg200"].append(None) + data["avg240"].append(None) + df_temp = pd.DataFrame(data) + df = pd.concat([df, df_temp]) move_changeLine = list(changeLine.values) for i in range(len(move_LeadingSpan2) - len(move_changeLine)): move_changeLine.append(None) diff --git a/stockpredictor/crawler/sQLite/Crawler.py b/stockpredictor/crawler/sQLite/Crawler.py index 931569f..48a558c 100644 --- a/stockpredictor/crawler/sQLite/Crawler.py +++ b/stockpredictor/crawler/sQLite/Crawler.py @@ -46,14 +46,17 @@ print("\n[종목 다운로드]") stockCrawler = StockCrawler(START_DATE) stockCrawler.crawl_etf_stocks(stockFileName) stockCrawler.crawl_stocks(stockFileName) -stockCrawler.crawl_special_stocks(stockFileName) +#stockCrawler.crawl_special_stocks(stockFileName) print("\n[종목 분석]") # S: 분석까지 진행 inFileName = PROJECT_HOME + '/resources/stock.db' analyzerSqlite = AnalyzerSqlite(PROJECT_HOME, stockFileName) -analyzerSqlite.analyze() + +stocks = analyzerSqlite.analyzeDaily() +analyzerSqlite.analyzeWeekly(stocks) +analyzerSqlite.analyzeMonthly(stocks) analyzerSqlite = AnalyzerSqlite(PROJECT_HOME, stockFileName) print("\n[종목 결정]")