init
This commit is contained in:
dosangyoon
@@ -1,4 +1,4 @@
|
||||
import json
|
||||
import os
|
||||
import datetime
|
||||
import requests
|
||||
import sqlite3
|
||||
@@ -12,75 +12,6 @@ class MetaCrawler:
|
||||
def __init__(self):
|
||||
return
|
||||
|
||||
# 참고) http://blog.naver.com/PostView.nhn?blogId=koko8624&logNo=221288761509
|
||||
def crawl_meterials(self, inFileName):
|
||||
tableName = 'meta_1'
|
||||
conn = sqlite3.connect(inFileName)
|
||||
cursor = conn.cursor()
|
||||
|
||||
# 테이블 생성
|
||||
cursor.execute("CREATE TABLE IF NOT EXISTS " + tableName + " (CODE text, NAME text, ymd text, close REAL, diff REAL, rate REAL)")
|
||||
|
||||
# 키 생성
|
||||
create_key = "CREATE INDEX IF NOT EXISTS " + tableName + "_idx on " + tableName + " (CODE, ymd) "
|
||||
cursor.execute(create_key)
|
||||
|
||||
inputs = []
|
||||
inputs.append( {'NAME':'WTI', 'CODE':'OIL_CL', 'URL':'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=OIL_CL&fdtc=2'} ) # WTI
|
||||
inputs.append( {'NAME':'GOLD', 'CODE':'CMDT_GC', 'URL':'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=CMDT_GC&fdtc=2'} ) # 국제 금
|
||||
inputs.append({'NAME': 'COPPER', 'CODE': 'CMDT_CDY','URL': 'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=CMDT_CDY&fdtc=2'}) # 구리
|
||||
inputs.append({'NAME': 'NATURALGAS', 'CODE': 'CMDT_NG','URL': 'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=CMDT_NG&fdtc=2'}) # 천연가스
|
||||
inputs.append({'NAME': 'CORN', 'CODE': 'CMDT_C','URL': 'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=CMDT_C&fdtc=2'}) # 국제 옥수수
|
||||
inputs.append({'NAME': 'SOYBEAN', 'CODE': 'CMDT_S','URL': 'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=CMDT_S&fdtc=2'}) # 국제 대두
|
||||
|
||||
for i in range(len(inputs)):
|
||||
input = inputs[i]
|
||||
NAME = input['NAME']
|
||||
CODE = input['CODE']
|
||||
|
||||
cursor.execute('SELECT ymd FROM ' + tableName + ' WHERE CODE=? order by ymd desc', (CODE,))
|
||||
result = cursor.fetchone()
|
||||
if result == None:
|
||||
lastDay = "1900.01.01"
|
||||
else:
|
||||
lastDay = result[0]
|
||||
|
||||
finish = False
|
||||
for i in range(1, self.limit_page_count):
|
||||
#html = pd.read_html(input['URL'] + '&page=%s' % i, header=0)
|
||||
html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header).text)
|
||||
|
||||
# 마지막 페이지 까지 받기
|
||||
if len(html[0].날짜.values) <= 1:
|
||||
break
|
||||
|
||||
for j in range(0, len(html[0].values)):
|
||||
item = html[0].values[j]
|
||||
|
||||
if item[0] <= lastDay:
|
||||
finish = True
|
||||
break
|
||||
ymd = item[0] # 날짜
|
||||
close = item[1] # 종가
|
||||
diff = item[2] # 전일대비
|
||||
rate = item[3] # 등락율
|
||||
|
||||
cursor.execute('SELECT * FROM ' + tableName + ' WHERE CODE=? and ymd=?', (CODE,ymd,))
|
||||
result = cursor.fetchone()
|
||||
if result == None:
|
||||
cursor.execute("INSERT INTO " + tableName + "(CODE, NAME, ymd, close, diff, rate) VALUES(?, ?, ?, ?, ?, ?)", (CODE, NAME, ymd, close, diff, rate))
|
||||
#else:
|
||||
# cursor.execute("UPDATE " + tableName + " SET close=?, diff=?, rate=? WHERE CODE=? and ymd=?", (close, diff, rate, CODE, ymd))
|
||||
|
||||
print(CODE, NAME, ymd)
|
||||
if finish:
|
||||
break
|
||||
|
||||
conn.commit()
|
||||
cursor.close()
|
||||
conn.close()
|
||||
return
|
||||
|
||||
# 참고) http://blog.naver.com/PostView.nhn?blogId=koko8624&logNo=221288761509
|
||||
def crawl_exchange(self, inFileName):
|
||||
tableName = 'meta_1'
|
||||
@@ -171,6 +102,10 @@ class MetaCrawler:
|
||||
# 테이블 생성
|
||||
cursor.execute("CREATE TABLE IF NOT EXISTS "+tableName+" (ymd text PRIMARY KEY, pri integer, fori integer, ins integer, ins0 integer, ins1 integer, ins2 integer, ins3 integer, ins4 integer, ins5 integer, cor integer)")
|
||||
|
||||
# 키 생성
|
||||
create_key = "CREATE INDEX IF NOT EXISTS "+tableName+"_idx on "+tableName+" (ymd) "
|
||||
cursor.execute(create_key)
|
||||
|
||||
cursor.execute('SELECT ymd FROM ' + tableName + ' order by ymd desc')
|
||||
result = cursor.fetchone()
|
||||
if result == None:
|
||||
@@ -365,3 +300,98 @@ class MetaCrawler:
|
||||
cursor.close()
|
||||
conn.close()
|
||||
return
|
||||
|
||||
# 참고) http://blog.naver.com/PostView.nhn?blogId=koko8624&logNo=221288761509
|
||||
def crawl_meterials(self, inFileName):
|
||||
tableName = 'meta_5'
|
||||
conn = sqlite3.connect(inFileName)
|
||||
cursor = conn.cursor()
|
||||
|
||||
# 테이블 생성
|
||||
cursor.execute("CREATE TABLE IF NOT EXISTS " + tableName + " (CODE text, NAME text, ymd text, close REAL, diff REAL, rate REAL)")
|
||||
|
||||
# 키 생성
|
||||
create_key = "CREATE INDEX IF NOT EXISTS " + tableName + "_idx on " + tableName + " (CODE, NAME, ymd) "
|
||||
cursor.execute(create_key)
|
||||
|
||||
inputs = []
|
||||
inputs.append( {'NAME':'WTI', 'CODE':'OIL_CL', 'URL':'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=OIL_CL&fdtc=2'} ) # WTI
|
||||
inputs.append( {'NAME':'GOLD', 'CODE':'CMDT_GC', 'URL':'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=CMDT_GC&fdtc=2'} ) # 국제 금
|
||||
inputs.append({'NAME': 'COPPER', 'CODE': 'CMDT_CDY','URL': 'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=CMDT_CDY&fdtc=2'}) # 구리
|
||||
inputs.append({'NAME': 'NATURALGAS', 'CODE': 'CMDT_NG','URL': 'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=CMDT_NG&fdtc=2'}) # 천연가스
|
||||
inputs.append({'NAME': 'CORN', 'CODE': 'CMDT_C','URL': 'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=CMDT_C&fdtc=2'}) # 국제 옥수수
|
||||
inputs.append({'NAME': 'SOYBEAN', 'CODE': 'CMDT_S','URL': 'http://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=CMDT_S&fdtc=2'}) # 국제 대두
|
||||
|
||||
for i in range(len(inputs)):
|
||||
input = inputs[i]
|
||||
NAME = input['NAME']
|
||||
CODE = input['CODE']
|
||||
|
||||
cursor.execute('SELECT ymd FROM ' + tableName + ' WHERE CODE=? order by ymd desc', (CODE,))
|
||||
result = cursor.fetchone()
|
||||
if result == None:
|
||||
lastDay = "1900.01.01"
|
||||
else:
|
||||
lastDay = result[0]
|
||||
|
||||
finish = False
|
||||
for i in range(1, self.limit_page_count):
|
||||
#html = pd.read_html(input['URL'] + '&page=%s' % i, header=0)
|
||||
html = pd.read_html(requests.get(input['URL'] + '&page=%s' % i, headers=self.header).text)
|
||||
|
||||
# 마지막 페이지 까지 받기
|
||||
if len(html[0].날짜.values) <= 1:
|
||||
break
|
||||
|
||||
for j in range(0, len(html[0].values)):
|
||||
item = html[0].values[j]
|
||||
|
||||
if item[0] <= lastDay:
|
||||
finish = True
|
||||
break
|
||||
ymd = item[0] # 날짜
|
||||
close = item[1] # 종가
|
||||
diff = item[2] # 전일대비
|
||||
rate = item[3] # 등락율
|
||||
|
||||
cursor.execute('SELECT * FROM ' + tableName + ' WHERE CODE=? and ymd=?', (CODE,ymd,))
|
||||
result = cursor.fetchone()
|
||||
if result == None:
|
||||
cursor.execute("INSERT INTO " + tableName + "(CODE, NAME, ymd, close, diff, rate) VALUES(?, ?, ?, ?, ?, ?)", (CODE, NAME, ymd, close, diff, rate))
|
||||
#else:
|
||||
# cursor.execute("UPDATE " + tableName + " SET close=?, diff=?, rate=? WHERE CODE=? and ymd=?", (close, diff, rate, CODE, ymd))
|
||||
|
||||
print(CODE, NAME, ymd)
|
||||
if finish:
|
||||
break
|
||||
|
||||
conn.commit()
|
||||
cursor.close()
|
||||
conn.close()
|
||||
return
|
||||
|
||||
if __name__ == "__main__":
|
||||
PROJECT_HOME = os.path.join(os.path.dirname(os.path.join(os.path.dirname(os.path.join(os.path.dirname(os.path.join(os.path.dirname(__file__))))))))
|
||||
|
||||
metaCrawler = MetaCrawler()
|
||||
|
||||
print("\n[환율 (USD, JPY, EUR, CNY)]")
|
||||
inFileName = PROJECT_HOME + '/resources/stock.db'
|
||||
metaCrawler.crawl_exchange(inFileName)
|
||||
|
||||
print("\n[투자자별 매매동향(Trading_Trend)]")
|
||||
inFileName = PROJECT_HOME + '/resources/stock.db'
|
||||
metaCrawler.crawl_trading_trend(inFileName)
|
||||
|
||||
print("\n[증시자금동향 (신용잔고, 펀드자금 잔고)]")
|
||||
inFileName = PROJECT_HOME + '/resources/stock.db'
|
||||
metaCrawler.crawl_money_trend(inFileName)
|
||||
|
||||
print("\n[국내 시장금리]")
|
||||
inFileName = PROJECT_HOME + '/resources/stock.db'
|
||||
metaCrawler.crawl_interest_rates(inFileName)
|
||||
|
||||
print("\n[원유 (WTI), 국제금, COPPER, NATURALGAS, CORN, SOYBEAN]")
|
||||
inFileName = PROJECT_HOME + '/resources/stock.db'
|
||||
metaCrawler.crawl_meterials(inFileName)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user