from bs4 import BeautifulSoup
from pandas import DataFrame
import requests as re
import pandas as pd
import os
import sqlite3

class FnGuideCrawler:
    header = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36'}
    START_DATE = None

    def __init__(self, START_DATE):
        self.START_DATE = START_DATE
        return

    def getStockInfo(self):
        code_df = pd.read_html('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', header=0)[0]
        #code_df = pd.read_html(requests.get('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13', headers=self.header).text)

        # 종목코드가 6자리이기 때문에 6자리를 맞춰주기 위해 설정해줌
        code_df.종목코드 = code_df.종목코드.map('{:06d}'.format)

        # 우리가 필요한 것은 회사명과 종목코드이기 때문에 필요없는 column들은 제외해준다.
        code_df = code_df[['회사명', '종목코드']]

        # 한글로된 컬럼명을 영어로 바꿔준다.
        code_df = code_df.rename(columns={'회사명': 'name', '종목코드': 'code'})
        ###print (code_df.head())

        return code_df

    # FnGuide에서 크롤링한 KOSPI 상장기업의 재무제표
    # http://blog.naver.com/PostView.nhn?blogId=koko8624&logNo=221294884955&parentCategoryNo=&categoryNo=&viewDate=&isShowPopularPosts=false&from=postView
    def get_fnguide_table(self, code):
        url = re.get('http://comp.fnguide.com/SVO2/ASP/SVD_main.asp?pGB=1&gicode=A%s'%(code.strip()))
        url = url.content

        html = BeautifulSoup(url,'html.parser')
        body = html.find('body')

        try:
            fn_body = body.find('div', {'class': 'fng_body asp_body'})
            ur_table = fn_body.find('div', {'id': 'div15'})
            table = ur_table.find('div', {'id': 'highlight_D_Y'})

            tbody = table.find('tbody')
            tr = tbody.find_all('tr')
            Table = DataFrame()
        except:
            return {}

        for i in tr:
            ''' 자료 항목 가져오기'''
            category = i.find('span', {'class': 'txt_acd'})

            if category == None:
                category = i.find('th')

            category = category.text.strip()

            '''값 가져오기'''
            value_list = []

            j = i.find_all('td', {'class': 'r'})

            for value in j:
                temp = value.text.replace(',', '').strip()

                try:
                    temp = float(temp)
                    value_list.append(temp)
                except:
                    value_list.append(0)

            Table['%s' % (category)] = value_list

            ''' 기간 가져오기 '''
            thead = table.find('thead')
            tr_2 = thead.find('tr', {'class': 'td_gapcolor2'}).find_all('th')

            year_list = []

            for i in tr_2:
                try:
                    temp_year = i.find('span', {'class': 'txt_acd'}).text
                except:
                    temp_year = i.text

                temp_year = temp_year.replace("/",".")+".01"
                year_list.append(temp_year)

            Table.index = year_list

        return Table.T.to_dict()

    def crawl_fnguide(self, inFileName):
        tableName = 'fnguide'
        conn = sqlite3.connect(inFileName, isolation_level=None)
        cursor = conn.cursor()

        # 테이블 생성
        create_sql = "CREATE TABLE IF NOT EXISTS "+tableName+" (CODE text, NAME text, ymd text, "
        create_sql += " type text, sales REAL, net_business_profits REAL, business_profits REAL, business_profits_release REAL, "
        create_sql += " net_profit REAL, significant_shareholder_net_profit REAL, "
        create_sql += " none_significant_shareholder_net_profit REAL, total_assets REAL, total_debt REAL, "
        create_sql += " total_ownership_interest REAL, equity_holdings REAL, none_equity_holdings REAL, capital REAL, "
        create_sql += " debt_ratio REAL, reserve_ratio REAL, business_profits_ratio REAL, "
        create_sql += " significant_shareholder_profits_ratio REAL, ROA REAL, ROE REAL, EPS REAL, BPS REAL, DPS REAL, "
        create_sql += " PER REAL, PBR REAL, share_outstanding REAL, dividend_rate REAL)"
        cursor.execute(create_sql)

        # 키 생성
        create_key = "CREATE INDEX IF NOT EXISTS "+tableName+"_idx on "+tableName+" (CODE, ymd) "
        cursor.execute(create_key)

        code_df = self.getStockInfo()
        idx = 0
        for item in code_df.values:
            item_name = item[0]
            item_code = item[1]

            idx += 1
            print(idx, item_code, item_name, 'http://comp.fnguide.com/SVO2/ASP/SVD_main.asp?pGB=1&gicode=A%s'%(item_code.strip()))

            fnGuideData = self.get_fnguide_table(item_code)

            """
            매출액: sales
            순영업수익: net_business_profits REAL,
            영업이익: business_profits
            영업이익(발표기준): business_profits_release
            당기순이익: net_profit
            지배주주순이익: significant_shareholder_net_profit
            비지배주주순이익: none_significant_shareholder_net_profit
            자산총계: total_assets
            부채총계: total_debt
            자본총계: total_ownership_interest
            지배주주지분: equity_holdings
            비지배주주지분: none_equity_holdings
            자본금: capital
            부채비율: debt_ratio
            유보율: reserve_ratio
            영업이익률: business_profits_ratio
            지배주주순이익률: significant_shareholder_profits_ratio
            ROA: ROA
            ROE: ROE
            EPS(원): EPS
            BPS(원): BPS
            DPS(원): DPS
            PER: PER
            PBR: PBR
            발행주식수: share_outstanding
            배당수익률: dividend_rate
"""
            for key_ymd in fnGuideData:
                ymd = key_ymd.replace('(P)', '').replace('(E)', '')
                if key_ymd.find('P') > 0:
                    type = 'P'
                elif key_ymd.find('E') > 0:
                    type = 'E'
                else:
                    type = ''
                if '매출액' in fnGuideData[key_ymd]:
                    sales = fnGuideData[key_ymd]['매출액']
                else:
                    sales = 0
                if '순영업수익' in fnGuideData[key_ymd]:
                    net_business_profits = fnGuideData[key_ymd]['순영업수익']
                else:
                    net_business_profits = 0
                business_profits = fnGuideData[key_ymd]['영업이익']
                business_profits_release = fnGuideData[key_ymd]['영업이익(발표기준)']
                net_profit = fnGuideData[key_ymd]['당기순이익']
                significant_shareholder_net_profit = fnGuideData[key_ymd]['지배주주순이익']
                none_significant_shareholder_net_profit = fnGuideData[key_ymd]['비지배주주순이익']
                total_assets = fnGuideData[key_ymd]['자산총계']
                total_debt = fnGuideData[key_ymd]['부채총계']
                total_ownership_interest = fnGuideData[key_ymd]['자본총계']
                equity_holdings = fnGuideData[key_ymd]['지배주주지분']
                none_equity_holdings = fnGuideData[key_ymd]['비지배주주지분']
                capital = fnGuideData[key_ymd]['자본금']
                debt_ratio = fnGuideData[key_ymd]['부채비율']
                reserve_ratio = fnGuideData[key_ymd]['유보율']
                business_profits_ratio = fnGuideData[key_ymd]['영업이익률']
                significant_shareholder_profits_ratio = fnGuideData[key_ymd]['지배주주순이익률']
                ROA = fnGuideData[key_ymd]['ROA']
                ROE = fnGuideData[key_ymd]['ROE']
                if 'EPS(원)' in fnGuideData[key_ymd]:
                    EPS = fnGuideData[key_ymd]['EPS(원)']
                else:
                    EPS = fnGuideData[key_ymd]['EPS']
                if 'BPS(원)' in fnGuideData[key_ymd]:
                    BPS = fnGuideData[key_ymd]['BPS(원)']
                else:
                    BPS = fnGuideData[key_ymd]['BPS']
                if 'DPS(원)' in fnGuideData[key_ymd]:
                    DPS = fnGuideData[key_ymd]['DPS(원)']
                else:
                    DPS = fnGuideData[key_ymd]['DPS']
                PER = fnGuideData[key_ymd]['PER']
                PBR = fnGuideData[key_ymd]['PBR']
                share_outstanding = fnGuideData[key_ymd]['발행주식수']
                dividend_rate = fnGuideData[key_ymd]['배당수익률']

                cursor.execute('SELECT * FROM '+tableName+' WHERE CODE=? and ymd=?', (item_code, ymd))
                result = cursor.fetchone()
                if result == None:
                    cursor.execute("INSERT INTO "+tableName+"(CODE, NAME, ymd, type, sales, net_business_profits, business_profits, business_profits_release, net_profit, significant_shareholder_net_profit, none_significant_shareholder_net_profit, total_assets, total_debt, total_ownership_interest, equity_holdings, none_equity_holdings, capital, debt_ratio, reserve_ratio, business_profits_ratio, significant_shareholder_profits_ratio, ROA, ROE, EPS, BPS, DPS, PER, PBR, share_outstanding, dividend_rate) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (item_code, item_name, ymd, type, sales, net_business_profits, business_profits, business_profits_release, net_profit, significant_shareholder_net_profit, none_significant_shareholder_net_profit, total_assets, total_debt, total_ownership_interest, equity_holdings, none_equity_holdings, capital, debt_ratio, reserve_ratio, business_profits_ratio, significant_shareholder_profits_ratio, ROA, ROE, EPS, BPS, DPS, PER, PBR, share_outstanding, dividend_rate))
                else:
                    # cursor.execute("UPDATE "+tableName+" SET ymd=?, type=?, sales=?, net_business_profits=?, business_profits=?, business_profits_release=?, net_profit=?, significant_shareholder_net_profit=?, none_significant_shareholder_net_profit=?, total_assets=?, total_debt=?, total_ownership_interest=?, equity_holdings=?, none_equity_holdings=?, capital=?, debt_ratio=?, reserve_ratio=?, business_profits_ratio=?, significant_shareholder_profits_ratio=?, ROA=?, ROE=?, EPS=?, BPS=?, DPS=?, PER=?, PBR=?, share_outstanding=?, dividend_rate=? WHERE CODE=?", (ymd, type, sales, net_business_profits, business_profits, business_profits_release, net_profit, significant_shareholder_net_profit, none_significant_shareholder_net_profit, total_assets, total_debt, total_ownership_interest, equity_holdings, none_equity_holdings, capital, debt_ratio, reserve_ratio, business_profits_ratio, significant_shareholder_profits_ratio, ROA, ROE, EPS, BPS, DPS, PER, PBR, share_outstanding, dividend_rate, item_code))
                    break

        cursor.close()
        conn.close()
        return

if __name__ == "__main__":
    PROJECT_HOME = os.path.join(os.path.dirname(os.path.join(os.path.dirname(os.path.join(os.path.dirname(os.path.join(os.path.dirname(__file__))))))))
    inFnguideFileName = PROJECT_HOME + '/resources/stock.db'
    crawler = FnGuideCrawler()
    crawler.crawl_fnguide(inFnguideFileName)
