YahooCSV.py

Go to the documentation of this file.

## @package serverportfolio.Parsers.YahooCSV
# @brief Define the class YahooCSV to use the 'old' Yahoo API
#
# LastChanged $Id: YahooCSV.py 13 2015-04-12 19:45:14Z michael $

# Note: CSV API, https://code.google.com/p/yahoo-finance-managed/wiki/csvHistQuotesDownload
# Adjusted Close includes both dividends and splits
# With HistoricalPrice, hp, You can download dividends by using &g=v in place of the three price variables
# yahoo.finance.dividend history use ichart/table.csv g=v to get dividend

# adjusted close, present in yahoo CSV
# http://www.captaineconomics.fr/-dividende-fractionnement-effet-cours-action-bourse

import copy, types, logging
import datetime, time

# for dates_intro, maybe to move to an other HTML parser
from lxml import html, etree

from serverportfolio import GlobalDicts
from serverportfolio.GlobalDicts import EAction
from serverportfolio.StockTemplates import StTmpl
from serverportfolio import Utils
# if missing run, but strange bug after...
from serverportfolio.PortfolioException import PortfolioError, ParserError

from serverportfolio.Parsers.Abstract import AbstractParser
from serverportfolio.Parsers import UtilsParsers
        
## @class YahooCSV
# @brief Derived parser class for retrieving divers data from the 'old' Yahoo API
#
# Implement actions:
#   - HistPrice : Historical Price
#   - DivSplit : Dividend/Split history (not fully implemented)
#   - DatesIntro : Get the first date available for the historical price
#   - Fundamental : YQL is preferred for this action. 
class YahooCSV( AbstractParser ):
    
    # option g=v gives dividends (but no split)
    __HISTORY_QUOTES_URL="http://ichart.yahoo.com/table.csv?"
    
    # other access http://ichart.finance.yahoo.com/x , contains data and dividends and split stock
    # did not find documentation, would need to compare the data, make more test
    # format is csv but different from table.csv, date format %y%m%d + keyword DIVIDEND/SPLIT
    # http://ichart.finance.yahoo.com/x?s=GSZ.PA&d=1&e=16&f=2015&g=v&a=1&b=2&c=2000 (dates important)
    # DIVIDEND, 20110506,0.750000
    # 20110506,169.03,170.15,168.24,168.89,4839400,157.75
    # 20110505,170.47,170.87,167.50,168.46,7332000,156.64
    # SPLIT, 19970528,2:1
    # 19970528,90.50,91.88,88.50,90.12,17898800,36.26
    # 19970527,174.00,179.25,174.00,179.25,12950800,36.06
    __HISTORY_QUOTES_X_URL="http://ichart.finance.yahoo.com/x?"
    
    # for currency discussion
    # http://stackoverflow.com/questions/5108399/yahoo-finance-all-currencies-quote-api-documentation
    #
    # page for querying max date of quotations, particularly the first date of introduction
    # related to pure HTML page
    # us page is correct, but data are wrong when downloaded! 5 minutes after ok !
    # http://finance.yahoo.com/q/hp?s=^FCHI&a=11&b=1&c=2014&d=01&e=9&f=2015&g=d
    __HISTORY_DATES_URL="http://fr.finance.yahoo.com/q/hp?"
    
    # page to retrieve fundamental data, old CVS API
    __QUOTES_CSV_URL="http://download.finance.yahoo.com/d/quotes.csv?" #s=AAPL,^
    
    ## @brief Constructor.
    # @param action enumeration of the EAction to perform
    def __init__(self, e_action ):              
        # init parent with action
        super(YahooCSV, self).__init__(e_action)
        
        # only logger as internal data member
        self.logger = logging.getLogger('SP.YahooCSV')
        self.logger.debug("Init class YahooCSV")
        
        self.source = 'YCSV'
       
    ## @brief Create the url according to the action and the stock(s).
    # Fundamental accepts multiple stocks.\n
    # @todo Check day, with HistPrice before use day+1 but very dangerous, now C++ should deal with overlap
    def create_url(self):
        self.logger.debug("YahooCSV::create_url")
        self.logger.debug("local stock: %s", self.local_stock)
        
        # extract symbol and stock from data member, correct output for one or multi stocks
        code_yahoo = ','.join( stock.get_action('Static','code_yahoo') for stock in self.local_stock.values() )
        self.logger.debug("code yahoo: %s" % code_yahoo)

        if self.e_action == EAction.HistPrice:
            self.logger.debug("YahooCSV with EAction.HistPrice, only one stock")
            assert ( len(self.local_stock) == 1 ), "YahooCSV with HistPrice deals only with one Stock at a time"
            
            symbol = self.local_stock.keys()[0]
            self.logger.debug("full dictionary symbol %s: %s" % (symbol, self.local_stock[symbol].get_action()))
            
            # last date in file, first_date is wrong set to 0
            lastdate = self.local_stock[symbol].get_action('HistPrice','last_date')
            # first date in file, not important here
            firstdate = self.local_stock[symbol].get_action('HistPrice','first_date')
            
            self.logger.debug("lastdate %s" % lastdate) # 1970-01-01
            (year,month,day)=(lastdate.year,lastdate.month,lastdate.day)
            
            # use today as default for last day, if use datetime get all hour..secs
            today = datetime.date.today()
            self.logger.debug("max_date 0, today %s" % today)
            (tyear,tmonth,tday)=(today.year,today.month,today.day)
            self.logger.debug("today day %d" % tday)
            
            # wrong here a,b,c start, d,e,f end
            # s=symbol &d=begin_month(-1) &e=begin_day &f= begin_year &g=d (for daily) 
            # &a= end_month(-1) &b=end_day &c=end_year
            
            # shift 1 for day to not copy again the same line ! Dangerous end of month ! Better checked in merging the data
            # Seems ok, no more shift
            self.url = self.__HISTORY_QUOTES_URL + "s=" + code_yahoo \
                   + "&d=" + str(tmonth-1) + "&e=" + str(tday) + "&f=" + str(tyear) \
                   + "&g=d" \
                   + "&a=" + str(month-1) + "&b=" + str(day) + "&c=" + str(year)
                   #+ "&a=" + str(month-1) + "&b=" + str(day+1) + "&c=" + str(year)
            
    # some common code with HistPrice, about dates
        elif self.e_action == EAction.DivSplit :
            
            assert ( len(self.local_stock) == 1 ), "YahooCSV with DivSplit deals only with one Stock at a time"
            symbol = self.local_stock.keys()[0]
            self.logger.debug("full dictionary symbol %s: %s" % (symbol, self.local_stock[symbol].get_action()))
            
            # last date in file, first_date is wrong set to 0 at the moment
            lastdate = self.local_stock[symbol].get_action('DivSplit','last_date')
            # first date in file, not important here
            firstdate = self.local_stock[symbol].get_action('DivSplit','first_date')
            
            self.logger.debug("lastdate %s" % lastdate) # 1970-01-01
            (year,month,day)=(lastdate.year,lastdate.month,lastdate.day)
            self.logger.debug("lastdate.day %d" % day)
            
            # update until now
            today = datetime.date.today()
            self.logger.debug("max_date today %s" % today)
            (tyear,tmonth,tday)=(today.year,today.month,today.day)
            self.logger.debug("today day %s" % tday)
            
            # g=v to print only Div and Split, dates optional restrict correctly
            self.url = self.__HISTORY_QUOTES_X_URL + "s=" + code_yahoo + "&g=v" \
                + "&a=" + str(month-1) + "&b=" + str(day) + "&c=" + str(year) \
                + "&d=" + str(tmonth-1) + "&e=" + str(tday) + "&f=" + str(tyear)
                
        # set method for getting introduction date, first available date in CSV
        elif self.e_action == EAction.DatesIntro :
            assert ( len(self.local_stock) == 1 ), "YahooCSV with DatesIntro deals only with one Stock at a time" 
            self.url = self.__HISTORY_DATES_URL + "s=" + code_yahoo
            
        # this action can be performed with multiple stocks at a time
        elif self.e_action == EAction.Fundamental :
            # Ask not used but give correct value for EURUS
            # symbol(s),name(n),Ask,MarketCapitalization(j1),SharesOwned(s1),DividendYield(d0),
            #  BVPS(b4),PriceBook(p6),PER(r0),PEG(r5), Error code e1
            self.url = self.__QUOTES_CSV_URL + "s=" + code_yahoo + '&f=snaj1s1d0b4p6r0r5e1&e=.csv'

        else :
            # not really an error, bug if not implemented, not exception here, assert better
            self.logger.critical("ERROR method parser not valid %s" % self.e_action)
            raise ParserError("YahooCSV not a valid e_action %s" % (self.e_action), \
                              self.local_stock.keys(), self.e_action.name, 'None' )

        self.logger.debug("url: %s",self.url)

    ## Parser implementation for YahooCSV, action dependent
    # @param s page return by web_query
    def parse( self, s):
        self.logger.debug("parse()") #, s
        self.logger.debug("self.e_action.name %s " % self.e_action.name )
        # print the complete webpage
        # print "s:\n", s
        try:
             if self.e_action == EAction.HistPrice:
                self.parse_hp(s)
                self.logger.debug("size of listCSV: %d" % 
                                  len(self.list_return_data['action_templ']['HistPrice']['list_csv']))

             elif self.e_action == EAction.DivSplit:
                self.parse_div(s)
                 
             elif self.e_action == EAction.Fundamental:
                self.parse_fund(s)
            
             elif self.e_action == EAction.DatesIntro:
                 self.parse_dates_intro(s)
                 
             else:
                 self.logger.error("Specific parse() function not defined for action: %s" % self.e_action)
                 raise ParserError("Specific parse() function not defined for action: %s" % (self.e_action),\
                                   self.local_stock.keys()[0], self.e_action.name, self.url)

        except ParserError as ex:
            self.logger.error("parse() caught ParserError ex: %s\n" % ex )
            raise
        
        # ok, works well
        except Exception as ex:
            self.logger.error("parse() caught Exception ex: %s\n" % ex )
            raise ParserError(ex, self.local_stock.keys()[0], self.e_action.name, self.url)

    ## @brief Specific parsing function to read a list of CSV, only daily values. 
    # To fit with design, must set a dict. Before never saved the listCSV only passed as argument
    # keep a simple dictionary (first, last, error in data(to write), listCSV specific in self
    # dictionary must be called dict_parser
    # @param all_line the page returned by web_query 
    def parse_hp(self, all_line): #lastCSV, max_date = None ):
        self.logger.debug("parse_hp()")
            
        # return all values in a list included in the template
        listCSV=[]
        # split line
        line_split=all_line.split("\n")
        
        # pass the header, last line, not sure
        for line_tmp in line_split[1:-1]:
            #print "newline ",line_tmp
            tmp_CSV=[]
            data_split=line_tmp.split(",")
            
            compt=0
            for data_tmp in data_split[:-1]:
                # deal with date for first in the list, translate to timestamp for C++
                # CSV on yahoo always date, no time
                if compt == 0:
                    tmp_date=time.strptime(data_tmp,"%Y-%m-%d")
                    data_tmp=int( round(time.mktime(tmp_date)))
                    tmp_CSV.append(data_tmp)
                    #print "tmp_date ",tmp_date
                    #print "data_tmp ",data_tmp
                    #self.logger.debug("parse_hp date_tmp %s" % (Utils.timestamp_to_string(data_tmp)) )
                #all other values convert to float (double for python,diff for C++)
                else:
                    tmp_CSV.append(float(data_tmp))
            
                compt+=1
            # add to the new list of data
            listCSV.append(tmp_CSV)
            #self.logger.debug("tmp_CSV %s" % tmp_CSV)
        self.logger.debug("listCSV.size %d ", len(listCSV))
             
        # include the template and symbol, only one stock here, could be in parse ?
        self.list_return_data = StTmpl.get_template_parser( self.e_action.name )
        self.list_return_data['symbol'] = self.local_stock.keys()[0]
        # assign data, must be specific, self.first not needed anymore
        tmp_templ = self.list_return_data['action_templ'][ self.e_action.name ]
        #tmp_templ['first_date'] = #listCSV[-1][0] #self.first_date
        tmp_templ['last_date'] = datetime.datetime.fromtimestamp( listCSV[0][0] )   #self.last_date
        tmp_templ['list_csv'] = listCSV 
        
        self.logger.debug("list_data_return %s" % self.list_return_data)
        # not used self.error
        #tmp_templ['error'] = None
        
    ## @brief Specific parsing function for dividends/split.
    # Save data in dictionary template DivSplit.list_div, list_split
    # @param all_line the page returned by web_query 
    def parse_div(self, all_line):
        self.logger.debug("parse_div()")
        print "all_line ", all_line
            
        # return all values in lists, test after if empty, wrong
        list_div=[]
        list_split=[]
        
        # split by lines
        line_split=all_line.split("\n")
        
        # pass the header, "Date,Dividends"
        # format : DIVIDEND, 20141013,0.500000
        # last line, not sure...
        for line_tmp in line_split[1:]:
            #print "newline ",line_tmp
            data_split=line_tmp.split(",")
            
            if data_split[0] == 'DIVIDEND':
                # datetime.strptime return a datetime
                # date = datetime.datetime.strptime( data_split[1].strip(), "%Y%m%d" )
                # time.strptime return a time_struct
                date = time.strptime( data_split[1].strip(), "%Y%m%d" )
                value = float(data_split[2])
                self.logger.debug("dividend: %s %f" % (date, value))
                self.logger.debug("mktime %d", int (time.mktime(date)))
                date = int ( time.mktime( date ) )
                list_div.append( [date, value])
                
            elif data_split[0] == 'SPLIT':
                date = int ( time.mktime( time.strptime( data_split[1].strip(), "%Y%m%d" ) ) )
                # format "1:4", always 1:X ??
                value = data_split[2]
                self.logger.debug("split: %s %s" % (date, value))
                list_split.append( [date, value])
            # check the STATUS
            else:
                #print "other "
                print line_tmp
                if data_split[0] == "STATUS":
                    assert( data_split[1].strip() == '0'  )
                
# new, first part could be done in parse()
        self.list_return_data = StTmpl.get_template_parser( self.e_action.name )
        # set the standard template with symbol
        self.list_return_data['symbol'] = self.local_stock.keys()[0]
        
        tmp_template = self.list_return_data['action_templ'][ self.e_action.name ]
        # must check for both not empty
        if list_div :
            # assign data, must be specific
            #tmp_template['first_date'] = #listCSV[-1][0] #self.first_date
            tmp_template['last_date'] = max( datetime.datetime.fromtimestamp( list_div[0][0] ), \
                                             datetime.datetime.fromtimestamp( list_div[0][0] ) )
            tmp_template['list_div'] = list_div
            tmp_template['list_split'] = list_split
            
        self.logger.debug("list_return_data %s" % self.list_return_data)
        self.logger.debug("list_div.size %d ", len(list_div))
        self.logger.debug("list_split.size %d ", len(list_split))
        
    ## @brief Parse fundamental data with old API, CSV data.
    # First implementation with fields specified in url, see Yahoo YQL for a better option.\n
    #
    # Support call with multiple stocks, one line by stock\n
    # Name is also available for Info\n
    # List of data:\n
    # - symbol(s),name(n),Ask,MarketCapitalization(j1),SharesOwned(s1),DividendYield(d0),
    #  BVPS(b4),PriceBook(p6),PER(r0),PEG(r5), new added e1 for error in code
    def parse_fund(self, all_line):
        self.logger.debug("parse_fund()")
        self.logger.debug("all_line %s" % all_line)
        
        # for debug with name containing ","
        # line:  "FSLR","First Solar, Inc.",... string format marked between double quotes...
        # re.findall(r'"([^"]*)"', inputString), extract all string in double quotes, what happens if empty...
        
        # main dictionary to save in parser, list of template parsers
        self.list_return_data = list()
        
        # parse line by line. One stock by line
        line_split = all_line.split("\n")
        for line in line_split :
            # test empty line, seems last line is empty
            if not line:
                self.logger.warning("1 line is empty")
                break
            self.logger.debug("line: %s" % line)
           
            # split comma separated line, problem with name ex: "FSLR","First Solar, Inc.",...
            elem_line = line.split(',')
            #self.logger.debug("elem_line: %s" % elem_line )
            
            # module csv can do it also, maybe other option to use
            yahoo_code = elem_line[0].replace('"', '').strip()
            self.logger.debug("yahoo_code: %s" % yahoo_code )
            
            # need to retrieve the symbol from the yahoo_code
            # raise a ParserError if cannot retrieve the symbol
            symbol = self.get_symbol_from_code( yahoo_code, 'code_yahoo')
           
            # creates standard and specific template for each Stock
            dict_data_stock = None
            dict_data_stock = StTmpl.get_template_parser( self.e_action.name )
            dict_data_stock['symbol'] = symbol
            
# It is the part specific to this action

            # for both fundamental (YQL and CSV) can get an error about the code for devise 
            if UtilsParsers.check_nan(elem_line[10]) == False:
                self.logger.error("error in parsing csv element[10] :%s" % line )
                #return
            
            # make shortcut 
            tmp_dict = dict_data_stock['action_templ'][ self.e_action.name ]
            
            # test, can extract the name
            # Not in template, but should work as. Tricky name contains "," First Solar, Inc.
            # Should count number of field, or test is string...
            # Could move name at the last position
            #tmp_dict['Name'] = elem_line[1]
            
            
            # B is deleted, maybe better function specific for Yahoo
            tmp_dict['MarketCapitalization'] = UtilsParsers.extract_number(elem_line[3]) #'45.753B'
            # did not pass because '-'
            tmp_dict['SharesOwned'] = UtilsParsers.extract_number( elem_line[4] )
            # return a float
            tmp_dict['DividendYield'] = UtilsParsers.extract_number( elem_line[5] )
            tmp_dict['BVPS'] = UtilsParsers.extract_number( elem_line[6] )
            tmp_dict['PriceBook'] = UtilsParsers.extract_number( elem_line[7] )
            tmp_dict['PER'] = UtilsParsers.extract_number (elem_line[8]) # 'N/A'
            tmp_dict['PEG'] = UtilsParsers.extract_number (elem_line[9]) # 'N/A'
            
            # append data for all stocks
            self.list_return_data.append( dict_data_stock )
            
        # end loop over line
        self.logger.debug("self.list_return_data:%s" % self.list_return_data)
        self.logger.debug("pretty test:\n%s" % Utils.pretty_dict( self.list_return_data ) )
        
    ## @brief To get introduction date of a stock, (first available on Yahoo) and last available.
    # need to make more test
    # @param html_page returned by web_query
    def parse_dates_intro( self, html_page ):
        
        htmltree = html.document_fromstring ( html_page )
        input_extracted = htmltree.xpath("//input[@id='startday']")
        day_start = input_extracted[0].get("value")
        
        # get main_parent for other research
        main_parent = input_extracted[0].getparent().getparent()
        option_selected = main_parent.xpath("td/select/option[@selected]")
        month_start = option_selected[0].get("value")
        year_selected=main_parent.xpath("td/input[@id='startyear']")
        year_start=year_selected[0].get("value")
        
        first_day = year_start+'-'+str(int(month_start)+1)+'-'+day_start
        # transform to a struct_tm
        date_start =  time.strptime(first_day,"%Y-%m-%d")
        datestamp_start=int( round(time.mktime(date_start)))
        # return struct_tm
        self.logger.debug("date_start: %s" % date_start)
        # return integer
        self.logger.debug("datestamp_start: %d" % datestamp_start)
        
# again common, except for Fundamental, because multiple queries
        # set the default template and symbol
        self.list_return_data = StTmpl.get_template_parser( self.e_action.name )
        # set the standard template with symbol
        self.list_return_data['symbol'] = self.local_stock.keys()[0]
        
# get specific template for this action
        self.list_return_data['action_templ'][self.e_action.name]['intro'] = datestamp_start
        self.list_return_data['action_templ'][self.e_action.name]['intro_str'] = \
            datetime.datetime.fromtimestamp( datestamp_start ).strftime('%Y-%m-%d %H:%M:%S')

# ###### Main for test
if __name__ == "__main__":
    
    from serverportfolio.DictionaryStocks import DictionaryStocks
    import sys, traceback
    
    print "main module YahooCSV"
    
    logging.basicConfig(level=logging.DEBUG)
    logging.getLogger('SP')
    m_logger = logging.getLogger("SP.main")
    
    try:
        m_stock = sys.argv[1]
        m_action = sys.argv[2]
    except IndexError:
        print " Only for debugging, should call ./Run_Parsers for all options "
        print " Usage:"
        print " YahooCSV CAC40 HistPrice"
        print " Action: HistPrice / Fundamental / DatesIntro / DivSplit"
        print " For Fundamental it is possible to query multi-stocks CAC40,GSZ,FSLR"
        sys.exit(1)
        
    # transform to a list, from format "Stock1,Stock2,.."
    m_list_stock = m_stock.split(',')
    
    try :
        if type(m_action) == types.StringType:
            m_e_action = EAction[m_action]
        #isisntance(EAction)
        else: 
            m_e_action = m_action
    except :
        m_logger.error("Exception from EActione numeration, cannot assign an e_action from action: %s" % m_action)
        raise
    
    m_logger.info("list_stock: %s" % m_list_stock)
    m_logger.info("e_action: %s " % m_e_action)    
    m_parserCSV = YahooCSV( m_e_action )
    #parserCSVsource = 'YCSV'
    
    m_logger.debug("\n== store copy of object in parser")
    try :
        m_parserCSV.store_stock_copy( m_list_stock )
        
    except ParserError as ex:
        print "Caught ParserError ex", ex.get_format_string()
        sys.exit(1)
        
    except Exception as ex:
        print "Got error in store_stock_copy"
        print "ex :", ex
        sys.exit(1)
        
# Preprocessing for HistPrice, DivSplit to set date range already available
    if ( m_action == EAction.HistPrice.name ) | ( m_action == EAction.DivSplit.name ):
        print "\n Preprocess for Action ", m_action
        DictionaryStocks().get_stocks( m_list_stock[0] ).last_CSV( m_action )
    
    m_logger.debug("\n== create_url default 0 0")
    try:
        m_parserCSV.create_url()
        
    except PortfolioError as ex:
        print "Catch PortfolioError from create_url: ", ex
        print ex.get_format_string()
        sys.exit(1)
        
    except Exception as ex:
        print "Catch Exception from create_url: ", ex
        # copy from parser stocks
        exc_type, exc_value, exc_traceback = sys.exc_info()
        print "*** print_tb:"
        traceback.print_tb(exc_traceback, limit=1, file=sys.stdout)
        # not working with print
        #print "trace1 ", traceback.print_tb( sys.exc_info()[2] )
        # return a tuple
        print "exec_info ", sys.exc_info()
        print "exec_info[2] ", sys.exc_info()[2]
        print "trace: "
        # ok 
        traceback.print_tb( sys.exc_info()[2] )
        sys.exit(1)
        
    m_logger.debug("\n== web_query ")
    try:
        m_s = UtilsParsers.web_query( m_parserCSV.url )
    # raise urllibERROR at the moment
    except Exception as ex:
        print "Got exception from web_query: ", ex
        print "exit "
        sys.exit(1)
       
    m_logger.debug("\n== parse the web page")
    try:
        m_parserCSV.parse(m_s)
        
    except ParserError as ex:
        print "Catch PaserError ", ex
        print "ex.get_format_string"
        # here nice output with all information
        print ex.get_format_string()
        sys.exit(1)
        
    except Exception as ex:
        print "Catch Exception from parse() ", ex
        sys.exit(1)
        
    m_logger.debug("\n== update the stock(s)")
    
    # argument of run_parser, set by UpdateStocks
    #m_parserCSV.option_post = { 
    #        'interactive': False,
    #        'to_save' : True
    #}
    
    m_parserCSV.update_stock()
    
    m_logger.debug("\n== print stock(s)")
    # only print the dictionary as final processing
    for stock in m_list_stock:
        print Utils.pretty_dict( DictionaryStocks().get_stocks( stock ).get_action() )