StockTemplates.py

Go to the documentation of this file.

## @package serverportfolio.StockTemplates
# @brief Define the global variable  @ref serverportfolio.StockTemplates.StTmpl "StockTemplates.StTmpl" and dictionary templates.
#
# Last Changed $Id: StockTemplates.py 29 2015-05-05 11:09:54Z michael $

import copy, datetime, logging

from serverportfolio.PortfolioException import *
from serverportfolio import Utils

## @class StockTemplates
# @brief Define template dictionaries to describe the data structure.
#
# Templates are named by the "action" to be performed by the parsers.\n
# XML output extends the action template.\n
#
# It is used by a global variable which must be accessed by StockTemplates.StTmpl\n
# Group templates, by use of dictionaries, to be included by Stock when they run a specific request (EAction, e.g. InstValue, HistPrice..)\n
# 'Static' data are a particular template which is read from the configuration file (dictstocks.txt by default) by DictionaryStocks.\n
class StockTemplates(object):
    
    # logger as class member
    _logger = logging.getLogger("SP.StockTemplate")
    _logger.debug( "Load StockTemplate" )

    # import math
    # x=float('nan') math.isnan
    # YahooCSV, all entries specified
    # YQL given entry with name
    ## Fundamental data, used by YQL and YahooCSV
    # @note to copy with copy.deepcopy( templ_dict[ key ] )

    ## @brief Template for fundamental data, used by YahooYQL(default) and YahooCSV
    _templ_fundamental = {
        # fixed fields
        'MarketCapitalization' : float(0), #append=false, not here
        'SharesOwned' : int(0),
        'DividendYield' : float(0),
        'BVPS' : float(0),
        'PriceBook' : float(0),
        'PER' : float(0),
        'PEG' : float(0),
        # variable field, only YQL may report other entries, YahooCSV query could be modified as well for test
        'other' : {},
        #'error' : None,
        # add source for each action to be saved in XML. Common, to include by default by an other template ?(one_value_xml)
        # new added, strange, can delete ?, used by parser to save the source. created on the fly
        # 'source' : str()
    }

    ## @brief Normally these entries should not change.
    # Executed by YQL, quotes or quote table give the name
    _templ_info = {
        # Name of the company
        'Name' : str(),
        # sector/ industry could be an enum (maybe can extend on the fly??)
        # YahooAPI CSV defines 9 sectors
        'Sector' : str(),
        # Each sector has many industries (not so clear)
        'Industry' : str(),
        # place of cotation (Paris, NasdaqNM)
        'StockExchange' : str(),
        # with YQL other entries are provided
        'other' : {},
        # test error
        #'error' : None,
        # new added, in each template, not too clear, necessary for parser
        #'source' : str()
    }

    # maybe better to save in dictionary after running check_list_CSV, can be separate from the parser
    # clearer for parser also.

    ## Historical Prices, the complete list (CSV) is written to file but not saved in dictionary
    # format of date in template, string/timestamp/datetime.datetime ?
    # C++ always timestamp, but datetime.datetime more convenient in python. If C++ read timestamp.
    # internal C++ datetime.datetime easier, datetime if extended later to inst ?
    _templ_hp = {
        'first_date' : datetime.datetime.fromtimestamp(0),
        'last_date' : datetime.datetime.fromtimestamp(0),
        # yahoo, correct format epoch for list_csv
        'list_csv' : None
        #'error' : None
    }

    ## @brief For instantaneous value. Only Boursorama parser implemented (plan to extend to Yahoo_HTML)
    _templ_inst_value={
        'value':0,
        'variation':0,
        # format Bourso store string: "2015-02-01 12:10:45"
        # to change to one datetime like others 
        'time':0,
        'date':0,
        'volume':0,
        'ouverture':0,
        'plushaut':0,
        'plusbas':0,
        # if not parsed, must be available, DEFAULT/initiliazed, ERROR, CLOSED, OPEN
        'state' : 'CLOSED',
        # maybe error to add, nice for checking
        # needed to add to avoid bug with ServerPortfolio, to check where exactly !
        #'source' : None
    }

    ## Dividend/Split same as HP, need dates certainly
    # maybe easier to split div and split ?
    _templ_div = {
        'first_date' : datetime.datetime.fromtimestamp(0),
        'last_date' : datetime.datetime.fromtimestamp(0),
        # maybe to keep all in one list ? easier to recompute adjusted close ? [ ['dividend',date, value],['dividend',date,value],[ SPLIT, date, value(1:4)] ]
        'list_div' : None,
        'list_split' : None
        #'error' : None
    }

# can be merged with templ_hp, but still 2 actions...
    ## DatesIntro, first CSV date available (in CSV), integer format (extends to string? last date?)
    _templ_intro = {
        'intro' : int(0),
        'intro_str' : str()
    }

    ## @brief Template for one XML value, stored in a list (of _templ_spec_xml)
    # Option specify the default value in case of a new file
    # date : str(), force each entry to provide a value. None can use default. Could be assumed date is ALWAYS required (I think it is now)  
    _templ_one_value_xml = {
        'value' : None,
        'attrib' : { 'append' : False, 'source' : None, 'date' : None } #date should be always present
    }

# really necessary ? could fill only _templ_dict['Static']['template'] (not use of get_template to avoid to have a copy)
# like the other, just to be sure to maintain the link between the dictionary after modification
# if reloaded, need clean and rewrite
# if copied, not problem with a link to maintain,... but problem with update.

    ## @brief Static data, keys are initialised only when reading the configuration file in DictionaryStocks::init.
    # Must care about maintaining the link, and to not create a different dictionary 
    _templ_static = {}

# if delete _templ_static, what to do of invalid version ?
    ## @brief Static data for InvalidStock, all fields are set to 'invalid'.
    _templ_static_invalid = {}

# Choice to be created in specific function _check_new_X, or use template as well ?
# It does not really extend dict_data, maybe it should ? It is defined as a separate Stock._dict_interactive at the moment
# If bound to Stock.dict_data , dict_data only with parser. valid_xml knows about XML input, deal with xml output

# dict_data -> dict_data_parser 
# dict_interactive :only for validation -> dict_validation better ? certainly. If only intermediate can split: Stock.Validation.dict_validation ?
# _templ_valid_xml (dict_xml) -> dict_data_saved, dict_data_file, dict_xml restrictive, but not needed when csv. Stock.Serialize.dict_xml ?

# for xml dict_interactive/ dict_xml are very close
# for csv dict_interactive/ no equivalent, (never read and) written by c++ from dict_data (through dict_interactive)
# for interactive need new_value, previous_value
# not limited to xml, to change to _templ_valid_value
# used only in ValidXML

    # introduced for dist_interactive, maybe not needed in fact, all in dict_xml..?
    # maybe to extend other class
    # describe only one data for list_interactive, not sure how to assign the key here(without explicitly copy all...)
    _templ_valid_xml_value = {
        'new_value' : None,
        'xml_value' : None,
        'source' : None,
        # important to save ! can store the old date ? previous date if interactive ?
        'date' : datetime.datetime.fromtimestamp(0), #if timestamp all float/int
        # to replace/add/append in dictionary, done partly automatic - partly by user input
        # keep status after check. 2:accept, 1:accept or user validation, 0: reject or user validation, -1: reject/discard directly(None,empty...)
        'repl_add'  : int(0)
    }

    # similar
    # _templ_valid_csv = 

    ## @brief Skeleton for the dictionaries of each action.
    # Store:
    #  -a specific "template" for the parser
    #  - a config entry for the management of the sauvegarde (save : 'xml'/'csv')
    #  - optional xml attributes for the update of data in case of 'save: : 'xml'  
    #
    # Set default values, save_xml, append, date, ...
    # could add:
    #   - date of the last modification (avoid to rerun web queries)
    _templ_action = {
        # not clear it is needed here, it is independent of the other actions. 
        # It is included by default in Stock._dict_data . Could keep in DictStocks(easier for update). 
        # Make apart in Stock._dict_static in creation (need its own get_ or combine for parser query)
        'Static' : { 'template' : _templ_static,
                     # Static, not saved the same way, set to False at the moment 
                     #'config' : { 'save_xml' : False } # 'save' : 'xml'/'file'/'both' or ['xml','file'],['xml']
                     'config' : { 'save' : False } # can add option, can add config option in dictstocks as well
                     #          option for csv as well, for dcsv DAY : 365 days, Inst : 15 days...
                     # error would be fine here, updated with add_dict_parser, error code from parser optional
                     #'error' : None
                   },

        # default value, source == str() : force each one_value to provide its source (Info, 2 sources)
        # append False, for parent action default, for one_value optional, only if different
        'Info' : { 'template' : _templ_info,
                   #'config' : {'save' : ['xml']  }, #can add option, can add in dictstoc
                   # make test, all available from stock.get_config
                   'config' : {'save' : ['xml'], 'attrib': {'append' : False, 'source' : None } }
                   # 
                   #'attrib'  : {'append' : False, 'source' : None }, # 'attrib_xml', 'attrib_sql'
                 },

        # append equivalent to date ! to differentiate
        # default append True, source None (not str()), may use the default values in action attributes
        'Fundamental' : { 'template' : _templ_fundamental,
                          #'source' : None,
                          # only internal use
                          'config' : { 'save' : ['xml'], 'attrib':{'append' : True, 'source' : None} }, #'append' : True
                          # default attributes for all values
                          #'attrib'  : {'append' : True, 'source' : None}
                          # source not accounted
                        },

        'HistPrice' : { 'template' : _templ_hp,
                        'config' : {'save' : ['csv']}
                      },

        'InstValue' : { 'template' : _templ_inst_value,
                        'config' : {'save' : ['csv']} # gui_c++ , to postpone the writing to file ?
                        #'config' : {'save_xml' : False }
                      },
        # belongs to both xml/csv, C++ needs fast access for one specific action only(adjust close) not saved at the moment.
        # convenient if stored in both format.           
        'DivSplit' :  { 'template' : _templ_div,
                        'config' : { 'save' : ['xml','csv'] } # both or a list ['xml','csv','sql'] easier to extend, 'gui_c++' ?
                        #'attrib' : {'append' : True, 'date' : True }
                      },
        # used to retrieve all previous data in yahoo, not (yet) saved. need to save ? or just used transiently by gui/or update python
        'DatesIntro' : { 'template' : _templ_intro,
                         'config' : {'save' : False}
                       }

# for sure unrelated 
    #  ###### to move, unrelated to Action to save, Validation class, still can use templates, more specific.
    #
    # only used by dict_interactive, which is now used if interactive or save is present, default save=True is present
    #  each action could import ValidXML/ValidCSV template
        ## unrelated to EAction, to valid new data in XML
#         'ValidXML' : { 'template' : _templ_valid_xml_value, #no really need template
#                        'config' : { 'save' : False } #query dict_data if 'xml' _templ_valid_xml, 'csv' _templ_valid_csv 
#                      }
    }

    ## @brief Define a generic template to be included by all parsers, for each stock.
    # symbol allows the parser to associate the results to each stock (case of multiple stocks query)\n
    # action_templ stores templates of the actions performed\n
    # @todo errors may be common or included by each, only for transient storage, destroyed after parsing is done 
    # see get_template_parser()
    _templ_parser = {
        ## @brief If parsing multiple stocks, associate the symbol
        'symbol' : str(),
        ## @brief One of action template: _templ_inst_value, _templ_hp.
        'action_templ' : {}

        # error could be common as well or used to report all errors
        # indicates the action, e.g. 'HistPrice' or a list
        # here not the best place, used only in dict_return_data, 
        # possible check in UpdateStock before clean parser, in add_dict_data is best

        #'global_error' : None , not global, one templ_dict_data for each Stock symbol 
        # global_error can only be UpdateStock, local in Stock (stock_error) can check in add_dict_data

        # source ?? best place, this template only used by parser, need variable to store in dict_data also
        # done actually, send with add_dict_parser
        #'source' : str()
    }

    #_templ_valid_xml = {
                 #'template' : _templ_valid_xml_value                   
    #}

    ## @brief Reset the state of templates to the original.
    # @warning Should not be used in the application, only for unit_test at this point, but may need later for reload.
    # to add dynamically in unit-test ?
    @classmethod
    def _reset(cls):
        cls._logger.debug("_reset static template")
        #print "id(self._templ_static)", id(cls._templ_static)
        #print "id(self._templ_dict['Static']['template'] ", id(cls._templ_dict['Static']['template'])
        assert id(cls._templ_static) == id(cls._templ_action['Static']['template']), \
               "Id templ_static differ before clear"
        # maintain links
        cls._templ_static.clear()
        cls._templ_static_invalid.clear()
        #print "id(self._templ_static)", id(cls._templ_static)
        #print "id(self._templ_dict['Static']['template'] ", id(cls._templ_dict['Static']['template'])
        assert id(cls._templ_static) == id(cls._templ_action['Static']['template']), \
               "Id templ_static differ after clear"

# get_template_stock: with access to config, to save or not, which format, pre/post-process available
# no need of 'config': 'attrib'

    ## @brief Return a copy of an action template for stocks, include 'template' and 'config', delete 'attrib' entry
    # @param action EAction or EAction.name
    @classmethod
    def get_template_stock( cls, action, for_xml=False ):
        cls._logger.debug("Entry get_template, action: %s" % action)
        # correctly handle exceptions in Utils
        e_action = Utils.stringToEAction( action )

        # check argument, needed ? not anymore ! could provide option with the template
        if e_action.name in cls._templ_action:
            tmp_dict = copy.deepcopy( cls._templ_action[e_action.name] )
            # 'attrib' entry not needed for stock, but needed for xml
            # new added
            if not for_xml:
                if 'attrib' in tmp_dict['config']:
                    del tmp_dict['config']['attrib']
            # else send back everything if called by get_template_xml
            return tmp_dict
            # original version
            # return copy.deepcopy( cls._templ_action[e_action.name] )

        # error if wrong action
        raise PortfolioError("e_action %s has no template in StockTemplates")

    ## @brief Return the template for parser to be included by all parsers.
    # Include the symbol of the stock (used for multiple stocks query),\n
    # and the template specific an action.\n
    # In multiple stock query, these template are stored in a list (AbstractParser.list_return_data)\n
    # @note Must be included before any other templates (usually in the parse() function of the parser)
    @classmethod
    def get_template_parser( cls, action = None ):
        # Possible to test if already existing, for not overwriting
        # template parser includes 'symbol'
        tmp_dict = copy.deepcopy( cls._templ_parser)

        # a priori, always called with one action
        assert action is not None,"get_template_parser called with action None"
        if action is None:
            return tmp_dict

        e_action = Utils.stringToEAction( action )
        # could almost avoid string and keep enum for the rest of the parsing
        tmp_dict['action_templ'][e_action.name] = copy.deepcopy(cls._templ_action[ e_action.name ]['template'])
        return tmp_dict

# xml need access to attribute (in config) and accept list, not only one value
# new added, call by Stock_fill_dict_xml(read_xml) , rename ext_xml, extended xml ?
#                and create_default_template_xml when creating a new empty XML file

    ## @brief Specific for xml dictionary, replace each value by a list.
    # @return dictionary template action for xml data
    @classmethod
    def get_template_xml(cls, action):
        cls._logger.debug("Entry get_template_xml %s" % action)
        # get 'template' and 'config', config:'save' not needed a priori
        # miss attrib, has been deleted from templ_action, added again
        # new added option for_xml, keep originam (with attrib)
        tmp_xml = cls.get_template_stock( action, for_xml=True )
        # no need of config, only attributes
        #tmp_xml['attrib'] = tmp_xml['config']['attrib']
        del tmp_xml['config']

        cls._logger.debug("template_xml: %s" % tmp_xml)

        # replace values by an empty list on the fly 
        for name in tmp_xml['template'].keys():
            print "name", name
            # always discard other,error
            # if name in StTmpl().discard_entries
            if (name == "other") | (name == "error" ):
                continue
            tmp_xml['template'][name] = list()
        cls._logger.debug("return template_xml %s" % tmp_xml)
        return tmp_xml

    ## @brief Return a template for each xml value, include 'value' and 'atrib' for each one
    # Or/And, with action, name can set default append/date
    @classmethod
    def get_template_one_value_xml(cls):
        return copy.deepcopy(cls._templ_one_value_xml)
    
    ## @brief Retrun a template for the validation template, include 'new_value','xml_value','repl_add','date'
    @classmethod
    def get_template_valid_xml_value(cls):
        return copy.deepcopy(cls._templ_valid_xml_value)

    ## @brief Set the internal template for the static data, also for invalid stock.
    #
    # Set cls._templ_static with keys read from dictstocks.txt\n
    # It should be executed only once when reading the configuration file.\n
    #
    # @post cls._tmpl_dict['Static'] should be link to the same object (to allow to use get_template('Static') safely)
    #
    # @param list_key keys to insert in the template for static data. Set default as empty string.
    @classmethod
    def set_tmpl_static(cls, list_key ):

        if any(cls._templ_static):
            cls._logger.warning("_tmpl_static already loaded, do nothing, should not happen !")
            return

        # here create a new dictionary, 2 different id's, before working ?? not really !!
        #cls._templ_static = {}
        #cls._templ_static_invalid = {}
        # was a bad idea
        #cls._templ_dict['Static']['template'] = {}

        # ok, cls._templ_dict['Static']['template'] updated at the same time
        for item in list_key:
            cls._templ_static[item] = str()
            cls._templ_static_invalid[item] = 'invalid'

        #print "set _tmpl_static ",cls._templ_static 
        #print "cls._tmpl_dict['Static'] ", cls._templ_dict['Static']

        assert id(cls._templ_static) == id(cls._templ_action['Static']['template']), \
               "templ_static not pointing to the same object"

    ## @brief Return the invalid static template.
    # Called at the creation of InvalidStock
    @classmethod
    def get_tmpl_static_invalid(cls):
        return cls._templ_static_invalid

# to delete saved_as replace this function
# now return 'xml'/'csv', to move to Stock, can modify the XML manually
    ## @brief Query to know if this template must be saved in XML file.
    #
    # @return bool
    @classmethod
    def to_save_xml(cls, action):
        cls._logger.debug("Entry to_save_xml")
        cls._logger.debug("to save action %s, %s" % (action, cls._templ_action[ action]) )
        return cls._templ_action[ action ]['config']['save']

    ## @brief Query the format in which the data must be saved.
    # @param action
    # @param specific
    @classmethod
    def saved_as(cls, action, specific=None):
        cls._logger.debug("Entry saved_as")
        
        if specific == None:
            return cls._templ_action[ action ]['config']['save']
        # if wrong entry return False
        else:
            # case of Static 'save': False
            #if specific in cls._templ_action[ action ]['config']['save'] == False:
            if cls._templ_action[ action ]['config']['save'] == False:
                return False
            if specific in cls._templ_action[ action ]['config']['save']:
                return True
            return False
           
        self.logger.error("saved_as Should not arrive here")
        assert 1==1, "Should not arrive here"
        
    ## @brief Create a default xml dictionary, called when creating a new XML file.
    # Assign same templates as stock, with value replaced by a list (one_value_xml)\n
    # Apply only to action with a save option 'xml' in 'config'
    # @post All one_values_xml template have the default attributes of the action
    @classmethod
    def create_default_template_xml(cls):
        cls._logger.debug("Entry create_default_template_xml")
        default_xml = {}
        for action in cls._templ_action.keys():
            print "call to_save_xml action ", action
            #if cls.to_save_xml(action): #to save (action, 'xml'/'csv'/'sql'
            if cls.saved_as(action,'xml'):
                
                # strange logic here, get_template_xml already replace value by list
                # here copy also config, not needed
                default_xml[action] = cls.get_template_xml(action)
                print "get_template_xml ", default_xml
                # in check new xml, access to xml_data['attrib']['source']

                # replace all values in template by a default one_value_xml, replace value by a list
                for key_value in default_xml[action]['template'].keys():
                    # skip entry other and error
                    if (key_value == 'other') | (key_value == 'error'):
                        continue
                    
                    # append 'value' / 'attrib' to each key_value
                    #print default_xml[action]['template'][key_value]
                    default_xml[action]['template'][key_value].append(cls.get_template_one_value_xml())

                    # here best way to override default attr in one_value_xml ? Yes, this way could read data from _templ_action : attrib
                    # attrib has been deleted from stock, add default in one_value_xml
                    #for key_attr_action in default_xml[action]['template'][key_value][0]['attrib']:
                    #    print "key_attr ", key_attr_action, default_xml[action]['attrib'][key_attr_action]
                    #    default_xml[action]['template'][key_value][0]['attrib'][key_attr_action] = default_xml[action]['attrib'][key_attr_action]
                    # seems correct now
                    print key_value, default_xml[action]['template'][key_value][0]['attrib']
                    # default_xml:
                    # {'Info': {'config': {'save': ['xml']}, 'template': {'Sector': [{'attrib': {'date': None, 'source': None, 'append': False}, 'value': None}], 'StockExchange': [], 'Industry': [], 'other': {}, 'Name': []}}}

        cls._logger.debug("default xml %s" % default_xml)
        return default_xml

# here or in Stock, static method, maybe more logical with the template
    def make_header(self, action_name):
        
        if (action_name == 'InstValue'):
            return "#name value variation time date volume ouverture +haut +bas state\n"
        elif (action_name == 'HistPrice'):
            return "#name    first_date    last_date\n"
        elif (action_name == 'Info'):
            return "# symbol   name  sector   industry\n"
        elif (action_name == 'DatesIntro'):
            return "# symbol first available data in Yahoo\n"
        elif (action_name == 'Fundamental'):
            return "# symbol ....\n"
        ## Div Split
        
        else :
            return "# header to write in StokTemplates for action %s\n" % action_name

## @brief Initialise a global variable when the module is loaded.
# if executed in main, logger can be loaded, class can be derived
# or executed in DictStocks, used by Parsers/Stocks...StockXML
StTmpl = StockTemplates()

# ################ For test, certainly to delete
if __name__ == "__main__":
    
    print "Main StockTemplates"
    
    print 'load default'
    d_static = StTmpl.get_dict_static()
    print "d_static ", d_static
    
    d_static = StTmpl.get_dict_static( "CAC40" )
    print "d_static", d_static