Stock.py

Go to the documentation of this file.

## @package serverportfolio.Stock
#
# Define the classes Stock and InvalidStock
#
# Last Changed $Id: Stock.py 29 2015-05-05 11:09:54Z michael $

# generator for interactive sessions ?
# http://www.jeffknupp.com/blog/2013/04/07/improve-your-python-yield-and-generators-explained/
# 

# namedtuple can avoid nested dictionary, from C++ ? for efficiency ?
# http://stackoverflow.com/questions/12119612/nicer-way-to-iterate-to-dictionary-in-python-to-avoid-many-nested-for-loops

import sys, os, logging, types
import time, datetime, math

from lxml import objectify, etree
 
# for eclipse debugging with multithread, example in _add_inst_value
#import pydevd

from serverportfolio.PortfolioException import *
from serverportfolio import Utils
import serverportfolio.GlobalDicts as GlobalDicts
from serverportfolio.GlobalDicts import EAction
from serverportfolio.StockTemplates import StTmpl
# C++ extension, could check here or in ROOT (to put in PYTHONPATH also)
from serverportfolio import wrapPyStock

# cannot import, recursive import
#from serverportfolio.DictionaryStocks import DictionaryStocks
#from serverportfolio.UpdateStocks import UpdateStocks

from serverportfolio.ValidStockUpdate import ValidStockUpdate

# to delete, shoudl be called only from ValidStockUpdate ?
from Validation import ValidationTkinter #, ValidationTkHP


## @class Stock
# @brief Store data and functions related to one stock.
# 
# Stock is a collection of dictionary templates copied from StockTemplates:
#   - _dict_stock : Store data coming from the parsers (same action templates are used). 
# Data are kept in memory(need for Static) except for HistPrice and DivSplit (delete long 'list_X').
# It contains also the 'Static' part (to make apart: Stock._dict_static (with get_static()), but like ?)
#
#   - _dict_xml : used for reading/writing xml, extends _dict_stock (need (a list of) previous and new data to be inserted after validation.
# Data to keep or delete ? To load/keep after parsing for further queries. 
#
#   - _dict_interactive : intermediate dictionary to validate an update of the files, semi-automated or validated by user.
# 2 versions/templates needed: xml/csv. Validation and merging are different. 2 sets of functions also
# Only intermediate for the validation stage, could be in a Validation class. Actually filled, no template existing. 
# 
#
# Provides functions to read/save data:
#   - HistoricalPrice / InstValue/ DivSplit are saved in CSV format with the use of C++ wrapper functions from wrapPyStock.so
#   - Info / Fundamental / (DivSplit) are read and saved in XML format, only use python code.
#
# Functions for querying and updating data from the web are implemented in UpdateStocks 
#
# Stock could have derived class (asset, FX, indice...), only done InvalidStock
class Stock(object):

    ## @brief Constructor.
    # Creates the stock and save the 'Static' data read in the configuration/definition file "dictstocks.txt".\n
    # Other (sub-)dictionaries will be created on demand: get_dict_parser(), get_dict_xml()\n
    #
    # If needed, one Stock can be constructed this way:\n
    # @code stock = Stock( "CAC40", DictionaryStocks().get_dict_static("CAC40") ) @endcode
    # But the use of DictionaryStocks is more convenient to store multiple Stock objects. 
    #
    # @param symbol of the stock as defined in dictstocks.txt
    # @param dict_static 'Static' template containing the data read from dictstocks.txt
    def __init__(self, symbol, dict_static = None):
        self._logger = logging.getLogger('SP.stock')
        self._logger.debug("Init Stock %s" % symbol)

        ## @brief Main key for all dictionaries
        self._symbol = symbol

        ## @brief Set of templates to store results from parsers, include 'Static' data
        self._dict_stock=None
        self._dict_stock={}
        # dict_static has the correct format, not new_data from parser
        if dict_static:
            self.set_action('Static', None, dict_static)
            #self._dict_stock['Static'] = dict_static
        ## @brief Dictionary for storing data for XML input/output, extends _dict_parser templates.
        # Initialised in read_xml()
        self._dict_xml = None

# almost all done in ValidStockUpdate, just problem with save_hist_price. Should use self.dict_stock, updated from ValidStockUpdate 
        ## @brief Store old/new values before update which may be applied to XML.\n
        # In case of interactive mode, it can be validated by the user
        # could be in separate class Validation() only intermediate data
        self._dict_interactive = None
        ## @brief Validation is done by a ValidStockUpdate object.
        self.valid_update = None

# ### Static method

    ## @brief Similar to Utils.to_list but deals with Stock or list of Stocks
    # @param one stock symbol (String) or Stock object, or a list of them
    # @return Always a list of stock symbols or stock objects
    @staticmethod
    def to_list(stock):
        # already a list is input
        if type(stock) == types.ListType :
            return stock

        # if one element, make a list, extended to InvalidStock
        if ( type(stock) == types.StringType) | ( isinstance(stock,Stock)) | ( isinstance(stock,InvalidStock)) :
            #print "it is a string"
            list_stock = []
            list_stock.append( stock )
            return list_stock

        # error, not tested ?
        return None

    ## @brief Return if the Stock is valid.
    # InvalidStock implements the same function, but return False
    def is_valid(self):
        return True

    ## @name Getter/Setter for internal dictionaries, provide initialisation on request
    ## @{

    ## @brief Return the action template part from _dict_stock or one of its data value.
    # If action is None, return the root dictionary _dict_stock\n
    # If key is None return _dict_stock[action]['template']\n
    # If _dict_stock[ action ] is not existing, create an 'action' entry with the default template and return a link\n
    # If the key is not exiting, it is created (the action as well if needed), set it up to None and return it\n
    #
    #
    # The purpose of this design is to provide lazy initialiation of the action / key, but some care must be taken when calling the function,\n
    # get functions can modify the internal dictionary\n
    #
    # Example of usage:\n
    # @code
    # from DictionaryStocks import DictionaryStocks
    # from Stock import Stock
    #
    # # Creates a Stock with 'Static' data
    # stock = Stock('CAC40', DictionaryStocks().get_dict_static('CAC40'))
    #
    # # Extract the 'Static' template and one specific value, do not modify the dictionary 
    # templ_static = stock.get_action('Static')
    # print templ_static # >>> {'code_bourso': '1rPCAC', 'code_yahoo': '^FCHI', 'market': 'europe', 'format_bourso': 'Pts'}
    # code_yah = stock.get_action('Static','code_yahoo')
    # print code_yah # >>> ^FCHI
    #
    # # Test a non-existing action, without modifying the internal dictionary
    # if 'InstValue' in stock.get_action():
    #    print "InstValue is present"
    # >>>
    #
    # # Only Static is present
    # # Equivalent to stock._dict_stock
    # stock.get_action() # >>> {'Static': {'config': {'save': False}, 'template': {'code_bourso': '1rPCAC', 'code_yahoo': '^FCHI',...}}}
    #
    # # Test a non-existing entry, action (and key) are created with default values the first time (default sate 'CLOSED')
    # state = stock.get_action('InstValue','state')
    # state # >>> CLOSED
    # # InstValue action has been loaded
    # print stock.get_action() # >>> {'InstValue': {'config': {'save': ['csv']}, 'template': {'variation': 0, ..}}, 'Static': {...}}
    #
    # # To not modify the dictionary, one needs 2 explicit loops
    # if 'HistPrice' in stock.get_action():
    #     if 'list_csv' in stock.get_action('HistPrice'): # not executed
    #         list_csv='list_value'
    # stock.get_action() # >>> No 'HistPrice' entry 
    # @endcode
    #
    # @note if the function return a link to a dictionary, if the link is modified the original dictionary is modified also.\n
    # If the function return a value, a copy is returned.\n
    # see unit_test.test_stock for more case
    # 
    # @param action optional string corresponding to a GlobalDicts.EAction name
    # @param key optional string to get the value of _templ_action['template'][key]
    def get_action(self, action = None, key = None):
        # create all on demand, can skip always dictionary
        #if self._dict_parser == None:
        #    self._dict_parser = {}

        # return the full dictionary
        if action == None:
            return self._dict_stock

        if action in self._dict_stock:

            if key is None:
                # to create or set to None by default ?
                return self._dict_stock[action]['template']

            elif key in self._dict_stock[action]['template']:
                return self._dict_stock[action]['template'][key]

            # create a new key, needed for 'source', other, or to declare in all templates
            else:
                self._dict_stock[action]['template'][key] = None
                return self._dict_stock[action]['template'][key]

        # create an action entry with default template and config
        # should be called only once during the lifetime of the object
        e_action = Utils.stringToEAction( action )
        self._dict_stock[ e_action.name ] = StTmpl.get_template_stock( e_action.name  )

        # recursive call, seems working... need more tests
        return self.get_action( action, key )

    ## @brief Fill _dict_stock['action'] with new_data.
    # If already existing overwrite the data, otherwise create the necessary action
    # @pre action != None, key can be None\n
    # new_data must follow the format of a template _templ_X 
    # @post _dict_parser[ action ] = new_data
    # @param action name
    # @param key name of the data
    # @param new_data to insert into _dict_stock[action]['template'] or ['template']['key']
    # @throw PortfolioError if wrong action
    def set_action(self, action, key, new_data):
        self._logger.debug("set_action, new_data %s" % new_data)
        assert action != None, 'set_action, action cannot be None'

        # create on demand with empty dictionary
        self.get_action( action, key )
        #d = self.get_action( action, key )

        # case set_action( action, None, StTmpl.get_template_stock() )
        # overwrite everything, template and config
        # crash with string or other, two solutions
        # 1. hasattr([1,2,3,4], '__iter__')
        # 2. duck typing
        try:
            if 'template' in new_data:
                self._dict_stock[action] = new_data
                return self._dict_stock[action]['template']
        except: #TypeError
            self._logger.debug("object not iterable")
            pass

        # sure it exists, if not empty, overwrite
        if key is None:
            self._dict_stock[ action ]['template'] = new_data
            #d = new_data
        else:
            self._dict_stock[ action ]['template'][key] = new_data
            #d = new_data
        return self._dict_stock[action]['template']

    ## @brief return the 'config' dictionary or one of its key from _dict_stock
    #        
    def get_config( self, action, key=None):
        self._logger.debug("get_config action %s, key %s" % (action, key))
        # Do not check if it exists
        if action in self._dict_stock:
            if key is None:
                return self._dict_stock[action]['config']
            elif key in self._dict_stock[action]['config']:
                return self._dict_stock[action]['config'][key]
            # else creates a new key ? maybe later
        # if action does not exist
        return None

    ## @brief query the entry 'config' : 'save' of an action
    # If specifc is indicated return a boolean value
    # @code
    # if stock.saved_as('Info','xml'):
    #    update_xml
    # @endcode
    # If specific is None return the list or False
    # @code
    # if 'xml' in stock.saved_as('Info'):
    #    update_xml(..)
    # @endcode
    #
    # Check first for the entry in _dict_stock, if not available query the default template through StTmpl.saved_as  
    # @param action
    # @param specific optional entry, valid values are 'csv' and 'xml' 
    # @return the list of format to save or a boolean
    def saved_as(self, action, specific=None):
        # query the local setup of the stock
        dict_config = self.get_config( action )
        # None if not existing
        if dict_config:
            if specific == None:
                return dict_config['save']
            else:
                # case of Static 'save': False
                if dict_config['save'] == False:
                    return False
                # 'save' : [list]
                if specific in dict_config['save']:
                    return True
                return False
        # else query the default template
        else:
            return StTmpl.saved_as(action, specific)
        # only for tests
        self._logger.error("saved_as Should not arrive here")
        assert True, "Should not arrive here"

    ## @brief Return the dictionary of data for XML input/output.
    # Similar to get_action but templates are extended with sub-dictionary\n
    # Always return a valid dictionary, maybe empty
    # @param action name or EAction
    # @throw PortfolioError if invalid action
    def get_dict_xml(self, action = None):
        self._logger.debug("Entry get_dict_xml")
        # create on demand
        if self._dict_xml == None:
            self._dict_xml = {}
        if action == None:
            return self._dict_xml
        if action in self._dict_xml:
            return self._dict_xml[action]
        # need to create an entry
        # check it is valid action
        e_action = Utils.stringToEAction( action )
        self._dict_xml[ e_action.name ] = {}
        return self._dict_xml[ e_action.name ]

    ## @brief Similar to set_action with extended dictionary.
    # \pre action cannot be None
    # first implementation, action is necessary
    def set_dict_xml(self, action, new_dict_xml ):
        assert action != None, 'set_dict_xml, action cannot be None'
        self.get_dict_xml( action )
        self._dict_xml[ action ] = new_dict_xml

    ## @brief Return a formatted error.
    # To generalise to action as well ? 
    # Only convenient way to get them from C++ ?
    def get_error(self, action = None):
        self._logger.debug("get_error, action %s", action)
        #print self._dict_parser

        if action is not None:
            return self.get_action( action )['error']

        # global error, not anymore, only in dict_return_data
        if self.get_action()['global_error']:
            self._logger.info("Error in action: %s" %  self.get_action()['global_error'])
            error_action = self.get_action()['global_error']
            error = self.get_action( error_action )['error']
            print "Error ", error

            # transform to string ? more complex structure ? 
            return error

        #else:
        # default no error
        return None
    ## @}
        
    ## @name Functions/properties to access some common data from _dict_parser (Static and InstValue)
    ## @{
    
    ## @brief Getter for the symbol
    @property
    def symbol(self):
        return self._symbol
    
    ## @brief Setter raises an exception, it cannot be changed 
    @symbol.setter
    def symbol(self, value):
        raise Exception("Cannot change the value of a stock symbol !! stock: %s" % self._symbol)
 
    ## @brief Getter function for the state. Possible state OPEN/CLOSED/WAIT_OPEN, default CLOSED.
    def get_state(self):
        #print "getter state id ", id(self), " ",self._dict_parser['InstValue']['state'] 
        return self.get_action('InstValue','state')

    def set_state(self, state):
        #print "setter sate id ", id(self), " ",state
        self.set_action('InstValue','state',state)
        
    # ok working with python 2.7
    # not sure how to set the property here with @state.setter. only available with python 3...?
    
    ## @brief Define both setter and getter, can use stock.state = new_state or print stock.state
    state = property(get_state, set_state)
    
    ## @brief Get the market on which the stock belongs. e.g. US, FR...
    def get_market( self ):
        return self.get_action('Static','market')
    
    ## @brief Return the last modification of the Instantaneous value
    def get_last_modification( self ):
        return self.get_action('InstValue','last_modification')
    ## @}
    
#     @property
#    def get_error( self ):
#        return self.dictstocks[stock]['error']
    
#    def set_error(self, error ):
#        self.dict_parser[stock]['error'] = error
    
    ## @brief Function to update Stock with the data extracted by the Parsers.
    # Update _dict_stock with the new_data from Parser._list_return_data\n
    # @param dict_new_data dictionary ('template' part only) containing the data to insert, similar template format that Stock._dict_stock
    # @param source name of the parser used to retrieved the data, inserted in _dict_stock
    def add_dict_stock(self, dict_new_data, source):
        self._logger.debug("add_dict_stock() dict_new_data: %s" % dict_new_data )
        assert ( dict_new_data is not None ),"dict_new_data is not defined in Stock.add_dict_stock"

        # loop over all actions in dict_new_data (only one action tested)
        for key,value in dict_new_data.iteritems():

            # for use with serverportfolio, need to update the state OPEN/WAIT_OPEN/CLOSED
            if key == "InstValue":
                self._add_instvalue_data( value )

            # need to update correctly the date
            elif key == "HistPrice":
                self._add_histprice_data( value )

            # all other, just copy the new data, want to copy others as well
            else:
                self._logger.debug("default in add_dict_parser, copy all values, without check")
                self.set_action(key, None, value)

            # assign the source, global to each action
            self.set_action(key,'source', source)

        self._logger.debug("End add_dict_stock, self.get_action: %s" % (self.get_action()))

    ## @brief Update InstValue data. 
    # With server running, the state (OPEN/CLOSED) must be updated correctly 
    # @param dict_new_data, templ_action
    def _add_instvalue_data( self, template_action):
        self._logger.debug("_add_instvalue_data")
        self._logger.debug("template_action: %s", template_action)
        #self._logger.debug("previous data: %s", self.get_action('InstValue'))

        # eclipse debugging with threads
        #pydevd.settrace( 5678 )

        # new, either default CLOSED or previous state
        prev_state = self.get_action('InstValue','state')
        # to compare new and previous state
        new_state = template_action['state']

        if new_state == 'CLOSED' :
            if prev_state == 'WAIT_OPEN':
                self._logger.info("CLOSED and WAIT_OPEN pass, prev_state %s", self.state)
                pass
            else:
                if (prev_state == 'OPEN'):
                    self._logger.info("new_state CLOSED, OPEN->CLOSED %s, copy the state" % (self.symbol))
                prev_state = new_state

        # ML WAIT_CLOSED pass if still open (but same value not recorded to file ?
        # WAIT_OPEN->OPEN
        elif new_state == 'OPEN':
            # all case, copy the new_state
            # self._logger.debug("OPEN copy the state")
            if prev_state == 'WAIT_OPEN':
                self._logger.info("WAIT_OPEN->OPEN %s" % self.symbol)
            prev_state = new_state

        # update all other values, config was incorrect here, empty{}
        self.set_action('InstValue', None, template_action)
        # case wait open bad name, final state better
        # maybe copy data before the OPEN/CLOSED tests and overwritte in case of WAIT_OPEN
        self.state = prev_state
        self._logger.debug("updated inst_value: %s", self.get_action('InstValue') )

    ## @brief Update Historical Price data. 
    # The field date (last date) must be updated correctly 
    def _add_histprice_data( self, template_action ):
        self._logger.debug("_add_instvalue_data")
        #self._logger.debug("previous data: %s", self.get_dict_parser('HistPrice'))
        # load default template if needed
        hp = self.get_action('HistPrice')
        # first_date not changed, but no set up correctly in last_CSV (0)
        # should correspond to the first loaded date in file now
        #hp['first_date'] = dict_new_data['HistPrice']['first_date']
        hp['last_date'] = template_action['last_date']
        hp['list_csv'] = template_action['list_csv']
        self._logger.debug('hp %s' % hp)

    ## @brief Pre-processing only for HistPrice, may extend to all 'csv' later.
    # The function is called by AbstractParser.store_stock_copy
    # @param e_action EAction to process
    def pre_process(self, e_action):
        self._logger.debug("pre_process e_action.name: %s" % ( e_action.name))
        # HistPrice, retrieve the date of the last update from the file
        if ( e_action == EAction.HistPrice ):
            self.last_CSV( e_action.name )

    ## @brief Post process the update: check, validate and save the data after one (unique) action is performed.
    # Delegate this functionality to a ValidStockUpdate object
    # @param e_action EAction executed by a parser
    # @param option_post post-process option coming from UpdateStocks/RunParser
    def post_process(self, e_action, option_post):
        self._logger.debug("Entry Stock.post_process e_action / option_post: %s / %s " % (e_action, option_post))
        # extra option parameters 
        interactive = option_post['interactive']
        to_save = option_post['to_save']
        # create object, with the tk_manager reference, None if no interactive 
        if self.valid_update is None:
            self.valid_update = ValidStockUpdate(self,option_post['tk_manager'])
        try:
            self.valid_update.post_process_one_action(e_action, option_post)
        except PortfolioError as ex:
            self._logger.debug("Caught PortfolioError %s" % ex.get_format_string())
            raise
        except Exception as ex:
            self._logger.debug("Caught Exception %s" % ex)
            raise
        # with GSZ HistPrice, example of output without error, to use in test
        # new_data / _dict_parser {'source': 'YCSV', 'last_date': datetime.datetime(2015, 3, 2, 0, 0), 
        # 'first_date': datetime.datetime(1970, 1, 1, 1, 0), 
        # 'list_csv': [[1425250800, 19.8, 19.86, 19.37, 19.44, 6279800.0], [1424991600, 19.46, 19.88, 19.35, 19.88, 8207000.0], [1424905200, 19.51, 19.74, 19.39, 19.48, 6426300.0], [1424818800, 19.25, 19.45, 19.17, 19.29, 5575500.0], [1424732400, 18.86, 19.35, 18.62, 19.24, 8116600.0], [1424646000, 18.8, 18.84, 18.59, 18.81, 5173700.0], [1424386800, 18.66, 18.74, 18.44, 18.59, 6623200.0], [1424300400, 18.53, 18.69, 18.43, 18.68, 5664000.0], [1424214000, 18.91, 18.98, 18.56, 18.68, 4637300.0], [1424127600, 18.51, 18.76, 18.27, 18.74, 4519600.0], [1424041200, 18.71, 18.72, 18.54, 18.6, 3649000.0], [1423782000, 18.89, 18.92, 18.72, 18.72, 6893900.0], [1423695600, 18.75, 18.86, 18.5, 18.8, 5891100.0], [1423609200, 18.99, 19.0, 18.67, 18.77, 5178200.0], [1423522800, 18.75, 18.99, 18.5, 18.96, 4967600.0], [1423436400, 18.9, 18.94, 18.53, 18.68, 6571700.0], [1423177200, 19.19, 19.21, 19.0, 19.01, 4635800.0], [1423090800, 19.33, 19.4, 19.1, 19.15, 8390900.0], [1423004400, 19.63, 19.73, 19.46, 19.6, 5781600.0], [1422918000, 19.9, 20.05, 19.55, 19.7, 9223500.0], [1422831600, 19.72, 19.88, 19.59, 19.83, 4641200.0]], 
        #  'error': None}

    ## @name Use C++ wrapPyStock.so library to read/write CSV data to file  
    ## @{
    ## All functions check for the global variable GlobalDicts.TEST_MODE to write output in tmp directory 

    ## @brief Save instantaneous values into a file.
    # Use C++ wrapPyStock.so module\n
    # C++ insures a correct format, correct header, correct directory (including TEST_MODE for debugging and unit-test)
    #
    # @todo Be sure only new data are recorded, not done in C++ ? Only check date and values, ok FSLR, not for DJ.
    def save_inst_value( self ):
        self._logger.debug("Entry save_inst_value call C++ wrapper stock: %s TEST_MODE: %s " % \
                          (self.symbol, GlobalDicts.TEST_MODE))
        #print self.print_stock("InstValue")
        # need to transform in timestamp, equivalent int ? Need util function, check timestamp with python
        full_datetime = self.get_action('InstValue','date')+" "+ self.get_action('InstValue','time')
        self._logger.debug("full_datetime %s" % full_datetime)
        # strptime time return a struct_time
        time_struct = time.strptime( full_datetime, "%Y-%m-%d %H:%M:%S")
        #print "time_struct ", time_struct
        # transform timestamp, epoch for C++
        date_tmp = int( round(time.mktime(time_struct)))
        #print 'date_tmp', date_tmp
        try:
            #print self.get_action('InstValue','value')
            #print self.get_action('InstValue','volume')
            retour = wrapPyStock.SaveInst( self.symbol,[ date_tmp, self.get_action('InstValue','value'), \
                                                         self.get_action('InstValue','volume') ], GlobalDicts.TEST_MODE )
            self._logger.debug("after wrapper C++ retour %d", retour)
        # Make    
        except Exception as ex:
            self.loogger.debug("Catch Exception in Python: %s", ex)
            # raise rethrow the same exception, otherwise will just continue
            raise
        # send status, no data append. Header dates certianly do not fit
        if retour != 0 :
            #self._logger.debug("SaveInst did not append data !!")
            self._logger.info("SaveInst did not append data !!")
        return retour

# Need to extend with DivSplit, try with an action argument
# Saving to file not writen yet, simple file or XML (makes sense to save with other "fundamental" data: SQL also)

    ## @brief Get the date of the last data from the file saved on disk.
    # Use C++ code with wrapStock.so, including TEST_MODE variable for unit-test\n
    # Tested for daily data only.\n
    #
    # file searched in /home/michael/workspace_kepler/ROOT_application/data_test/historical/CAC40.DAY.csv (with TEST_MODE)
    # Store date as datetime in tmeplate HistPrice
    def last_CSV( self, action ):
        
        self._logger.debug("last_CSV symbol %s " % self.symbol )
        self._logger.debug("action %s" % action)
        
        assert( (action == 'HistPrice') | (action == "DivSplit") ),\
            "last_CSV can be called only with HistPrice or DivSplit"
        
        # need to add transform from action name to EAction
        e_action = Utils.stringToEAction( action )
        # (re-) initialise
        #self.set_action( action, None, StTmpl.get_template_stock( e_action))
        self.get_action( action )    
        self._logger.debug("dict_stock action %s: %s" % (action, self.get_action(action)))
        
        # to check if already exists, and up to day
        if action == 'HistPrice':
            # To get both, first and last dates !
            last_date_in_file = wrapPyStock.LastCSVUpdate( self.symbol, GlobalDicts.TEST_MODE )
            # C++ return always timestamp date
            self._logger.debug("last_CSV C++ return last_date_in_file %d" % last_date_in_file)
        
            # can load in the template, timestamp number of second
            self.set_action('HistPrice','last_date', datetime.datetime.fromtimestamp( last_date_in_file ))
            #self.get_dict_parser('HistPrice')['first_date'] = datetime.datetime(0,0,0)
            self._logger.debug("last date in dict_parser %s" % self.get_action('HistPrice','last_date'))
            
            # for unit-test, return timestamp, to delete if no use later
            return last_date_in_file
            
        elif action == 'DivSplit':
            print "Action DivSplit, get first and last date available TO IMPLEMENT"
            # default correct
            #self._dict_parser[ action]['last_date'] = datetime.datetime(0,0,0)
            #self._dict_parser[ action]['first_date'] = datetime.datetime(0,0,0)
            
# Error with gen_part_of_list_csv, not declared here !!    
    ## @brief Save historical price in CSV format in ROOT_data/hp/symbol.DAY.CSV
    # Use C++ wrapPyStock.UpdateCSV function. Function improved flexibility can reread/append, deal with partial overlaps.
    def save_hist_price(self): #interactive ?
        self._logger.debug("Entry save_hist_price")
        #self._logger.debug("TEST_MODE %s" % GlobalDicts.TEST_MODE )
        # check error, here ? before ? otherwise created by default
        templ_hp = self.get_action('HistPrice')
        list_csv_memory = self.get_action('HistPrice','list_csv')
        # a copy of the validated data is created and saved into the file
        # bug test_stock, list_csv_copy not filled if not interactive
        list_csv_copy = []
        if (self._dict_interactive is not None) and ( 'HistPrice' in self._dict_interactive):
            # shortcut
            d_inter = self._dict_interactive['HistPrice']
            # what to send first ? no matter do not enter the function, just assign the generator
            # work with explicit None
            sub_list_generator = self.gen_part_of_list_csv(None) #0
            # here enter
            tmp = sub_list_generator.send(None)
            print "tmp ", tmp
            # easier to sort in inverse for the last key to the first
            for group in sorted( d_inter.keys(), reverse=True):
                print "group ", group, d_inter[group]
                value = d_inter[group]
                # accepted update, butt only the first/last date are available in dict_directive
                # could extend with the position in the list list_csv[x:y] easier..., need to be sure not modified
                if value['repl_add'] == 2:
                    # here it is inverted !!!
                    first_date = Utils.string_to_timestamp( value['first_date'] )
                    last_date = Utils.string_to_timestamp( value['last_date'] )
                    print "first date/last date ", first_date, last_date
                    print "first date/last date ", Utils.timestamp_to_string(first_date), Utils.timestamp_to_string(last_date)
                    # return a list from the previous call to last_date (included)
                    #list_csv_copy.append( sub_list_generator.send( first_date ) )
                    sub_list = sub_list_generator.send( first_date )
                    print "sub_list ", sub_list
                    list_csv_copy += sub_list
                    #list_csv_copy.append() 
                elif value['repl_add'] == 0:
                    print "This group was not validated, stop the copy"
                    break
        # bug test_stock, list_csv_copy not filled if not interactive       
        else:
            list_csv_copy = list_csv_memory
        # NO bug should accept same initial data (overlap 1), Order must be inverted ! No check !
        # C++ output, find overlap but Data differ ?? they are the same, to check
        if any(list_csv_copy):
            retour = wrapPyStock.UpdateCSV( self.symbol, sorted(list_csv_copy, reverse=True), GlobalDicts.TEST_MODE )

    ## @}

    ## @brief Compute in how long time the stock will open, to test for setting the state 'WAIT_OPEN' in AutoParsers.
    # @return datetime a delta time
    def time_to_open( self ):
        #if isinstance(self[stock], InvalidStock):
        #    return None
        market = self.get_market()
        # Opening time stored in a static dictionary
        time_str = GlobalDicts.OpeningMarket[market]
        #print "opening", time_str[0]
        # print "closing", time_str[1] #not implemented ??
        
        # extract the time of the opening from string
        time_open=time.strptime(time_str[0],"%Hh%Mmn")
        #print "time_open ", time_open

        # compute difference of the opening time with the now() time
        time_now = datetime.datetime.now()
        # keep identical day with replace
        # format (2010, 2, 18, 13, 56, 17, 27245),(year,month,day,hour,minute,seconde,ms)
        time_proba_open = time_now.replace(hour=int(time_open[3]),minute=int(time_open[4]))
        # time will open in => datetime.deltatime(days,seconds,microseconds)
        time_to_open = time_proba_open - time_now
        self._logger.debug("time to open: %s" % time_to_open)
        return time_to_open

    ## @brief Check date and time for ML(Marche libre) specifically. 
    # 
    # Stcks from the "Marche Libre" have only a quotation by day (15H30 in Paris).\n
    # Need only the data-date and time info
    # If is 'open' assume than work has been done, to check...
    # @param stock : name of the stock
    def test_ML_done( self ):
        self._logger.debug("date %s", self.get_action('InstValue','date') )
        self._logger.debug("time %s", self.get_action('InstValue','time') )
        last_date = self.get_action('InstValue','date')
        date_last = datetime.date(*time.strptime(last_date, "%d/%m/%y")[0:3])
        today= datetime.date.today()
        last_time = self.get_action('InstValue','time')
        time_last= datetime.time(*time.strptime(last_time, "%H:%M:%S")[3:6])
        now = datetime.time(*time.localtime()[3:6])

        if (date_last==today) and (now > time_last):
            return True
        else:
            return False

    ## @brief Create a one line string with the values stored in a dictionary.
    # If the dictionary is not loaded, return a string message with the symbol and a message starting with 'ERROR'\n
    # Usually called by DictionaryStocks.make_print_stocks() or UpdateStocks
    # @param action the EAction name of the data to retrieve
    # @return a formatted one line string  
    def print_stock( self, action ):
        # maybe test standard template entry for error
        # should print if empty, error or invalid
        # print "to test for error"
        # assert any( self._dict_stock[action]),"Stock:print_stock action is empty !"

        # test the data have been updated
        # always created by get_action, now with default value
        #if  (self.get_action(action)):
        if action not in self._dict_stock:
            stock_line = self.symbol + " ERROR NO %s loaded\n" % action
            return stock_line

        # sure it exists, make short_cut to action/template
        sub_data = self.get_action(action)
        #print sub_data

# to add clear error, InvalidStock can be used as well now
        # Added only in _SetVolatileData, not present if only reading the stock
        # In fact all ! nothing loaded
        # if self[stock].has_key('error'):
        #    if self.dictstocks[stock]['error'] != 0 :
        #        stock_line = stock + " ERROR BOURSO\n"
        #        return stock_line

        if ( action == 'InstValue'):
            #print "test self.dictstocks ",self.dictstocks
            stock_line = self.symbol + " " + str(sub_data['value']) + " "  +  str(sub_data['variation']) + " "
            stock_line += str(sub_data['time']) + " "  + str(sub_data['date']) + " "
            stock_line += str(sub_data['volume']) + " " + str(sub_data['ouverture']) + " "
            stock_line += str(sub_data['plushaut']) + " " + str(sub_data['plusbas']) + " "
            stock_line += str(self.state) + "\n"
            print "stock_line ", stock_line
            #return stock_line

        elif ( action == 'Fundamental'):
            #sub_data = self.get_dict_parser(action)
            stock_line = self.symbol + " " + str(sub_data['MarketCapitalization']) + " " +  str(sub_data['SharesOwned']) + " "
            stock_line += str(sub_data['DividendYield']) + " " + str(sub_data['BVPS']) + " " +str(sub_data['PriceBook'])
            #stock_line += str(sub_data[]) + " " + str(sub_data[]) + " " +str(sub_data[]
            stock_line += '\n'

        elif ( action == 'Info'):
            stock_line = self.symbol + ", " + str(sub_data['Name']) + ", " + str(sub_data['Sector']) + ", "
            stock_line += str(sub_data['Industry']) + "\n"

        elif ( action == 'HistPrice'):
            first_str = sub_data['first_date'].strftime('%Y-%m-%d %H:%M:%S')
            last_str = sub_data['last_date'].strftime('%Y-%m-%d %H:%M:%S')
            stock_line = self.symbol + " " + first_str + " " + last_str + "\n"

        elif (action == 'DatesIntro'):
            stock_line = self.symbol + " " + sub_data['intro_str'] + "\n"

        else:
            stock_line =  self.symbol + " ERROR action %s Not implemented in Stock\n" % (action)

        return stock_line

# ########## XML input/output ####################

# first test to process all stages of XML data update: read / check_new_data / (validation_stage) /
# merge_after_validation / write XML
# Focus on Fundamental / Info / (DivSplit ??)

# logic similar for HistPrice (looks more DivSplit for validation)
# InstValue : cannot validate every 30 sec., may check / clean data before transforming in 5/10/30 minutes 
# for test/debugging PATH_DATA_ROOT_TEST ?

    ## @brief ReadXML data from file into internal xml_dict
    # @note self._dict_xml is (re-)initialised in this function 
    def read_xml(self):
        self._logger.debug("Entry read_xml")

        filename = GlobalDicts.PATH_DATA_ROOT + os.sep + "xml" + os.sep + self.symbol + ".xml"
        self._logger.debug("to read filename %s", filename)

        if not os.path.isfile(filename):
            self._logger.debug("File %s does not exist !" % filename )
            # should create a default XML ? pass
            #self.get_dict_xml()
            self._dict_xml = StTmpl.create_default_template_xml()
            # new
            #self.set_dict_xml( action, StTmpl.create_default_template_xml() )
            return

        # an existing file should be valid...
        try :
            # parse return a TreeElement
            doc = etree.parse( filename )
            root = doc.getroot()

        except etree.XMLSyntaxError, e:
            self._logger.debug("Caught a etree.XMLSyntaxError")
            raise PortfolioError("PortfolioEror from :%s" % e)

        print(etree.tostring(root, pretty_print=True))

        # (re-)initialise the internal dict_xml,get_dict_stock.clear()
        self.get_dict_xml()
        print "self.get_dict_xml()", self.get_dict_xml()

        # test read symbol and compare to symbol
        # can assert, but not store in dict_xml
        #self._dict_xml['symbol'] = root.find("symbol").text
        #assert ( self._dict_xml['symbol'] == self.symbol ), \
        assert root.find("symbol").text == self.symbol,\
            "Symbol %s does not correspond in file %s !!" % ( self._dict_xml['symbol'], filename )

        # loop over all element from root
        # loop over action childs
        for el_action in root:
            #print("%s - %s" % (element.tag, element.text))
            if el_action.tag == 'symbol':
                continue

            print "process the child tag %s", el_action.tag
            print("%s - %s" % (el_action.tag, el_action.text))
            try :
                # create and fill the template_xml
                dict_action = self._fill_dict_xml( el_action )
            except PortfolioError as ex:
                # can use extra_message also
                hl_exception = PortfolioError("PortfolioError in Stock.ReadXML for symbol: %s\n%s" % (self.symbol,ex))
                #print "new_traceback:\n", hl_exception.traceback
                # can modify the traceback ?
                hl_exception.traceback =  hl_exception.traceback + '....\n' +  ex.traceback
                #print "full traceback\n", hl_exception.traceback
                # type and value given by one instance
                #print "\n== Raise hl_exception"
                raise hl_exception

                # here ok, create a new Exception, value None keep the original(very accurate), traceback is high-level
                # identical, traceback is re-assigned in Constructor
                #raise PortfolioError("new PortfolioError throw in ReadXML"), None, sys.exc_info()[2]
                #raise PortfolioError("new PortfolioError throw in ReadXML")
            # other possible exception ?
            except Exception as ex:
                self._logger.debug("Caught Exception in read_xml")
                print "sys.exc_info()[2] ", sys.exc_info()[2]
                print "traceback ", traceback.format_exc(3)
                raise
                #print "End get_template, re-raise"

            # update self
            self._dict_xml[el_action.tag] = dict_action
            print "self._dict_xml", self._dict_xml
            print Utils.pretty_dict( self._dict_xml )

    ## @brief Create an empty template in _dict_xml and fill with the data in the xml file
    # @param element etree.Element corresponding to an action 
    def _fill_dict_xml(self, el_action ):
        self._logger.debug("Entry element tag %s", el_action.tag )
        # could throw a PortfolioError / XMLError if action does not exist
        action = el_action.tag
        # get the template for the action, XML format: template, config, attrib
        tmp_dict = StTmpl.get_template_xml( action )
        print "tmp_dict default ", tmp_dict
        print "attributes action ", el_action.attrib
        # test, make a full  copy, default may be lost 
        tmp_dict['attrib'] = el_action.attrib

        # loop over child of the action element
        # now list, should make a loop over Set (getchildren ), findall ) 
        for elem_data in el_action:
            self._logger.debug("elem_data.tag/ text/ attrib: %s / %s / %s" \
                % (elem_data.tag, elem_data.text, elem_data.attrib))
            # template for one_value xml, here default (no action)
            one_value_xml = StTmpl.get_template_one_value_xml()
        # Maybe update value after setting the attributes, case append True/False
            one_value_xml['value'] = elem_data.text
            # to invert the logic ? attributes action may be used as default ?
            # loop over one_value, check elem_data, else check parent
            for key_attr in one_value_xml['attrib'].keys():
                # look if attrib set in the line 
                if key_attr in elem_data.attrib:
                    one_value_xml['attrib'][key_attr] = elem_data.attrib[key_attr]
                # else look at the parent attrib, can inherit, delete if set explicitely
                elif key_attr in el_action.attrib:
                    one_value_xml['attrib'][key_attr] = el_action.attrib[key_attr]
                # else problem, some option not setup !
                else:
                    assert( 1==1), "One attribute is neither one_value, netiher default action"
            # append this one_value
            tmp_dict['template'][ elem_data.tag ].append( one_value_xml )

        #print "tmp_dict ", tmp_dict
        print Utils.pretty_dict(tmp_dict)
        # assign here or return to caller
        return tmp_dict

# action is missing, all dict_stock is checked. Maybe not so good if some stay in memory.
#
# More convenient for post_process, action executed one by one:
#
# write another function check_new_data( action, opt_interactive )
#                           -> _valid_xml(action)
#                           -> _valid_list_csv(action)
#
# if interactive:
#    ValidTk() or deriv
#
# same for save_new_data(action)
#          -> save_xml
#          -> save hp/div/inst

# apply only to XML ?

    ## @brief Check the validity of the new retrieved data (in _dict_stock), store update in _dict_interactive.
    # Executed before an automatic or interactive update of the XML data (_dict_xml)\n
    # Updated data are stored in selff.dict_interactive
    # @param opt_interactive optional ask validation to user
#     def check_new_xml_data(self, opt_interactive = False):
#         self._logger.debug("opt_interactive %s", opt_interactive)
#         
#         dict_interactive = {}
#         # create a list of data to update, need specialization by template ??
#         # at best, check if some data are present !!
#         self._logger.debug("_dict_stock: %s", self._dict_stock)
#         
#         # check global error ?
#         #if self.get_dict_parser()['global_error']:
#         #    self._logger.debug("Global Error ", self.get_dict_parser()['global_error'] )
#         
#         # loop over the recent action retrieved by parser
#         for key,value in self.get_action().iteritems() :
#             self._logger.debug("key/value %s %s" % (key, value))
#             
#             # to check for template only, discard config
#             
#             # Static/InstValue not updated
#             # info present in _dict_xml[action][config]
#             #if (key == 'Static') | (key == 'InstValue') :
#             # can create on the fly, not really good idea
#             #if not self.get_dict_xml( key )['config']['save_xml']:
#             
#             # can change to to_save
#             # if not StTmpl.to_save_xml( key ):
#             # if not self.saved_as( key, 'xml' ):
#             #    continue
#             
#             if self.saved_as( key, 'xml' ):
#                 # should not throw exception if possible, better error for interactive mode
#                 dict_action_inter = self._valid_new_value_xml( key, opt_interactive )
#             
#                 if any(dict_action_inter):
#                     dict_interactive[ key ] = dict_action_inter
#              
#             #else pass
               
# keep as example   
#             fname = "check_data_tmpl_" + key
#             # throw AttributeError (getattr)
#             try:
#                 getattr(self, fname)( list_interactive, opt_interactive ) # key in the funct. name
#             except Exception as ex:
#                 self._logger.error("Caught Exception: %s", ex)
#                 self._logger.error("Function %s not implemented " % fname)
#                 raise
#     
#         self._logger.debug("\nFinal dict_interactive:\n%s" % dict_interactive)
#         print Utils.pretty_dict( dict_interactive )
#         self._dict_interactive = dict_interactive
        
    ## @brief Final merge (after user/auto validation) and write XML file.
    # This step includes the merging of the data in self.dist_interactive,\n
    # and (re-)write the file by a call to write_xml, include TEST_MODE for unit-test
#     def save_new_xml_data(self):
#         self._logger.debug("Entry save_new_data_to_xml")
#         
#         # loop over the data in dict_interactive (have been validated)
#         for (key_action, value_action) in self._dict_interactive.iteritems():
#             self._logger.debug("key_action/value_action %s / %s" % (key_action, value_action))
#             
#             # dictionary to update
#             dict_xml = self.get_dict_xml( key_action )
#             
#             # for each eaction loop over key_data and value_data, 
#             # check if modifed to 0 or 2
#             for (key_data, value_data) in value_action.iteritems():
#                 self._logger.debug("\nkey_data / value_data %s / %s" % (key_data, value_data))
#                 
#                 # certainly to save, or to write into file for feedback
#                 if value_data['repl_add'] > 0:
#                     self._logger.info("update xml value / key_data : %s / %s / %s " % \
#                                      (self.symbol, key_data, value_data['new_value']) )
#                     
#                     
#                     # value can overwite the default, use the last one ...?
#                     #b_append_data = bool( dict_xml['template'][key_data][-1]['attrib']['append'] )
#                     # Not working if new created file from template
#                     #if dict_xml['template'][key_data][-1]['attrib']['append'] == "True":
#                     if dict_xml['template'][key_data][-1]['attrib']['append'] == True:
#                         #b_append_data = True
#                         print "append data"
#                         new_tmpl_value = StTmpl.get_template_one_value_xml()
#                         new_tmpl_value['value'] = value_data['new_value']
#                         new_tmpl_value['attrib']['date'] = value_data['date']
#                         new_tmpl_value['attrib']['source'] = value_data['source']
#                         new_tmpl_value['attrib']['append'] = 'True'
#                         # append to the list
#                         dict_xml['template'][ key_data ].append( new_tmpl_value )
#                         
#                     # replace the last value, do not modify the previous entries
#                     else:
#                         dict_xml['template'][ key_data ][-1]['value'] = value_data['new_value']
#                         # format timestamp for C++, transform on the fly ? date added automatically
#                         dict_xml['template'][ key_data ][-1]['attrib']['date'] = value_data['date']
#                         dict_xml['template'][ key_data ][-1]['attrib']['source'] = value_data['source']
#                     
#                 # delete entry ? just pass   
#                 #else:
#                     
#         self._logger.debug("dict_xml before wirte_xml:\n%s " % Utils.pretty_dict(self.get_dict_xml()) )
#         self._logger.debug("dict_xml before wirte_xml:\n%s " % (self.get_dict_xml()) )
#         self.write_xml()

    ## @brief Write the data stored in _dict_xml into a XML file.
    # At this point, write only the data with a valid 'source', avoid to write the default template.\n
    # No parameter action, want to keep all previous and valid entries
    def write_xml(self):
        self._logger.debug("Entry write_xml")
        self._logger.debug("TEST_MODE %s" % GlobalDicts.TEST_MODE )

        # can happen ? not a nomal use
        if not any(self.get_dict_xml()):
            self._logger.error("write xml with an emptyy dict_xml, do nothing")
            return

        # should save in xml_tmp in test
        if GlobalDicts.TEST_MODE :
            filename = GlobalDicts.PATH_DATA_ROOT_TEST + os.sep + "xml_tmp" + os.sep + self.symbol + ".xml"
        else :
            filename = GlobalDicts.PATH_DATA_ROOT + os.sep + "xml" + os.sep + self.symbol + ".xml"
        self._logger.debug("to write filename %s", filename)
        # maybe backup ?
        self._logger.debug("Create a new XML file")

        # create a root Element
        root = etree.Element("root")
        doc = etree.ElementTree(root)

        # child symbol, not necessary, can check it is correct in reading...
        # always present
        ch_symbol = etree.SubElement(root, "symbol")
        ch_symbol.text = self.symbol

        # loop over the action
        for key_action, value_action in self.get_dict_xml().iteritems():
            self._logger.debug("key_action %s" % key_action)

            # symbol still present, if dict_xml read from file. No need to store (only assert in read_xml is enought
            #if key_action == 'symbol':
            #    continue
            
            # set for all actions, still neeeded ? in case of modification default maybe ?
            # user can modify this ??? not implemented, if in write_xml should be fine
            #if not self.get_dict_xml( key_action )['config']['save_xml']:
            #    self._logger.info("save_xml True for stock %s. Does the original setup have been modified ?")
            #    continue
            
            # create sub-tree action in helper function
            
    # if create default template for all actions when creating a new file, DivSplit not updated
    # now version only one action, still problem if new xml file/can check source
            # need to know if values has been updated by a parser, or print all default

            # test is wrong, if source is correct, data have been updated, or add an argument in write_xml
            #if 'attrib' in value_action['config']:
            # main attrib, for each action
            if 'attrib' in value_action:
                #if value_action['config']['attrib']['source'] != None:
                if value_action['attrib']['source'] != None:
                    sub_tree_action = self._create_xml_action( key_action, value_action )
                    root.append( sub_tree_action )

# need more test, need maybe a generator, get back next item and call self.create_xml_action
# what to do if intermediate result ( root.append( sub_tree_action ) )
# this way implement only for function for value
#        sub_tree_action = self._apply_loop_dict( self.create_xml_action )
            
        print(etree.tostring(root, pretty_print=True))
                
        self._logger.debug("write the file")
        with open( filename , 'w') as out_file:
            doc.write(out_file, pretty_print=True)
    
    # could be static, or pass the sub_dictionary only
    ## @brief Helper function, generate xml sub-tree for one action
    # @param key_action action name
    # @param value_action associated dictionary to the dictionary key key_action 
    def _create_xml_action(self, key_action, value_action ):
        self._logger.debug("Entry %s %s", sys._getframe().f_code.co_name, self.symbol )
        self._logger.debug("key_action %s" % key_action)
        # if create default template for all actions when creating a new file, DivSplit not updated
        # create sub_tree for the action
        action_tree = etree.Element( key_action )
        # deal with attribute of the action root
        action_attr = action_tree.attrib
        #for key_attr,value_attr in value_action['config']['attrib'].iteritems():
        for key_attr,value_attr in value_action['attrib'].iteritems():
            action_attr[key_attr] = str(value_attr)

        b_action_append = action_attr['append']
        b_action_source = action_attr['source']

        # dict_xml, values_data always in a list
        for key_data, values_data in value_action['template'].iteritems():
            # never print these entries
            if (key_data == "error") | (key_data == "other"):
                continue
            # values_data is a list, StockTemplate._templ_one_value_xml, only last one interested here ? 
            for one_value_data in values_data:
                print "one_value_data ", one_value_data
                child = etree.SubElement(action_tree, key_data)
                # to change float to string ? certainly better ?
                child.text = str(one_value_data['value'])
                one_value_attr = child.attrib
                #one_value_attrib : date / source / append
                for key_attr in one_value_data['attrib']:
                    # always print the date
                    if key_attr == 'date':
                        one_value_attr[key_attr] = str(one_value_data['attrib'][key_attr])
                        continue
                    # case source, print only if different from default
                    if key_attr == 'source':
                        if str(one_value_data['attrib'][key_attr]) != str(b_action_source):
                            one_value_attr[key_attr] = str(one_value_data['attrib'][key_attr])
                        # else use default
                        continue
                    if key_attr == 'append':
                        if str(one_value_data['attrib'][key_attr]) != str(b_action_append):
                            one_value_attr[key_attr] = str(one_value_data['attrib'][key_attr])
                        #    print "append is True"
                        #else:
                        #print "append is False"
                    else:
                        print "one_value_xml do not repeat attribute %s" % key_attr
        print(etree.tostring(action_tree, pretty_print=True))
        return action_tree

# #### maybe convenient, loop over action, loop over value, apply specific function f_todo (opt, loop over ValidXML template)
# tricky for retour, cannot insert intermediate. Need a generator ? thread-safe  ?
#     def _apply_loop_dict(self, f_todo):
#         self.logger.debug("Entry %s %s", sys._getframe().f_code.co_name, self.symbol )
#         print "f_todo ", f_todo
#         #print "value ", value
#         
#         for key_action, value_action in self.get_dict_xml().iteritems():
#             print "key_action ", key_action
#             print "value_action"
#              
#             if key_action == 'symbol':
#                 continue
#             
#             retour = f_todo( key_action, value_action )
#         
#         return retour

# case of interactive, need to modify self._dict_parser, list of all data to update
# or during selection delete the entry we do not want to add, easier for listCSV
# all data correct in self._dict_parser

# namedtuple can avoid nested dictionary
# http://stackoverflow.com/questions/12119612/nicer-way-to-iterate-to-dictionary-in-python-to-avoid-many-nested-for-loops

# choice, I pass opt_interactive again, or if interactive false all values 1 are treated 0 before
 
    # brief Last step to merge the data present in _dict_parser to _dict_xml.
    # The replacement is done only for the data present in list_interactive after potential user validation.
    # If non interactive mode, data have been discarded in list_update_xml_value_generic. Just need to check > 0  
#    def _merge_data_to_xml(self):
#        self.logger.debug("Entry _merge_data_to_xml")

## @class InvalidStock
# @brief Class template for invalid stocks, if a stock symbol is not found in the configuration file.
# If mistakes are done in some input, the code can continue to run without breaking.
# Test could be done in DictStocks for errors.
class InvalidStock(Stock):

    ## @brief Constructor.
    # Need the invalid template in creation, could be hard-coded in init
    def __init__(self, symbol, dict_static_invalid ):
        # init base, with specific static dictionary in entry

        # modif in templates, need tmp
        tmp = StTmpl.get_template_stock('Static')
        tmp['template'] = StTmpl.get_tmpl_static_invalid()
        #super(InvalidStock, self).__init__(symbol, dict_static_invalid)
        super(InvalidStock, self).__init__(symbol, tmp)
        # own logger
        self._logger = logging.getLogger('SP.InvalidStock')
        self._logger.warning("Init InvalidStock %s" % symbol)

        print "init Invalid, dict_static_invalid", dict_static_invalid
        print "tmp ", tmp

    def is_valid(self):
        return False

    ## @brief Override method of base class, return None    
    def time_to_open(self):
        return None

    ## @brief Override method of base class, return string containing ERROR. 
    def print_stock(self, action):
        return "%s ERROR InvalidStock\n" % self.symbol

if  __name__ == "__main__":
    # only for test, until now Dictionary not imported, circular import
    # http://effbot.org/zone/import-confusion.htm
    from serverportfolio.DictionaryStocks import DictionaryStocks
    
    logging.basicConfig(level=logging.DEBUG)
    logging.getLogger('SP')
    m_logger = logging.getLogger("SP.main")
    
    print "Main Stock.py"
    # create a Stock object
    #stock = Stock('CAC40', DictionaryStocks().get_dict_static('CAC40'))
    m_stock = Stock('GSZ', DictionaryStocks().get_dict_static('GSZ'))
    print "m_stock._dict_stock ", m_stock._dict_stock
    
    # some access to the static data
    print "symbol: ", m_stock.symbol
    print "market: ", m_stock.get_market()
    print "dict_stock: ", m_stock.get_action()
    print "dict_stock[static] ", m_stock.get_action('Static')
    
    print "\n== last_CSV deactivated for dev."
    #print "date ", stock.last_CSV()," string ",datetime.datetime.fromtimestamp( stock.last_CSV() ).strftime('%Y-%m-%d %H:%M:%S')
    #print "date string ", datetime.datetime.fromtimestamp( stock.last_CSV('HistPrice') ).strftime('%Y-%m-%d %H:%M:%S')
#     try :
#         stock.last_CSV('HistPrice')
#     except PortfolioError as ex:
#         print "Caught a PortfolioError "
#         print ex.get_format_string()
#         sys.exit(1)
#     print "date string ", stock.get_dict_parser('HistPrice')['last_date'].strftime('%Y-%m-%d %H:%M:%S')
    
    # dict_return_data a bit more complex
    # config are missing here ? but seems correct in output !
    m_dict_info = {
        'Name' : 'GDF-Suez', #str(),
        'Sector' : 'Energy', # str(),
        'Industry' : 'EnergyGaz', #str(),
        'StockExchange' : 'Paris', #str(),
        # variable, YQL may report other entries
        'other' : {},
        'error' : None,
        # source must be present
        'source' : 'YQL'
    }
    m_stock.set_action( 'Info', None, m_dict_info)
    
    # should use template function for correctness, keep explicit for unit-test
    m_dict_fund = {
        'MarketCapitalization' : float(20000),
        'SharesOwned' : int(15000),
        'DividendYield' : float(3.5),
        'BVPS' : float(5.5),
        'PriceBook' : float(3.0),
        'PER' : float(0),
        'PEG' : float(1.0),
        # variable field, only YQL may report other entry
        'other' : {},
        'error' : None,
        'source' : 'YQL'
    }
    m_stock.set_action('Fundamental', None, m_dict_fund)
    
    #dict_div = {
    #    'first_date' : '2015-01-01',
    #    'last_date' : '20115-02-01',
    #    'list_div' :  
    #}
    print "_dict_stock ", m_stock.get_action()
    print Utils.pretty_dict( m_stock.get_action() )
    
    #print "\n== WriteXML 1"
    #m_stock.writeXML()
    
    print "\n== read_xml"
    try :
        m_stock.read_xml()
        
    except PortfolioError as ex:
        print "Caught PortfolioError in main, ex:\n ", ex
        print "\n",ex.get_format_string()
        sys.exit(1)
    
    except Exception, ex:
        print "Caught an Exception ", ex
        print "sys.exc_info()[2] ", sys.exc_info()[2]
        print "traceback ", traceback.format_exc(3)
        sys.exit(1)
    
#     print "\n== write_xml"
#     try :
#         m_stock.write_xml()
#         
#     except PortfolioError as ex:
#         print "Caught PortfolioError in main, ex:\n ", ex
#         print "\n",ex.get_format_string()
#         sys.exit(1)
#     
#     except Exception, ex:
#         print "Caught an Exception ", ex
#         print "sys.exc_info()[2] ", sys.exc_info()[2]
#         print "traceback ", traceback.format_exc(3)
#         sys.exit(1)
#         
#     sys.exit(1)
       
    print "\n==Interactive simulation"
    print "_dict_parser:"
    print Utils.pretty_dict( m_stock.get_action() )
    
    try :
        # test update of mutliple action
        m_stock.check_new_xml_data( opt_interactive = False )
        # should test complete post_process function
        # m_stock.post_process(e_action, option_post)
    except Exception as ex:
        print "Caught an Exception from checkNewData", ex
        #print "sys.exc_info()[2] ", sys.exc_info()[2]
        print "traceback ", traceback.format_exc(3)
    
    #sys.exit(1)
    #print "dict_parser"
    #print Utils.pretty_dict( m_stock.get_dict_parser )
    #for dict in m_stock.list_interactive
    #print self.get_dict_parser( dict_action_list.keys()[0])
    #sys.exit(1)
    
    # if interactive == False, print directly the data to the file
    try :
        m_stock.save_new_xml_data()
        
    except PortfolioError as ex:
        print "Caught PortfolioError ", ex
        print ex.get_format_string()
        
    except Exception as ex:
        print "Caught an Exception from save_new_data", ex
        # print "sys.exc_info()[2] ", sys.exc_info()[2]
        print "traceback ", traceback.format_exc(3)
        
    print "\n== Create new XML"
    m_stock = Stock("FP", DictionaryStocks().get_dict_static('FP'))
    m_stock.read_xml()
    
    print "\n== Write new empty XML"
    m_stock.write_xml()

#    print '\n== Test property on state\n'
#     m_stock._dict_parser[ 'InstValue' ] = { 'state' : 'CLOSED' }
#     print m_stock.get_dict_parser()
#     print 'initial state ', m_stock.state #get_state() #state
#     print "Open the stock"
#     m_stock.state = 'OPEN' # 
#     print 'final state ', m_stock.state #get_state() #state
#     print "modify state "
#     m_stock._dict_parser[ 'InstValue' ]['state'] = 'CLOSED'
#     print "real ", m_stock._dict_parser[ 'InstValue' ]['state']
#     print "wrong ", m_stock.state #get_state()
#     
#     print "init symbol ", m_stock.symbol
#     m_stock.symbol = 'toto' ### Error cannot change symbol
#     print "new symbol", m_stock.symbol