Abstract.py

Go to the documentation of this file.

## @package serverportfolio.Parsers.Abstract
# Define an abstract base class for specific Parsers
#
# Last Changed $Id: Abstract.py 27 2015-04-25 16:16:47Z michael $

# https://docs.python.org/2/library/abc.html
# http://stackoverflow.com/questions/13646245/is-it-possible-to-make-abstract-classes-in-python

from abc import ABCMeta, abstractmethod
import types, logging


from serverportfolio import GlobalDicts
from serverportfolio.GlobalDicts import EAction
from serverportfolio import Utils
from serverportfolio.Parsers import UtilsParsers

from serverportfolio.DictionaryStocks import DictionaryStocks
# only needed for Stock.to_list ?
from serverportfolio.Stock import Stock, InvalidStock
# only for assert, may be deleted later
#from serverportfolio.Stock import InvalidStock
from serverportfolio.PortfolioException import ParserError, QueryError, PortfolioError

## @class AbstractParser
#
# Define 2 abstract methods which need to be overridden by the Parsers\n
# and a generic algorithm (run_parser) for parsing the data
class AbstractParser(object):
    __metaclass__ = ABCMeta

    ## Constructor
    # @param e_action GlobalDicts.EAction 
    def __init__( self, e_action ):
        
        ## save url, useful for reporting errors and exceptions
        self.url = None
        # action convenient for Yahoo_CSV (multiple action), set in constructor
        ## enumeration (EAction) of the type of query to perform
        self.e_action = e_action
        
    # TO DELETE AT THE END, end ParserStocks, end UpdateStock
        ## @brief Store links to stocks to update, dictionary 'symbol' : Stock object 
        self.local_stock = None
    
        ## @brief Contain all parsed data by one query, include multiple stocks query
        # List of template_parser
        self.list_return_data = None
        
        ## @brief Source of the data is the parser name : YQL/YCSV/BOURSO, set in ParserFactory
        self.source = None
    
    ## Abstract method to create the URL
    @abstractmethod
    def create_url(self):
        pass

    ## Parse the text file (html, CSV,...)
    @abstractmethod
    def parse(self, s):
        pass
    
# ################ possibility for some parser to overwrite

# option_post, post-process in update_stock to_save(default config, ok?), case to_save easier to implement with access to config 
# interactive, important, case multi-thread, need to protect, how ? case interactive, more tricky (with mutlithread)
#              need singleton, serialize access to TkValid interface 

    ## \brief Main function to call for running the parser.
    # Regroup the different steps in one call:
    # - store_stock_copy (in AbstractParser)
    # - create_url (abstractmethod, to implement in each parser)
    # - web_query (in UtilsParsers)
    # - parse (abstractmethod, to implement in each parser)
    # - update_stock (in AbstractParser)
    # - (post_process?)
    # - clean (in AbstractParser)
    #
    # @param stock a stock symbol or a list of st.symb. "CAC40" or ["CAC40","GSZ"] 
    # @return dict_data : dictionary with extracted data
    # throw PortfolioError
    def run_parser( self, stock, option_post=None ):
        self.logger.debug("run_parser stock: %s", Utils.to_list( stock ) )
        
        if option_post != None:
            self.option_post = option_post
            self.logger.debug('option_post is present: %s' % option_post)
        
        # store a linked to the stock(s) to update
        # Check the size, if no a valid stock, just return (no crash)
        try :
            self.store_stock_copy( Utils.to_list(stock) )
            # could return an exception... but tricky to run the post-processing
            if len(self.local_stock) == 0:
                self.logger.debug("No valid Stock after store_stock_copy")
                return
        except Exception as ex:
            self.logger.debug("Caught Exception in store_stock_copy: ex", ex)
            raise
       
        self.logger.debug("\n== create_url")
        try :
            self.create_url()
        # AssertionError is forwarded
        except Exception as ex:
            self.logger.debug("Caught Exception in create_url: %s", ex)
            raise ParserError( ex, stock, self.e_action.name, self.url)
            
        # load web page, works for all
        s = None
        self.logger.debug("\n== web_query")
        try :
            s = UtilsParsers.web_query( self.url )
        except Exception as ex:
            self.logger.error("Got a general Exception from web_query %s" % ex )
            # should raise a QueryError normally
            raise ParserError( ex, stock, self.e_action.name, self.url )
    
        # parse the page
        self.logger.debug("\n== parse the page")
        try:
            self.parse(s)

        except ParserError:
            self.logger.error("run_parser() catch a ParserError")
            raise
        # check for an other specific error, not sure used
        except PortfolioError as ex:
            self.logger.error("run_parser() catch a PortfolioError: %s", ex)
            raise
        
        # many error of parsing may happen, catch all
        except Exception as ex :
            # critical for parsing and AutoParser
            self.logger.error("Catch exception in run_parser from parse(): %s " % ex)
            raise ParserError( 'ParserError from myparser.parse(): ' + str(ex), stock, self.e_action.name, self.url )
            
        # new add a new commom part, to insert with the other ? at the end of parser ?
        # to do in DictionnaryStocks ? not really. But Dictionary Main Functions could check for error, message, write, TCP...
        # to do inside parse ?
        self.logger.debug("\n== update the stock(s)")
        self.update_stock()
        
        # clean parser in case of re-use, maybe for re-use it is better
        # maybe on top of the function, to use some data in post-processing
        self.logger.debug("\n== clean parser for re-use")
        self.clean()

    ## @brief Reset internal data members after a parsing.
    # Because parsers can be re-used, it deletes previous data\n
    # Most (if not all) of the data member belongs to the abtract base class, the default may be sufficient for most of the specific parsers
    def clean(self):
        # cannot reset e_action defined in the init function if reuse, other ok
        self.url = None
        #self.e_action = None
        self.local_stock = None
        # delete all associated data
        self.list_return_data = None
        
    ## @brief Make a local copy of the Stock objects (linked of the original in DictionaryStocks in fact) into a local dictionary.
    # @pre Stock must be valid (not InvalidStock) changed accept InvalidStock but does not add to the local dictionary
    # @param list of stock symbols
    def store_stock_copy( self, list_stock ):
        self.logger.debug("Entry store_stock_copy")
        self.logger.debug("list_stock: %s" % list_stock)
        
        list_obj_stocks = DictionaryStocks().get_stocks( list_stock ) 
        self.local_stock = {}
        
        for elem in Stock.to_list(list_obj_stocks):
            # allow InvalidStock, no need to filter before, but check the size before running parser
            if ( isinstance(elem, InvalidStock) == False ):
                # new added, pre_process here
                # try
                elem.pre_process( self.e_action )
                self.local_stock[ elem.symbol ] = elem
                
        self.logger.debug("local_stock %s" % self.local_stock)
    
    ## @brief Update the local stocks with the new retrieved data.
    # This version in the base class calls Stock.add_dict_stock(), which implements the update. \n
    # More tests could be done if a specific parser override it.\n
    def update_stock( self ):
        self.logger.debug("Entry update_stock")

        list_dict = Utils.to_list( self.list_return_data )

        # a list of template dictionary, added source parser
        for elem in list_dict:
            symbol = elem['symbol']
            self.local_stock[ symbol ].add_dict_stock( elem['action_templ'], self.source )
            # post-process can be run from here now, if config is correct
            print "\n== After add_dict_stok"
            # which action ? self.e_action ok + option_post from update_stock
            #print "stock._dict_stock", self.local_stock[symbol].get_action()

            # new added, post_process can be done by each parser
            #if self.option_post:
            if hasattr(self,'option_post'):
                print "option_post present"
                self.local_stock[ symbol ].post_process( self.e_action, self.option_post )
            else:
                print "option_post not present"

# To move into DictionaryStocks if used by others modules that parsers

    ## @brief Retrieve the symbol (or Stock) from the code_yahoo or code_bourso.
    # Needed for parsing multiple CSV data, if order is not fixed.\n
    # @param code read from the CSV line
    # @param code_type 'code_yahoo' or 'code_bourso'
    def get_symbol_from_code( self, code, code_type ):
        self.logger.debug("get_symbol_from_code: %s" % code )
        assert( ( code_type == 'code_yahoo') | (code_type == 'code_bourso') ),\
            "Error in argument for code_type: %s" % code_type 
            
        for key, value in self.local_stock.iteritems():
            if value.get_action('Static',code_type) == code:
                self.logger.debug("return symbol: %s" % key)
                return key
        # do not generate traceback
        raise ParserError("Cannot retrieve symbol from '%s': %s" %(code_type, code), \
                          self.local_stock.keys(), self.e_action.name, self.url)