Run_Parsers.py

Go to the documentation of this file.

#!/usr/bin/python

## @package Run_Parsers
#
# Standalone program, implement all possible functions with or without the Server running.\n
# If the server is running call the server functions (not always need to query the web page)\n
# or make a call to Parserstocks through UpdateStocks\n
# 
# Should be called simply from apache/ROOT_application with a minimum wrapper\n
# Return a simple string to the caller
#
#
# Last Changed $Id$

## @page doc_runparsers Page documentation for RunParsers
#  
#  Implement all functions with or without the Server running.
#  Always return simple strings of data, easy to parse.
#  
#  \section sec1 Exemples
#
#  ./Run_Parsers.py --source=Boursorama --action=GET --list_stock CAC40,ORA 
#
#  ./Run_Parsers.py --s=Boursorama --a GET  CAC40 ORA
#
#  ./Run_Parsers.py --a RELOAD 
#
#  ./Run_Parsers.py -t

import sys, time
import logging, argparse

#from serverportfolio.GlobalDicts import DEBUG_MODE
#from serverportfolio.DictionaryStocks import DictionaryStocks

from serverportfolio.UpdateStocks import UpdateStocks
#from serverportfolio.Parsers.ParserStocks import ParserStocks
import serverportfolio.SocketServer_Client as SocketServer_Client
from serverportfolio.PortfolioException import PortfolioError

# global data to the module
verbose = None
#list_stock = list()
#source = None
#action = None

# old usage, keep to remember the options
# def usage():
#     print '''
# ./Run_Parsers --test_server / -t no argument
#               --source Boursorama/Yahoo
#               --action GET (instantaneous values) / UPDATE_CSV (missing CSV data) / INTRO ( first date in yahoo )
#                        RELOAD (reload dictionary)
#               --list_stock X,X,X
#               -h / --help
# ex.
# ./Run_Parsers.py --source=Boursorama --action=GET --list_stock CAC40,ORA
# ./Run_Parsers.py --s=Boursorama --a GET  CAC40 ORA
# ./Run_Parsers.py --a RELOAD
# ./Run_Parsers.py -t
# '''
    
# def main():
#     
#     # necessary to get from main
#     global verbose
#     global list_stock
#     global source
#     global action
#        
#     # try reading arguments
#     try:
#         opts, args = getopt.getopt(sys.argv[1:], "hlsa:tv", ["help", "list_stock=", "source=", "action=","test_server","reload"])
#         #print "opts ",opts
#         #print "args ",args
#         if (opts==[] ) & (args==[] ):
#            usage()
#            sys.exit(2) 
#     except getopt.GetoptError, err:
#         # print help information and exit:
#         print "error parsing arguments"
#         print str(err) # will print something like "option -a not recognized"
#         usage()
#         sys.exit(2)
#     
#     #verbose = None
#     
#     for o, a in opts:
#         #print " o, a ", o, a
#         if o == "-v":
#             #print "ask verbose "
#             verbose = True
#         elif o in ("-h", "--help"):
#             #print "ask help"
#             usage()
#             sys.exit()
#         elif o in ("-l", "--list_stock"):
#             #print "ask list stock a=", a
#             list_stock = str.split(a,",")
#             
#         elif o in ("-s", "--source"):
#             #print "ask source a= ", a
#             source = a
#         
#         elif o in ("-a", "--action"):
#             #print "ask action a= ", a
#             action = a
#             
#         # special action, make an argument for it    
#         elif o in ("-t","--test_server"):
#             print "special t arguemnt"
#             action = 'TEST_SERVER'
#             
#         elif o in ("-r","--reload"):
#             print "special r argument"
#             action = 'RELOAD'
#             
#         else:
#             # miss something ?? import or other ??
#             #assert False, "unhandled option"
#             print "cannot parse option"
#             usage()
#             sys.exit(2)
#             
#     for a in args:
#         list_stock.append( a )
#     
#     # test required argument
#     if ( action != 'TEST_SERVER' ) & ( action != 'RELOAD' ):
#         # else all field required 
#         if ( action == None ) | ( source == None ) | (list_stock == [] ) :
#              print "Missing action, source or list_stock"
#              usage()
#              sys.exit(2)
             
# work for test, could be more general. All action could return a simple line of data (with optional header?)
# all depends on Stock, calling should be transparent..
# function could be in UpdateStocks get_string_data()
# whose responsible if server is running ? (server may need UpdateStocks/ParserStocks for sure)

# 1 specific function to call the server/ 1 specific function to call updatestocks, makes sense
# to see regarding to ouput, to get a list of strings good for python/C++

# named CallGET before 

## @brief Get the last instantaneous data of the list of stocks.
#
# Only used with Boursorama at this point
#
# @method method
# @param list_stock of the stock symbols
# @param action
# @param header if True print a header before the data     
def send_to_server( method, action, list_stock,  header=False ):
    
    # not working ??
    #received = str()
    received = ''
    #print "send_to_server server_running ", server_running
    main_logger.debug("send_to_server server_running %s" % server_running)
    # if server is running, must format the query. Can be done by UpdateStocks, just do not run the parser parse. 
    if server_running :
        
        print "server running call it directly"
        print 'method ', method
        str=''
        
        if method:
            str += method + ' '
        if action:
            str += action + ' '
        if list_stock:
            str += ' '.join(stock for stock in list_stock)
        #print "str: ", str
        #str = action
        # could make better
        #for stock in list_stock:
        #    str += ' ' +  stock
            
        print "str in CallGET ", str
        received = SocketServer_Client.SendData( str ) #action
        #print "received in CallGET ", received
        #return received
    
    # server not running, parse in loop
    else :
        print "server not running call parsers"
        main_logger.debug("server not running call parsers")
        
        # first test, better to call UpdateStocks, here action is known
        update_stock = UpdateStocks( action )
        # first test, better to call UpdateStocks ?
        
        # reuse of the internal parser if multi-stock, new accept option multithread
        received += update_stock.get_string_data( list_stock, opt_header = header)
            
    # only one return, shoud deal with error, better exception
    #print "generak received ", received
    return received

## @brief Make calls to YahooAPI module.
# Not tested yet, work only for one stock
#
# Implements:
#  - LAST_CSV_UPDATE : 
#  - UPDATE_CSV : Download stock values from last in file to today
#  - INTRO : Download stock data from first available to first in file
def CallYahooCSV( dict_stocks, stock, action ):
    
    if DEBUG_MODE :
        print "Entry CallYahooCSV"
        print "action ", action
        print "stock ", stock
        
    received = 'output'
    # strange format, no space !
    str = action + stock
    if DEBUG_MODE :
        print "str ", str
    
    # server do not implement a better method, do not use thread for this
    # should implement action in AutoParser, and thread
    if server_running :
        received = SocketServer_Client.SendData( str )
        
    else :
        # can overwrite source, Yahoo_CSV for all methods implemented here
        parserstocks = ParserStocks( stock, source )
        #print "manual "
        
        # need to get time of the last update
        if action == 'LAST_CSV_UPDATE' :
            time_t = parserstocks.LastCSV()
            #print " time_t ", time_t
            received = time_t
            
        # make the update, get time inside the function
        elif action == 'UPDATE_CSV' :
            received = parserstocks.UpdateCSV()
                
        # get intro
        elif action == 'INTRO' :
            print "INTRO"
            time_t = parserstocks.GetDateYIntro()
            print "time_t", time_t
            received = time_t
        
    return received

# no more main section,should call directly correct parsers/UpdateStocks functions
# focus on available option/action defined in UpdateStocks, no care of parsers/thread, impl of post-process


# parsing with argparse, variable defined with add_argument, parser bad name here !!
arg_parser = argparse.ArgumentParser(description="Main executable for parsing data from web page")

# if not provided, should treat all stocks
arg_parser.add_argument("-stock", dest="list_stock", required=False, nargs='+', help="The list of stock symbol to update")

# GET/FORCE, optional ? if server running should use it, otherwise call to get
arg_parser.add_argument("-method", dest="method",required=True, \
                        help='method GET / FORCE / UPDATE_CSV / TEST_SERVER')

# all action should be available, maybe keep action for InstValue, ... like in the code 
arg_parser.add_argument("-a","--action", \
                        help="Action to perform, available: EAction")

# this is good, change the level in debug. Print to a file better for real usage
arg_parser.add_argument('-v','--verbose', dest='verbose', action='store_true', default="False",\
                        help='Set logging to debug mode, otherwise error/critical mode activated by default')

# need to force boolean type default
arg_parser.set_defaults(verbose=False)

try :
    option = arg_parser.parse_args()
# parse_args return an exit
except SystemExit :
    sys.exit(1)

print "option.verbose", option.verbose
# logging from verbose option, could define a file output by default ?
# file is good, can test the call in production with server (read the output)
if option.verbose:
    logging.basicConfig(level=logging.DEBUG)
else:
    logging.basicConfig(level=logging.ERROR)

# in all case, it is better to split log and returned data(used by apache server,by C++...?)
# logging.basicConfig(filename='run_parser.log', level=logging.INFO) # filemode='w' to not append to file

# create loggers for modules and main, not working !
logger = logging.getLogger('SP')
main_logger = logging.getLogger("SP.run_parser")

main_logger.debug("Main Run_Parsers")
main_logger.debug("OK, list_stock: %s" % option.list_stock)
main_logger.debug("method : %s" % option.method)
main_logger.debug("action : %s" % option.action)
main_logger.debug("option.verbose : %s" % option.verbose)

# test if a server is running
server_running = SocketServer_Client.Connect()
main_logger.debug("server_running: %s", server_running)

#print 'status ', status
# special case, no real action required, exit at the end
# server_running = False
if option.method == 'TEST_SERVER' :
    if server_running == False :
        print "NOT RUNNING"
    else :
        print "RUNNING"
    sys.exit(0)
        
#
elif option.method == 'RELOAD' :
    print "RELOAD to implement"

# transform to EAction not needed here, done later

# get or force, one option, method is about action  
#elif ( option.method == 'GET' ) | ( option.method == 'FORCE' ) :
#    received = CallGET( list_stock, option.action )

# call local function to Run_Parsers
try:
    received = send_to_server( option.method, option.action, option.list_stock )
except PortfolioError as ex:
    print "Caught PortfolioError in RunParser"
    print "ex: ", ex.get_format_string()
    print "ERROR"
    sys.exit(1)
    
    
main_logger.debug("final output")
print received

# ###################    main function ##########
# no more main a priori
# if __name__ == "__main__":
    
    
#     
#     if DEBUG_MODE :
#         print "Entry Run_Parser main "
#     
#     server_running = False
#     # parse argument
#     main()
#     
#     
#     if DEBUG_MODE :
#         print "verbose ", verbose
#         print "source ", source
#         print "list_stock ", list_stock
#         print "action ", action
#     
#     # make singleton object for stocks
#     dict_stocks = DictionaryStocks()
#     
#     # test if server running
#     status = SocketServer_Client.Connect()
#     #print "status ",status
#     
#      # if not, need to load the dictionary
#     if ( status == False ) :
#         server_running = False
#     else :
#         server_running = True
#         #print " will use server "
#         
#     # special case, no real action required, exit at the end
#     if action == 'TEST_SERVER' :
#         if server_running == False :
#             print "NOT RUNNING"
#         else :
#             print "RUNNING"
#         sys.exit(0)
#         
#     elif action == 'RELOAD' :
#         if server_running == False :
#             print "server no running, RELOAD action has no effect"
#             sys.exit(0)
#         else :
#             if DEBUG_MODE :
#                 print "Reload done from the server call"
#             str = "RELOAD"
#             #print "str for RELOAD ", str
#             received = SocketServer_Client.SendData( str )
#             print received
#             sys.exit(0) 
#         
#     # for the moment, work only with one stock for some actions, less and less
#     # seems ok now, can delete
#     #stock = list_stock[0]
#     #print "stock ", stock
#         
#     # call GET or Force call function, will deal with all case
#     # create the parser if needed only
#     if ( action == 'GET' ) | ( action == 'FORCE' ) :
#         received = CallGET( dict_stocks, list_stock )
#         #print "final out", received
#         print received
#         #sys.exit(0)
#         
#     # same can do it, or use server and thread
#     # combine with LAST_CSV_UPDATE and INTRO in a function CallYahooCSV
#     elif ( action == 'UPDATE_CSV') | ( action== 'LAST_CSV_UPDATE') | \
#          ( action == 'INTRO' ):
#         
#         # @todo make a loop over the list or inside
#         received = CallYahooCSV( dict_stocks, list_stock[0], action )
#         if received != None :
#             print received
#             
#     else :
#         print "action not implemented ",action
#         #sys.exit(0)