UtilsParsers.py

Go to the documentation of this file.

## @package serverportfolio.Parsers.UtilsParsers
# @brief Utilities functions specific to the parsers
#
# Last Changed $Id: UtilsParsers.py 13 2015-04-12 19:45:14Z michael $
 
import sys, types, re
import urllib2

from serverportfolio.PortfolioException import QueryError

## @brief Execute the url (GET call only, could extend)
# @return web page in a string format
def web_query( url ):
    print "Entry Utils.web_query url ", url
    # contains the whole download page
    s = None
    # load web page, works for all
    try :
        f = urllib2.urlopen(url)
        s = f.read()
        # should be finally
        f.close()   
    # do report the bug seen at work ??
    # not sure about those errors
    except urllib2.URLError, e:              
        # re-throw error
        #raise
        #print "Caught urllib2.URLError "
        message_str="QueryError with"
        if hasattr(e, 'reason'):
            message_str += ' reason: %s,' % e.reason
        if hasattr(e, 'code'):
            message_str += ' error code: %s' % e.code
        raise QueryError( message_str, url)
    # should not happen
    except Exception, ex:
        print "Got general exception from urllib2 ", ex
        raise
    #finally:
    #    print "Error close the file"
    #    f.close()
    return s

# not sure general is good, local to Parsers will be more specific (too many different cases)
# copy from Parser_Bourso, made general here or keep in Yahoo/Specific Yahoo YQL

## @brief Extract a float from a string.
# General function which checks for 'N/A', '-' and delete blank characters.
# Specific parsers may implement more pseific functions.
# @param line string to parse
# @return float or nan
# may throw different type of error
def extract_number ( line ):
    #print "UtilsParsers.extract_number "
    #print "line: ", line
    tmp = ''
    if ('N/A' in line) | (line.strip() == '-'):
        #print "Not a number"
        return float('nan')
    #print "result: ", float(tmp.join(re.findall(r'([\d+\s\.\+\-])',line)).replace(' ',''))
    # do not run, more check to do with re. Should compile the query
    #print "result: ", float(tmp.join(re.findall(r'([\d+\s\.\+\-\[N\\\A]])',line)).replace(' ',''))
    #return float(tmp.join(re.findall(r'([\d+\s\.\+\-])',line)).replace(' ',''))
    return float(tmp.join(re.findall(r'([\d+\s\.\+\-])',line)).replace(' ',''))

## @brief Check for N/A in Yahoo output, 'N/A' no error
# @param line string to parse
def check_nan( line ):
    if ('N/A' in line):
        return True
    return False