StockwrapPyParser.cpp

Go to the documentation of this file.

 // Split apart, by this way rootcint do not see this include, which makes problem
 // Send a mail to ROOT team ?? Check before

#ifndef Python_for_stock
#define Python_for_stock
//#include <Python.h>
// eclipse recognize this one
#include <python2.7/Python.h>
#endif

using namespace TimeScale;

// declare internal function, could be in header Stock.h
void GetListDataFromPyList( PyObject * pyListCSV, ldata_candle * new_listdata_csv, ldata_volume * new_listdata_vol );

// simple test : works, in calling the python in workspace, need PYTHONPATH set correctly
//may extend function around the dictionary ( print content, get data )
// could be a class in itself
// Load python class DictionaryStocks.
//
// to call after PyInitialze() ( to check, seems logic )
// load module DictionaryStocks and call constructor DictionaryStocks()
// name of the txt file, now use default only
//
int Stock::LoadDictionary ( std::string dict )
{
#ifdef DEBUG_CPP
    std::cout << "Entry Stock::LoadDictionary dict " << dict << std::endl;
#endif

    PyObject *pModule;
    PyObject *pFunc_Init;
    // not used but needed for calling the function
    PyObject *pArgs=0;

    pModule = PyImport_ImportModule( "DictionaryStocks" );

    // modify order of test, easier
    if (pModule == NULL) {
        std::cout << "\n\tCannot load module DictionaryStocks\n" << std::endl;
        return 1;
    }

#ifdef DEBUG_CPP
    std::cout << "DictionaryStocks imported " << std::endl;
#endif
    //get the constructor ParserStock
    //pFunc_Init = PyObject_GetAttrString(pModule, "Init");
    pFunc_Init = PyObject_GetAttrString(pModule, "DictionaryStocks");

    if (pFunc_Init && PyCallable_Check(pFunc_Init)) {
        //see later for arguments, now use 0=NULL
        PyObject_CallObject( pFunc_Init, pArgs );
    }
    //std::cout << " function Init/Constructor done " << std::endl;

    // deallocate python pointer, may be not here in DeleteDictionary ??
    // ok can delete the module still working, until now...
    //Py_DECREF(pArgs);
    Py_DECREF ( pModule );
    Py_DECREF ( pFunc_Init );
    //std::cout << "End LadDictionnary return 0 " << std::endl;
    return 0;
}

// Should split more : LoadModule and Function
//                     make Stock logic

// need to know which source Boursorama or Yahoo_CSV, implemented only DAY and Yahoo
// to extend to Bourso, for Progress DAY.
//
// For update, in-memory like header < now : update all data->Update in-memory (Save new data? yes, consuming time Parser)
//             in-memory shorter than header : update from Header to now->Do not update in-memory (but Save new data)
//                                             do not recomputeTA, need message to the user. updated but not apply by default to in-memory
//             in.memory > header ( because server, not really for DAY, yes but only progress DAY bourso!) :
//                For Yahoo, (call update after loading=>Automatically if written to file, should not happen)
//                For Bourso, only instantaneous, always get the data. For saving can check hour is not 00:00:00...
//
// In summary expects from LastCSVUpdate: last header is crucial.
//            before LastCSVUpdate loaded always the Stock, but could be done here also...
//            If only header (not-in-memory) is available load from header. Will be a problem of Save !
//
// create an object ParserStock( stock, "source" )
// call directly Parser function.
// implemented call to GetListCSV ( redirect to Yahoo_CSV )
//
// only with tmscl == DAY now !
int Stock::UpdateCSVFromWeb ( const std::string& source, Utils::firstlast_dates opt_dates )
        throw (StockException)
        //throw (StockPythonException,StockFileError) // by Read_HeaderCSV
 {
#ifdef DEBUG_CPP
    std::cout << "\n\tEntry Stock::UpdateCSVFromParser source " << source << std::endl;
    std::cout << "dates first " << opt_dates.first << " " << Utils::Time_tToString( opt_dates.first ) << std::endl;
    std::cout << "dates last  " << opt_dates.last << " " << Utils::Time_tToString( opt_dates.last ) << std::endl;
#endif

 std::string namestock=GetName();
 // problem with Py_DECREF, maybe because argument
 //std::string source_copy = source;

 // first date to extract
 time_t first_time = 0; //, opt_date_max;
 int new_size=0;
 int status = 0;
 // set default 0,0 not equivalent to today ! if used with default argument (normal case)
 Utils::firstlast_dates header_dates = Utils::get_fldate_default();
 Utils::firstlast_dates in_memory_dates = Utils::get_fldate_default();
 // always check with today
 //time_t now;
 
 ldata_candle *new_listdata_csv = 0;
 ldata_volume *new_listdata_vol = 0;
 
 // Could create directly the ListDataStock in Function GetList
 PyObject *pStockName, *pSource;            //string stockname, source
 PyObject *pModule;                         // module ParserStocks.py
 PyObject *pFunc_Parser, *pInstance;        // func constructor ParserStock, instance ParserStock
 // *pFunc_LastCSV, *pFunc_Update ;
 PyObject *pFunc_GetListCSV, *pReturn_ListCSV; //funct and return value of GetListCSV
 PyObject *pArgs=0, *pArg_lasttime;            // args for ParserStock

 Read_HeaderCSV( ETime::DAY, header_dates );

 // default date first, last from header
 if ( opt_dates.first == 0 ) {
         first_time = header_dates.last;
 #ifdef DEBUG_CPP
          std::cout << "Set First time, get the last time from header " << first_time << std::endl;
 #endif
 }

// if default date last, set to today, Utils::GetTime_tToday()
 if ( opt_dates.last == 0 ) {
         opt_dates.last = time( NULL );
 #ifdef DEBUG_CPP
         std::cout << "Set last time, get to now " <<   Utils::Time_tToString( opt_dates.last ) << std::endl;
 #endif
 }

 #ifdef DEBUG_CPP
     std::cout << "opt_dates.last " << opt_dates.last << " " << Utils::Time_tToString( opt_dates.last ) << std::endl;
     std::cout << "Set first_time " << first_time << " " << Utils::Time_tToString( first_time ) << std::endl;
 #endif

 // not care about .last, optional if zero up to now if already up now no problem... ??

 // Check if update is possible, with state in memory (in memory loaded until last header, or more, but not less)
 if ( CheckLoaded( ETime::DAY ) ) {
         in_memory_dates.first = GetListData( ETime::DAY, Indicator::StockCSV )->GetFirstTime();
         in_memory_dates.last = GetListData( ETime::DAY, Indicator::StockCSV )->LastUpdate();
 #ifdef DEBUG_CPP
         std::cout << "In memory first date " << in_memory_dates.first << " " << Utils::Time_tToString( in_memory_dates.first,true ) << std::endl;
         std::cout << "In memory last date  " << in_memory_dates.last << " " << Utils::Time_tToString( in_memory_dates.last,true ) << std::endl;
#endif

         // Check if update impossible, only a sub-range of initial data has been read.
         // In memory is missing last ones, problem with saving
         //if ( in_memory_dates.last > header_dates.last ) {
         if ( header_dates.last > in_memory_dates.last ) {
                 std::cout << "CANNOT_UPDATE UpdateFromWeb, return -1" << std::endl;
                 std::cout << "in_memory_last " << in_memory_dates.last << " " << Utils::Time_tToString(in_memory_dates.last, true) << std::endl;
                 std::cout << "header_last " << header_dates.last << " " << Utils::Time_tToString(header_dates.last, true) << std::endl;
                 return -1;
         }
         // do not reload what is in memory
         first_time = in_memory_dates.last;
}

 //always Initialize the interpreter first
 Py_Initialize();
 
 // Load global dictionary in python interpreter, use default dictionary
 // Could throw exception directly from function
 status = LoadDictionary("");
 //std::cout << "status " << status << std::endl;
 if ( status != 0 ) {
     std::cout << "Could not load the dictionary" << std::endl;
     throw ( StockPythonException( namestock, "Error in UpdateCSVFromWeb in loading Python dictionary" ) ) ;
     //return -1;
 }

 // extract arguments for ParserStocks initialization
 pStockName = PyString_FromString( namestock.c_str() );
 pSource = PyString_FromString( source.c_str() );
 //pSource = PyString_FromString( source_copy.c_str() ); // try problem with Py_DECREF, change nothing
#ifdef DEBUG_CPP
 std::cout << "source " << source << std::endl;
 std::cout << "namestock " << namestock << std::endl;
#endif

 //std::cout << "Before import module" << std::endl;
 pModule = PyImport_ImportModule( "ParserStocks" );
 //std::cout << "After import module ParserStocks" << pModule << std::endl;
 // easier to check here
 if ( pModule == NULL ) {
     PyErr_Print();
     std::cout << "Error in loading module" << std::endl;
     Py_Finalize();
     throw ( StockPythonException( namestock, "Error in UpdateCSVFromWeb in loading Python module" ) ) ;
     //return -1;
 }

 // else module loaded
 //get the constructor ParserStock
 pFunc_Parser = PyObject_GetAttrString(pModule, "ParserStocks");
 //std::cout << "pFunc  " << pFunc_Parser << std::endl;

 //call constructor of ParserStock
 if (pFunc_Parser && PyCallable_Check(pFunc_Parser)) {

     pArgs = PyTuple_New(2);
     // test better after ??
     if (!pArgs) {
         Py_DECREF(pArgs);
         Py_DECREF(pModule);
         //fprintf(stderr, "Cannot convert argument, return\n");
         throw ( StockPythonException( namestock, "Error in UpdateCSVFromWeb in calling Python" ) ) ;
         //return -1;
     }
     //Set value of Tuple
     /* pArgs reference stolen here: */
     PyTuple_SetItem(pArgs, 0, pStockName);
     PyTuple_SetItem(pArgs, 1, pSource);

     //Call function constructor and create instance
     //std::cout << "CallObject " << std::endl;
     pInstance = PyObject_CallObject( pFunc_Parser, pArgs );
     //std::cout << "After CallObject " << std::endl;
     //should Py_DECREF-ed the arguments just after
     Py_DECREF(pArgs);

     // Check if null, not enough
     // do not check if an error occurred like the symbol does not exist in the dictionary
     if ( pInstance == NULL ) {
         Py_DECREF(pFunc_Parser);
         Py_DECREF(pModule);
         //Py_DECREF(pInstance);
         PyErr_Print();
         //fprintf(stderr,"Call failed\n");
         std::cout << "Call failed return -1" << std::endl;
         throw ( StockPythonException( namestock, "Error in UpdateCSVFromWeb in loaing Python module" ) ) ;
         //return -1;
     }

     // here need a way to access the potential error:
     // symbol not in dictionary...
     // a simple SymbolExists with a bool value, or the name of the symbol, error by default
     // ...

 } //end if pFunc && PyCallable_Check(pFunc)

 // prepare function for calling GetListCSV
 pFunc_GetListCSV = PyObject_GetAttrString( pInstance, "GetListCSV");

 // check if instance is correct and if callable
 if ( pFunc_GetListCSV && PyCallable_Check(pFunc_GetListCSV) ) {

     //PrintMap();
#ifdef DEBUG_CPP
     std::cout << "Call python with argument first time " << Utils::Time_tToString(first_time) << std::endl;
     std::cout << "Call python with argument last_dates " << Utils::Time_tToString(opt_dates.last) << std::endl;
#endif

     pArg_lasttime = Py_BuildValue("(ll)", (long int) first_time, (long int) opt_dates.last);

     // here or before need to test if symbol exists !
     //std::cout << "Call Python GetListCSV " << std::endl;

     pReturn_ListCSV = PyObject_CallObject( pFunc_GetListCSV, pArg_lasttime);
     Py_DECREF( pArg_lasttime );
#ifdef DEBUG_CPP
     std::cout << " Size of pReturn_ListCSV " << PyList_Size( pReturn_ListCSV ) << std::endl;
#endif
     // here is a test, but maybe never created by python ??
     // ok test working ( should improve intern python, avoid to execute if clearly wrong), return
     if ( PyList_Size( pReturn_ListCSV ) == 0 ) {
#ifdef DEBUG_CPP
         std::cout << "Size of updated stockCSV == 0 " << std::endl;
#endif
         //need to clean before return -1, listdatastock not yet created
         Py_DECREF(pModule);
         Py_DECREF(pInstance);
         Py_Finalize();
         // before retrun -1
         //return -1;
         // now send an exception, but not sure of the reason here !
         // in test symbol not in dictionnary, but error in internet call possible also
         // need a test, before calling the function to test the symbol
         throw ( StockPythonException( namestock, "Error in UpdateCSVFromWeb in calling Python" ) ) ;
     }
#ifdef DEBUG_CPP
     else {
         std::cout << "GetListCSV return with size " << PyList_Size( pReturn_ListCSV ) << std::endl;
     }
#endif

 // function not callable, exit
 } else {
     std::cout << "error in calling pFunc_GetListCSV" << std::endl;
     Py_DECREF(pModule);
     Py_DECREF(pInstance);
     PyErr_Print();
     fprintf(stderr,"Call failed\n");
     return -1;
 }
#ifdef DEBUG_CPP
 std::cout << "Extract data from the list " << std::endl;
#endif

 // get ListCSV from python to C, do not know the size here ?
 // always create a new list to insert, before sure it was loaded, but now ??
 new_listdata_csv = new ldata_candle( "StockCSV", ETime::DAY );
 new_listdata_vol = new ldata_volume( "Volume", ETime::DAY );

 // extract the data and fill the lists, helper function
 GetListDataFromPyList( pReturn_ListCSV, new_listdata_csv, new_listdata_vol );
 /*
 std::cout << "After GetListDataFromPyList " << std::endl;
 new_listdata_csv->PrintData();
 new_listdata_vol->PrintData();
 */

 // crash, not coming from here ??
 // cannot delete variable in C++ way (delete) will crash, should use Py_DECREF
 // should delete all arguments ?? no garbage if embedded, but do not create errors in valgrind ?
 Py_DECREF ( pModule );
 Py_DECREF ( pInstance );
 Py_DECREF ( pFunc_Parser );
 Py_DECREF ( pArgs );
 Py_DECREF ( pArg_lasttime );

 // crash here, exactly same, but before argument, seems order dependent !!
 // if comented ok... other can pass. Check logic here with arguments stolen
 //Py_DECREF ( pSource ); // 1 decrease number of error in valgrind ??
 //Py_DECREF ( pStockName ); // origin, delete StockName, no pSource

 // new added, seems to pass, no error if included or no ?
 // if Py_CREF pSource is commented, they can be deleted without error valgrind
 Py_DECREF ( pFunc_GetListCSV );
 Py_DECREF ( pReturn_ListCSV ); // change nothing

 // always close interpreter, not really needed but good habit, from doc
 Py_Finalize();
 
 new_size = new_listdata_vol->GetSize();

 //std::cout << "will add listdatatomap new_size " << new_size << std::endl;
 //std::cout << "first date " << ( new_listdata_csv->At(0) )->GetDate() << std::endl;
 //std::cout << "size " << new_listdata_csv->GetSize() << " date " << ( new_listdata_csv->At( new_listdata_csv->GetSize()-1 ) )->GetDate()  << std::endl;
 if ( new_listdata_vol->GetSize() ) {

     // re-order if necessary
     new_listdata_csv->Order( EPChrono::NO_CHRONO );
     new_listdata_vol->Order( EPChrono::NO_CHRONO );
     // set dates into the list, real dates for this list here
     // set date before update in AddToList, or done inside ?? need to pass test inside
     new_listdata_csv->SetDates();
     new_listdata_vol->SetDates();


     // only if in-memory !
     // 1 means update, so need to delete the list
     if ( AddListDataToMap( ETime::DAY, new_listdata_csv ) == 1 ) {
         delete new_listdata_csv; new_listdata_csv=0;
     }
     if ( AddListDataToMap( ETime::DAY, new_listdata_vol ) == 1 ) {
         delete new_listdata_vol; new_listdata_vol=0;
     }

     // now, in all case save right now !
     //SaveCSV( ETime::DAY );
 }
#ifdef DEBUG_CPP
 std::cout << "StockwrapPyStock::Update return new_size " << new_size << std::endl;
#endif
 return new_size;
}

//function very similar, copy from, wrapPyStock.cpp wrapUpdateCSV. Should be called by the wrapper,
// one function == one mehod
void GetListDataFromPyList( PyObject * pyListCSV, ldata_candle * new_listdata_csv, ldata_volume * new_listdata_vol )
{
#ifdef DEBUG_CPP
 std::cout << "Entry Stock::GetListDataFromPyList" << std::endl;
#endif
 long int size;
 //create temporary object for data, will be save in the vector
 DataCSV data_csv = DataCSV();
 SimpleData data_vol = SimpleData();
 //equivalent for python
 PyObject *pyListData;

 time_t date;
 float op,pl,mo,cl;
 float vol;

 size=PyList_GET_SIZE(pyListCSV);
#ifdef DEBUG_CPP
 std::cout << "size data pyListCSV reserve for list" << size << std::endl;
#endif
 //allocate the list
 new_listdata_csv->Reserve( size );
 new_listdata_vol->Reserve( size );
  
 for ( int i=0; i<size; i++ ) {
    pyListData = PyList_GET_ITEM(pyListCSV,i);
    //should ??
    Py_XINCREF(pyListData);

    date = (time_t) PyInt_AsLong(PyList_GET_ITEM(pyListData,0));
    op = PyFloat_AsDouble(PyList_GET_ITEM(pyListData,1));
    pl = PyFloat_AsDouble(PyList_GET_ITEM(pyListData,2));
    mo = PyFloat_AsDouble(PyList_GET_ITEM(pyListData,3));
    cl = PyFloat_AsDouble(PyList_GET_ITEM(pyListData,4));
    vol = PyFloat_AsDouble(PyList_GET_ITEM(pyListData,5));
    //std::cout << "date op " << date << " " << op << std::endl;
           
    //set data
    data_csv.SetData(date,op,pl,mo,cl);
    data_vol.SetData(date,vol);

    new_listdata_csv ->AddData ( data_csv );
    new_listdata_vol ->AddData ( data_vol );
    //clear the object, will see missing data
    data_csv.SetData(0,0.,0.,-1000.0,0.);
    data_vol.SetData(0,0.);

    //if incr should decref ??
    Py_XDECREF(pyListData);
   } 
 //pyListData = 0; ??
}

// test function
int Stock::TestwrapPython( std::string source )
{
 std::cout << "entry testwrappython " << std::endl;

 std::string namestock=GetName();
 std::cout << "name  " << namestock << "source " << source << std::endl;
 //time_t last_time;
 //int new_size=0;

 //ListDataStock<DataCSV> *new_listdata_csv=0;
 //ListDataStock<SimpleData> *new_listdata_vol=0;

 // Could create directly the ListDataStock in Function GetList
 PyObject *pStockName;
 PyObject *pSource;
 PyObject *pModule;
 //PyObject *pFunc_Parser, *pFunc_GetListCSV; // *pFunc_LastCSV, *pFunc_Update ;
 //PyObject *pInstance, *pReturn_ListCSV;
 //PyObject *pArgs, *pArg_lasttime;
 
 //always Initialize the interpreter, if we want to embedded the interpreter
 //needed for PyImport_ImportModule
 Py_Initialize();

 pSource = PyString_FromString( source.c_str() );
 Py_INCREF (pSource); // not necessary
 pStockName = PyString_FromString( namestock.c_str() );
 Py_INCREF (pStockName); // not necessary

 
 //pModule = PyImport_ImportModule( "ParserStocks" );
 pModule = PyImport_ImportModule( "ModTest" );

 if (pModule != NULL) {
       //get the constructor ParserStock
       //pFunc_Parser = PyObject_GetAttrString(pModule, "ParserStocks");
       //std::cout << "pFunc  " << pFunc_Parser << std::endl;
       std::cout << " toto " << std::endl;

 } //end if pModule != NULL

 Py_DECREF(pModule); 

 //namestock.clear();
 Py_DECREF ( pStockName );
 Py_DECREF ( pSource );
 pStockName = 0;
 pSource = 0;

 //always close interpreter, not really needed but good habit, from doc
 Py_Finalize();

 return 1;
}

// 2 nd version, add until error valgrind
int Stock::TestwrapPython2( std::string source ) //throw (std::exception)
{
 std::cout << "entry testwrappython 2 " << std::endl;

 std::string namestock=GetName();
 std::cout << "name  " << namestock << " source " << source << std::endl;
 //time_t last_time;
 //int new_size=0;

 //ListDataStock<DataCSV> *new_listdata_csv=0;
 //ListDataStock<SimpleData> *new_listdata_vol=0;

 // Could create directly the ListDataStock in Function GetList
 PyObject *pStockName;
 PyObject *pSource;
 PyObject *pModule = NULL;
 //PyObject *pFunc_Parser, *pFunc_GetListCSV; // *pFunc_LastCSV, *pFunc_Update ;
 //PyObject *pInstance, *pReturn_ListCSV;
 //PyObject *pArgs, *pArg_lasttime;

 //always Initialize the interpreter, if we want to embedded the interpreter
 //needed for PyImport_ImportModule
 Py_Initialize();

 pSource = PyString_FromString( source.c_str() );
 Py_INCREF (pSource); // not necessary
 pStockName = PyString_FromString( namestock.c_str() );
 Py_INCREF (pStockName); // not necessary


 //pModule = PyImport_ImportModule( "ParserStocks" ); //source of error
 //pModule = PyImport_ImportModule( "ModTest" );

 pModule = PyImport_ImportModule( "ModTest2" );
 std::cout << "pModule " << pModule << std::endl;

  // Cannot do here if the module does not exist!
  //Py_INCREF ( pModule );

 if (pModule != NULL) {
           // if error in module return 0
           Py_INCREF ( pModule );
       //get the constructor ParserStock
       //pFunc_Parser = PyObject_GetAttrString(pModule, "ParserStocks");
       //std::cout << "pFunc  " << pFunc_Parser << std::endl;
       std::cout << " loaded ModTest2 " << std::endl;

 } //end if pModule != NULL
 else {
     std::cout << " Module not loaded " << std::endl;
     namestock.clear();
     Py_DECREF ( pStockName );
     Py_DECREF ( pSource );
     throw ( StockPythonException( namestock, "Error in testwrap2 python" ) ) ;
 }

 Py_DECREF(pModule);

 namestock.clear();
 Py_DECREF ( pStockName );
 Py_DECREF ( pSource );
 Py_DECREF ( pModule );
 pStockName = 0;
 pSource = 0;

 //always close interpreter, not really needed but good habit, from doc
 // seems no change in valgrind
 Py_Finalize();

 return 1;
}