ListDataStock_timpl.h

Go to the documentation of this file.

#include <cassert>

// to get << ETime, but creates ambiguous with some other as_uint(EDS) not so logic ??
//using namespace TimeScale;




template<
class T,
template <class> class PolicyChronologic
>
void ListDataStock<T,PolicyChronologic>::Clear()
{
#ifdef DEBUG_CPP
  std::cout << "List<T> Clear()" << std::endl;
#endif
 // clear internal vector
 ListData.clear();
 // reinitialize data member of base class
 name = "";
 tmscl = ETime::not_a_time;
 dates = Utils::get_fldate_default ();
 fullname = "";
}


// Append data from a new list ( coming from update or ReadCSV normally ) to the existing list
//
// \pre Order policy is not checked, they are consider identical (condition to include)
// \post new_list is not modified, should be deleted by the calling function (add const)
//
// First implementation, consider it is an update (only more recent dates)
// PolicyChronologic should make the correct work, when updating the vector
//
// The test are:
//  - if GetFirstTime() are equal: full update,\n
//     The last data is removed by default (can be update in the last candle) first_to_skip = GetSize()-1
//  - else if this->LastDate != new->FristDate(), assume posterior dates for the new_data (added assert)\n
//  - else (LastDate == new->FirstDate()) \n
//     Check LastData HasChanged (same date, but different Close() only, e.g., update M5)
//     if yes : replace_last = true, EraseLastData
//     if no : first_to_skip +=1
//
//  test size to insert, if new->GetSize = first_to_skip, nothing to do ?
//
//  // never a check about the policy of the entry
//  // add assert, first, transform if not
//  // or to do in AddChronoToList, better, all other tests independent of the policy
//  Finally call Policy::AddChronoToList ( param first_to_skip )
//
// Effect expected:
// - FirstDates are equal : full update
// - first date of new == last date of old : check values (only Close) and set replace_last ( values changed )
// - else no overlap, add only the new data
//
// Always call to Policy::AddChronoToList( new_list, first_to_skip )
// Maybe replace would work also ?? Or all-in-one
//
// @param ListDataStockBase* : list to append, never modified by the function
//
// could return some infos number of updates. Maybe a parameter would be better, bool_replace,  bool_update ?
// should be the same type, but not necessary the same policy
template<
class T,
template <class> class PolicyChronologic
>
int ListDataStock<T,PolicyChronologic>::AddToList( const ListDataStockBase * new_list )
{
#ifdef DEBUG_CPP
  std::cout << "List<T> AddToList" << std::endl;
#endif

  bool replace_last = false;
  std::vector<T> vec_new;
  // default take all the data of new_list
  unsigned int first_to_skip = 0;

  // 2 cases :
  // - first dates are equal, replace all, call Replace
  //   return 0;
  //
  // - if first date of new equal last date of old :
  // - Check for replace_last: true delete last, false set first_to_skip to 1
  // - Continue (real) update
  //    - Set last date from the new_list
  //    - call AddToListChrono
  //    - return 0;
  
  assert ( new_list != nullptr );
  assert ( new_list->GetSize() > 0 );

  if ( ( !new_list ) || ( new_list->GetSize() == 0 ) ) {
#ifdef DEBUG_CPP
        std::cout << "Pointer not valid or size is null" << std::endl;
#endif
    // return a error, should not happen internally
        return 1;
  }

  // check if FirstTime are equal, will replace the full list (keep pointer identical)
  if ( GetFirstTime() == new_list->GetFirstTime() ) {
#ifdef DEBUG_CPP
      std::cout << " GetFirstTime are equal, replace all data " << std::endl;
#endif
      // cannot if call AddChronoToList, must clear the data before, like in Replace !
      // replace all data, by full copy, dates are updated
      Replace ( new_list );
      return 0;
  }

  // Try to append data
  // test last date of old and first date of new overlap
  if ( NoChronologicAt(0).GetDate() ==  new_list->ChronologicAt(0).GetDate() ) {
#ifdef DEBUG_CPP
      std::cout << "Last date of old == first date of new" << std::endl;
      std::cout << "first to skip " << first_to_skip << std::endl;
#endif
      // check if there is a common date, and if the last data must be updated
      // only Close value is checked
      replace_last = NoChronologicAt(0)
          .HasChanged ( new_list->ChronologicAt( first_to_skip ) );
#ifdef DEBUG_CPP
      std::cout << "replace_last " << replace_last << std::endl;
#endif
      // same date and different data, delete the last ( input is correct same date )
      if ( replace_last ) {
          // no need full Policy, but still more clear, compiler dependent ??
          PolicyChronologic<T>::Erase_LastChrono( ListData );
      // or same values, shift skip to 1
      } else {
          // no need of the first data
          first_to_skip += 1;
      }
   }
#ifdef DEBUG_CPP
  // else no overlap, it is a real update, makes some assert here
  else {
     // should apply
     std::cout << "First dates, no overlap of dates, will append all data" << std::endl;
     std::cout << "LastUpdate " << Utils::Time_tToString( LastUpdate(), true ) << std::endl;
     std::cout << "GetFirstTime() " << Utils::Time_tToString( new_list->GetFirstTime(), true ) << std::endl;
     // keep all, first_to_skip still 0, replace_last still false
     // here assume that new dates are larger, may include the case if needed
     assert ( LastUpdate() < new_list->GetFirstTime() );
  }
  // assert coherent new_list input, data member and datastock in vector
  // could be done at any moment ?? even at start ??
  assert ( new_list->GetFirstTime() == new_list->GetFirstDate() );
  assert ( new_list->LastUpdate() == new_list->GetLastDate() );
#endif
  //}

  // need to update the date, can vary in this case, even if no new candle
  // if update reassign the last date only,
  // if test not necessary, should never be smaller ?
  if ( new_list->GetLastDate() > dates.last ) {
#ifdef DEBUG_CPP
      std::cout << "update only dates.last" << std::endl;
#endif
      dates.last = new_list->GetLastDate();
  }

  // if only one data and no update of the last nothing to do
  if ( new_list->GetSize() - first_to_skip == 0 ) {
#ifdef DEBUG_CPP
        std::cout << " size_new_list_update == 0, nothing to do " << std::endl;
#endif
      // it is not an error, 1 candle not updated
      return 0;
  }

#ifdef DEBUG_CPP
  std::cout << "call AddChronoToList with first_to_skip " << first_to_skip << std::endl;
#endif
  // extract the vector in the same policy order as this
  new_list->GetVectorData ( vec_new, bpolicy() );
  // add the vector(now correctly ordered) to the list
  PolicyChronologic<T>::AddChronoToList( ListData, vec_new, first_to_skip);

#ifdef DEBUG_CPP
  std::cout << "End ListDataStock::AddToList " << std::endl;
#endif
  return 0;
}

// better stl for this copy ? problem to check policy is correct
// should be the same type, but not necessary the same policy
template <
  class T,
  template <class> class PolicyChronologic
>
void ListDataStock<T,PolicyChronologic>::Replace ( const ListDataStockBase * ldata_in) {
#ifdef DEBUG_CPP
      std::cout << "ListDataStock::Replace " << std::endl;
#endif
      // delete previous data, to test with swap
      ListData.clear();
      //  get the vector of data to insert, in the same order as this.policy
      std::vector<T> new_vector;
      ldata_in->GetVectorData( new_vector, bpolicy() );
      // insert all new data
      // or use swap, more efficient, data deleted later anyway...
      ListData.insert( ListData.end(), new_vector.begin(), new_vector.end());
      // reset the new dates by default, or copy from ldata_in
      SetDates();
}


// can add data from multiple ListDataStock
// no offsets here, but simple to introduce
/*
template <
  class T,
  template <class> class PolicyChronologic
>
template <
  class U
>
unsigned int ListDataStock<T,PolicyChronologic>::GetCArray2( U **arraydata_c, const EPChrono b_order,
         const unsigned int no ) const
{
#ifdef DEBUG_CPP
        std::cout << "Entry GetCArray2" << std::endl;
    std::cout << "  b_order " << b_order << std::endl;
    std::cout << "  no " << no << std::endl;
    std::cout << "  size_listdata " << GetSize() << std::endl;
#endif
        // size of input
        size_t size_data = GetSize();

    // allocate the array, U float/double
        *arraydata_c = new U[size_data];

    // fill in chronological order
        if ( b_order == EPChrono::CHRONO ) {
            for ( size_t i = 0; i < size_data; ++i ) {
            // Return no from the vector
                // GetValues return a copy of the vector, with MultiData could be a ref, not for the others
                //std::cout << "GetValues()[no] " << ChronologicAt( i ).GetValues()[no] << std::endl;
                (*arraydata_c)[i] = ChronologicAt( i ).GetValues()[no];
            }
    // fill in non-chronological order
        } else {
            for ( size_t i = 0; i < size_data; ++i ) {
            (*arraydata_c)[i] = NoChronologicAt( i ).GetValues()[no];
            }
        }
        return size_data;
}
*/
template <
  class T,
  template <class> class PolicyChronologic
>
unsigned int ListDataStock<T,PolicyChronologic>::GetCArray( var_types arraydata_c, const EPChrono b_order,
         const unsigned int no ) const
{
#ifdef DEBUG_CPP
        std::cout << "Entry GetCArray" << std::endl;
    std::cout << "  b_order " << b_order << std::endl;
    std::cout << "  no " << no << std::endl;
    std::cout << "  size_listdata " << GetSize() << std::endl;
#endif
        // size of input
        size_t size_data = GetSize();
        
    // allocate the array, U float/double
        //*arraydata_c = new U[size_data];
         boost::apply_visitor( visitor_get_array_allocate(size_data), arraydata_c );
         std::cout << "After allocation memory " << std::endl;


    // fill in chronological order
        if ( b_order == EPChrono::CHRONO ) {
            for ( size_t i = 0; i < size_data; ++i ) {
                // GetValues return a copy of the vector, with MultiData could be a ref, not for the others
                //std::cout << "GetValues()[no] " << ChronologicAt( i ).GetValues()[no] << std::endl;
                //(*arraydata_c)[i] = ChronologicAt( i ).GetValues()[no];
                boost::apply_visitor( visitor_get_array_set_value(i,ChronologicAt( i ).GetValues()[no]), arraydata_c );
            }
    // fill in non-chronological order
        } else {
            for ( size_t i = 0; i < size_data; ++i ) {
            //(*arraydata_c)[i] = NoChronologicAt( i ).GetValues()[no];
            boost::apply_visitor( visitor_get_array_set_value(i,NoChronologicAt( i ).GetValues()[no]), arraydata_c );
            }
        }


        return size_data;
}

template <
  class T,
  template <class> class PolicyChronologic
>
unsigned int ListDataStock<T,PolicyChronologic>::GetCArray( var_types2 arraydata_c, const EPChrono b_order,
         std::initializer_list<unsigned int> list_no ) const
{

#ifdef DEBUG_CPP
        std::cout << "GetCArray2 with initializer_list" << std::endl;
    std::cout << "b_order " << b_order << std::endl;
    std::cout << "number in list " << list_no.size() << std::endl;
    std::cout << "size_data " << GetSize() << std::endl;
#endif
        // store values of the list
        std::vector<unsigned int> vec_indic;
        // number of colomn to extract
        unsigned int nb_indic = list_no.size();

        // to do
        // Check nb_indic <= GetNbValues() ?
        
        // save values from the input list, list has no []
        for (auto& value : list_no) {
                vec_indic.push_back(value);
        }
        // allocate the first array, U float/double
        //*arraydata_c = new U*[ nb_indic ];
        boost::apply_visitor( visitor_get_array_allocate(nb_indic), arraydata_c );

        // indice in list may be random
        unsigned int count = 0;
        // call the other function, will allocate second dimension
        for ( auto &nb_indic : vec_indic ) {
                //this->GetCArray2( &((*arraydata_c)[count]), b_order, nb_indic );
                try {
                        if ( float*** pf = boost::get<float***>(arraydata_c) )
                                this->GetCArray( &((*pf)[count]), b_order, nb_indic );
                } catch (  boost::bad_get &e ) {
                //} catch (  std::exception &e ) {
                        std::cout << "Got exception for float, try float " << std::endl;

                        try {
                                if ( double*** pd = boost::get<double***>(arraydata_c) )
                                        this->GetCArray( &((*pd)[count]), b_order, nb_indic );
                        } catch (  boost::bad_get &e ) {
                                std::cout << "Got exception for double, try int " << std::endl;

                                try {
                                        if ( int*** pi = boost::get<int***>(arraydata_c) )
                                                this->GetCArray( &((*pi)[count]), b_order, nb_indic );
                                } catch (  boost::bad_get &e ) {
                                        std::cout << "Got exception for int, rethrow " << std::endl;

                                        throw;
                                }
                        }
                }
                ++count;
        }
        return nb_indic;
}

template <
  class T,
  template <class> class PolicyChronologic
>
int ListDataStock<T,PolicyChronologic>::PutCArray (
      const ListDataStockBase & ldata_date,
          const unsigned int nb_data,
      const unsigned size_indic,
      var_types arraydata_c,
      const unsigned int offset_begin,
      const unsigned int offset_end,
      const EPChrono b_chrono,
      bool full_array )
{

 using namespace TimeScale;

#ifdef DEBUG_CPP
 std::cout << "In ListDataStock::PutCArray with offset_begin/offset_end " << std::endl;
 std::cout << "nb_data " << nb_data << std::endl;
 std::cout << "offset_begin " << offset_begin << std::endl;
 std::cout << "offset_end " << offset_end << std::endl;
 std::cout << "size_indic " << size_indic << std::endl;
 std::cout << "bool chrono " << b_chrono << std::endl;
 std::cout << "full_array " << full_array << std::endl;
#endif

 Reserve( nb_data );

 // check, assert some size problem (nb_indic mainly)
 // size_indic should fit in c-array, cannot check here

 // size of nb_data, should be smaller than size c-array, but no access here
 // should fit with ldata_date, maybe very strong but sure it is working like this
 // maybe later only smaller ? normally nb_data given by TALib
 assert ( nb_data == ldata_date.GetSize() - offset_begin - offset_end );

 if ( tmscl != ldata_date.GetTimeScale() ) {
  if ( tmscl != ETime::not_a_time )
          std::cout << "Warning, TimeScale has been reset to " << ldata_date.GetTimeScale() << std::endl;
  tmscl = ldata_date.GetTimeScale();
 }

 // need this constructor for all, only used by MultiData, other fixed ?
 T sdata( size_indic );
 //std::cout << sdata << std::endl;

 // use intermediate vector to setup all data stock
 // conversion between input and type_value_data done automatically
 std::vector<type_value_data> vec_values;
 //type_value_data tmp_values;
 //visitor_type_array Visitor;
 unsigned int shift_data = 0;

 // in case of full array of data, need to apply some shift
 if ( full_array ) {

         if ( ( b_chrono == EPChrono::CHRONO ) && offset_begin > 0 ) {
                 shift_data = offset_begin;
         } else if ( ( b_chrono == EPChrono::NO_CHRONO ) && offset_end > 0 ) {
                 shift_data = offset_end;
         }
    // the other cases pass without shift
 }

 // check order of the array in entry important for the dates
 // new version, talib output is chronologic, but there is a shift !
 for ( unsigned int i= 0; i < nb_data; ++i ) {

         vec_values.clear();
         for ( unsigned int j=0; j < size_indic; ++j ) {

                 vec_values.push_back( boost::apply_visitor( visitor_type_array(j,i+shift_data), arraydata_c ) );
         }
         //std::cout << "vec_values size " << vec_values.size() << std::endl;

         if ( b_chrono == EPChrono::CHRONO ) {
                 // no change needed if offset_end
                 sdata.SetData( ldata_date.ChronologicAt(i+offset_begin).GetDate(), vec_values );
         } else {
                 // no change if offset_begin
                 sdata.SetData( ldata_date.NoChronologicAt(i+offset_end).GetDate(), vec_values );
         }
         // Add always by push back, efficient if reserve cannot control chronology
         AddData(sdata);
 }
 // this algo loop in the order of input
 // will come back to the policy chronology of this, depending of the order given in input
 Order( b_chrono );
 // Set Dates after reordering
 SetDates();
 return 0;
}

template <
  class T,
  template <class> class PolicyChronologic
>
void ListDataStock<T,PolicyChronologic>::PrintData ( const unsigned int verbose ) const
{
        using namespace TimeScale;

    std::cout << "\tname: " << GetName() << " size: " << GetSize() << \
                        " ,capacity: " << Capacity() << policy() << std::endl;
    std::cout << "\ttimescale: " << GetTimeScale() << std::endl;
    std::cout << "\tfullname: " << GetFullName() << std::endl;
    std::cout << "\tdates: " << Utils::Time_tToString( GetFirstDate(), true )
              << " " << Utils::Time_tToString( GetLastDate(), true ) << std::endl;
    // check size, better here than in Stock
    if ( ListData.size() == 0 ) {
        std::cout << "\tList is empty" << std::endl;
        return;
    }
    // if 0 (default) only first and last
    if ( verbose == 0 ) {
      std::cout << "\t  First: " << std::endl;
      this->ChronologicAt(0).PrintData( 1 );
    
    //if ( verbose == 0 )
      std::cout << "\t   Last: " << std::endl;        
      this->ChronologicAt( ListData.size()-1 ).PrintData( 1 );
    }

    // print all the data, no header 
    if ( verbose == 1 ) {
        std::cout << "Will call ListDataStock::PrintData(0), no header in loop " << std::endl;
        for (unsigned int i =0; i < ListData.size(); ++i ) {
                this->ChronologicAt( i ).PrintData( 0 );
        }
    }
    return;
}


namespace ListDataStock_Utils {


// here possible to extend for use of M5 to... H1 or M5 to DAY as well
// make a copy

// functions called in every loop :
//  // here ref certainly good as well
//  new_candle.AdditionData( NoChronologicAt( i ) );
//  here ref NoChrono is certainly good
//  tmp_volume += in_vol->NoChronologicAt ( i ).GetValue();
//
//  if candle changed:
//    new_candle.SetOpen( NoChronologicAt( i-1 ).GetOpen() );
//    new_candle.SetDate( NoChronologicAt( i-1 ).GetDate());
//    new_vol.SetData ( new_candle.GetDate(), tmp_volume );
//    //copy data to the new list, ref is ok, because makes a copy
//    out_csv.AddData( new_candle );
//    out_vol.AddData( new_vol );
//    //Avoid reconstruction of object, Reset not bad..
//    //new_candle.SetData(0,0.,0.,100000.,0.); / new_candle.Reset();
//    new_vol.SetData(0,0.);
template <
  template <class> class Policy1,
  template <class> class Policy2,
  template <class> class Policy3,
  template <class> class Policy4
>
int TransformTimeScale( const ETime new_tmscl, const ListDataStock<DataCSV,Policy1> & list_csv_in,
      const ListDataStock<SimpleData,Policy2> & list_vol_in, 
      ListDataStock<DataCSV,Policy3> & list_csv_out, ListDataStock<SimpleData,Policy4> & list_vol_out  )
{
  // to use operator << Etime
  using namespace TimeScale;

#ifdef DEBUG_CPP
  std::cout << "\tTransformTimeScale, input DataCSV\n" << std::endl;
#endif

  unsigned int size_input_csv= list_csv_in.GetSize();

  // pass all as run time check, makes sense because friend, public function
  // assert first
  //assert ( list_csv_in.GetSize() == list_vol_in.GetSize() );
  //assert ( new_tmscl != ETime::not_a_time );
  // pass without forcing int made as
  //assert ( new_tmscl > ETime::DAY );

  // Cannot reproduce results when using WEEK intermediates, works for others
  //assert ( list_csv_in.GetTimeScale() != ETime::WEEK );
  //assert ( list_vol_in.GetTimeScale() != ETime::WEEK );

  if ( list_csv_in.GetTimeScale() != list_vol_in.GetTimeScale() ) {
          std::cout << "Error, TimeScale of the input are different, return 1 " << std::endl;
          return 1;
  }

  // to set correct name and timescale
  std::string shortname_csv = ShortName::MakeShortName( Indicator::StockCSV );
  std::string shortname_vol = ShortName::MakeShortName( Indicator::Volume );

  // check input, WEEK forbidden by assert now
  // better warning, if WEEK
  if ( list_csv_in.GetTimeScale() == ETime::WEEK )
          std::cout << "Warning, Transform TimeScale from WEEK leads to different results " << std::endl;

  if ( ( new_tmscl < ETime::M5 ) || ( new_tmscl == ETime::not_a_time ) ) {
          std::cout << "Error, new tmscl must be >= ETime::M5, your input: " << new_tmscl << std::endl;
          return 1;
  }

  if ( list_csv_in.GetSize() != list_vol_in.GetSize() ) {
          std::cout << "Error, Input lists must have the same size " << std::endl;
          return 1;
  }

  // Set or reset correct TimeScale and Name if not correct
  if ( list_csv_out.GetName() != shortname_csv ) {
          // not a warning if empty
          if ( ! list_csv_out.GetName().empty() )
                  std::cout << "Warning Set correct name for list_csv_out " << shortname_csv << std::endl;
          list_csv_out.SetName( shortname_csv );
  }

  if ( list_csv_out.GetTimeScale() != new_tmscl ) {
          // not a warning if not_a_time
          if ( list_csv_out.GetTimeScale() != ETime::not_a_time )
                  // namespace TimeScale to get << ETime, but creates ambiguous with some other as_uint(EDS) not so logic ??
                  // use namespace in function only, ok
                  //std::cout << "Warning Set correct TimeScale to list_csv_out " << TimeScale::GetTimeScaleName(new_tmscl) << std::endl;
                  std::cout << "Warning Set correct TimeScale to list_csv_out " << new_tmscl << std::endl;
          // not an error if not_a_time, no warning to print
          list_csv_out.SetTimeScale( new_tmscl );
  }

  if ( list_vol_out.GetName() != shortname_vol ) {
          // not a warning if empty
          if ( ! list_vol_out.GetName().empty() )
                  std::cout << "Warning Set correct name for list_vol_out " << shortname_vol << std::endl;
          list_vol_out.SetName( shortname_vol );
  }

  if ( list_vol_out.GetTimeScale() != new_tmscl ) {
          // not a warning if not_a_time
      if ( list_vol_out.GetTimeScale() != ETime::not_a_time )
                  std::cout << "Warning Set correct TimeScale to list_csv_out " << new_tmscl << std::endl;
          // not an error if not_a_time, no warning to print
          list_vol_out.SetTimeScale( new_tmscl );
  }

  // used to save previous week, month....
  char tmp_str[20];
  bool change_candle= false;
  struct tm time_tm;

  // to be sure than a change_candle is first true
  // need to track hour and date at the minimum, if server stops for a small time
  // change order week:day:hour:minute
  int prev_value[3]={-1,-1,-1};
  // for larger timescale, no problem of missing some datas
  int previous_value = -1;

  //create temporary data, will be copy and added in the loop
  DataCSV new_candle;
  SimpleData new_vol;
  //for updating tmp_volume correctly in ListDataStock<DataCSV>, if day must sum
  // typedef defined in DataStock.h float/double
  type_value_data tmp_volume=0;

  // Reserve as much as the original data, swap could be called at the end
  // really better ? could optimize depending of the transformation
  list_csv_out.Reserve(size_input_csv);
  list_vol_out.Reserve(size_input_csv);

  // Set first candle, get a full copy of the last one: some values are correct, other will be set when close
  new_candle = list_csv_in.NoChronologicAt( 0 );
  
  //give the first value to sum of volume
  tmp_volume = list_vol_in.NoChronologicAt( 0 ).GetValue();
  // date is correct with this, but not needed
  //new_vol = in_vol->NoChronologicAt( 0 )
  
  // Utils::GetDateYMD, need really to check (if shift -1 for month)
  // tm structure to get  minute/hour/day/WEEK/month/year number
  // could use a tm structure to store previous, same logic
  time_tm = new_candle.GetDateTm();

  // changed to day, hour, min
  prev_value[1] = time_tm.tm_hour;
  // here as well problem with first value, need case
  // in case H1 only
  prev_value[2] = time_tm.tm_min;
  // test
  prev_value[0] = time_tm.tm_mday;

  switch ( new_tmscl ) {
         //case ETime::M5 : prev_value[1] = time_tm.tm_min /5; break;
     case ETime::M10 : prev_value[2] = time_tm.tm_min /10; break;
     case ETime::M30 : prev_value[2] = time_tm.tm_min /30; break;
     // forgot this case ? in purpose ? yes always assigned before the switch
     // need for the switch
     case ETime::H1  : prev_value[1] = time_tm.tm_hour; break;
    //case ETime::DAY : prev_value=time_tm.tm_mday; break; same before the switch
    case ETime::WEEK :
      strftime ( tmp_str,3,"%W", &time_tm);
      previous_value=atoi(tmp_str); break;

    case ETime::MONTH : previous_value = time_tm.tm_mon; break;
    case ETime::TRIM : previous_value = (int)(time_tm.tm_mon/3); break;
    case ETime::YEAR : previous_value = time_tm.tm_year; break;
    default: std::cout << "Error switch new_tmscl, return 1" << std::endl; return 1;
  }

  // ok correct here
  //std::cout << "first prev value " << prev_value << std::endl;


  // algo should be independent of order, now always fill in Non-chronologic order
  // start at 1, but zero not completely filled
  // if candle changed, use info of the previous (i-1)
  for (unsigned int i=1; i< size_input_csv; i++) {
      //std::cout << "Date: " << NoChronologicAt(i).GetDateString() << std::endl;
      //std::cout << "loop i: " << i << std::endl;
      
      // new_tmscl is fixed here, always the same case, 
      // template better ?? or function pointer ?? function in DataStock ??
      // template function bool test_new_candle<ETime>( time_tm, &prev_value, &new_value ),
      //                   new_value test_new_candle<ETime>( time_tm, &prev_value
      // and Template will optimize the switch ? (avoid to write explicit specialization)
      // dispatch compile time solutions:
      // http://stackoverflow.com/questions/1384326/c-optimize-using-templates-variables
      // always same test, could avoid switch inside the loop
      // template with enums
      // http://stackoverflow.com/questions/20289397/add-method-to-class-by-template-parameter
      // pointer function

        
      time_tm = list_csv_in.NoChronologicAt ( i ).GetDateTm();

      // check from struct tm if a new candle must be opened
      // here assumes all data are available
      // no extra tests (test of year if we are dealing with week...)
      switch ( new_tmscl ) {
            // M10
                        case ETime::M10 :
                                //if ( (minute/10) != prev_value[1] ) {
                                if ( ( time_tm.tm_min/10) != prev_value[2] ) {
                                        change_candle=true;
                                        //prev_value[1]= minute/10;
                                        prev_value[2] = time_tm.tm_min / 10;
                                }
                                break;
                        // M30
                        case ETime::M30 :
                                //std::cout << "timescale 3 min/30=" <<minute/30 << std::endl;
                                //if ( (minute/30) != prev_value[1] ) {
                                if ( ( time_tm.tm_min/30) != prev_value[2] ) {
                                        change_candle=true;
                                        //prev_value[1]= minute/30;
                                        prev_value[2] = time_tm.tm_min / 30 ;
                                }
                                break;
                        // H1
                        case ETime::H1 :
                                // changed from 0 to 1
                                if ( time_tm.tm_hour != prev_value[1]) {
                                        change_candle=true;
                                        prev_value[1] = time_tm.tm_hour;
                                }
                                break;

          // day, test when we need a new candle, will appear if we store hours from server
          /*
          case ETime::DAY :
              if ( time_tm.tm_mday != prev_value ) {
                change_candle=true;
                prev_value=time_tm.tm_mday;
              }
              break;
          */
          case ETime::WEEK :
              // no week information in time_tm ??
              strftime ( tmp_str,3,"%W", &time_tm);
              if ( atoi(tmp_str) != previous_value ) {
                change_candle=true;
                previous_value=atoi(tmp_str);
              }
              break;
          case ETime::MONTH :
              if ( time_tm.tm_mon != previous_value ) {
                change_candle=true;
                previous_value = time_tm.tm_mon;
              }
              break;
          case ETime::TRIM :
              if ( (int)(time_tm.tm_mon/3) != previous_value ) {
                  change_candle=true;
                  previous_value = (int)(time_tm.tm_mon/3);
              }
              break;
          case ETime::YEAR :
              if ( time_tm.tm_year != previous_value ) {
                  change_candle = true;
                  previous_value = time_tm.tm_year;
              }
              break;
          // check as run time, sure cannot happen ?
          default:
              std::cout << "Error in TimeScale Default Switch in TransformTimeScale " << std::endl;
              return 1;
      } // end switch

      if ( change_candle ) {
          //std::cout << " Change Candle" << std::endl;
          // set next loop to false by default
          change_candle = false;

          // here assure last values are correct, could be here or when AdditionData
          // Set date and open, always coming from the previous data 
          new_candle.SetOpen( list_csv_in.NoChronologicAt( i-1 ).GetOpen() );
          //close previous, give the date open of opening, previous one
          new_candle.SetDate( list_csv_in.NoChronologicAt( i-1 ).GetDate());
          // for volume, set date and the sum of volume
          new_vol.SetData ( new_candle.GetDate(), tmp_volume );
          // copy data to the lists
          list_csv_out.AddData( new_candle );
          list_vol_out.AddData( new_vol );

          // setup new candle and volume
          // avoid a call to Reset() in making a copy of all, low and high for sure correct
          // Open and date are wrong, but will be updated before adding to vector
          new_candle = list_csv_in.NoChronologicAt( i );
          //std::cout << "SetClose " << new_candle.GetClose() << "i=" << i << std::endl;
          tmp_volume = 0;
      }
      // always update the candle and volume, modify only High and Low
      new_candle.AdditionData( list_csv_in.NoChronologicAt( i ) );
      // volume must be sum up in this case
      tmp_volume += list_vol_in.NoChronologicAt ( i ).GetValue();
  } //end for loop

  // last candle never saved
  new_candle.SetOpen( list_csv_in.NoChronologicAt( size_input_csv-1 ).GetOpen() );
  new_candle.SetDate( list_csv_in.NoChronologicAt( size_input_csv-1 ).GetDate());
  // same for volume
  new_vol.SetData( new_candle.GetDate(), tmp_volume); 
  // append the last candle
  list_csv_out.AddData( new_candle );
  list_vol_out.AddData( new_vol );

  // need to reorder in the correct policy
  // has been filled in non-chronologic order, maybe need to reorder
  // possible to combine with swap ?
  list_csv_out.Order( EPChrono::NO_CHRONO );
  list_vol_out.Order( EPChrono::NO_CHRONO );
  
  // assign the new dates from the data or from the input ?
  // use initial candle, maybe wrong if later update
  //out_csv.SetDates( dates.first, dates.last );
  //out_vol.SetDates( dates.first, dates.last );
  // not exactly the same date in this case, test this new implementation
  // here real date of the new candles
  list_csv_out.SetDates();
  list_vol_out.SetDates();

  // Swap efficient ? to reduce capacity.
  // may need to test on the timescale, for M10 wants to keep some space
  list_csv_out.Swap();
  list_vol_out.Swap();
  return 0;
}

template <
  template <class> class Policy1,
  template <class> class Policy2,
  template <class> class Policy3,
  template <class> class Policy4
>
int TransformTimeScale( const ETime new_tmscl, const ListDataStock<SimpleData,Policy1> & list_dcsv_in,
      const ListDataStock<SimpleData,Policy2> & list_vol_in, 
      ListDataStock<DataCSV,Policy3> & list_csv_out, ListDataStock<SimpleData,Policy4> & list_vol_out  )
{
  using namespace TimeScale;

  unsigned int size_input_csv = list_dcsv_in.GetSize();

#ifdef DEBUG_CPP
  std::cout << "\n\tTransformTimeScale with input SimpleData\n" << std::endl;
  std::cout << "Size ListData input " << size_input_csv << std::endl;
  //std::cout << "Test direct GetName, GetSize " << GetName() << " " << GetSize() << std::endl;
  //std::cout << "TestSize new Vol " << size_input_vol << std::endl;
#endif

  // to set correct name and timescale
  std::string shortname_csv = ShortName::MakeShortName( Indicator::StockCSV );
  std::string shortname_vol = ShortName::MakeShortName( Indicator::Volume );

  // error ? maybe works, check StockCSV ?? also could test not_a_time also
  if ( ( list_dcsv_in.GetTimeScale() != ETime::INST ) ||
           ( list_vol_in.GetTimeScale() != ETime::INST ) ) {
          std::cout << "Error, Input must be a ETime::INST" << std::endl;
          return 1;
  }

  if ( ( new_tmscl >= ETime::DAY ) || ( new_tmscl == ETime::not_a_time ) ) {
          std::cout << "Error, new tmscl must be < ETime::DAY, your input: " << new_tmscl << std::endl;
          return 1;
  }

  if ( list_dcsv_in.GetSize() != list_vol_in.GetSize() ) {
        std::cout << "Error, Input lists must have the same size " << std::endl;
        return 1;
  }

  // Set or reset correct TimeScale and Name if not correct
  if ( list_csv_out.GetName() != shortname_csv ) {
          // not a warning if empty
          if ( ! list_csv_out.GetName().empty() )
                  std::cout << "Warning Set correct name for list_csv_out " << shortname_csv << std::endl;
          list_csv_out.SetName( shortname_csv );
  }

  if ( list_csv_out.GetTimeScale() != new_tmscl ) {
          // not a warning if not_a_time
          if ( list_csv_out.GetTimeScale() != ETime::not_a_time )
                  std::cout << "Warning Set correct TimeScale to list_csv_out " << new_tmscl << std::endl;
          // not an error if not_a_time, no warning to print
          list_csv_out.SetTimeScale( new_tmscl );
  }

  if ( list_vol_out.GetName() != shortname_vol ) {
          // not a warning if empty
          if ( ! list_vol_out.GetName().empty() )
                  std::cout << "Warning Set correct name for list_vol_out " << shortname_vol << std::endl;
          list_vol_out.SetName( shortname_vol );
  }

  if ( list_vol_out.GetTimeScale() != new_tmscl ) {
          // not a warning if not_a_time
          if ( list_vol_out.GetTimeScale() != ETime::not_a_time )
                  std::cout << "Warning Set correct TimeScale to list_csv_out " << new_tmscl << std::endl;
                  // not an error if not_a_time, no warning to print
          list_vol_out.SetTimeScale( new_tmscl );
  }


  // reserve enough space
  list_csv_out.Reserve(size_input_csv);
  list_vol_out.Reserve(size_input_csv);

  bool change_candle=false;
  
  int minute; //use for calcul, could be deleted ?
  struct tm time_tm;
  // to be sure than a change_candle is first true
  // keep hour:minute:day (day to avoid problem between missing data I guess)
  int prev_value[3]={-1,-1,-1};

  //create temporary data, will be copy and added in the loop
  DataCSV new_candle;
  SimpleData new_vol;

  //for updating tmp_volume correctly, here use difference
  // typedef defined in DataStock.h float/double
  type_value_data tmp_volume=0;

  // keep same logic non-chronologic, open new_candle
  tmp_volume = list_vol_in.NoChronologicAt(0).GetValue();

  new_candle.SetDate ( list_dcsv_in.NoChronologicAt( 0 ).GetDate() );
  new_candle.SetClose( list_dcsv_in.NoChronologicAt( 0 ).GetValue() );
  // set all explicitely, cannot use assignment because different types here
  new_candle.SetHigh ( list_dcsv_in.NoChronologicAt( 0 ).GetValue() );
  new_candle.SetLow  ( list_dcsv_in.NoChronologicAt( 0 ).GetValue() );

  // set prev_value for all cases
  time_tm = new_candle.GetDateTm();
  // not logic the order, better min, hour, day
  prev_value[0] = time_tm.tm_hour;
  // here as well problem with first value, need case
  // in case H1 only
  prev_value[1] = time_tm.tm_min;
  switch ( new_tmscl ) {
    case ETime::M5 : prev_value[1] = time_tm.tm_min /5; break;
    case ETime::M10 : prev_value[1] = time_tm.tm_min /10; break;
    case ETime::M30 : prev_value[1] = time_tm.tm_min /30; break;
    // forgot this case ? in purpose ? yes always assigned before the switch
    //case ETime::H1  :
    default : break;
  }
  prev_value[2] = time_tm.tm_mday;

  //std::cout << "first candle update tmp_volume " << tmp_volume << std::endl;
  //std::cout << "new candle GetClose() " << new_candle.GetClose() << std::endl;
  //std::cout << "Date: " << NoChronologicAt( 0 ).GetDateString() << " tmp_volume "<< tmp_volume  << std::endl;
  //std::cout << "begin loop " << std::endl;

  // output non-chronologic ? certainly in the output, can reorder after
  for ( unsigned int i = 1 ; i < size_input_csv  ; ++i ) {

      time_tm = list_dcsv_in.NoChronologicAt( i ).GetDateTm();
      //std::cout << "Date: " << NoChronologicAt( i ).GetDateString() << " "
      //    << in_vol->NoChronologicAt( i ).GetValue() << std::endl ;
      //use a modulo for bool to change candle
      minute = time_tm.tm_min;

      // 0 added: dcvs, cannot change to this, begin to 1
      //default MANDATORY otherwise take value ???? bug /???
      switch (new_tmscl) {
          // M5
          case ETime::M5 :
              // change prev store the real minute
              // time_tm.tm_min/5
              if ( (minute/5) != prev_value[1] ) {
                  //std::cout << " M5 change (minute/5)= " << (minute/5) << " prev_value[1]/5 " << prev_value[1]/5 << std::endl;
                  change_candle=true;
                  prev_value[1] = minute/5 ;
              }
              break;
          // M10
          case ETime::M10 :
              if ( (minute/10) != prev_value[1] ) {
                  change_candle=true;
                  prev_value[1]= minute/10;
              }
              break;
          // M30
          case ETime::M30 :
              //std::cout << "timescale 3 min/30=" <<minute/30 << std::endl;
              if ( (minute/30) != prev_value[1] ) {
                  change_candle=true;
                  prev_value[1]= minute/30;
              }
              break;
          // H1
          case ETime::H1 :
              if ( time_tm.tm_hour != prev_value[0]) {
                  change_candle=true;
                  prev_value[0] = time_tm.tm_hour;
              }
              break;

          //to write other, more or egal than 1 day
          default:
              std::cout << "Error in switch timescale, return 1" << std::endl;
              //break;
              return 1;
      } //end switch
      //all case, check day also !
      // possible change day, but minute or hour did not change and prev_value not updated
      // seems fine, to make unit_test
      if ( time_tm.tm_mday != prev_value[2]) {
          //std::cout << "change day" << std::endl;
          change_candle=true;
          prev_value[2] = time_tm.tm_mday;
      }

      if ( change_candle ) {
          //std::cout << " Change Candle" << std::endl;
          change_candle = false;

          // close the candle to the previous value
          new_candle.SetOpen ( list_dcsv_in.NoChronologicAt(i-1).GetValue() );
          //std::cout << "SetOpen " << new_candle.GetOpen() << std::endl; //GetDateString()
          // assign date get previous one, still ok if i==1
          new_candle.SetDate( list_dcsv_in.NoChronologicAt(i-1).GetDate() );
          // update volume, here makes the difference
          new_vol.SetData ( new_candle.GetDate(), tmp_volume - (list_vol_in.NoChronologicAt(i-1).GetValue()) );

          // save in output listdata
          list_csv_out.AddData( new_candle);
          list_vol_out.AddData( new_vol);

          // clear the min max of candle, or set explicit
          //new_candle.Reset();
          new_candle.SetHigh ( list_dcsv_in.NoChronologicAt( i ).GetValue() );
          new_candle.SetLow  ( list_dcsv_in.NoChronologicAt( i ).GetValue() );
          new_candle.SetClose( list_dcsv_in.NoChronologicAt( i ).GetValue() );
          // initialze volume, here always last
          tmp_volume = list_vol_in.NoChronologicAt(i-1).GetValue();

      } // end change_candle
      // all case update min-max values
      new_candle.AdditionData ( list_dcsv_in.NoChronologicAt( i ) );
      // nothing to do for volume
  } // end loop i

  // must close last candle
  new_candle.SetOpen ( list_dcsv_in.NoChronologicAt( size_input_csv -1 ).GetValue() );
  new_candle.SetDate( list_dcsv_in.NoChronologicAt( size_input_csv -1 ).GetDate() );
  // tmp_volume initialized only when opened, tmp_volume not really needed
  new_vol.SetData ( new_candle.GetDate(),
                    tmp_volume - (list_vol_in.NoChronologicAt(size_input_csv -1).GetValue()) );

  // add the last data
  list_csv_out.AddData( new_candle );
  list_vol_out.AddData( new_vol );
  
  // has been filled in non chronologic order, maybe need to reorder
  list_csv_out.Order( EPChrono::NO_CHRONO );
  list_vol_out.Order( EPChrono::NO_CHRONO );

  // may swap also here, but may need to extend regularly

  // assign the new dates from the CSV or dcsv
  // in fact the last is interesting, want the real last date
  //out_csv.SetDates( dates.first, dates.last );
  //out_vol.SetDates( dates.first, dates.last );
  list_csv_out.SetDates();
  list_vol_out.SetDates();

#ifdef DEBUG_CPP
  std::cout << "TestSize out_csv " << list_csv_out.GetSize() << std::endl;
  std::cout << "TestSize out_vol " <<  list_vol_out.GetSize() << std::endl;
#endif
  return 0;
}

} //end namespace_Utils