//+------------------------------------------------------------------+
//|                                    Meta COT: Sript Agregator.mq4 |
//|   Copyright © 2010, C-4 (Vasiliy Sokolov, Russia, St.-Petersburg,|
//|                                2009, 2010), All Rights Reserved. |
//|                                                   vs-box@mail.ru |
//+------------------------------------------------------------------+
//

//#property show_inputs
#property copyright "Copyright © 2010, C-4 (Vasiliy Sokolov, SPb), All Rights Reserved."
#property link      "vs-box@mail.ru"

#property show_inputs
//Àññîöèàöèè
#define Market_and_Exchange_Names 1
#define As_of_Date_in_Form_YYMMDD 2
#define As_of_Date_in_Form_YYYY_MM_DD 3
#define CFTC_Contract_Market_Code 4
#define CFTC_Market_Code_in_Initials 5
#define CFTC_Region_Code 6
#define CFTC_Commodity_Code 7
#define Open_Interest_All 8
#define Nonc_Positions_Long_All 9
#define Nonc_Positions_Short_All 10
#define Nonc_Positions_Spreading_All 11
#define Commercial_Positions_Long_All 12
#define Commercial_Positions_Short_All 13
#define Total_Reportable_Pos_Long 14
#define Total_Reportable_Pos_Short 15
#define Nonrep_Positions_Long_All 16
#define Nonrep_Positions_Short_All 17

extern string name_list="names.ini";
//string normalize_name;
 int a;
int start()
{
   int CFileNames,
       cot_file,
       tools_data;
   
   int column,
       column_string;
   int cot;
   bool IsOldInfo=true;
   string name_cotfile;
   string data, data2,
          normalize_name,
          string_cot;
   CFileNames=FileOpen(name_list,FILE_READ|FILE_CSV);
   if(TrueFileName(CFileNames, name_list)==false)return(1);
   while(FileIsEnding(CFileNames)==false){
      cot++;
      name_cotfile=FileReadString(CFileNames);
      cot_file=FileOpen(name_cotfile,FILE_READ|FILE_CSV, ",");
      if(TrueFileName(cot_file, name_cotfile)==false)return(1);
      else Print("Parsing " + name_cotfile + "...");
      // Skip first string
      while(FileIsLineEnding(cot_file)==false){
         column++;
         FileReadString(cot_file);
      }
      
      while(FileIsEnding(cot_file)==false){
         data=FileReadString(cot_file);
         column_string++;
         if(FileIsLineEnding(cot_file)==true||FileIsEnding(cot_file)==true){
            column_string=0;
            FileWrite(tools_data, string_cot);
            FileClose(tools_data);
            string_cot="";     
         }
         else{
            if(TrueDataNew(column_string)==true){//Here check superfluous comma in file
               data=NormalizeData(data);
               if(column_string==Market_and_Exchange_Names){   
                  
                  data2=FileReadString(cot_file);        // 
                  if(CheckValidColumn(data2)==true)      //
                     data=data+NormalizeColumn(data2);   //
                  else column_string++;
                  //data=NormalizeNames(data);
                  tools_data=FileOpen("COT - "+data+".csv",FILE_READ|FILE_WRITE|FILE_CSV);
                  TrueFileName(tools_data, data);
                  FileSeek(tools_data, 0, SEEK_END);
                  string_cot=string_cot+data;
               }
               else{ 
                  if(column_string==As_of_Date_in_Form_YYYY_MM_DD)data=ConvertData(data);
                  string_cot=string_cot+";"+data;
               }
            }
         }
      }
   }
   Print("Parsing completed. Check the new files and run concatenate script.");
   return(0);
}

string ConvertData(string data)
{
   string c_data="";
   int strlen=StringLen(data);
   int charter;
   for(int i=0;i<strlen;i++){
      charter=StringGetChar(data,i);
      if(charter=='-')c_data=c_data+".";
      else c_data=StringConcatenate(c_data,CharToStr(charter));
   }
   return(c_data);
}


bool TrueDataNew(int column_string)
{
   switch(column_string){
      case Market_and_Exchange_Names:
      case As_of_Date_in_Form_YYYY_MM_DD:
      case Open_Interest_All:
      case Nonc_Positions_Long_All:
      case Nonc_Positions_Short_All:
      case Nonc_Positions_Spreading_All:
      case Commercial_Positions_Long_All:
      case Commercial_Positions_Short_All:
      case Total_Reportable_Pos_Long:
      case Total_Reportable_Pos_Short:
      case Nonrep_Positions_Long_All:
      case Nonrep_Positions_Short_All:
           return(true);
      default:
           return(false);
   }
}

string NormalizeData(string file_name)
{
   
   int charter;
   int strlen;
   string normalize_name="";
   string normalize_name_p="";
   strlen=StringLen(file_name);
   for(int i=0;i<strlen;i++){
      charter=StringGetChar(file_name,i);
      switch(charter){
         case '/':
         case '\\':
         case ':':
         case '*':
         case '\"':
         case '<':
         case '>':
         case '|':
         case ';':
            break;
         default: 
            normalize_name=StringConcatenate(normalize_name,CharToStr(charter));
      }
   }
   return(normalize_name);
}

string NormalizeNames(string data)
{
   int strlen=StringLen(data);
   data=StringTrimRight(data);
   if(CharToStr(StringGetChar(data,strlen-1))==".")
      StringSetChar(data,strlen-1,"");
   return(data);
}

bool TrueFileName(int handle, string filename)
{
   if(handle==-1){
      Print("The file has not been opened: ", filename, " Last error: ", GetLastError());
      return(false);
   }
   else return(true);  
}

bool CheckValidColumn(string column_name)
{
   int strlen=StringLen(column_name);
   if(StringFind(column_name, "\"")!=-1){a++;return(true);}
   return(false);
}

string NormalizeColumn(string column)
{
   string n_column;
   int charter;
   int strlen=StringLen(column);
   for(int i=0;i<strlen;i++){
      charter=StringGetChar(column,i);
      if(charter=='\"')continue;
      else n_column=StringConcatenate(n_column,CharToStr(charter));
   }
   return(n_column);
}

