Friday 31 July 2009

read and write data


   1:  using System;
   2:  using System.Collections.Generic;
   3:  using System.IO;
   4:  using System.Text;
   5:  using System.Windows.Forms;
   6:  namespace ReadData
   7:  {
   8:      public partial class Form1 : Form
   9:      {
  10:          static readonly StringBuilder outData = new StringBuilder();
  11:          readonly List<double> a = new List<double>(); // the 2 input TSs
  12:          readonly List<double> b = new List<double>();
  13:          public Form1()
  14:          {
  15:              InitializeComponent();
  16:          }
  17:          void Form1_Load(object sender, EventArgs e)
  18:          {
  19:              LoadData();
  20:              int totalFound = 0;
  21:              double lastDiff = 0;
  22:              bool haveEntry = false;
  23:              const int window = 100; // how many values we use to estimate mean and stdDev
  24:              for (int i = window; i <= a.Count; i++)
  25:              //for (int i = 144; i < 145; i++) // one sample only, for testing
  26:              {
  27:                  double meanA = GetMean(a, i - window, i);
  28:                  double meanB = GetMean(b, i - window, i);
  29:                  double stdA = GetStd(a, i - window, i, meanA);
  30:                  double stdB = GetStd(b, i - window, i, meanB);
  31:                  double aNorm = (a[i - 1] - meanA) / stdA;
  32:                  double bNorm = (b[i - 1] - meanB) / stdB;
  33:                  double diff = aNorm - bNorm;
  34:                  double absDiff = Math.Abs(diff);
  35:                  if (haveEntry &amp;&amp; ((lastDiff > 1 &amp;&amp; diff <> -1)))
  36:                  {
  37:                      Console.WriteLine("==== Exit index " + i);
  38:                      haveEntry = false;
  39:                  }
  40:                  //outData.AppendLine(i + ", " + diff); // for writing to file
  41:                  // outData.AppendLine(diff.ToString()); // for writing to file
  42:                  outData.AppendLine(string.Format("{0:d4}", i-window+1) + " " + diff); // for writing to file
  43:                  if (absDiff > 15)
  44:                  {
  45:                      totalFound++;
  46:                      Console.WriteLine("Found at index " + i + " diff " + diff);
  47:                      if (absDiff < Math.Abs(lastDiff) &amp;&amp; !haveEntry)
  48:                      {
  49:                          Console.WriteLine("==== Entry index " + i);
  50:                          haveEntry = true;
  51:                      }
  52:                  }
  53:                  lastDiff = diff;
  54:              }
  55:              Console.WriteLine("total found " + totalFound);
  56:              WriteData();
  57:          }
  58:          static void WriteData()
  59:          {
  60:              const string fileout = "out.csv";
  61:              if (File.Exists(fileout))
  62:              {
  63:                  File.Delete(fileout);
  64:              }
  65:              using (StreamWriter sw = new StreamWriter(fileout))
  66:              {
  67:                  sw.Write(outData.ToString());
  68:              }
  69:          }
  70:          static double GetStd(IList<double> array, int start, int limit, double mean)
  71:          {
  72:              double sum = 0;
  73:              for (int i = start; i < limit; i++)
  74:              {
  75:                  sum += (array[i] - mean) * (array[i] - mean);
  76:              }
  77:              sum = Math.Sqrt(sum);
  78:              sum /= (limit - start);
  79:              return sum;
  80:          }
  81:          static double GetMean(IList<double> array, int start, int limit)
  82:          {
  83:              double sum = 0;
  84:              for (int i = start; i < limit; i++)
  85:              {
  86:                  sum += array[i];
  87:              }
  88:              sum /= (limit - start);
  89:              return sum;
  90:          }
  91:          /// <summary>
  92:          /// Assume input file has 2 columns, one for each TS we want to track.
  93:          /// </summary>
  94:          void LoadData()
  95:          {
  96:              using (StreamReader sr = new StreamReader("5m-db.csv"))
  97:              {
  98:                  string newData;
  99:                  a.Clear();
 100:                  b.Clear();
 101:                  while (sr.Peek() != -1)
 102:                  {
 103:                      newData = sr.ReadLine();
 104:                      string[] line = newData.Split(',');
 105:                      a.Add(double.Parse(line[0]));
 106:                      b.Add(double.Parse(line[1]));
 107:                  }
 108:                  Console.WriteLine("Loaded " + a.Count + " data points.");
 109:              }
 110:          }
 111:      }

No comments: