将CSV格式的文件导入influxdb

现有csv格式文件

文件格式:

第一行是字段名,其中时间在第二列,第二行以后都是数值(...表示后续数据省略)

turbId,recTime,giWecRunContion,giFaultInformation,grRotorSpeedPDM,...

10004110,2018-07-20 23:54:36,3,0,10.994,488,-0.5 ...
10004110,2018-07-20 23:54:38,3,0,11.191,505,-0.5 ...

要将此文件导入到influxdb中

第一步,将此文件转化成influx导入要求的格式

第二步,用bat脚本进行导入。

 

第一步的实现

先读取csv到DataTable中,再对时间进行特殊处理,转化成unix nano second格式。

转换后的格式:

windFarm作为influxDB中的measurement, turbId作为tag key, 后面的作为field key 

windFarm,turbId=10004110 giWecRunContion="3",giFaultInformation="0",grRotorSpeedPDM="10.994",grCAN_GeneratorTorque="488",... simulate_PowerSetpointFlag="1" 1532102076000000000
windFarm,turbId=10004110 giWecRunContion="3",giFaultInformation="0",grRotorSpeedPDM="9.24",grCAN_GeneratorTorque="488",... ,simulate_PowerSetpointFlag="1" 1532102082000000000

具体代码:

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Data;


namespace influxRW
{
    class Program
    {
        static void Main(string[] args)
        {
            DataTable dt = OpenCSV("10004110_20180721.csv");
            try
            {
                SaveData(dt, "data.txt");
                SaveDataNotString(dt, "data-notstring.txt");
            }
            catch
            {

            }

            Console.WriteLine("写入完成");
            Console.ReadKey();
        }

        ///   
        /// DateTime时间格式转换为Unix时间戳格式  
        ///   
        ///  DateTime时间格式  
        /// Unix时间戳格式  
        public static int ConvertDateTimeInt(System.DateTime time)
        {
            System.DateTime startTime = TimeZone.CurrentTimeZone.ToLocalTime(new System.DateTime(1970, 1, 1));
            return (int)(time - startTime).TotalSeconds;
        }


        public static DataTable OpenCSV(string filePath)//从csv读取数据返回table
        {
            System.Text.Encoding encoding = Encoding.ASCII;//GetType(filePath); //
            DataTable dt = new DataTable();
            System.IO.FileStream fs = new System.IO.FileStream(filePath, System.IO.FileMode.Open,
                System.IO.FileAccess.Read);

            System.IO.StreamReader sr = new System.IO.StreamReader(fs, encoding);

            //记录每次读取的一行记录
            string strLine = "";
            //记录每行记录中的各字段内容
            string[] aryLine = null;
            string[] tableHead = null;
            //标示列数
            int columnCount = 0;
            //标示是否是读取的第一行
            bool IsFirst = true;
            //逐行读取CSV中的数据
            while ((strLine = sr.ReadLine()) != null)
            {
                if (IsFirst == true)
                {
                    tableHead = strLine.Split(',');
                    IsFirst = false;
                    columnCount = tableHead.Length;
                    //创建列
                    for (int i = 0; i < columnCount; i++)
                    {
                        DataColumn dc = new DataColumn(tableHead[i]);
                        dt.Columns.Add(dc);
                    }
                }
                else
                {
                    aryLine = strLine.Split(',');
                    DataRow dr = dt.NewRow();
                    for (int j = 0; j < columnCount; j++)
                    {
                        dr[j] = aryLine[j];
                    }
                    dt.Rows.Add(dr);
                }
            }
            if (aryLine != null && aryLine.Length > 0)
            {
                dt.DefaultView.Sort = tableHead[0] + " " + "asc";
            }

            sr.Close();
            fs.Close();
            return dt;
        }

        public static void SaveData(DataTable dt, string fullPath)//table数据写入data
        {
            System.IO.FileInfo fi = new System.IO.FileInfo(fullPath);
            if (!fi.Directory.Exists)
            {
                fi.Directory.Create();
            }
            System.IO.FileStream fs = new System.IO.FileStream(fullPath, System.IO.FileMode.Create,
                System.IO.FileAccess.Write);
            System.IO.StreamWriter sw = new System.IO.StreamWriter(fs, System.Text.Encoding.ASCII);
            string data = "";

            for (int i = 0; i < dt.Rows.Count; i++) //写入各行数据
            {
                data = "";
                data += "windFarm" + ",";
                data += dt.Columns[0].ColumnName +"="+ dt.Rows[i][0].ToString();
                string s = dt.Rows[i][1].ToString();//时间单元格内容
                DateTime time = Convert.ToDateTime(s);
                string unixtime = ConvertDateTimeInt(time).ToString() + "000000000";
                data += " ";


                for (int j = 2; j < dt.Columns.Count; j++)
                {
                    string temp = dt.Columns[j].ColumnName + "=" + "\"" + dt.Rows[i][j].ToString() + "\"";
                 
                    if (j < dt.Columns.Count - 1)
                    {
                        temp += ",";
                    }
                    data += temp;
                }

                data += " "+ unixtime;
                sw.WriteLine(data);
                   
            }
            sw.Close();
            fs.Close();
        }


        public static void SaveDataNotString(DataTable dt, string fullPath)//table数据写入data
        {
            System.IO.FileInfo fi = new System.IO.FileInfo(fullPath);
            if (!fi.Directory.Exists)
            {
                fi.Directory.Create();
            }
            System.IO.FileStream fs = new System.IO.FileStream(fullPath, System.IO.FileMode.Create,
                System.IO.FileAccess.Write);
            System.IO.StreamWriter sw = new System.IO.StreamWriter(fs, System.Text.Encoding.ASCII);
            string data = "";

            for (int i = 0; i < dt.Rows.Count; i++) //写入各行数据
            {
                data = "";
                //windFarm,turbId="10004110" giWecRunContion="3",
                //giFaultInformation="0",grRotorSpeedPDM="10.994" 1533022504000000000
                data += "windFarm" + ",";
                data += dt.Columns[0].ColumnName + "=" + dt.Rows[i][0].ToString();
                string s = dt.Rows[i][1].ToString();//时间单元格内容
                DateTime time = Convert.ToDateTime(s);
                string unixtime = ConvertDateTimeInt(time).ToString() + "000000000";
                data += " ";


                for (int j = 2; j < dt.Columns.Count; j++)
                {
                    string temp = dt.Columns[j].ColumnName + "="  + dt.Rows[i][j].ToString() ;

                    if (j < dt.Columns.Count - 1)
                    {
                        temp += ",";
                    }
                    data += temp;
                }

                data += " " + unixtime;
                sw.WriteLine(data);

            }
            sw.Close();
            fs.Close();
        }


 
    }
}

第二步,用bat文件导入influxdb

 

上一步生成的文件名称为data-2.txt(以\n结尾),新建一个up.bat,内容如下:

for /f "delims=@" %%i in (data-2.txt) do curl -i -XPOST "http://localhost:8086/write?db=db20053" --data-binary "%%i"

下载curl.exe放到当前目录下,双击运行up.bat,即可实现按行导入数据到influxDB中。

 

 

实际测试

导入速度约为每秒3条,有点低,主要是一行数据点较多,有128个。

后续可以考虑实现一次写入多行数据,提高数据写入效率。

 

你可能感兴趣的:(influxDB)