将CSV格式的文件导入influxdb
现有csv格式文件
文件格式：
第一行是字段名，其中时间在第二列，第二行以后都是数值(...表示后续数据省略）
turbId,recTime,giWecRunContion,giFaultInformation,grRotorSpeedPDM,...
10004110,2018-07-20 23:54:36,3,0,10.994,488,-0.5 ...
10004110,2018-07-20 23:54:38,3,0,11.191,505,-0.5 ...要将此文件导入到influxdb中
第一步，将此文件转化成influx导入要求的格式
第二步，用bat脚本进行导入。
第一步的实现
先读取csv到DataTable中，再对时间进行特殊处理，转化成unix nano second格式。
转换后的格式：
windFarm作为influxDB中的measurement， turbId作为tag key, 后面的作为field key
windFarm,turbId=10004110 giWecRunContion="3",giFaultInformation="0",grRotorSpeedPDM="10.994",grCAN_GeneratorTorque="488",... simulate_PowerSetpointFlag="1" 1532102076000000000
windFarm,turbId=10004110 giWecRunContion="3",giFaultInformation="0",grRotorSpeedPDM="9.24",grCAN_GeneratorTorque="488",... ,simulate_PowerSetpointFlag="1" 1532102082000000000具体代码：
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Data;
namespace influxRW
{
class Program
{
static void Main(string[] args)
{
DataTable dt = OpenCSV("10004110_20180721.csv");
try
{
SaveData(dt, "data.txt");
SaveDataNotString(dt, "data-notstring.txt");
}
catch
{
}
Console.WriteLine("写入完成");
Console.ReadKey();
}
///
/// DateTime时间格式转换为Unix时间戳格式
///
/// DateTime时间格式
/// Unix时间戳格式
public static int ConvertDateTimeInt(System.DateTime time)
{
System.DateTime startTime = TimeZone.CurrentTimeZone.ToLocalTime(new System.DateTime(1970, 1, 1));
return (int)(time - startTime).TotalSeconds;
}
public static DataTable OpenCSV(string filePath)//从csv读取数据返回table
{
System.Text.Encoding encoding = Encoding.ASCII;//GetType(filePath); //
DataTable dt = new DataTable();
System.IO.FileStream fs = new System.IO.FileStream(filePath, System.IO.FileMode.Open,
System.IO.FileAccess.Read);
System.IO.StreamReader sr = new System.IO.StreamReader(fs, encoding);
//记录每次读取的一行记录
string strLine = "";
//记录每行记录中的各字段内容
string[] aryLine = null;
string[] tableHead = null;
//标示列数
int columnCount = 0;
//标示是否是读取的第一行
bool IsFirst = true;
//逐行读取CSV中的数据
while ((strLine = sr.ReadLine()) != null)
{
if (IsFirst == true)
{
tableHead = strLine.Split(',');
IsFirst = false;
columnCount = tableHead.Length;
//创建列
for (int i = 0; i < columnCount; i++)
{
DataColumn dc = new DataColumn(tableHead[i]);
dt.Columns.Add(dc);
}
}
else
{
aryLine = strLine.Split(',');
DataRow dr = dt.NewRow();
for (int j = 0; j < columnCount; j++)
{
dr[j] = aryLine[j];
}
dt.Rows.Add(dr);
}
}
if (aryLine != null && aryLine.Length > 0)
{
dt.DefaultView.Sort = tableHead[0] + " " + "asc";
}
sr.Close();
fs.Close();
return dt;
}
public static void SaveData(DataTable dt, string fullPath)//table数据写入data
{
System.IO.FileInfo fi = new System.IO.FileInfo(fullPath);
if (!fi.Directory.Exists)
{
fi.Directory.Create();
}
System.IO.FileStream fs = new System.IO.FileStream(fullPath, System.IO.FileMode.Create,
System.IO.FileAccess.Write);
System.IO.StreamWriter sw = new System.IO.StreamWriter(fs, System.Text.Encoding.ASCII);
string data = "";
for (int i = 0; i < dt.Rows.Count; i++) //写入各行数据
{
data = "";
data += "windFarm" + ",";
data += dt.Columns[0].ColumnName +"="+ dt.Rows[i][0].ToString();
string s = dt.Rows[i][1].ToString();//时间单元格内容
DateTime time = Convert.ToDateTime(s);
string unixtime = ConvertDateTimeInt(time).ToString() + "000000000";
data += " ";
for (int j = 2; j < dt.Columns.Count; j++)
{
string temp = dt.Columns[j].ColumnName + "=" + "\"" + dt.Rows[i][j].ToString() + "\"";
if (j < dt.Columns.Count - 1)
{
temp += ",";
}
data += temp;
}
data += " "+ unixtime;
sw.WriteLine(data);
}
sw.Close();
fs.Close();
}
public static void SaveDataNotString(DataTable dt, string fullPath)//table数据写入data
{
System.IO.FileInfo fi = new System.IO.FileInfo(fullPath);
if (!fi.Directory.Exists)
{
fi.Directory.Create();
}
System.IO.FileStream fs = new System.IO.FileStream(fullPath, System.IO.FileMode.Create,
System.IO.FileAccess.Write);
System.IO.StreamWriter sw = new System.IO.StreamWriter(fs, System.Text.Encoding.ASCII);
string data = "";
for (int i = 0; i < dt.Rows.Count; i++) //写入各行数据
{
data = "";
//windFarm,turbId="10004110" giWecRunContion="3",
//giFaultInformation="0",grRotorSpeedPDM="10.994" 1533022504000000000
data += "windFarm" + ",";
data += dt.Columns[0].ColumnName + "=" + dt.Rows[i][0].ToString();
string s = dt.Rows[i][1].ToString();//时间单元格内容
DateTime time = Convert.ToDateTime(s);
string unixtime = ConvertDateTimeInt(time).ToString() + "000000000";
data += " ";
for (int j = 2; j < dt.Columns.Count; j++)
{
string temp = dt.Columns[j].ColumnName + "=" + dt.Rows[i][j].ToString() ;
if (j < dt.Columns.Count - 1)
{
temp += ",";
}
data += temp;
}
data += " " + unixtime;
sw.WriteLine(data);
}
sw.Close();
fs.Close();
}
}
}
第二步，用bat文件导入influxdb
上一步生成的文件名称为data-2.txt（以\n结尾），新建一个up.bat，内容如下：
for /f "delims=@" %%i in (data-2.txt) do curl -i -XPOST "http://localhost:8086/write?db=db20053" --data-binary "%%i"下载curl.exe放到当前目录下，双击运行up.bat，即可实现按行导入数据到influxDB中。
实际测试
导入速度约为每秒3条，有点低，主要是一行数据点较多，有128个。
后续可以考虑实现一次写入多行数据，提高数据写入效率。