使用tensorflow处理csv文件
源代码
from numpy import genfromtxt
import numpy as np
import csv
from itertools import islice
import tensorflow as tf
SUMMARY_DIR = "log/slope_log"
TENSOR_NUM = 100
ACC_ELM_NUM = 1000
CSV_FILE = "BMIdata.csv"
def file_len(fname):
with open(fname)as f:
# count = len(open(filepath, 'r').readlines()) 可以统计文件的行数，但是比较慢
# enumerate 统计文件的行数
for i, l in enumerate(f):
pass
return i + 1
def main():
filename_queue = tf.train.string_input_producer([CSV_FILE])
file_length = file_len(CSV_FILE)
reader = tf.TextLineReader(skip_header_lines=7)
key, value = reader.read(filename_queue)
print(key)
print(value)
# Default values, in case of empty columns. Also specifies the type of the
# decoded result.
# 定义列
record_defaults = [['null'], ['null'], ['null'], ['null'], ['null'], ['null'], ['null'], ['null'], ['null'], ['null'], ['null'], ['null'], ['null']]
# record_defaults = [['null'], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0]]
# 编码
time, acc_x_s, acc_y_s, acc_z_s, acc_x_g, acc_y_g, acc_z_g, status0, status1, status2, status3, time_sensor, comment = tf.decode_csv(value, record_defaults)
features = tf.stack([time, acc_x_s, acc_y_s, acc_z_s, acc_x_g, acc_y_g, acc_z_g, status0, status1, status2, status3, time_sensor, comment])
print(features)
with tf.name_scope('input'):
tf_acc_x = tf.placeholder(tf.float32, name="tf_Acc_X-input")
tf_acc_y = tf.placeholder(tf.float32, name="tf_Acc_Y-input")
tf_acc_z = tf.placeholder(tf.float32, name="tf_Acc_Z-input")
tf.summary.scalar('tf_Acc_X', tf_acc_x)
tf.summary.scalar('tf_Acc_Y', tf_acc_y)
tf.summary.scalar('tf_Acc_Z', tf_acc_z)
init_op = tf.global_variables_initializer()
local_init_op = tf.local_variables_initializer() # local variables like epoch_num, batch_size 可以不初始化local
merged_summary_op = tf.summary.merge_all()
with tf.Session() as sess:
sess.run(init_op)
sess.run(local_init_op)
#写文件到log中
summary_writer = tf.summary.FileWriter(SUMMARY_DIR, sess.graph)
total_step = 0
#以下是读取数据文件
coord = tf.train.Coordinator() # 创建一个协调器，管理线程
threads = tf.train.start_queue_runners(coord=coord) # 启动QueueRunner, 此时文件名队列已经进队。
try:
for i in range(file_length):
acc_x_item = tf.string_to_number(acc_x_s, tf.float32)
# #加上decode可以去掉字符串前面的b
# acc_x_item = sess.run(tf_acc_x_tmp).decode()
# acc_x_item = acc_x_item.replace(' ', '').replace('\n', ',').replace('\t', '').replace('\'', '')
# print('acc_x_item: ', sess.run(acc_x_item))
acc_y_item = tf.string_to_number(acc_y_s, tf.float32)
acc_z_item = tf.string_to_number(acc_z_s, tf.float32)
tf_acc_x_var, tf_acc_y_var, tf_acc_z_var = sess.run([acc_x_item, acc_y_item, acc_z_item])
print('total_step: ', i, ', acc_x_item: ', tf_acc_x_var, ', acc_y_item: ', tf_acc_y_var, ', acc_z_item: ', tf_acc_z_var)
#添加到tensorboard
summary_str = sess.run(merged_summary_op, feed_dict={tf_acc_x: tf_acc_x_var, tf_acc_y: tf_acc_y_var, tf_acc_z: tf_acc_z_var})
summary_writer.add_summary(summary_str, total_step)
except tf.errors.OutOfRangeError:
print('Epochs Complete!')
finally:
coord.request_stop()
coord.request_stop()
coord.join(threads)
summary_writer.close()
if __name__ == '__main__':
main()
打印结果
total_step: 0 , acc_x_item: 0.6147 , acc_y_item: -0.5142 , acc_z_item: 10.0179
total_step: 1 , acc_x_item: 0.5818 , acc_y_item: -0.5429 , acc_z_item: 10.1107
total_step: 2 , acc_x_item: 0.5842 , acc_y_item: -0.5387 , acc_z_item: 10.1299
total_step: 3 , acc_x_item: 0.5836 , acc_y_item: -0.5357 , acc_z_item: 10.073
total_step: 4 , acc_x_item: 0.6147 , acc_y_item: -0.5531 , acc_z_item: 10.0149
total_step: 5 , acc_x_item: 0.6554 , acc_y_item: -0.498 , acc_z_item: 10.0054
total_step: 6 , acc_x_item: 0.68 , acc_y_item: -0.4788 , acc_z_item: 10.1167
total_step: 7 , acc_x_item: 0.6961 , acc_y_item: -0.4603 , acc_z_item: 10.1778
total_step: 8 , acc_x_item: 0.641 , acc_y_item: -0.4645 , acc_z_item: 10.1095
total_step: 9 , acc_x_item: 0.6063 , acc_y_item: -0.4848 , acc_z_item: 10.0946
total_step: 10 , acc_x_item: 0.677 , acc_y_item: -0.5285 , acc_z_item: 10.1418
total_step: 11 , acc_x_item: 0.6871 , acc_y_item: -0.5142 , acc_z_item: 10.1113
total_step: 12 , acc_x_item: 0.6087 , acc_y_item: -0.5351 , acc_z_item: 10.0114
total_step: 13 , acc_x_item: 0.5297 , acc_y_item: -0.5584 , acc_z_item: 10.0904
total_step: 14 , acc_x_item: 0.5884 , acc_y_item: -0.565 , acc_z_item: 10.1652
total_step: 15 , acc_x_item: 0.6381 , acc_y_item: -0.5836 , acc_z_item: 9.9886
total_step: 16 , acc_x_item: 0.5764 , acc_y_item: -0.5836 , acc_z_item: 9.8509
total_step: 17 , acc_x_item: 0.5638 , acc_y_item: -0.5387 , acc_z_item: 9.9563
total_step: 18 , acc_x_item: 0.6135 , acc_y_item: -0.5944 , acc_z_item: 10.1323
total_step: 19 , acc_x_item: 0.5638 , acc_y_item: -0.5393 , acc_z_item: 10.2801
total_step: 20 , acc_x_item: 0.516 , acc_y_item: -0.4178 , acc_z_item: 10.486
total_step: 21 , acc_x_item: 0.5387 , acc_y_item: -0.5094 , acc_z_item: 10.4956
total_step: 22 , acc_x_item: 0.6494 , acc_y_item: -0.5692 , acc_z_item: 10.2065
total_step: 23 , acc_x_item: 0.6273 , acc_y_item: -0.5339 , acc_z_item: 9.7432
进程已结束,退出代码0
测试文件
#
#
#
#
#
#
# Time Stamp,Accel X-axis - M/s2,Accel Y-axis - M/s2,Accel Z-axis - M/s2,Accel X-axis - g,Accel Y-axis - g,Accel Z-axis - g,Accel Interrupt Status Byte 0 - Hex,Accel Interrupt Status Byte 1 - Hex,Accel Interrupt Status Byte 2 - Hex,Accel Interrupt Status Byte 3 - Hex,Sensor Time - µs,
14:58:45.6089935, 0.6147, -0.5142, 10.0179, 0.0627, -0.0524, 1.0215, 00, 00, 00, 00, 173601046.1513,
14:58:45.6089935, 0.5818, -0.5429, 10.1107, 0.0593, -0.0554, 1.0310, 00, 00, 00, 00, 173611045.8313,
14:58:45.6150098, 0.5842, -0.5387, 10.1299, 0.0596, -0.0549, 1.0330, 00, 00, 00, 00, 173621045.5113,
14:58:45.6340593, 0.5836, -0.5357, 10.0730, 0.0595, -0.0546, 1.0272, 00, 00, 00, 00, 173631045.1913,
14:58:45.6360979, 0.6147, -0.5531, 10.0149, 0.0627, -0.0564, 1.0212, 00, 00, 00, 00, 173641005.8100,
14:58:45.6451799, 0.6554, -0.4980, 10.0054, 0.0668, -0.0508, 1.0203, 00, 00, 00, 00, 173651005.4900,
14:58:45.6551173, 0.6800, -0.4788, 10.1167, 0.0693, -0.0488, 1.0316, 00, 00, 00, 00, 173661005.1700,
14:58:45.6661442, 0.6961, -0.4603, 10.1778, 0.0710, -0.0469, 1.0378, 00, 00, 00, 00, 173671004.8500,
14:58:45.6801842, 0.6410, -0.4645, 10.1095, 0.0654, -0.0474, 1.0309, 00, 00, 00, 00, 173681004.5300,
14:58:45.6841937, 0.6063, -0.4848, 10.0946, 0.0618, -0.0494, 1.0294, 00, 00, 00, 00, 173690965.1488,
14:58:45.6952215, 0.6770, -0.5285, 10.1418, 0.0690, -0.0539, 1.0342, 00, 00, 00, 00, 173700964.8288,
14:58:45.7072535, 0.6871, -0.5142, 10.1113, 0.0701, -0.0524, 1.0311, 00, 00, 00, 00, 173710964.5088,
14:58:45.7153667, 0.6087, -0.5351, 10.0114, 0.0621, -0.0546, 1.0209, 00, 00, 00, 00, 173720964.1888,
14:58:45.7333240, 0.5297, -0.5584, 10.0904, 0.0540, -0.0569, 1.0289, 00, 00, 00, 00, 173730924.8075,
14:58:45.7363310, 0.5884, -0.5650, 10.1652, 0.0600, -0.0576, 1.0366, 00, 00, 00, 00, 173740924.4875,
14:58:45.7434377, 0.6381, -0.5836, 9.9886, 0.0651, -0.0595, 1.0186, 00, 00, 00, 00, 173750963.2288,
14:58:45.7554724, 0.5764, -0.5836, 9.8509, 0.0588, -0.0595, 1.0045, 00, 00, 00, 00, 173760923.8475,
14:58:45.7654083, 0.5638, -0.5387, 9.9563, 0.0575, -0.0549, 1.0153, 00, 00, 00, 00, 173770923.5275,
14:58:45.7824551, 0.6135, -0.5944, 10.1323, 0.0626, -0.0606, 1.0332, 00, 00, 00, 00, 173780923.2075,
14:58:45.7844589, 0.5638, -0.5393, 10.2801, 0.0575, -0.0550, 1.0483, 00, 00, 00, 00, 173790922.8875,
14:58:45.7954886, 0.5160, -0.4178, 10.4860, 0.0526, -0.0426, 1.0693, 00, 00, 00, 00, 173800883.5063,
14:58:45.8055146, 0.5387, -0.5094, 10.4956, 0.0549, -0.0519, 1.0703, 00, 00, 00, 00, 173810883.1863,
14:58:45.8155418, 0.6494, -0.5692, 10.2065, 0.0662, -0.0580, 1.0408, 00, 00, 00, 00, 173820882.8663,
14:58:45.8336015, 0.6273, -0.5339, 9.7432, 0.0640, -0.0544, 0.9935, 00, 00, 00, 00, 173830843.4850,
14:58:45.8355990, 0.7661, -0.6099, 9.5199, 0.0781, -0.0622, 0.9708, 00, 00, 00, 00, 173840843.1650,
14:58:45.8466263, 0.8266, -0.6901, 9.4631, 0.0843, -0.0704, 0.9650, 00, 00, 00, 00, 173850842.8450,
参考链接
http://www.jb51.net/article/134146.htm