apache iotdb 初步使用以及遇到的问题

0，遇到的问题

 1）日志显示跟flink的冲突，需要排除

    org.apache.iotdb
    flink-iotdb-connector
    0.11.0
    
        
            logback-classic
            ch.qos.logback
        
    

2) 在flink任务重启的时候会报错：（这里是验证flink并行度 1跟大于1的时候）

存储组已经存在。

需要将 77行要给try-catch起来，修改源码再编译，不过官方已经解决发布了

 

 

3）源码问题

大家可以测试，会报错

错误地点在tablet类

目前官方已经修改。

 

 

1，安装iotdb 

1）下载zip包 

https://iotdb.apache.org/Download/

unzip 解压

启动：

2，Flink代码写入iotdb

import com.google.common.collect.Lists;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
import org.apache.iotdb.flink.DefaultIoTSerializationSchema;
import org.apache.iotdb.flink.IoTDBOptions;
import org.apache.iotdb.flink.IoTDBSink;
import org.apache.iotdb.flink.IoTSerializationSchema;

import java.util.*;

/**
 * @program: flink-neiwang-dev
 * @description: iotdb测试
 * @author: Mr.Wang
 * @create: 2020-12-01 15:38
 **/


public class IoTDBSinkInsertTest {
    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

        IoTDBOptions options = new IoTDBOptions();
        options.setHost("192.168.6.36");
        options.setPort(6667);
        options.setUser("root");
        options.setPassword("root");
        options.setStorageGroup("root.test1");
        options.setTimeseriesOptionList(
                Lists.newArrayList(
                        new IoTDBOptions.TimeseriesOption("root.test1.D01.aaa"),
                        new IoTDBOptions.TimeseriesOption("root.test1.D01.bbb"),
                        new IoTDBOptions.TimeseriesOption("root.test1.D01.ccc")
                )
        );

        IoTSerializationSchema serializationSchema = new DefaultIoTSerializationSchema();
        IoTDBSink ioTDBSink = new IoTDBSink(options, serializationSchema)
                // enable batching
                .withBatchSize(10) ;


        env.addSource(new SensorSource())
                .name("sensor-source")
                .setParallelism(1)
                .addSink(ioTDBSink)
                .name("iotdb-sink")
                .setParallelism(1);


        env.execute("iotdb-flink-example");
    }
    private static class SensorSource implements SourceFunction< Map> {
        private static final long serialVersionUID = 1L;


        private volatile boolean isRunning = true;
        private int counter = 0;

        /**
         * @Description:
         * @Param:
         * @return:
         * @Author: intsmaze
         * @Date: 2019/1/5
         */
        @Override
        public void run(SourceFunction.SourceContext< Map> ctx) throws Exception {
            Random random = new Random();
            String[] strArray={"aaa","bbb","ccc"};

        while (true){
            Thread.sleep(5000);
            int index = random.nextInt(3);
            Map tuple = new HashMap();
            tuple.put("device", "root.test1.D01");
            long timeMillis = System.currentTimeMillis();
            tuple.put("timestamp", String.valueOf(timeMillis));
            tuple.put("measurements", "aa,bb,cc");

            tuple.put("types", "TEXT,TEXT,TEXT");
            tuple.put("values", "1,2,3");
//            tuple.put("values", String.valueOf(index));

            ctx.collect(tuple);
        }

        }

        @Override
        public void cancel() {
            isRunning = false;
        }
    }


}

需要导入依赖：

     org.apache.iotdb
     flink-iotdb-connector
     0.11.0
 

 

3，进入客户端查看结果

 

4，初步理解

如上图，我们理解为：

电力集团层-电厂层-设备层-传感器层

最左侧路径对应的时间序列名称为ROOT.ln.wf01.wt01.status   解释为：电力集团in-电厂wf01-设备wt01-传感器status

在上面的代码里面，每次写入iotdb是Map的格式。

在代码层面没有List>的概率，在下面 的设置中就是批的意思

这个实际上就是攒多少条的意思，攒够10条才调用接口就发送一次。

 

 

5，案例代码：

 

 

https://github.com/apache/iotdb/blob/master/example/session/src/main/java/org/apache/iotdb/SessionExample.java

package org.apache.iotdb;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import org.apache.iotdb.rpc.BatchExecutionException;
import org.apache.iotdb.rpc.IoTDBConnectionException;
import org.apache.iotdb.rpc.StatementExecutionException;
import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.iotdb.session.Session;
import org.apache.iotdb.session.SessionDataSet;
import org.apache.iotdb.session.SessionDataSet.DataIterator;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.write.record.Tablet;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;

public class SessionExample {

  private static Session session;
  private static final String ROOT_SG1_D1_S1 = "root.sg1.d1.s1";
  private static final String ROOT_SG1_D1_S2 = "root.sg1.d1.s2";
  private static final String ROOT_SG1_D1_S3 = "root.sg1.d1.s3";
  private static final String ROOT_SG1_D1_S4 = "root.sg1.d1.s4";
  private static final String ROOT_SG1_D1_S5 = "root.sg1.d1.s5";
  private static final String ROOT_SG1_D1 = "root.sg1.d1";


  public static void main(String[] args)
      throws IoTDBConnectionException, StatementExecutionException {
    session = new Session("127.0.0.1", 6667, "root", "root");
    session.open(false);

    try {
      session.setStorageGroup("root.sg1");
    } catch (StatementExecutionException e) {
      if (e.getStatusCode() != TSStatusCode.PATH_ALREADY_EXIST_ERROR.getStatusCode())
        throw e;
    }

    createTimeseries();
    createMultiTimeseries();
    insertRecord();
    insertTablet();
    insertTablets();
    insertRecords();
    nonQuery();
    query();
    rawDataQuery();
    queryByIterator();
    deleteData();
    deleteTimeseries();
    session.close();
  }

  private static void createTimeseries()
      throws IoTDBConnectionException, StatementExecutionException {

    if (!session.checkTimeseriesExists(ROOT_SG1_D1_S1)) {
      session.createTimeseries(ROOT_SG1_D1_S1, TSDataType.INT64, TSEncoding.RLE,
          CompressionType.SNAPPY);
    }
    if (!session.checkTimeseriesExists(ROOT_SG1_D1_S2)) {
      session.createTimeseries(ROOT_SG1_D1_S2, TSDataType.INT64, TSEncoding.RLE,
          CompressionType.SNAPPY);
    }
    if (!session.checkTimeseriesExists(ROOT_SG1_D1_S3)) {
      session.createTimeseries(ROOT_SG1_D1_S3, TSDataType.INT64, TSEncoding.RLE,
          CompressionType.SNAPPY);
    }

    // create timeseries with tags and attributes
    if (!session.checkTimeseriesExists(ROOT_SG1_D1_S4)) {
      Map tags = new HashMap<>();
      tags.put("tag1", "v1");
      Map attributes = new HashMap<>();
      tags.put("description", "v1");
      session.createTimeseries(ROOT_SG1_D1_S4, TSDataType.INT64, TSEncoding.RLE,
          CompressionType.SNAPPY, null, tags, attributes, "temperature");
    }

    // create timeseries with SDT property, SDT will take place when flushing
    if (!session.checkTimeseriesExists(ROOT_SG1_D1_S5)) {
      // compDev is required
      // compMax and compMin are optional and their unit is ms
      Map props = new HashMap<>();
      props.put("loss", "sdt");
      props.put("compDev", "0.01");
      props.put("compMin", "2");
      props.put("compMax", "10");
      session.createTimeseries(ROOT_SG1_D1_S5, TSDataType.INT64, TSEncoding.RLE,
          CompressionType.SNAPPY, props, null, null, null);
    }
  }

  private static void createMultiTimeseries()
      throws IoTDBConnectionException, BatchExecutionException, StatementExecutionException {

    if (!session.checkTimeseriesExists("root.sg1.d2.s1") && !session
        .checkTimeseriesExists("root.sg1.d2.s2")) {
      List paths = new ArrayList<>();
      paths.add("root.sg1.d2.s1");
      paths.add("root.sg1.d2.s2");
      List tsDataTypes = new ArrayList<>();
      tsDataTypes.add(TSDataType.INT64);
      tsDataTypes.add(TSDataType.INT64);
      List tsEncodings = new ArrayList<>();
      tsEncodings.add(TSEncoding.RLE);
      tsEncodings.add(TSEncoding.RLE);
      List compressionTypes = new ArrayList<>();
      compressionTypes.add(CompressionType.SNAPPY);
      compressionTypes.add(CompressionType.SNAPPY);

      List> tagsList = new ArrayList<>();
      Map tags = new HashMap<>();
      tags.put("unit", "kg");
      tagsList.add(tags);
      tagsList.add(tags);

      List> attributesList = new ArrayList<>();
      Map attributes = new HashMap<>();
      attributes.put("minValue", "1");
      attributes.put("maxValue", "100");
      attributesList.add(attributes);
      attributesList.add(attributes);

      List alias = new ArrayList<>();
      alias.add("weight1");
      alias.add("weight2");

      session
          .createMultiTimeseries(paths, tsDataTypes, tsEncodings, compressionTypes, null, tagsList,
              attributesList, alias);
    }
  }

  private static void insertRecord() throws IoTDBConnectionException, StatementExecutionException {
    String deviceId = ROOT_SG1_D1;
    List measurements = new ArrayList<>();
    List types = new ArrayList<>();
    measurements.add("s1");
    measurements.add("s2");
    measurements.add("s3");
    types.add(TSDataType.INT64);
    types.add(TSDataType.INT64);
    types.add(TSDataType.INT64);

    for (long time = 0; time < 100; time++) {
      List