opentsdb性能测试

opentsdb存在的大坑: 

    value只支持数字(整型和浮点数),不支持英文,汉字, 现场景出现需要存汉字的data数据, 单opentsdb不支持,只能手动存储在hbase中,

    tags标签不支持汉字,特殊字符, 每次汉字都要转成unicode码再存入opentsdb

测试环境:

单机服务器:8核12G

发送的结构体:

json = {
    "metric": 16,
    "timestamp": 1551285963,
    "value": "sdfasd",
    "tags": {
       "operationValue": "HDFF88FFF",
       "gateMac": "00E2690CDFFD",
    }
}

发送上述的结构体,进行3分钟的测试:

java测试结果:1秒大概10万条数据

python测试结果: 1秒大概1万多条数据

java测试代码:

package com.flink;

import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.mortbay.util.ajax.JSON;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class sendTest {
    private static CloseableHttpClient httpClient;
//    private static HttpPost httpPost;

    public static void  main(String[] args) throws Exception {
        int Total= 1000000;
        int Num = 0;
        httpClient = HttpClients.createDefault();
        long Starttime = System.currentTimeMillis();

        for(int i = 0; i< Total; i++){
            List List = new ArrayList<>();
            Map map1 = new HashMap<>();
            Map tags = new HashMap<>();
            tags.put("operationValue", "HDFF88FFF");
            tags.put("gateMac", "00E2690CDFFD");
            map1.put("metric", 10);
            map1.put("value", i);
            map1.put("tags", tags);

//            System.out.println(i + "条数据");
            map1.put("timestamp", Starttime/1000 + i + 3000000);
            List.add(map1);
            Num += 1;
            if (Num >= 200) {
                long Starttime1 = System.currentTimeMillis();
                sendPost("http://192.168.3.101:4242/api/put?async", List);
                long Endtime1 = System.currentTimeMillis();
                System.out.println("==============================opentsdb写入耗时=============================: " + (Endtime1-Starttime1));
                Num = 0;
                List.clear();
            }
        }

        long EndTime = System.currentTimeMillis();
        System.out.println(Starttime);
        System.out.println(EndTime);
        System.out.println("处理" + Total + "条数据, 消耗: " + (EndTime - Starttime));
    }



    public static void sendPost(final String url, List list) throws IOException {
        final HttpPost httpPost = new HttpPost(url);
        if (! list.isEmpty()) {
//            System.out.println(list);
            String string = JSON.toString(list);
            StringEntity entity = new StringEntity(string, "utf-8");
            entity.setContentEncoding("utf-8");
            entity.setContentType("application/json");
            httpPost.setEntity(entity);
            httpClient.execute(httpPost);
        }
    }
}

 

python测试代码

#coding:utf-8
import time
import requests
import uuid
import math
import random
import time, threading
start = time.time()

start_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))


def get_value(num):
    return math.sin(num)+1


def send_json(json, s, ip):
    url = "http://%s:4242/api/put" % ip
    r = s.post(url, json=json)
    return r.text


def main(metric, ip, Num):
    s = requests.Session()
    a = int(time.time() + 2000000) - Num
    # print(a)
    ls = []
    k = random.randint(0, 10000)

    value = "转速"
    # value = "转速".encode('unicode_escape')
    # value = value.replace("\\u", "")
    for i in range(1, Num):
        print("%s: %s条数据" % (ip, i))
        time.sleep(2)
        json = {
            "metric": metric,
            "timestamp": a,
            "value": "sdfasd",
            "tags": {
	            "operationValue": "HDFF88FFF",
	            "gateMac": "00E2690CDFFD",
            }
        }
        a += 1
        k += 0.01
        ls.append(json)

        if len(ls) == 1:
            send_json(ls, s, ip)
            ls = []
    send_json(ls, s, ip)
    print("%s: %s条数据" % (ip, Num))
    print("开始时间: %s" % start_time)
    print("时间: %s" % time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())))


import json


def read_opentsdb():
    queries = [{
        "metric": 53,
        "tags": {"operationValue": "WeldCurrent"},
        "aggregator": "none",
    }]
    url = "http://192.168.3.101:4242/api/query"
    post_dict = {
        "start": 1551285963,
        "end": 1551285965,
        "queries": queries
    }

    ret = requests.post(url, data=json.dumps(post_dict))
    data_ret = ret.json()
    print(data_ret)


def start_opentsdb():
    main("12", "192.168.3.101", 1000000)

if __name__ == "__main__":
    start_opentsdb()
    # read_opentsdb()

测试结果: 1秒大概1万多条数据

 

你可能感兴趣的:(opentsdb,opetnsdb性能测试)