列式存储infobright

1、数据库安装

安装

#查看默认路径 rpm -qpl infobright-4.5.0-4-x86_64-eval.rpm
rpm -ivh --relocate /usr/local = /data/server/ infobright-4.5.0-4-x86_64- eval .rpm

设置权限

GRANT ALL PRIVILEGES ON *.* TO 'root' @ '192.168.%' IDENTIFIED BY 'password' WITH GRANT OPTION ;
flush privileges ;

编辑配置文件，修改数据文件路径、默认表引擎 编码

vim /etc/my-ib.cnf 

port            = 3337
basedir= /data/server/infobright
datadir = /data/data/infobright
log-error = /data/data/infobright/bh .err

default-storage-engine=MyISAM
#collation_server=latin1_bin
#character_set_server=latin1
collation_server=utf8_general_ci
character_set_server=utf8

启动数据库

/etc/init .d /mysqld-ib start

#/etc/init.d/mysqld-ib stop




2、数据导入

要求为标准csv格式，通过load data导入

LOAD DATA INFILE '/data/tmp/01/temp.csv'
INTO TABLE log_epg
FIELDS
TERMINATED BY ','
ENCLOSED BY '"'
ESCAPED BY '\\'
;

3、数据分析――示例1

 

文件格式csv，以下两种格式合法

/data/tmp/rwdata/0209/0151.txt

"015159014000001","0C:C6:55:24:9F:97"

"015159014000002" , "0C:C6:55:24:9F:98"
"015159014000003" , "0C:C6:55:24:9F:99"
"015159014000004" , "0C:C6:55:24:9F:9A"
"015159014000005" , "0C:C6:55:24:9F:9B"
"015159014000006" , "0C:C6:55:24:9F:9C"
"015159014000007" , "0C:C6:55:24:9F:9D"
"015159014000008" , "0C:C6:55:24:9F:9E"
"015159014000009" , "0C:C6:55:24:9F:9F"
"015159014000010" , "0C:C6:55:24:9F:A0"
"015159014000011" , "0C:C6:55:24:9F:A1"

/data/tmp/rwdata/0209/aaamac.csv

010121011011501,0C:C6:55:1E:BC:B7

010121011011502 ,0C:C6: 55 :1E:BC:B8
010121011011503 ,0C:C6: 55 :1E:BC:B9
010121011011504 ,0C:C6: 55 :1E:BC:BA
010121011011505 ,0C:C6: 55 :1E:BC:BB
010121011011506 ,0C:C6: 55 :1E:BC:BC
010121011011507 ,0C:C6: 55 :1E:BC:BD
010121011011508 ,0C:C6: 55 :1E:BC:BE

如果有非标准字符，需要进行预处理

#sed -i "s/|/,/g" 0151.txt

"015159014000001" | "0C:C6:55:24:9F:97"
"015159014000002" | "0C:C6:55:24:9F:98"
"015159014000003" | "0C:C6:55:24:9F:99"
"015159014000004" | "0C:C6:55:24:9F:9A"
"015159014000005" | "0C:C6:55:24:9F:9B"

表创建

use rwdata
CREATE TABLE `t0151` (
  `icntvid` varchar (255) DEFAULT '' ,
  `mac` varchar (255) DEFAULT ''
) ENGINE=BRIGHTHOUSE DEFAULT CHARSET=utf8;

CREATE TABLE `aaamac` (
  `icntvid` varchar (255) DEFAULT '' ,
  `mac` varchar (255) DEFAULT ''
) ENGINE=BRIGHTHOUSE DEFAULT CHARSET=utf8;

数据导入

set @bh_dataformat = 'txt_variable' ;

LOAD DATA INFILE '/data/tmp/rwdata/0209/aaamac.csv'
INTO TABLE aaamac
FIELDS
TERMINATED BY ','
ENCLOSED BY '"'
ESCAPED BY '\\'
;

LOAD DATA INFILE '/data/tmp/rwdata/0209/0151.txt '
INTO TABLE t0151
FIELDS
TERMINATED BY ','
ENCLOSED BY '"'
ESCAPED BY '\\'
;

查询

SELECT t0151.*,aaamac.mac AS aaa_mac FROM t0151,aaamac WHERE t0151.icntvid=aaamac.icntvid AND t0151.mac!=aaamac.mac

确认无误后，将查询结果生成表，然后导出

CREATE TABLE data_result AS SELECT t0151.*,aaamac.mac AS aaa_mac FROM t0151,aaamac WHERE t0151.icntvid=aaamac.icntvid AND t0151.mac!=aaamac