sphinx 采用c扩展xmlpipe2数据源 .

由于要采用sphinx 作为全文检索引擎，但是我的很多数据都存在硬盘上，如果这些数据都读入数据库那就太痛苦了，数据量很大，mysql数据库存储这些数据，只能是为了全文检索，那也过于浪费了，所以决定采用xmlpipe2数据源，由于c语言的io操作能力超强为了效率问题也就是用，从不会写的c语言。废话少说，代码如下

 

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <mysql.h>
#include <iconv.h>

#include "dictionary.h"
#include "iniparser.h"

//配置文件名称
#define INI_FILE_NAME "sanshi_xmlpipe.ini"
//sql多条语句分隔符号
#define SQL_SIGN "|"
//设置字段里名字与字段值的分隔符
#define ATTRIBUTE_SIGN ":"
//设置多个字段间的分隔符
#define DIELD_SIGN ","
//设置读取文件的长度
#define READFILE_MAX_LEN 1024
//设置路径+文件的最大长度
#define FILE_NAME_MAX_LEN 1024

typedef struct tag_sanshi_ini_config
{
	char * db_name;
	char * db_user;
	char * db_pwd;
	char * db_host;
	char * sql_query_pre;
	char * sql_query;
	char * sql_query_post;
	char * sphinx_schema;
	char *	sphinx_id;
	char * sphinx_file;
	char * sphinx_other_field;
	char * file_dir;
}sanshi_ini_config;

sanshi_ini_config conf={
	"db:db_name",
	"db:db_user",
	"db:db_pwd",
	"db:db_host",
	"sql:sql_query_pre",
	"sql:sql_query",
	"sql:sql_query_post",
	"xml:schema",
	"xml:index_id",
	"xml:file_field",
	"xml:other_field",
	"file:base_dir"
	};

typedef struct tag_sanshi_field
{
	char *field;
	int  id;
} sanshi_field;

sanshi_field parser_field(char * field_str)
{
	sanshi_field temp;
	char * buf = strstr( field_str, ATTRIBUTE_SIGN);
	temp.id = atoi(buf + strlen(ATTRIBUTE_SIGN));
	buf[0]='/0';
	temp.field = field_str;
	//temp.id = buf + strlen(ATTRIBUTE_SIGN);
	//buf = strstr( field_str, ATTRIBUTE_SIGN);
	//temp.id = atoi(buf + strlen(ATTRIBUTE_SIGN));
	//printf("%s %s %d/n",field_str,temp.field,temp.id);
	/*
	char * p;
	char field_tmp[strlen(field_str)+1];
	strcpy(field_tmp,field_str);
	//temp.field = strtok( field_str, ATTRIBUTE_SIGN);
	//field_str = strtok( NULL, ATTRIBUTE_SIGN);
	//temp.id = atoi(p);
	printf("%s /n",field_tmp);
	field_str =NULL;
	*/
	return temp;
}

void print_file_content(char * file_name)
{
	FILE *fp;
	char line[READFILE_MAX_LEN];
	fp=fopen(file_name,"r");
	if(fp!=NULL)
	{
		while(fgets(line,READFILE_MAX_LEN,fp)!=NULL)
		{
			printf(line);
		}
		fclose(fp);
	}
}

void exec_mysql_query(MYSQL * mysql_con,char * sql_str)
{
	char* token = strtok( sql_str, SQL_SIGN);
	while( token != NULL )
    {
		int query_error_no=0;
		query_error_no = mysql_query(mysql_con,token);
		if(query_error_no !=0)
		{
			printf("ERROR sql=%s /nERROR NO=%d /nERROR msg= %s /n",token,query_error_no,mysql_error(mysql_con));
			mysql_close(mysql_con);
			exit(0);
		}
		mysql_free_result(mysql_store_result(mysql_con));
		//printf("exec sql : %s /n",token);
        token = strtok( NULL, SQL_SIGN);
    }
}

int main(int argc,char * argv[])
{
	dictionary * ini;
	
	MYSQL mysql,*mysql_con;
	MYSQL_RES *result;
	MYSQL_ROW row;
	int query_error_no,sphinx_id;
	
	sanshi_field file_field;

	ini = iniparser_load(INI_FILE_NAME);
	//get ini config mysql set 
	conf.db_name = iniparser_getstring(ini,conf.db_name,"test");
	conf.db_user = iniparser_getstring(ini,conf.db_user,"root");
	conf.db_pwd = iniparser_getstring(ini,conf.db_pwd,"");
	conf.db_host = iniparser_getstring(ini,conf.db_host,"localhost");
	conf.sql_query_pre = iniparser_getstring(ini,conf.sql_query_pre,NULL);
	conf.sql_query = iniparser_getstring(ini,conf.sql_query,NULL);
	conf.sql_query_post = iniparser_getstring(ini,conf.sql_query_post,NULL);
	conf.sphinx_schema = iniparser_getstring(ini,conf.sphinx_schema,NULL);
	sphinx_id = iniparser_getint(ini,conf.sphinx_id,0);
	conf.sphinx_file = iniparser_getstring(ini,conf.sphinx_file,NULL);
	file_field = parser_field(conf.sphinx_file);

	conf.sphinx_other_field = iniparser_getstring(ini,conf.sphinx_other_field,NULL);
	conf.file_dir = iniparser_getstring(ini,conf.file_dir,"./");
	
	//printf("db_name=%s /t db_user=%s /t db_pwd=%s /t db_host=%s /n",conf.db_name,conf.db_user,conf.db_pwd,conf.db_host);
	//mysql connect 
	mysql_init(&mysql);
	mysql_con =mysql_real_connect(&mysql,conf.db_host,conf.db_user,conf.db_pwd,conf.db_name,0,NULL,0);
	if(mysql_con == NULL)
	{
		printf("ERROR: connect mysql fail! plaese check ini file in set/n %s /n",mysql_error(&mysql));
		exit(0);
	}
	//printf("mysql connect suc!/n");

	//exec sql
	exec_mysql_query(mysql_con,conf.sql_query_pre);
	
	query_error_no = mysql_query(mysql_con,conf.sql_query);
	
	if(query_error_no !=0)
	{
		printf("ERROR sql=%s /nERROR NO=%d /nERROR msg= %s /n",conf.sql_query,query_error_no,mysql_error(mysql_con));
		mysql_close(mysql_con);
		exit(0);
	}
	//printf("exec sql_query : %s /n",conf.sql_query);
	result = mysql_store_result(mysql_con);
	//echo xml header
	printf("<?xml version=/"1.0/" encodeing=/"utf-8/"?>/n<sphinx:docset>/n%s/n",conf.sphinx_schema);
	//printf("%s/n",conf.sphinx_other_field);
	while(row = mysql_fetch_row(result))
	{
		printf("<sphinx:document id=/"%d/">/n",(row[sphinx_id]?row[sphinx_id]:0));
		char * field_str;
		char field_tmp[strlen(conf.sphinx_other_field)+1];
		char temp_file_name[FILE_NAME_MAX_LEN];
		memcpy(field_tmp,conf.sphinx_other_field,strlen(conf.sphinx_other_field)+1);
		field_str = strtok( field_tmp, DIELD_SIGN);
		while( field_str != NULL )
		{
			//printf("%s/n",field_str);
			sanshi_field other_field = parser_field(field_str);
			printf("

 

本人第一次写c，有些地方优化部够，还望各位指点

 

配置文件如下：

[db]
db_host=127.0.0.1
db_name=test
db_pwd=123456
db_user=root
[sql]
sql_query_pre=select * from log|select * from log
sql_query = select * from log
sql_query_post =
[xml]
schema=/
<sphinx:schema>/
<sphinx:field name="LogActionType"/>/
<sphinx:field name="LogDataType"/>/
<sphinx:attr name="LogTime" type="timestamp"/>/
<sphinx:attr name="LogIP" type="int" bits="16" default="1"/>/
</sphinx:schema>
index_id=0
file_field=LogActionType:1
other_field=LogDataType:2,LogTime:6,LogIP:4
[file]
base_dir=

 

 

 

备注：该程序的ini解析用到的是iniparser3.0b 的源代码，也就是依赖的2个h文件

#include "dictionary.h"

#include "iniparser.h"

       编译的命令为：

gcc -I /data/app/mysql/include/mysql/ -L /data/app/mysql/lib/mysql/ -l mysqlclient -g -o sanshi sanshi_xmlpipe.c dictionary.c iniparser.c

注意msql的库路径，已经文件名