搜狗细胞词库处理代码（可用于scel转txt）
搜狗细胞词库处理代码（可用于scel转txt）
2010-07-05 18:46
http://hi.baidu.com/guangbinw/blog/item/86d61009de07f28fd0581b11.html
今天先贴个简单代码，稍后再详细叙述……
基于QT实现，主要是考虑Unicode字符处理的方便，
可以稍加处理用于C或C++语言。
// 取连续两字节，转换为short类型的值，字节顺序是低字节-高字节
inline unsigned short GetUShort(QByteArray &fData, int startPos)
{
unsigned char low = fData.at(startPos);
unsigned char high = fData.at(startPos + 1);
unsigned short st = low + (high * 256);
return st;
}
//把 Unicode编码的值转换为字符串
inline QString GetStrvalue(QByteArray &fData, int startPos, int len)
{
QString temp;
for (int i = 0 ;i < len ; i+=2)
{
unsigned short st = GetUShort(fData,startPos + i);
QChar ch = QChar(st);
temp.append(ch);
if (st == 0)
{
break;
}
}
return temp;
}
//临时保存结果
class CHanzi
{
public:
void SetPy(QByteArray &py)
{
m_py = py;
}
void SetHz(QByteArray &hz)
{
m_hz = GetStrvalue(hz,0,hz.size());
}
void Debug()
{
QString py = "";
for (int i = 0 ; i < m_py.length() ; i += 2)
{
py += pyList.at(GetUShort(m_py,i));
}
qDebug() << m_hz << ":" << py;
}
private:
QByteArray m_py;
QString m_hz;
};
//1、读取拼音表
inline void ReadPyTable(QByteArray &fData,QList<QString> &pyList)
{
int startPos = 0x1540;
QByteArray fFlag = fData.mid(startPos,4);
if( fData.data() == "/x9D/x01/x00/x00")
{
qDebug() << "读取词库拼音表失败！";
return;
}
int pos = 4;
while (true)
{
//取bit的索引号，用于表示一个拼音信息
unsigned short n = GetUShort(fData,startPos + pos);
pos += 2;
//取bit的拼音长度，字母数的倍
unsigned short len = GetUShort(fData,startPos + pos);
pos += 2;
//每个字母占bit,Unicode编码
QString py = GetStrvalue(fData,startPos + pos,len);
qDebug() << n << ":" << py;
pyList.push_back(py);
pos += len;
//zuo是最后一个拼音，处理完成
if (py == "zuo" || pos + startPos >= fData.length())
{
break;
}
}
}
//2、读取汉字表
inline void ReadHzTable(QByteArray &fData,QList<CHanzi> &hzList)
{
int startPos = 0x2628;
int pos = 0;
while (true)
{
//偏移加权,16bit,用于计算下一记录的位置
unsigned short offset = GetUShort(fData,startPos + pos) - 1;
pos += 2;
//拼音长度,16bit,每个拼音用一个bit的short表示,其值是拼音表的索引号
unsigned short len1 = GetUShort(fData,startPos + pos);
pos += 2;
CHanzi hanzi;
//取拼音索引信息
hanzi.SetPy(fData.mid(startPos + pos,len1));
pos += len1;
//索引之后是词组的长度，字节数，字数乘
unsigned short len2 = GetUShort(fData,startPos + pos);
pos += 2;
//Unicode编码，每个汉字bit
hanzi.SetHz(fData.mid(startPos + pos,len2));
hzList.push_back(hanzi);
pos += len2;
//到下个字的偏移位置，没有再细分析，可能是词频或者类似信息
pos += (12 + offset * (12 + len2 + 2));
if (pos + startPos >= fData.length())
{
break;
}
}
}
//3、读取名称、类别、信息、示例
inline void ReadFileInfo(QByteArray &fData)
{
QByteArray fFlag = fData.mid(0,8);
if(fFlag.data() == "/x40/x15/x00/x00/x44/x43/x53/x01")
{
qDebug() << "确认你选择的是搜狗(.scel)词库?";
return ;
}
QString strName;
strName = GetStrvalue(fData,0x130,128);
qDebug() << "词库名：" << strName;
strName = GetStrvalue(fData,0x338,128);
qDebug() << "词库类型：" << strName;
strName = GetStrvalue(fData,0x540,128);
qDebug() << "描述信息：" << strName;
strName = GetStrvalue(fData,0xd40,128);
qDebug() << "词库示例：" << strName;
}