使用java将搜狗词库.scel文件转化为.txt文件

需求：批量将.scel文件转化为可视的txt文件(支持1对1，多对1，多对多)，并从中提取中文词(去重)，支持追加内容。

成果：

   

使用：

package com.hxl.files;

import java.io.IOException;

public class Demo {
	
	public static void main(String[] args) {
		//单个scel文件转化  
		FileProcessing scel=new SougouScelFileProcessing();
		scel.parseFile("/Users/ST_iOS/Desktop/test/ciku/89个节日.scel", "/Users/ST_iOS/Desktop/test/ciku/txt/89个节日.txt", true);

		//多个scel文件转化为一个txt (格式：拼音字母 词)
		try {
			scel.parseFiles("/Users/ST_iOS/Desktop/test/ciku", "/Users/ST_iOS/Desktop/test/ciku/txt/汇总.txt", false);
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		//多个scel文件转化为多个txt文件
		scel.setTargetDir("/Users/ST_iOS/Desktop/test/ciku/多对多");//转化后文件的存储位置
		scel.parseFile("/Users/ST_iOS/Desktop/test/ciku",false);
		
	}
}

以上代码是将.scel文件转化为.txt文件的代码，效果类似于图1和图2的，如果要提取中文词，将scel=new TxtFileProcessing()；调用一样的方法，即可实现txt词库文件提取中文词。

注意事项：

SougouScelFileProcessing 解析的源文件必须为.scel文件类型，解析后的文件必须为.txt文件类型，参数以绝对路径进行传递

TxtFileProcessing   解析的源文件必须为.txt，格式如图2（拼音 中文），解析后的文件必须为.txt文件类型，参数以绝对路径进行传递

源码：一共4个class，需要的朋友可自行拷入项目中使用，代码解释的也挺清晰的，就不多说了。

1、FileProcessing

package com.hxl.files;

import java.io.File;
import java.io.IOException;

public abstract class FileProcessing {
	// 解析后存放的文件夹
	protected String targetDir;

	/**
	 * 解析单个文件
	 * 
	 * @param filePath
	 *            要解析的源文件路径
	 * @param targetFilePath
	 *            解析后的文件路径
	 * @param isAppend
	 *            是否为内容追加，不追加则会覆盖内容
	 */
	public abstract void parseFile(String filePath, String targetFilePath, boolean isAppend);

	/**
	 * 合并解析多个文件
	 * 
	 * @param fileDirPath
	 *            要解析的源文件夹路径
	 * @param targetFilePath
	 *            解析后的文件路径
	 * @param isAppend
	 *            是否为内容追加，不追加则会覆盖内容
	 * @throws IOException 
	 */
	public abstract void parseFiles(String fileDirPath, String targetFilePath, boolean isAppend) throws IOException;

	/**
	 * 解析单个或者多个文件，如果是多个文件则生成对应的txt文件，{@link #setTargetDir(String)}，
	 * 如果targetDir不设置，则在当前文件夹下生成相应的txt文件
	 * 
	 * @param filePath
	 *            源文件路径
	 * @param isAppend
	 *            false：覆盖内容 true：附加内容
	 */
	public abstract void parseFile(String filePath, boolean isAppend);
	
	/**
	 * 创建文件夹
	 * 
	 * @param filePath
	 *            目标文件
	 * @return
	 */
	protected void createParentDir(String targetFilePath) {
		if (!targetFilePath.endsWith(".txt")) {
			throw new IllegalStateException("文件格式错误，后缀必须为.txt，此格式为   " + targetFilePath);
		}
		String path = targetFilePath.substring(0, targetFilePath.lastIndexOf("/") + 1);
		File file = new File(path);
		if (!file.exists()) {
			file.mkdirs();
		}
	}
	
	/**
	 * 解析单个文件
	 * @param filePath   文件路径
	 */
	public void parseFile(String filePath){
		parseFile(filePath,false);
	}

	public String getTargetDir() {
		return targetDir;
	}

	/**
	 * 解析后的txt文件存放路径
	 * 
	 * @param targetDir
	 *            文件夹路径
	 */
	public void setTargetDir(String targetDir) {
		this.targetDir = targetDir;
	}

}

2、SougouScelFileProcessing 继承FileProcessing

package com.hxl.files;

import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;


public class SougouScelFileProcessing extends FileProcessing {
	private static final Logger log = Logger.getLogger("SougouScelFileProcessing");
	protected static String encoding = "UTF-16LE";
	protected ByteArrayOutputStream output = new ByteArrayOutputStream();

	/**
	 * 解析单个或者多个文件，如果是多个文件则生成对应的txt文件，{@link #setTargetDir(String)}，
	 * 如果targetDir不设置，则在当前文件夹下生成相应的txt文件
	 * 
	 * @param filePath
	 *            源文件路径
	 * @param isAppend
	 *            false：覆盖内容 true：附加内容
	 */
	public void parseFile(String filePath, boolean isAppend) {
		File file = new File(filePath);
		if (file.isDirectory()) {
			File items[] = file.listFiles();
			for (int i = 0; i < items.length; i++) {
				if (!items[i].getName().endsWith(".scel")) {
					continue;
				}

				if (targetDir == null) {
					parseFile(items[i].getAbsolutePath(), items[i].getAbsolutePath().replace(".scel", ".txt"),
							isAppend);
				} else {
					parseFile(items[i].getAbsolutePath(), targetDir + "/" + items[i].getName().replace(".scel", ".txt"),
							isAppend);
				}

			}
		} else {
			parseFile(filePath, file.getAbsolutePath().replace(".scel", ".txt"), isAppend);
		}

	}

	/**
	 * 解析单个scel文件
	 * 
	 * @param filePath
	 *            源文件路径
	 * @param targetFilePath目标文件路径
	 * @param isAppend
	 *            false：覆盖内容 true：附加内容
	 */
	public void parseFile(String filePath, String targetFilePath, boolean isAppend) {
		if (!targetFilePath.endsWith(".txt")) {
			throw new IllegalStateException("文件格式错误，后缀必须为.txt，此格式为   " + targetFilePath);
		}
		if (!filePath.endsWith(".scel")) {
			return;
		}
		File input = new File(filePath);
		if (input.length() < 8) {
			// 假如文件小于8字节，不去考虑它
			return;
		}
		FileInputStream in = null;
		SougouScelMdel model = null;
		try {
			in = new FileInputStream(input);
			model = read(in);
			if (model == null) {
				return;
			}
			writeToTargetFile(model, targetFilePath, isAppend);
		} catch (IOException e) {
			log.log(Level.SEVERE, e.getMessage());
			e.printStackTrace();
		}

	}

	/**
	 * 解析多个文件夹，将解析后的内容放到一个文件里
	 * 
	 * @param fileDirPath
	 *            源文件夹路径
	 * @param targetFilePath
	 *            目标文件路径
	 * @param isAppend
	 *            false：覆盖内容 true：附加内容
	 * @throws FileNotFoundException
	 */
	public void parseFiles(String fileDirPath, String targetFilePath, boolean isAppend) throws IOException {
		if (!targetFilePath.endsWith(".txt")) {
			throw new IllegalStateException("文件格式错误，后缀必须为.txt，此格式为   " + targetFilePath);
		}
		File dir = new File(fileDirPath);
		if (!dir.exists() || !dir.isDirectory()) {
			throw new IllegalStateException("scel文件夹路径错误   " + targetFilePath);
		}
		File scels[] = dir.listFiles();
		ArrayList models = new ArrayList<>();
		for (int i = 0; i < scels.length; i++) {
			if (!scels[i].getName().endsWith(".scel")) {
				continue;
			}
			FileInputStream in = null;
			SougouScelMdel model = null;
			in = new FileInputStream(scels[i]);
			model = read(in);
			if (model != null) {
				models.add(model);
			}
		}
		writeToTargetFile(models, targetFilePath, isAppend);
	}

	private void writeToTargetFile(SougouScelMdel model, String targetFilePath, boolean isAppend) throws IOException {
		List models = new ArrayList<>();
		models.add(model);
		writeToTargetFile(models, targetFilePath, isAppend);

	}

	/**
	 * 将搜狗scel文件解析后的内容写入txt文件
	 * 
	 * @param models
	 * @param targetFilePath
	 * @param isAppend
	 * @throws IOException
	 */
	private void writeToTargetFile(List models, String targetFilePath, boolean isAppend)
			throws IOException {
		createParentDir(targetFilePath);
		FileOutputStream out = new FileOutputStream(targetFilePath, isAppend);
		int count = 0;
		for (int k = 0; k < models.size(); k++) {
			Map> words = models.get(k).getWordMap(); // 词<拼音,词>
			Set>> set = words.entrySet();
			Iterator>> iter = set.iterator();
			if (isAppend) {
				out.write("\r\n".getBytes());
			}
			while (iter.hasNext()) {
				Entry> entry = iter.next();
				List list = entry.getValue();

				int size = list.size();
				for (int i = 0; i < size; i++) {
					String word = list.get(i);
					out.write((entry.getKey() + " ").getBytes());
					out.write((word + "\n").getBytes());// 写入txt文件
					count++;

				}
			}

		}
		out.close();
		log.info("生成" + targetFilePath.substring(targetFilePath.lastIndexOf("/") + 1) + "成功！,总计写入: " + count + " 条数据！");

	}

	private SougouScelMdel read(InputStream in) {
		SougouScelMdel model = new SougouScelMdel();
		DataInputStream input = new DataInputStream(in);
		int read;
		try {
			byte[] bytes = new byte[4];
			input.readFully(bytes);
			assert (bytes[0] == 0x40 && bytes[1] == 0x15 && bytes[2] == 0 && bytes[3] == 0);
			input.readFully(bytes);
			int flag1 = bytes[0];
			assert (bytes[1] == 0x43 && bytes[2] == 0x53 && bytes[3] == 0x01);
			int[] reads = new int[] { 8 };
			model.setName(readString(input, 0x130, reads));
			model.setType(readString(input, 0x338, reads));
			model.setDescription(readString(input, 0x540, reads));
			model.setSample(readString(input, 0xd40, reads));
			read = reads[0];
			input.skip(0x1540 - read);
			read = 0x1540;
			input.readFully(bytes);
			read += 4;
			assert (bytes[0] == (byte) 0x9D && bytes[1] == 0x01 && bytes[2] == 0 && bytes[3] == 0);
			bytes = new byte[128];
			Map pyMap = new LinkedHashMap();
			while (true) {
				int mark = readUnsignedShort(input);
				int size = input.readUnsignedByte();
				input.skip(1);
				read += 4;
				assert (size > 0 && (size % 2) == 0);
				input.readFully(bytes, 0, size);
				read += size;
				String py = new String(bytes, 0, size, encoding);
				// System.out.println(py);
				pyMap.put(mark, py);
				if ("zuo".equals(py)) {
					break;
				}
			}
			if (flag1 == 0x44) {
				input.skip(0x2628 - read);
			} else if (flag1 == 0x45) {
				input.skip(0x26C4 - read);
			} else {
				throw new RuntimeException("出现意外，联系作者");
			}
			StringBuffer buffer = new StringBuffer();
			Map> wordMap = new LinkedHashMap>();
			while (true) {
				int size = readUnsignedShort(input);
				if (size < 0) {
					break;
				}
				int count = readUnsignedShort(input);
				int len = count / 2;
				assert (len * 2 == count);
				buffer.setLength(0);
				for (int i = 0; i < len; i++) {
					int key = readUnsignedShort(input);
					buffer.append(pyMap.get(key)).append("'");
				}
				buffer.setLength(buffer.length() - 1);
				String py = buffer.toString();
				List list = wordMap.get(py);
				if (list == null) {
					list = new ArrayList();
					wordMap.put(py, list);
				}
				for (int i = 0; i < size; i++) {
					count = readUnsignedShort(input);
					if (count > bytes.length) {
						bytes = new byte[count];
					}
					input.readFully(bytes, 0, count);
					String word = new String(bytes, 0, count, encoding);
					// 接下来12个字节可能是词频或者类似信息
					input.skip(12);
					list.add(word);
				}
			}
			model.setWordMap(wordMap);
			return model;
		} catch (IOException e) {
			log.log(Level.SEVERE, e.getMessage());
			e.printStackTrace();
		} finally {
			try {
				in.close();
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}
		return null;
	}

	protected String readString(DataInputStream input, int pos, int[] reads) throws IOException {
		int read = reads[0];
		input.skip(pos - read);
		read = pos;
		output.reset();
		while (true) {
			int c1 = input.read();
			int c2 = input.read();
			read += 2;
			if (c1 == 0 && c2 == 0) {
				break;
			} else {
				output.write(c1);
				output.write(c2);
			}
		}
		reads[0] = read;
		return new String(output.toByteArray(), encoding);
	}

	protected final int readUnsignedShort(InputStream in) throws IOException {
		int ch1 = in.read();
		int ch2 = in.read();
		if ((ch1 | ch2) < 0) {
			return Integer.MIN_VALUE;
		}
		return (ch2 << 8) + (ch1 << 0);
	}

}

3、SougouScelModel

package com.hxl.files;

import java.util.List;
import java.util.Map;

public class SougouScelModel {
	private Map> wordMap;

    private String name;
    private String type;
    private String description;
    private String sample;

    public Map> getWordMap() {
        return wordMap;
    }

    void setWordMap(Map> wordMap) {
        this.wordMap = wordMap;
    }

    public String getType() {
        return type;
    }

    public void setType(String type) {
        this.type = type;
    }

    public String getDescription() {
        return description;
    }

    public void setDescription(String description) {
        this.description = description;
    }

    public String getSample() {
        return sample;
    }

    public void setSample(String sample) {
        this.sample = sample;
    }

    public String getName() {
        return name;
    }

    public void setName(String name) {
        this.name = name;
    }
}

4、TxtFileProcessing 继承于FileProcessing

package com.hxl.files;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;


public class TxtFileProcessing extends FileProcessing {
	private static final Logger log = Logger.getLogger("TxtFileProcessing");
	// 文字编码
	private String encoding = "UTF-8";

	@Override
	public void parseFile(String filePath, String targetFilePath, boolean isAppend) {
		if (!targetFilePath.endsWith(".txt")) {
			throw new IllegalStateException("文件格式错误，后缀必须为.txt，此格式为   " + targetFilePath);
		}
		if (!filePath.endsWith(".txt")) {
			return;
		}
		File inputFile = new File(filePath);
		if (!inputFile.exists()) {
			log.log(Level.SEVERE,filePath + "   文件不存在");
		} else {
			ArrayList content = new ArrayList<>();
			HashSet set = new HashSet<>();
			createParentDir(targetFilePath);
			File outputFile = new File(targetFilePath);
			if (!isAppend) {
				// 假如不是附加内容,删除
				if (outputFile.exists()) {
					log.info(outputFile.getAbsolutePath() + "   文件存在，删除...");
					outputFile.delete();
				}
			} else {
				// 读取原有的txt文件内容
				content.addAll(readTargetFile(outputFile));
			}
			content.addAll(readSourceFile(inputFile));
			// 去重
			for (int i = 0; i < content.size(); i++) {
				set.add(content.get(i));
			}
			// 写入目标文件
			writeToTargetFile(set, outputFile);

		}

	}

	@Override
	public void parseFile(String filePath, boolean isAppend) {
		File file = new File(filePath);
		if (file.isDirectory()) {
			File items[] = file.listFiles();
			for (int i = 0; i < items.length; i++) {
				if (!items[i].getName().endsWith(".txt")) {
					continue;
				}

				if (targetDir == null) {
					parseFile(items[i].getAbsolutePath(), items[i].getAbsolutePath().replace(".txt", "解析.txt"),
							isAppend);
				} else {
					parseFile(items[i].getAbsolutePath(), targetDir + "/" + items[i].getName(),
							isAppend);
				}

			}
		} else {
			parseFile(filePath, file.getAbsolutePath().replace(".txt", "解析.txt"), isAppend);
		}
	}
	
	@Override
	public void parseFiles(String fileDirPath, String targetFilePath, boolean isAppend) throws IOException {
		if (!targetFilePath.endsWith(".txt")) {
			throw new IllegalStateException("文件格式错误，后缀必须为.txt，此格式为   " + targetFilePath);
		}
		File fileDir = new File(fileDirPath);
		if (!fileDir.isDirectory() || !fileDir.exists()) {
			throw new IllegalStateException("文件夹路径错误   " + targetFilePath);
		}
		File file[] = fileDir.listFiles();
		ArrayList content = new ArrayList<>();
		HashSet set = new HashSet<>();
		createParentDir(targetFilePath);
		File outputFile = new File(targetFilePath);
		if (!isAppend) {
			// 假如不是附加内容,删除
			if (outputFile.exists()) {
				log.info(outputFile.getAbsolutePath() + "   文件存在，删除...");
				outputFile.delete();
			}
		} else {
			// 读取原有的txt文件内容
			content.addAll(readSourceFile(outputFile));
		}
		for (int i = 0; i < file.length; i++) {
			if (file[i].getName().endsWith(".txt")) {
				content.addAll(readSourceFile(file[i]));
			}
		}
		// 去重
		for (int i = 0; i < content.size(); i++) {
			set.add(content.get(i));
		}
		// 写入目标文件
		writeToTargetFile(set, outputFile);

	}

	/**
	 * 将内容写入目标文件
	 * 
	 * @param set
	 *            词库合集
	 * @param outputFile
	 *            目标文件
	 */
	private void writeToTargetFile(HashSet set, File outputFile) {
		StringBuffer buff = new StringBuffer();
		for (String content : set) {
			buff.append(content);
			buff.append("\r\n");
		}
		String content = buff.toString();

		FileOutputStream out = null;
		try {
			out = new FileOutputStream(outputFile);
			out.write(content.getBytes());

		} catch (IOException e) {
			log.log(Level.SEVERE, e.getMessage());
			e.printStackTrace();
		} finally {
			try {
				out.close();
			} catch (IOException e) {
				log.log(Level.SEVERE, e.getMessage());
				e.printStackTrace();
			}
		}
		log.info("生成" + outputFile.getName() + "成功！,总计写入: " + set.size() + " 条数据！");
	}

	

	

	/**
	 * 读取源文件，获取中文词库
	 * 
	 * @param file
	 *            源文件
	 * @return 中文词库集合
	 */
	private List readSourceFile(File file) {
		ArrayList content = new ArrayList<>();
		try {
			if (file.isFile() && file.exists()) { // 判断文件是否存在
				InputStreamReader read = new InputStreamReader(new FileInputStream(file), encoding);// 考虑到编码格式
				BufferedReader bufferedReader = new BufferedReader(read);
				String lineTxt = null;

				while ((lineTxt = bufferedReader.readLine()) != null) {
					String newStr = new String(lineTxt.getBytes("UTF-8"));
					String split[] = newStr.split(" ");
					for (int i = 0; i < split.length; i++) {
						if (i % 2 == 0) {
							// 拼音字母
						} else {
							// 中文词库
							content.add(split[i]);
						}
					}
				}
				bufferedReader.close();
				read.close();
			} else {
				log.log(Level.SEVERE, "找不到源文件   " + file.getAbsolutePath());
			}
		} catch (Exception e) {
			log.log(Level.SEVERE, e.getMessage());
			e.printStackTrace();
		}
		return content;

	}

	/**
	 * 读取已解析好的的词库文件
	 * 
	 * @param file
	 *            词库文件
	 * @return 词库内容
	 */
	private List readTargetFile(File file) {
		ArrayList content = new ArrayList<>();
		try {
			if (file.isFile() && file.exists()) { // 判断文件是否存在
				InputStreamReader read = new InputStreamReader(new FileInputStream(file), encoding);// 考虑到编码格式
				BufferedReader bufferedReader = new BufferedReader(read);
				String lineTxt = null;

				while ((lineTxt = bufferedReader.readLine()) != null) {
					String newStr = new String(lineTxt.getBytes("UTF-8"));
					if (!newStr.trim().isEmpty()) {
						content.add(newStr);
					}
				}
				bufferedReader.close();
				read.close();
			} else {
				System.err.println("找不到目标文件  " + file.getAbsolutePath());
			}
		} catch (Exception e) {
			log.log(Level.SEVERE, e.getMessage());
			e.printStackTrace();
		}
		return content;

	}

}