找出多个文本中频率高的单词（2）
接上篇，我打算用用concurrent包里的CountDownLatch类去实现。
还是直接上代码吧：
Main.java
package com.anders.thread;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
public class Main {
public static void main(String[] args) {
int threadNumber = Integer.parseInt(PropertiesUtil.get("ThreadNumber"));
ExecutorService es = Executors.newFixedThreadPool(threadNumber);
SingleThreadStatistics[] threads = new SingleThreadStatistics[threadNumber];
try {
CountDownLatch doneSignals = new CountDownLatch(threadNumber);
// 这是在 文件数比线程数多的情况下，若文件比线程数少的话，加个判断就可以了
for (int i = 0; i < threadNumber; i++) {
threads[i] = new SingleThreadStatistics(doneSignals);
es.execute(threads[i]);
}
doneSignals.await();
Map<String, Integer> map = mergeThreadMap(threads);
display(map);
} catch (InterruptedException e) {
e.printStackTrace();
} finally {
es.shutdown();
}
}
private static Map<String, Integer> mergeThreadMap(SingleThreadStatistics[] threads) {
Map<String, Integer> map = new HashMap<String, Integer>();
for (SingleThreadStatistics singleThreadStatistics : threads) {
Map<String, Integer> threadMap = singleThreadStatistics.getMap();
for (Map.Entry<String, Integer> entry : threadMap.entrySet()) {
String threadWord = entry.getKey();
Integer threadWordCount = entry.getValue();
Integer wordCount = map.get(threadWord);
if (wordCount == null) {
map.put(threadWord, threadWordCount);
} else {
map.put(threadWord, threadWordCount + wordCount);
}
}
}
return map;
}
private static void display(Map<String, Integer> map) {
for (Map.Entry<String, Integer> entry : map.entrySet()) {
System.out.print(entry.getKey());
System.out.println(" ," + entry.getValue());
}
}
}
SingleThreadStatistics.java
package com.anders.thread;
import java.io.File;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
public class SingleThreadStatistics implements Runnable {
private Map<String, Integer> map = new HashMap<String, Integer>();
private CountDownLatch doneSignals;
public SingleThreadStatistics(CountDownLatch doneSignals) {
this.doneSignals = doneSignals;
}
@Override
public void run() {
while (true) {
File file = FileManager.getFile();
if (file == null) {
break;
}
FileManager.parseFile(file, map);
}
doneSignals.countDown();
}
// --------getter/setter------------
public Map<String, Integer> getMap() {
return map;
}
}
FileManager.java
package com.anders.thread;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Manage files and offer single for every thread
*
* @author Anders
*
*/
public class FileManager {
private static List<File> fileList;
private static int index = 0;
static {
String dirPath = PropertiesUtil.get("DirName");
String path = FileManager.class.getClassLoader().getResource(dirPath).getPath();
fileList = getFiles(path);
}
public synchronized static File getFile() {
if (index == fileList.size()) {
return null;
}
File file = fileList.get(index);
index++;
return file;
}
private static List<File> getFiles(String dirPath) {
File dir = new File(dirPath);
if (!dir.exists() || !dir.isDirectory()) {
return Collections.emptyList();
}
File[] files = dir.listFiles();
//判断 是不是 以txt结尾的文件
Pattern pattern = Pattern.compile(PropertiesUtil.get("FileType"));
List<File> list = new ArrayList<File>();
for (File file : files) {
Matcher matcher = pattern.matcher(file.getName());
if (matcher.matches()) {
list.add(file);
}
}
return list;
}
//读取文件 使用的是java.nio的filechannel 和bytebuffer
public static void parseFile(File file, Map<String, Integer> map) {
FileInputStream ins = null;
try {
ins = new FileInputStream(file);
FileChannel fIns = ins.getChannel();
ByteBuffer buffer = ByteBuffer.allocate(1024);
while (true) {
buffer.clear();
int r = fIns.read(buffer);
if (r == -1) {
break;
}
buffer.flip();
buffer2word(buffer, map);
}
fIns.close();
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
if (ins != null) {
ins.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
//这个是 将读取的内容，提取出 英语字母
private static void buffer2word(ByteBuffer buffer, Map<String, Integer> map) {
StringBuilder str = new StringBuilder();
for (int i = 0; i < buffer.limit(); i++) {
byte b = buffer.get();
if (isEnglishChar(b)) {
str.append((char) b);
} else {
word2map(str.toString(), map);
str = new StringBuilder();
}
}
}
//将 英语单词放到Map中
private static void word2map(String word, Map<String, Integer> map) {
Integer count = map.get(word);
if (null == count) {
map.put(word, 1);
} else {
map.put(word, ++count);
}
}
//看看是否是 英语字符
private static boolean isEnglishChar(byte b) {
//通过ASCLL码 判断
if (b > 65 && b < 91) {
return true;
}
if (b > 97 && b < 123) {
return true;
}
return false;
}
}
ThreadNumber=3 DirName=txt FileType=.*.txt
以上是使用CountDownLatch 实现的 当然也可以使用future+ExecutorService 实现，下一篇就是
其实我觉得最重要的代码是 FileManager里的
public synchronized static File getFile() {
if (index == fileList.size()) {
return null;
}
File file = fileList.get(index);
index++;
return file;
}
这部分代码，因为只要 每个thread 分别得到不同的文件，就可以了。
而且还有一个很重要的一点就是 验证index是否已经读取完所有的文件 要和index++放在一个同步块里面，不然会引起线程安全问题