使用Java的WatchService监控文件上传
很多有效的业务系统中，采用了传说中万能的dirwatch解决方案，所以讨论在java下的对目录下的文件监控是挺有意义的事情。
WatchService里面提供了对文件夹监控的标准接口WatchService，但是这个接口只提供了Delete,Modify和Create三种事件的监控。
之前，我们一直使用的JNotify，也是只提供了这几个接口事件。
如果我们在Java的网站中，对用户刚刚上传完毕的文件进行立即处理的情况下， 应对对文件的完整性进行确认。
WatchService里面提供给我们的三个事件，是否能让我们判断这个文件是否完整呢？
我写了一个测试用例，代码如下：
/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
package fswatch;
import com.kamike.message.FsWatchService;
/**
*
* @author THiNk
*/
public class FsWatch {
/**
* @param args the command line arguments
*/
public static void main(String[] args) {
// TODO code application logic here
FsWatchService.start("D:\\temp\\");
}
}
/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
package com.kamike.message.fswatch;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
public class PathEvents {
private final List<PathEvent> pathEvents = new ArrayList<PathEvent>();
private final Path watchedDirectory;
private final boolean isValid;
PathEvents(boolean valid, Path watchedDirectory) {
isValid = valid;
this.watchedDirectory = watchedDirectory;
}
public boolean isValid(){
return isValid;
}
public Path getWatchedDirectory(){
return watchedDirectory;
}
public List<PathEvent> getEvents() {
return Collections.unmodifiableList(pathEvents);
}
public void add(PathEvent pathEvent) {
pathEvents.add(pathEvent);
}
}
/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
package com.kamike.message.fswatch;
import java.nio.file.Path;
import java.nio.file.WatchEvent;
/**
*
* @author THiNk
*/
public class PathEvent {
private final Path eventTarget;
private final WatchEvent.Kind type;
PathEvent(Path eventTarget, WatchEvent.Kind type) {
this.eventTarget = eventTarget;
this.type = type;
}
public Path getEventTarget() {
return eventTarget;
}
public WatchEvent.Kind getType() {
return type;
}
}
/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
package com.kamike.message.fswatch;
import com.google.common.base.Function;
import java.io.IOException;
import java.nio.file.FileVisitResult;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
/**
*
* @author THiNk
*/
public class FunctionVisitor extends SimpleFileVisitor<Path> {
Function<Path,FileVisitResult> pathFunction;
public FunctionVisitor(Function<Path, FileVisitResult> pathFunction) {
this.pathFunction = pathFunction;
}
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
return pathFunction.apply(file);
}
}
/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
package com.kamike.message.fswatch;
import com.google.common.eventbus.EventBus;
import java.io.IOException;
import java.nio.file.FileSystems;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import static java.nio.file.StandardWatchEventKinds.ENTRY_CREATE;
import static java.nio.file.StandardWatchEventKinds.ENTRY_DELETE;
import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY;
import java.nio.file.WatchEvent;
import java.nio.file.WatchKey;
import java.nio.file.WatchService;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.FutureTask;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
*
* @author THiNk
*/
public class FsWatcher {
private FutureTask<Integer> watchTask;
private EventBus eventBus;
private WatchService watchService;
private volatile boolean keepWatching = true;
private Path startPath;
public FsWatcher(EventBus eventBus, Path startPath) {
this.eventBus = Objects.requireNonNull(eventBus);
this.startPath = Objects.requireNonNull(startPath);
}
public void start() throws IOException {
initWatchService();
registerDirectories();
createWatchTask();
startWatching();
}
public boolean isRunning() {
return watchTask != null && !watchTask.isDone();
}
public void stop() {
keepWatching = false;
}
public void close()
{
try {
this.stop();
this.watchService.close();
} catch (IOException ex) {
Logger.getLogger(FsWatcher.class.getName()).log(Level.SEVERE, null, ex);
}
}
//Used for testing purposes
Integer getEventCount() {
try {
return watchTask.get();
} catch (InterruptedException e) {
throw new RuntimeException(e);
} catch (ExecutionException ex) {
throw new RuntimeException(ex);
}
}
private void createWatchTask() {
watchTask = new FutureTask<Integer>(new Callable<Integer>() {
private int totalEventCount;
@Override
public Integer call() throws Exception {
while (keepWatching) {
WatchKey watchKey = watchService.poll(10, TimeUnit.SECONDS);
if (watchKey != null) {
List<WatchEvent<?>> events = watchKey.pollEvents();
Path watched = (Path) watchKey.watchable();
// PathEvents pathEvents = new PathEvents(watchKey.isValid(), watched);
// for (WatchEvent event : events) {
// pathEvents.add(new PathEvent((Path) event.context(), event.kind()));
// totalEventCount++;
// }
// watchKey.reset();
//
for(WatchEvent event : events)
{
PathEvent pathEvent= new PathEvent((Path) event.context(), event.kind());
eventBus.post(pathEvent);
}
watchKey.reset();
}
}
return totalEventCount;
}
});
}
private void startWatching() {
new Thread(watchTask).start();
}
private void registerDirectories() throws IOException {
Files.walkFileTree(startPath, new WatchServiceRegisteringVisitor());
}
private WatchService initWatchService() throws IOException {
if (watchService == null) {
watchService = FileSystems.getDefault().newWatchService();
}
return watchService;
}
private class WatchServiceRegisteringVisitor extends SimpleFileVisitor<Path> {
@Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
//dir.register(watchService, ENTRY_CREATE, ENTRY_DELETE, ENTRY_MODIFY);
dir.register(watchService, ENTRY_CREATE, ENTRY_DELETE, ENTRY_MODIFY);
return FileVisitResult.CONTINUE;
}
}
}
/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
package com.kamike.message.fswatch;
import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
import java.util.Iterator;
import java.util.concurrent.Callable;
import java.util.concurrent.FutureTask;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;
/**
*
* @author THiNk
*/
public class FileDirectoryStream {
File startDirectory;
String pattern;
private LinkedBlockingQueue<File> fileLinkedBlockingQueue = new LinkedBlockingQueue<File>();
private boolean closed = false;
private FutureTask<Void> fileTask;
private FilenameFilter filenameFilter;
public FileDirectoryStream(String pattern, File startDirectory) {
this.pattern = pattern;
this.startDirectory = startDirectory;
this.filenameFilter = getFileNameFilter(pattern);
}
public Iterator<File> glob() throws IOException {
confirmNotClosed();
startFileSearch(startDirectory, filenameFilter);
return new Iterator<File>() {
File file = null;
@Override
public boolean hasNext() {
try {
file = fileLinkedBlockingQueue.poll();
while (!fileTask.isDone() && file == null) {
file = fileLinkedBlockingQueue.poll(5, TimeUnit.MILLISECONDS);
}
return file != null;
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
return false;
}
@Override
public File next() {
return file;
}
@Override
public void remove() {
throw new UnsupportedOperationException("Remove not supported");
}
};
}
private void startFileSearch(final File startDirectory, final FilenameFilter filenameFilter) {
fileTask = new FutureTask<Void>(new Callable<Void>() {
@Override
public Void call() throws Exception {
findFiles(startDirectory, filenameFilter);
return null;
}
});
start(fileTask);
}
private void findFiles(final File startDirectory, final FilenameFilter filenameFilter) {
File[] files = startDirectory.listFiles(filenameFilter);
for (File file : files) {
if (!fileTask.isCancelled()) {
if (file.isDirectory()) {
findFiles(file, filenameFilter);
}
fileLinkedBlockingQueue.offer(file);
}
}
}
private FilenameFilter getFileNameFilter(final String pattern) {
return new FilenameFilter() {
Pattern regexPattern = Pattern.compile(pattern);
@Override
public boolean accept(File dir, String name) {
return new File(dir, name).isDirectory() || regexPattern.matcher(name).matches();
}
};
}
public void close() throws IOException {
if (fileTask != null) {
fileTask.cancel(true);
}
fileLinkedBlockingQueue.clear();
fileLinkedBlockingQueue = null;
fileTask = null;
closed = true;
}
private void start(FutureTask<Void> futureTask) {
new Thread(futureTask).start();
}
private void confirmNotClosed() {
if (closed) {
throw new IllegalStateException("File Iterator has already been closed");
}
}
}
/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
package com.kamike.message.fswatch;
import com.google.common.base.Function;
import java.io.IOException;
import java.nio.file.DirectoryStream;
import java.nio.file.DirectoryStream.Filter;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Iterator;
import java.util.Objects;
import java.util.concurrent.Callable;
import java.util.concurrent.FutureTask;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
/**
*
* @author THiNk
*/
public class AsynchronousRecursiveDirectoryStream implements DirectoryStream<Path> {
private LinkedBlockingQueue<Path> pathsBlockingQueue = new LinkedBlockingQueue<Path>();
private boolean closed = false;
private FutureTask<Void> pathTask;
private Path startPath;
private Filter filter;
public AsynchronousRecursiveDirectoryStream(Path startPath, String pattern) throws IOException {
this.startPath = Objects.requireNonNull(startPath);
}
@Override
public Iterator<Path> iterator() {
confirmNotClosed();
findFiles(startPath, filter);
return new Iterator<Path>() {
Path path;
@Override
public boolean hasNext() {
try {
path = pathsBlockingQueue.poll();
while (!pathTask.isDone() && path == null) {
path = pathsBlockingQueue.poll(5, TimeUnit.MILLISECONDS);
}
return (path != null);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
return false;
}
@Override
public Path next() {
return path;
}
@Override
public void remove() {
throw new UnsupportedOperationException("Removal not supported");
}
};
}
private void findFiles(final Path startPath, final Filter filter) {
pathTask = new FutureTask<Void>(new Callable<Void>() {
@Override
public Void call() throws Exception {
Files.walkFileTree(startPath, new FunctionVisitor(getFunction(filter)));
return null;
}
});
start(pathTask);
}
private Function<Path, FileVisitResult> getFunction(final Filter<Path> filter) {
return new Function<Path, FileVisitResult>() {
@Override
public FileVisitResult apply(Path input) {
try {
if (filter.accept(input.getFileName())) {
pathsBlockingQueue.offer(input);
}
} catch (IOException e) {
throw new RuntimeException(e.getMessage());
}
return (pathTask.isCancelled()) ? FileVisitResult.TERMINATE : FileVisitResult.CONTINUE;
}
};
}
@Override
public void close() throws IOException {
if(pathTask !=null){
pathTask.cancel(true);
}
pathsBlockingQueue.clear();
pathsBlockingQueue = null;
pathTask = null;
filter = null;
closed = true;
}
private void start(FutureTask<Void> futureTask) {
new Thread(futureTask).start();
}
private void confirmNotClosed() {
if (closed) {
throw new IllegalStateException("DirectoryStream has already been closed");
}
}
}
/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
package com.kamike.message;
import com.google.common.eventbus.AllowConcurrentEvents;
import com.google.common.eventbus.Subscribe;
import com.kamike.message.fswatch.FsWatcher;
import com.kamike.message.fswatch.PathEvent;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.concurrent.locks.ReentrantLock;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.commons.io.FilenameUtils;
/**
*
* @author THiNk
*/
public class FsWatchService {
private static FsWatcher fsw;
private static volatile ReentrantLock lock = new ReentrantLock();
public FsWatchService()
{
}
@Subscribe
@AllowConcurrentEvents
public void proc(PathEvent event)
{
try {
Path path = event.getEventTarget();
String fileName = FilenameUtils.concat("D:\\temp\\", path.toString());
if (fileName.endsWith(".aspx")) {
String fullPath = FilenameUtils.getFullPath(fileName);
String srcName = FilenameUtils.getBaseName(fileName);
}
} catch (Error e) {
e.printStackTrace();
}
}
public static void start(String path) {
if (fsw == null) {
lock.lock();
if (fsw == null) {
try {
fsw = new FsWatcher(EventInst.getInstance().getAsyncEventBus(),
Paths.get(path));
try {
fsw.start();
} catch (IOException ex) {
Logger.getLogger(FsWatchService.class.getName()).log(Level.SEVERE, null, ex);
}
} finally {
lock.unlock();
}
}
}
}
}
/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
package com.kamike.message;
import com.google.common.eventbus.AsyncEventBus;
import com.google.common.eventbus.EventBus;
import java.util.concurrent.Executors;
/**
*
* @author THiNk
*/
public class EventInst {
private static volatile EventInst eventInst = new EventInst();
private EventBus eventBus;
private AsyncEventBus asyncEventBus;
private FsWatchService fs=new FsWatchService();
private EventInst() {
eventBus=new EventBus();
asyncEventBus = new AsyncEventBus(Executors.newFixedThreadPool(50));
asyncEventBus.register(fs);
}
public static EventInst getInstance() {
return eventInst;
}
/**
* @return the eventBus
*/
public EventBus getEventBus() {
return eventBus;
}
/**
* @return the eventBus
*/
public AsyncEventBus getAsyncEventBus() {
return asyncEventBus;
}
}
通过对上面的代码的测试发现：
在Http的文件上传和文件正常拷贝过程中，会触发一次create,一次紧接着create的modify，然后一直传输，最后结束时的触发一次modify.
在真实的业务场景里面，Http的上传，往往文件非常小，最多几M，如果出现中断，用户一般会重传，所以采用记录create,modify，然后等待第二次modify的方法，可以确切的在文件传输完毕后，对第二次modify进行响应，及时进行处理。
而在ftp上传过程中，如果出现了传输中断，或者是文件拷贝过程中被用户终止，在发送这种中断时，也会触发一次modify，此时并不能判断文件是否传完。
在这种情况下，采用对第二次modify进行响应的做法，并不能准确起作用。
但是，某些断点续传功能的ftp server和传输软件（比如迅雷）会在未传输成功的文件的同一个目录下，建立.cfg文件，记录文件的传输进度，当传输成功后，这个.cfg会被删除，这种情况下，可以对这个.cfg的delete事件进行响应。