计算一个文件大小的几种实现方式
1、串行计算
import java.io.File; /** * 串行计算 */ public class TotalFileSizeSequential { private long getTotalSizeOfFilesInDir(File file) { if(file.isFile()) return file.length(); long length = 0; File []children = file.listFiles(); if(children != null) for(final File child : children) { length += getTotalSizeOfFilesInDir(child); } return length; } public static void main(String[] args) { final long start = System.nanoTime(); final File file = new File("C:\\Windows"); TotalFileSizeSequential total = new TotalFileSizeSequential(); long size = total.getTotalSizeOfFilesInDir(file); final long end = System.nanoTime(); System.out.println("total size:" + size); System.out.println("total token:"+(end - start) /1.0e9); } }
2、线程协作，分子任务，启动新线程（目录不是很深的时候没问题）
import java.io.File; import java.util.ArrayList; import java.util.List; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; /** * 每当扫描一个子任务的时候，就将子任务调度给其它线程 * 目录不是很深的时候没问题 * */ public class NativelyConcurrentTotalFileSize { private long getTotalSizeOfFile(File file) throws InterruptedException, ExecutionException, TimeoutException { ExecutorService service = Executors.newFixedThreadPool(100); long size = getTotalSizeOfFilesInDir(service, file); return size; } private long getTotalSizeOfFilesInDir(final ExecutorService service, final File file) throws InterruptedException, ExecutionException, TimeoutException { if(file.isFile()) return file.length(); long size = 0; File []children = file.listFiles(); if(children != null) { List<Future<Long>> futures = new ArrayList<Future<Long>>() ; for(final File child : children) futures.add(service.submit((new Callable<Long>() { public Long call() throws Exception { return getTotalSizeOfFilesInDir(service, child); } }) )); for(Future<Long> future : futures) { size += future.get(10,TimeUnit.SECONDS); } return size; } return size; } public static void main(String[] args) throws InterruptedException, ExecutionException, TimeoutException { final long start = System.nanoTime(); final File file = new File("C:\\Windows"); NativelyConcurrentTotalFileSize total = new NativelyConcurrentTotalFileSize(); long size = total.getTotalSizeOfFile(file); final long end = System.nanoTime(); System.out.println("total size:" + size); System.out.println("total token:"+(end - start) /1.0e9); } }
3、解决第二种方法出现线程死锁超时的问题，每次计算文件夹中文件大小和保存子目录
import java.io.File; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; public class ConcurrentTotalFileSize { /** * 保存子目录和文件大小（设计思想较为复杂） */ class SubDirectoriesAndSize { public long size; public List<File> subDirectories; public SubDirectoriesAndSize(final long size, final List<File> subDirectories) { this.size = size; this.subDirectories = Collections.unmodifiableList(subDirectories); } } /** * 获取总大小 */ private long getTotalSizeOfFile(File file) throws InterruptedException, ExecutionException, TimeoutException { ExecutorService service = Executors.newFixedThreadPool(100); long size = getTotalSizeOfFilesInDir(service, file); return size; } /** * 获取子目录和文件大小 */ private SubDirectoriesAndSize getSubDirectoriesAndSize(final File file) { long total = 0; List<File> subDirectories = new ArrayList<File>(); if(file.isDirectory()) { File []children = file.listFiles(); if(children != null) for(final File child : children) { if(child.isFile()) total += child.length(); else subDirectories.add(child); } } return new SubDirectoriesAndSize(total, subDirectories); } private long getTotalSizeOfFilesInDir(ExecutorService service, File file) throws InterruptedException, ExecutionException, TimeoutException { if(file.isFile()) return file.length(); long size = 0; List<File> directories = new ArrayList<File>(); directories.add(file); try { while(!directories.isEmpty()) { List<Future<SubDirectoriesAndSize>> futures = new ArrayList<Future<SubDirectoriesAndSize>>() ; for(final File child : directories) { futures.add(service.submit(new Callable<SubDirectoriesAndSize>() { @Override public SubDirectoriesAndSize call() throws Exception { return getSubDirectoriesAndSize(child); } })); } directories.clear();//清除原有子目录 for(Future<SubDirectoriesAndSize> future : futures) { size += future.get().size; directories.addAll(future.get(100,TimeUnit.SECONDS).subDirectories); } } } finally { service.shutdown(); } return size; } public static void main(String[] args) throws InterruptedException, ExecutionException, TimeoutException { final long start = System.nanoTime(); final File file = new File("C:\\Windows");//文件夹路径自定 ConcurrentTotalFileSize total = new ConcurrentTotalFileSize(); long size = total.getTotalSizeOfFile(file); final long end = System.nanoTime(); System.out.println("total size:" + size); System.out.println("total token:"+(end - start) /1.0e9); } }
4、数据交换，使用ArrayBlockingQueue，在线程间互发多组数据
import java.io.File; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicLong; /** * 数据交换 * 使用ArrayBlockingQueue，在线程间互发多组数据 */ public class ConcurrentTotalFileSizeWQueue { private ExecutorService service; final private BlockingQueue<Long> fileSizes = new ArrayBlockingQueue<Long>(500);//简单的阻塞队列 final private AtomicLong pendingFileVisits = new AtomicLong(); private long getTotalSizeOfFile(File file) throws InterruptedException { service = Executors.newFixedThreadPool(100); try { startExploreFile(file); long totalSize = 0; while(pendingFileVisits.get() > 0 || fileSizes.size() > 0) { final long size = fileSizes.poll(10, TimeUnit.SECONDS); totalSize += size; } return totalSize; } finally { service.shutdown(); } } private void startExploreFile(final File file) { pendingFileVisits.incrementAndGet(); service.execute(new Runnable() { public void run() { exploreDir(file); } }); } private void exploreDir(final File file) { long fileSize = 0; if(file.isFile()) fileSize = file.length(); else { File[] children = file.listFiles(); if(children != null) { for(File child : children) { if(child.isFile()) fileSize += child.length(); else startExploreFile(child); } } } try { fileSizes.put(fileSize); } catch (InterruptedException e) { e.printStackTrace(); } pendingFileVisits.decrementAndGet(); } public static void main(String[] args) throws InterruptedException, ExecutionException, TimeoutException { final long start = System.nanoTime(); final File file = new File("C:\\Windows");//文件夹路径自定 ConcurrentTotalFileSizeWQueue total = new ConcurrentTotalFileSizeWQueue(); long size = total.getTotalSizeOfFile(file); final long end = System.nanoTime(); System.out.println("total size:" + size); System.out.println("total token:"+(end - start) /1.0e9); } }
5、CountDownLatch实现，不返回结果，共享状态，代码简洁
/** * CountDownLatch的使用 * 线程不再返回一个结果，AtomicLong提供了更改并取回一个简单long型变量值的线程安全方法 * java.util.concurrent.atomic包中定义的其它原子操作类对于处理单个共享数据的值来说非常有用 */ public class ConcurrentTotalSizeWLatch { private ExecutorService service; private AtomicLong pendingFileVisits = new AtomicLong();//作用是提供当前当前待访问的文件（或子目录）的数量 private AtomicLong totalSize = new AtomicLong();//计算大小 private CountDownLatch latch = new CountDownLatch(1); private long getTotalSizeOfFile(File file) { service = Executors.newFixedThreadPool(100); pendingFileVisits.incrementAndGet(); updateTotalSizeOfFileInDir(file); try { latch.await(100, TimeUnit.SECONDS);//当前线程等待直到latch减为0，除非线程被打断或超时 } catch (InterruptedException e) { e.printStackTrace(); } finally { service.shutdown(); } return totalSize.longValue(); } private void updateTotalSizeOfFileInDir(File file) { long fileSize = 0; if(file.isFile()) fileSize = file.length(); if(file.isDirectory()) { File[] children = file.listFiles(); if(children != null) { for(final File child : children) { if(file.isFile()) fileSize += child.length(); else { pendingFileVisits.incrementAndGet(); service.execute(new Runnable() { public void run() { updateTotalSizeOfFileInDir(child); } }); } } } } totalSize.addAndGet(fileSize); if(pendingFileVisits.decrementAndGet() == 0) latch.countDown();//释放线程闩 } public static void main(String[] args) { final long start = System.nanoTime(); final File file = new File("C:\\Windows"); ConcurrentTotalSizeWLatch total = new ConcurrentTotalSizeWLatch(); long size = total.getTotalSizeOfFile(file); final long end = System.nanoTime(); System.out.println("total size:" + size); System.out.println("total token:"+(end - start) /1.0e9); }
此学习内容来自<Java虚拟机并发编程>一书。