瓦片数据入库之MongoDB + Java 版本

本示例暂时使用了GridFS的存储方式，目前还没有很好的考虑大数据量下，瓦片索引的机制

PackageManager.java

import java.io.File;
import java.net.UnknownHostException;
import java.util.Queue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;

import com.mongodb.DB;
import com.mongodb.Mongo;
import com.mongodb.MongoException;
import com.mongodb.gridfs.GridFS;
import com.mongodb.gridfs.GridFSInputFile;

public class PackageManager {
	private final ExecutorService mThreadPool;

	private final Queue<File> mQueue = new LinkedBlockingQueue<File>();
	Mongo mg;
	DB db;
	GridFS gridFS;
	// ===========================================================
	// Constructors
	// ===========================================================

	public PackageManager(final int mThreads) {
		
		this.mThreadPool = Executors.newFixedThreadPool(mThreads);
		
		try {
            mg = new Mongo();
            //mg = new Mongo("localhost", 27017);
        } catch (UnknownHostException e) {
            e.printStackTrace();
        } catch (MongoException e) {
            e.printStackTrace();
        }
        db = mg.getDB("beijingImg");
        gridFS= new GridFS(db);
	}

	// ===========================================================
	// Getter & Setter
	// ===========================================================

	public synchronized void add(final File pTileInfo){
		this.mQueue.add(pTileInfo);
		spawnNewThread();
	}

	private synchronized File getNext(){
		final File tile = this.mQueue.poll();

		final int remaining = this.mQueue.size();
		if(remaining % 100 == 0 && remaining > 0) {
			System.out.print("(" + remaining +")");
		} else {
			System.out.print(".");
		}

		this.notify();
		return tile;
	}

	public synchronized void waitEmpty() throws InterruptedException {
		while(this.mQueue.size() > 0){
			this.wait();
		}
	}

	public void waitFinished() throws InterruptedException {
		waitEmpty();
		this.mThreadPool.shutdown();
		this.mThreadPool.awaitTermination(6, TimeUnit.HOURS);
	}

	// ===========================================================
	// Methods from SuperClass/Interfaces
	// ===========================================================

	// ===========================================================
	// Methods
	// ===========================================================

	private void spawnNewThread() {
		this.mThreadPool.execute(new DownloadRunner());
	}

	// ===========================================================
	// Inner and Anonymous Classes
	// ===========================================================


	private class DownloadRunner implements Runnable {

		private File mTile;
		private String saveFileName;
		//private File mDestinationFile;

		public DownloadRunner() {
		}

		private void init(final File pTileInfo) {
			this.mTile = pTileInfo;
			
			String fileName = mTile.getName().substring(0, mTile.getName().indexOf("."));
			//System.out.println("fileName：" + fileName);
			
			
			String row = mTile.getParentFile().getPath().substring(mTile.getParentFile().getPath().lastIndexOf("\\") + 1);
			//System.out.println("row：" + row);
			
			String level = mTile.getParentFile().getParentFile().getPath().substring(mTile.getParentFile().getParentFile().getPath().lastIndexOf("\\") + 1);
			//System.out.println("level：" + level);
			
			
			saveFileName = level + "_" + row + "_" + fileName;
			
			
		}

		@Override
		public void run() {
			try{
				init(PackageManager.this.getNext());
				// gridFS= new GridFS(db);
				GridFSInputFile gfs = gridFS.createFile(mTile);
				//GridFSInputFile gridFSInputFile = myFS.createFile(inputStream);
				
				gfs.put("filename", saveFileName);
				gfs.save();
			}catch(Exception e){
				
			}
		}
	}
}

TilePackager.java

import java.io.File;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;

import org.junit.Test;

public class TilePackager {
	
	public static List<File> files = new ArrayList<File>();
	PackageManager packageManager;
	public int count = 0;
	public static void main(String[] args) {
		/*File ss = new File("E:\\瓦片数据\\北京影像");
		
		String fileName = ss.getName().substring(0, ss.getName().indexOf("."));
		System.out.println("fileName：" + fileName);
		
		
		String row = ss.getParentFile().getPath().substring(ss.getParentFile().getPath().lastIndexOf("\\") + 1);
		System.out.println("row：" + row);
		
		String level = ss.getParentFile().getParentFile().getPath().substring(ss.getParentFile().getParentFile().getPath().lastIndexOf("\\") + 1);
		System.out.println("level：" + level);*/
		
		//入库
		/*TilePackager tp = new TilePackager();
		tp.packageManager = new PackageManager(30);
		
		tp.getTiles(new File("E:\\瓦片数据\\北京影像"));
		System.out.println(files.size());*/
		
		TilePackager tp = new TilePackager();
		tp.getDirectorySize(new File("E:\\瓦片数据\\北京矢量"), tp.list);
	}
	//测试成功
	@Test
	public void listFiles() {
		long count = 0;
		long countd = 0;
		// 链表
		LinkedList<File> list = new LinkedList<File>();
		File dir = new File("D:\\opt");
		File[] file = dir.listFiles();
		for (int i = 0; i < file.length; i++) {
			if (file[i].isDirectory())
				// 把第一层的目录，全部放入链表
				list.add(file[i]);
			else
				count++;
			System.out.println("文件" + count + ":" + file[i].getAbsolutePath());
		}
		File tmp = null;
		// 循环遍历链表
		while (!list.isEmpty()) {
			// 把链表的第一个记录删除
			tmp = list.removeFirst();
			// 如果删除的目录是一个路径的话
			if (tmp.isDirectory()) {
				// 列出这个目录下的文件到数组中
				file = tmp.listFiles();
				if (file == null)
					continue;
				// 遍历文件数组
				for (int i = 0; i < file.length; i++) {
					if (file[i].isDirectory())
						// 如果遍历到的是目录，则继续加入链表
						list.add(file[i]);
					else
						count++;
					System.out.println("文件" + count + ":" + file[i].getAbsolutePath());
				}
			} else {
				countd++;
				System.out.println("目录[" + countd + "]路径:" + tmp.getAbsolutePath());
			}
		}
	}
	
	public void getTiles(File file){
		File[] files = file.listFiles();
		if (files == null || files.length == 0) {
			return;
		}
		for(File childFile : files){
			if(childFile.isDirectory()){
				getTiles(childFile);
			}else{
				String fileName = childFile.getName();
				String filePath = childFile.getPath();
				if(fileName.endsWith("png") || fileName.endsWith("jpg")){
					packageManager.add(childFile);
				}
			}
		}
	}
	
	LinkedList<File> list = new LinkedList<File>(); // 保存待遍历文件夹的列表
	// 非递归遍历
	private void getDirectorySize(File file, LinkedList<File> list) {
		
		getOneDir(file, list); // 调用遍历文件夹根目录文件的方法

		File tmp;
		while (!list.isEmpty()) {
			tmp = (File) list.removeFirst();
			// 这个地方的判断有点多余，但是为了保险还是给个判断了，正常情况列表中是只有文件夹的
			// 但是不排除特殊情况，例如：本身是文件夹的目标在压入堆栈之后变成了文件
			if (tmp.isDirectory()) {
				getDirectorySize(tmp, list);
			} else {
				System.out.println("file==>" + tmp.getAbsolutePath());
			}
		}
	}

	int sumdir = 0;

	// 遍历指定文件夹根目录下的文件
	private void getOneDir(File file, LinkedList<File> list) {
		// 每个文件夹遍历都会调用该方法
		System.out.println("Dir==>" + file.getAbsolutePath());
		File[] files = file.listFiles();
		sumdir += 1;
		if (files == null || files.length == 0) {
			return;
		}

		for (File f : files) {
			if (f.isDirectory()) {
				list.add(f);
			} else {
				// 这里列出当前文件夹根目录下的所有文件
				System.out.println("file==>" + f.getAbsolutePath());
			}
		}
	}
}