千万级别瓦片文件夹遍历及导入mongodb

客官°小女子只卖身不卖艺 2022-03-09 07:16 414阅读 0赞

本示例暂时使用了GridFS的存储方式，目前还没有很好的考虑大数据量下，瓦片索引的机制

import java.io.File;

import java.net.UnknownHostException;

import java.util.Queue;

import java.util.concurrent.ExecutorService;

import java.util.concurrent.Executors;

import java.util.concurrent.LinkedBlockingQueue;

import java.util.concurrent.TimeUnit;

import com.mongodb.DB;

import com.mongodb.Mongo;

import com.mongodb.MongoException;

import com.mongodb.gridfs.GridFS;

import com.mongodb.gridfs.GridFSInputFile;

public class PackageManager \{

private final ExecutorService mThreadPool;

private final Queue<File> mQueue = new LinkedBlockingQueue<File>();

Mongo mg;

DB db;

GridFS gridFS;

// ===========================================================

// Constructors

// ===========================================================

public PackageManager(final int mThreads) \{

this.mThreadPool = Executors.newFixedThreadPool(mThreads);

try \{

mg = new Mongo();

//mg = new Mongo("localhost", 27017);

\} catch (UnknownHostException e) \{

e.printStackTrace();

\} catch (MongoException e) \{

e.printStackTrace();

db = mg.getDB("beijingImg");

gridFS= new GridFS(db);

// ===========================================================

// Getter & Setter

// ===========================================================

public synchronized void add(final File pTileInfo)\{

this.mQueue.add(pTileInfo);

spawnNewThread();

private synchronized File getNext()\{

final File tile = this.mQueue.poll();

final int remaining = this.mQueue.size();

if(remaining % 100 == 0 && remaining > 0) \{

System.out.print("(" + remaining +")");

\} else \{

System.out.print(".");

this.notify();

return tile;

public synchronized void waitEmpty() throws InterruptedException \{

while(this.mQueue.size() > 0)\{

this.wait();

public void waitFinished() throws InterruptedException \{

waitEmpty();

this.mThreadPool.shutdown();

this.mThreadPool.awaitTermination(6, TimeUnit.HOURS);

// ===========================================================

// Methods from SuperClass/Interfaces

// ===========================================================

// Methods

// ===========================================================

private void spawnNewThread() \{

this.mThreadPool.execute(new DownloadRunner());

// ===========================================================

// Inner and Anonymous Classes

// ===========================================================

private class DownloadRunner implements Runnable \{

private File mTile;

private String saveFileName;

//private File mDestinationFile;

public DownloadRunner() \{

private void init(final File pTileInfo) \{

this.mTile = pTileInfo;

String fileName = mTile.getName().substring(0, mTile.getName().indexOf("."));

//System.out.println("fileName：" + fileName);

String row = mTile.getParentFile().getPath().substring(mTile.getParentFile().getPath().lastIndexOf("\\\\") + 1);

//System.out.println("row：" + row);

String level = mTile.getParentFile().getParentFile().getPath().substring(mTile.getParentFile().getParentFile().getPath().lastIndexOf("\\\\") + 1);

//System.out.println("level：" + level);

saveFileName = level + "\_" + row + "\_" + fileName;

@Override

public void run() \{

try\{

init(PackageManager.this.getNext());

// gridFS= new GridFS(db);

GridFSInputFile gfs = gridFS.createFile(mTile);

//GridFSInputFile gridFSInputFile = myFS.createFile(inputStream);

gfs.put("filename", saveFileName);

gfs.save();

\}catch(Exception e)\{

import java.io.File;

import java.util.ArrayList;

import java.util.LinkedList;

import java.util.List;

import org.junit.Test;

public class TilePackager \{

public static List<File> files = new ArrayList<File>();

PackageManager packageManager;

public int count = 0;

public static void main(String\[\] args) \{

/\*File ss = new File("E:\\\\瓦片数据\\\\北京影像");

String fileName = ss.getName().substring(0, ss.getName().indexOf("."));

System.out.println("fileName：" + fileName);

String row = ss.getParentFile().getPath().substring(ss.getParentFile().getPath().lastIndexOf("\\\\") + 1);

System.out.println("row：" + row);

String level = ss.getParentFile().getParentFile().getPath().substring(ss.getParentFile().getParentFile().getPath().lastIndexOf("\\\\") + 1);

System.out.println("level：" + level);\*/

//入库

/\*TilePackager tp = new TilePackager();

tp.packageManager = new PackageManager(30);

tp.getTiles(new File("E:\\\\瓦片数据\\\\北京影像"));

System.out.println(files.size());\*/

TilePackager tp = new TilePackager();

tp.getDirectorySize(new File("E:\\\\瓦片数据\\\\北京矢量"), tp.list);

//测试成功

@Test

public void listFiles() \{

long count = 0;

long countd = 0;

// 链表

LinkedList<File> list = new LinkedList<File>();

File dir = new File("D:\\\\opt");

File\[\] file = dir.listFiles();

for (int i = 0; i < file.length; i++) \{

if (file\[i\].isDirectory())

// 把第一层的目录，全部放入链表

list.add(file\[i\]);

else

count++;

System.out.println("文件" + count + ":" + file\[i\].getAbsolutePath());

File tmp = null;

// 循环遍历链表

while (!list.isEmpty()) \{

// 把链表的第一个记录删除

tmp = list.removeFirst();

// 如果删除的目录是一个路径的话

if (tmp.isDirectory()) \{

// 列出这个目录下的文件到数组中

file = tmp.listFiles();

if (file == null)

continue;

// 遍历文件数组

for (int i = 0; i < file.length; i++) \{

if (file\[i\].isDirectory())

// 如果遍历到的是目录，则继续加入链表

list.add(file\[i\]);

else

count++;

System.out.println("文件" + count + ":" + file\[i\].getAbsolutePath());

\} else \{

countd++;

System.out.println("目录\[" + countd + "\]路径:" + tmp.getAbsolutePath());

public void getTiles(File file)\{

File\[\] files = file.listFiles();

if (files == null || files.length == 0) \{

return;

for(File childFile : files)\{

if(childFile.isDirectory())\{

getTiles(childFile);

\}else\{

String fileName = childFile.getName();

String filePath = childFile.getPath();

if(fileName.endsWith("png") || fileName.endsWith("jpg"))\{

packageManager.add(childFile);

LinkedList<File> list = new LinkedList<File>(); // 保存待遍历文件夹的列表

// 非递归遍历

private void getDirectorySize(File file, LinkedList<File> list) \{

getOneDir(file, list); // 调用遍历文件夹根目录文件的方法

File tmp;

while (!list.isEmpty()) \{

tmp = (File) list.removeFirst();

// 这个地方的判断有点多余，但是为了保险还是给个判断了，正常情况列表中是只有文件夹的

// 但是不排除特殊情况，例如：本身是文件夹的目标在压入堆栈之后变成了文件

if (tmp.isDirectory()) \{

getDirectorySize(tmp, list);

\} else \{

System.out.println("file==>" + tmp.getAbsolutePath());

int sumdir = 0;

// 遍历指定文件夹根目录下的文件

private void getOneDir(File file, LinkedList<File> list) \{

// 每个文件夹遍历都会调用该方法

System.out.println("Dir==>" + file.getAbsolutePath());

File\[\] files = file.listFiles();

sumdir += 1;

if (files == null || files.length == 0) \{

return;

for (File f : files) \{

if (f.isDirectory()) \{

list.add(f);

\} else \{

// 这里列出当前文件夹根目录下的所有文件

System.out.println("file==>" + f.getAbsolutePath());

千万级别瓦片文件夹遍历及导入mongodb

发表评论取消回复

还没有评论，来说两句吧...

相关阅读

相关 C++遍历文件夹名

相关 java 遍历文件夹

相关 Python3遍历指定文件夹下所有文件及文件夹

相关 Mysql千万级别数据优化方案

相关 PHP遍历文件夹内容

相关 java千万级别数据处理-千万级别FTP下载

相关 php遍历文件夹、文件

相关 windows 遍历文件夹

相关 java千万级别数据处理 2 -千万级别FTP下载

相关千万级别瓦片文件夹遍历及导入mongodb