三台机器的IP地址分别为
192.168.100.205
192.168.100.206
192.168.100.207
操作系统为centos
1. 将mongodb-org-server-4.4.4-1.el7.x86_64.rpm上传到三台机器上
2. 使用yum install mongodb-org-server-4.4.4-1.el7.x86_64.rpm 安装到三台机器上
3. 三台机器上创建数据存储目录/data/mongodb
[root@test1 ~]# mkdir -p /data/mongodb
##修改数据目录所在用户和组是因为yum安装mongodb的时候已经创建了mongod用户,并且使用systemctl start mongod启动服务的时候后台是以mongod运行的,如果该目录mongod用户没有读写权限,服务会启动失败
[root@test1 ~]# chown -R mongod:mongod /data/mongodb/
[root@test2 ~]# mkdir -p /data/mongodb
[root@test2 ~]# chown -R mongod:mongod /data/mongodb
[root@test3 ~]# mkdir -p /data/mongodb
[root@test3 ~]# chown -R mongod:mongod /data/mongodb
4. 修改配置文件内容
默认安装目录为
/usr/bin/mongod
配置文件的路径
/etc/mongod.conf
修改器内容如下:
# mongod.conf
# for documentation of all options, see:
# http://docs.mongodb.org/manual/reference/configuration-options/
# where to write logging data.
# 修改日志文件路径
systemLog:
destination: file
logAppend: true
path: /data/mongodb/mongod.log
# Where and how to store data.
# 修改数据存储路径
storage:
dbPath: /data/mongodb
journal:
enabled: true
# engine:
# wiredTiger:
# how the process runs
processManagement:
fork: true # fork and run in background
pidFilePath: /var/run/mongodb/mongod.pid # location of pidfile
timeZoneInfo: /usr/share/zoneinfo
# network interfaces
net:
port: 27017
# 修改绑定地址
bindIp: 0.0.0.0 # Enter 0.0.0.0,:: to bind to all IPv4 and IPv6 addresses or, alternatively, use the net.bindIpAll setting.
#security:
#operationProfiling:
#注意key和value之间的空格
replication:
replSetName: "rs0"
#sharding:
## Enterprise-Only Options
#auditLog:
#snmp:
5. 启动服务
systemctl start mongod (当然也可以直接用mongod -f /etc/mongod.conf启动)
6. 关闭MongoDB
[root@test1 ~]# mongod --shutdown -f /etc/mongod.conf
killing process with pid: 35667
[root@test2 ~]# mongod --shutdown -f /etc/mongod.conf
killing process with pid: 27659
[root@test3 ~]# mongod --shutdown -f /etc/mongod.conf
killing process with pid: 29149
###使用mongo命令连接到三台机器中的一台
> rs.initiate( {
... _id : "rs0",
... members: [
... { _id: 0, host: "test1:27017" },
... { _id: 1, host: "test2:27017" },
... { _id: 2, host: "test3:27017" }
... ]
... })
{
"ok" : 1,
"$clusterTime" : {
"clusterTime" : Timestamp(1632281074, 1),
"signature" : {
"hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
"keyId" : NumberLong(0)
}
},
"operationTime" : Timestamp(1632281074, 1)
}
rs.status()
{
"set" : "rs0",
"date" : ISODate("2021-09-22T03:25:50.205Z"),
"myState" : 1,
"term" : NumberLong(1),
"syncSourceHost" : "",
"syncSourceId" : -1,
"heartbeatIntervalMillis" : NumberLong(2000),
"majorityVoteCount" : 2,
"writeMajorityCount" : 2,
"votingMembersCount" : 3,
"writableVotingMembersCount" : 3,
"optimes" : {
"lastCommittedOpTime" : {
"ts" : Timestamp(1632281145, 1),
"t" : NumberLong(1)
},
"lastCommittedWallTime" : ISODate("2021-09-22T03:25:45.153Z"),
"readConcernMajorityOpTime" : {
"ts" : Timestamp(1632281145, 1),
"t" : NumberLong(1)
},
"readConcernMajorityWallTime" : ISODate("2021-09-22T03:25:45.153Z"),
"appliedOpTime" : {
"ts" : Timestamp(1632281145, 1),
"t" : NumberLong(1)
},
"durableOpTime" : {
"ts" : Timestamp(1632281145, 1),
"t" : NumberLong(1)
},
"lastAppliedWallTime" : ISODate("2021-09-22T03:25:45.153Z"),
"lastDurableWallTime" : ISODate("2021-09-22T03:25:45.153Z")
},
"lastStableRecoveryTimestamp" : Timestamp(1632281145, 1),
"electionCandidateMetrics" : {
"lastElectionReason" : "electionTimeout",
"lastElectionDate" : ISODate("2021-09-22T03:24:45.126Z"),
"electionTerm" : NumberLong(1),
"lastCommittedOpTimeAtElection" : {
"ts" : Timestamp(0, 0),
"t" : NumberLong(-1)
},
"lastSeenOpTimeAtElection" : {
"ts" : Timestamp(1632281074, 1),
"t" : NumberLong(-1)
},
"numVotesNeeded" : 2,
"priorityAtElection" : 1,
"electionTimeoutMillis" : NumberLong(10000),
"numCatchUpOps" : NumberLong(0),
"newTermStartDate" : ISODate("2021-09-22T03:24:45.134Z"),
"wMajorityWriteAvailabilityDate" : ISODate("2021-09-22T03:24:46.128Z")
},
"members" : [
{
"_id" : 0,
"name" : "test1:27017",
"health" : 1,
"state" : 1,
"stateStr" : "PRIMARY",
"uptime" : 636,
"optime" : {
"ts" : Timestamp(1632281145, 1),
"t" : NumberLong(1)
},
"optimeDate" : ISODate("2021-09-22T03:25:45Z"),
"syncSourceHost" : "",
"syncSourceId" : -1,
"infoMessage" : "",
"electionTime" : Timestamp(1632281085, 1),
"electionDate" : ISODate("2021-09-22T03:24:45Z"),
"configVersion" : 1,
"configTerm" : 1,
"self" : true,
"lastHeartbeatMessage" : ""
},
{
"_id" : 1,
"name" : "test2:27017",
"health" : 0,
"state" : 8,
"stateStr" : "(not reachable/healthy)",
"uptime" : 0,
"optime" : {
"ts" : Timestamp(0, 0),
"t" : NumberLong(-1)
},
"optimeDurable" : {
"ts" : Timestamp(0, 0),
"t" : NumberLong(-1)
},
"optimeDate" : ISODate("1970-01-01T00:00:00Z"),
"optimeDurableDate" : ISODate("1970-01-01T00:00:00Z"),
"lastHeartbeat" : ISODate("2021-09-22T03:25:49.184Z"),
"lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"),
"pingMs" : NumberLong(0),
"lastHeartbeatMessage" : "Our replica set configuration is invalid or does not include us",
"syncSourceHost" : "",
"syncSourceId" : -1,
"infoMessage" : "",
"configVersion" : -2,
"configTerm" : -1
},
{
"_id" : 2,
"name" : "test3:27017",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 75,
"optime" : {
"ts" : Timestamp(1632281145, 1),
"t" : NumberLong(1)
},
"optimeDurable" : {
"ts" : Timestamp(1632281145, 1),
"t" : NumberLong(1)
},
"optimeDate" : ISODate("2021-09-22T03:25:45Z"),
"optimeDurableDate" : ISODate("2021-09-22T03:25:45Z"),
"lastHeartbeat" : ISODate("2021-09-22T03:25:49.169Z"),
"lastHeartbeatRecv" : ISODate("2021-09-22T03:25:50.167Z"),
"pingMs" : NumberLong(0),
"lastHeartbeatMessage" : "",
"syncSourceHost" : "test1:27017",
"syncSourceId" : 0,
"infoMessage" : "",
"configVersion" : 1,
"configTerm" : 1
}
],
"ok" : 1,
"$clusterTime" : {
"clusterTime" : Timestamp(1632281145, 1),
"signature" : {
"hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
"keyId" : NumberLong(0)
}
},
"operationTime" : Timestamp(1632281145, 1)
}
1. 分别下载 cmake,boost,mongo-c driver, mongo-cxx-driver (GitHub - mongodb/mongo-cxx-driver at releases/v3.4)
2. boost下载的版本为60,放在e:\info\boost目录下,编译boost,步骤如下
a. cd到 X64 native Tools Command Prompt for VS2017
b. cd到e:/info/boost/boost_1_60_0目录下,执行如下命令
:\info\boost\boost_1_60_0>bootstrap.bat
Building Boost.Build engine
Bootstrapping is done. To build, run:
.\b2
To adjust configuration, edit 'project-config.jam'.
Further information:
- Command line help:
.\b2 --help
- Getting started guide:
http://boost.org/more/getting_started/windows.html
- Boost.Build documentation:
http://www.boost.org/build/doc/html/index.html
c 执行如下命令
bjam stage --toolset=msvc-14.1 --with-date_time --with-filesystem --with-program_options --with-system --with-thread link=static runtime-link=static threading=multi address-model=64 debug release
3. 编译mongo-c-driver,源码放在E:\info\mongodb\mongo-c-driver-1.13.0目录中
a. cd到E:\info\mongodb\mongo-c-driver-1.13.0\build目录中
b. 执行如下命令
E:\info\cmak\cmake-3.13.3-win64-x64\bin\cmake -G "Visual Studio 15 2017 Win64" "-DCMAKE_INSTALL_PREFIX=e:\info\mongodb\mongo-driver" "-DCMAKE_PREFIX_PATH=e:\info\mongodb\mongo-driver" ..
b. 进入E:\info\mongodb\mongo-c-driver-1.13.0\build目录中,使用vs打开mongo-c-driver.sln后,编译all_build工程后,编译install工程,编译后的结果会放到e:\info\mongodb\mongo-driver目录中。
3. 编译mongodb-c++ driver,源码放在E:\info\mongodb\mongo-cxx-driver-releases-v3.4目录中
a. cd到E:\info\mongodb\mongo-cxx-driver-releases-v3\build目录中。
b. 执行如下命令
E:\info\cmak\cmake-3.13.3-win64-x64\bin\cmake -G "Visual Studio 15 2017 Win64" -DCMAKE_INSTALL_PREFIX=E:\info\mongodb\mongo-cxx-driver -DCMAKE_PREFIX_PATH=e:\info\mongodb\mongo-driver -DBOOST_ROOT=e:\info\boost\boost_1_60_0 -DCMAKE_CXX_STANDARD=11 -DCMAKE_CXX_FLAGS="/Zc:__cplusplus" ..
注释:
-DBOOST_ROOT是boost的根目录
c,用vs打开E:\info\mongodb\mongo-cxx-driver-releases-v3.4\MONGO_CXX_DRIVER.sln文件
d 编译 all_build工程,编译过程中会出现一些错误,需要所有工程修改如下信息
2. bsoncxx工程需要添加_ENABLE_EXTENDED_ALIGNED_STORAGE预编译条件
最后可以参考
windows下VS2017编译mongoDB c、c++API_sqcfj的专栏-CSDN博客
linux下编译以及编程步骤,参考
a. 从https://github.com/mongodb/mongo-c-driver/releases/download/1.17.4/mongo-c-driver-1.17.4.tar.gz下载源码到/home/mongodb并加压到/home/mongodb/mongo-c-driver-1.17.4目录中。
b. cd /home/mongodb/mongo-c-driver-1.17.4
c. mkdir cmake-build
d. cmake -DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF -DCMAKE_INSTALL_PREFIX=/home/mongodb/mongo-c-driver/build -DCMAKE_PREFIX_PATH=/home/mongodb/mongo-c-driver/build ..
e. cmake --build . (编译,类似make命令)
f. cmake --build --target install(类似make install),该操作会在/home/mongodb/mongo-c-driver/build目录中放置include,bin,lib64,share四个目录
hello_mongoc.c内容如下
#include
int
main (int argc, char *argv[])
{
const char *uri_string = "mongodb://localhost:27017";
mongoc_uri_t *uri;
mongoc_client_t *client;
mongoc_database_t *database;
mongoc_collection_t *collection;
bson_t *command, reply, *insert;
bson_error_t error;
char *str;
bool retval;
/*
* * Required to initialize libmongoc's internals
* */
mongoc_init ();
/*
* * Optionally get MongoDB URI from command line
* */
if (argc > 1) {
uri_string = argv[1];
}
/*
* * Safely create a MongoDB URI object from the given string
* */
uri = mongoc_uri_new_with_error (uri_string, &error);
if (!uri) {
fprintf (stderr,
"failed to parse URI: %s\n"
"error message: %s\n",
uri_string,
error.message);
return EXIT_FAILURE;
}
/*
* * Create a new client instance
* */
client = mongoc_client_new_from_uri (uri);
if (!client) {
return EXIT_FAILURE;
}
/*
* * Register the application name so we can track it in the profile logs
* * on the server. This can also be done from the URI (see other examples).
* */
mongoc_client_set_appname (client, "connect-example");
/*
* * Get a handle on the database "db_name" and collection "coll_name"
* */
database = mongoc_client_get_database (client, "test1");
collection = mongoc_client_get_collection (client, "test1", "test");
/*
* * Do work. This example pings the database, prints the result as JSON and
* * performs an insert
* */
command = BCON_NEW ("ping", BCON_INT32 (1));
retval = mongoc_client_command_simple (
client, "admin", command, NULL, &reply, &error);
if (!retval) {
fprintf (stderr, "%s\n", error.message);
return EXIT_FAILURE;
}
str = bson_as_json (&reply, NULL);
printf ("%s\n", str);
insert = BCON_NEW ("hello", BCON_UTF8 ("world"));
if (!mongoc_collection_insert_one (collection, insert, NULL, NULL, &error)) {
fprintf (stderr, "%s\n", error.message);
}
bson_destroy (insert);
bson_destroy (&reply);
bson_destroy (command);
bson_free (str);
/*
* * Release our handles and clean up libmongoc
* */
mongoc_collection_destroy (collection);
mongoc_database_destroy (database);
mongoc_uri_destroy (uri);
mongoc_client_destroy (client);
mongoc_cleanup ();
return EXIT_SUCCESS;
}
g. 编译并运行
gcc -o hello_mongoc hello_mongoc.c -I/home/mongodb/mongo-c-driver/build/include/libbson-1.0 -I/home/mongodb/mongo-c-driver/build/include/libmongoc-1.0 -L/home/mongodb/mongo-c-driver/build/lib64 -lmongoc-1.0 -lbson-1.0
./hello_mongoc
pg的数据结构如下:
sde=# \d landsat
数据表 "sde.landsat"
栏位 | 类型 | 校对规则 | 可空的 | 预设
------------+------------------------+----------+----------+------
objectid | integer | | not null |
entityid | character varying(254) | | |
acquisitio | character varying(254) | | |
cloudcover | numeric(38,8) | | |
processing | character varying(254) | | |
path | integer | | |
row | integer | | |
min_lat | numeric(38,8) | | |
min_lon | numeric(38,8) | | |
max_lat | numeric(38,8) | | |
max_lon | numeric(38,8) | | |
download_u | character varying(254) | | |
shape | st_point | | |
索引:
"r48_sde_rowid_uk" UNIQUE, btree (objectid) WITH (fillfactor='75')
"a35_ix1" gist (shape)
"landsat_entityid_idx" btree (entityid)
记录数量为
sde=# select count(*) from landsat;
count
---------
1059757
(1 行记录)
PG到MongoDB的c的录入程序
#pragma comment(lib,"bson-1.0.lib")
#pragma comment(lib,"mongoc-1.0.lib")
#pragma comment(lib,"bsoncxx.lib")
#pragma comment(lib,"mongocxx.lib")
#pragma comment(lib,"libpq.lib")
#pragma warning (disable:4819)
#include
#include "bsoncxx/builder/stream/document.hpp"
#include "mongocxx/instance.hpp"
#include "mongocxx/uri.hpp"
#include "mongocxx/client.hpp"
#include
#include
#include
#include
using namespace std;
int main(int argc,char* argv[])
{
//mongo连接字符串
string uri_string = "mongodb://192.168.100.138:27017";
//连接信息结构体
mongoc_uri_t* uri = NULL;
//客户端结构体
mongoc_client_t* client = NULL;
//客户端结构体
mongoc_database_t* database = NULL;
//collection结构体
mongoc_collection_t* collection = NULL;
//bson结构体
bson_t* command = NULL, reply, *insert = NULL;
bson_error_t error;
char* str = NULL;
bool retval;
PGconn* m_pgconn = NULL;
string connectinfo = "hostaddr=127.0.0.1 port=5432 dbname=sde user=sde password=sde";
m_pgconn = PQconnectdb(connectinfo.c_str());
ConnStatusType t = PQstatus(m_pgconn);
if (PQstatus(m_pgconn) != CONNECTION_OK)
{
return -1;
}
//初始化mongoc环境
mongoc_init();
if (argc > 1)
{
uri_string = argv[1];
}
//初始化uri对象
uri = mongoc_uri_new_with_error(uri_string.c_str(), &error);
if (!uri)
{
fprintf(stderr, "failed to parse uri %s error message %s\n",uri_string.c_str(),error.message);
mongoc_cleanup();
PQfinish(m_pgconn);
return EXIT_FAILURE;
}
//建立新连接
client = mongoc_client_new_from_uri(uri);
if (!client)
{
if (collection)
mongoc_collection_destroy(collection);
if (database)
mongoc_database_destroy(database);
if (uri)
mongoc_uri_destroy(uri);
if (client)
mongoc_client_destroy(client);
mongoc_cleanup();
PQfinish(m_pgconn);
fprintf(stderr, "failed to connect mongodb");
return EXIT_FAILURE;
}
mongoc_client_set_appname(client, "connect example");
//从client对象中获取database对象
database = mongoc_client_get_database(client, "test");
if (!database)
{
if (collection)
mongoc_collection_destroy(collection);
if (database)
mongoc_database_destroy(database);
if (uri)
mongoc_uri_destroy(uri);
if (client)
mongoc_client_destroy(client);
mongoc_cleanup();
PQfinish(m_pgconn);
return EXIT_FAILURE;
}
从client对象中获取collection对象
collection = mongoc_client_get_collection(client, "test", "landsat1");
if (!collection)
{
if (collection)
mongoc_collection_destroy(collection);
if (database)
mongoc_database_destroy(database);
if (uri)
mongoc_uri_destroy(uri);
if (client)
mongoc_client_destroy(client);
mongoc_cleanup();
PQfinish(m_pgconn);
return EXIT_FAILURE;
}
//command = BCON_NEW("ping", BCON_INT32(1));
string sql = "select t.id,row_to_json(t) from (select objectid::text as id,entityid,cloudcover,processing,path,row,st_y(shape)||','||st_x(shape) as shape from landsat) as t";
PGresult* result = PQexec(m_pgconn, sql.c_str());
for (int i = 0; i < PQntuples(result); i++)
{
string json = PQgetvalue(result, i, 1);
//从json字符串转换成bson对象
bson_t* bson = bson_new_from_json((const uint8_t *)json.c_str(), -1, &error);
if (!bson)
continue;
//插入一条到mongodb中
if (!mongoc_collection_insert_one(collection, bson, NULL, NULL, &error))
{
fprintf(stderr, "%s\n", error.message);
bson_destroy(bson);
continue;
}
bson_destroy(bson);
printf("%d\n", i);
}
//释放各种对象
if(collection)
mongoc_collection_destroy(collection);
if(database)
mongoc_database_destroy(database);
if(uri)
mongoc_uri_destroy(uri);
if(client)
mongoc_client_destroy(client);
mongoc_cleanup();
PQfinish(m_pgconn);
return 0;
}
录入mongodb信息如下所示:
查询前20条记录
Mon Mar 29 2021 09:18:35 GMT+0800-test> db.landsat.find()
{ "_id" : ObjectId("605d9baeaa96d82bb40059a2"), "id" : "1", "entityid" : "LC80101172015002LGN00", "cloudcover" : 80.81, "processing" : "L1GT", "path" : 10, "row" : 117, "shape" : "-79.09922999966409,-139.6608200003422" }
{ "_id" : ObjectId("605d9baeaa96d82bb40059a3"), "id" : "2", "entityid" : "LC80260392015002LGN00", "cloudcover" : 90.84, "processing" : "L1GT", "path" : 26, "row" : 39, "shape" : "29.23105999991992,-97.48576000002339" }
{ "_id" : ObjectId("605d9baeaa96d82bb40059a4"), "id" : "3", "entityid" : "LC82270742015002LGN00", "cloudcover" : 83.44, "processing" : "L1GT", "path" : 227, "row" : 74, "shape" : "-21.285979999676442,-59.277360000163185" }
{ "_id" : ObjectId("605d9baeaa96d82bb40059a5"), "id" : "4", "entityid" : "LC82270732015002LGN00", "cloudcover" : 52.29, "processing" : "L1T", "path" : 227, "row" : 73, "shape" : "-19.843650000146624,-58.93257999966653" }
{ "_id" : ObjectId("605d9baeaa96d82bb40059a6"), "id" : "5", "entityid" : "LC82270622015002LGN00", "cloudcover" : 38.85, "processing" : "L1T", "path" : 227, "row" : 62, "shape" : "-3.9529399998803,-55.3889599997035" }
{ "_id" : ObjectId("605d9baeaa96d82bb40059a7"), "id" : "6", "entityid" : "LC82111152015002LGN00", "cloudcover" : 22.93, "processing" : "L1GT", "path" : 211, "row" : 115, "shape" : "-78.54179000030763,-79.3614800002012" }
{ "_id" : ObjectId("605d9baeaa96d82bb40059a8"), "id" : "7", "entityid" : "LC81791202015002LGN00", "cloudcover" : 7.67, "processing" : "L1GT", "path" : 179, "row" : 120, "shape" : "-82.84798999973805,-65.4305599996436" }
{ "_id" : ObjectId("605d9baeaa96d82bb40059a9"), "id" : "8", "entityid" : "LC82111112015002LGN00", "cloudcover" : 43.43, "processing" : "L1GT", "path" : 211, "row" : 111, "shape" : "-73.85498999968809,-65.85499999994937" }
{ "_id" : ObjectId("605d9baeaa96d82bb40059aa"), "id" : "9", "entityid" : "LC81950292015002LGN00", "cloudcover" : 21.02, "processing" : "L1T", "path" : 195, "row" : 29, "shape" : "43.482350000353165,5.347290000234182" }
{ "_id" : ObjectId("605d9baeaa96d82bb40059ab"), "id" : "10", "entityid" : "LC81790452015002LGN00", "cloudcover" : 1.92, "processing" : "L1T", "path" : 179, "row" : 45, "shape" : "20.589169999996386,24.04268999984447" }
{ "_id" : ObjectId("605d9baeaa96d82bb40059ac"), "id" : "11", "entityid" : "LC81951112015002LGN00", "cloudcover" : 81.56, "processing" : "L1GT", "path" : 195, "row" : 111, "shape" : "-73.58112000018718,-40.430350000351496" }
{ "_id" : ObjectId("605d9baeaa96d82bb40059ad"), "id" : "12", "entityid" : "LC81950372015002LGN00", "cloudcover" : 0, "processing" : "L1T", "path" : 195, "row" : 37, "shape" : "32.105319999619155,2.1216200000637855" }
{ "_id" : ObjectId("605d9baeaa96d82bb40059ae"), "id" : "13", "entityid" : "LC81790442015002LGN00", "cloudcover" : 0, "processing" : "L1T", "path" : 179, "row" : 44, "shape" : "22.03263999993692,24.373359999864988" }
{ "_id" : ObjectId("605d9baeaa96d82bb40059af"), "id" : "14", "entityid" : "LC81790412015002LGN00", "cloudcover" : 38.22, "processing" : "L1T", "path" : 179, "row" : 41, "shape" : "26.36084000036658,25.398240000376575" }
{ "_id" : ObjectId("605d9baeaa96d82bb40059b0"), "id" : "15", "entityid" : "LC81951122015002LGN00", "cloudcover" : 54.9, "processing" : "L1GT", "path" : 195, "row" : 112, "shape" : "-74.85996999981495,-43.25487000044944" }
{ "_id" : ObjectId("605d9baeaa96d82bb40059b1"), "id" : "16", "entityid" : "LC81790402015002LGN00", "cloudcover" : 28.34, "processing" : "L1T", "path" : 179, "row" : 40, "shape" : "27.80228000022271,25.751709999617674" }
{ "_id" : ObjectId("605d9baeaa96d82bb40059b2"), "id" : "17", "entityid" : "LC81630432015002LGN00", "cloudcover" : 1.57, "processing" : "L1T", "path" : 163, "row" : 43, "shape" : "23.484589999729565,49.441580000417616" }
{ "_id" : ObjectId("605d9baeaa96d82bb40059b3"), "id" : "18", "entityid" : "LC81630422015002LGN00", "cloudcover" : 2.49, "processing" : "L1T", "path" : 163, "row" : 42, "shape" : "24.926129999700606,49.782079999593236" }
{ "_id" : ObjectId("605d9baeaa96d82bb40059b4"), "id" : "19", "entityid" : "LC81630402015002LGN00", "cloudcover" : 1.13, "processing" : "L1T", "path" : 163, "row" : 40, "shape" : "27.801170000296167,50.48511999988898" }
{ "_id" : ObjectId("605d9baeaa96d82bb40059b5"), "id" : "20", "entityid" : "LC80900752015002LGN00", "cloudcover" : 7.18, "processing" : "L1GT", "path" : 90, "row" : 75, "shape" : "-22.72508000044087,152.04860000039082" }
Type "it" for more
Mon Mar 29 2021 09:18:47 GMT+0800-test> db.landsat.count()
1059757
//查询entityid值为LC81630422015002LGN00的记录
Mon Mar 29 2021 10:24:35 GMT+0800-test> db.landsat.find({"entityid":"LC81630422015002LGN00"})
{ "_id" : ObjectId("605d9baeaa96d82bb40059b3"), "id" : "18", "entityid" : "LC81630422015002LGN00", "cloudcover" : 2.49, "processing" : "L1T", "path" : 163, "row" : 42, "shape" : "24.926129999700606,49.782079999593236" }
//_id值为605d9baeaa96d82bb40059aa的记录
db.landsat.find({"_id":ObjectId("605d9baeaa96d82bb40059aa")})
{ "_id" : ObjectId("605d9baeaa96d82bb40059aa"), "id" : "9", "entityid" : "LC81950292015002LGN00", "cloudcover" : 21.02, "processing" : "L1T", "path" : 195, "row" : 29, "shape" : "43.482350000353165,5.347290000234182" }
//查询cloudcover值为21.02的记录数量
db.landsat.find({"cloudcover":21.02}).count()
118
//查询大于21.02小于30的cloudcover的记录数量
db.landsat.find({"cloudcover":{"$gte":21.02,"$lte":30}}).count()
76892
查看执行计划,可以查看执行路径以及执行时间,此事例中走的是索引扫描,执行时间为163毫秒
Mon Mar 29 2021 10:40:02 GMT+0800-test> db.landsat.find({"cloudcover":{"$gte":21.02,"$lte":30}}).explain("executionStats")
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "test.landsat",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"cloudcover" : {
"$lte" : 30
}
},
{
"cloudcover" : {
"$gte" : 21.02
}
}
]
},
"winningPlan" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"cloudcover" : 1
},
"indexName" : "cloudcover_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"cloudcover" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"cloudcover" : [
"[21.02, 30.0]"
]
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 76892,
"executionTimeMillis" : 163,
"totalKeysExamined" : 76892,
"totalDocsExamined" : 76892,
"executionStages" : {
"stage" : "FETCH",
"nReturned" : 76892,
"executionTimeMillisEstimate" : 24,
"works" : 76893,
"advanced" : 76892,
"needTime" : 0,
"needYield" : 0,
"saveState" : 76,
"restoreState" : 76,
"isEOF" : 1,
"docsExamined" : 76892,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 76892,
"executionTimeMillisEstimate" : 5,
"works" : 76893,
"advanced" : 76892,
"needTime" : 0,
"needYield" : 0,
"saveState" : 76,
"restoreState" : 76,
"isEOF" : 1,
"keyPattern" : {
"cloudcover" : 1
},
"indexName" : "cloudcover_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"cloudcover" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"cloudcover" : [
"[21.02, 30.0]"
]
},
"keysExamined" : 76892,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0
}
}
},
"serverInfo" : {
"host" : "bigdataserver.esrichina.com",
"port" : 27017,
"version" : "4.4.4",
"gitVersion" : "8db30a63db1a9d84bdcad0c83369623f708e0397"
},
"ok" : 1
}
将相关索引删除掉后的变化,执行计划走了全表扫描(COLLSCAN),执行时间也变成了517ms
db.landsat.dropIndex('cloudcover_1')
{ "nIndexesWas" : 3, "ok" : 1 }
Mon Mar 29 2021 11:27:19 GMT+0800-test> db.landsat.find({"cloudcover":{"$gte":21.02,"$lte":30}}).explain("executionStats")
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "test.landsat",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"cloudcover" : {
"$lte" : 30
}
},
{
"cloudcover" : {
"$gte" : 21.02
}
}
]
},
"winningPlan" : {
"stage" : "COLLSCAN",
"filter" : {
"$and" : [
{
"cloudcover" : {
"$lte" : 30
}
},
{
"cloudcover" : {
"$gte" : 21.02
}
}
]
},
"direction" : "forward"
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 76892,
"executionTimeMillis" : 517,
"totalKeysExamined" : 0,
"totalDocsExamined" : 1059757,
"executionStages" : {
"stage" : "COLLSCAN",
"filter" : {
"$and" : [
{
"cloudcover" : {
"$lte" : 30
}
},
{
"cloudcover" : {
"$gte" : 21.02
}
}
]
},
"nReturned" : 76892,
"executionTimeMillisEstimate" : 42,
"works" : 1059759,
"advanced" : 76892,
"needTime" : 982866,
"needYield" : 0,
"saveState" : 1059,
"restoreState" : 1059,
"isEOF" : 1,
"direction" : "forward",
"docsExamined" : 1059757
}
},
"serverInfo" : {
"host" : "bigdataserver.esrichina.com",
"port" : 27017,
"version" : "4.4.4",
"gitVersion" : "8db30a63db1a9d84bdcad0c83369623f708e0397"
},
"ok" : 1
}
查看collection的索引以及大小信息
分别在_id,entityid,cloudcover三个字段上建有索引,其中_id_为系统自带索引,剩下两个为自定义索引
Mon Mar 29 2021 09:32:48 GMT+0800-test> db.landsat.getIndexes()
[
{
"v" : 2,
"key" : {
"_id" : 1
},
"name" : "_id_"
},
{
"v" : 2,
"key" : {
"entityid" : 1
},
"name" : "entityid_1"
},
{
"v" : 2,
"key" : {
"cloudcover" : 1
},
"name" : "cloudcover_1"
}
]
获取collection的大小
//整个colletion存储空间,包括索引以及已分配的空闲空间
Mon Mar 29 2021 10:18:06 GMT+0800-test> db.landsat.totalSize()
115109888
//单collection存储空间,包括已分配的空闲空间
Mon Mar 29 2021 10:18:20 GMT+0800-test> db.landsat.storageSize()
79790080
//索引的存储空间
Mon Mar 29 2021 10:18:38 GMT+0800-test> db.landsat.totalIndexSize()
35319808
与放在pg中进行比较
\dt+ landsat
关联列表
架构模式 | 名称 | 类型 | 拥有者 | 大小 | 描述
----------+---------+--------+--------+--------+------
sde | landsat | 数据表 | sde | 360 MB |
pg明显比mongo占用空间多的原因是pg每条记录都包括头信息,同时有些字段还包括字段值大小,另外shape由于存储方式不同,因此占用的空间也大不相同。
可以使用mongodump和mongorestore这对工具进行导出和导入。
也可以使用mongoexport和mongoimport这对工具进行导出和导入。
mongodump是以bson形式导出,可以以库为单位导出,也可以单个collection导出,如果以库为单位导出,如下所示
指定导出目录为e盘,但是会根据库名字自动建立库名字命名的文件夹,然后将库中每个collection进行导出,
每个collection会生成两个文件,一个是collection.bson,一个是collection.metadata.json,bson后缀的文件包含实际collection的数据,json后缀的文件包括collection的元信息,包括索引等信息
c:\Program Files\MongoDB\Server\4.4\bin>mongodump /out:e:/ /db:test mongodb://192.168.100.138:27017
2021-03-31T14:18:36.748+0800 writing test.movies to e:\test\movies.bson
2021-03-31T14:18:36.769+0800 writing test.test to e:\test\test.bson
2021-03-31T14:18:36.770+0800 writing test.landsat to e:\test\landsat.bson
2021-03-31T14:18:36.806+0800 done dumping test.movies (10 documents)
2021-03-31T14:18:36.806+0800 done dumping test.test (2 documents)
2021-03-31T14:18:39.729+0800 [######################..] test.landsat 996782/1059757 (94.1%)
2021-03-31T14:18:39.779+0800 [########################] test.landsat 1059757/1059757 (100.0%)
2021-03-31T14:18:39.779+0800 done dumping test.landsat (1059757 documents)
mongorestore恢复过程
使用nongo工具删除库
Wed Mar 31 2021 14:28:34 GMT+0800-test> db.dropDatabase()
{ "dropped" : "test", "ok" : 1 }
恢复如下,工具会自动到相关目录中搜索bson以及json文件进行自动处理
mongorestore mongodb://192.168.100.138:27017 /db:test e:/test
2021-03-31T14:29:29.688+0800 The --db and --collection flags are deprecated for this use-case; please use --nsInclude instead, i.e. with --nsInclude=${DATABASE}.${COLLECTION}
2021-03-31T14:29:29.710+0800 building a list of collections to restore from e:\test dir
2021-03-31T14:29:29.713+0800 reading metadata for test.landsat from e:\test\landsat.metadata.json
2021-03-31T14:29:29.713+0800 reading metadata for test.movies from e:\test\movies.metadata.json
2021-03-31T14:29:29.714+0800 reading metadata for test.test from e:\test\test.metadata.json
2021-03-31T14:29:30.001+0800 restoring test.landsat from e:\test\landsat.bson
2021-03-31T14:29:30.109+0800 restoring test.movies from e:\test\movies.bson
2021-03-31T14:29:30.120+0800 no indexes to restore
2021-03-31T14:29:30.120+0800 finished restoring test.movies (10 documents, 0 failures)
2021-03-31T14:29:30.184+0800 restoring test.test from e:\test\test.bson
2021-03-31T14:29:30.196+0800 no indexes to restore
2021-03-31T14:29:30.196+0800 finished restoring test.test (2 documents, 0 failures)
2021-03-31T14:29:32.687+0800 [####....................] test.landsat 32.5MB/183MB (17.8%)
2021-03-31T14:29:35.687+0800 [#########...............] test.landsat 69.2MB/183MB (37.9%)
2021-03-31T14:29:38.688+0800 [##############..........] test.landsat 107MB/183MB (58.5%)
2021-03-31T14:29:41.687+0800 [##################......] test.landsat 141MB/183MB (77.1%)
2021-03-31T14:29:44.687+0800 [#######################.] test.landsat 178MB/183MB (97.3%)
2021-03-31T14:29:45.075+0800 [########################] test.landsat 183MB/183MB (100.0%)
2021-03-31T14:29:45.075+0800 restoring indexes for collection test.landsat from metadata
2021-03-31T14:29:47.325+0800 finished restoring test.landsat (1059757 documents, 0 failures)
2021-03-31T14:29:47.325+0800 1059769 document(s) restored successfully. 0 document(s) failed to restore.
恢复完成后
Wed Mar 31 2021 14:31:20 GMT+0800-test> show collections
landsat
movies
test
Wed Mar 31 2021 14:31:24 GMT+0800-test> db.landsat.count()
1059757
mongoexport(可以将collection导出成json或者csv格式,只支持collection级别的导出,并且不会导出collection的元信息)
导出成json格式
mongoexport /out:e:/test/landat.json /db:test /collection:landsat mongodb://192.168.100.138:27017
2021-03-31T16:14:02.893+0800 connected to: mongodb://192.168.100.138:27017
2021-03-31T16:14:03.916+0800 [#.......................] test.landsat 64000/1059757 (6.0%)
2021-03-31T16:14:04.917+0800 [###.....................] test.landsat 136000/1059757 (12.8%)
2021-03-31T16:14:05.917+0800 [####....................] test.landsat 208000/1059757 (19.6%)
2021-03-31T16:14:06.918+0800 [######..................] test.landsat 280000/1059757 (26.4%)
2021-03-31T16:14:07.916+0800 [#######.................] test.landsat 352000/1059757 (33.2%)
2021-03-31T16:14:08.917+0800 [#########...............] test.landsat 440000/1059757 (41.5%)
2021-03-31T16:14:09.916+0800 [###########.............] test.landsat 512000/1059757 (48.3%)
2021-03-31T16:14:10.916+0800 [#############...........] test.landsat 576000/1059757 (54.4%)
2021-03-31T16:14:11.916+0800 [##############..........] test.landsat 648000/1059757 (61.1%)
2021-03-31T16:14:12.917+0800 [################........] test.landsat 720000/1059757 (67.9%)
2021-03-31T16:14:13.917+0800 [##################......] test.landsat 800000/1059757 (75.5%)
2021-03-31T16:14:14.916+0800 [###################.....] test.landsat 880000/1059757 (83.0%)
2021-03-31T16:14:15.917+0800 [#####################...] test.landsat 952000/1059757 (89.8%)
2021-03-31T16:14:16.917+0800 [#######################.] test.landsat 1032000/1059757 (97.4%)
2021-03-31T16:14:17.211+0800 [########################] test.landsat 1059757/1059757 (100.0%)
2021-03-31T16:14:17.211+0800 exported 1059757 records
mongoimport导入流程
Wed Mar 31 2021 16:37:21 GMT+0800-test> db.landsat.drop()
true
mongoimport /db:test /collection:landsat mongodb://192.168.100.138:27017 e:/test/landsat.json
2021-03-31T16:46:28.978+0800 connected to: mongodb://192.168.100.138:27017
2021-03-31T16:46:31.979+0800 [###.....................] test.landsat 32.9MB/200MB (16.5%)
2021-03-31T16:46:34.979+0800 [########................] test.landsat 68.2MB/200MB (34.1%)
2021-03-31T16:46:37.979+0800 [############............] test.landsat 105MB/200MB (52.4%)
2021-03-31T16:46:40.979+0800 [#################.......] test.landsat 142MB/200MB (71.0%)
2021-03-31T16:46:43.978+0800 [#####################...] test.landsat 179MB/200MB (89.5%)
2021-03-31T16:46:45.659+0800 [########################] test.landsat 200MB/200MB (100.0%)
2021-03-31T16:46:45.659+0800 1059757 document(s) imported successfully. 0 document(s) failed to import.
aggregate:
db.dltb.aggregate([{$match:{zldwmc:"围村村委会"}},{$project:{_id:0,shape_leng:1,tbmj1:"$tbmj"}},{$sort:{shape_leng:1}},{$skip:10},{$limit:5}]).pretty()
{ "shape_leng" : 72.02594049, "tbmj1" : 180.15 }
{ "shape_leng" : 72.94600482, "tbmj1" : 313.03 }
{ "shape_leng" : 76.98691841, "tbmj1" : 392.64 }
{ "shape_leng" : 79.50763485, "tbmj1" : 416.67 }
{ "shape_leng" : 80.62169182, "tbmj1" : 131.19 }
select zldwmc,max(shape_leng),min(shape_leng) from dltb group by zldwmc;
db.dltb.aggregate([{$group : {_id : "$zldwmc", max : {$max : "$shape_len"},min:{$min:"$shape_len"}}}])
db.dltb.aggregate([{$match:{zldwmc:"河边村村委会"}},{$group : {_id : "$zldwmc", putresult: {$push : "$shape_leng"}}}])
{ "_id" : "河边村村委会", "putresult" : [ 99.40712538, 102.36057367, 104.54855408, 116.24146144, 119.35984173, 125.48743115, 125.62949464, 126.42127865, 128.92065772, 129.70264764, 131.170776, 131.77536112, 132.04617498, 136.24933499, 136.98661635, 137.72935085, 138.41202079, 139.05893734, 141.15677031, 144.5034053, 145.0191613, 145.03473565, 146.11873848, 147.43097363, 152.1220918, 152.57367265, 154.07369476, 155.5130736, 158.31569483, 160.08748463, 162.29848267, 162.9647573, 163.31695951, 171.69619333, 176.12958515, 176.3020228, 178.29178324, 182.51725166, 185.92057129, 186.36180633, 186.4674956, 187.31812123, 187.74772456, 188.03341752, 189.29474394, 189.88874338, 190.98409935, 191.26693228, 191.99302483, 192.53095055, 192.60807368, 194.34164883, 196.59779562, 199.91707304, 202.03983356, 203.83659298, 206.49870674, 206.67429068, 206.98980224, 209.72779394, 211.92738115, 213.37018956, 213.6247351, 214.1895983, 214.90288623, 216.22596842, 217.37149513, 217.97446154, 218.54891216, 221.24838385, 222.43653477, 223.66592261, 225.87440157, 228.72740555, 230.05915178, 230.86532784, 231.37716221, 232.35732707, 234.16537307, 236.90284773, 240.47073571, 240.88440101, 241.09838194, 241.46829632, 241.51522213, 245.8834796, 249.71509711, 249.96682727, 250.56505492, 251.71244681, 256.45277654, 257.76221186, 259.19999048, 259.5180976, 260.42810978, 263.74174412, 266.24240954, 270.0267575, 273.47214562, 274.74935514, 275.24139076, 277.94334263, 281.30860928, 282.29692266, 287.74030798, 288.46435405, 289.38632528, 290.0487507, 292.45216351, 293.46034424, 294.71099858, 295.37155526, 296.15800066, 296.87974872, 297.85707847, 299.37225774, 300.35709887, 300.8409395, 302.53316957, 303.86917884, 304.61376049, 305.66795155, 307.50706647, 309.26193962, 309.66691146, 312.77576476, 313.77553476, 314.90217649, 320.13710571, 321.67117777, 322.93670009, 323.03661894, 323.49288037, 323.98611128, 325.67538111, 330.00665229, 330.02289156, 330.16727499, 333.9926748, 334.37407119, 340.03909586, 340.12853224, 342.11860841, 343.10470978, 345.00823062, 346.59617852, 351.2072364, 352.42306459, 353.27109909, 354.65864891, 360.81264728, 361.71370937, 361.83430892, 364.9586323, 365.16960213, 365.23055043, 366.15480555, 371.25166243, 372.89383972, 374.90343367, 376.2437189, 376.85409985, 377.60775871, 378.44269886, 378.57153797, 380.32467001, 384.22572297, 384.42656714, 384.7248453, 385.82794842, 386.98929864, 390.39018474, 393.54451658, 401.3330031, 401.71657434, 403.5062881, 406.6432758, 407.31379446, 408.38454544, 411.98419378, 412.8020323, 414.75187346, 419.2303266, 420.7914334, 425.09051403, 425.13546676, 427.15478017, 427.81401429, 427.96368212, 436.30607157, 437.61320349, 439.52314964, 443.91188027, 449.00423414, 450.24742687, 450.39923999, 452.76348386, 453.05906796, 456.51356834, 459.05892656, 459.21703335, 459.4174793, 461.48882947, 464.24790282, 468.59546979, 468.59634968, 468.83598294, 470.39275595, 470.7626457, 471.36562, 474.57704127, 475.70458578, 476.18340352, 478.40169861, 490.24300713, 490.81264915, 493.74025682, 495.39607232, 499.73241627, 500.57138784, 501.8063027, 521.35162282, 523.13963482, 524.69568747, 525.14900933, 530.39444289, 536.45266998, 541.76358939, 542.66313045, 546.58544334, 547.02706657, 547.56217317, 555.35432816, 558.48295156, 560.84521632, 563.61562404, 571.72267746, 579.5851921, 586.38954961, 588.09626845, 592.04617101, 601.25771374, 604.51040936, 609.21679846, 611.7999161, 614.35519806, 621.78143856, 624.15932672, 629.2161524, 629.44031167, 634.28568971, 642.9130542, 654.05198968, 656.1776533, 661.52971559, 662.24986482, 665.25585308, 674.38579717, 677.1313134, 678.48172186, 679.43010437, 680.21186381, 688.51957737, 700.32158396, 706.86375025, 707.07443218, 708.66209989, 711.35615071, 722.11206155, 726.55510027, 732.96837938, 736.26656198, 738.89251708, 740.65771181, 751.14986554, 755.67160388, 761.26112137, 761.84438219, 765.14930986, 778.02798937, 805.4269242, 812.07828185, 824.54834521, 834.57022234, 843.18728273, 845.92364668, 850.45623974, 856.06457945, 861.26372955, 862.72068767, 863.24060065, 875.38983886, 879.75447891, 880.22346842, 898.77788635, 902.83538544, 907.61740631, 910.12235553, 914.14471559, 929.01556018, 935.37489117, 942.91568717, 946.52860066, 961.81244628, 963.51326341, 1003.43671822, 1004.31753955, 1010.9637607, 1013.66111662, 1030.80883021, 1031.83238975, 1039.29017885, 1046.86865943, 1049.04895191, 1051.14704428, 1061.29316601, 1069.79825956, 1097.20777079, 1114.29773758, 1123.16719335, 1139.67465374, 1142.34475825, 1144.21962266, 1149.15819694, 1159.09597922, 1160.71181, 1183.01860673, 1204.73488915, 1223.53097743, 1244.54910989, 1267.87018212, 1285.463447, 1295.58929975, 1295.71708521, 1300.44742469, 1318.16444488, 1325.03335706, 1337.51525862, 1341.71204045, 1356.91281671, 1372.83440168, 1414.1369536, 1503.81236767, 1604.79385463, 1674.24213093, 1737.71668168, 1762.64786327, 1782.59566472, 1840.93250353, 1893.80145543, 1956.96688366, 2036.3711091, 2101.45435379, 2115.81633062, 2183.94727517, 2192.28618557, 2230.86886654, 2264.28300561, 2377.91857196, 2407.49394482, 2416.03426706, 2667.60192168, 2881.36017826, 3004.25414866, 3099.34361499, 3100.19235926, 3206.58452128, 3481.43259945, 3709.19269136, 3905.40399032, 4483.05581909, 5462.74776216, 5893.17849218, 8285.45580338, 8373.95567826, 10134.0080367, 10892.0628387, 11306.8393023, 16829.716139, 28715.9040926, 32886.9005115 ] }