云备份
项目介绍
该项目完成一个自动对指定目录下的文件进行备份的功能；
- 客户端对指定目录进行监控，对每个文件生成etag信息，鉴别是否需要备份
- 客户端将需要备份的文件基于http协议的PUT请求上传到服务器端
- 服务端对于PUT上传的文件进行备份到指定目录下
- 整个通信的过程使用SSL/TLS加密传输
服务端设计
实现功能：
- 提供解析基于https协议的put请求，将文件数据进行备份
- 提供浏览器能够查看服务器上文件信息功能
- 提供浏览器能够下载服务器上文件功能
- 提供对后台长时间无访问文件的压缩存储功
服务端代码：
运行环境：Linux平台
1 #include "httplib.h"
2 #include
3 #include
4 #include
5 #include
6
7 #define SERVER_BASE_DIR "www"
8 #define SERVER_BACKUP_DIR SERVER_BASE_DIR"/list/"
9 #define SERVER_ADDR "0.0.0.0"
10 #define SERVER_PORT 9527
11
12 namespace bf = boost::filesystem;
13
14 class CloudServer
15 {
16 private:
17 httplib::Server srv;
18 public:
19 CloudServer()
20 {
21 bf::path base_path(SERVER_BASE_DIR);
22 if(!bf::exists(base_path)){
23 bf::create_directory(base_path);
24 }
25 bf::path list_path(SERVER_BACKUP_DIR);
26 if(!bf::exists(list_path)){
27 bf::create_directory(list_path);
28 }
29 }
30 bool Start(){
31 srv.set_base_dir(SERVER_BASE_DIR);
32 srv.Get("/(list(/){0,1}){0,1}", GetFileList);
33 srv.Get("/list/(.*)", GetFileData);
34 srv.Put("/list/(.*)", PutFileData);
35 srv.listen(SERVER_ADDR, SERVER_PORT);
36 return true;
37 }
38 private:
39 static void GetFileList(const httplib::Request &req, httplib::Response &rsp)
40 {
41 bf::path list(SERVER_BASE_DIR);
42 bf::directory_iterator item_begin(list);
43 bf::directory_iterator item_end;
44
45 std::string body;
46 body = "
";
47 for(; item_begin != item_end; ++item_begin){
48 if(bf::is_directory(item_begin->status())){
49 continue;
50 }
51 std::string file = item_begin->path().filename().string();
52 std::string uri = "/list/" + file;
53 body += "- ";
54 body += "";
57 body += file;
58 body += "";
59 body += "
";
60 }
61 body += "
";
62 rsp.set_content(&body[0], "text/html");
63 return;
64 }
65 static void GetFileData(const httplib::Request &req, httplib::Response &rsp)
66 {
67 std::string file = SERVER_BASE_DIR + req.path;
68 if(!bf::exists(file)){
69 std::cerr << "file" << file << "is not exists";
70 rsp.status = 404;
71 return;
72 }
73 std::ifstream ifile(file, std::ios::binary);
74 if(!ifile.is_open()){
75 std::cerr << "open file " << file << "error\n";
76 rsp.status = 500;
77 return ;
78 }
79 std::string body;
80 int64_t fsize = bf::file_size(file);
81 body.resize(fsize);
82 ifile.read(&body[0], fsize);
83 if(!ifile.good()){
84 std::cerr << "read file" << file << " body error\n";
85 rsp.status = 500;
86 return ;
87 }
88 rsp.set_content(body, "text/html");
89 }
90 static void PutFileData(const httplib:: Request &req, httplib::Response &rsp)
91 {
92 if(!req.has_header("Range")){
93 rsp.status = 400;
94 return ;
95 }
96 std::string range = req.get_header_value("Range");
97 int64_t range_start;
98 if(RangeParse(range, range_start) == false){
99 rsp.status = 400;
100 return ;
101 }
102 std::cout << "backup file:[" << req.path << "]range:[" << range << "]\n";
103 std::string real = SERVER_BASE_DIR + req.path;
104 int fd = open(real.c_str(),O_CREAT|O_WRONLY, 0664);
105 if(fd < 0){
106 std::cerr << "open file" << real << " error\n";
107 rsp.status = 500;
108 return ;
109 }
110 lseek(fd, range_start, SEEK_SET);
111 int ret = write(fd, &req.body[0], req.body.size());
112 if(ret != req.body.size()){
113 std::cerr << "file write body error\n";
114 rsp.status = 500;
115 return ;
116 }
117 close(fd);
118
119 return ;
120 }
121 static bool RangeParse(std::string &range, int64_t &start){
122 size_t pos1 = range.find("=");
123 size_t pos2 = range.find("-");
124 if(pos1 == std::string::npos || pos2 == std::string ::npos){
125 std::cerr << "Range:[" << range << "] format error\n";
126 return false;
127 }
128 std::stringstream rs;
129 rs << range.substr(pos1+1, pos2 - pos1 - 1);
130 rs >> start;
131 return true;
132 }
138 };
140 int main()
141 {
142 CloudServer srv;
143 srv.Start();
144 return 0;
145 }
客户端设计
实现功能：
- 提供监控目录的功能，能够获取目录下文件信息，鉴别文件是否需要备份
- 备份文件，基于HTTPS协议PUT请求，实现文件多线程分块上传功能
- 文件的信息记录，便于文件是否需要备份的鉴别
客户端代码：
运行环境：vs2015
#ifndef __M_CLOUD_H__
#define __M_CLOUD_H__
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include "httplib.h"
#define CLIENT_BACKUP_DIR "backup"
#define CLIENT_BACKUP_INFO_FILE "back.list"
#define RANGE_MAX_SIZE (10 << 20)
#define SERVER_IP "192.168.194.129"
#define SERVER_PORT 9527
#define BACKUP_URI "/list/"
namespace bf = boost::filesystem;
class ThrBackup
{
private:
std::string _file;
int64_t _range_start;
int64_t _range_len;
public:
bool _res;
public:
ThrBackup(const std::string &file, int64_t start, int64_t len):_res(true),
_file(file), _range_start(start), _range_len(len){}
void Start() {
//获取文件的range分块数据
std::ifstream path(_file, std::ios::binary);
if (!path.is_open()) {
std::cerr << "range backup file" << _file << "failed\n";
_res = false;
return;
}
//跳转到range的起始位置
path.seekg(_range_start, std::ios::beg);
std::string body;
body.resize(_range_len);
//读取文件中range分块的文件数据
path.read(&body[0], _range_len);
if (!path.good()) {
std::cerr << "read file " << _file << "range data failed\n";
_res = false;
return;
}
path.close();
//上传range数据
bf::path name(_file);
//组织上传的url路径
std::string uri = BACKUP_URI + name.filename().string();
//实例化一个httplib的客户端对象
httplib::Client cli(SERVER_IP, SERVER_PORT);
httplib::Headers hdr;
hdr.insert(std::make_pair("Content-Length", std::to_string(_range_len)));
std::stringstream tmp;
tmp << "bytes=" << _range_start << "-" << (_range_start + _range_len - 1);
hdr.insert(std::make_pair("Range", tmp.str().c_str()));
//通过实例化的Client想服务端
auto rsp = cli.Put(uri.c_str(), hdr, body, "text/plain");
if (rsp && rsp->status != 200) {
_res = false;
}
std::stringstream ss;
ss << "backup file" << _file << "range:[" << _range_start << "-" << _range_len << "] backup success\n";
std::cout << ss.str();
return ;
}
};
class CloudClient
{
private:
std::unordered_map _backup_list;
private:
bool GetBackupInfo()
{
bf::path path(CLIENT_BACKUP_INFO_FILE);
if (!bf::exists(path) ){
std::cerr << "list file" << path.string() << "is not exist" << std::endl;
return false;
}
int64_t fsize = bf::file_size(path);
if (fsize == 0) {
std::cerr << "have no backup info\n";
return false;
}
std::string body;
body.resize(fsize);
std::ifstream file(CLIENT_BACKUP_INFO_FILE, std::ios::binary);
if (!file.is_open()) {
std::cerr << "list file open error\n";
return false;
}
file.read(&body[0], fsize);
if (!file.good()) {
std::cerr << "read list file body error\n";
return false;
}
file.close();
std::vector list;
boost::split(list, body,boost::is_any_of ("\n"));
for (auto i : list) {
size_t pos = i.find(" ");
if (pos == std::string::npos) {
continue;
}
std::string key = i.substr(0, pos);
std::string val = i.substr(pos + 1);
_backup_list[key] = val;
}
return true;
}
bool SetBackupInfo()
{
std::string body;
for (auto i : _backup_list) {
body += i.first + " " + i.second + "\n";
}
std::ofstream file(CLIENT_BACKUP_INFO_FILE, std::ios::binary);
if (!file.is_open()) {
std::cerr << "open list file error\n";
return false;
}
file.write(&body[0], body.size());
if (!file.good()) {
std::cerr << "set backup info error\n";
return false;
}
file.close();
return true;
}
bool BackupDirListen(const std::string &path)
{
bf::directory_iterator item_begin(path);
bf::directory_iterator item_end;
for (; item_begin != item_end; ++item_begin) {
if (bf::is_directory(item_begin->status())) {
BackupDirListen(item_begin->path().string());
continue;
}
if (FileIsNeedBackup(item_begin->path().string()) == false) {
continue;
}
if (PutFileData(item_begin->path().string()) == false) {
continue;
}
AddBackupInfo(item_begin->path().string());
}
return true;
}
bool AddBackupInfo(const std::string &file)
{
std::string etag;
if (GetFileEtag(file, etag) == false) {
return false;
}
_backup_list[file] = etag;
}
bool GetFileEtag(const std::string &file, std::string &etag)
{
bf::path path(file);
if (!bf::exists(path)) {
std::cerr << "get file" << file << "etag error\n";
return false;
}
int64_t fsize = bf::file_size(path);
int64_t mtime = bf::last_write_time(path);
std::stringstream tmp;
tmp << std::hex << fsize << "-" << std::hex << mtime;
etag = tmp.str();
return 0;
}
bool PutFileData(const std::string &file)
{
//1.获取文件大小
int64_t fsize = bf::file_size(file);
if (fsize <= 0) {
std::cerr << "file" << file << "unnecessary backup\n";
return false;
}
//2.计算总共需要分多少块，得到每块大小以及起始位置
//3.循环创建线程，在线程中上传文件数据
int count = fsize / RANGE_MAX_SIZE;
std::vector thr_res;
std::vector thr_list;
std::cerr << "file:[" << file << "] fsize:[" << fsize << "] count:[" << count + 1 << "]\n";
for (int i = 0; i <= count; i++) {
int64_t range_start = i * RANGE_MAX_SIZE;
int64_t range_end = (i + 1) * RANGE_MAX_SIZE - 1;
if (i == count) {
range_end = fsize - 1;
}
int64_t range_len = range_end - range_start + 1;
ThrBackup backup_info(file, range_start, range_len);
std::cerr << "file:[" << file << "]range:[" << range_start << "-" << range_end << "]-" << range_len << "\n";
thr_res.push_back(backup_info);
}
for (int i = 0; i <= count; i++) {
thr_list.push_back(std::thread(thr_start, &thr_res[i]));
}
//4.等待线程退出，判断文件上传结果
bool ret = true;
for (int i = 0; i <= count; i++) {
thr_list[i].join();
if (thr_res[i]._res == true) {
continue;
}
ret = false;
}
//5.上传成功，则添加文件的备份信息记录
if (ret == false) {
return false;
}
std::cerr << "file:[" << file << "] backup success\n";
return true;
}
bool FileIsNeedBackup(const std::string &file)
{
std::string etag;
if (GetFileEtag(file, etag) == false) {
return false;
}
auto it = _backup_list.find(file);
if (it != _backup_list.end() && it->second == etag) {
return false;
}
return true;
}
static void thr_start(ThrBackup *backup_info) {
backup_info->Start();
return;
}
public:
bool Start() {
GetBackupInfo();
while (1) {
BackupDirListen(CLIENT_BACKUP_DIR);
SetBackupInfo();
Sleep(10);
}
return true;
}
};
#endif
httplib基本使用
/*test.cpp*/ #include "httplib.h"
static void HelloWorld(const httplib::Request &req, httplib::Response &rsp)
{
rsp.set_content("hello world", "text/html"); return;
}
int main()
{
httplib::Server srv;
srv.set_base_dir("./www");
srv.Get("/", HelloWorld);
srv.listen("0.0.0.0", 9000);
}
/*g++ -std=c++0x test.cpp -o test -lpthread -lboost_filesystem -lboost_system*/
boost库下载路径：boost_1_70_0-msvc-14.1-64.exe