关于从网页上下载数据

  1 #include "curl/curl.h"  
  2 
  3 #include  
  4 #include   
  5 #include 
  6 #include <string>
  7 #include 
  8 #include  
  9 
 10 #pragma comment(lib, "libcurl.lib")   
 11 #pragma comment(lib, "wldap32.lib")   
 12 #pragma comment(lib, "ws2_32.lib")   
 13 #pragma comment(lib, "winmm.lib")   
 14 
 15 size_t write_func(void *ptr, size_t size, size_t nmemb, FILE *stream)
 16 {
 17     return fwrite(ptr, size, nmemb, stream);
 18 }
 19 
 20 
 21 std::string analyze_name;
 22 std::string analyze_url;
 23 std::string cplocation;
 24 std::string httpurl;
 25 
 26 //得到每条链接的短地址和名字
 27 void get_name_url(std::string line)
 28 {
 29     //File d;
 30     char *linkhead = "";
 31     char *linkmiddle = "\"";
 32     char *linktail = "";
 33     std::string head = "";
 34 
 35     char *s = strstr((char*)line.data(), linkhead);
 36     for (int i = 0; i<9; i++)
 37         *s++;
 38     int urloffset = 0;
 39     while (*s != *linkmiddle)
 40     {
 41         *s++;
 42         urloffset++;
 43     }
 44     analyze_url = line.substr(line.find(head) + 9, urloffset);
 45     for (int i = 0; i<2; i++)
 46         *s++;
 47     int nameoffset = 0;
 48     while (*s != *linktail)
 49     {
 50         *s++;
 51         nameoffset++;
 52     }
 53     analyze_name = line.substr(line.find(head) + 11 + urloffset, nameoffset);
 54 
 55     //return d;
 56 }
 57 
 58 //根据http网址下载网页到本机指定位置
 59 int http_get(std::string httpurl, std::string location)
 60 {
 61     CURL *curl;
 62     CURLcode res;
 63     FILE *outfile;
 64     errno_t err;
 65     err = fopen_s(&outfile, location.c_str(), "wb");
 66     //outfile = fopen(location.c_str(), "wb");
 67     const char *url = httpurl.c_str();
 68     curl = curl_easy_init();
 69     if (curl)
 70     {
 71         curl_easy_setopt(curl, CURLOPT_URL, url);
 72         curl_easy_setopt(curl, CURLOPT_WRITEDATA, outfile);
 73         curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_func);
 74         res = curl_easy_perform(curl);
 75         fclose(outfile);
 76         curl_easy_cleanup(curl);
 77     }
 78     return 0;
 79 }
 80 
 81 
 82 int linkextract(std::string httpfile, std::string httpurl, std::string cplocation)
 83 {
 84     std::ifstream infile;
 85     std::string line, link, name;
 86     infile.open(httpfile, std::ios::in);
 87     std::string directory = "alt=\"[DIR]\"";
 88     std::string file = "alt=\"[   ]\"";
 89 
 90     while (!infile.eof())
 91     {
 92         getline(infile, line, '\n');
 93         //if (line.find(directory) != -1)//目录
 94         //{
 95         //    get_name_url(line);
 96         //    analyze_name = analyze_name.substr(0, analyze_name.length() - 1);
 97         //    std::string content = cplocation + "/" + analyze_name;
 98         //    _mkdir((char*)content.c_str());
 99         //    httpurl = httpurl + analyze_url;
100         //    cplocation = cplocation + "/" + analyze_name;
101         //    http_get(httpurl, cplocation + "/" + analyze_name + ".txt");
102         //    linkextract(cplocation + "/" + analyze_name + ".txt", httpurl, cplocation);
103         //}
104         //else if (line.find(file) != -1)//文件
105         {
106             //get_name_url(line);
107             if (line.find("href=\"Tile")==-1)
108                 continue;
109 
110             char *linkhead = "";
111             char *linkmiddle = "\"";
112             char *linktail = "";
113             std::string head = "";
114 
115             char *s = strstr((char*)line.data(), linkhead);
116             for (int i = 0; i<9; i++)
117                 *s++;
118             int urloffset = 0;
119             while (*s != *linkmiddle)
120             {
121                 *s++;
122                 urloffset++;
123             }
124             analyze_url = line.substr(line.find(head) + 9, urloffset);
125             for (int i = 0; i<2; i++)
126                 *s++;
127             int nameoffset = 0;
128             while (*s != *linktail)
129             {
130                 *s++;
131                 nameoffset++;
132             }
133             analyze_name = line.substr(line.find(head) + 11 + urloffset, nameoffset);
134 
135 
136             http_get(httpurl + analyze_url, cplocation + "/" + analyze_name);
137         }
138         /*else
139             continue;*/
140     }
141     infile.close();
142 
143     return 0;
144 }
145 
146 
147 
148 
149 int main()
150 {
151     std::string httpurl = "http://服务地址";
152     std::string content = "../DownloadDirectory";
153     _mkdir((char*)content.c_str());
154     std::string localdir = "../DownloadDirectory/DownloadDirectory.txt";
155     http_get(httpurl, localdir);
156     linkextract(localdir, httpurl, content);
157     return 0;
158 }

只能下载href里面的链接文件，而且很多网页不能下载；

其中，需要下载curl并配置路径，关于下载编译curl有很好的博客，暂时找不见了；