go语言实现爬虫采集联想词
/****************************************************
go语言实现爬虫_联想词 咨询:qq:1465376564
黄哥python培训班所写
python北京周末培训班
https://github.com/pythonpeixun/article/blob/master/beijing_weekend.md
python
上海周末培训班
https://github.com/pythonpeixun/article/blob/master/shanghai_weekend.md
*****************************************************/
package main
import (
"fmt"
"io/ioutil"
"net/http"
"regexp"
)
//定义新的数据类型
type Spider struct {
url string
header map[string]string
}
//定义 Spider的方法
func (keyword Spider) get_html_header() string {
client := &http.Client{}
req, err := http.NewRequest("GET", keyword.url, nil)
if err != nil {
}
for key, value := range keyword.header {
req.Header.Add(key, value)
}
resp, err := client.Do(req)
if err != nil {
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
}
return string(body)
}
func main() {
header := map[string]string{"Host": "sug.so.360.cn",
"Referer": "http://www.so.com/",
"DNT": "1",
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36",
"Cookie": "__huid=104rl%2B0HjG2YltBarbPPIz2w7HTbLrv43gETLeVtBdIEI%3D",
}
keyword := "科技"
url := "http://sug.so.360.cn/suggest?callback=suggest_so&encodein=utf-8&encodeout=utf-8&format=json&fields=word,obdata&word=" + keyword
spider := &Spider{url, header}
html := spider.get_html_header()
rp1 := regexp.MustCompile("{(.*?)}")
find_txt := rp1.FindAllString(html, -1)
fmt.Println(find_txt)
}