首次 golang爬虫插件gocolly/colly 使用经历
涉及问题
- 各种包 例如：golang.org/x/net, golang/x/test 等的下载问题，其实 golang 在 github 上建立了一个镜像库，如 https://github.com/golang/net 即是 https://golang.org/x/net 的镜像库。
- golang 与数据库交互
- golang 文件读写
- golang 多线程使用
- golang 字符编码的转换
- js 标签选择器
参考
系列文章 https://www.cnblogs.com/majianguo/p/8146110.html
start
js 选择器的问题
1）. 一个controller 可以定义多个OnHTML 回调函数 2）. "div > p" --> div 的直接子元素, "div p" --> div 下的所有子元素
golang数据库连接问题
参考 ： https://blog.csdn.net/webxscan/article/details/70174658
import (
"database/sql"
_ "github.com/go-sql-driver/mysql"
)
//数据库配置
const (
userName = "root"
password = ""
ip = "127.0.0.1"
port = "3306"
dbName = "dbName"
)
path := strings.Join([]string{userName, ":", password, "@tcp(",ip, ":", port, ")/", dbName, "?charset=utf8"}, "")
DB, _ := sql.Open("mysql", path)
//验证连接
if errConn := DB.Ping(); errConn != nil{
fmt.Println("open database fail")
return
}
fmt.Println("connnect success")
defer DB.Close()
stmt, err := DB.Prepare("insert into user(name age) values(?, ?)")
if err != nil {
fmt.Println(err)
}
res, err := stmt.Exec("username", 18)
if err != nil {
fmt.Println(err)
}
// 获取新插入行的id
fmt.Println(res.LastInsertId())
golang文件写入
var fileName = "./flag.txt"
var file *os.File
var err error
func main() {
file = openFile(fileName)
writeFile(file, "keep coding!!")
}
func openFile(fileName string) *os.File {
if checkFileIsExist(fileName) {
//如果文件存在
file, err = os.OpenFile(fileName, os.O_APPEND, 0666)
} else {
//创建文件
file, err = os.Create(fileName)
}
check(err)
return file
}
func writeFile(file *os.File, content string) {
writer := bufio.NewWriter(file)
writer.WriteString(content)
writer.WriteString("\r\n")
writer.Flush()
}
func check(e error) {
if e != nil {
panic(e)
}
}
func checkFileIsExist(filename string) bool {
var exist = true
if _, err := os.Stat(filename); os.IsNotExist(err) {
exist = false
}
return exist
}
golang字符编码的转换
抓取链家网数据时，网页本身是utf8编码，没有问题，可是抓取房天下的数据时，网页本身是gb2312，折腾了好一会， F**K
//src为要转换的字符串
func coverGBKToUTF8(src string) string {
// 网上搜有说要调用translate函数的，实测不用
return mahonia.NewDecoder("gbk").ConvertString(src)
}
golang多线程，控制消息同步
附上测试代码，便于理解
func main() {
// 我们还可以创建一个带缓冲的channel：
//c := make(chan int, 1024)
// 从带缓冲的channel中读数据
//for i:= range c {
//}
//此时，创建一个大小为1024的int类型的channel，即使没有读取方，写入方也可以一直往channel里写入，在缓冲区被填完之前都不会阻塞。
chs := make([] chan int, 10)
for i := 0; i < 10; i ++ {
go func(i int) {
chs[i] = make(chan int)
count(chs[i], i)
//fmt.Println("run thread ", i) // 打印要放在向信道发消息之前
}(i)
}
for _, ch := range chs {
value := <- ch
close(ch)
fmt.Println(value, " thread done")
}
fmt.Println("All done")
}
func count(ch chan int, i int) {
ch <- i // 向信道发消息的过程一定要放在协程内部，才不会被主进程阻塞
}
##最后，附上抓取放天下小区信息的源代码
package main
import (
"database/sql"
"fmt"
"github.com/PuerkitoBio/goquery"
"github.com/axgle/mahonia"
_ "github.com/go-sql-driver/mysql"
"github.com/gocolly/colly"
"reflect"
"strconv"
"strings"
)
//数据库配置
const (
userName = "root"
password = ""
ip = "127.0.0.1"
port = "3306"
dbName = "rixin"
)
type XQinfo struct {
name string
addr string
area string
price string
imgs string
post_code string
property_right string
property_type string
build_time int
developer string
build_type string
build_area string
build_struct string
floor_space string
house_count string
manager_company string
green_ratio string
plot_ratio string
proprety_fee string
additional_info string
water_supply string
heat_supply string
elec_supply string
gas string
security string
environment string
parking_space string
other_info string
district string // 区
}
var flagCh = make(chan int)
var infos = make([]XQinfo, 0)
var count = 1
var page = 10
func main() {
path := strings.Join([]string{userName, ":", password, "@tcp(",ip, ":", port, ")/", dbName, "?charset=utf8"}, "")
max := 10
DB, _ := sql.Open("mysql", path)
//验证连接
if errConn := DB.Ping(); errConn != nil{
fmt.Println("open database fail")
return
}
fmt.Println("connnect success")
defer DB.Close()
for i := 0; i < max; i ++ {
for j := 1; j <= page; j ++ {
link := "https://tj.esf.fang.com/housing/__0_0_0_0_" + strconv.Itoa(i * 10 + j) + "_0_0_0/"
go work(link, i * 10 + j)
}
}
for {
<- flagCh
count++
if count < max * page {
fmt.Println("<- receive the " + strconv.Itoa(count) + " thread ending flag")
} else {
for _, info := range infos{
insertDB(DB, info)
}
fmt.Println("insert DB == ", len(infos))
break
}
}
fmt.Println("All "+ strconv.Itoa(count) + " has done")
}
func work(url string, page int) {
c := colly.NewCollector()
// 小区列表页
c.OnHTML("body", func(e *colly.HTMLElement) {
e.DOM.Find(".plotListwrap").Each(func(i int, selection *goquery.Selection) {
info := XQinfo{}
link, _ := selection.Find(".plotListwrap > dt > a").Attr("href")
// 行政区名称
//district := selection.Find(".plotListwrap > dd > p:nth-child(2) > a:nth-child(1)").Text();
//info.district = district
detailVisit(info, link)
})
})
c.OnRequest(func(r *colly.Request) {
fmt.Println("Visiting", r.URL.String())
})
c.OnScraped(func(response *colly.Response) {
fmt.Println("the " + strconv.Itoa(page) + " thread sending end flag ->")
flagCh <- 1
})
c.Visit(url)
}
func detailVisit(info XQinfo, link string) {
detailLink := colly.NewCollector()
detailController := detailLink.Clone()
detailLink.OnHTML("body", func(e *colly.HTMLElement) {
// 详情链接
link1, _ := e.DOM.Find("#kesfxqxq_A01_03_01 > a").Attr("href")
// 图片地址
img, _ := e.DOM.Find(".bannerbg_pos > a > img").Attr("src")
info.imgs = img
// 价格
//price := e.DOM.Find(".prib").Text()
//info.price = price + "元/㎡ "
detailController.OnHTML("body", func(e *colly.HTMLElement) {
// 小区名称
name := e.DOM.Find(".ceninfo_sq > h1 > a").Text()
info.name = coverString(name)
price := e.DOM.Find(".detaiLtop > dl:nth-child(1) > dd > span").Text()
info.price = price + "元/㎡ "
e.DOM.Find(".inforwrap").Each(func(i int, selection *goquery.Selection) {
// 模块名称
modelName := coverString(selection.Prev().Find("h3").Text())
switch modelName {
case "基本信息":
dealInfo(selection, &info)
case "配套设施":
dealInfo(selection, &info)
case "周边信息":
selection.Find("dl dt").Each(func(_ int, otherSelect *goquery.Selection) {
tab := coverString(otherSelect.Text())
del := strings.Index(tab, "本段合作")
if del == -1 {
info.other_info = info.other_info + tab + "|"
}
})
}
})
infos = append(infos, info)
})
// 访问小区详情
detailController.Visit(link1)
})
detailLink.Visit(link)
}
// 处理小区基础信息
func dealInfo(selection *goquery.Selection, info *XQinfo) {
selection.Find("dl dd").Each(func(_ int, selectionbase *goquery.Selection) {
setXQinfo(selectionbase, info)
})
selection.Find("dl dt").Each(func(_ int, selectionbase *goquery.Selection) {
setXQinfo(selectionbase, info)
})
}
func setXQinfo(selectionbase *goquery.Selection, info *XQinfo) {
orgKey := coverString(selectionbase.Find("strong").Text())
index := strings.Index(orgKey, "：")
var key string
if index > 0 {
key = orgKey[:index]
} else {
key = orgKey
}
var value string
var fullValue string
value,ok := selectionbase.Attr("title")
if ok {
value = coverString(value)
} else {
fullValue = coverString(selectionbase.Text())
value = fullValue[strings.Index(fullValue, "：") + 3:]
}
switch key {
case "小区地址":
info.addr = value
case "所属区域":
info.area = value
info.district = value[:strings.Index(value, " ")]
case "邮编":
info.post_code = value
case "产权描述":
info.property_right = value
case "物业类别":
info.property_type = value
case "建筑年代":
year,_ := strconv.Atoi(string([]rune(value)[:4]))
info.build_time = year
case "开 发 商":
info.developer = value
case "建筑结构":
info.build_struct = value
case "建筑类型":
info.build_type = value
case "建筑面积":
info.build_area = value
case "占地面积":
info.floor_space = value
case "物业公司":
info.manager_company = value
case "房屋总数":
info.house_count = value
case "绿 化 率":
info.green_ratio = value
case "容 积 率":
info.plot_ratio = value
case "物 业 费":
info.proprety_fee = value
case "附加信息":
info.additional_info = value
case "供水":
info.water_supply = value
case "供暖":
info.heat_supply = value
case "供电":
info.elec_supply = value
case "燃气":
info.gas = value
case "安全管理":
info.security = value
case "卫生服务":
info.environment = value
case "停 车 位":
info.parking_space = value
}
}
//src为要转换的字符串
func coverGBKToUTF8(src string) string {
return mahonia.NewDecoder("gbk").ConvertString(src)
}
func replaceNullHtml(src string) string {
temp := strings.Replace(src, "聽", "", -1)
temp = strings.Replace(temp, "小区网", "", -1)
return temp
}
func coverString(src string) string {
return replaceNullHtml(coverGBKToUTF8(src))
}
func insertDB(DB *sql.DB, info XQinfo) {
t := reflect.TypeOf(info)
v := reflect.ValueOf(info)
sql1 := "insert into rx_xiaoqu_more("
sql2 := ") values ("
sql3 := ")"
for i := 0; i < t.NumField(); i++ {
sql1 = sql1 + t.Field(i).Name
if t.Field(i).Name == "build_time" {
sql2 = fmt.Sprintf("%s%s%d%s", sql2, "'", v.Field(i), "'")
} else {
sql2 = fmt.Sprintf("%s%s%s%s", sql2, "'", v.Field(i), "'")
}
if i != t.NumField() - 1 {
sql1 = sql1 + ", "
sql2 = sql2 + ", "
}
}
stmt, err := DB.Prepare(sql1 + sql2 + sql3)
if err != nil {
fmt.Println(sql1 + sql2 + sql3)
fmt.Println(err)
}
res, err := stmt.Exec()
if err != nil {
fmt.Println(sql1 + sql2 + sql3)
fmt.Println(err)
}
fmt.Println(res.LastInsertId())
}