单线程版本
package main
import (
"fmt"
"strconv"
"net/http"
"os"
)
func pachong(start, end int) {
//明确爬的地址
url := "https://tieba.baidu.com/f?kw=%E6%AE%B5%E5%AD%90&ie=utf-8&pn="
for i := start; i < end+1; i++ {
fmt.Printf("当前爬取第 %d 页\n",i)
res, err := curl(url + strconv.Itoa(i*50))
if err != nil {
fmt.Println("你有错误:", err)
}
name:=strconv.Itoa(i)+".html"
//创建文件
f,err1:=os.Create(name)
if err1!=nil {
fmt.Println("文件创建失败:",err1)
}
//内容写入文件
f.WriteString(res)
//关闭文件
f.Close()
}
}
//爬取网页内容
func curl(url string) (reques string, err error) {
resq,err1:=http.Get(url)
if err1 !=nil {
err=err1
return
}
//最后关闭
defer resq.Body.Close()
str:=make([]byte,1024*5)
//循环读取数据
for {
number,_:=resq.Body.Read(str)
if number == 0 {
fmt.Println("爬取页面成功!")
break
}
reques+=string(str[:number])
}
return
}
func main() {
var start, end int
//var c chan int
fmt.Println("[请输入起始页面:]")
//获取输入
fmt.Scan(&start)
fmt.Println("[请输入终止页面:]")
fmt.Scan(&end)
pachong(start, end)
}
并发版
package main
import (
"fmt"
"strconv"
"net/http"
"os"
)
func pachong(start, end int) {
//明确爬的地址
c:=make(chan int)
url := "https://tieba.baidu.com/f?kw=%E6%AE%B5%E5%AD%90&ie=utf-8&pn="
for i := start; i < end+1; i++ {
go paing(i, url+strconv.Itoa(i*50),c)
}
for v:=range c{
fmt.Printf("当前爬完第%d页 \n", v)
}
}
func paing(i int, url string,c chan int) {
fmt.Printf("当前爬取第 %d 页,%s\n", i, url)
res, err := curl(url)
if err != nil {
fmt.Println("你有错误:", err)
return
}
name := strconv.Itoa(i) + ".html"
//创建文件
f, err1 := os.Create(name)
if err1 != nil {
fmt.Println("文件创建失败:", err1)
return
}
//内容写入文件i
f.WriteString(res)
//关闭文件
f.Close()
c <- i
}
//爬取网页内容
func curl(url string) (reques string, err error) {
resq, err1 := http.Get(url)
if err1 != nil {
err = err1
return
}
//最后关闭
defer resq.Body.Close()
str := make([]byte, 1024*5)
//循环读取数据
for {
number, _ := resq.Body.Read(str)
if number == 0 {
//fmt.Println("爬取页面成功!")
break
}
reques += string(str[:number])
}
return
}
func main() {
var start, end int
fmt.Println("[请输入起始页面:]")
//获取输入
fmt.Scan(&start)
fmt.Println("[请输入终止页面:]")
fmt.Scan(&end)
pachong(start, end)
}
pengfu爬虫
package main
import (
"fmt"
"strconv"
"net/http"
"regexp"
"strings"
"os"
)
func pachong(start, end int) {
//明确爬的地址
c:=make(chan int)
for i := start; i < end+1; i++ {
url := "https://www.pengfu.com/xiaohua_"+strconv.Itoa(i)+".html"
go paing(i,url,c)
}
for v:=range c{
fmt.Printf("当前爬完第%d页 \n", v)
}
}
func paing(i int, url string,c chan int) {
fmt.Printf("当前爬取第 %d 页,%s\n", i, url)
res, err := curl(url)
if err != nil {
fmt.Println("你有错误:", err)
return
}
//aaa,_:=json.Marshal(res)
//fmt.Println("json数据是:",string(aaa))
//组装数据,写入txt中
for k,v:=range res["title"]{
//fmt.Println("标题是:",v[1])
//a:=strings.Replace(res["content"][k][1], "\n", "", -1)
//fmt.Println("内容是:",a)
xie(v[1],res["content"][k][1])
}
c<-i
}
//写入内容
func xie(title,content string) {
name := title + ".txt"
//创建文件
f, err1 := os.Create(name)
if err1 != nil {
fmt.Println("文件创建失败:", err1)
return
}
//内容写入文件i
f.WriteString(content)
//关闭文件
f.Close()
}
//爬取网页内容
func curl(url string) (res map[string][][]string,err error){
res=make(map[string][][]string)
resq, err := HttpGet(url)
if err != nil {
return res,err
}
// 去除换行符
resq = strings.Replace(resq, "\t", "", -1)
resq = strings.Replace(resq, "\r", "", -1)
resq = strings.Replace(resq, "
", "", -1)
//解析题目
rel1:=regexp.MustCompile(`
