go并发爬取图片

package main
 
import (
    "fmt"
    "strconv"
    "net/http"
    "io/ioutil"
    "regexp"
    "strings"
)
 
func main(){
    writeChan := make([]chan int,26)
    for i:=0;i<26;i++{
        writeChan[i] = make(chan int)
        go DownloadOnePage(i,writeChan[i])
    }
    for _,ch := range writeChan{
        <-ch 
    }
    
}
 
func DownloadOnePage(index int, writeChan chan int){
    var url = "http://www.umei.cc/p/gaoqing/cn/"+strconv.Itoa(index+1)+".htm"
    var pageStr = GetStrByUrl(url)
    var match = regexp.MustCompile(`src="(https?:.*.jpg)"`)
    var results = match.FindAllStringSubmatch(pageStr,-1)
    for _, result := range results{
        fmt.Println("title:", result[1])
        var imageUrl = result[1] 
        var fileName = strings.Split(imageUrl,"/")
        var filePath = "D:/go_project/practice/image/"+fileName[len(fileName)-1]
        DownloadImg(result[1],filePath)
    }
    writeChan <- 1
}
 
func GetStrByUrl(url string) string {
    resp, err := http.Get(url)
    fmt.Println(resp)
    if nil != err{
        fmt.Println(err)
    }
    defer resp.Body.Close()
    fmt.Println(resp.StatusCode)
    respBytes, err :=ioutil.ReadAll(resp.Body)
    if nil != err{
        fmt.Println(err)
    }
    respStr := string(respBytes)
    return respStr
}
 
func DownloadImg(imageUrl, fileName string){
    resp, err := http.Get(imageUrl)
    fmt.Println(resp)
    if nil != err{
        fmt.Println(err)
        return 
    }
    defer resp.Body.Close()
    respBytes, err :=ioutil.ReadAll(resp.Body)
    if nil != err{
        fmt.Println(err)
        return
    }
    ioutil.WriteFile(fileName, respBytes, 0666)
}
 

你可能感兴趣的:(go语言)