• golang colly下载图片


    package main
    
    import (
        "bytes"
        "fmt"
        "github.com/gocolly/colly"
        "io/ioutil"
        "log"
        "strconv"
        "strings"
        "sync"
    )
    
    func down(i int, group *sync.WaitGroup) {
        //func down(i int) {
        var c = colly.NewCollector()
        c.OnRequest(func(r *colly.Request) {
            r.Headers.Set("cookie", "__yjs_duid=1_54772224670dec235da1fe2f38979cd31642487126183; Hm_lvt_526caf4e20c21f06a4e9209712d6a20e=1642487108,1642487119,1642487123,1642487131; zkhanecookieclassrecord=%2C54%2C53%2C; yjs_js_security_passport=7f28c9b52fda683cc710865e91dcec0027759517_1642495580_js; Hm_lpvt_526caf4e20c21f06a4e9209712d6a20e=1642496144")
            r.Headers.Set("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36")
    
        })
        c.OnError(func(_ *colly.Response, err error) {
            log.Println("Something went wrong:", err)
        })
        c.OnHTML("#main .slist ul li img[src]", func(e *colly.HTMLElement) {
            fmt.Println(e.Attr("src"))
            url := "https://pic.netbian.com" + e.Attr("src")
    
            var c = colly.NewCollector()
            c.OnRequest(func(r *colly.Request) {
                r.Headers.Set("cookie", "__yjs_duid=1_54772224670dec235da1fe2f38979cd31642487126183; Hm_lvt_526caf4e20c21f06a4e9209712d6a20e=1642487108,1642487119,1642487123,1642487131; zkhanecookieclassrecord=%2C54%2C53%2C; Hm_lpvt_526caf4e20c21f06a4e9209712d6a20e=1642494377; yjs_js_security_passport=7f28c9b52fda683cc710865e91dcec0027759517_1642495580_js")
                r.Headers.Set("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36")
    
            })
            c.OnResponse(func(r *colly.Response) {
                reader := bytes.NewReader(r.Body)
                body, _ := ioutil.ReadAll(reader)
                //读取图片内容
                index := strings.LastIndex(url, "/")
                s := url[index+1:]
                err := ioutil.WriteFile("./img/"+s, body, 0755)
                fmt.Println(url, err)
            })
            c.Visit(url)
    
        })
        c.OnRequest(func(r *colly.Request) {
            fmt.Println("Visiting", r.URL.String())
        })
        c.OnScraped(func(r *colly.Response) {
            group.Done()
        })
        c.Visit("https://pic.netbian.com/4kmeinv/index_" + strconv.Itoa(i) + ".html")
    
    }
    
    func main() {
        group := sync.WaitGroup{}
        for i := 2; i < 100; i++ {
            group.Add(1)
            go down(i, &group)
        }
        group.Wait()
        fmt.Println("下载完整")
    }
  • 相关阅读:
    logging模板日志格式
    MariaDB修改默认字符集
    Django之表单验证
    Django之定制属于自己的admin
    sympy-高数可以这么学
    matplotlib01
    mysql---- 用户权限管理
    django----JSONP知识回顾
    django----文件上传
    数据库结构备份
  • 原文地址:https://www.cnblogs.com/chengfengchi/p/15818917.html
Copyright © 2020-2023  润新知