沙滩星空的博客沙滩星空的博客

使用chromedp爬取xhr数据

package main

import (
    "context"
    "fmt"
    "github.com/chromedp/cdproto/cdp"
    "github.com/chromedp/cdproto/network"
    "github.com/chromedp/chromedp"
    "io/ioutil"
    "log"
    "time"
)

func main() {
    ////create context
    options := []chromedp.ExecAllocatorOption{
        chromedp.Flag("headless", true), // debug使用
        chromedp.Flag("blink-settings", "imagesEnabled=false"),
        chromedp.UserAgent(`Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36`),
    }
    options = append(chromedp.DefaultExecAllocatorOptions[:], options...)
    c, _ := chromedp.NewExecAllocator(context.Background(), options...)

    // create context
    ctx, cancel := chromedp.NewContext(c, chromedp.WithLogf(log.Printf))
    defer cancel()

    // run task list
    chromedp.ListenTarget(
        ctx,
        func(ev interface{}){
            if ev, ok := ev.(*network.EventResponseReceived); ok {
                fmt.Println("event received:")
                fmt.Println(ev.Type)

                if ev.Type != "XHR" {
                    return
                }

                fmt.Println(ev.Response.URL)

                go func () {
                    // print response body
                    c := chromedp.FromContext(ctx)
                    rbp := network.GetResponseBody(ev.RequestID)
                    body, err := rbp.Do(cdp.WithExecutor(ctx, c.Target))
                    if err != nil {
                        fmt.Println(err)
                    }
                    if err = ioutil.WriteFile(ev.RequestID.String(), body, 0644); err != nil {
                        log.Fatal(err)
                    }
                    if err == nil {
                        fmt.Printf("%s\n", body)
                    }
                }()

            }
        },
    )


    //

    // navigate to a page, wait for an element, click
    err := chromedp.Run(ctx,
        network.Enable(),
        chromedp.Navigate("https://v.pptv.com/show/ta6aGel9ZKIFgibs.html"),
        chromedp.Sleep(time.Second*15),
    )
    if err != nil {
        log.Fatal(err)
    }
}

go语言学习例子No.34-chromedp爬取xhr数据 https://www.foxzc.com/archives/2021/07/21/1185
未经允许不得转载:沙滩星空的博客 » 使用chromedp爬取xhr数据

评论 抢沙发

  • 昵称 (必填)
  • 邮箱 (必填)
  • 网址