源地址:https://github.com/chromedp/chromedp/issues/543
可以用来爬取视频网站的视频地址,例如m3u8,或者隐藏在js文件里的m3u8(例如爱奇艺),或者直接就是mp4的链接
package main
import (
"context"
"fmt"
"github.com/chromedp/cdproto/cdp"
"github.com/chromedp/cdproto/network"
"github.com/chromedp/chromedp"
"io/ioutil"
"log"
"time"
)
func main() {
////create context
options := []chromedp.ExecAllocatorOption{
chromedp.Flag("headless", true), // debug使用
chromedp.Flag("blink-settings", "imagesEnabled=false"),
chromedp.UserAgent(`Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36`),
}
options = append(chromedp.DefaultExecAllocatorOptions[:], options...)
c, _ := chromedp.NewExecAllocator(context.Background(), options...)
// create context
ctx, cancel := chromedp.NewContext(c, chromedp.WithLogf(log.Printf))
defer cancel()
// run task list
chromedp.ListenTarget(
ctx,
func(ev interface{}){
if ev, ok := ev.(*network.EventResponseReceived); ok {
fmt.Println("event received:")
fmt.Println(ev.Type)
if ev.Type != "XHR" {
return
}
fmt.Println(ev.Response.URL)
go func () {
// print response body
c := chromedp.FromContext(ctx)
rbp := network.GetResponseBody(ev.RequestID)
body, err := rbp.Do(cdp.WithExecutor(ctx, c.Target))
if err != nil {
fmt.Println(err)
}
if err = ioutil.WriteFile(ev.RequestID.String(), body, 0644); err != nil {
log.Fatal(err)
}
if err == nil {
fmt.Printf("%s\n", body)
}
}()
}
},
)
//
// navigate to a page, wait for an element, click
err := chromedp.Run(ctx,
network.Enable(),
chromedp.Navigate("https://v.pptv.com/show/ta6aGel9ZKIFgibs.html"),
chromedp.Sleep(time.Second*15),
)
if err != nil {
log.Fatal(err)
}
}
ListenTarget 添加了一个函数,只要在 chromedp context中接收到目标事件,就会调用该函数。最好再里面用goroutine运行