go语言学习例子No.36-chromedp爬取某艺网视频

先说下思路：

打开F12的network选xhr找到下面的

这个里面包含了m3u8

那我们就搜索xhr里包含https://cache.video.iqiyi.com/dash的字符串

然后链接里包含bid=500应该是超清还是高清吧，我们就选这个再筛选一下

然后再正则匹配出m3u8写到本地，然后用N_m3u8DL这个工具下载

反正是写给自己用的，还是有点麻烦的

我先把链接都拷贝到1.txt，因为太多集会分页，又涉及到点击js动态生成，好麻烦

我就右键复制到1.txt里读取就好了

代码如下

package main

import (
	"context"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"regexp"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
	"github.com/chromedp/cdproto/cdp"
	"github.com/chromedp/cdproto/network"
	"github.com/chromedp/chromedp"
)

func main() {
	////create context
	name := make(chan string)
	finish := make(chan bool)
	options := []chromedp.ExecAllocatorOption{
		chromedp.Flag("headless", false), // debug使用
		//chromedp.Flag("blink-settings", "imagesEnabled=false"),
		chromedp.UserAgent(`Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36`),
	}
	options = append(chromedp.DefaultExecAllocatorOptions[:], options...)
	c, _ := chromedp.NewExecAllocator(context.Background(), options...)

	// create context
	ctx, cancel := chromedp.NewContext(c, chromedp.WithLogf(log.Printf))
	defer cancel()

	// run task list
	//var urls string
	chromedp.ListenTarget(
		ctx,
		func(ev interface{}) {
			if ev, ok := ev.(*network.EventResponseReceived); ok {
				//fmt.Println("event received:")
				//fmt.Println(ev.Type)
				//fmt.Println(ev.Response.URL)

				if ev.Type == "XHR" {
					rep := regexp.MustCompile(`https://cache.video.iqiyi.com/dash`)
					matchbool := rep.MatchString(ev.Response.URL)
					if matchbool != false {

						rep := regexp.MustCompile(`bid=500`)
						matchbool = rep.MatchString(ev.Response.URL)
						if matchbool != false {

							fmt.Println(ev.Response.URL)

							go func(ctx context.Context) {
								// print response body

								// if err := chromedp.Run(ctx,
								// 	chromedp.Title(&title),
								// ); err != nil {
								// 	panic(err)
								// }
								title := <-name

								c := chromedp.FromContext(ctx)
								rbp := network.GetResponseBody(ev.RequestID)
								body, err := rbp.Do(cdp.WithExecutor(ctx, c.Target))
								if err != nil {
									fmt.Println(err)
								}
								reg := regexp.MustCompile(`#EXTM3U[\s\S]+#EXT-X-ENDLIST`)//匹配出m3u8
								result := reg.Find(body)
								resultstring := strings.Replace(string(result), `\n`, "\n", -1)//替换js里m3u8多余字符串
								resultstring = strings.Replace(resultstring, `\/`, "/", -1)//替换js里m3u8多余字符串
								if err = ioutil.WriteFile(title+".m3u8", []byte(resultstring), 0644); err != nil {
									fmt.Println("写入m3u8文件失败！", err)
								}
								runcmd(title)
								finish <- true

							}(ctx)
							return
						}

						return
					}
				}

			}

		},
	)

	//

	// navigate to a page, wait for an element, click
	//var res string
	err := chromedp.Run(ctx,
		network.Enable(),
		chromedp.Navigate("https://www.iqiyi.com"),
		chromedp.Sleep(time.Second*30), //30s时间自己手动登录vip，当然可以不登录，但是会有广告。。。
	)
	if err != nil {
		log.Fatal(err)
	}
	file, err := os.Open("1.txt") //1.txt存放网址源码
	if err != nil {
		fmt.Println("读取文件失败！")
	}
	doc, err := goquery.NewDocumentFromReader(file)
	if err != nil {
		fmt.Println("goquery读取失败")
	}
	doc.Find("li").Each(func(i int, s *goquery.Selection) {
		num := s.Find("span").Text()
		title := s.Find("a").Text()

		src, _ := s.Find("a").Attr("href")
		if src == "" {
			return
		}
		err := chromedp.Run(ctx,
			chromedp.Navigate("https:"+src),
		)
		if err != nil {
			fmt.Println("跳转失败！", err)
		}
		if num == "" {
			name <- (title) //发送文件名等待协程取出
		} else {
			name <- (num + title) //发送文件名等待协程取出
		}

		<-finish //等待下载完成后协程发送true后继续下一次跳转

	})
}


func runcmd(name string) {
	command := `D:\soft\m3u8DL\N_m3u8DL-CLI_v2.9.7.exe "` + name + `.m3u8" --saveName "` + name + `" --enableDelAfterDone`

	cmd := exec.Command("PowerShell.exe", "/c", command)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Run()
	fmt.Println(name + ".mp4 finished")
}

本人基础不好，东拼西凑用的还是笨办法，不喜勿喷，谢谢