其实本章算不上逆向教程 只是介绍golang的colly框架而已

列表页分析

根据关键字搜索
通过抓包分析可知
下一页所请求的参数如下

上图标红的代表所需参数
所以其实我们真正需要的也就是SearchSql
但是你多观察即可知 这个SearchSql在第一页访问的时候就传给了
所以也就是
爬取第一页获取第二页的请求参数——爬取第二页获取第三页的请求参数

详情页分析

详情页链接在列表页给你的链接是错误的
真是的链接如下图所示

由此图可知我们需要很多参数才能完成爬取
但是列表页给我们的链接也不是毫无用处

我们用正则提取中间所需要的参数即可完成爬取

代码

由于代码是用Go写的 用colly 还不是特别熟练

package main

import (
	"fmt"
	"github.com/gocolly/colly"
	"regexp"
	"time"
)


func main() {
	PostUrl := "https://kns.cnki.net/kns8/Brief/GetGridTableHtml"
	c, contentCollector := initCollector()

	c.OnRequest(func(r *colly.Request) {
		fmt.Println(r)
		fmt.Println(r.Method)
		r.Headers.Set("Accept", "text/html, */*; q=0.01")
		r.Headers.Set("Accept-Language", "zh-CN,zh;q=0.9")
		r.Headers.Set("Connection", "keep-alive")
		r.Headers.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
		r.Headers.Set("Origin", "https://kns.cnki.net")
		r.Headers.Set("Referer", "https://kns.cnki.net/kns8/defaultresult/index")
		r.Headers.Set("Sec-Fetch-Dest", "empty")
		r.Headers.Set("Sec-Fetch-Mode", "cors")
		r.Headers.Set("Sec-Fetch-Site", "same-origin")
		r.Headers.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36")
		r.Headers.Set("X-Requested-With", "XMLHttpRequest")

	})

	// 解析列表页
	c.OnHTML(".result-table-list tbody", func(element *colly.HTMLElement) {

		element.ForEach("tr", func(i int, element *colly.HTMLElement) {
			// 获取链接地址
			titleUrl, _ := element.DOM.Find("a").Attr("href")
			// 获取时间
			titleDate := element.DOM.Find("td.date").Text()
			// 拿到标题名
			s := element.DOM.Find("a").Text()
			// 切片取值
			if len(s) < 0 {
				_ = fmt.Errorf("匹配失败,长度为：%v\n", len(s))
			}

			titleName := s[:len(s)-4]

			// 正则取 decode的值
			//deCodeRe := `DbCode=(.*?)&`
			// 正则 fileName 的值
			fileNameRe := `FileName=(.*?)&`
			// 正则 取DbName 的值
			dbNameRe := `DbName=(.*?)&`
			isMatched, _ := regexp.MatchString(fileNameRe, titleUrl)
			if !isMatched {
				_ = fmt.Errorf("匹配失败,%v\n", isMatched)
			}
			fileName := regexp.MustCompile(fileNameRe).FindStringSubmatch(titleUrl)[1]
			dbName := regexp.MustCompile(dbNameRe).FindStringSubmatch(titleUrl)[1]

			trueUrl := fmt.Sprintf(`https://kns.cnki.net/kcms/detail/detail.aspx?dbcode=CCND&dbname=%s&filename=%s&uniplatform=NZKPT`, dbName, fileName)

			fmt.Println(titleName)
			fmt.Println(titleDate)
			fmt.Println(trueUrl)

			_ = contentCollector.Visit(trueUrl)
		})

		sqlVal, _ := element.DOM.Find("#sqlVal").Attr("value")

		_ = c.Post(PostUrl, map[string]string{
			"IsSearch":          "false",
			"QueryJson":         `{"Platform":"","DBCode":"CFLS","KuaKuCode":"CJFQ,CDMD,CIPD,CCND,CISD,SNAD,BDZK,CCJD,CCVD,CJFN","QNode":{"QGroup":[{"Key":"Subject","Title":"","Logic":1,"Items":[{"Title":"主题","Name":"SU","Value":"矿","Operate":"%=","BlurType":""}],"ChildItems":[]}]}}`,
			"SearchSql":         sqlVal,
			"PageName":          "defaultresult",
			"HandlerId":         "16",
			"DBCode":            "CFLS",
			"KuaKuCodes":        "CJFQ,CDMD,CIPD,CCND,CISD,SNAD,BDZK,CCJD,CCVD,CJFN",
			"CurPage":           "2",
			"RecordsCntPerPage": "20",
			"CurDisplayMode":    "listmode",
			"CurrSortField":     "",
			"CurrSortFieldType": "desc",
			"IsSortSearch":      "false",
			"IsSentenceSearch":  "false",
			"Subject":           "",
		})

	})

	// 解析详情页
	contentCollector.OnHTML(".brief", func(element *colly.HTMLElement) {
		contentHtml := element.Text
		fmt.Printf(contentHtml)
	})

	_ = c.Post(PostUrl, map[string]string{
		"IsSearch":          "true",
		"QueryJson":         `{"Platform":"","DBCode":"CFLS","KuaKuCode":"CJFQ,CDMD,CIPD,CCND,CISD,SNAD,BDZK,CCJD,CCVD,CJFN","QNode":{"QGroup":[{"Key":"Subject","Title":"","Logic":1,"Items":[{"Title":"主题","Name":"SU","Value":"矿","Operate":"%=","BlurType":""}],"ChildItems":[]}]}}`,
		"PageName":          "defaultresult",
		"DBCode":            "CFLS",
		"KuaKuCodes":        "CJFQ,CDMD,CIPD,CCND,CISD,SNAD,BDZK,CCJD,CCVD,CJFN",
		"CurPage":           "1",
		"RecordsCntPerPage": "20",
		"CurDisplayMode":    "listmode",
		"CurrSortField":     "",
		"CurrSortFieldType": "desc",
		"IsSentenceSearch":  "false",
		"Subject":           "",
	})

}

func initCollector() (*colly.Collector, *colly.Collector) {
	// 1. 创建收集器
	c := colly.NewCollector(
		colly.UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.125 Safari/537.36"),
		colly.AllowURLRevisit(),
	)

	// 设置抓取频率限制
	_ = c.Limit(&colly.LimitRule{
		DomainGlob:  "*",
		RandomDelay: 5 * time.Second, // 随机延迟
	})

	// 设置clone内容 解析
	contentCollector := c.Clone()
	return c, contentCollector
}

原文地址：http://www.cnblogs.com/zichliang/p/16790994.html