目录字符 数量限定 其他 爬虫 补充:正则表达式加golang爬虫爬取经典案例豆瓣top250最近学习Go,爬取网站数据用到正则表达式,做个总结; Go中正则表达式采用RE2语法(具
Go中正则表达式采用RE2语法(具体是啥咱也不清楚);
方法
//参数正则字符串,返回值*Regexp
str := regexp.MustCompile(string)
//参数要查找的数据,查找次数-1为全局,返回值二维数组,查找出的字符串+正则字符串
var result [][]string = str.FindAllStringSubmatch(data, -1)
爬取博客园所有文章阅读量,评论,推荐;
package main
import (
"fmt"
"io"
"net/Http"
"regexp"
"strconv"
)
var readCount int = 0
var commentCount int = 0
var digGCount int = 0
//http读取网页数据写入result返回
func HttpGet(url string) (result string, err error) {
resp, err1 := http.Get(url)
if err1 != nil {
err = err1
return
}
defer resp.Body.Close()
buf := make([]byte, 4096)
for {
n, err2 := resp.Body.Read(buf)
//fmt.Println(url)
if n == 0 {
break
}
if err2 != nil && err2 != io.EOF {
err = err2
return
}
result += string(buf[:n])
}
return result, err
}
//横向纵向爬取文章标题数据,并累计数值
func SpiderPageDB(index int, page chan int) {
url := "https://www.cnblogs.com/littleperilla/default.html?page=" + strconv.Itoa(index)
result, err := HttpGet(url)
if err != nil {
fmt.Println("HttpGet err:", err)
return
}
str := regexp.MustCompile("post-view-count\">阅读[(](?s:(.*?))[)]</span>")
alls := str.FindAllStringSubmatch(result, -1)
for _, j := range alls {
temp, err := strconv.Atoi(j[1])
if err != nil {
fmt.Println("string2int err:", err)
}
readCount += temp
}
str = regexp.MustCompile("post-comment-count\">评论[(](?s:(.*?))[)]</span>")
alls = str.FindAllStringSubmatch(result, -1)
for _, j := range alls {
temp, err := strconv.Atoi(j[1])
if err != nil {
fmt.Println("string2int err:", err)
}
commentCount += temp
}
str = regexp.MustCompile("post-digg-count\">推荐[(](?s:(.*?))[)]</span>")
alls = str.FindAllStringSubmatch(result, -1)
for _, j := range alls {
temp, err := strconv.Atoi(j[1])
if err != nil {
fmt.Println("string2int err:", err)
}
diggCount += temp
}
page <- index
}
//主要工作方法
func working(start, end int) {
fmt.Printf("正在从%d到%d爬取中...\n", start, end)
//channel通知主线程是否所有go都结束
page := make(chan int)
//多线程go程同时爬取
for i := start; i <= end; i++ {
go SpiderPageDB(i, page)
}
for i := start; i <= end; i++ {
fmt.Printf("拉取到%d页\n", <-page)
}
}
//入口函数
func main() {
//输入爬取的起始页
var start, end int
fmt.Print("startPos:")
fmt.Scan(&start)
fmt.Print("endPos:")
fmt.Scan(&end)
working(start, end)
fmt.Println("阅读:", readCount)
fmt.Println("评论:", commentCount)
fmt.Println("推荐:", diggCount)
}
package main
import (
"fmt"
"io"
"net/http"
"os"
"regexp"
"strconv"
)
func savToFile(index int, filmName, filmScore [][]string) {
f, err := os.Create("第" + strconv.Itoa(index) + "页.txt")
if err != nil {
fmt.Println("os create err", err)
return
}
defer f.Close()
// 查出有多少条
n := len(filmName)
// 先写抬头 名称 评分
f.WriteString("电影名称" + "\t\t\t" + "评分" + "\n")
for i := 0; i < n; i++ {
f.WriteString(filmName[i][1] + "\t\t\t" + filmScore[i][1] + "\n")
}
}
func main() {
var start, end int
fmt.Print("请输入要爬取的起始页")
fmt.Scan(&start)
fmt.Print("请输入要爬取的终止页")
fmt.Scan(&end)
working(start, end)
}
func working(start int, end int) {
fmt.Printf("正在爬取%d到%d页", start, end)
for i := start; i <= end; i++ {
SpiderPage(i)
}
}
// 爬取一个豆瓣页面数据信息保存到文档
func SpiderPage(index int) {
// 获取url
url := "https://movie.douban.com/top250?start=" + strconv.Itoa((index-1)*25) + "&filter="
// 爬取url对应页面
result, err := HttpGet(url)
if err != nil {
fmt.Println("httpget err", err)
return
}
//fmt.Println("result=", result)
// 解析,编译正则表达式 ---电影名称
ret := regexp.MustCompile(`<img width="100" alt="(?s:(.*?))"`)
filmName := ret.FindAllStringSubmatch(result, -1)
for _, name := range filmName {
fmt.Println("name", name[1])
}
ret2 := regexp.MustCompile(`<span class="rating_num" property="v:average">(?s:(.*?))<`)
filmScore := ret2.FindAllStringSubmatch(result, -1)
for _, score := range filmScore {
fmt.Println("score", score[1])
}
savToFile(index, filmName, filmScore)
}
// 爬取指定url页面,返回result
func HttpGet(url string) (result string, err error) {
req, _ := http.NewRequest("GET", url, nil)
// 设置头部信息
req.Header.Set("User-Agent", "Mozilla/5.0 (windows NT 10.0; Win64; x64) AppleWEBKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36 OPR/66.0.3515.115")
resp, err1 := (&http.Client{}).Do(req)
//resp, err1 := http.Get(url) //此方法已经被豆瓣视为爬虫,返回状态吗为418,所以必须伪装头部用上述办法
if err1 != nil {
err = err1
return
}
defer resp.Body.Close()
buf := make([]byte, 4096)
//循环爬取整页数据
for {
n, err2 := resp.Body.Read(buf)
if n == 0 {
break
}
if err2 != nil && err2 != io.EOF {
err = err2
return
}
result += string(buf[:n])
}
return
}
到此这篇关于Golang爬虫及正则表达式的实现示例的文章就介绍到这了,更多相关Golang爬虫及正则表达式 内容请搜索编程网以前的文章或继续浏览下面的相关文章希望大家以后多多支持编程网!
--结束END--
本文标题: Golang爬虫及正则表达式的实现示例
本文链接: https://www.lsjlt.com/news/160643.html(转载时请注明来源链接)
有问题或投稿请发送至: 邮箱/279061341@qq.com QQ/279061341
下载Word文档到电脑,方便收藏和打印~
2024-04-05
2024-04-05
2024-04-05
2024-04-04
2024-04-05
2024-04-05
2024-04-05
2024-04-05
2024-04-04
回答
回答
回答
回答
回答
回答
回答
回答
回答
回答
0