糗事百科爬虫源代码1.0
'''
#=============================================================================
# FileName: qiushibaike.py
# Desc:
# Author: modys
# Email: http://www.modys.top
# HomePage: http://www.modys.top
# Version: 1.0
# LastChange: 2020-08-23 19:11:02
# History:
#=============================================================================
'''
# 爬去糗事百科段子
from urllib import request
import re
headers = ('User-Agent','Mozilla/5.0 (Windows NT 6.3; Win64; x64; rv:79.0) Gecko/20100101 Firefox/79.0')
opener = request.build_opener()
opener.addheaders = [headers]
# 安装全局
request.install_opener(opener)
fh = open('C:\\Users\\Administrator\\Desktop\\qiushibaike.txt', 'w', encoding='utf-8')
# 爬去前5页
for i in range(0, 5):
url = 'https://www.qiushibaike.com/text/page/' + str(1+i)
req = request.urlopen(url).read().decode('utf-8', 'ignore')
print(len(req))
print(url)
pat = '<div class="content">.*?<span>(.*?)</span>.*?</a>'
rst = re.compile(pat, re.S).findall(req)
for i in range(0, len(rst)):
# print(rst[i].strip())
# print(rst[i].strip() + "\n", end="---------\n")
fh.write(str(rst[i].strip()) + "\n-----------\n")
fh.close()
