爬虫—网页小说

# 怎么发送请求
# pip install requests
import requests
# pip install lxml
from lxml import etree


# 发送给谁
url = 'http://www.cits0871.com/booktxt/25529/14443163.html'

while True:
    # 伪装自己
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36'
    }
    # 发送请求
    resp = requests.get(url, headers=headers)
    # 设置编码
    resp.encoding = 'utf-8'
    # 响应信息
    #print(resp.text)

    a = '//div[@class="bottem1"]/a[4]/@href'
    e = etree.HTML(resp.text)
    info = '\n'.join(e.xpath('//div[@id="content"]/p/text()'))
    title = e.xpath('//h1/text()')[0]
    url = f'http://www.cits0871.com{e.xpath(a)[0]}'
    #print(info)
    print(title)
    # 保存
    with open('斗罗大陆.txt', 'a',encoding='utf-8') as f:
        f.write(title + '\n\n' + info + '\n\n')