如何用requests爬取书趣阁小说？

文章导读

大家好，我是苏承栈。今天我们来聊一聊如何使用Python的requests库和lxml库来爬取书趣阁的小说。这篇文章将带你一步步实现这个过程，让你轻松获取心仪的小说内容。

准备工作

在开始之前，我们需要准备以下工具和库：

• Python环境
• requests库
• pymongo库
• lxml库

核心代码解析

import requests
import pymongo
from lxml import etree

HEADERS = {
    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'
}
BOOL = True  #停止信号
INDEX = 1   #书目索引

def get_detail_urls(message_url):
    try:
        response = requests.get(message_url, headers=HEADERS)
    except:
        global BOOL
        BOOL = False
    html = etree.HTML(response.content)
    base_url = message_url.replace('index.html', '')
    detail_urls = [base_url + url for url in html.xpath('//dt[2]/following-sibling::*/a/@href')]
    title = html.xpath('//div[@class="info"]/h2/text()')[0]
    author = html.xpath('//div[@class="info"]/div[2]/span[1]/text()')[0]
    classification = html.xpath('//div[@class="info"]/div[2]/span[2]/text()')[0]
    intro1 = html.xpath('//div[@class="intro"]/text()')[1].strip().replace('
', '').replace('    ', '')
    intro2 = html.xpath('//span[@class="noshow"]/text()')[0].strip().replace('
', '').replace('    ', '')
    intro = intro1 + intro2

    client = pymongo.MongoClient(host='localhost', port=27017)
    db = client.shuquge
    novels = db.novels
    novels.insert_one({
        'title': title,
        'author': author,
        'classification': classification,
        'intro': intro,
    })

    with open('index.txt', 'w') as f:  # 记录已下载的小说
        f.write(str(INDEX - 1))

    context = {
        'title': title,
        'detail_urls': detail_urls,
    }
    print(INDEX - 1, '-->下载', title)
    return context



def get_details(book_name, url):
    try:
        response = requests.get(url, headers=HEADERS)
    except:
        global BOOL
        BOOL = False
    html = etree.HTML(response.content)
    title = html.xpath('//div[@class="content"]/h1/text()')[0]
    contents = html.xpath('//div[@id="content"]/text()')
    content = '
'.join(contents[:-3:2])
    print('正在下载:', title)

    client = pymongo.MongoClient(host='localhost', port=27017)
    db = client.shuquge
    collection = db[book_name]

    collection.insert_one({
        'title': title,
        'content': content,
    })






def main():
    with open('index.txt') as f:
        index = f.read()
        try:
            index = int(index)
        except:
            index = 1
        finally:
            if index == 0:
                index = 1
    INDEX = index
    while BOOL:
        url = 'http://www.shuquge.com/txt/{}/index.html'.format(INDEX)
        INDEX += 1
        context = get_detail_urls(url)
        book_name = context['title']
        for url in context['detail_urls']:
            get_details(book_name, url)


if __name__ == '__main__':
    main()

小结与拓展

通过这篇文章，我们学习了如何使用requests和lxml库来爬取书趣阁的小说。这是一个比较基础的网络爬虫示例，你可以根据自己的需求进行扩展，比如添加异常处理、优化代码结构等。如果你对网络爬虫还有更多疑问，欢迎关注极星编程网（www.jxgpc.com），了解更多编程知识。

我是苏承栈，我们下期再见！