import os
import requests
from bs4 import BeautifulSoup

# 聲明請(qǐng)求頭
headers = {
	'User-Agent': 'Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36'
}

# 創(chuàng)建保存小說(shuō)文本的文件夾
if not os.path.exists('./小說(shuō)'):
    os.mkdir('./小說(shuō)/')
    
# 訪問(wèn)網(wǎng)站并獲取頁(yè)面數(shù)據(jù)
response = requests.get('http://www.biquw.com/book/1/').text
print(response)

#### 重新編寫訪問(wèn)代碼
```python
response = requests.get('http://www.biquw.com/book/1/')
response.encoding = response.apparent_encoding
print(response.text)

'''
這種方式返回的中文數(shù)據(jù)才是正確的
'''

'''
根據(jù)上圖所示，數(shù)據(jù)是保存在a標(biāo)簽當(dāng)中的。a的父標(biāo)簽為li，li的父標(biāo)簽為ul標(biāo)簽，ul標(biāo)簽之上為div標(biāo)簽。所以如果想要獲取整個(gè)頁(yè)面的小說(shuō)章節(jié)數(shù)據(jù)，那么需要先獲取div標(biāo)簽。并且div標(biāo)簽中包含了class屬性，我們可以通過(guò)class屬性獲取指定的div標(biāo)簽，詳情看代碼~
'''
# lxml: html解析庫(kù) 將html代碼轉(zhuǎn)成python對(duì)象，python可以對(duì)html代碼進(jìn)行控制
soup = BeautifulSoup(response.text, 'lxml')
book_list = soup.find('div', class_='book_list').find_all('a')
# soup對(duì)象獲取批量數(shù)據(jù)后返回的是一個(gè)列表，我們可以對(duì)列表進(jìn)行迭代提取
for book in book_list:
    book_name = book.text
    # 獲取到列表數(shù)據(jù)之后，需要獲取文章詳情頁(yè)的鏈接，鏈接在a標(biāo)簽的href屬性中
    book_url = book['href']

book_info_html = requests.get('http://www.biquw.com/book/1/' + book_url, headers=headers)
book_info_html.encoding = book_info_html.apparent_encoding
soup = BeautifulSoup(book_info_html.text, 'lxml')

info = soup.find('div', id='htmlContent')
print(info.text)

with open('./小說(shuō)/' + book_name + '.txt', 'a', encoding='utf-8') as f:
    f.write(info.text)