import requests

protocol://host[:port]/path/[?query]#fragment
http://www.itcast.cn/index.html?name=andy&age=18#link

# 首先引入所需要的包
import requests
from bs4 import BeautifulSoup
import pandas as pd
import json
# 發(fā)送 GET 請(qǐng)求獲取網(wǎng)頁(yè)內(nèi)容
url = 'http://map.amap.com/subway/index.html'
response = requests.get(url)
# 第一步：爬取兩個(gè) div 中的城市數(shù)據(jù)（包括 ID 和拼音），生成城市集合
if response.status_code == 200:
    # 解碼
    response_content = response.content.decode('utf-8')
    # 使用 Beautiful Soup 解析網(wǎng)頁(yè)內(nèi)容
    soup = BeautifulSoup(response_content, 'html.parser')
    # 從這里開(kāi)始，你可以使用 Beautiful Soup 對(duì)象（soup）來(lái)提取所需的信息
    # 例如，查找標(biāo)題
    title = soup.title
    # 通過(guò)Beautiful Soup來(lái)找到城市信息元素，并提取這個(gè)元素的信息
    for soup_a in soup.find('div', class_='city-list fl').find_all('a'):
        city_name_py = soup_a['cityname']
        city_id = soup_a['id']
        city_name_ch = soup_a.get_text()
        city_info_list.append({'name_py': city_name_py, 'id': city_id, 'name_ch': city_name_ch})
        # 獲取未顯示出來(lái)的城市列表
    for soup_a in soup.find('div', class_='more-city-list').find_all('a'):
        city_name_py = soup_a['cityname']
        city_id = soup_a['id']
        city_name_ch = soup_a.get_text()
        city_info_list.append({'name_py': city_name_py, 'id': city_id, 'name_ch': city_name_ch})
        print(city_info_list)
else:
    print("無(wú)法獲取網(wǎng)頁(yè)內(nèi)容")
for city_info in city_info_list:
    city_id = city_info.get("id")
    city_name = city_info.get("name_py")
    city_name_ch = city_info.get("name_ch")
    print("開(kāi)始爬取城市" + city_name_ch + "的數(shù)據(jù)")
    city_lines_list = []
    # 第二步：遍歷城市集合，構(gòu)造每一個(gè)城市的 url,并下載數(shù)據(jù)
    # 構(gòu)造每個(gè)城市的url
    url = "http://map.amap.com/service/subway?_1717380520536&srhdata=" + city_id + '_drw_' + city_name + '.json'
    res = requests.get(url)
    content = res.content.decode('utf-8')
    # 將內(nèi)容字符串轉(zhuǎn)換成json對(duì)象
    content_json = json.loads(content)
    # 提取該城市的所有地鐵線list
    line_info_list = content_json.get("l")
    # 第三步：開(kāi)始處理每一個(gè)地鐵線，提取內(nèi)容到dataframe中
    for line_info in line_info_list:
        # 地鐵線名字
        line_name = line_info["kn"]
        # 處理地鐵線站點(diǎn)
        df_per_zd = pd.DataFrame(line_info["st"])
        df_per_zd = df_per_zd[['n', 'sl', 'poiid', 'sp', 't', 'su', 'sid']]
        df_per_zd['gd經(jīng)度'] = df_per_zd['sl'].apply(lambda x: x.split(',')[0])
        df_per_zd['gd緯度'] = df_per_zd['sl'].apply(lambda x: x.split(',')[1])
        df_per_zd.drop('sl', axis=1, inplace=True)
        df_per_zd['路線名稱'] = line_info['ln']
        df_per_zd['城市名稱'] = city_name_ch
        df_per_zd.rename(columns={"n": "站點(diǎn)名稱", "poiid": "POI編號(hào)", "sp": "拼音名稱", "t": "換乘標(biāo)志 1:換乘，0:不可換乘", "su": "su", "sid": "sid編號(hào)"}, inplace=True)
        # 先將這條地鐵線處理過(guò)的dataframe存起來(lái)，我們后面給他放到一張表里
        city_lines_list.append(df_per_zd)
    # 這段代碼就是將地鐵線數(shù)據(jù)列表聚合到一張表里，形成每個(gè)城市的地鐵站數(shù)據(jù)
    city_subway_data = pd.concat(city_lines_list, ignore_index=True)
    # 第四步：將處理好的文件保存為xlsx
    city_subway_data.to_excel(city_name_ch + '.xlsx', sheet_name='Sheet1')