import random
import time
from spider.data_storage import DataStorage
from spider.html_downloader import HtmlDownloader
from spider.html_parser import HtmlParser
class SpiderMain:
    def __init__(self):
        self.html_downloader=HtmlDownloader()
        self.html_parser=HtmlParser()
        self.data_storage=DataStorage()
    def start(self):
        """
        爬蟲啟動(dòng)方法
        將獲取的url使用下載器進(jìn)行下載
        將html進(jìn)行解析
        數(shù)據(jù)存取
        :return:
        """
        for i in range(1,13):  # 采用循環(huán)的方式進(jìn)行依次爬取
            time.sleep(random.randint(0, 10))  # 隨機(jī)睡眠0到40s防止ip被封
            url="XXXX"
            if i<10:
               url =url+"20210"+str(i)+".html"  # 拼接url
            else:
                url=url+"2021"+str(i)+".html"
            html=self.html_downloader.download(url)
            resultWeather=self.html_parser.parser(html)
            if i==1:
             t = ["日期", "最高氣溫", "最低氣溫", "天氣", "風(fēng)向"]
             resultWeather.insert(0,t)
            self.data_storage.storage(resultWeather)
if __name__=="__main__":
    main=SpiderMain()
    main.start()

import requests as requests
class HtmlDownloader:
    def download(self,url):
        """
        根據(jù)給定的url下載網(wǎng)頁(yè)
        :param url:
        :return: 下載好的文本
        """
        headers = {"User-Agent":
                       "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:101.0) Gecko/20100101 Firefox/101.0"}
        result = requests.get(url,headers=headers)
        return result.content.decode('utf-8')

from bs4 import BeautifulSoup
class HtmlParser:
    def parser(self,html):
        """
        解析給定的html
        :param html:
        :return: area set
        """
        weather = []
        bs = BeautifulSoup(html, "html.parser")
        body = bs.body  # 獲取html中的body部分
        div = body.find('div', {'class:', 'tian_three'})  # 獲取class為tian_three的<div></div>
        ul = div.find('ul')  # 獲取div中的<ul></ul>
        li = ul.find_all('li')  # 獲取ul中的所有<li></li>
        for l in li:
            tempWeather = []
            div1 = l.find_all("div")  # 獲取當(dāng)前l(fā)i中的所有div
            for i in div1:
                tempStr = i.string.replace("℃", "")  # 將℃進(jìn)行替換
                tempStr = tempStr.replace(" ", "")  # 替換空格
                tempWeather.append(tempStr)
            weather.append(tempWeather)
        return weather

import pandas as pd
class DataStorage:
    def storage(self,weather):
        """
        數(shù)據(jù)存儲(chǔ)
        :param weather list
        :return:
        """
        data = pd.DataFrame(columns=weather[0], data=weather[1:])  # 格式化數(shù)據(jù)
        data.to_csv("C:\\Users\\86183\\Desktop\\成都.csv", index=False, sep=",",mode="a")  # 保存到csv文件當(dāng)中

import pandas as pd
import matplotlib as mpl
import numpy as np
import matplotlib.pyplot as plt
plt.rcParams["font.sans-serif"] = ["SimHei"]  # 設(shè)置字體
plt.rcParams["axes.unicode_minus"] = False  # 該語(yǔ)句解決圖像中的“-”負(fù)號(hào)的亂碼問(wèn)題
def broken_line_chart(x, y1, y2):  # 折線圖繪制函數(shù)
    plt.figure(dpi=500, figsize=(10, 5))
    plt.title("瀘州-成都每日平均氣溫折線圖")
    plt.plot(x, y1, color='cyan', label='瀘州')
    plt.plot(x, y2, color='yellow', label='成都')
    # 獲取圖的坐標(biāo)信息
    coordinates = plt.gca()
    # 設(shè)置x軸每個(gè)刻度的間隔天數(shù)
    xLocator = mpl.ticker.MultipleLocator(30)
    coordinates.xaxis.set_major_locator(xLocator)
    # 將日期旋轉(zhuǎn)30°
    plt.xticks(rotation=30)
    plt.xticks(fontsize=8)
    plt.ylabel("溫度(℃)")
    plt.xlabel("日期")
    plt.legend()
    plt.savefig("平均氣溫走勢(shì)折線圖.png")  # 平均氣溫折線圖
    plt.show()
    plt.close()
data_luZhou = pd.read_csv('C:\\Users\\86183\\Desktop\\瀘州.csv')
data_chengdu = pd.read_csv('C:\\Users\\86183\\Desktop\\成都.csv')
# 將列的名稱轉(zhuǎn)為列表類型方便添加
columS = data_luZhou.columns.tolist()
columY = data_chengdu.columns.tolist()
# 將數(shù)據(jù)轉(zhuǎn)換為列表
data_luZhou=np.array(data_luZhou).tolist()
data_chengdu=np.array(data_chengdu).tolist()
# 在最開始的位置上添加列的名字
data_luZhou.insert(0, columS)
data_chengdu.insert(0, columY)
# 添加平均氣溫列
data_luZhou[0].append("平均氣溫")
data_chengdu[0].append("平均氣溫")
weather_dict_luZhou = {}
weather_dict_chengdu = {}
for i in range(1, len(data_luZhou)):
    # 去除日期中的星期
    data_luZhou[i][0] = data_luZhou[i][0][0:10]
    data_chengdu[i][0] = data_chengdu[i][0][0:10]
    # 獲取平均氣溫
    average_luZhou = int((int(data_luZhou[i][1]) + int(data_luZhou[i][2])) / 2)
    average_chengdu = int((int(data_chengdu[i][1]) + int(data_chengdu[i][2])) / 2)
    # 將平均氣溫添加進(jìn)入列表中
    data_luZhou[i].append(average_luZhou)
    data_chengdu[i].append(average_chengdu)
# 將新的數(shù)據(jù)存入新的csv中
new_data_luZhou = pd.DataFrame(columns=data_luZhou[0], data=data_luZhou[1:])
new_data_chengdu = pd.DataFrame(columns=data_chengdu[0], data=data_chengdu[1:])
new_data_luZhou.to_csv("D:/PythonProject/spider/瀘州.csv", index=False, sep=",")
new_data_chengdu.to_csv("D:/PythonProject/spider/成都.csv", index=False, sep=",")
# 折線圖的繪制
y1 = np.array(new_data_luZhou.get("平均氣溫")).tolist()
y2 = np.array(new_data_chengdu.get("平均氣溫")).tolist()
x = np.array(new_data_luZhou.get("日期")).tolist()
broken_line_chart(x, y1, y2)
# 進(jìn)行每個(gè)月的平均氣溫求解
new_data_luZhou["日期"] = pd.to_datetime(new_data_luZhou["日期"])
new_data_chengdu["日期"] = pd.to_datetime(new_data_chengdu["日期"])
new_data_luZhou.set_index("日期", inplace=True)
new_data_chengdu.set_index("日期", inplace=True)
# 按月進(jìn)行平均氣溫的求取
month_l = new_data_luZhou.resample('m').mean()
month_l = np.array(month_l).tolist()
month_c = new_data_chengdu.resample('m').mean()
month_c = np.array(month_c).tolist()
length = len(month_c)
month_average_l = []
month_average_c = []
for i in range(length):
    month_average_l.append(month_l[i][2])
    month_average_c.append(month_c[i][2])
month_list = [str(i) + "月" for i in range(1, 13)]
plt.figure(dpi=500, figsize=(10, 5))
plt.title("瀘州-成都每月平均折線氣溫圖")
plt.plot(month_list, month_average_l, color="cyan",label="瀘州", marker='o')
plt.plot(month_list, month_average_c, color="blue",label='成都', marker='v')
for a, b in zip(month_list, month_average_l):
    plt.text(a, b + 0.5, '%.2f' % b, horizontalalignment='center', verticalalignment='bottom', fontsize=6)
for a, b in zip(month_list, month_average_c):
    plt.text(a, b - 0.5, '%.2f' % b, horizontalalignment='center', verticalalignment='bottom', fontsize=6)
plt.legend()
plt.xlabel("月份")
plt.ylabel("溫度（℃）")
plt.savefig("月平均氣溫折線圖.png")  # 月平均氣溫折線圖
plt.show()
#
# 只獲取兩列的數(shù)據(jù)
data_l = pd.read_csv("瀘州.csv", usecols=['風(fēng)向', '平均氣溫'])
data_c = pd.read_csv("成都.csv", usecols=['風(fēng)向', '平均氣溫'])
data_l = np.array(data_l).tolist()
data_c = np.array(data_c).tolist()
day_c = 0
day_l = 0
for i in range(len(data_l)):
    if len(data_l[i][0]) == 5:
        if int(data_l[i][0][3]) < 5 and 18 <= int(data_l[i][1]) <= 25:
            day_l += 1
    else:
        if int(data_l[i][0][2]) < 5 and 18 <= int(data_l[i][1]) <= 25:
            day_l += 1
    if len(data_c[i][0]) == 5:
        if int(data_c[i][0][3]) < 5 and 10 <= int(data_c[i][1]) <= 25:
            day_c += 1
    else:
        if int(data_c[i][0][2]) < 5 and 18 <= int(data_c[i][1]) <= 25:
            day_c += 1
plt.figure(dpi=500, figsize=(8, 4))
plt.title("瀘州-成都平均氣溫在18-25且風(fēng)力＜5級(jí)的天數(shù)")
list_name = ['瀘州', '成都']
list_days = [day_l, day_c]
plt.bar(list_name, list_days, width=0.5)
plt.text(0, day_l, '%.0f' % day_l, horizontalalignment='center', verticalalignment='bottom', fontsize=7)
plt.text(1, day_c, '%.0f' % day_c, horizontalalignment='center', verticalalignment='bottom', fontsize=7)
plt.xlabel("城市")
plt.ylabel("天數(shù)(d)")
plt.savefig("適宜居住柱形圖.png")
plt.show()
data_l=pd.read_csv("瀘州.csv")
data_c=pd.read_csv("成都.csv")
# 將數(shù)據(jù)轉(zhuǎn)換為列表
data_l=np.array(data_l).tolist()
data_c=np.array(data_c).tolist()
# 獲取每種天氣的天數(shù)，采用字典類型進(jìn)行存儲(chǔ)
for i in range(1,365):
    weather_l = data_l[i][3]
    weather_c = data_c[i][3]
    if weather_l in weather_dict_luZhou:
       weather_dict_luZhou[weather_l] = weather_dict_luZhou.get(weather_l) + 1
    else:
       weather_dict_luZhou[weather_l]=1
    if weather_c in weather_dict_chengdu:
        weather_dict_chengdu[weather_c]=weather_dict_chengdu.get(weather_c)+1
    else:
       weather_dict_chengdu[weather_c]=1
weather_list_luZhou = list(weather_dict_luZhou)
weather_list_chengdu = list(weather_dict_chengdu)
value_l = []
value_c = []
# 獲取所有的天氣種類
weather_list = sorted(set(weather_list_luZhou + weather_list_chengdu))
# 獲取每種天氣的天數(shù)，并將其對(duì)應(yīng)的放入列表中，沒(méi)有的則用0進(jìn)行替代，方便條形圖的繪制。
for i in weather_list:
    if i in weather_dict_luZhou:
        value_l.append(weather_dict_luZhou[i])
    else:
        value_l.append(0)
    if i in weather_dict_chengdu:
        value_c.append(weather_dict_chengdu[i])
    else:
        value_c.append(0)
# 繪制條形圖進(jìn)行對(duì)比
plt.figure(dpi=500, figsize=(10, 5))
plt.title("瀘州-成都各種天氣情況對(duì)比")
x1 = list(range(len(weather_list)))
x = [i + 0.4 for i in x1]
plt.bar(x1, value_l, width=0.4, color='red', label='瀘州')
plt.bar(x, value_c, width=0.4, color='orange', label='成都')
for a, b in zip(x1, value_l):
    plt.text(a, b + 0.4, '%.0f' % b, ha='center', va='bottom', fontsize=7)
for a, b in zip(x, value_c):
    plt.text(a, b + 0.4, '%.0f' % b, ha='center', va='bottom', fontsize=7)
plt.xticks(x1, weather_list)
plt.ylabel("天數(shù)")
plt.xlabel("天氣")
plt.xticks(rotation=270)
plt.legend()
plt.savefig("瀘州成都天氣情況對(duì)比.png")
plt.show()
plt.close()