用python+sklearn(机器学习)实现天气预测

1
# -*- coding: utf-8 -*-
2
# @Time: 2020/12/16
3
# @Author: Eritque arcus
4
# 功能: 爬取数据
5
import urllib3
6

7

8
class GetData:
9
    url = ""
10
    headers = ""
11

12
    def __init__(self, url, header=""):
13
        """
14
        :param url: 获取的网址
15
        :param header: 请求头，默认已内置
16
        """
17
        self.url = url
18
        if header == "":
19
            self.headers = {
20
                'Connection': 'Keep-Alive',
21
                'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,'
22
                          '*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
23
                'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
24
                'Accept-Encoding': 'gzip, deflate',
25
                'User-Agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, '
26
                              'like Gecko) Chrome/87.0.4280.66 Mobile Safari/537.36 ',
27
                'Host': 'www.meteomanz.com'
28
            }
29
        else:
30
            self.headers = header
31

32
    def Get(self):
33
        """
34
        :return: 网址对应的网页内容
35
        """
36
        http = urllib3.PoolManager()
37
        return http.request('GET', self.url, headers=self.headers).data

1
# -*- coding: utf-8 -*-
2
# @Time: 2020/12/16
3
# @Author: Eritque arcus
4
import re
5
from bs4 import BeautifulSoup
6
from GetData import GetData
7
import datetime as DT
8
import csv
9

10
def a(t):
11
    return t.replace(" - ", "0")
12

13
# 功能: 写csv
14
def write(years, b, c):
15
    """
16
    :param years: [开始日期距离现在的年份, 结束日期距离现在的年份]
17
    :param b: [开始日期距离现在日期的天数, 结束日期距离现在日期的天数]
18
    :param c: csv文件名
19
    :return: None
20
    """
21
    # 1. 创建文件对象
22
    f = open(c, 'w', encoding='utf-8', newline='')
23

24
    # 2. 基于文件对象构建 csv写入对象
25
    csv_writer = csv.writer(f)
26

27
    # 3. 构建列表头
28
    # , "negAve", "negMax", "negMin"
29
    csv_writer.writerow(["Time", "Ave_t", "Max_t", "Min_t", "Prec", "SLpress", "Winddir", "Windsp", "Cloud"])
30
    # 取现在日期
31
    today = DT.datetime.now()
32
    # 取20天前日期
33
    week_ago = (today - DT.timedelta(days=b[0])).date()
34
    # 20天后
35
    week_pre = (today + DT.timedelta(days=b[1])).date()
36
    # 城市id 广州59287 青岛 54857
37
    id = "59287"
38
    # 爬取数据链接
39
    url = "http://www.meteomanz.com/sy2?l=1&cou=2250&ind=" + id + "&d1=" + str(week_ago.day).zfill(2) + "&m1=" + str(
40
        week_ago.month).zfill(2) + "&y1=" + str(week_ago.year - years[0]) + "&d2=" + str(week_pre.day).zfill(
41
        2) + "&m2=" + str(week_pre.month).zfill(2) + "&y2=" + str(week_pre.year - years[1])
42
    # 显示获取数据集的网址
43
    print(url)
44
    g = GetData(url).Get()
45
    # beautifulsoup解析网页
46
    soup = BeautifulSoup(g, "html5lib")
47
    # 取<tbody>内容
48
    tb = soup.find(name="tbody")
49
    # 取tr内容
50
    past_tr = tb.find_all(name="tr")
51
    for tr in past_tr:
52
        # 取tr内每个td的内容
53
        text = tr.find_all(name="td")
54
        flag = False
55
        negA = negMax = negMin = False
56
        for i in range(0, len(text)):
57
            if i == 0:
58
                text[i] = text[i].a.string
59
                # 网站bug，会给每个月第0天，比如 00/11/2020,所以要drop掉
60
                if "00/" in text[i]:
61
                    flag = True
62
            elif i == 8:
63
                # 把/8去掉，网页显示的格式
64
                text[i] = text[i].string.replace("/8", "")
65
            elif i == 5:
66
                # 去掉单位
67
                text[i] = text[i].string.replace(" Hpa", "")
68
            elif i == 6:
69
                # 去掉风力里括号内的内容
70
                text[i] = re.sub(u"[º(.*?|N|W|E|S)]", "", text[i].string)
71
            else:
72
                # 取每个元素的内容
73
                text[i] = text[i].string
74
            # 丢失数据都取2(简陋做法)
75
            # 这么做 MAE=3.6021
76
            text[i] = "2" if text[i] == "-" else text[i]
77
            text[i] = "2" if text[i] == "Tr" else text[i]
78
        text = text[0:9]
79
        # ext += [str(int(negA)), str(int(negMax)), str(int(negMin))]
80
        # 4. 写入csv文件内容
81
        if not flag:
82
            csv_writer.writerow(text)
83
    # 5. 关闭文件
84
    f.close()

1
# -*- coding: utf-8 -*-
2
# @Time: 2020/12/16
3
# @Author: Eritque arcus
4
# 功能: 爬取数据
5
import urllib3
6

7

8
class GetData:
9
    url = ""
10
    headers = ""
11

12
    def __init__(self, url, header=""):
13
        """
14
        :param url: 获取的网址
15
        :param header: 请求头，默认已内置
16
        """
17
        self.url = url
18
        if header == "":
19
            self.headers = {
20
                'Connection': 'Keep-Alive',
21
                'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,'
22
                          '*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
23
                'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
24
                'Accept-Encoding': 'gzip, deflate',
25
                'User-Agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, '
26
                              'like Gecko) Chrome/87.0.4280.66 Mobile Safari/537.36 ',
27
                'Host': 'www.meteomanz.com'
28
            }
29
        else:
30
            self.headers = header
31

32
    def Get(self):
33
        """
34
        :return: 网址对应的网页内容
35
        """
36
        http = urllib3.PoolManager()
37
        return http.request('GET', self.url, headers=self.headers).data

1
# -*- coding: utf-8 -*-
2
# @Time: 2020/12/16
3
# @Author: Eritque arcus
4
from Write import Write
5
import pandas as pd
6
from sklearn.model_selection import train_test_split
7
from sklearn.impute import SimpleImputer
8
import seaborn as sns
9
import matplotlib.pyplot as plt
10

11

12
# 功能: 数据预处理
13
def ProcessData():
14
    """
15
    :return:
16
        [X_train X训练数据集,
17
        X_valid X训练数据集的验证集,
18
        y_train Y训练数据集,
19
        y_valid Y训练数据集的验证集,
20
        imputed_X_test 预测数据集]
21
    """
22
    # 用近几年的数据做训练集
23
    # 如 [1,1], [20, 0]就是用2019年的今天的20天前到2019年的今天数据做训练集
24
    # 写入csv
25
    Write([1, 1], [20, 0], "weather_train_train.csv")
26
    Write([1, 1], [0, 20], "weather_train_valid.csv")
27
    Write([0, 0], [20, 0], "weather_test.csv")
28
    X_test = pd.read_csv("weather_test.csv", index_col="Time", parse_dates=True)
29
    # 读取测试集和验证集
30
    X = pd.read_csv("weather_train_train.csv", index_col="Time", parse_dates=True)
31
    y = pd.read_csv("weather_train_valid.csv", index_col="Time", parse_dates=True)
32
    # 把全部丢失的数据都drop，MAE=3.7又高了，所以去掉了
33
    # dxtcol = [col for col in X_test.columns
34
    #           if X_test[col].isnull().all()]
35
    # dxcol = [col for col in X.columns
36
    #          if X[col].isnull().all()]
37
    # dycol = [col for col in y.columns
38
    #          if y[col].isnull().all()]
39
    # for a1 in [dxtcol, dxcol, dycol]:
40
    #     for a2 in a1:
41
    #         if a2 in X_test.columns:
42
    #             X_test = X_test.drop(a2, axis=1)
43
    #         if a2 in X.columns:
44
    #             X = X.drop(a2, axis=1)
45
    #         if a2 in y.columns:
46
    #             y = y.drop(a2, axis=1)
47
    # 数据归一化和标准化，无法还原不用
48
    # scaler = preprocessing.StandardScaler()
49
    # pars = [cols for cols in X.columns if cols != "Time"]
50
    # for data in [X, y, X_test]:
51
    #     for par in pars:
52
    #         data[par] = scaler.fit_transform(data[par].values.reshape(-1, 1))
53
    #         # temp = scaler.fit(data[par].values.reshape(-1, 1))
54
    #         # data[par] = scaler.fit_transform(data[par].values.reshape(-1, 1), temp)
55

56
    # 填充缺少的数值用方差，不清楚效果如何
57
    my_imputer = SimpleImputer()
58
    X_train, X_valid, y_train, y_valid = train_test_split(X, y, train_size=0.8, test_size=0.2, random_state=0)
59
    imputed_X_train = pd.DataFrame(my_imputer.fit_transform(X_train))
60
    imputed_X_valid = pd.DataFrame(my_imputer.transform(X_valid))
61
    imputed_X_train.columns = X_train.columns
62
    imputed_X_valid.columns = X_valid.columns
63
    imputed_y_train = pd.DataFrame(my_imputer.fit_transform(y_train))
64
    imputed_y_valid = pd.DataFrame(my_imputer.transform(y_valid))
65
    imputed_y_train.columns = y_train.columns
66
    imputed_y_valid.columns = y_valid.columns
67
    imputed_X_test = pd.DataFrame(my_imputer.fit_transform(X_test))
68

69
    # 画折线图
70
    # sns.lineplot(data=X)
71
    # plt.show()
72
    # sns.lineplot(data=y)
73
    # plt.show()
74
    # sns.lineplot(data=X_test)
75
    # plt.show()
76
    # 返回分割后的数据集
77
    return [imputed_X_train, imputed_X_valid, imputed_y_train, imputed_y_valid, imputed_X_test]

1
# -*- coding: utf-8 -*-
2
# @Time: 2020/12/16
3
# @Author: Eritque arcus
4
from sklearn.ensemble import RandomForestRegressor
5
import joblib
6
from sklearn.metrics import mean_absolute_error
7
from ProcessData import ProcessData
8

9

10
# 训练并保存模型
11
def GetModel(a="Model.pkl"):
12
    """
13
    :param a: 模型文件名
14
    :return:
15
        [socre: MAE评估结果,
16
        X_test: 预测数据集]
17
    """
18
    # 取到数据
19
    [X_train, X_valid, y_train, y_valid, X_test] = ProcessData()
20
    # 用XGB模型，不过用有bug
21
    # modelX = XGBRegressor(n_estimators=1000, learning_rate=0.05, random_state=0, n_jobs=4)
22
    # # model.fit(X_train_3, y_train_3)
23
    # # model.fit(X_train_2, y_train_2)
24
    # col = ["Ave_t", "Max_t", "Min_t", "Prec","SLpress", "Winddir", "Windsp", "Cloud"]
25
    # modelX.fit(X_train, y_train,
26
    #           early_stopping_rounds=5,
27
    #           eval_set=[(X_valid, y_valid)],
28
    #           verbose=False)
29
    # 随机树森林模型
30
    model = RandomForestRegressor(random_state=0, n_estimators=1001)
31
    # 训练模型
32
    model.fit(X_train, y_train)
33
    # 预测模型，用上个星期的数据
34
    preds = model.predict(X_valid)
35
    # 用MAE评估
36
    score = mean_absolute_error(y_valid, preds)
37
    # 保存模型到本地
38
    joblib.dump(model, a)
39
    # 返回MAE
40
    return [score, X_test]

1
# -*- coding: utf-8 -*-
2
# @Time: 2020/12/16
3
# @Author: Eritque arcus
4
import joblib
5
import datetime as DT
6
from GetModel import GetModel
7
import matplotlib.pyplot as plt
8

9

10
# 训练并保存模型并返回MAE
11
r = GetModel()
12
print("MAE:", r[0])
13
# 读取保存的模型
14
model = joblib.load('Model.pkl')
15

16
# 最终预测结果
17
preds = model.predict(r[1])
18
# 反归一化或标准化，不过出bug了，不用
19
# for cols in range(0, len(preds)):
20
#     preds[cols] = scaler.inverse_transform(preds[cols])
21
# sns.lineplot(data=preds)
22
# plt.show()
23
# 打印结果到控制台
24
print("未来7天预测")
25
print(preds)
26
all_ave_t = []
27
all_high_t = []
28
all_low_t = []
29
for a in range(1, 7):
30
    today = DT.datetime.now()
31
    time = (today + DT.timedelta(days=a)).date()
32
    print(time.year, '/', time.month, '/', time.day,
33
          ': 平均气温',  preds[a][0],
34
          '最高气温', preds[a][1],
35
          '最低气温', preds[a][2],
36
          "降雨量", preds[a][3],
37
          "风力", preds[a][4])
38
    all_ave_t.append(preds[a][0])
39
    all_high_t.append(preds[a][1])
40
    all_low_t.append(preds[a][2])
41
temp = {"ave_t": all_ave_t, "high_t": all_high_t, "low_t": all_low_t}
42
# 绘画折线图
43
plt.plot(range(1, 7), temp["ave_t"], color="green", label="ave_t")
44
plt.plot(range(1, 7), temp["high_t"], color="red", label="high_t")
45
plt.plot(range(1, 7), temp["low_t"], color="blue", label="low_t")
46
plt.legend()  # 显示图例
47
plt.ylabel("Temperature(°C)")
48
plt.xlabel("day")
49
# 显示
50
plt.show()