老师，请问使用正规方程计算时矩阵不可逆怎么办呢？

[eanson023]

代码如下：

#! python3
# -*- encoding: utf-8 -*-
"""
Created on Mon Sep  26 08:02:10 2022

@author: eanson
"""
import pandas as pd
import torch
import math

import os

# 训练集比例
train_proportion = 0.7

#####################################################################
data_file = os.path.join('HW1 linear model', 'data1.txt')
data = pd.read_csv(data_file, header=None, names=[
                   "Room Size", "Room Amount", "Price"])
# 训练集按照比例划分
m = math.floor(len(data)*train_proportion)
X_train_pd, y_train_pd, X_cv_pd, y_cv_pd = data.iloc[:m, 0:2], data.iloc[:m,
                                                                         2], data.iloc[m+1:, 0:2], data.iloc[m+1:, 2]
# 转换为张量格式
X_train, y_train = torch.tensor(
    X_train_pd.values), torch.tensor(y_train_pd.values)
X_cv, y_cv = torch.tensor(X_cv_pd.values), torch.tensor(y_cv_pd.values)

############################################################################
# 正规方程
"""
$$
w=(X^TX)^{-1}X^Ty
$$
"""
w = X_train.t().mm(X_train).inverse().mm(X_train.t()).mm(y_train)

报错:

(d2l) yanshengdeMacBook-Pro:MachineLearning_HW_CQUT eanson$  cd /Users/eanson/Documents/ml/MachineLearning_HW_CQUT ; /usr/bin/env /Users/eanson/opt/miniconda3/envs/d2l/bin/python /Users/eanson/.vscode/extensions/ms-python.python-2021.2.633441544/pythonFiles/lib/python/debugpy/launcher 59765 -- /Users/eanson/Documents/ml/MachineLearning_HW_CQUT/linear_regression.py 
Traceback (most recent call last):
  File "/Users/eanson/Documents/ml/MachineLearning_HW_CQUT/linear_regression.py", line 37, in <module>
    w = X_train.t().mm(X_train).inverse().mm(X_train.t()).mm(y_train)
RuntimeError: "linalg_inv_out_cpu" not implemented for 'Long'

具体就是调用Inverse函数时，前面的矩阵不可逆😄

[mousecpn]

代码如下：

#! python3
# -*- encoding: utf-8 -*-
"""
Created on Mon Sep  26 08:02:10 2022

@author: eanson
"""
import pandas as pd
import torch
import math

import os

# 训练集比例
train_proportion = 0.7

#####################################################################
data_file = os.path.join('HW1 linear model', 'data1.txt')
data = pd.read_csv(data_file, header=None, names=[
                   "Room Size", "Room Amount", "Price"])
# 训练集按照比例划分
m = math.floor(len(data)*train_proportion)
X_train_pd, y_train_pd, X_cv_pd, y_cv_pd = data.iloc[:m, 0:2], data.iloc[:m,
                                                                         2], data.iloc[m+1:, 0:2], data.iloc[m+1:, 2]
# 转换为张量格式
X_train, y_train = torch.tensor(
    X_train_pd.values), torch.tensor(y_train_pd.values)
X_cv, y_cv = torch.tensor(X_cv_pd.values), torch.tensor(y_cv_pd.values)

############################################################################
# 正规方程
"""
$$
w=(X^TX)^{-1}X^Ty
$$
"""
w = X_train.t().mm(X_train).inverse().mm(X_train.t()).mm(y_train)

报错:

(d2l) yanshengdeMacBook-Pro:MachineLearning_HW_CQUT eanson$  cd /Users/eanson/Documents/ml/MachineLearning_HW_CQUT ; /usr/bin/env /Users/eanson/opt/miniconda3/envs/d2l/bin/python /Users/eanson/.vscode/extensions/ms-python.python-2021.2.633441544/pythonFiles/lib/python/debugpy/launcher 59765 -- /Users/eanson/Documents/ml/MachineLearning_HW_CQUT/linear_regression.py 
Traceback (most recent call last):
  File "/Users/eanson/Documents/ml/MachineLearning_HW_CQUT/linear_regression.py", line 37, in <module>
    w = X_train.t().mm(X_train).inverse().mm(X_train.t()).mm(y_train)
RuntimeError: "linalg_inv_out_cpu" not implemented for 'Long'

具体就是调用Inverse函数时，前面的矩阵不可逆😄

看报错好像并不是矩阵不可逆，而是inverse这个函数不能对Long型变量操作。
可以先把数据转成float型变量

[eanson023]

还真是，我改了之后能跑起来了,谢谢老师

[eanson023]

老师，我这个loss虽然下降了，但是太大了，您能帮我看下具体哪出了问题吗?

输出：

iter:9991,loss:2224352768.0
iter:9992,loss:2224352768.0
iter:9993,loss:2224352768.0
iter:9994,loss:2224352768.0
iter:9995,loss:2224352768.0
iter:9996,loss:2224352768.0
iter:9997,loss:2224352768.0
iter:9998,loss:2224352768.0
iter:9999,loss:2224352768.0
iter:10000,loss:2224352768.0

loss图:

代码如下:

#! python3
# -*- encoding: utf-8 -*-
"""
Created on Mon Sep  26 08:02:10 2022

@author: eanson
"""
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import torch
import math

import os

# 训练集比例
train_proportion = 0.7
figure_idx = 1

############################################################################

# 读取数据集
data_file = os.path.join('HW1 linear model', 'data1.txt')
data = pd.read_csv(data_file, header=None, names=[
                   "Room Size", "Room Amount", "Price"])
# 训练集按照比例划分
m = math.floor(len(data)*train_proportion)
X_train_pd, y_train_pd, X_cv_pd, y_cv_pd = data.iloc[:m, 0:2], data.iloc[:m,
                                                                         2], data.iloc[m+1:, 0:2], data.iloc[m+1:, 2]
# 转换为张量格式 引入特征 x0=1 （方便之后向量化）
X_train, y_train = torch.cat([torch.ones(m, 1), torch.tensor(
    X_train_pd.values, dtype=torch.float)], dim=1), torch.tensor(y_train_pd.values, dtype=torch.float).reshape(-1, 1)
X_cv, y_cv = torch.cat([torch.ones(len(X_cv_pd), 1), torch.tensor(
    X_cv_pd.values, dtype=torch.float)], dim=1), torch.tensor(y_cv_pd.values, dtype=torch.float).reshape(-1, 1)

# 可视化
def visualize(X, y, w, title_suffix):
    global figure_idx
    # 计算验证集预测值
    y_pred = torch.mm(X, w)
    x_points = X[:, 1].numpy()
    y_points = y.detach().numpy()
    y_pred_points = y_pred.detach().numpy()

    plt.figure(figure_idx)
    figure_idx += 1
    plt.title("house price prediction "+title_suffix)
    plt.xlabel("room size(m²)")
    plt.ylabel("house price($)")
    plt.plot(x_points, y_points, 'o', c='b')
    plt.plot(x_points, y_pred_points, 'o', c='r')
    plt.show()


############################################################################
# 梯度下降法
# feature scaling 将房间大小进行特征缩放
"""
$$
x_{n} = \frac{x_{n}-u_{n}}{s_{n}} （其中u_{n}是平均值，s_{n}是标准差）
$$
"""
X_train[:, 1] = (X_train[:, 1]-X_train[:, 1].mean())/X_train[:, 1].std()

# 迭代次数
iterations = 10000
# 学习率
lr = 0.1
# 初始化theta
w = torch.rand((X_train.size(1), 1))  # size: feature_size+1 x 1


def loss_function(y_pred, y):
    m = len(y_pred)
    return 1/(2*m)*torch.sum((y_pred-y)**2)


log = []
for i in range(iterations):
    # 假设函数
    hypothesis = torch.mm(X_train, w)
    # 向量化
    grad = lr*torch.mean((hypothesis-y_train)*X_train, dim=0).reshape(-1, 1)
    # 梯度下降
    w = w - grad
    loss = loss_function(hypothesis, y_train)
    print(f'iter:{i+1},loss:{loss}')
    log.append([i+1, loss])

# 画下loss
log = np.array(log)
plt.figure(figure_idx)
figure_idx += 1
plt.plot(log[:, 0], log[:, 1])
plt.show()

visualize(X_cv, y_cv, w, 'on cross validation data with gradient descent')

[mousecpn]

老师，我这个loss虽然下降了，但是太大了，您能帮我看下具体哪出了问题吗?

输出：

iter:9991,loss:2224352768.0
iter:9992,loss:2224352768.0
iter:9993,loss:2224352768.0
iter:9994,loss:2224352768.0
iter:9995,loss:2224352768.0
iter:9996,loss:2224352768.0
iter:9997,loss:2224352768.0
iter:9998,loss:2224352768.0
iter:9999,loss:2224352768.0
iter:10000,loss:2224352768.0

loss图:

代码如下:

#! python3
# -*- encoding: utf-8 -*-
"""
Created on Mon Sep  26 08:02:10 2022

@author: eanson
"""
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import torch
import math

import os

# 训练集比例
train_proportion = 0.7
figure_idx = 1

############################################################################

# 读取数据集
data_file = os.path.join('HW1 linear model', 'data1.txt')
data = pd.read_csv(data_file, header=None, names=[
                   "Room Size", "Room Amount", "Price"])
# 训练集按照比例划分
m = math.floor(len(data)*train_proportion)
X_train_pd, y_train_pd, X_cv_pd, y_cv_pd = data.iloc[:m, 0:2], data.iloc[:m,
                                                                         2], data.iloc[m+1:, 0:2], data.iloc[m+1:, 2]
# 转换为张量格式 引入特征 x0=1 （方便之后向量化）
X_train, y_train = torch.cat([torch.ones(m, 1), torch.tensor(
    X_train_pd.values, dtype=torch.float)], dim=1), torch.tensor(y_train_pd.values, dtype=torch.float).reshape(-1, 1)
X_cv, y_cv = torch.cat([torch.ones(len(X_cv_pd), 1), torch.tensor(
    X_cv_pd.values, dtype=torch.float)], dim=1), torch.tensor(y_cv_pd.values, dtype=torch.float).reshape(-1, 1)

# 可视化
def visualize(X, y, w, title_suffix):
    global figure_idx
    # 计算验证集预测值
    y_pred = torch.mm(X, w)
    x_points = X[:, 1].numpy()
    y_points = y.detach().numpy()
    y_pred_points = y_pred.detach().numpy()

    plt.figure(figure_idx)
    figure_idx += 1
    plt.title("house price prediction "+title_suffix)
    plt.xlabel("room size(m²)")
    plt.ylabel("house price($)")
    plt.plot(x_points, y_points, 'o', c='b')
    plt.plot(x_points, y_pred_points, 'o', c='r')
    plt.show()


############################################################################
# 梯度下降法
# feature scaling 将房间大小进行特征缩放
"""
$$
x_{n} = \frac{x_{n}-u_{n}}{s_{n}} （其中u_{n}是平均值，s_{n}是标准差）
$$
"""
X_train[:, 1] = (X_train[:, 1]-X_train[:, 1].mean())/X_train[:, 1].std()

# 迭代次数
iterations = 10000
# 学习率
lr = 0.1
# 初始化theta
w = torch.rand((X_train.size(1), 1))  # size: feature_size+1 x 1


def loss_function(y_pred, y):
    m = len(y_pred)
    return 1/(2*m)*torch.sum((y_pred-y)**2)


log = []
for i in range(iterations):
    # 假设函数
    hypothesis = torch.mm(X_train, w)
    # 向量化
    grad = lr*torch.mean((hypothesis-y_train)*X_train, dim=0).reshape(-1, 1)
    # 梯度下降
    w = w - grad
    loss = loss_function(hypothesis, y_train)
    print(f'iter:{i+1},loss:{loss}')
    log.append([i+1, loss])

# 画下loss
log = np.array(log)
plt.figure(figure_idx)
figure_idx += 1
plt.plot(log[:, 0], log[:, 1])
plt.show()

visualize(X_cv, y_cv, w, 'on cross validation data with gradient descent')

没做归一化