import matplotlib as mpl
import matplotlib.pyplot as plt
%matplotlib inline
import numpy as np
import sklearn
import pandas as pd
import os
import sys
import time
from tqdm.auto import tqdm
import torch
import torch.nn as nn
import torch.nn.functional as F

print(sys.version_info)
for module in mpl, np, pd, sklearn, torch:
    print(module.__name__, module.__version__)
    
device = torch.device("cuda:0") if torch.cuda.is_available() else torch.device("cpu")
print(device)

这里使用sklearn提供的加利福尼亚房价的数据集：

from sklearn.datasets import fetch_california_housing

housing = fetch_california_housing(data_home='data')

划分成三个：

from sklearn.model_selection import train_test_split

#拆分训练集和测试集，random_state是随机种子,同样的随机数种子，是为了得到同样的随机值
x_train_all, x_test, y_train_all, y_test = train_test_split(
    housing.data, housing.target, random_state = 7)
x_train, x_valid, y_train, y_valid = train_test_split(
    x_train_all, y_train_all, random_state = 11)
# 训练集
print(x_train.shape, y_train.shape)
# 验证集
print(x_valid.shape, y_valid.shape)
# 测试集
print(x_test.shape, y_test.shape)

dataset_maps = {
    "train": [x_train, y_train], #训练集
    "valid": [x_valid, y_valid], #验证集
    "test": [x_test, y_test], #测试集
} #把3个数据集都放到字典中

标准化：

from sklearn.preprocessing import StandardScaler



scaler = StandardScaler() #标准化
scaler.fit(x_train) #fit和fit_transform的区别，fit是计算均值和方差，fit_transform是先fit，然后transform

构建数据集：

#构建私有数据集，就需要用如下方式
from torch.utils.data import Dataset

class HousingDataset(Dataset):
    def __init__(self, mode='train'):
        self.x, self.y = dataset_maps[mode] #x,y都是ndarray类型
        self.x = torch.from_numpy(scaler.transform(self.x)).float() #from_numpy将NumPy数组转换成PyTorch张量
        self.y = torch.from_numpy(self.y).float().reshape(-1, 1) #处理为多行1列的tensor类型,因为__getitem__切片时需要
            
    def __len__(self): #必须写
        return len(self.x) #返回数据集的长度,0维的size
    
    def __getitem__(self, idx): #idx是索引，返回的是数据和标签，这里是一个样本
        return self.x[idx], self.y[idx]
    
#train_ds是dataset类型的数据，与前面例子的FashionMNIST类型一致
train_ds = HousingDataset("train")
valid_ds = HousingDataset("valid")
test_ds = HousingDataset("test")

至于为什么要把y变成二维的，是因为：

原本y是那么多行，转换成张量后，就变成一维的了，一行多列
需要将标签数据转换为张量，并调整形状以适应模型训练的要求，.float的作用在于如果原始标签是整数，那么需要显式转换为浮点数

然后是DataLoader

from torch.utils.data import DataLoader

#放到DataLoader中的train_ds, valid_ds, test_ds都是dataset类型的数据
batch_size = 16 #batch_size是可以调的超参数
train_loader = DataLoader(train_ds, batch_size=batch_size, shuffle=True)
val_loader = DataLoader(valid_ds, batch_size=batch_size, shuffle=False)
test_loader = DataLoader(test_ds, batch_size=batch_size, shuffle=False)

验证和测试就不随机了