House Prices, End to End

库与画图主题：import / rcParams

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from scipy import stats
import glob, matplotlib
from matplotlib import font_manager as fm
for f in glob.glob(r'F:\2026fall_skd\练习\画图字体\NotoSerifSC-*.ttf'):
    fm.fontManager.addfont(f)              # 本机字体；别的电脑没有这个文件夹，自动跳过
have = {f.name for f in fm.fontManager.ttflist}

# ---- WYX Only：粉蓝马卡龙 ----
CN = [f for f in ['Noto Serif SC', 'Source Han Serif SC', 'Songti SC', 'STSong', 'SimSun', 'PingFang SC', 'Microsoft YaHei', 'Noto Sans CJK SC', 'WenQuanYi Zen Hei'] if f in have]   # 宋体优先，没有再退到黑体
MAIN, MAIN_D, MAIN_L = '#a9c8ea', '#6f93c6', '#c9dcf2'   # 主色：浅蓝 / 深蓝 / 很浅的蓝
ACCENT, ACCENT_D = '#f4b6ca', '#e0819f'                 # 强调色：浅粉 / 深粉
GRAY, BG = '#a89dab', '#fffbfc'                         # 注释文字 / 带粉的白底
CMAP = sns.blend_palette([ACCENT_D, '#fbe1ea', BG, MAIN_L, '#86abdb', '#4f7fb8', '#2d4a78'], as_cmap=True)  # 负粉 → 白 → 正蓝
sns.set_theme(style='white')
plt.rcParams.update({
    'font.family': CN + ['DejaVu Sans'], 'axes.unicode_minus': False,
    'figure.facecolor': BG, 'axes.facecolor': BG, 'savefig.facecolor': BG,
    'figure.dpi': 110, 'figure.constrained_layout.use': True,
    'axes.spines.top': False, 'axes.spines.right': False,
    'axes.edgecolor': '#e3d6dc', 'axes.linewidth': 0.9,
    'axes.grid': True, 'axes.grid.axis': 'y', 'axes.axisbelow': True,
    'grid.color': '#f3e9ee', 'grid.linewidth': 0.8,
    'axes.titlesize': 14, 'axes.titleweight': 'bold', 'axes.titlepad': 12, 'axes.titlecolor': '#4a4150',
    'axes.labelsize': 11, 'axes.labelcolor': '#7a6f80',
    'xtick.color': GRAY, 'ytick.color': GRAY, 'xtick.labelsize': 10, 'ytick.labelsize': 10,
    'xtick.major.size': 0, 'ytick.major.size': 0,
    'legend.frameon': False, 'legend.fontsize': 10, 'legend.labelcolor': '#6b6070',
    'axes.prop_cycle': matplotlib.cycler(color=[MAIN, ACCENT, MAIN_D, ACCENT_D]),
})

建模用的库：sklearn

from sklearn.model_selection import train_test_split
from sklearn.preprocessing import MinMaxScaler
from sklearn.linear_model import LinearRegression, Ridge
from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score
from IPython.display import display                  # 表格显示得好看；print 也行
pd.set_option('display.max_columns', 12)
pd.set_option('display.float_format', lambda x: f'{x:,.3f}')
SEED = 42

统一列名：rename

COLUMN_MAP = {
    'Id': 'id', 'SalePrice': 'price', 'GrLivArea': 'area', 'Neighborhood': 'neighborhood',
    'OverallQual': 'quality', 'OverallCond': 'condition', 'LotArea': 'lot_area',
    'LotFrontage': 'lot_frontage', 'YearBuilt': 'year_built', 'YearRemodAdd': 'year_remodel',
    'YrSold': 'sale_year', 'MoSold': 'sale_month', 'TotRmsAbvGrd': 'rooms',
    'TotalBsmtSF': 'basement_area', 'GarageCars': 'garage_cars', 'PoolArea': 'pool_area',
    'MasVnrArea': 'veneer_area', 'CentralAir': 'central_air', 'KitchenQual': 'kitchen_quality',
}

读入数据：read_csvtricky

read_options = dict(keep_default_na=False, na_values=['', '?', 'null', 'NaN', 'nan'])
raw = pd.read_csv('house.csv', **read_options)
print(raw.shape)
raw.iloc[:3, :8]

Output(1460, 81)
   Id  MSSubClass MSZoning  LotFrontage  LotArea Street Alley LotShape
0   1          60       RL       65.000     8450   Pave   NaN      Reg
1   2          20       RL       80.000     9600   Pave   NaN      Reg
2   3          60       RL       68.000    11250   Pave   NaN      IR1

缺失、类型与重复：isna / dtypes / duplicated

print(raw.dtypes.astype(str).value_counts().to_dict())   # 各类型各有几列
missing = pd.DataFrame({
    'missing_n': raw.isna().sum(),
    'missing_rate': raw.isna().mean(),                 # 0–1 的比例，要百分数再乘 100
}).sort_values('missing_n', ascending=False)
display(missing[missing['missing_n'] > 0].head(8))
print('完全重复的行：', raw.duplicated().sum())
# raw.info() 也能看，但它只打印、返回 None

Output{'object': 43, 'int64': 35, 'float64': 3}
             missing_n  missing_rate
PoolQC            1453         0.995
MiscFeature       1406         0.963
Alley             1369         0.938
Fence             1179         0.808
MasVnrType         872         0.597
FireplaceQu        690         0.473
LotFrontage        259         0.177
GarageYrBlt         81         0.055
完全重复的行： 0

规范列名与类型：rename / to_numeric

clean = raw.copy()                                     # raw 留着不动
clean.columns = clean.columns.str.strip()
clean = clean.rename(columns=COLUMN_MAP)

# 文本列：去首尾空格，'' 和 '?' 统一成缺失
for col in clean.select_dtypes(include='object').columns:
    clean[col] = clean[col].str.strip().replace({'': np.nan, '?': np.nan})

# 应该是数字的列：去掉 $ 和逗号再转，转不了的变 NaN
num_cols = ['price', 'area', 'quality', 'condition', 'lot_area', 'lot_frontage', 'year_built',
            'year_remodel', 'sale_year', 'sale_month', 'rooms', 'basement_area', 'garage_cars',
            'pool_area', 'veneer_area']
for col in num_cols:
    text = clean[col].astype(str).str.replace(r'[$,]', '', regex=True)
    clean[col] = pd.to_numeric(text, errors='coerce')
clean[num_cols].dtypes.value_counts()

Outputint64      13
float64     2

删无效目标、标出不合理值：loc

n0 = len(clean)
clean = clean.drop_duplicates()                        # 确认是重复导入才删；本例 0 条
clean = clean[clean['price'].notna() & (clean['price'] > 0)].copy()
clean.loc[clean['area'] <= 0, 'area'] = np.nan
for col in ['lot_area', 'rooms', 'garage_cars', 'basement_area']:
    clean.loc[clean[col] < 0, col] = np.nan
clean.loc[~clean['quality'].between(1, 10), 'quality'] = np.nan
print(n0, '→', len(clean))

Output1460 → 1460

NA 是「没有」还是「不知道」：isna & eq(0)tricky

facilities = {'PoolQC': 'pool_area', 'GarageType': 'garage_cars',
              'BsmtQual': 'basement_area', 'MasVnrType': 'veneer_area'}   # 文字列 : 对应的数量列
for text_col, amount_col in facilities.items():
    clean[text_col] = clean[text_col].replace({'NA': np.nan, 'None': 'Absent'})
    absent = clean[text_col].isna() & clean[amount_col].eq(0)
    clean.loc[absent, text_col] = 'Absent'
clean[['pool_area', 'PoolQC', 'veneer_area', 'MasVnrType']].head(3)

Output   pool_area  PoolQC  veneer_area MasVnrType
0          0  Absent      196.000    BrkFace
1          0  Absent        0.000     Absent
2          0  Absent      162.000    BrkFace

派生房龄：列相减

clean['house_age'] = clean['sale_year'] - clean['year_built']
clean['remodel_age'] = clean['sale_year'] - clean['year_remodel']
clean.loc[clean['house_age'] < 0, 'house_age'] = np.nan     # 负数是记录问题，改成未知
clean['house_age'].describe()[['min', 'mean', 'max']]

Outputmin      0.000
mean    36.548
max    136.000

训练 / 验证 / 测试：train_test_splittricky

trainval, test_data = train_test_split(clean, test_size=0.2, random_state=SEED)
train_data, val_data = train_test_split(trainval, test_size=0.25, random_state=SEED)   # 0.8 × 0.25 = 0.2
df = train_data.copy()                                 # 后面 EDA 只用训练部分
print(len(train_data), len(val_data), len(test_data))
# 只做 EDA、不建模时：df = clean.copy() 就行

Output876 292 292

组内中位数填补：groupby + transform

fill_demo = df.copy()
group_median = df.groupby('neighborhood')['lot_frontage'].transform('median')   # 每行 = 自己街区的中位数
fill_demo['lot_frontage'] = (df['lot_frontage'].fillna(group_median)
                                               .fillna(df['lot_frontage'].median()))   # 整个街区都缺，再用总体
print(df['lot_frontage'].isna().sum(), '→', fill_demo['lot_frontage'].isna().sum())

Output163 → 0

IQR 标记异常值：quantile

rows = []
for col in ['price', 'area']:
    q1, q3 = df[col].quantile([0.25, 0.75])
    low, high = q1 - 1.5 * (q3 - q1), q3 + 1.5 * (q3 - q1)
    flag = (df[col] < low) | (df[col] > high)
    df[f'{col}_iqr_flag'] = flag
    rows.append({'column': col, 'lower': low, 'upper': high, 'flag_n': flag.sum()})
display(pd.DataFrame(rows))
df.loc[df['area_iqr_flag'], ['id', 'area', 'price', 'quality', 'neighborhood']].nlargest(3, 'area')
# 面积最大的两套才卖 16–18 万：Edwards 的特殊交易，是要重点看的离群点

Output  column     lower       upper  flag_n
0  price 7,593.750 336,343.750      37
1   area   199.250   2,745.250      19
        id      area   price  quality neighborhood
1298  1299 5,642.000  160000   10.000      Edwards
523    524 4,676.000  184750   10.000      Edwards
1182  1183 4,476.000  745000   10.000      NoRidge

条件筛选与排序：loc / sort_values

mask = (df['area'] >= 1500) & (df['price'] < 200000)
cols = ['id', 'price', 'area', 'quality', 'neighborhood']
display(df.loc[mask, cols].sort_values('price').head(3))
print('符合条件', mask.sum(), '套，占', f'{mask.mean():.1%}')
# df.query('area >= 1500 and price < 200000')       同一个筛选
# df[df['neighborhood'].isin(['NAmes', 'OldTown'])]  属于某几类；前面加 ~ 是不属于
# df[df['price'].between(100000, 200000)]           闭区间

Output        id  price      area  quality neighborhood
88      89  85000 1,526.000    3.000       IDOTRR
1186  1187  95000 1,699.000    3.000      OldTown
749    750  98000 1,529.000    4.000      Edwards
符合条件 180 套，占 20.5%

取最大的几条：nlargest / idxmax / argsorttricky

display(df.nlargest(3, 'price')[cols])
print(df['price'].idxmax())                            # index 标签
top = np.argsort(-df['price'].to_numpy())[:3]          # 位置
df.iloc[top][cols]

Output        id   price      area  quality neighborhood
1182  1183  745000 4,476.000   10.000      NoRidge
803    804  582933 2,822.000    9.000      NridgHt
440    441  555000 2,402.000   10.000      NridgHt
1182
        id   price      area  quality neighborhood
1182  1183  745000 4,476.000   10.000      NoRidge
803    804  582933 2,822.000    9.000      NridgHt
440    441  555000 2,402.000   10.000      NridgHt

按条件改值：loc 赋值

demo = df[['price']].copy()
demo.loc[mask, 'scenario_price'] = demo.loc[mask, 'price'] * 0.95    # 一次 loc 写完
demo[mask].head(3)
# 别写 df[mask]['price'] = ...：链式赋值，改不到原表

Output      price  scenario_price
463  188700     179,265.000
657  149000     141,550.000
248  180000     171,000.000

分组汇总：groupby + agg

g = (df.groupby('neighborhood')
       .agg(n=('price', 'size'), mean_price=('price', 'mean'),
            median_price=('price', 'median'), mean_area=('area', 'mean'))
       .sort_values('median_price', ascending=False))
display(g.head(5))
big = g[g['n'] >= 15]                                  # n ≥ 15 的街区
rich = big[big['mean_price'] > df['price'].mean()]
print('n≥15 的街区', len(big), '个；其中均价高于全体的', len(rich), '个')

Output               n  mean_price  median_price  mean_area
neighborhood                                         
StoneBr       16 302,423.562   298,500.000  1,867.750
NoRidge       26 310,781.462   290,000.000  2,381.500
NridgHt       48 309,658.896   283,000.000  1,884.500
Somerst       43 230,791.395   227,000.000  1,580.465
Timber        26 233,917.308   222,250.000  1,707.154
n≥15 的街区 19 个；其中均价高于全体的 10 个

每行减去组均值：transform

group_mean = df.groupby('neighborhood')['price'].transform('mean')
df[['neighborhood', 'price']].assign(group_mean=group_mean, diff=df['price'] - group_mean).head(3)
# agg → 每组一行；transform → 和原表一样长；filter → 留下满足条件的组的全部行

Output     neighborhood   price  group_mean        diff
683       CollgCr  285000 201,290.430  83,709.570
1077        NAmes  138800 147,220.269  -8,420.269
463       Crawfor  188700 212,734.286 -24,034.286

对数与单价：np.log

df['log_price'] = np.log(df['price'])                  # 有 0 用 np.log1p；还原用 np.exp / np.expm1
df['price_per_area'] = df['price'] / df['area']
print('偏度', round(df['price'].skew(), 2), '→', round(df['log_price'].skew(), 2))
print('等权平均单价', round(df['price_per_area'].mean(), 1),
      '；面积加权', round(df['price'].sum() / df['area'].sum(), 1))

Output偏度 1.8 → 0.1
等权平均单价 119.7 ；面积加权 118.3

分档：cut / qcuttricky

df['quality_band'] = pd.cut(df['quality'], bins=[0, 5, 7, 10],
                            labels=['低 1-5', '中 6-7', '高 8-10'])   # 右闭：(0,5] (5,7] (7,10]
df['price_quartile'] = pd.qcut(df['price'], q=4)       # 按分位数切，每档套数差不多
print(df['quality_band'].value_counts(sort=False).to_dict())
# ⚠ log_price、price_per_area、price_quartile 都含答案，不能放进 X

Output{'低 1-5': 310, '中 6-7': 431, '高 8-10': 135}

编码与条件列：map / np.where / get_dummies

order = {'Po': 1, 'Fa': 2, 'TA': 3, 'Gd': 4, 'Ex': 5}
df['kitchen_score'] = df['kitchen_quality'].map(order)       # 有顺序的类别；对不上的会变 NaN
df['big_house'] = np.where(df['area'] >= 1500, 1, 0)
pd.get_dummies(df['neighborhood'], prefix='nb', dtype=int).iloc[:3, :4]   # 无顺序的类别：独热

Output      nb_Blmngtn  nb_BrDale  nb_BrkSide  nb_ClearCr
683            0          0           0           0
1077           0          0           0           0
463            0          0           0           0

按年汇总：to_datetime / resampleoptional

df['sale_date'] = pd.to_datetime(dict(year=df['sale_year'], month=df['sale_month'], day=1))
df.set_index('sale_date').resample('YS')['price'].agg(['size', 'median'])
# 只有年月，day=1 是凑的；2010 年只记到 7 月，套数少不代表市场冷

Output            size      median
sale_date                   
2006-01-01   193 164,000.000
2007-01-01   193 165,000.000
2008-01-01   177 165,500.000
2009-01-01   211 163,500.000
2010-01-01   102 166,750.000

目标变量分布：histplot / boxplot

fig, axes = plt.subplots(1, 3, figsize=(14, 3.8))
sns.histplot(data=df, x='price', bins=35, color=MAIN, ax=axes[0])
axes[0].axvline(df['price'].median(), color=ACCENT_D, ls='--', label='中位数')
axes[0].set(title='房价分布', xlabel='成交价（美元）', ylabel='套数')
axes[0].legend()
sns.histplot(data=df, x='log_price', bins=30, color=ACCENT, ax=axes[1])
axes[1].set(title='取对数后', xlabel='ln(价格)', ylabel='套数')
sns.boxplot(data=df, x='price', color=MAIN_L, ax=axes[2])
axes[2].set(title='箱线图：看上尾', xlabel='成交价（美元）')
plt.show()

数值 × 数值：scatterplot / regplot

fig, axes = plt.subplots(1, 2, figsize=(12, 4.5))
sns.scatterplot(data=df, x='area', y='price', hue='quality_band', palette=[MAIN, ACCENT_D, MAIN_D],
                s=24, alpha=.6, edgecolor=None, ax=axes[0])
axes[0].set(title='面积与房价', xlabel='地上居住面积（平方英尺）', ylabel='成交价（美元）')
sns.regplot(x=np.log(df['area']), y=df['log_price'], ci=None, ax=axes[1],
            scatter_kws={'s': 14, 'alpha': .35, 'color': MAIN}, line_kws={'color': ACCENT_D})
axes[1].set(title='取对数后接近直线', xlabel='ln(面积)', ylabel='ln(价格)')
plt.show()

各组统计量：barh / countplot

top = big.head(10).sort_values('median_price')         # big：上面 n≥15 的街区
fig, axes = plt.subplots(1, 2, figsize=(12, 4.6))
bars = axes[0].barh(top.index, top['median_price'], color=MAIN)
axes[0].bar_label(bars, labels=[f'n={n}' for n in top['n']], padding=4, fontsize=9, color=GRAY)
axes[0].set_xlim(0, top['median_price'].max() * 1.2)
axes[0].set(title='中位价最高的 10 个街区', xlabel='中位价（美元）')
order = df['neighborhood'].value_counts().head(10).index
sns.countplot(data=df, y='neighborhood', order=order, color=ACCENT, ax=axes[1])
axes[1].set(title='成交最多的 10 个街区', xlabel='套数', ylabel='')
plt.show()

类别 × 数值：barplot + boxplot

fig, axes = plt.subplots(1, 2, figsize=(11, 4))
sns.barplot(data=df, x='quality_band', y='price', estimator='median', errorbar=None, color=MAIN, ax=axes[0])
axes[0].set(title='各质量档中位价', xlabel='质量档', ylabel='中位价（美元）')
sns.boxplot(data=df, x='quality_band', y='price', color=ACCENT, fliersize=2, ax=axes[1])
axes[1].set(title='组内差异', xlabel='质量档', ylabel='成交价（美元）')
plt.show()
df.groupby('quality_band', observed=True)['price'].agg(['size', 'mean', 'median'])

Output              size        mean      median
quality_band                              
低 1-5          310 126,976.945 128,000.000
中 6-7          431 182,242.898 179,000.000
高 8-10         135 302,603.067 285,000.000

相关系数热力图：corr + heatmap

corr_cols = ['price', 'area', 'quality', 'lot_area', 'house_age', 'garage_cars', 'rooms', 'basement_area']
pearson = df[corr_cols].corr()
spearman = df[corr_cols].corr(method='spearman')
display(pd.DataFrame({'Pearson': pearson['price'], 'Spearman': spearman['price']})
          .drop('price').sort_values('Pearson', key=abs, ascending=False))
fig, ax = plt.subplots(figsize=(8, 6))
tri = np.triu(np.ones_like(pearson, dtype=bool), k=1)  # 只画下三角
sns.heatmap(pearson, mask=tri, annot=True, fmt='.2f', cmap=CMAP, center=0, vmin=-1, vmax=1,
            square=True, linewidths=.5, ax=ax)
ax.set_title('数值变量相关系数')
plt.show()

Output               Pearson  Spearman
quality          0.779     0.795
area             0.684     0.721
garage_cars      0.644     0.688
basement_area    0.588     0.599
rooms            0.531     0.527
house_age       -0.513    -0.634
lot_area         0.244     0.479

透视表热力图：pivot_table + heatmap

common = df['neighborhood'].value_counts().head(10).index
sub = df[df['neighborhood'].isin(common)]
price_pv = sub.pivot_table(index='neighborhood', columns='quality_band', values='price',
                           aggfunc='median', observed=True)
count_pv = sub.pivot_table(index='neighborhood', columns='quality_band', values='price',
                           aggfunc='count', observed=True).reindex_like(price_pv).fillna(0)
fig, ax = plt.subplots(figsize=(8, 5.5))
sns.heatmap(price_pv.where(count_pv >= 5) / 1000, annot=True, fmt='.0f',
            cmap=sns.light_palette(MAIN_D, as_cmap=True), linewidths=.5, ax=ax)
ax.set(title='街区 × 质量档 中位价（千美元）', xlabel='质量档', ylabel='')
plt.show()

交叉表比例热力图：crosstabtricky

display(pd.crosstab(df['central_air'], df['price_quartile']))
share = pd.crosstab(df['central_air'], df['price_quartile'], normalize='index')
fig, ax = plt.subplots(figsize=(9, 3.2))
sns.heatmap(share, annot=True, fmt='.1%', cmap=sns.light_palette(MAIN_D, as_cmap=True), vmin=0, vmax=1, ax=ax)
ax.set(title='有无中央空调：各价位占比', xlabel='价格四分位', ylabel='中央空调')
plt.show()

Outputprice_quartile  (34899.999, 130875.0]  (130875.0, 164945.0]  (164945.0, 213062.5]  (213062.5, 745000.0]
central_air                                                                                            
N                                  50                    11                     1                     3
Y                                 169                   208                   218                   216

把输出写成结论：f-string

r = df['area'].corr(df['price'])
best = big.index[0]
print(f"范围：训练部分 {len(df)} 套，IQR 标出的极端值保留。")
print(f"证据：中位价 {df['price'].median():,.0f} 美元，均价 {df['price'].mean():,.0f}，偏度 {df['price'].skew():.2f}；"
      f"面积与房价 r = {r:.3f}（见散点图）。")
print(f"解释：n≥15 的街区里 {best} 中位价最高（{big.loc[best, 'median_price']:,.0f}，n={big.loc[best, 'n']}），"
      "可能与面积、质量构成有关。")
print("限制：相关不等于因果；街区差异可能来自质量和面积不同，需要分层或回归。")

Output范围：训练部分 876 套，IQR 标出的极端值保留。
证据：中位价 164,945 美元，均价 181,234，偏度 1.80；面积与房价 r = 0.684（见散点图）。
解释：n≥15 的街区里 StoneBr 中位价最高（298,500，n=16），可能与面积、质量构成有关。
限制：相关不等于因果；街区差异可能来自质量和面积不同，需要分层或回归。

特征、填补、拟合：fit / predicttricky

FEATURES = ['area', 'quality', 'house_age']           # 先少量、能解释的数值列
X_train, X_val, X_test = train_data[FEATURES], val_data[FEATURES], test_data[FEATURES]
y_train, y_val, y_test = train_data['price'], val_data['price'], test_data['price']

med = X_train.median()
X_train, X_val, X_test = X_train.fillna(med), X_val.fillna(med), X_test.fillna(med)

lr = LinearRegression()
lr.fit(X_train, y_train)
pred_val = lr.predict(X_val)
print('截距', round(lr.intercept_, 1))
pd.Series(lr.coef_, index=FEATURES).round(1)
# area 系数：其他变量不变时，面积多 1 平方英尺，预测价高多少美元（相关，不是因果）

Output截距 -35380.7
area            57.600
quality     24,252.000
house_age     -531.000

误差指标与基线：MAE / RMSE / R²

def scores(y, pred):
    return {'MAE': mean_absolute_error(y, pred),
            'RMSE': np.sqrt(mean_squared_error(y, pred)),
            'R2': r2_score(y, pred)}

baseline = np.full(len(y_val), y_train.mean())         # 只猜训练均价：最低标准
pd.DataFrame({'线性回归': scores(y_val, pred_val), '均价基线': scores(y_val, baseline)}).T
# MAE：平均差多少美元；RMSE：更怕大误差；R² 可以为负，不是「准确率」

Output            MAE       RMSE     R2
线性回归 26,005.295 37,513.660  0.765
均价基线 57,793.599 77,389.346 -0.000

归一化：MinMaxScalertricky

scaler = MinMaxScaler()
X_train_s = scaler.fit_transform(X_train)
X_val_s = scaler.transform(X_val)
X_test_s = scaler.transform(X_test)
X_train_s.min(axis=0), X_train_s.max(axis=0)

Output(array([0., 0., 0.]), array([1., 1., 1.]))

验证集上比较：for 循环 + Ridge

models = {'线性回归': LinearRegression(), 'Ridge α=0.1': Ridge(alpha=0.1),
          'Ridge α=1': Ridge(alpha=1), 'Ridge α=10': Ridge(alpha=10)}
rows = []
for name, m in models.items():
    m.fit(X_train_s, y_train)
    row = scores(y_val, m.predict(X_val_s))
    row['train_RMSE'] = np.sqrt(mean_squared_error(y_train, m.predict(X_train_s)))
    rows.append(pd.Series(row, name=name))
comparison = pd.DataFrame(rows).sort_values('RMSE')
display(comparison)
best_name = comparison.index[0]
best_model = models[best_name]
print('按验证 RMSE 选：', best_name)
# 训练误差小、验证误差大 → 过拟合；两个都大 → 欠拟合

Output                   MAE       RMSE    R2  train_RMSE
线性回归        26,005.295 37,513.660 0.765  40,985.489
Ridge α=0.1 25,988.191 37,566.071 0.764  40,986.970
Ridge α=1   25,995.464 38,093.476 0.758  41,104.837
Ridge α=10  29,260.260 43,787.730 0.680  44,912.479
按验证 RMSE 选： 线性回归

测试集评价：predict

test_pred = best_model.predict(X_test_s)
test_results = pd.DataFrame({'选中模型': scores(y_test, test_pred),
                             '均价基线': scores(y_test, np.full(len(y_test), y_train.mean()))}).T
test_results
# 看完测试分数不再回头调参数

Output            MAE       RMSE     R2
选中模型 28,022.460 43,411.052  0.754
均价基线 62,516.213 87,613.117 -0.001

真实 vs 预测、残差图：scatter

resid = y_test.to_numpy() - test_pred
fig, axes = plt.subplots(1, 2, figsize=(12, 4.5))
axes[0].scatter(y_test, test_pred, s=22, alpha=.6, color=MAIN)
lo, hi = min(y_test.min(), test_pred.min()), max(y_test.max(), test_pred.max())
axes[0].plot([lo, hi], [lo, hi], ls='--', color=ACCENT_D, label='完美预测')
axes[0].set(title='真实价 vs 预测价', xlabel='真实（美元）', ylabel='预测（美元）')
axes[0].legend()
axes[1].scatter(test_pred, resid, s=22, alpha=.6, color=MAIN_D)
axes[1].axhline(0, ls='--', color=ACCENT_D)
axes[1].set(title='残差图', xlabel='预测（美元）', ylabel='残差 = 真实 − 预测')
plt.show()

分价位看误差：cut + groupby

audit = pd.DataFrame({'actual': y_test, 'pred': test_pred})
audit['resid'] = audit['actual'] - audit['pred']
audit['abs_err'] = audit['resid'].abs()
edges = [-np.inf, *y_train.quantile([.25, .5, .75]), np.inf]   # 档位边界用训练集定
audit['band'] = pd.cut(audit['actual'], bins=edges)
display(audit.groupby('band', observed=True)
             .agg(n=('actual', 'size'), MAE=('abs_err', 'mean'), mean_resid=('resid', 'mean')))
audit.nlargest(3, 'abs_err')

Output                       n        MAE  mean_resid
band                                           
(-inf, 130875.0]      84 20,778.633  -4,661.329
(130875.0, 164945.0]  78 21,607.998  -4,415.156
(164945.0, 213062.5]  59 24,864.196 -16,436.746
(213062.5, inf]       71 46,263.967  34,770.330
      actual        pred       resid     abs_err             band
691   755000 449,028.739 305,971.261 305,971.261  (213062.5, inf]
898   611657 318,626.981 293,030.019 293,030.019  (213062.5, inf]
1046  556581 347,679.650 208,901.350 208,901.350  (213062.5, inf]

建模结论：f-string

r = test_results.loc['选中模型']
gain = 1 - r['RMSE'] / test_results.loc['均价基线', 'RMSE']
print(f"用 {len(FEATURES)} 个数值特征，在验证集比较后选 {best_name}。测试集 n={len(y_test)}："
      f"MAE={r['MAE']:,.0f}，RMSE={r['RMSE']:,.0f}，R²={r['R2']:.3f}，RMSE 比均价基线低 {gain:.1%}。")
print("最贵一档误差最大，而且系统性低估；结论只代表这份样本，不代表别的城市和未来市场。")

Output用 3 个数值特征，在验证集比较后选 线性回归。测试集 n=292：MAE=28,022，RMSE=43,411，R²=0.754，RMSE 比均价基线低 50.5%。
最贵一档误差最大，而且系统性低估；结论只代表这份样本，不代表别的城市和未来市场。

类别入模：get_dummies + reindextricky

tr_d = pd.get_dummies(train_data['neighborhood'], prefix='nb', dtype=int)
va_d = pd.get_dummies(val_data['neighborhood'], prefix='nb', dtype=int).reindex(columns=tr_d.columns, fill_value=0)
m = LinearRegression().fit(pd.concat([X_train, tr_d], axis=1), y_train)
pd.DataFrame({'数值 + 街区': scores(y_val, m.predict(pd.concat([X_val, va_d], axis=1)))}).T

Output               MAE       RMSE    R2
数值 + 街区 22,564.546 33,830.161 0.809

预测 log(价格)：np.log / np.exp

log_m = LinearRegression().fit(X_train_s, np.log(y_train))
price_pred = np.exp(log_m.predict(X_val_s))            # 还原回美元再算误差
pd.DataFrame({'log 目标': scores(y_val, price_pred)}).T

Output              MAE       RMSE    R2
log 目标 23,370.203 33,515.731 0.812

逻辑回归与混淆矩阵：LogisticRegressiontricky

from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, confusion_matrix
cut = y_train.median()                                 # 高于训练中位数 = 高价房（1）
yc_train, yc_test = (y_train > cut).astype(int), (y_test > cut).astype(int)
clf = LogisticRegression(max_iter=2000).fit(X_train_s, yc_train)
pc = clf.predict(X_test_s)
for name, f in [('Accuracy', accuracy_score), ('Precision', precision_score),
                ('Recall', recall_score), ('F1', f1_score)]:
    print(name, round(float(f(yc_test, pc)), 3))
cm = confusion_matrix(yc_test, pc)
fig, ax = plt.subplots(figsize=(4.5, 3.5))
sns.heatmap(cm, annot=True, fmt='d', cmap=sns.light_palette(MAIN_D, as_cmap=True), cbar=False,
            xticklabels=['其他', '高价'], yticklabels=['其他', '高价'], ax=ax)
ax.set(title='混淆矩阵', xlabel='预测', ylabel='真实')
plt.show()

OutputAccuracy 0.901
Precision 0.88
Recall 0.9
F1 0.89

聚类：KMeans

from sklearn.cluster import KMeans
km = KMeans(n_clusters=3, n_init=10, random_state=SEED)
view = train_data[FEATURES + ['price']].copy()
view['cluster'] = km.fit_predict(X_train_s)
view.groupby('cluster').median().assign(n=view['cluster'].value_counts())

Output             area  quality  house_age       price    n
cluster                                               
0       1,218.000    5.000     43.000 143,000.000  357
1       1,680.000    7.000      5.000 219,500.000  345
2       1,429.000    5.000     84.000 125,250.000  174

梯度下降：theta -= lr * grad

A_train = np.column_stack([np.ones(len(X_train_s)), X_train_s])
A_val = np.column_stack([np.ones(len(X_val_s)), X_val_s])
y_k = y_train.to_numpy() / 1000                       # 千美元，数值好看

def loss(theta, A, y):
    return np.mean((A @ theta - y) ** 2) / 2
def grad(theta, A, y):
    return A.T @ (A @ theta - y) / len(y)

theta = np.zeros(A_train.shape[1])
for step in range(3000):
    theta = theta - 0.1 * grad(theta, A_train, y_k)
print('最终损失', round(loss(theta, A_train, y_k), 1))
pd.DataFrame({'手写 GD': scores(y_val, A_val @ theta * 1000)}).T

Output最终损失 842.5
             MAE       RMSE    R2
手写 GD 26,176.916 37,775.779 0.762

小批量 SGD：permutation

rng = np.random.default_rng(SEED)
theta = np.zeros(A_train.shape[1])
for epoch in range(300):
    order = rng.permutation(len(y_k))                  # 每轮打乱
    for start in range(0, len(y_k), 32):               # 每次 32 条
        batch = order[start:start + 32]
        theta = theta - 0.1 / np.sqrt(epoch + 1) * grad(theta, A_train[batch], y_k[batch])
pd.DataFrame({'小批量 SGD': scores(y_val, A_val @ theta * 1000)}).T

Output               MAE       RMSE    R2
小批量 SGD 27,571.399 40,360.779 0.728

交给 SciPy：minimize

from scipy.optimize import minimize
res = minimize(loss, x0=np.zeros(A_train.shape[1]), args=(A_train, y_k), jac=grad)
print(res.success)
pd.DataFrame({'SciPy': scores(y_val, A_val @ res.x * 1000)}).T

OutputTrue
             MAE       RMSE    R2
SciPy 26,005.296 37,513.659 0.765

无标签新数据：clone / transformtricky

from sklearn.base import clone
all_med = clean[FEATURES].median()
final_scaler = MinMaxScaler().fit(clean[FEATURES].fillna(all_med))
final_model = clone(best_model).fit(final_scaler.transform(clean[FEATURES].fillna(all_med)), clean['price'])

new = raw.head(3).rename(columns=COLUMN_MAP)           # 考试换成 pd.read_csv('test.csv', **read_options)
new['house_age'] = new['sale_year'] - new['year_built']   # 和训练时一样的清洗、派生
X_new = final_scaler.transform(new[FEATURES].fillna(all_med))
pd.DataFrame({'id': new['id'], 'predicted_price': final_model.predict(X_new)})
# prediction.to_csv('submission.csv', index=False)    题目要求才导出

Output   id  predicted_price
0   1      231,424.189
1   2      165,460.574
2   3      235,141.324