Sklearn 集成学习

sklearn.ensemble 提供 Bagging、Boosting、Voting、Stacking 等集成方法。

---

🌲 Bagging 类方法

1. RandomForestClassifier — 随机森林分类器 ⭐

from sklearn.ensemble import RandomForestClassifier

model = RandomForestClassifier(
    n_estimators=100,       # 树的数量
    criterion='gini',       # 'gini' 或 'entropy' 或 'log_loss'
    max_depth=None,         # 树的最大深度
    min_samples_split=2,    # 内部节点再划分所需最小样本数
    min_samples_leaf=1,     # 叶节点最少样本数
    min_weight_fraction_leaf=0.0,
    max_features='sqrt',    # 每棵树随机选择的特征数
    # 'sqrt', 'log2', None, int, float(比例)
    max_leaf_nodes=None,    # 最大叶节点数
    min_impurity_decrease=0.0,
    bootstrap=True,         # 是否 bootstrap 采样
    oob_score=False,        # 是否计算袋外分数
    n_jobs=-1,
    random_state=42,
    verbose=0,
    warm_start=False,
    class_weight=None,      # None, 'balanced', 'balanced_subsample', dict
    ccp_alpha=0.0,          # 最小代价复杂度剪枝参数
    max_samples=None        # bootstrap 样本数（None=全部）
)

model.fit(X, y)

# 核心属性
print(model.feature_importances_)  # 特征重要性 ⭐
print(model.oob_score_)            # 袋外分数（oob_score=True）
print(model.estimators_)           # 所有决策树列表
print(model.classes_)              # 类别
print(model.n_classes_)            # 类别数
print(model.n_features_in_)        # 特征数

# 预测
y_pred = model.predict(X)
y_prob = model.predict_proba(X)
y_log_proba = model.predict_log_proba(X)

# 应用到新数据
model.apply(X)  # 返回每个样本每棵树的叶节点索引

---

2. RandomForestRegressor — 随机森林回归器 ⭐

from sklearn.ensemble import RandomForestRegressor

model = RandomForestRegressor(
    n_estimators=100,
    criterion='squared_error',  # 'squared_error','absolute_error','friedman_mse','poisson'
    max_depth=None,
    min_samples_split=2,
    min_samples_leaf=1,
    max_features=1.0,      # 回归建议用 'sqrt' 或 None(=n_features)
    bootstrap=True,
    oob_score=False,
    n_jobs=-1,
    random_state=42,
    max_samples=None
)

model.fit(X, y)
print(model.feature_importances_)  # 特征重要性
print(model.oob_prediction_)       # 袋外预测（oob_score=True）

---

3. ExtraTreesClassifier / ExtraTreesRegressor — 极端随机树

与随机森林的区别：分裂阈值完全随机（而不是选择最优）。

from sklearn.ensemble import ExtraTreesClassifier, ExtraTreesRegressor

model = ExtraTreesClassifier(
    n_estimators=100,
    criterion='gini',
    max_features='sqrt',
    bootstrap=False,       # 默认不用 bootstrap
    oob_score=False,       # 默认不开
    n_jobs=-1,
    random_state=42
)
model.fit(X, y)

---

4. BaggingClassifier / BaggingRegressor — 通用 Bagging ⭐

from sklearn.ensemble import BaggingClassifier, BaggingRegressor

model = BaggingClassifier(
    estimator=None,        # None=DecisionTreeClassifier, 或传入任意估计器
    n_estimators=10,
    max_samples=1.0,       # 每轮采样的样本比例
    max_features=1.0,      # 每轮采样的特征比例
    bootstrap=True,        # True=有放回, False=无放回(pasting)
    bootstrap_features=False,
    oob_score=False,
    warm_start=False,
    n_jobs=-1,
    random_state=42
)

# 使用 SVM 作为基学习器
from sklearn.svm import SVC
bagging_svm = BaggingClassifier(
    estimator=SVC(),
    n_estimators=10,
    max_samples=0.5,
    max_features=0.5
)

---

🚀 Boosting 类方法

1. GradientBoostingClassifier — 梯度提升分类器 ⭐

from sklearn.ensemble import GradientBoostingClassifier

model = GradientBoostingClassifier(
    loss='log_loss',        # 'log_loss' 或 'exponential'
    learning_rate=0.1,      # 学习率（步长）
    n_estimators=100,       # 提升阶段数（树的数量）
    subsample=1.0,          # 每棵树的样本比例（stochastic GB）
    criterion='friedman_mse',
    min_samples_split=2,
    min_samples_leaf=1,
    min_weight_fraction_leaf=0.0,
    max_depth=3,            # 树的深度（较大的值会增加交互）
    min_impurity_decrease=0.0,
    init=None,              # 初始估计器（或 'zero'）
    random_state=42,
    max_features=None,      # 特征子采样
    verbose=0,
    max_leaf_nodes=None,
    warm_start=False,
    validation_fraction=0.1, # 早停验证集比例
    n_iter_no_change=None,  # 早停容忍轮数
    tol=1e-4,
    ccp_alpha=0.0
)

model.fit(X, y)

# 核心属性
print(model.feature_importances_)    # 特征重要性
print(model.train_score_)            # 每轮训练分数
print(model.oob_improvement_)        # 袋外改善（subsample < 1 时）
print(model.estimators_)             # 每棵树的列表（len=n_estimators）

# 分阶段预测
y_pred_staged = list(model.staged_predict(X_test))
y_prob_staged = list(model.staged_predict_proba(X_test))

---

2. GradientBoostingRegressor — 梯度提升回归器 ⭐

from sklearn.ensemble import GradientBoostingRegressor

model = GradientBoostingRegressor(
    loss='squared_error',
    # 'squared_error','absolute_error','huber','quantile'
    learning_rate=0.1,
    n_estimators=100,
    subsample=1.0,
    criterion='friedman_mse',
    max_depth=3,
    alpha=0.9,              # huber 和 quantile 损失的参数
    random_state=42
)

model.fit(X, y)

# 分阶段预测
y_pred_iterable = model.staged_predict(X)  # 生成器

---

3. AdaBoostClassifier / AdaBoostRegressor — AdaBoost

from sklearn.ensemble import AdaBoostClassifier, AdaBoostRegressor

model = AdaBoostClassifier(
    estimator=None,         # None=DecisionTreeClassifier(max_depth=1)
    n_estimators=50,
    learning_rate=1.0,
    algorithm='SAMME.R',    # 'SAMME' 或 'SAMME.R'（需概率支持）
    random_state=42
)
model.fit(X, y)

print(model.estimator_weights_)  # 每轮权重
print(model.estimator_errors_)   # 每轮误差
print(model.feature_importances_)

---

4. HistGradientBoostingClassifier — 直方图梯度提升 ⭐

基于直方图的快速实现，原生支持缺失值和类别特征。

from sklearn.ensemble import HistGradientBoostingClassifier

model = HistGradientBoostingClassifier(
    loss='log_loss',        # 'log_loss' 或 'auto'
    learning_rate=0.1,
    max_iter=100,           # 等效于 n_estimators
    max_leaf_nodes=31,
    max_depth=None,
    min_samples_leaf=20,
    l2_regularization=0.0,
    max_bins=255,           # 连续特征的分箱数
    categorical_features=None,  # 类别特征的索引列表
    monotonic_cst=None,     # 单调约束
    interaction_cst=None,   # 交互约束
    warm_start=False,
    early_stopping='auto',  # True, False, 'auto'
    scoring='loss',
    validation_fraction=0.1,
    n_iter_no_change=10,   # 早停容忍轮数
    tol=1e-7,
    verbose=0,
    random_state=42,
    class_weight=None
)

model.fit(X, y)
print(model.feature_importances_)
print(model.n_iter_)  # 实际迭代数（早停后）

---

5. HistGradientBoostingRegressor — 直方图梯度提升回归器

from sklearn.ensemble import HistGradientBoostingRegressor

model = HistGradientBoostingRegressor(
    loss='squared_error',
    quantile=0.5,            # quantile 损失时使用
    learning_rate=0.1,
    max_iter=100,
    max_leaf_nodes=31,
    min_samples_leaf=20,
    l2_regularization=0.0,
    max_bins=255,
    monotonic_cst=None,
    early_stopping='auto',
    random_state=42
)
model.fit(X, y)

---

6. VotingClassifier / VotingRegressor — 投票集成 ⭐

from sklearn.ensemble import VotingClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.tree import DecisionTreeClassifier

voting = VotingClassifier(
    estimators=[
        ('lr', LogisticRegression()),
        ('svc', SVC(probability=True)),  # soft voting 需要 predict_proba
        ('dt', DecisionTreeClassifier())
    ],
    voting='soft',          # 'hard'(多数投票) 或 'soft'(概率加权)
    weights=[1, 2, 1],     # 各模型权重（soft voting 时）
    n_jobs=-1,
    flatten_transform=True  # transform 方法
)

voting.fit(X_train, y_train)
y_pred = voting.predict(X_test)

# 查看各模型的类别概率
probs = voting.transform(X_test)
# 属性
print(voting.named_estimators_)       # 模型字典
print(voting.estimators_)             # 模型列表

回归版本:

from sklearn.ensemble import VotingRegressor
from sklearn.linear_model import LinearRegression
from sklearn.ensemble import RandomForestRegressor

voting_reg = VotingRegressor(
    estimators=[
        ('lr', LinearRegression()),
        ('rf', RandomForestRegressor())
    ],
    weights=[1, 2]
)
voting_reg.fit(X_train, y_train)

---

7. StackingClassifier / StackingRegressor — 堆叠集成 ⭐

from sklearn.ensemble import StackingClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.tree import DecisionTreeClassifier

stacking = StackingClassifier(
    estimators=[
        ('svc', SVC(probability=True)),
        ('dt', DecisionTreeClassifier()),
    ],
    final_estimator=LogisticRegression(),  # 元学习器
    cv=5,                    # 交叉验证折叠数（生成第一层预测）
    stack_method='auto',     # 'auto','predict_proba','decision_function','predict'
    n_jobs=-1,
    passthrough=False,       # 是否将原始 X 也传给元学习器
    verbose=0
)

stacking.fit(X_train, y_train)
y_pred = stacking.predict(X_test)

# 属性
print(stacking.final_estimator_)        # 元学习器
print(stacking.named_estimators_)       # 基学习器
print(stacking.stack_method_)           # 实际使用的 stack_method

回归版本:

from sklearn.ensemble import StackingRegressor
from sklearn.linear_model import Ridge

stacking_reg = StackingRegressor(
    estimators=[
        ('rf', RandomForestRegressor()),
        ('gb', GradientBoostingRegressor()),
    ],
    final_estimator=Ridge(),
    cv=5,
    passthrough=True
)

---

8. IsolationForest — 孤立森林（异常检测）

from sklearn.ensemble import IsolationForest

model = IsolationForest(
    n_estimators=100,
    max_samples='auto',     # 每棵树的样本数
    contamination='auto',   # 异常比例（float）或 'auto'
    max_features=1.0,
    bootstrap=False,
    n_jobs=-1,
    random_state=42,
    verbose=0
)

model.fit(X)

# 预测: 1=正常, -1=异常
y_pred = model.predict(X)

# 异常分数（越低越异常）
scores = model.decision_function(X)
print(model.offset_)  # 决策偏移量

# 异常分数转为概率
scores = model.score_samples(X)  # 负分数，越高越正常

---

📊 特征重要性

所有树模型都支持特征重要性：

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.ensemble import RandomForestClassifier

model = RandomForestClassifier(n_estimators=100, random_state=42)
model.fit(X, y)

# 获取特征重要性
importances = model.feature_importances_
indices = np.argsort(importances)[::-1]

# 可视化
plt.figure(figsize=(10, 6))
plt.bar(range(X.shape[1]), importances[indices])
plt.xticks(range(X.shape[1]),
           [f'Feature {i}' for i in indices], rotation=90)
plt.title('Feature Importances')
plt.tight_layout()
plt.show()

基于置换的特征重要性（模型无关）:

from sklearn.inspection import permutation_importance

result = permutation_importance(
    model, X_test, y_test,
    n_repeats=10,
    random_state=42,
    n_jobs=-1
)

print(result.importances_mean)
print(result.importances_std)

---

📝 实践选择指南

场景	推荐模型
通用分类/回归基线	RandomForestClassifier / RandomForestRegressor
追求准确率	GradientBoostingClassifier / HistGradientBoostingClassifier
大数据集 + 快速	HistGradientBoostingClassifier
多模型融合	VotingClassifier / StackingClassifier
自定义基学习器	BaggingClassifier
异常检测	IsolationForest
特征含缺失值	HistGradientBoostingClassifier（原生支持）
类别特征	HistGradientBoostingClassifier + categorical_features

---

[[sklearn-总览|← 返回总览]]