Python 数据科学高级应用指南

1. 数据科学基础

数据科学是一门融合了统计学、计算机科学和领域知识的学科，它使用各种技术和方法来从数据中提取有价值的信息。Python 是数据科学领域最流行的编程语言之一，拥有丰富的库和工具。

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

# 创建示例数据
data = pd.DataFrame({
    'age': [25, 30, 35, 40, 45],
    'income': [50000, 60000, 70000, 80000, 90000]
})

# 数据可视化
plt.scatter(data['age'], data['income'])
plt.title('Age vs Income')
plt.xlabel('Age')
plt.ylabel('Income')
plt.show()

2. 数据预处理

2.1 数据清洗

import pandas as pd
import numpy as np

# 创建包含缺失值的数据
data = pd.DataFrame({
    'age': [25, np.nan, 35, 40, 45],
    'income': [50000, 60000, np.nan, 80000, 90000],
    'gender': ['M', 'F', 'M', 'F', 'M']
})

# 查看缺失值
print(data.isnull())

# 填充缺失值
data['age'].fillna(data['age'].mean(), inplace=True)
data['income'].fillna(data['income'].median(), inplace=True)

# 查看处理后的数据
print(data)

2.2 特征工程

import pandas as pd
from sklearn.preprocessing import StandardScaler, OneHotEncoder

# 创建数据
data = pd.DataFrame({
    'age': [25, 30, 35, 40, 45],
    'income': [50000, 60000, 70000, 80000, 90000],
    'gender': ['M', 'F', 'M', 'F', 'M']
})

# 标准化数值特征
scaler = StandardScaler()
data[['age', 'income']] = scaler.fit_transform(data[['age', 'income']])

# 编码分类特征
encoder = OneHotEncoder(drop='first', sparse=False)
gender_encoded = encoder.fit_transform(data[['gender']])
gender_df = pd.DataFrame(gender_encoded, columns=['gender_M'])
data = pd.concat([data, gender_df], axis=1)
data.drop('gender', axis=1, inplace=True)

# 查看处理后的数据
print(data)

3. 机器学习

3.1 监督学习

from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_error
import pandas as pd
import numpy as np

# 创建数据
data = pd.DataFrame({
    'age': [25, 30, 35, 40, 45, 50, 55, 60],
    'income': [50000, 60000, 70000, 80000, 90000, 95000, 98000, 100000]
})

# 准备特征和标签
X = data[['age']]
y = data['income']

# 拆分数据
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# 训练模型
model = LinearRegression()
model.fit(X_train, y_train)

# 预测
y_pred = model.predict(X_test)

# 评估
mse = mean_squared_error(y_test, y_pred)
print(f"Mean Squared Error: {mse}")

# 查看模型系数
print(f"Coefficient: {model.coef_[0]}")
print(f"Intercept: {model.intercept_}")

3.2 无监督学习

from sklearn.cluster import KMeans
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

# 创建数据
np.random.seed(42)
data = pd.DataFrame({
    'x': np.random.normal(0, 1, 100),
    'y': np.random.normal(0, 1, 100)
})

# 训练 K-Means 模型
kmeans = KMeans(n_clusters=3, random_state=42)
data['cluster'] = kmeans.fit_predict(data[['x', 'y']])

# 可视化聚类结果
plt.scatter(data['x'], data['y'], c=data['cluster'], cmap='viridis')
plt.scatter(kmeans.cluster_centers_[:, 0], kmeans.cluster_centers_[:, 1], s=100, c='red')
plt.title('K-Means Clustering')
plt.show()

4. 深度学习

4.1 神经网络基础

import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
import numpy as np

# 创建数据
X = np.array([[0, 0], [0, 1], [1, 0], [1, 1]])
y = np.array([[0], [1], [1], [0]])

# 创建模型
model = Sequential([
    Dense(4, activation='relu', input_shape=(2,)),
    Dense(1, activation='sigmoid')
])

# 编译模型
model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])

# 训练模型
model.fit(X, y, epochs=1000, verbose=0)

# 预测
predictions = model.predict(X)
print(predictions)

4.2 卷积神经网络

import tensorflow as tf
from tensorflow.keras.datasets import mnist
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Conv2D, MaxPooling2D, Flatten

# 加载数据
(x_train, y_train), (x_test, y_test) = mnist.load_data()

# 数据预处理
x_train = x_train.reshape(-1, 28, 28, 1) / 255.0
x_test = x_test.reshape(-1, 28, 28, 1) / 255.0
y_train = tf.keras.utils.to_categorical(y_train, 10)
y_test = tf.keras.utils.to_categorical(y_test, 10)

# 创建模型
model = Sequential([
    Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)),
    MaxPooling2D((2, 2)),
    Conv2D(64, (3, 3), activation='relu'),
    MaxPooling2D((2, 2)),
    Flatten(),
    Dense(128, activation='relu'),
    Dense(10, activation='softmax')
])

# 编译模型
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])

# 训练模型
model.fit(x_train, y_train, epochs=5, batch_size=32, validation_split=0.2)

# 评估模型
loss, accuracy = model.evaluate(x_test, y_test)
print(f"Test accuracy: {accuracy}")

5. 实际应用场景

5.1 预测分析

import pandas as pd
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt

# 加载数据
data = pd.read_csv('house_prices.csv')

# 准备特征和标签
X = data[['sqft_living', 'bedrooms', 'bathrooms']]
y = data['price']

# 拆分数据
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# 训练模型
model = LinearRegression()
model.fit(X_train, y_train)

# 预测
y_pred = model.predict(X_test)

# 可视化预测结果
plt.scatter(y_test, y_pred)
plt.plot([y_test.min(), y_test.max()], [y_test.min(), y_test.max()], 'k--', lw=2)
plt.xlabel('Actual Price')
plt.ylabel('Predicted Price')
plt.title('Actual vs Predicted House Prices')
plt.show()

5.2 自然语言处理

import nltk
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.cluster import KMeans

# 示例文本
texts = [
    "Python is a popular programming language",
    "Java is used for enterprise applications",
    "C++ is a compiled language",
    "Python is used for data science",
    "Java is used for Android development",
    "C++ is used for system programming"
]

# 文本预处理
nltk.download('punkt')
nltk.download('stopwords')
stop_words = set(stopwords.words('english'))

processed_texts = []
for text in texts:
    tokens = word_tokenize(text.lower())
    filtered_tokens = [token for token in tokens if token.isalpha() and token not in stop_words]
    processed_texts.append(' '.join(filtered_tokens))

# 特征提取
vectorizer = TfidfVectorizer()
X = vectorizer.fit_transform(processed_texts)

# 聚类
kmeans = KMeans(n_clusters=3, random_state=42)
clusters = kmeans.fit_predict(X)

# 查看结果
for i, text in enumerate(texts):
    print(f"Text: {text}")
    print(f"Cluster: {clusters[i]}")
    print()

6. 最佳实践

1. 数据质量：确保数据质量，处理缺失值和异常值。
2. 特征工程：合理选择和处理特征，提高模型性能。
3. 模型选择：根据问题类型选择合适的模型。
4. 模型评估：使用合适的评估指标评估模型性能。
5. 超参数调优：通过交叉验证等方法调优模型超参数。
6. 可解释性：关注模型的可解释性，理解模型决策过程。

7. 总结

Python 是数据科学领域的强大工具，它提供了丰富的库和工具，从数据处理到机器学习和深度学习。通过掌握这些工具的高级应用，我们可以从数据中提取有价值的信息，解决复杂的问题。

在实际应用中，我们可以使用 Python 进行预测分析、自然语言处理、图像处理等多种任务，为业务决策提供支持。

希望本文对你理解和应用 Python 数据科学有所帮助！