Classification Models

import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
import warnings
warnings.filterwarnings('ignore')
 
# Set seed for reproducible results
np.random.seed(42)
 
# Generate sample data for customer churn prediction
n_customers = 1000
 
customer_features = pd.DataFrame({
    'user_id': range(1, n_customers + 1),
    'total_orders': np.random.poisson(5, n_customers),
    'total_items': np.random.poisson(15, n_customers),
    'total_spent': np.random.exponential(500, n_customers),
    'avg_order_value': np.random.exponential(100, n_customers),
    'order_span_days': np.random.randint(1, 365, n_customers),
    'unique_categories': np.random.randint(1, 10, n_customers),
    'unique_brands': np.random.randint(1, 20, n_customers),
    'days_since_last_order': np.random.exponential(60, n_customers)
})
 
# Churn definition: Churned if no purchase for 90+ days (+ random noise)
churn_prob = 1 / (1 + np.exp(-(customer_features['days_since_last_order'] - 90) / 30))
customer_features['churned'] = (np.random.random(n_customers) < churn_prob).astype(int)
 
print(f"Total customers: {len(customer_features)}")
print(f"Churned customers: {customer_features['churned'].sum()}")
print(f"Churn rate: {customer_features['churned'].mean():.1%}")

# Separate features and target
feature_cols = ['total_orders', 'total_items', 'total_spent', 'avg_order_value',
                'order_span_days', 'unique_categories', 'unique_brands']
X = customer_features[feature_cols]
y = customer_features['churned']
 
# Train/test split (80:20)
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42, stratify=y
)
 
print(f"Training set: {len(X_train)} samples")
print(f"Test set: {len(X_test)} samples")
print(f"Training set churn rate: {y_train.mean():.1%}")
print(f"Test set churn rate: {y_test.mean():.1%}")

# Scaling is required for Logistic Regression, SVM, etc.
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)
 
# Tree-based models don't need scaling
# RandomForest, XGBoost can use original values
print("Scaling complete!")
print(f"X_train_scaled mean: {X_train_scaled.mean():.4f}")
print(f"X_train_scaled std: {X_train_scaled.std():.4f}")

from sklearn.linear_model import LogisticRegression
from sklearn.metrics import classification_report, confusion_matrix
 
# Model training
lr_model = LogisticRegression(random_state=42, max_iter=1000)
lr_model.fit(X_train_scaled, y_train)
 
# Prediction
y_pred_lr = lr_model.predict(X_test_scaled)
y_prob_lr = lr_model.predict_proba(X_test_scaled)[:, 1]
 
# Evaluation
print("=== Logistic Regression Results ===")
print(classification_report(y_test, y_pred_lr, target_names=['Retained', 'Churned']))

import matplotlib.pyplot as plt
 
# Coefficients by feature (influence)
coef_df = pd.DataFrame({
    'feature': feature_cols,
    'coefficient': lr_model.coef_[0]
})
coef_df['abs_coef'] = coef_df['coefficient'].abs()
coef_df = coef_df.sort_values('abs_coef', ascending=False)
 
print("Feature Importance (Coefficients):")
print(coef_df.to_string(index=False))
 
# Visualization
plt.figure(figsize=(10, 6))
colors = ['green' if c > 0 else 'red' for c in coef_df['coefficient']]
plt.barh(coef_df['feature'], coef_df['coefficient'], color=colors)
plt.xlabel('Coefficient (positive: increases churn, negative: decreases churn)')
plt.title('Logistic Regression Feature Importance', fontsize=14, fontweight='bold')
plt.axvline(x=0, color='black', linestyle='-', linewidth=0.5)
plt.tight_layout()
plt.show()

from sklearn.tree import DecisionTreeClassifier, plot_tree
 
# Model training
dt_model = DecisionTreeClassifier(
    max_depth=5,           # Prevent overfitting
    min_samples_split=20,  # Minimum samples to split
    random_state=42
)
dt_model.fit(X_train, y_train)
 
# Prediction
y_pred_dt = dt_model.predict(X_test)
y_prob_dt = dt_model.predict_proba(X_test)[:, 1]
 
# Evaluation
print("=== Decision Tree Results ===")
print(classification_report(y_test, y_pred_dt, target_names=['Retained', 'Churned']))

# Decision tree visualization
plt.figure(figsize=(20, 10))
plot_tree(
    dt_model,
    feature_names=feature_cols,
    class_names=['Retained', 'Churned'],
    filled=True,
    rounded=True,
    fontsize=10,
    max_depth=3  # Limit depth for visualization
)
plt.title('Decision Tree Visualization (up to depth 3)', fontsize=16, fontweight='bold')
plt.tight_layout()
plt.show()

from sklearn.ensemble import RandomForestClassifier
 
# Model training
rf_model = RandomForestClassifier(
    n_estimators=100,      # Number of trees
    max_depth=10,          # Maximum depth
    min_samples_split=10,
    random_state=42,
    n_jobs=-1              # Parallel processing
)
rf_model.fit(X_train, y_train)
 
# Prediction
y_pred_rf = rf_model.predict(X_test)
y_prob_rf = rf_model.predict_proba(X_test)[:, 1]
 
# Evaluation
print("=== Random Forest Results ===")
print(classification_report(y_test, y_pred_rf, target_names=['Retained', 'Churned']))

# Feature importance
importance_df = pd.DataFrame({
    'feature': feature_cols,
    'importance': rf_model.feature_importances_
}).sort_values('importance', ascending=False)
 
print("Feature Importance:")
print(importance_df.to_string(index=False))
 
# Visualization
plt.figure(figsize=(10, 6))
plt.barh(importance_df['feature'], importance_df['importance'], color='steelblue')
plt.xlabel('Importance')
plt.title('Random Forest Feature Importance', fontsize=14, fontweight='bold')
plt.tight_layout()
plt.show()

from xgboost import XGBClassifier
 
# Model training
xgb_model = XGBClassifier(
    n_estimators=100,
    max_depth=6,
    learning_rate=0.1,
    subsample=0.8,         # Row sampling
    colsample_bytree=0.8,  # Column sampling
    random_state=42,
    eval_metric='logloss'
)
xgb_model.fit(X_train, y_train)
 
# Prediction
y_pred_xgb = xgb_model.predict(X_test)
y_prob_xgb = xgb_model.predict_proba(X_test)[:, 1]
 
# Evaluation
print("=== XGBoost Results ===")
print(classification_report(y_test, y_pred_xgb, target_names=['Retained', 'Churned']))

from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay
 
# Confusion matrix visualization
fig, axes = plt.subplots(1, 3, figsize=(15, 4))
models = [
    ('Logistic Regression', y_pred_lr),
    ('Random Forest', y_pred_rf),
    ('XGBoost', y_pred_xgb)
]
 
for ax, (name, y_pred) in zip(axes, models):
    cm = confusion_matrix(y_test, y_pred)
    disp = ConfusionMatrixDisplay(cm, display_labels=['Retained', 'Churned'])
    disp.plot(ax=ax, cmap='Blues', values_format='d')
    ax.set_title(name)
 
plt.tight_layout()
plt.show()

from sklearn.metrics import roc_curve, roc_auc_score
 
plt.figure(figsize=(10, 8))
 
# ROC curve for each model
for name, y_prob in [('Logistic Regression', y_prob_lr),
                      ('Random Forest', y_prob_rf),
                      ('XGBoost', y_prob_xgb)]:
    fpr, tpr, _ = roc_curve(y_test, y_prob)
    auc = roc_auc_score(y_test, y_prob)
    plt.plot(fpr, tpr, linewidth=2, label=f'{name} (AUC={auc:.3f})')
 
# Baseline (random prediction)
plt.plot([0, 1], [0, 1], 'k--', linewidth=1, label='Random (AUC=0.500)')
 
plt.xlabel('False Positive Rate', fontsize=12)
plt.ylabel('True Positive Rate (Recall)', fontsize=12)
plt.title('ROC Curve Comparison', fontsize=14, fontweight='bold')
plt.legend(loc='lower right')
plt.grid(True, alpha=0.3)
plt.tight_layout()
plt.show()

from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score
 
# Model performance comparison
results = []
for name, y_pred, y_prob in [('Logistic Regression', y_pred_lr, y_prob_lr),
                               ('Random Forest', y_pred_rf, y_prob_rf),
                               ('XGBoost', y_pred_xgb, y_prob_xgb)]:
    results.append({
        'Model': name,
        'Accuracy': accuracy_score(y_test, y_pred),
        'Precision': precision_score(y_test, y_pred),
        'Recall': recall_score(y_test, y_pred),
        'F1': f1_score(y_test, y_pred),
        'AUC': roc_auc_score(y_test, y_prob)
    })
 
results_df = pd.DataFrame(results).round(3)
print("=== Model Performance Comparison ===")
print(results_df.to_string(index=False))

# Method 1: class_weight adjustment
rf_balanced = RandomForestClassifier(
    n_estimators=100,
    class_weight='balanced',  # Weight for minority class
    random_state=42
)
rf_balanced.fit(X_train, y_train)
y_pred_balanced = rf_balanced.predict(X_test)
 
print("=== class_weight='balanced' Results ===")
print(f"Original recall: {recall_score(y_test, y_pred_rf):.3f}")
print(f"Balanced recall: {recall_score(y_test, y_pred_balanced):.3f}")
 
# Method 2: Threshold adjustment
threshold = 0.3  # Lower from default 0.5
y_pred_adjusted = (y_prob_xgb >= threshold).astype(int)
 
print(f"\n=== Threshold Adjustment (0.5 → 0.3) ===")
print(f"Original recall: {recall_score(y_test, y_pred_xgb):.3f}")
print(f"Adjusted recall: {recall_score(y_test, y_pred_adjusted):.3f}")
print(f"Original precision: {precision_score(y_test, y_pred_xgb):.3f}")
print(f"Adjusted precision: {precision_score(y_test, y_pred_adjusted):.3f}")