Senior Data Scientist Interview Questions: Complete Guide

import numpy as np
from sklearn.model_selection import learning_curve
from sklearn.tree import DecisionTreeRegressor
import matplotlib.pyplot as plt

# Generate data
X = np.random.rand(100, 1) * 10
y = 2 * X + 3 + np.random.randn(100, 1) * 2

# High bias model (max_depth=1)
high_bias = DecisionTreeRegressor(max_depth=1)

# High variance model (max_depth=20)
high_variance = DecisionTreeRegressor(max_depth=20)

# Optimal model (max_depth=3)
optimal = DecisionTreeRegressor(max_depth=3)

# Learning curves show bias-variance tradeoff
train_sizes, train_scores, val_scores = learning_curve(
    optimal, X, y.ravel(), cv=5, train_sizes=np.linspace(0.1, 1.0, 10)
)

print(f"Training score: {train_scores.mean():.2f}")
print(f"Validation score: {val_scores.mean():.2f}")

from sklearn.linear_model import Lasso, Ridge, ElasticNet
from sklearn.datasets import make_regression
from sklearn.model_selection import train_test_split
import numpy as np

# Generate data with many features
X, y = make_regression(n_samples=100, n_features=20, n_informative=10, random_state=42)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3)

# L1 Regularization (Lasso)
lasso = Lasso(alpha=1.0)
lasso.fit(X_train, y_train)
print(f"Lasso coefficients: {np.sum(lasso.coef_ != 0)} non-zero out of {len(lasso.coef_)}")

# L2 Regularization (Ridge)
ridge = Ridge(alpha=1.0)
ridge.fit(X_train, y_train)
print(f"Ridge coefficients: {np.sum(ridge.coef_ != 0)} non-zero out of {len(ridge.coef_)}")

# Elastic Net (L1 + L2)
elastic = ElasticNet(alpha=1.0, l1_ratio=0.5)
elastic.fit(X_train, y_train)

print(f"\nLasso score: {lasso.score(X_test, y_test):.3f}")
print(f"Ridge score: {ridge.score(X_test, y_test):.3f}")
print(f"Elastic Net score: {elastic.score(X_test, y_test):.3f}")

from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split, cross_val_score

# Load data
data = load_breast_cancer()
X_train, X_test, y_train, y_test = train_test_split(
    data.data, data.target, test_size=0.3, random_state=42
)

# Bagging - Random Forest
rf = RandomForestClassifier(n_estimators=100, random_state=42)
rf.fit(X_train, y_train)
rf_score = rf.score(X_test, y_test)

# Boosting - Gradient Boosting
gb = GradientBoostingClassifier(n_estimators=100, random_state=42)
gb.fit(X_train, y_train)
gb_score = gb.score(X_test, y_test)

print(f"Random Forest (Bagging) accuracy: {rf_score:.3f}")
print(f"Gradient Boosting accuracy: {gb_score:.3f}")

# Cross-validation
rf_cv = cross_val_score(rf, data.data, data.target, cv=5)
gb_cv = cross_val_score(gb, data.data, data.target, cv=5)

print(f"\nRF CV scores: {rf_cv.mean():.3f} (+/- {rf_cv.std():.3f})")
print(f"GB CV scores: {gb_cv.mean():.3f} (+/- {gb_cv.std():.3f})")

from sklearn.model_selection import cross_val_score, KFold, StratifiedKFold
from sklearn.linear_model import LogisticRegression
from sklearn.datasets import load_iris

data = load_iris()
X, y = data.data, data.target

model = LogisticRegression(max_iter=200)

# Standard K-Fold
kfold = KFold(n_splits=5, shuffle=True, random_state=42)
scores = cross_val_score(model, X, y, cv=kfold)
print(f"K-Fold CV scores: {scores}")
print(f"Mean: {scores.mean():.3f} (+/- {scores.std():.3f})")

# Stratified K-Fold (preserves class distribution)
stratified_kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
stratified_scores = cross_val_score(model, X, y, cv=stratified_kfold)
print(f"\nStratified K-Fold scores: {stratified_scores}")
print(f"Mean: {stratified_scores.mean():.3f} (+/- {stratified_scores.std():.3f})")

# Custom cross-validation
from sklearn.model_selection import cross_validate

cv_results = cross_validate(
    model, X, y, cv=5,
    scoring=['accuracy', 'precision_macro', 'recall_macro'],
    return_train_score=True
)

print(f"\nTest accuracy: {cv_results['test_accuracy'].mean():.3f}")
print(f"Test precision: {cv_results['test_precision_macro'].mean():.3f}")
print(f"Test recall: {cv_results['test_recall_macro'].mean():.3f}")

from sklearn.decomposition import PCA
from sklearn.manifold import TSNE
from sklearn.datasets import load_digits
import matplotlib.pyplot as plt

# Load high-dimensional data
digits = load_digits()
X, y = digits.data, digits.target

print(f"Original shape: {X.shape}")

# PCA - reduce to 2 dimensions
pca = PCA(n_components=2)
X_pca = pca.fit_transform(X)
print(f"PCA shape: {X_pca.shape}")
print(f"Explained variance ratio: {pca.explained_variance_ratio_}")
print(f"Total variance explained: {pca.explained_variance_ratio_.sum():.3f}")

# t-SNE - reduce to 2 dimensions
tsne = TSNE(n_components=2, random_state=42)
X_tsne = tsne.fit_transform(X)
print(f"t-SNE shape: {X_tsne.shape}")

# PCA for feature extraction (keep 95% variance)
pca_95 = PCA(n_components=0.95)
X_reduced = pca_95.fit_transform(X)
print(f"\nComponents for 95% variance: {pca_95.n_components_}")
print(f"Reduced shape: {X_reduced.shape}")

from sklearn.metrics import roc_curve, roc_auc_score, auc
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.datasets import load_breast_cancer
import matplotlib.pyplot as plt

# Load data
data = load_breast_cancer()
X_train, X_test, y_train, y_test = train_test_split(
    data.data, data.target, test_size=0.3, random_state=42
)

# Train model
model = LogisticRegression(max_iter=10000)
model.fit(X_train, y_train)

# Get probability predictions
y_pred_proba = model.predict_proba(X_test)[:, 1]

# Calculate ROC curve
fpr, tpr, thresholds = roc_curve(y_test, y_pred_proba)
roc_auc = auc(fpr, tpr)

print(f"AUC: {roc_auc:.3f}")

# Alternative: direct AUC calculation
auc_score = roc_auc_score(y_test, y_pred_proba)
print(f"AUC (direct): {auc_score:.3f}")

# Find optimal threshold (Youden's J statistic)
optimal_idx = np.argmax(tpr - fpr)
optimal_threshold = thresholds[optimal_idx]
print(f"Optimal threshold: {optimal_threshold:.3f}")

from sklearn.preprocessing import StandardScaler, OneHotEncoder, PolynomialFeatures
import pandas as pd
import numpy as np

# Sample data
df = pd.DataFrame({
    'age': [25, 30, 35, 40, 45],
    'salary': [50000, 60000, 75000, 80000, 90000],
    'department': ['IT', 'HR', 'IT', 'Finance', 'HR'],
    'date': pd.date_range('2023-01-01', periods=5)
})

# One-hot encoding
df_encoded = pd.get_dummies(df, columns=['department'], prefix='dept')

# Scaling
scaler = StandardScaler()
df_encoded[['age_scaled', 'salary_scaled']] = scaler.fit_transform(
    df_encoded[['age', 'salary']]
)

# Binning
df_encoded['age_group'] = pd.cut(df['age'], bins=[0, 30, 40, 100], labels=['young', 'mid', 'senior'])

# Date features
df_encoded['year'] = df['date'].dt.year
df_encoded['month'] = df['date'].dt.month
df_encoded['day_of_week'] = df['date'].dt.dayofweek

# Polynomial features
poly = PolynomialFeatures(degree=2, include_bias=False)
poly_features = poly.fit_transform(df[['age', 'salary']])

# Interaction features
df_encoded['age_salary_interaction'] = df['age'] * df['salary']

print(df_encoded.head())

from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import classification_report, confusion_matrix
from imblearn.over_sampling import SMOTE
from imblearn.under_sampling import RandomUnderSampler

# Create imbalanced dataset
X, y = make_classification(
    n_samples=1000, n_features=20, n_informative=15,
    n_classes=2, weights=[0.9, 0.1], random_state=42
)

print(f"Class distribution: {np.bincount(y)}")

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)

# 1. Without handling imbalance
model_baseline = LogisticRegression()
model_baseline.fit(X_train, y_train)
y_pred_baseline = model_baseline.predict(X_test)
print("\nBaseline (no handling):")
print(classification_report(y_test, y_pred_baseline))

# 2. SMOTE (Synthetic Minority Over-sampling)
smote = SMOTE(random_state=42)
X_train_smote, y_train_smote = smote.fit_resample(X_train, y_train)
print(f"\nAfter SMOTE: {np.bincount(y_train_smote)}")

model_smote = LogisticRegression()
model_smote.fit(X_train_smote, y_train_smote)
y_pred_smote = model_smote.predict(X_test)
print("\nWith SMOTE:")
print(classification_report(y_test, y_pred_smote))

# 3. Class weights
model_weighted = LogisticRegression(class_weight='balanced')
model_weighted.fit(X_train, y_train)
y_pred_weighted = model_weighted.predict(X_test)
print("\nWith class weights:")
print(classification_report(y_test, y_pred_weighted))

from sklearn.feature_selection import SelectKBest, chi2, RFE, SelectFromModel
from sklearn.ensemble import RandomForestClassifier
from sklearn.datasets import load_breast_cancer
from sklearn.preprocessing import MinMaxScaler

# Load data
data = load_breast_cancer()
X, y = data.data, data.target

# 1. Filter Method - SelectKBest with chi-square
X_scaled = MinMaxScaler().fit_transform(X)
selector_chi2 = SelectKBest(chi2, k=10)
X_chi2 = selector_chi2.fit_transform(X_scaled, y)
print(f"Original features: {X.shape[1]}")
print(f"Selected features (chi2): {X_chi2.shape[1]}")

# 2. Wrapper Method - Recursive Feature Elimination
rf = RandomForestClassifier(n_estimators=100, random_state=42)
rfe = RFE(estimator=rf, n_features_to_select=10)
X_rfe = rfe.fit_transform(X, y)
print(f"Selected features (RFE): {X_rfe.shape[1]}")
print(f"Feature ranking: {rfe.ranking_}")

# 3. Embedded Method - Tree-based feature importance
rf.fit(X, y)
importances = rf.feature_importances_
indices = np.argsort(importances)[::-1]

print("\nTop 10 features by importance:")
for i in range(10):
    print(f"{i+1}. {data.feature_names[indices[i]]}: {importances[indices[i]]:.4f}")

# SelectFromModel
selector_model = SelectFromModel(rf, threshold='median', prefit=True)
X_selected = selector_model.transform(X)
print(f"\nSelected features (importance): {X_selected.shape[1]}")

import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split

# Sample data with high cardinality
df = pd.DataFrame({
    'city': np.random.choice([f'City_{i}' for i in range(100)], 1000),
    'target': np.random.randint(0, 2, 1000)
})

print(f"Unique cities: {df['city'].nunique()}")

# 1. Target Encoding
target_means = df.groupby('city')['target'].mean()
df['city_target_encoded'] = df['city'].map(target_means)

# 2. Frequency Encoding
freq = df['city'].value_counts()
df['city_frequency'] = df['city'].map(freq)

# 3. Grouping rare categories
freq_threshold = 10
rare_cities = freq[freq < freq_threshold].index
df['city_grouped'] = df['city'].apply(lambda x: 'Other' if x in rare_cities else x)

print(f"\nAfter grouping: {df['city_grouped'].nunique()} unique values")

# 4. Hash encoding (using category_encoders library)
# from category_encoders import HashingEncoder
# encoder = HashingEncoder(cols=['city'], n_components=10)
# df_hashed = encoder.fit_transform(df)

print(df[['city', 'city_target_encoded', 'city_frequency', 'city_grouped']].head())

# 1. Train and save model
from sklearn.ensemble import RandomForestClassifier
from sklearn.datasets import load_iris
import joblib

# Train model
data = load_iris()
model = RandomForestClassifier()
model.fit(data.data, data.target)

# Save model
joblib.dump(model, 'model.joblib')

# 2. Create API with FastAPI
from fastapi import FastAPI
import numpy as np

app = FastAPI()

# Load model
model = joblib.load('model.joblib')

@app.post("/predict")
def predict(features: list):
    # Convert to numpy array
    X = np.array(features).reshape(1, -1)
    prediction = model.predict(X)
    probability = model.predict_proba(X)
    
    return {
        "prediction": int(prediction[0]),
        "probability": probability[0].tolist()
    }

# 3. Dockerfile
"""
FROM python:3.9-slim
WORKDIR /app
COPY requirements.txt .
RUN pip install -r requirements.txt
COPY . .
CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]
"""

# 4. Usage
# curl -X POST "http://localhost:8000/predict" \
#      -H "Content-Type: application/json" \
#      -d '{"features": [5.1, 3.5, 1.4, 0.2]}'

import numpy as np
from scipy import stats

# Simulate production data
training_data = np.random.normal(0, 1, 1000)
production_data = np.random.normal(0.5, 1.2, 1000)  # Drifted

# Detect data drift using Kolmogorov-Smirnov test
statistic, p_value = stats.ks_2samp(training_data, production_data)

print(f"KS Statistic: {statistic:.4f}")
print(f"P-value: {p_value:.4f}")

if p_value < 0.05:
    print("Data drift detected! Consider retraining the model.")
else:
    print("No significant drift detected.")

# Monitor model performance
class ModelMonitor:
    def __init__(self, model):
        self.model = model
        self.predictions = []
        self.actuals = []
        
    def log_prediction(self, X, y_pred, y_true=None):
        self.predictions.append(y_pred)
        if y_true is not None:
            self.actuals.append(y_true)
    
    def get_accuracy(self):
        if len(self.actuals) == 0:
            return None
        return np.mean(np.array(self.predictions) == np.array(self.actuals))
    
    def check_drift(self, new_data, reference_data):
        statistic, p_value = stats.ks_2samp(new_data, reference_data)
        return p_value < 0.05

# Usage
monitor = ModelMonitor(model)
# monitor.log_prediction(X, y_pred, y_true)
# accuracy = monitor.get_accuracy()

import numpy as np
from scipy import stats

# Simulate A/B test results
# Control group (Model A)
control_conversions = 520
control_visitors = 10000

# Treatment group (Model B)
treatment_conversions = 580
treatment_visitors = 10000

# Calculate conversion rates
control_rate = control_conversions / control_visitors
treatment_rate = treatment_conversions / treatment_visitors

print(f"Control conversion rate: {control_rate:.4f}")
print(f"Treatment conversion rate: {treatment_rate:.4f}")
print(f"Lift: {((treatment_rate - control_rate) / control_rate * 100):.2f}%")

# Statistical significance test (two-proportion z-test)
pooled_rate = (control_conversions + treatment_conversions) / (control_visitors + treatment_visitors)
se = np.sqrt(pooled_rate * (1 - pooled_rate) * (1/control_visitors + 1/treatment_visitors))
z_score = (treatment_rate - control_rate) / se
p_value = 2 * (1 - stats.norm.cdf(abs(z_score)))

print(f"\nZ-score: {z_score:.4f}")
print(f"P-value: {p_value:.4f}")

if p_value < 0.05:
    print("Result is statistically significant!")
    if treatment_rate > control_rate:
        print("Treatment (Model B) is better.")
    else:
        print("Control (Model A) is better.")
else:
    print("No statistically significant difference.")

# Sample size calculation
from statsmodels.stats.power import zt_ind_solve_power

required_sample = zt_ind_solve_power(
    effect_size=0.02,  # Minimum detectable effect
    alpha=0.05,
    power=0.8,
    alternative='two-sided'
)
print(f"\nRequired sample size per group: {int(required_sample)}")

# Example: MLflow for experiment tracking
import mlflow
import mlflow.sklearn
from sklearn.ensemble import RandomForestClassifier
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score

# Load data
data = load_iris()
X_train, X_test, y_train, y_test = train_test_split(
    data.data, data.target, test_size=0.3, random_state=42
)

# Start MLflow run
with mlflow.start_run():
    # Log parameters
    n_estimators = 100
    max_depth = 5
    mlflow.log_param("n_estimators", n_estimators)
    mlflow.log_param("max_depth", max_depth)
    
    # Train model
    model = RandomForestClassifier(
        n_estimators=n_estimators,
        max_depth=max_depth,
        random_state=42
    )
    model.fit(X_train, y_train)
    
    # Evaluate
    y_pred = model.predict(X_test)
    accuracy = accuracy_score(y_test, y_pred)
    
    # Log metrics
    mlflow.log_metric("accuracy", accuracy)
    
    # Log model
    mlflow.sklearn.log_model(model, "random_forest_model")
    
    print(f"Accuracy: {accuracy:.3f}")
    print(f"Run ID: {mlflow.active_run().info.run_id}")

# Version control with DVC
"""
# Initialize DVC
dvc init

# Track data
dvc add data/train.csv
git add data/train.csv.dvc .gitignore
git commit -m "Add training data"

# Track model
dvc add models/model.pkl
git add models/model.pkl.dvc
git commit -m "Add trained model"
"""

import tensorflow as tf
from tensorflow import keras
from sklearn.datasets import load_digits
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

# Load data
digits = load_digits()
X, y = digits.data, digits.target

# Preprocess
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)
X_train, X_test, y_train, y_test = train_test_split(
    X_scaled, y, test_size=0.3, random_state=42
)

# Build neural network
model = keras.Sequential([
    keras.layers.Dense(128, activation='relu', input_shape=(64,)),
    keras.layers.Dropout(0.2),
    keras.layers.Dense(64, activation='relu'),
    keras.layers.Dropout(0.2),
    keras.layers.Dense(10, activation='softmax')
])

# Compile
model.compile(
    optimizer='adam',
    loss='sparse_categorical_crossentropy',
    metrics=['accuracy']
)

# Train
history = model.fit(
    X_train, y_train,
    epochs=50,
    batch_size=32,
    validation_split=0.2,
    verbose=0
)

# Evaluate
test_loss, test_acc = model.evaluate(X_test, y_test, verbose=0)
print(f"Test accuracy: {test_acc:.3f}")

# Model summary
model.summary()

from tensorflow.keras.applications import VGG16
from tensorflow.keras import layers, models

# Load pre-trained model (without top classification layer)
base_model = VGG16(
    weights='imagenet',
    include_top=False,
    input_shape=(224, 224, 3)
)

# Freeze base model layers
base_model.trainable = False

# Add custom classification layers
model = models.Sequential([
    base_model,
    layers.GlobalAveragePooling2D(),
    layers.Dense(256, activation='relu'),
    layers.Dropout(0.5),
    layers.Dense(10, activation='softmax')  # 10 classes
])

model.compile(
    optimizer='adam',
    loss='categorical_crossentropy',
    metrics=['accuracy']
)

print(f"Trainable parameters: {sum([tf.size(w).numpy() for w in model.trainable_weights])}")
print(f"Non-trainable parameters: {sum([tf.size(w).numpy() for w in model.non_trainable_weights])}")

# Fine-tuning: Unfreeze some layers
base_model.trainable = True
for layer in base_model.layers[:-4]:
    layer.trainable = False

model.compile(
    optimizer=tf.keras.optimizers.Adam(1e-5),  # Lower learning rate
    loss='categorical_crossentropy',
    metrics=['accuracy']
)

import numpy as np
import matplotlib.pyplot as plt

# Simple gradient descent implementation
def gradient_descent(X, y, learning_rate=0.01, epochs=1000):
    m, n = X.shape
    theta = np.zeros(n)
    losses = []
    
    for epoch in range(epochs):
        # Predictions
        predictions = X.dot(theta)
        
        # Loss (MSE)
        loss = np.mean((predictions - y) ** 2)
        losses.append(loss)
        
        # Gradient
        gradient = (2/m) * X.T.dot(predictions - y)
        
        # Update parameters
        theta -= learning_rate * gradient
    
    return theta, losses

# Generate data
np.random.seed(42)
X = np.random.rand(100, 1)
y = 2 * X.squeeze() + 1 + np.random.randn(100) * 0.1

# Add bias term
X_b = np.c_[np.ones((100, 1)), X]

# Run gradient descent
theta, losses = gradient_descent(X_b, y, learning_rate=0.1, epochs=1000)

print(f"Learned parameters: {theta}")
print(f"Final loss: {losses[-1]:.4f}")

# Compare with different learning rates
for lr in [0.001, 0.01, 0.1]:
    _, losses_lr = gradient_descent(X_b, y, learning_rate=lr, epochs=1000)
    print(f"LR={lr}: Final loss = {losses_lr[-1]:.4f}")

from tensorflow import keras
from tensorflow.keras import layers

# Model with Batch Normalization
model_bn = keras.Sequential([
    layers.Dense(128, input_shape=(784,)),
    layers.BatchNormalization(),
    layers.Activation('relu'),
    layers.Dense(64),
    layers.BatchNormalization(),
    layers.Activation('relu'),
    layers.Dense(10, activation='softmax')
])

# Model with Dropout
model_dropout = keras.Sequential([
    layers.Dense(128, activation='relu', input_shape=(784,)),
    layers.Dropout(0.5),
    layers.Dense(64, activation='relu'),
    layers.Dropout(0.3),
    layers.Dense(10, activation='softmax')
])

# Model with both
model_both = keras.Sequential([
    layers.Dense(128, input_shape=(784,)),
    layers.BatchNormalization(),
    layers.Activation('relu'),
    layers.Dropout(0.3),
    layers.Dense(64),
    layers.BatchNormalization(),
    layers.Activation('relu'),
    layers.Dropout(0.2),
    layers.Dense(10, activation='softmax')
])

# Compile
for model in [model_bn, model_dropout, model_both]:
    model.compile(
        optimizer='adam',
        loss='sparse_categorical_crossentropy',
        metrics=['accuracy']
    )

print("Batch Normalization model:")
model_bn.summary()