%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt
from matplotlib import rcParams

# Consistent, readable figures for every plot in this notebook.
rcParams['figure.figsize'] = (7, 4)
rcParams['figure.dpi'] = 110
rcParams['axes.grid'] = True
rcParams['grid.alpha'] = 0.3

print("imports ready")

imports ready

# A toy look at the four building blocks, written out explicitly.

# (3) A parameterized model f(x) with parameters w and b
def f(x, w, b):
    return w * x + b

# (1) A loss function: how wrong is the model on some data?
def mse_loss(x, y, w, b):
    return ((y - f(x, w, b)) ** 2).mean()

x_demo = np.array([1.0, 2.0, 3.0])
y_demo = np.array([2.0, 4.1, 5.9])   # roughly y = 2x

# (2) An optimization algorithm tweaks w to reduce the loss.
#     (Here we simply try several values and see which scores lowest.)
print("Trying different values of w (with b fixed at 0.5):")
for w_try in [0.5, 1.0, 1.5, 2.0, 2.5]:
    print(f"  w={w_try}: loss = {mse_loss(x_demo, y_demo, w_try, 0.5):.3f}")

# (4) An output: once we believe w=2 is best, the model predicts for new x
print("Prediction for x=4 with w=2, b=0.5:", f(4.0, 2.0, 0.5))

Trying different values of w (with b fixed at 0.5):
  w=0.5: loss = 7.657
  w=1.0: loss = 2.857
  w=1.5: loss = 0.390
  w=2.0: loss = 0.257
  w=2.5: loss = 2.457
Prediction for x=4 with w=2, b=0.5: 8.5

# Gradient descent from scratch on a simple convex (bowl-shaped) loss.
# Loss: L(w, b) = (w - 3)^2 + (b + 1)^2
# Its minimum is at w = 3, b = -1, where the loss is 0.

def loss(w, b):
    return (w - 3) ** 2 + (b + 1) ** 2

# The gradient: a vector of partial derivatives (points uphill).
def grad(w, b):
    dL_dw = 2 * (w - 3)   # dL/dw
    dL_db = 2 * (b + 1)   # dL/db
    return dL_dw, dL_db

# The gradient-descent loop: start somewhere, step downhill repeatedly.
def gradient_descent(eta, epochs, w0=0.0, b0=0.0):
    w, b = w0, b0
    history = [loss(w, b)]
    for _ in range(epochs):
        gw, gb = grad(w, b)
        w = w - eta * gw    # update rule:  w <- w - eta * gradient
        b = b - eta * gb
        history.append(loss(w, b))
    return w, b, history

# Run with a sensible learning rate.
w_opt, b_opt, hist = gradient_descent(eta=0.1, epochs=60)
print(f"Converged to w={w_opt:.4f}, b={b_opt:.4f}  (true minimum: w=3, b=-1)")
print(f"Final loss: {hist[-1]:.2e}")

fig, ax = plt.subplots()
ax.plot(hist, marker='o', ms=3)
ax.set_title("Gradient descent on $(w-3)^2 + (b+1)^2$  (learning rate $\eta=0.1$)")
ax.set_xlabel("Iteration")
ax.set_ylabel("Loss")
ax.set_yscale('log')
plt.show()

Converged to w=3.0000, b=-1.0000  (true minimum: w=3, b=-1)
Final loss: 2.35e-11

# Now use a learning rate that is TOO LARGE.
# For this quadratic, eta > 1 makes each step overshoot and climb the
# opposite side -- the loss grows instead of shrinks.

w_div, b_div, hist_div = gradient_descent(eta=1.1, epochs=30)

fig, ax = plt.subplots()
ax.plot(hist_div, marker='o', color='crimson', ms=4)
ax.set_title("Too-large learning rate ($\eta=1.1$): loss EXPLODES")
ax.set_xlabel("Iteration")
ax.set_ylabel("Loss")
plt.show()

print(f"Loss went from {hist_div[0]:.2e} to {hist_div[-1]:.2e} in 30 steps")

Loss went from 1.00e+01 to 5.63e+05 in 30 steps

# A tiny regression dataset: y = 1.8*x + 0.4 + small noise
np.random.seed(42)
x = np.linspace(0, 3, 30)
y = 1.8 * x + 0.4 + np.random.randn(30) * 0.15

N = len(x)
w, b = 0.0, 0.0          # start the parameters at zero
eta = 0.05               # learning rate
epochs = 200             # cap on passes over the data
losses = []

for epoch in range(epochs):
    preds = w * x + b
    error = y - preds
    # Gradients of the Mean Squared Error
    grad_w = (-2.0 / N) * np.sum(x * error)
    grad_b = (-2.0 / N) * np.sum(error)
    # Update rule
    w = w - eta * grad_w
    b = b - eta * grad_b
    # Record the loss AFTER this epoch's update
    losses.append(np.mean((y - (w * x + b)) ** 2))

print(f"Learned w={w:.3f}, b={b:.3f}  (true w=1.8, b=0.4)")
print(f"Final MSE: {losses[-1]:.4f}")

fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))
ax1.plot(losses)
ax1.set_title("Training loss (MSE) vs. epoch")
ax1.set_xlabel("Epoch"); ax1.set_ylabel("MSE")
ax2.scatter(x, y, s=25, label="data")
ax2.plot(x, w * x + b, 'r-', label=f"fit: y={w:.2f}x+{b:.2f}")
ax2.set_title("Data and learned regression line")
ax2.set_xlabel("x"); ax2.set_ylabel("y"); ax2.legend()
plt.show()

Learned w=1.746, b=0.454  (true w=1.8, b=0.4)
Final MSE: 0.0155

from sklearn.linear_model import SGDRegressor, LinearRegression

X = x.reshape(-1, 1)

# Practical version 1: scikit-learn's SGD-based regressor (same idea,
# library-grade, uses mini-batches internally).
sgd = SGDRegressor(learning_rate='constant', eta0=0.05, max_iter=200,
                   penalty=None, random_state=42)
sgd.fit(X, y)

# Practical version 2: the closed-form (normal equation) solution.
lr = LinearRegression().fit(X, y)

print("From-scratch GD  :  w = {:.3f}, b = {:.3f}".format(w, b))
print("SGDRegressor     :  w = {:.3f}, b = {:.3f}".format(
    sgd.coef_[0], sgd.intercept_[0]))
print("LinearRegression :  w = {:.3f}, b = {:.3f}".format(
    lr.coef_[0], lr.intercept_))

From-scratch GD  :  w = 1.746, b = 0.454
SGDRegressor     :  w = 1.753, b = 0.462
LinearRegression :  w = 1.749, b = 0.448

from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import make_pipeline
from sklearn.linear_model import LogisticRegression
from sklearn.datasets import make_classification

# A miniature end-to-end workflow on synthetic data.
X, y = make_classification(n_samples=300, n_features=4, n_informative=3,
                           n_redundant=0, random_state=42)

# 1) Split into train / test
X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=0.25,
                                          random_state=42)

# 2) Build a pipeline: scale features, then fit a model
pipe = make_pipeline(StandardScaler(),
                     LogisticRegression(max_iter=200, n_jobs=1))

# 3) Train (fit) on the training split
pipe.fit(X_tr, y_tr)

# 4) Evaluate on the held-out test split
print(f"Train accuracy: {pipe.score(X_tr, y_tr):.3f}")
print(f"Test accuracy : {pipe.score(X_te, y_te):.3f}")

Train accuracy: 0.849
Test accuracy : 0.867

# Exercise 7: gradient descent on a 1D quadratic
w = 0.0
eta = 0.1
for step in range(60):
    # TODO: gradient of L(w) = (w - 3)^2 is 2*(w - 3)
    grad = 0.0  # placeholder
    w -= eta * grad
print("final w =", w, " (expected ~3)")

# Exercise 8: divergence with too-large learning rate
w = 0.0
eta = 1.1
for step in range(10):
    loss = (w - 3) ** 2
    print(f"step {step}: w={w:.4f}, loss={loss:.4f}")
    # TODO: update w using grad = 2*(w - 3)
    grad = 0.0  # placeholder
    w -= eta * grad

# Exercise 9: from-scratch gradient descent for linear regression
import numpy as np
from sklearn.datasets import make_regression

X, y = make_regression(n_samples=100, n_features=1, noise=10, random_state=0)
x = X.ravel()
N = len(y)

w, b = 0.0, 0.0
eta = 0.1
epochs = 100

for epoch in range(epochs):
    # TODO: compute gradients and update w, b
    #   dL/dw = -(2/N) * sum( x_i * (y_i - w*x_i - b) )
    #   dL/db = -(2/N) * sum( y_i - w*x_i - b )
    dw = 0.0  # placeholder
    db = 0.0  # placeholder
    w -= eta * dw
    b -= eta * db

print(f"learned w={w:.3f}, b={b:.3f}")

# Exercise 10: compare GD vs LinearRegression vs SGDRegressor
import numpy as np
from sklearn.linear_model import LinearRegression, SGDRegressor
from sklearn.preprocessing import StandardScaler
from sklearn.datasets import make_regression

X, y = make_regression(n_samples=100, n_features=1, noise=10, random_state=0)

# Closed form
lr = LinearRegression().fit(X, y)
print("LinearRegression:", lr.coef_[0], lr.intercept_)

# SGD (scale features first)
scaler = StandardScaler().fit(X)
Xs = scaler.transform(X)
sgd = SGDRegressor(max_iter=1000, tol=1e-3, random_state=0).fit(Xs, y)
# TODO: convert SGD's slope/intercept back to the unscaled feature space, or just compare on scaled data
print("SGDRegressor (scaled space):", sgd.coef_[0], sgd.intercept_)

# TODO: paste your from-scratch GD w, b here and compare

# Exercise 11: mini end-to-end workflow
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LogisticRegression

X, y = make_classification(n_samples=500, n_features=5, n_informative=4,
                           n_redundant=0, random_state=0)
Xtr, Xte, ytr, yte = train_test_split(X, y, test_size=0.3, random_state=0)

# TODO: build make_pipeline(StandardScaler(), LogisticRegression(max_iter=500)) and fit on Xtr
pipe = None  # placeholder

# TODO: print .score on Xtr and Xte, and comment on what a big gap would mean

# Exercise 12: effect of feature scaling on kNN
import numpy as np
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
from sklearn.neighbors import KNeighborsClassifier
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import make_pipeline

X, y = make_classification(n_samples=400, n_features=3, n_informative=3,
                           n_redundant=0, random_state=0)
X_scaled_bad = X.copy()
X_scaled_bad[:, 0] *= 1000  # blow up one feature's scale

Xtr, Xte, ytr, yte = train_test_split(X_scaled_bad, y, test_size=0.3, random_state=0)

# TODO: fit a plain KNeighborsClassifier and a make_pipeline(StandardScaler(), KNeighborsClassifier)
# Print both test accuracies.
unscaled_acc = 0.0  # placeholder
scaled_acc = 0.0   # placeholder
print("unscaled test accuracy:", unscaled_acc)
print("scaled test accuracy  :", scaled_acc)

Variant	What it adds
SGD	Approximates the gradient with mini-batches — faster, slightly noisy
Momentum	Builds up velocity in a consistent direction to dampen oscillation
Adagrad	Adapts the learning rate per parameter based on gradient history
RMSprop	Adapts the rate using a moving average of squared gradients
Adam	Combines momentum + adaptive rates; the default choice for neural nets

Property	What it means	Example
Hyperparameters	Settings you choose before training	$C$ in SVM, tree depth, learning rate $\eta$
Input data type	Categorical vs. numerical features	Trees accept "red/yellow/green"; SVM & kNN need numbers
Class weighting	Let you care more about rare classes	SVM can up-weight a minority class
Output type	Label only vs. label + probability/score	SVM/kNN give a class; logistic regression gives a probability
Retraining	Rebuild from scratch vs. update incrementally	Trees/SVM/kNN rebuild; SGD models & Naive Bayes update online
Task types	Classification only, regression only, or both	Trees, SVM, kNN do both; logistic regression classifies
Speed & memory	Training/prediction time and RAM footprint	kNN is slow at predict time; trees are fast

Chapter 4 — Anatomy of a Learning Algorithm¶

4.1 Building Blocks of a Learning Algorithm¶

The four ingredients in plain words¶

4.2 Gradient Descent¶

Gradient descent for linear regression¶

4.3 How Machine Learning Engineers Work¶

4.4 Learning Algorithms' Particularities¶

Key Takeaways¶

What's Next¶

Exercises¶

Hands-On Coding Problems¶