Exercises Notebook

Converted from exercises.ipynb for web reading.

Linear Transformations - Exercises

This notebook contains 10 progressive exercises for 04-Linear-Transformations. Each exercise has a learner workspace followed by a complete reference solution. Use the solution cells after making a serious attempt.

Difficulty grows from direct computation to AI-facing interpretation. Formulas use LaTeX-in-Markdown with $...$ and `

$$...$$

`.

Code cell 2

import numpy as np
import matplotlib.pyplot as plt
import matplotlib as mpl

try:
    import seaborn as sns
    sns.set_theme(style="whitegrid", palette="colorblind")
    HAS_SNS = True
except ImportError:
    plt.style.use("seaborn-v0_8-whitegrid")
    HAS_SNS = False

mpl.rcParams.update({
    "figure.figsize":    (10, 6),
    "figure.dpi":         120,
    "font.size":           13,
    "axes.titlesize":      15,
    "axes.labelsize":      13,
    "xtick.labelsize":     11,
    "ytick.labelsize":     11,
    "legend.fontsize":     11,
    "legend.framealpha":   0.85,
    "lines.linewidth":      2.0,
    "axes.spines.top":     False,
    "axes.spines.right":   False,
    "savefig.bbox":       "tight",
    "savefig.dpi":         150,
})
np.random.seed(42)
print("Plot setup complete.")

Code cell 3

import numpy as np
import numpy.linalg as la
import scipy.linalg as sla
from scipy import stats

np.set_printoptions(precision=8, suppress=True)
np.random.seed(42)

COLORS = {
    "primary": "#0077BB",
    "secondary": "#EE7733",
    "tertiary": "#009988",
    "error": "#CC3311",
    "neutral": "#555555",
    "highlight": "#EE3377",
}

def header(title):
    print("\n" + "=" * len(title))
    print(title)
    print("=" * len(title))

def check_true(name, cond):
    ok = bool(cond)
    print(f"{'PASS' if ok else 'FAIL'} - {name}")
    return ok

def check_close(name, got, expected, tol=1e-8):
    ok = np.allclose(got, expected, atol=tol, rtol=tol)
    print(f"{'PASS' if ok else 'FAIL'} - {name}")
    if not ok:
        print("  got     =", got)
        print("  expected=", expected)
    return ok

def softmax(z, axis=-1):
    z = np.asarray(z, dtype=float)
    z = z - np.max(z, axis=axis, keepdims=True)
    e = np.exp(z)
    return e / np.sum(e, axis=axis, keepdims=True)

def gram_schmidt_columns(A, tol=1e-12):
    A = np.asarray(A, dtype=float)
    Q = []
    for j in range(A.shape[1]):
        v = A[:, j].copy()
        for q in Q:
            v -= (q @ v) * q
        n = la.norm(v)
        if n > tol:
            Q.append(v / n)
    return np.column_stack(Q) if Q else np.empty((A.shape[0], 0))

def projection_matrix(A):
    Q = gram_schmidt_columns(A)
    return Q @ Q.T

def numerical_rank(A, tol=1e-10):
    return int(np.sum(la.svd(np.asarray(A, dtype=float), compute_uv=False) > tol))

def stable_rank(A):
    s = la.svd(np.asarray(A, dtype=float), compute_uv=False)
    return float(np.sum(s**2) / (s[0]**2 + 1e-15))

def make_spd(n, seed=0, ridge=0.5):
    rng = np.random.default_rng(seed)
    A = rng.normal(size=(n, n))
    return A.T @ A + ridge * np.eye(n)

print("Chapter 03 helper setup complete.")

Exercise 1: Linearity Test

Verify additivity and homogeneity for a candidate map $T$.

Code cell 5

# Your Solution
# Exercise 1 - learner workspace
# Write your solution here, then run the reference solution below to compare.
print("Learner workspace ready for Exercise 1.")

Code cell 6

# Solution
# Exercise 1 - Linearity Test
header("Exercise 1: linearity test")
A = np.array([[2.0, -1.0], [0.5, 3.0]])
def T(x): return A @ x
u, v, a = np.array([1.0, 2.0]), np.array([-3.0, 1.0]), 2.5
check_close("additivity", T(u+v), T(u)+T(v))
check_close("homogeneity", T(a*u), a*T(u))
print("Takeaway: every matrix map is linear.")

Exercise 2: Kernel and Image

Compute rank and nullity and verify rank-nullity.

Code cell 8

# Your Solution
# Exercise 2 - learner workspace
# Write your solution here, then run the reference solution below to compare.
print("Learner workspace ready for Exercise 2.")

Code cell 9

# Solution
# Exercise 2 - Kernel and Image
header("Exercise 2: kernel image")
A = np.array([[1.0, 2.0, 3.0], [2.0, 4.0, 6.0]])
U, s, Vt = la.svd(A, full_matrices=True)
r = numerical_rank(A)
N = Vt[r:].T
print("rank", r, "nullity", N.shape[1])
check_close("A @ null = 0", A @ N, np.zeros((A.shape[0], N.shape[1])))
check_true("rank-nullity", r + N.shape[1] == A.shape[1])

Exercise 3: Matrix in a New Basis

Compute $[T]_B=P^{-1}AP$ for a basis matrix $P$.

Code cell 11

# Your Solution
# Exercise 3 - learner workspace
# Write your solution here, then run the reference solution below to compare.
print("Learner workspace ready for Exercise 3.")

Code cell 12

# Solution
# Exercise 3 - Matrix in a New Basis
header("Exercise 3: change basis")
A = np.array([[2.0, 1.0], [0.0, 3.0]])
P = np.array([[1.0, 1.0], [1.0, -1.0]])
A_B = la.inv(P) @ A @ P
x_B = np.array([2.0, -1.0])
check_close("coordinate consistency", P @ (A_B @ x_B), A @ (P @ x_B))
print("matrix in basis B:\n", A_B)

Exercise 4: Composition

Show that composition of linear maps corresponds to matrix multiplication.

Code cell 14

# Your Solution
# Exercise 4 - learner workspace
# Write your solution here, then run the reference solution below to compare.
print("Learner workspace ready for Exercise 4.")

Code cell 15

# Solution
# Exercise 4 - Composition
header("Exercise 4: composition")
A = np.array([[1.0, 2.0], [0.0, 1.0]])
B = np.array([[2.0, 0.0], [1.0, 3.0]])
x = np.array([1.5, -0.5])
check_close("A(Bx) = (AB)x", A @ (B @ x), (A @ B) @ x)
print("Takeaway: matrix multiplication is function composition.")

Exercise 5: Projection and Reflection

Build a projection matrix and a reflection matrix from a unit vector.

Code cell 17

# Your Solution
# Exercise 5 - learner workspace
# Write your solution here, then run the reference solution below to compare.
print("Learner workspace ready for Exercise 5.")

Code cell 18

# Solution
# Exercise 5 - Projection and Reflection
header("Exercise 5: projection reflection")
u = np.array([1.0, 2.0]); u = u / la.norm(u)
P = np.outer(u, u)
R = 2*P - np.eye(2)
check_close("P idempotent", P @ P, P)
check_close("R orthogonal", R.T @ R, np.eye(2))
check_close("R keeps u", R @ u, u)

Exercise 6: Affine Map as Homogeneous Linear Map

Represent $x \mapsto Ax+b$ by a larger matrix.

Code cell 20

# Your Solution
# Exercise 6 - learner workspace
# Write your solution here, then run the reference solution below to compare.
print("Learner workspace ready for Exercise 6.")

Code cell 21

# Solution
# Exercise 6 - Affine Map as Homogeneous Linear Map
header("Exercise 6: homogeneous affine map")
A = np.array([[2.0, 0.0], [1.0, 1.0]])
b = np.array([0.5, -1.0])
H = np.block([[A, b[:, None]], [np.zeros((1,2)), np.ones((1,1))]])
x = np.array([3.0, 2.0])
yh = H @ np.r_[x, 1.0]
check_close("homogeneous map", yh[:2], A @ x + b)
check_close("last coordinate", yh[-1], 1.0)

Exercise 7: Jacobian Linearization

Compare a nonlinear map to its first-order linear approximation.

Code cell 23

# Your Solution
# Exercise 7 - learner workspace
# Write your solution here, then run the reference solution below to compare.
print("Learner workspace ready for Exercise 7.")

Code cell 24

# Solution
# Exercise 7 - Jacobian Linearization
header("Exercise 7: Jacobian")
def f(x): return np.array([x[0]**2 + x[1], np.sin(x[0]) + x[1]**2])
def J(x): return np.array([[2*x[0], 1.0], [np.cos(x[0]), 2*x[1]]])
x0 = np.array([1.0, 0.5]); dx = np.array([1e-4, -2e-4])
lin = f(x0) + J(x0) @ dx
true = f(x0 + dx)
print("linearization error:", la.norm(true-lin))
check_true("first-order error small", la.norm(true-lin) < 1e-7)

Exercise 8: Softmax Jacobian

Derive and verify $J=\operatorname{diag}(p)-pp^T$.

Code cell 26

# Your Solution
# Exercise 8 - learner workspace
# Write your solution here, then run the reference solution below to compare.
print("Learner workspace ready for Exercise 8.")

Code cell 27

# Solution
# Exercise 8 - Softmax Jacobian
header("Exercise 8: softmax Jacobian")
z = np.array([1.0, 0.5, -0.2])
p = softmax(z)
J = np.diag(p) - np.outer(p, p)
check_close("rows sum zero", J.sum(axis=1), np.zeros(3))
eps = 1e-6; v = np.array([0.2, -0.4, 0.1])
fd = (softmax(z + eps*v) - softmax(z - eps*v)) / (2*eps)
check_close("finite difference", J @ v, fd, tol=1e-8)

Exercise 9: LoRA as Linear Map

Show $W+BA$ changes outputs through a rank-limited correction.

Code cell 29

# Your Solution
# Exercise 9 - learner workspace
# Write your solution here, then run the reference solution below to compare.
print("Learner workspace ready for Exercise 9.")

Code cell 30

# Solution
# Exercise 9 - LoRA as Linear Map
header("Exercise 9: LoRA linear map")
rng = np.random.default_rng(5)
W = rng.normal(size=(6, 5)); B = rng.normal(size=(6, 2)); A = rng.normal(size=(2, 5))
Delta = B @ A
x = rng.normal(size=5)
check_close("low-rank correction output", (W+Delta)@x - W@x, Delta@x)
check_true("rank bound", la.matrix_rank(Delta) <= 2)

Exercise 10: Attention Projection Shapes

Track query, key, and value projections as linear transformations.

Code cell 32

# Your Solution
# Exercise 10 - learner workspace
# Write your solution here, then run the reference solution below to compare.
print("Learner workspace ready for Exercise 10.")

Code cell 33

# Solution
# Exercise 10 - Attention Projection Shapes
header("Exercise 10: attention projections")
rng = np.random.default_rng(6)
X = rng.normal(size=(4, 8)); Wq = rng.normal(size=(8, 3)); Wk = rng.normal(size=(8, 3)); Wv = rng.normal(size=(8, 5))
Q, K, V = X@Wq, X@Wk, X@Wv
Aatt = softmax(Q @ K.T / np.sqrt(Q.shape[1]), axis=1)
O = Aatt @ V
print("Q", Q.shape, "K", K.shape, "V", V.shape, "O", O.shape)
check_true("attention rows stochastic", np.allclose(Aatt.sum(axis=1), 1.0))
check_true("output feature dimension equals value dimension", O.shape == (4,5))