Exercises Notebook

Converted from exercises.ipynb for web reading.

Determinants - Exercises

10 graded exercises covering the full linear algebra basics arc, from computation to ML-facing matrix workflows.

Difficulty: straightforward -> moderate -> challenging.

Code cell 2

import numpy as np
import matplotlib.pyplot as plt
import matplotlib as mpl

try:
    import seaborn as sns
    sns.set_theme(style="whitegrid", palette="colorblind")
    HAS_SNS = True
except ImportError:
    plt.style.use("seaborn-v0_8-whitegrid")
    HAS_SNS = False

mpl.rcParams.update({
    "figure.figsize":    (10, 6),
    "figure.dpi":         120,
    "font.size":           13,
    "axes.titlesize":      15,
    "axes.labelsize":      13,
    "xtick.labelsize":     11,
    "ytick.labelsize":     11,
    "legend.fontsize":     11,
    "legend.framealpha":   0.85,
    "lines.linewidth":      2.0,
    "axes.spines.top":     False,
    "axes.spines.right":   False,
    "savefig.bbox":       "tight",
    "savefig.dpi":         150,
})
np.random.seed(42)
print("Plot setup complete.")

Code cell 3

import numpy as np
import numpy.linalg as la
import scipy.linalg as sla
from scipy import stats

COLORS = {
    "primary": "#0077BB",
    "secondary": "#EE7733",
    "tertiary": "#009988",
    "error": "#CC3311",
    "neutral": "#555555",
    "highlight": "#EE3377",
}
HAS_MPL = True
np.set_printoptions(precision=8, suppress=True)
np.random.seed(42)

def header(title):
    print("\n" + "=" * len(title))
    print(title)
    print("=" * len(title))

def check_true(name, cond):
    ok = bool(cond)
    print(f"{'PASS' if ok else 'FAIL'} - {name}")
    return ok

def check_close(name, got, expected, tol=1e-8):
    ok = np.allclose(got, expected, atol=tol, rtol=tol)
    print(f"{'PASS' if ok else 'FAIL'} - {name}: got {got}, expected {expected}")
    return ok

def check(name, got, expected, tol=1e-8):
    return check_close(name, got, expected, tol=tol)

def softmax(z, axis=-1, tau=1.0):
    z = np.asarray(z, dtype=float) / float(tau)
    z = z - np.max(z, axis=axis, keepdims=True)
    e = np.exp(z)
    return e / np.sum(e, axis=axis, keepdims=True)

def cosine_similarity(a, b):
    a = np.asarray(a, dtype=float); b = np.asarray(b, dtype=float)
    return float(a @ b / (la.norm(a) * la.norm(b) + 1e-12))

def numerical_rank(A, tol=1e-10):
    return int(np.sum(la.svd(A, compute_uv=False) > tol))

def orthonormal_basis(A, tol=1e-10):
    Q, R = la.qr(A)
    keep = np.abs(np.diag(R)) > tol
    return Q[:, keep]

def null_space(A, tol=1e-10):
    U, S, Vt = la.svd(A)
    return Vt[S.size:,:].T if S.size < Vt.shape[0] else Vt[S <= tol,:].T



# Compatibility helpers used by the Chapter 02 theory and exercise cells.
def null_space(A, tol=1e-10):
    A = np.asarray(A, dtype=float)
    U, S, Vt = la.svd(A, full_matrices=True)
    rank = int(np.sum(S > tol))
    return Vt[rank:].T

svd_null_space = null_space

def gram_schmidt(vectors, tol=1e-10):
    A = np.asarray(vectors, dtype=float)
    if A.ndim == 1:
        A = A.reshape(1, -1)
    basis = []
    for v in A:
        w = v.astype(float).copy()
        for q in basis:
            w = w - np.dot(w, q) * q
        norm = la.norm(w)
        if norm > tol:
            basis.append(w / norm)
    return np.array(basis)

def projection_matrix_from_columns(A, tol=1e-10):
    Q = orthonormal_basis(np.asarray(A, dtype=float), tol=tol)
    return Q @ Q.T


def random_unit_vectors(n, d):
    X = np.random.randn(n, d)
    return X / np.maximum(la.norm(X, axis=1, keepdims=True), 1e-12)

def pairwise_distances(X):
    X = np.asarray(X, dtype=float)
    diff = X[:, None, :] - X[None, :, :]
    return la.norm(diff, axis=-1)


def normalize(x, axis=None, tol=1e-12):
    x = np.asarray(x, dtype=float)
    norm = la.norm(x, axis=axis, keepdims=True)
    return x / np.maximum(norm, tol)

def frobenius_inner(A, B):
    return float(np.sum(np.asarray(A, dtype=float) * np.asarray(B, dtype=float)))

def outer_sum_product(A, B):
    A = np.asarray(A, dtype=float)
    B = np.asarray(B, dtype=float)
    return sum(np.outer(A[:, k], B[k, :]) for k in range(A.shape[1]))

def softmax_rows(X):
    return softmax(X, axis=1)

def col_space(A, tol=1e-10):
    return orthonormal_basis(np.asarray(A, dtype=float), tol=tol)

def row_space(A, tol=1e-10):
    return orthonormal_basis(np.asarray(A, dtype=float).T, tol=tol).T

def rref(A, tol=1e-10):
    R = np.array(A, dtype=float, copy=True)
    m, n = R.shape
    pivots = []
    row = 0
    for col in range(n):
        pivot = row + int(np.argmax(np.abs(R[row:, col]))) if row < m else row
        if row >= m or abs(R[pivot, col]) <= tol:
            continue
        if pivot != row:
            R[[row, pivot]] = R[[pivot, row]]
        R[row] = R[row] / R[row, col]
        for r in range(m):
            if r != row:
                R[r] = R[r] - R[r, col] * R[row]
        pivots.append(col)
        row += 1
        if row == m:
            break
    R[np.abs(R) < tol] = 0.0
    return R, pivots

def nullspace_basis(A, tol=1e-10):
    A = np.asarray(A, dtype=float)
    U, S, Vt = la.svd(A, full_matrices=True)
    rank = int(np.sum(S > tol))
    return Vt[rank:].T, rank

print("Chapter helper setup complete.")

Exercise 1: Signed Area and Orientation *

For two vectors in the plane, the determinant of the matrix with those vectors as columns gives the signed area of the spanned parallelogram.

Task:

• Implement signed_area_2d(u, v)
• Return both the signed area and the orientation label: positive, negative, or degenerate
• Test it on one positively oriented pair, one negatively oriented pair, and one dependent pair

Code cell 5

# Your Solution
# Exercise 1 - learner workspace
# Write your solution here, then run the reference solution below to compare.
print("Learner workspace ready for Exercise 1.")

Code cell 6

# Solution
# Exercise 1 - reference solution

def signed_area_2d(u, v):
    u = np.array(u, dtype=float)
    v = np.array(v, dtype=float)
    area = u[0] * v[1] - u[1] * v[0]
    if np.isclose(area, 0.0):
        label = 'degenerate'
    elif area > 0:
        label = 'positive'
    else:
        label = 'negative'
    return area, label


header('Exercise 1 checks')
a1 = signed_area_2d([3, 1], [1, 2])
a2 = signed_area_2d([3, 1], [1, -2])
a3 = signed_area_2d([2, 1], [4, 2])
print(a1)
print(a2)
print(a3)
check_true('positive orientation detected', a1[1] == 'positive')
check_true('negative orientation detected', a2[1] == 'negative')
check_true('dependent pair detected', a3[1] == 'degenerate')
print('\nTakeaway: determinant sign tracks orientation, and absolute value tracks area.')

print("Exercise 1 solution complete.")

Exercise 2: Computation Strategy *

Choose the best determinant method for each matrix: direct formula, triangular-product rule, or elimination.

Task:

• Compute each determinant
• State which method is most natural and why
• Identify which matrix is singular

Code cell 8

# Your Solution
# Exercise 2 - learner workspace
# Write your solution here, then run the reference solution below to compare.
print("Learner workspace ready for Exercise 2.")

Code cell 9

# Solution
# Exercise 2 - reference solution

A = np.array([[4.0, 2.0], [1.0, 3.0]])
B = np.array([[2.0, -1.0, 4.0], [0.0, 3.0, 5.0], [0.0, 0.0, -2.0]])
C = np.array([[1.0, 2.0, 3.0], [2.0, 4.0, 6.0], [1.0, 0.0, 1.0]])
D = np.array([[2.0, 0.0, 0.0], [1.0, -3.0, 0.0], [4.0, 2.0, 5.0]])

header('Exercise 2 checks')
det_A = np.linalg.det(A)    # 2x2 closed form is natural
det_B = np.linalg.det(B)    # upper triangular -> product of diagonal
det_C = np.linalg.det(C)    # singular dense 3x3
det_D = np.linalg.det(D)    # lower triangular -> product of diagonal

print(f'det(A) = {det_A:.6f}  (2x2 formula)')
print(f'det(B) = {det_B:.6f}  (triangular product)')
print(f'det(C) = {det_C:.6f}  (dense; elimination or Sarrus)')
print(f'det(D) = {det_D:.6f}  (triangular product)')
check_true('C is singular', np.isclose(det_C, 0.0))
check_true('B is nonsingular', not np.isclose(det_B, 0.0))
check_true('D is nonsingular', not np.isclose(det_D, 0.0))
print('\nTakeaway: structure should decide the computation path before you start calculating.')

print("Exercise 2 solution complete.")

Exercise 3: Characteristic Polynomial and Cayley-Hamilton *

For

$$A = \begin{pmatrix}4 & 2 \\ 1 & 3\end{pmatrix},$$

compute the characteristic polynomial, the eigenvalues, and verify Cayley-Hamilton.

Code cell 11

# Your Solution
# Exercise 3 - learner workspace
# Write your solution here, then run the reference solution below to compare.
print("Learner workspace ready for Exercise 3.")

Code cell 12

# Solution
# Exercise 3 - reference solution

A = np.array([[2.0, 1.0], [3.0, 4.0]])

trace_A = np.trace(A)
det_A = np.linalg.det(A)
eigvals, eigvecs = np.linalg.eig(A)
cayley_residual = A @ A - trace_A * A + det_A * np.eye(2)

header('Exercise 3 checks')
print('trace(A) =', trace_A)
print('det(A)   =', det_A)
print('eigenvalues =', eigvals)
print('Cayley-Hamilton residual =\n', cayley_residual)
check_true('sum of eigenvalues matches trace', np.isclose(np.sum(eigvals), trace_A))
check_true('product of eigenvalues matches determinant', np.isclose(np.prod(eigvals), det_A))
check_close('Cayley-Hamilton', cayley_residual, np.zeros((2, 2)))
print('\nTakeaway: determinant is the constant-term spectral summary in the characteristic polynomial.')

print("Exercise 3 solution complete.")

Exercise 4: Cofactors, Adjugate, and the Inverse *

For

$$A = \begin{pmatrix}1 & 2 & 0 \\ 3 & 1 & 1 \\ 0 & 2 & 1\end{pmatrix},$$

compute the cofactor matrix, the adjugate, and reconstruct the inverse using

$$A^{-1} = \frac{\operatorname{adj}(A)}{\det(A)}.$$

Code cell 14

# Your Solution
# Exercise 4 - learner workspace
# Write your solution here, then run the reference solution below to compare.
print("Learner workspace ready for Exercise 4.")

Code cell 15

# Solution
# Exercise 4 - reference solution

A = np.array([[2.0, -1.0, 0.0], [1.0, 2.0, 1.0], [0.0, 1.0, 3.0]])

def cofactor_matrix(A):
    A = np.array(A, dtype=float)
    n = A.shape[0]
    C = np.zeros_like(A)
    for i in range(n):
        for j in range(n):
            sub = np.delete(np.delete(A, i, axis=0), j, axis=1)
            C[i, j] = ((-1) ** (i + j)) * np.linalg.det(sub)
    return C

C = cofactor_matrix(A)
adj = C.T
det_A = np.linalg.det(A)
A_inv = adj / det_A

header('Exercise 4 checks')
print('cofactor matrix =\n', C)
print('\nadjugate =\n', adj)
print('\ninverse from adjugate =\n', A_inv)
check_close('A @ adj(A) = det(A) I', A @ adj, det_A * np.eye(3))
check_close('adjugate inverse matches NumPy inverse', A_inv, np.linalg.inv(A))
print('\nTakeaway: cofactors are not just hand-computation artifacts; they encode the inverse identity.')

print("Exercise 4 solution complete.")

Exercise 5: SPD Matrices and Stable Log-Det **

For covariance matrices, the numerically correct quantity is almost always the log-determinant, preferably computed through Cholesky rather than direct determinant evaluation.

Code cell 17

# Your Solution
# Exercise 5 - learner workspace
# Write your solution here, then run the reference solution below to compare.
print("Learner workspace ready for Exercise 5.")

Code cell 18

# Solution
# Exercise 5 - reference solution

Sigma = np.array([[2.0, 0.4, 0.1], [0.4, 1.5, 0.2], [0.1, 0.2, 1.0]])

L = np.linalg.cholesky(Sigma)
logdet = 2.0 * np.sum(np.log(np.diag(L)))
eigvals = np.linalg.eigvalsh(Sigma)
logdet_eigs = np.sum(np.log(eigvals))

header('Exercise 5 checks')
print('Cholesky factor L =\n', L)
print('\nlog det via Cholesky =', logdet)
print('log det via eigenvalues =', logdet_eigs)
check_close('two log-det computations agree', logdet, logdet_eigs)
check_true('SPD determinant is positive', np.linalg.det(Sigma) > 0)
print('\nTakeaway: SPD structure turns determinant work into stable diagonal work after factorization.')

print("Exercise 5 solution complete.")

Exercise 6: Determinant Magnitude vs Conditioning **

A determinant can be tiny even when a matrix is perfectly well-conditioned. This exercise is about separating exact singularity logic from numerical stability logic.

Code cell 20

# Your Solution
# Exercise 6 - learner workspace
# Write your solution here, then run the reference solution below to compare.
print("Learner workspace ready for Exercise 6.")

Code cell 21

# Solution
# Exercise 6 - reference solution

A = 0.1 * np.eye(50)
B = np.diag(np.geomspace(1e-6, 1.0, 50))

det_A = np.linalg.det(A)
cond_A = np.linalg.cond(A)
det_B = np.linalg.det(B)
cond_B = np.linalg.cond(B)
sign_A, logabs_A = np.linalg.slogdet(A)

header('Exercise 6 checks')
print(f'det(0.1 I_50) = {det_A:.6e}')
print(f'cond(0.1 I_50) = {cond_A:.6f}')
print(f'slogdet(0.1 I_50) = (sign={sign_A}, logabsdet={logabs_A:.6f})')
print(f'\ndet(B) = {det_B:.6e}')
print(f'cond(B) = {cond_B:.6e}')
check_true('scaled identity is perfectly conditioned', np.isclose(cond_A, 1.0))
check_true('B is far more ill-conditioned than 0.1I', cond_B > cond_A * 1e3)
print('\nTakeaway: determinant magnitude alone is not a numerical conditioning diagnostic.')

print("Exercise 6 solution complete.")

Exercise 7: Matrix Determinant Lemma **

Use the rank-1 determinant update formula

$$\det(A + uv^T) = (1 + v^T A^{-1} u)\det(A)$$

to avoid recomputing a full determinant from scratch.

Code cell 23

# Your Solution
# Exercise 7 - learner workspace
# Write your solution here, then run the reference solution below to compare.
print("Learner workspace ready for Exercise 7.")

Code cell 24

# Solution
# Exercise 7 - reference solution

A = np.array([[3.0, 0.5], [0.5, 2.0]])
u = np.array([[1.0], [-0.3]])
v = np.array([[0.2], [0.7]])

lhs = np.linalg.det(A + u @ v.T)
rhs = (1.0 + (v.T @ np.linalg.inv(A) @ u).item()) * np.linalg.det(A)

header('Exercise 7 checks')
print('lhs =', lhs)
print('rhs =', rhs)
check_true('determinant lemma matches direct computation', np.isclose(lhs, rhs))
print('\nTakeaway: low-rank updates let you replace a big determinant with a tiny correction factor.')

print("Exercise 7 solution complete.")

Exercise 8: Sylvester and Schur Complement **

Verify two determinant identities that matter for block structure and rectangular products.

Code cell 26

# Your Solution
# Exercise 8 - learner workspace
# Write your solution here, then run the reference solution below to compare.
print("Learner workspace ready for Exercise 8.")

Code cell 27

# Solution
# Exercise 8 - reference solution

A_rect = np.array([[0.3, -0.2], [1.0, 0.5], [0.0, 0.7]])
B_rect = np.array([[0.2, 0.1, -0.4], [0.3, -0.2, 0.5]])
A_block = np.array([[3.0, 0.4], [0.4, 2.0]])
B_block = np.array([[0.5, -0.2], [0.1, 0.3]])
D_block = np.array([[2.5, 0.1], [0.1, 1.8]])

lhs_syl = np.linalg.det(np.eye(3) + A_rect @ B_rect)
rhs_syl = np.linalg.det(np.eye(2) + B_rect @ A_rect)

M = np.block([[A_block, B_block], [B_block.T, D_block]])
schur = D_block - B_block.T @ np.linalg.inv(A_block) @ B_block
lhs_schur = np.linalg.det(M)
rhs_schur = np.linalg.det(A_block) * np.linalg.det(schur)

header('Exercise 8 checks')
print('Sylvester lhs =', lhs_syl)
print('Sylvester rhs =', rhs_syl)
print('\nSchur lhs =', lhs_schur)
print('Schur rhs =', rhs_schur)
check_true('Sylvester identity holds', np.isclose(lhs_syl, rhs_syl))
check_true('Schur complement identity holds', np.isclose(lhs_schur, rhs_schur))
print('\nTakeaway: block and rectangular structure can move determinant work to smaller matrices.')

print("Exercise 8 solution complete.")

Exercise 9 (★★★): Log-Determinant and Gaussian Volume

For a positive definite covariance $\Sigma$, the log normalizer of a Gaussian includes

$$\frac{1}{2}\log\det\Sigma.$$

Compute it using both determinant and Cholesky factors.

Code cell 29

# Your Solution
# Exercise 9 - learner workspace
# Compare logdet from det and Cholesky diagonal.
print("Learner workspace ready for Exercise 9.")

Code cell 30

# Solution
# Exercise 9 - log determinant via Cholesky
header("Exercise 9: log determinant")

Sigma = np.array([[2.0, 0.4, 0.1], [0.4, 1.5, 0.2], [0.1, 0.2, 1.0]])
L = la.cholesky(Sigma)
logdet_direct = np.log(la.det(Sigma))
logdet_chol = 2 * np.sum(np.log(np.diag(L)))
print("logdet direct:", logdet_direct)
print("logdet Cholesky:", logdet_chol)
check_close("logdet agreement", logdet_direct, logdet_chol, tol=1e-12)
check_true("positive determinant", la.det(Sigma) > 0)
print("Takeaway: Cholesky logdet is stable and central to Gaussian likelihoods.")

Exercise 10 (★★★): Matrix Determinant Lemma

For invertible $A$ and vectors $u,v$,

$$\det(A+uv^\top)=\det(A)(1+v^\top A^{-1}u).$$

Verify the identity numerically and interpret it as a rank-one volume update.

Code cell 32

# Your Solution
# Exercise 10 - learner workspace
# Verify the determinant lemma for a rank-one update.
print("Learner workspace ready for Exercise 10.")

Code cell 33

# Solution
# Exercise 10 - matrix determinant lemma
header("Exercise 10: determinant lemma")

A = np.array([[3.0, 0.5], [0.5, 2.0]])
u = np.array([1.0, -0.3])
v = np.array([0.2, 0.7])
left = la.det(A + np.outer(u, v))
right = la.det(A) * (1 + v @ la.solve(A, u))
print("left:", left, "right:", right)
check_close("determinant lemma", left, right, tol=1e-12)
print("Takeaway: low-rank determinant updates avoid recomputing a full determinant.")

What to Review After Finishing

• Can you explain determinant as geometry before writing any formula?
• Do you know when to use cofactor expansion versus elimination?
• Can you separate exact invertibility from numerical conditioning?
• Can you derive eigenvalue relations from det(lambda I - A)?
• Do you understand why flow models are built around cheap Jacobian log-determinants?
• Can you recognize when low-rank or block structure makes determinant identities useful?

If you can do those six things cleanly, the chapter is doing its job.

References used in the chapter and notebook design:

• notes.md
• theory.ipynb
• MIT 18.06 Linear Algebra
• Stanford EE263
• Real NVP
• Glow
• FFJORD
• GPyTorch
• Determinantal Point Processes for Machine Learning