Theory Notebook

Converted from theory.ipynb for web reading.

Probability Measure Spaces

Probability measure spaces turn randomness into measure, random variables into measurable maps, and distributions into pushforward measures.

This notebook is the executable companion to notes.md. It uses finite spaces and synthetic measures so the measure-theoretic objects are visible.

Code cell 2

import numpy as np
import matplotlib.pyplot as plt
import matplotlib as mpl

try:
    import seaborn as sns
    sns.set_theme(style="whitegrid", palette="colorblind")
    HAS_SNS = True
except ImportError:
    plt.style.use("seaborn-v0_8-whitegrid")
    HAS_SNS = False

mpl.rcParams.update({
    "figure.figsize":    (10, 6),
    "figure.dpi":         120,
    "font.size":           13,
    "axes.titlesize":      15,
    "axes.labelsize":      13,
    "xtick.labelsize":     11,
    "ytick.labelsize":     11,
    "legend.fontsize":     11,
    "legend.framealpha":   0.85,
    "lines.linewidth":      2.0,
    "axes.spines.top":     False,
    "axes.spines.right":   False,
    "savefig.bbox":       "tight",
    "savefig.dpi":         150,
})
np.random.seed(42)
print("Plot setup complete.")

Code cell 3

COLORS = {
    "primary":   "#0077BB",
    "secondary": "#EE7733",
    "tertiary":  "#009988",
    "error":     "#CC3311",
    "neutral":   "#555555",
    "highlight": "#EE3377",
}

def header(title):
    print("\n" + "=" * 72)
    print(title)
    print("=" * 72)

def check_true(condition, name):
    ok = bool(condition)
    print(f"{'PASS' if ok else 'FAIL'} - {name}")
    assert ok, name

def check_close(value, target, tol=1e-8, name="value"):
    ok = abs(float(value) - float(target)) <= tol
    print(f"{'PASS' if ok else 'FAIL'} - {name}: got {float(value):.6f}, expected {float(target):.6f}")
    assert ok, name

def powerset(universe):
    items = tuple(universe)
    out = [frozenset()]
    for item in items:
        out += [s | {item} for s in out]
    return set(out)

def sigma_generated(universe, generators):
    universe = frozenset(universe)
    family = {frozenset(), universe} | {frozenset(g) for g in generators}
    changed = True
    while changed:
        old = set(family)
        family |= {universe - a for a in old}
        for a in old:
            for b in old:
                family.add(a | b)
                family.add(a & b)
        changed = len(family) != len(old)
    return family

def is_sigma_algebra(universe, family):
    universe = frozenset(universe)
    family = {frozenset(a) for a in family}
    if frozenset() not in family or universe not in family:
        return False
    for a in list(family):
        if universe - a not in family:
            return False
        for b in list(family):
            if a | b not in family:
                return False
    return True

def simple_integral(values, masses):
    values = np.asarray(values, dtype=float)
    masses = np.asarray(masses, dtype=float)
    return float(np.sum(values * masses))

def pushforward(prob, mapping):
    out = {}
    for omega, mass in prob.items():
        y = mapping[omega]
        out[y] = out.get(y, 0.0) + float(mass)
    return out

def product_measure(p, q):
    return {(a, b): float(pa) * float(qb) for a, pa in p.items() for b, qb in q.items()}

def rn_derivative(target, proposal):
    ratio = {}
    for key, p_mass in target.items():
        q_mass = proposal.get(key, 0.0)
        if p_mass > 0 and q_mass <= 0:
            raise ValueError("target is not absolutely continuous with respect to proposal")
        ratio[key] = 0.0 if q_mass == 0 else float(p_mass) / float(q_mass)
    return ratio

def expectation_under(prob, values):
    return float(sum(prob[k] * values[k] for k in prob))

print("Measure-theory helpers ready.")

Demo 1: Probability as measure with total mass one

This demo turns the approved TOC concept into a finite, executable measure-theory calculation.

Code cell 5

header("Demo 1 - Probability as measure with total mass one: generated sigma algebra")
universe = {0, 1, 2, 3}
generators = [{0, 1}]
family = sigma_generated(universe, generators)
print("Generated family:", sorted([sorted(s) for s in family]))
check_true(is_sigma_algebra(universe, family), "generated family is a sigma algebra")
print("Number of measurable events:", len(family))

Demo 2: Sample spaces events and observables

This demo turns the approved TOC concept into a finite, executable measure-theory calculation.

Code cell 7

header("Demo 2 - Sample spaces events and observables: measurable observation")
omega = ["raw_a", "raw_b", "raw_c", "raw_d"]
scores = {"raw_a": "safe", "raw_b": "safe", "raw_c": "risky", "raw_d": "risky"}
event = {w for w in omega if scores[w] == "risky"}
family = sigma_generated(omega, [event])
print("Risky preimage:", sorted(event))
check_true(frozenset(event) in family, "threshold event is measurable")
print("Observation-generated event count:", len(family))

Demo 3: Random variables as measurable maps

This demo turns the approved TOC concept into a finite, executable measure-theory calculation.

Code cell 9

header("Demo 3 - Random variables as measurable maps: simple-function integral")
values = np.array([0.2, 0.5, 1.5, 2.0])
masses = np.array([0.1, 0.2, 0.3, 0.4])
integral = simple_integral(values, masses)
print("Integral:", integral)
check_close(integral, 1.37, name="finite Lebesgue integral")
print("Interpretation: expected loss under a finite empirical measure.")

Demo 4: Laws and distributions as pushforward measures

This demo turns the approved TOC concept into a finite, executable measure-theory calculation.

Code cell 11

header("Demo 4 - Laws and distributions as pushforward measures: dominated convergence toy")
x = np.linspace(0, 1, 1001)
dx = x[1] - x[0]
integrals = []
for n in [1, 2, 5, 10, 20, 50]:
    f_n = x ** n
    integrals.append(float(np.sum(f_n) * dx))
print("Integrals of x^n:", [round(v, 4) for v in integrals])
check_true(integrals[-1] < integrals[0], "integrals shrink toward zero")
fig, ax = plt.subplots()
ax.plot([1, 2, 5, 10, 20, 50], integrals, color=COLORS["primary"], marker="o", label="Integral")
ax.set_title("Dominated convergence toy: integral of x^n")
ax.set_xlabel("n")
ax.set_ylabel("Approximate integral")
ax.legend()
fig.tight_layout()
plt.show()
plt.close(fig)
print("Interpretation: domination allows limit and integral to agree.")

Demo 5: Why ML often hides the base space

This demo turns the approved TOC concept into a finite, executable measure-theory calculation.

Code cell 13

header("Demo 5 - Why ML often hides the base space: pushforward law")
prob = {"a": 0.15, "b": 0.35, "c": 0.25, "d": 0.25}
mapping = {"a": "short", "b": "short", "c": "long", "d": "long"}
law = pushforward(prob, mapping)
print("Pushforward law:", law)
check_close(sum(law.values()), 1.0, name="pushforward total mass")
print("Interpretation: feature maps transform probability measures.")

Demo 6: Probability space $(\Omega,\mathcal{F},P)$

This demo turns the approved TOC concept into a finite, executable measure-theory calculation.

Code cell 15

header("Demo 6 - Probability space $(\\Omega,\\mathcal{F},P)$: product measure independence")
p = {"H": 0.6, "T": 0.4}
q = {"red": 0.25, "blue": 0.75}
joint = product_measure(p, q)
print("Joint entries:", joint)
check_close(joint[("H", "blue")], 0.45, name="product probability")
print("Interpretation: independence is factorization of the joint measure.")

Demo 7: Random element $X:\Omega\to\mathcal{X}$

This demo turns the approved TOC concept into a finite, executable measure-theory calculation.

Code cell 17

header("Demo 7 - Random element $X:\\Omega\\to\\mathcal{X}$: Radon-Nikodym ratios")
target = {"a": 0.2, "b": 0.5, "c": 0.3}
proposal = {"a": 0.4, "b": 0.4, "c": 0.2}
ratio = rn_derivative(target, proposal)
print("dP/dQ:", ratio)
values = {"a": 1.0, "b": 2.0, "c": 4.0}
direct = expectation_under(target, values)
weighted = sum(proposal[k] * ratio[k] * values[k] for k in proposal)
print("Direct E_P[f]:", direct)
print("Weighted E_Q[f dP/dQ]:", weighted)
check_close(direct, weighted, name="change of measure identity")

Demo 8: Distribution law $P_X$ as pushforward

This demo turns the approved TOC concept into a finite, executable measure-theory calculation.

Code cell 19

header("Demo 8 - Distribution law $P_X$ as pushforward: support mismatch")
target = {"a": 0.5, "b": 0.5}
proposal = {"a": 1.0, "b": 0.0}
try:
    rn_derivative(target, proposal)
    ok = False
except ValueError as exc:
    print("Caught expected error:", exc)
    ok = True
check_true(ok, "support mismatch blocks dP/dQ")
print("Interpretation: absolute continuity is a support condition.")

Demo 9: Independence via product measures

This demo turns the approved TOC concept into a finite, executable measure-theory calculation.

Code cell 21

header("Demo 9 - Independence via product measures: generated sigma algebra")
universe = {0, 1, 2, 3}
generators = [{0, 1}]
family = sigma_generated(universe, generators)
print("Generated family:", sorted([sorted(s) for s in family]))
check_true(is_sigma_algebra(universe, family), "generated family is a sigma algebra")
print("Number of measurable events:", len(family))

Demo 10: Conditional probability preview

This demo turns the approved TOC concept into a finite, executable measure-theory calculation.

Code cell 23

header("Demo 10 - Conditional probability preview: measurable observation")
omega = ["raw_a", "raw_b", "raw_c", "raw_d"]
scores = {"raw_a": "safe", "raw_b": "safe", "raw_c": "risky", "raw_d": "risky"}
event = {w for w in omega if scores[w] == "risky"}
family = sigma_generated(omega, [event])
print("Risky preimage:", sorted(event))
check_true(frozenset(event) in family, "threshold event is measurable")
print("Observation-generated event count:", len(family))

Demo 11: Expectation as $\int X\,dP$

This demo turns the approved TOC concept into a finite, executable measure-theory calculation.

Code cell 25

header("Demo 11 - Expectation as $\\int X\\,dP$: simple-function integral")
values = np.array([0.2, 0.5, 1.5, 2.0])
masses = np.array([0.1, 0.2, 0.3, 0.4])
integral = simple_integral(values, masses)
print("Integral:", integral)
check_close(integral, 1.37, name="finite Lebesgue integral")
print("Interpretation: expected loss under a finite empirical measure.")

Demo 12: Joint laws and product spaces

This demo turns the approved TOC concept into a finite, executable measure-theory calculation.

Code cell 27

header("Demo 12 - Joint laws and product spaces: dominated convergence toy")
x = np.linspace(0, 1, 1001)
dx = x[1] - x[0]
integrals = []
for n in [1, 2, 5, 10, 20, 50]:
    f_n = x ** n
    integrals.append(float(np.sum(f_n) * dx))
print("Integrals of x^n:", [round(v, 4) for v in integrals])
check_true(integrals[-1] < integrals[0], "integrals shrink toward zero")
fig, ax = plt.subplots()
ax.plot([1, 2, 5, 10, 20, 50], integrals, color=COLORS["primary"], marker="o", label="Integral")
ax.set_title("Dominated convergence toy: integral of x^n")
ax.set_xlabel("n")
ax.set_ylabel("Approximate integral")
ax.legend()
fig.tight_layout()
plt.show()
plt.close(fig)
print("Interpretation: domination allows limit and integral to agree.")

Demo 13: Almost sure events and null sets

This demo turns the approved TOC concept into a finite, executable measure-theory calculation.

Code cell 29

header("Demo 13 - Almost sure events and null sets: pushforward law")
prob = {"a": 0.15, "b": 0.35, "c": 0.25, "d": 0.25}
mapping = {"a": "short", "b": "short", "c": "long", "d": "long"}
law = pushforward(prob, mapping)
print("Pushforward law:", law)
check_close(sum(law.values()), 1.0, name="pushforward total mass")
print("Interpretation: feature maps transform probability measures.")

Demo 14: Modes of convergence: a.s. in probability $L^p$ and distribution

This demo turns the approved TOC concept into a finite, executable measure-theory calculation.

Code cell 31

header("Demo 14 - Modes of convergence: a.s. in probability $L^p$ and distribution: product measure independence")
p = {"H": 0.6, "T": 0.4}
q = {"red": 0.25, "blue": 0.75}
joint = product_measure(p, q)
print("Joint entries:", joint)
check_close(joint[("H", "blue")], 0.45, name="product probability")
print("Interpretation: independence is factorization of the joint measure.")

Demo 15: Law of large numbers as measure-theoretic statement

This demo turns the approved TOC concept into a finite, executable measure-theory calculation.

Code cell 33

header("Demo 15 - Law of large numbers as measure-theoretic statement: Radon-Nikodym ratios")
target = {"a": 0.2, "b": 0.5, "c": 0.3}
proposal = {"a": 0.4, "b": 0.4, "c": 0.2}
ratio = rn_derivative(target, proposal)
print("dP/dQ:", ratio)
values = {"a": 1.0, "b": 2.0, "c": 4.0}
direct = expectation_under(target, values)
weighted = sum(proposal[k] * ratio[k] * values[k] for k in proposal)
print("Direct E_P[f]:", direct)
print("Weighted E_Q[f dP/dQ]:", weighted)
check_close(direct, weighted, name="change of measure identity")

Demo 16: Data-generating distribution $P_{\mathrm{data}}$

This demo turns the approved TOC concept into a finite, executable measure-theory calculation.

Code cell 35

header("Demo 16 - Data-generating distribution $P_{\\mathrm{data}}$: support mismatch")
target = {"a": 0.5, "b": 0.5}
proposal = {"a": 1.0, "b": 0.0}
try:
    rn_derivative(target, proposal)
    ok = False
except ValueError as exc:
    print("Caught expected error:", exc)
    ok = True
check_true(ok, "support mismatch blocks dP/dQ")
print("Interpretation: absolute continuity is a support condition.")

Demo 17: Training samples as i.i.d. random elements

This demo turns the approved TOC concept into a finite, executable measure-theory calculation.

Code cell 37

header("Demo 17 - Training samples as i.i.d. random elements: generated sigma algebra")
universe = {0, 1, 2, 3}
generators = [{0, 1}]
family = sigma_generated(universe, generators)
print("Generated family:", sorted([sorted(s) for s in family]))
check_true(is_sigma_algebra(universe, family), "generated family is a sigma algebra")
print("Number of measurable events:", len(family))

Demo 18: Generalization as population vs empirical risk

This demo turns the approved TOC concept into a finite, executable measure-theory calculation.

Code cell 39

header("Demo 18 - Generalization as population vs empirical risk: measurable observation")
omega = ["raw_a", "raw_b", "raw_c", "raw_d"]
scores = {"raw_a": "safe", "raw_b": "safe", "raw_c": "risky", "raw_d": "risky"}
event = {w for w in omega if scores[w] == "risky"}
family = sigma_generated(omega, [event])
print("Risky preimage:", sorted(event))
check_true(frozenset(event) in family, "threshold event is measurable")
print("Observation-generated event count:", len(family))

Demo 19: Stochastic kernels for models and policies

This demo turns the approved TOC concept into a finite, executable measure-theory calculation.

Code cell 41

header("Demo 19 - Stochastic kernels for models and policies: simple-function integral")
values = np.array([0.2, 0.5, 1.5, 2.0])
masses = np.array([0.1, 0.2, 0.3, 0.4])
integral = simple_integral(values, masses)
print("Integral:", integral)
check_close(integral, 1.37, name="finite Lebesgue integral")
print("Interpretation: expected loss under a finite empirical measure.")

Demo 20: Sequence models and infinite product spaces

This demo turns the approved TOC concept into a finite, executable measure-theory calculation.

Code cell 43

header("Demo 20 - Sequence models and infinite product spaces: dominated convergence toy")
x = np.linspace(0, 1, 1001)
dx = x[1] - x[0]
integrals = []
for n in [1, 2, 5, 10, 20, 50]:
    f_n = x ** n
    integrals.append(float(np.sum(f_n) * dx))
print("Integrals of x^n:", [round(v, 4) for v in integrals])
check_true(integrals[-1] < integrals[0], "integrals shrink toward zero")
fig, ax = plt.subplots()
ax.plot([1, 2, 5, 10, 20, 50], integrals, color=COLORS["primary"], marker="o", label="Integral")
ax.set_title("Dominated convergence toy: integral of x^n")
ax.set_xlabel("n")
ax.set_ylabel("Approximate integral")
ax.legend()
fig.tight_layout()
plt.show()
plt.close(fig)
print("Interpretation: domination allows limit and integral to agree.")

Demo 21: Probability as measure with total mass one

This demo turns the approved TOC concept into a finite, executable measure-theory calculation.

Code cell 45

header("Demo 21 - Probability as measure with total mass one: pushforward law")
prob = {"a": 0.15, "b": 0.35, "c": 0.25, "d": 0.25}
mapping = {"a": "short", "b": "short", "c": "long", "d": "long"}
law = pushforward(prob, mapping)
print("Pushforward law:", law)
check_close(sum(law.values()), 1.0, name="pushforward total mass")
print("Interpretation: feature maps transform probability measures.")

Demo 22: Sample spaces events and observables

This demo turns the approved TOC concept into a finite, executable measure-theory calculation.

Code cell 47

header("Demo 22 - Sample spaces events and observables: product measure independence")
p = {"H": 0.6, "T": 0.4}
q = {"red": 0.25, "blue": 0.75}
joint = product_measure(p, q)
print("Joint entries:", joint)
check_close(joint[("H", "blue")], 0.45, name="product probability")
print("Interpretation: independence is factorization of the joint measure.")

Demo 23: Random variables as measurable maps

This demo turns the approved TOC concept into a finite, executable measure-theory calculation.

Code cell 49

header("Demo 23 - Random variables as measurable maps: Radon-Nikodym ratios")
target = {"a": 0.2, "b": 0.5, "c": 0.3}
proposal = {"a": 0.4, "b": 0.4, "c": 0.2}
ratio = rn_derivative(target, proposal)
print("dP/dQ:", ratio)
values = {"a": 1.0, "b": 2.0, "c": 4.0}
direct = expectation_under(target, values)
weighted = sum(proposal[k] * ratio[k] * values[k] for k in proposal)
print("Direct E_P[f]:", direct)
print("Weighted E_Q[f dP/dQ]:", weighted)
check_close(direct, weighted, name="change of measure identity")

Demo 24: Laws and distributions as pushforward measures

This demo turns the approved TOC concept into a finite, executable measure-theory calculation.

Code cell 51

header("Demo 24 - Laws and distributions as pushforward measures: support mismatch")
target = {"a": 0.5, "b": 0.5}
proposal = {"a": 1.0, "b": 0.0}
try:
    rn_derivative(target, proposal)
    ok = False
except ValueError as exc:
    print("Caught expected error:", exc)
    ok = True
check_true(ok, "support mismatch blocks dP/dQ")
print("Interpretation: absolute continuity is a support condition.")

Demo 25: Why ML often hides the base space

This demo turns the approved TOC concept into a finite, executable measure-theory calculation.

Code cell 53

header("Demo 25 - Why ML often hides the base space: generated sigma algebra")
universe = {0, 1, 2, 3}
generators = [{0, 1}]
family = sigma_generated(universe, generators)
print("Generated family:", sorted([sorted(s) for s in family]))
check_true(is_sigma_algebra(universe, family), "generated family is a sigma algebra")
print("Number of measurable events:", len(family))