Title: IB Statistics

Q: Bias-variance trade-off
A: The \emph{mean squared error} is
\begin{align*}
  \E_\theta[(\hat{\theta} - \theta)^2] &= \E_\theta[(\hat{\theta} - \E_\theta(\hat{\theta}) + \E_\theta(\hat{\theta}) - \theta)^2]\\
  &= \E_\theta[(\hat{\theta} - \E_\theta(\hat{\theta}))^2] + [\E_\theta(\hat{\theta}) - \theta]^2 + 2[\E_\theta(\hat{\theta}) - \theta]\E_\theta[\hat{\theta} - \E_\theta(\hat{\theta})]\\
  &= \var(\hat{\theta}) + \bias^2(\hat{\theta}).
\end{align*}

Q: Statistic
A: A function of sample data that is independent of the particular sample distribution

Q: Estimator
A: A statistic which is used to estimate a population parameter

Q: Estimate
A: The value of an estimator for a particular sample

Q: The factorization criterion
A: $T$ is sufficient for $\theta$ if and only if
  \begin{align*}
    f_{\mathbf{X}}(\mathbf{x} \mid \theta) = g(T(\mathbf{x}), \theta)h(\mathbf{x})
  \end{align*}
  for some functions $g$ and $h$.
\begin{proof}
($\Rightarrow$) If $f_\mathbf{X}(\mathbf{x}\mid \theta) = g(T(\mathbf{x}), \theta)h(\mathbf{x})$, then
  \begin{align*}
    f_{\mathbf{X}\mid T = t}(\mathbf{x}) &= \frac{\P_\theta(\mathbf{X} = \mathbf{x}, T(\mathbf{X}) = t)}{\P_\theta (T = t)}\\
    &= \frac{g(T(\mathbf{x}), \theta)h(\mathbf{x})}{\sum_{\{\mathbf{x}:T(\mathbf{x}) = t\}}g(T(\mathbf{x}), \theta)h(\mathbf{x})}\\
    &=\frac{g(t, \theta)h(\mathbf{x})}{g(t, \theta)\sum h(\mathbf{x})}\\
    &= \frac{h(\mathbf{x})}{\sum h(\mathbf{x})}
  \end{align*}
is independent of $\theta$.
($\Leftarrow$) If $T$ is sufficient, then
  \begin{align*}
    \P_\theta(\mathbf{X} = \mathbf{x}) &= \P_\theta(\mathbf{X} = \mathbf{x}, T=T(\mathbf{x})) \\
&= \P(\mathbf{X} = \mathbf{x}\mid T = T(\mathbf{x}))\P_\theta (T = T(\mathbf{x})) \\
&= h(\mathbf{x}) g(T(\mathbf{x}), \theta).
  \end{align*}
\end{proof}

Q: Minimal statistic criterion
A: Suppose $T = T(\mathbf{X})$ is a statistic that satisfies
  \begin{center}
    $\displaystyle\frac{f_\mathbf{X}(\mathbf{x}; \theta)}{f_\mathbf{X}(\mathbf{y}; \theta)}$ is independent of $\theta$ $\iff$ $T(\mathbf{x}) = T(\mathbf{y})$.
  \end{center}
  Then $T$ is minimally sufficient for $\theta$.
\begin{proof}
\textbf{Sufficient}. Firstly, for each possible $t$, pick a favourite $\mathbf{x}_t$ such that $T(\mathbf{x}_t) = t$.
  Now let $\mathbf{x}\in \mathcal{X}^N$ and let $T(\mathbf{x}) = t$. So $T(\mathbf{x}) = T(\mathbf{x}_t)$ and thus
  \begin{align*}
    f_\mathbf{X}(\mathbf{x}; \theta) = f_\mathbf{X}(\mathbf{x}_t; \theta) \frac{f_\mathbf{X}(\mathbf{x}; \theta)}{f_\mathbf{X}(\mathbf{x}_t; \theta)} = g(t, \theta) h(\mathbf{x}).
  \end{align*}
\textbf{Minimal}. Suppose that $S(\mathbf{X})$ is also sufficient. Then $\exists g_S, h_S$ such that
  \begin{align*}
    f_\mathbf{X}(\mathbf{x}; \theta) = g_S(S(\mathbf{x}), \theta) h_S(\mathbf{x}).
  \end{align*}
If $S(\mathbf{x}) = S(\mathbf{y})$, then
  \begin{align*}
    \frac{f_\mathbf{X}(\mathbf{x}; \theta)}{f_\mathbf{X}(\mathbf{y}; \theta)} = \frac{g_S(S(\mathbf{x}), \theta)h_S(\mathbf{x})}{g_S(S(\mathbf{y}), \theta)h_S(\mathbf{y})} = \frac{h_S(\mathbf{x})}{h_S(\mathbf{y})}
  \end{align*}
 is independent of $\theta$. So $T(\mathbf{x}) = T(\mathbf{y})$.
\end{proof}

Q: Rao-Blackwell Theorem
A: Let $T$ be a sufficient statistic for $\theta$ and let $\tilde{\theta}$ be an estimator for $\theta$ with $\E(\tilde{\theta}^2) < \infty$ for all $\theta$. Let $\hat{\theta}(\mathbf{x}) = \E[\tilde{\theta}(\mathbf{X})\mid T(\mathbf{X}) = T(\mathbf{x})]$. Then $\bias(\hat\theta)=\bias(\tilde\theta)$ and for all $\theta$,
  \begin{align*}
    \E[(\hat{\theta} - \theta)^2] \leq \E[(\tilde{\theta} - \theta)^2].
  \end{align*}
  The inequality is strict unless $\tilde{\theta}$ is a function of $T$.
\begin{proof}
\textbf{Bias}. $\E(\hat{\theta}) = \E[\E(\tilde{\theta}\mid T)] = \E(\tilde{\theta})$.
\textbf{Variance}.  By the conditional variance formula
  \begin{align*}
    \var(\tilde{\theta}) &= \E[\var (\tilde{\theta}\mid T)] + \var [\E(\tilde{\theta}\mid T)] \\
&= \E[\var(\tilde{\theta}\mid T)] + \var(\hat{\theta}) \\
& \geq \var (\hat{\theta})
  \end{align*}
with equality only if $\var(\tilde{\theta}\mid T) = 0$.
\end{proof}

Q: Conditional variance formula
A: $\var X = \E[\var X | Y] + \var\E[X|Y]$.
\begin{proof}
\begin{align*}
\var X &= \E[X^2]-\E[X]^2 \\
&= \E[\E[X^2 | Y]] - (\E[\E[X |Y]])^2 \\
&=  \E[\E[X^2 | Y]] - \E[\E[X|Y]^2] + \E[\E[X|Y]^2] - (\E[\E[X |Y]])^2 \\
&= \E[\var X | Y] + \var\E[X|Y].
\end{align*}
\end{proof}

Q: Pivot for Bernoulli probability
A: Suppose $X_1, \cdots, X_n$ are iid $\Bernoulli(p)$.
  The mle of $p$ is $\hat p = \sum X_i/n$.
  By the Central Limit theorem, $\displaystyle \frac{\sqrt{n}(\hat{p} - p)}{\sqrt{p(1 - p)}}$ is approximately $N(0, 1)$ for large $n$. For a conservative confidence interval, we can take the pivot
\begin{align*}
 \frac{\sqrt{n}(\hat{p} - p)}{1/2}
\end{align*}
and for a generous confidence interval we can use
\begin{align*}
 \frac{\sqrt{n}(\hat{p} - p)}{\sqrt{\hat{p}(1 - \hat{p})}}.
\end{align*}

Q: Laplace's Rule of Succession
A: Let $X_i\sim B(1,p)$ be a sequence of Bernoulli trials and $\mathbf{X}=(X_i)_{i=1}^n$ be the first $n$ trials. Then the likelihood of $p$ is\vspace{-0.2in}
\begin{align*}
    f_{\mathbf{X}}(\mathbf{x}\mid p) =p^{\sum x_i}(1 - p)^{n - \sum x_i}=p^{s}(1 - p)^{n - s}.
  \end{align*}
By the `principle of insufficient reason' we take the Laplace prior giving posterior distribution
\begin{align*}
\pi(p | \mathbf{x}) \propto p^{s}(1 - p)^{n - s}.
\end{align*}
Recognising Beta$(s+1,n-s+1)$ we have the posterior
\begin{align*}
\pi(p | \mathbf{x}) = \frac{p^{s}(1 - p)^{n - s}}{B(s+1,n-s+1)}
\end{align*}
with expected value equal to the Laplace estimator
\begin{align*}
\int_0^1 \pi(p| \mathbf{x})p \; \d x =  \frac{B(s+2,n-s+1)}{B(s+1,n-s+1)}=\frac{s+1}{n+2}.
\end{align*}
But by the law of total probability
\begin{align*}
\P(X_{n+1}=1 | \mathbf{X}=\mathbf{x}) = \int_0^1 \pi(p| \mathbf{x}) p\; \d x =  \frac{s+1}{n+2}.
\end{align*}

Q: Quadratic loss
A: The expected posterior loss
\begin{align*}
  h(a) = \int (a - \theta)^2 \pi(\theta\mid \mathbf{x})\;\d \theta.
\end{align*}
So $h'(a) = 0$ if
\begin{align*}
  \int (a - \theta) \pi(\theta\mid \mathbf{x})\;\d \theta = 0,
\end{align*}
or
\begin{align*}
  a=a \int \pi(\theta\mid \mathbf{x})\;\d \theta = \int \theta \pi(\theta\mid \mathbf{x})\;\d \theta.
\end{align*}
Then the Bayes estimator is $\hat{\theta} = \int \theta\pi(\theta\mid \mathbf{x})\;\d \theta$, the \emph{posterior mean}.

Q: Absolute error loss
A: The expected loss is
\begin{align*}
  h(a) &= \int |\theta - a|\pi(\theta\mid \mathbf{x})\;\d \theta\\
  &= \int_{-\infty}^a (a - \theta)\pi(\theta\mid \mathbf{x})\;\d \theta + \int_a^\infty (\theta - a)\pi(\theta\mid \mathbf{x})\;\d \theta.
\end{align*}
Differentiating under the integral sign
\begin{align*}
h'(a)=\int_{-\infty}^a \pi(\theta\mid \mathbf{x})\;\d \theta - \int_a^\infty \pi(\theta\mid \mathbf{x})\;\d \theta.
\end{align*}
So $h'(a)=0$ when each integral is $1/2$. So the Bayes estimator is the \emph{posterior median}.

Q: Simple and composite hypotheses
A: A \emph{simple hypothesis} $H$ specifies $f$ completely (i.e. $H: \theta = \theta_0$).
Otherwise, $H$ is a \emph{composite hypothesis}.

Q: Size and power of a simple test
A: When $H_0$ and $H_1$ are both simple, let
\begin{align*}
  \alpha = \P(\text{Type I error}) = \P(\mathbf{X}\in C\mid H_0).
\end{align*}
\begin{align*}
  \beta = \P(\text{Type II error}) = \P(\mathbf{X}\not\in C\mid H_1).
\end{align*}
The \emph{size} of the test is $\alpha$, and $1 - \beta$ is the \emph{power} of the test to detect $H_1$.

Q: LR test
A: The \emph{likelihood} of a simple hypothesis $H: \theta = \theta^*$ given data $\mathbf{x}$ is
  \begin{align*}
    L_\mathbf{x}(H) = f_\mathbf{X}(\mathbf{x}\mid \theta = \theta^*).
  \end{align*}
  The \emph{likelihood ratio} of two simple hypotheses $H_0, H_1$ given data $\mathbf{x}$ is
  \begin{align*}
    \Lambda_\mathbf{x}(H_0; H_1) = \frac{L_\mathbf{x}(H_1)}{L_\mathbf{x}(H_0)}.
  \end{align*}
  A \emph{likelihood ratio test} is one where the critical region $C$ is of the form
  \begin{align*}
    C = \{\mathbf{x}: \Lambda_\mathbf{x} (H_0; H_1) > k\}
  \end{align*}
  for some $k$. Note any $\Lambda_\mathbf{x}$ is a function $f$ of the sufficient statistic $T(\mathbf{x})$. If $f$ is increasing, then $\Lambda_\mathbf{x} > k \iff T(\mathbf{x})>t_0$ for some $t_0$.

Q: Neyman-Pearson lemma
A: Suppose $H_0: f = f_0$, $H_1: f = f_1$, where $f_0$ and $f_1$ are continuous densities that are nonzero on the same regions. Then the test with largest power and size $\leq\alpha$ is the LR test of size $\alpha$.
\begin{proof}
  Under the LR test, our critical region is
  \begin{align*}
    C = \{\mathbf{x}: f_1(\mathbf{x})/f_0(\mathbf{x}) > k\},
  \end{align*}
such that $\alpha= \int_C f_0(\mathbf{x})\;\d \mathbf{x}$ and $\beta = \int_{\bar C}f_1(\mathbf{x})\;\d \mathbf{x}$.
  Let $C^*$ be the critical region of any other test with size $\leq\alpha$. Let $\alpha^* = \P(X\in C^*\mid f_0)$ and $\beta^* = \P(\mathbf{X}\not\in C^*\mid f_1)$. Then
  \begin{align*}
    \beta - \beta^* &= \int_{\bar C}f_1(\mathbf{x}) \;\d \mathbf{x} - \int_{\bar C^*}f_1(\mathbf{x})\;\d \mathbf{x}\\
    &= \int_{\bar C\cap C^*} f_1(\mathbf{x}) \;\d \mathbf{x} + \cancel{\int_{\bar C\cap \bar C^*}f_1(\mathbf{x})\;\d \mathbf{x}} \\
    &\quad - \int_{\bar C^*\cap C} f_1(\mathbf{x}) \;\d \mathbf{x} - \cancel{\int_{\bar C^*\cap\bar C}f_1(\mathbf{x})\;\d \mathbf{x}}\\
    &= \int_{\bar C \cap C^*}f_1(\mathbf{x})\;\d \mathbf{x} - \int_{\bar C^*\cap C}f_1(\mathbf{x})\;\d \mathbf{x}\\
    &\leq k\int_{\bar C \cap C^*}f_0(\mathbf{x})\;\d \mathbf{x} - k\int_{\bar C^*\cap C}f_0(\mathbf{x})\;\d \mathbf{x}\\
    &= k\left\{\int_{\bar C\cap C^*}f_0(\mathbf{x})\;\d \mathbf{x} + \cancel{\int_{C \cap C^*}f_0(\mathbf{x})\;\d \mathbf{x}}\right\} \\
    &\quad- k\left\{\int_{\bar C^*\cap C}f_0(\mathbf{x})\;\d \mathbf{x} + \cancel{\int_{C\cap C^*}f_0(\mathbf{x})\;\d \mathbf{x}}\right\}\\
    &= k(\alpha^* - \alpha)\\
    &\leq 0.
  \end{align*}
\end{proof}

Q: Z-test
A: A hypothesis test where the test statistic is normal given the null hypothesis.

Q: p-value
A: The \emph{$p$-value}, or ``observed significance level'', of observed data $\mathbf{x}$ wrt a test statistic $T$, is $P=\P(T(\mathbf{X})<T(\mathbf{x}) | H_0)\sim U[0,1]$.
\begin{proof}
Under the null hypothesis, assume $T$ has increasing cdf $F(t)$. Then, assuming $H_0$, $P=F(T)$ has cdf
\begin{align*}
\P(P\leq p) =\P(F(T)\leq F(t))=\P(T\leq t)=p.
\end{align*}
\end{proof}

Q: Power function and size
A: The \emph{power function} is
  \begin{align*}
    W(\theta) = \P(\mathbf{X}\in C\mid \theta) = \P(\text{reject }H_0\mid \theta),
  \end{align*}
The size is then
\begin{align*}
    \alpha =\sup_{\theta\in \Theta_0}W(\theta).
  \end{align*}
For $\theta\in \Theta_1$, $1 - W(\theta) = \P(\text{Type II error}\mid \theta)$.

Q: Uniformly most powerful test
A: A test specified by a critical region $C$ is \emph{uniformly most powerful} (UMP) size $\alpha$ test of $H_0:\theta\in \Theta_0$ against $H_1: \theta \in \Theta_1$ if
  \begin{enumerate}
    \item $\sup_{\theta\in \Theta_0} W(\theta) = \alpha$.
    \item For any other test $C^*$ with size $\leq \alpha$ and power function $W^*$, we have $W(\theta) \geq W^*(\theta)$ for all $\theta\in \Theta_1$.
  \end{enumerate}

Q: LR of composite hypotheses
A: The \emph{likelihood} of a composite hypothesis $H:\theta\in \Theta$ given data $\mathbf{x}$ is
  \begin{align*}
    L_x(H) = \sup_{\theta\in \Theta}f(\mathbf{x}\mid \theta).
  \end{align*}
Taking $\Theta_1 = \Theta$ we have
\begin{align*}
  \Lambda_\mathbf{x} (H_0; H_1) = \frac{L_\mathbf{x}(H_1)}{L_\mathbf{x}(H_0)}=\frac{\sup_{\theta\in \Theta_1}f(\mathbf{x}\mid \theta)}{\sup_{\theta\in \Theta_0}f(\mathbf{x}\mid \theta)} \geq 1,
\end{align*}
with large values of $\Lambda$ indicating departure from $H_0$.

Q: Wilk's Theorem
A: If $\Theta = \{\theta: \theta = (\theta_1, \cdots, \theta_n)\}$ has $n$ free parameters and $\Theta_0$ has $n - p$ free parameters, e.g.
\begin{itemize}
  \item $H_0: \theta_{i_1} = a_1, \theta_{i_2} = a_2, \cdots \theta_{i_p} = a_p$; or
  \item $H_0: A\theta = \mathbf{b}$ (with $A$ $p\times n$, $\mathbf{b}$ $p\times 1$ given); or
  \item $H_0: \theta_i = f_i(\varphi), i = 1, \cdots, n$ for some $\varphi = (\varphi_1, \cdots, \varphi_{n - p})$,
\end{itemize}
then we write $|\Theta_0| = n - p$ and $|\Theta| = n$.
\begin{thm}
  Suppose $\Theta_0 \subseteq \Theta_1$ and $|\Theta_1| - |\Theta_0| = p$. Let $\mathbf{X} = (X_1, \cdots, X_n)$ with all $X_i$ iid. Then if $H_0$ is true, as $n\to \infty$,
  \begin{align*}
    2\log \Lambda_\mathbf{X}(H_0:H_1)\sim \chi_p^2.
  \end{align*}
In GLRT of size $\alpha$, we reject $H_0$ if $2\log \Lambda > \chi_p^2(\alpha)$.
\end{thm}

Q: Pearson's Chi-squared test
A: Let $(N_1,\cdots, N_k)\sim \multinomial(n; p_1, \cdots, p_n)$.
Under $H_0: p_i=p_i(\theta)$ and $H_1: p_i\geq 0, \sum_ip_i=1$. Hence $|\Theta_1| - |\Theta_0| = (k - 1)-|\Theta_0|$.
 Now $\like(p_1, \cdots, p_k) \propto p_1^{n_1}\cdots p_k^{n_k}$. So $\log\like = \text{constant} + \sum n_i \log p_i$. By Lagrange multipliers, subject to $\sum p_i = 1$, the mle is $\hat{p_i} = n_i/n$. So
 \begin{align*}
  2\log \Lambda = 2\log \left(\frac{\hat{p_1}^{n_1}\cdots \hat{p_k}^{n_k}}{p_1(\hat{\theta})^{n_1}\cdots p_k (\hat{\theta})^{n_1}}\right) = 2\sum n_i \log \left(\frac{n_i}{np_i(\hat{\theta})}\right).
\end{align*}
Let $o_i=n_i$ be observed, $e_i=np_i(\theta) $ be expected and $\delta_i = o_i - e_i$. Then
\begin{align*}
  2\log \Lambda &= 2\sum o_i \log \left(\frac{o_i}{e_i}\right)\\
  &= 2\sum (e_i + \delta_i) \log\left(1 + \frac{\delta_i}{e_i}\right)\\
  &= 2\sum (e_i + \delta_i) \left(\frac{\delta_i}{e_i} - \frac{\delta_i^2}{2e_i^2} + O(\delta_i^3)\right)\\
  &= 2\sum \left(\delta_i + \frac{\delta_i^2}{e_i} - \frac{\delta_i^2}{2e_i} + O(\delta_i^3)\right)\\
  \intertext{but $\sum \delta_i = 0$ since $n=\sum e_i = \sum o_i$ so}
  &\approx \sum \frac{\delta_i^2}{e_i}\\
  &= \sum\frac{(o_i - e_i)^2}{e_i}.
\end{align*}
So reject $H_0$ if $2\log \Lambda > \chi_{k - 1-|\Theta_0|}^2(\alpha)$.

Q: Contingency table
A: Consider a two-way contingency table with $r$ rows and $c$ columns. Let $p_{ij}$ be the probability that a sampled individual is classified in row $i$ and column $j$. So $p_{++} = 1$.
Suppose a random sample of $n$ individuals is taken and let $n_{ij}$ be the number of these classified in the $(i, j)$ cell of the table. So $n_{++} = n$.
We have that $(N_{ij}) \sim \multinomial (n; (p_{ij}))$.
We test $H_0: p_i = p_{i+}p_{+j}$ against $H_1: p_{++} = 1, p_{ij} \geq 0$.
Under $H_1$, the MLEs are $\hat{p_{ij}} = \frac{n_{ij}}{n}$.
Under $H_0$, the MLEs are $\hat{p}_{i+} = \frac{n_{i+}}{n}$ and $\hat{p}_{+j} = \frac{n_{+j}}{n}$.
Write $o_{ij} = n_{ij}$ and $e_{ij} = n\hat{p}_{i+}\hat{p}_{+j} = n_{i+}n_{+j}/n$.
Then
\begin{align*}
  2\log \Lambda = 2\sum_{i = 1}^r \sum_{j = 1}^c o_{ij}\log\left(\frac{o_{ij}}{e_{ij}}\right) \approx \sum_{i = 1}^r \sum_{j = 1}^c \frac{(o_{ij} - e_{ij})^2}{e_{ij}}.
\end{align*}
Now
\begin{align*}
  |\Theta_1| - |\Theta_0| = rc - 1 - [(r - 1) + (c - 1)] = (r - 1)(c - 1).
\end{align*}
So we compare to $\chi^2_{(r-1)(c-1)}(\alpha)$.

Q: Homogeneity test
A: We observe $(n_{ij})$, with $r$ rows and $c$ columns where
\begin{align*}
  (N_{i1}, \cdots, N_{ic}) \sim \multinomial(n_{i+}, p_{i1}, \cdots, p_{ic})
\end{align*}
independently for each $i = 1, \cdots, r$.
We test
\begin{align*}
  H_0: p_j:= p_{1j} = p_{2j} = \cdots = p_{rj},
\end{align*}
for $j = 1, \cdots, c$ and
\begin{align*}
  H_1: p_{ij}\geq 0, p_{i+}=1.
\end{align*}
Under $H_1$
\begin{align*}
  \log\like(p_{ij}) = \text{constant} + \sum_{i = 1}^r \sum_{j = 1}^c n_{ij}\log p_{ij}
\end{align*}
and $\hat{p}_{ij} = \frac{n_{ij}}{n_{i+}}$. Under $H_0$
\begin{align*}
  \log\like(p_j) = \text{constant} + \sum_{j = 1}^c n_{+j}\log p_j
\end{align*}
and $\hat{p}_j = \frac{n_{+j}}{n_{++}}$. So $o_{ij}= n_{ij}, e_{ij} = \frac{n_{i+}n_{+j}}{n_{++}}$.
Degrees of freedom are $|\Theta_1| - |\Theta_0| =  r(c - 1)- c - 1 = (r-1)(c-1)$.

Q: Confidence sets vs. Acceptance regions
A: \begin{thm}\leavevmode
  \begin{enumerate}
    \item Suppose that for every $\theta_0\in \Theta_0$ there is a size $\alpha$ test of $H_0: \theta = \theta_0$. Denote the acceptance region by $A(\theta_0)$. Then the set $I(\mathbf{X}) = \{\theta:\mathbf{X}\in A(\theta)\}$ is a $100(1 - \alpha)\%$ confidence set for $\theta$.
    \item Suppose $I(\mathbf{X})$ is a $100(1 - \alpha)\%$ confidence set for $\theta$. Then $A(\theta_0) = \{\mathbf{X}: \theta_0 \in I(\mathbf{X})\}$ is an acceptance region for a size $\alpha$ test of $H_0: \theta = \theta_0$.
  \end{enumerate}
\end{thm}
\begin{proof}
From their definitions
\begin{align*}
\theta_0\in I(\mathbf{X})\iff \mathbf{X}\in A(\theta_0)
\end{align*}
and thus
\begin{align*}
\P(\theta_0\in I(\mathbf{X})\mid \theta = \theta_0) =1-\alpha \iff \P(\mathbf{X}\in A(\theta_0)\mid \theta = \theta_0)=1- \alpha.
\end{align*}
\end{proof}

Q: Covariance of a linear combination
A: For any $A,B\in\Mat_{m,n}(\R)$, $X$ a 1 by $n$ vector of random variables
\begin{align*}
  \cov (A\mathbf{X}) = A\cov(\mathbf{X})A^T
\end{align*}
and
\begin{align*}
\cov(A\mathbf{X}, B\mathbf{X}) = A\cov (\mathbf{X}) B^T.
\end{align*}
\begin{proof}Using linearity of expectation
\begin{align*}
  \cov(A\mathbf{X}, B\mathbf{X}) &= \E[(A\mathbf{X} -\E[A\mathbf{X}])(B\mathbf{X} - \E[B\mathbf{X}])^T] \\
  &= \E[A(\mathbf{X} - \E\mathbf{X})(\mathbf{X} - \E\mathbf{X})^T B^T]\\
  &= A\E[(\mathbf{X} - \E\mathbf{X})(\mathbf{X} - \E\mathbf{X})^T]B^T \\
&=A\cov(\mathbf{X})B^T.
\end{align*}
Putting $B=A$ gives the first result.
\end{proof}

Q: Multivariate normal MGF
A: $\mathbf{X}$ is MVN if, for every $\mathbf{t}\in \R^n$, $\mathbf{t}^T\mathbf{X}$ is normal.
A normal $X\sim N(\mu, \sigma^2)$ has MGF
\begin{align*}
  M_X(s) = \E[e^{sX}] = \exp(\mu s + \frac{1}{2}\sigma^2 s^2).
\end{align*}
Hence for any $\mathbf{t}$, the MGF of $\mathbf{t}^T\mathbf{X}$ is given by
\begin{align*}
  M_{\mathbf{t}^T \mathbf{X}}(s) = \E[e^{s\mathbf{t}^T\mathbf{X}}] = \exp\left(\mathbf{t}^T\boldsymbol\mu s + \frac{1}{2}\mathbf{t}^T\Sigma \mathbf{t}s^2\right).
\end{align*}
Hence $\mathbf{X}$ has mgf
\begin{align*}
  M_\mathbf{X}(\mathbf{t}) = \E[e^{\mathbf{t}^T \mathbf{X}}]= M_{\mathbf{t}^T\mathbf{X}}(1) = \exp\left(\mathbf{t}^T\boldsymbol\mu + \frac{1}{2}\mathbf{t}^T \Sigma \mathbf{t}\right).
\end{align*}

Q: Multivariate normal PDF
A: Let $Z_1, \cdots, Z_n$ be iid $N(0, 1)$. Then their joint density is
\begin{align*}
f_\mathbf{Z}(\mathbf{z})= \frac{1}{(2\pi)^{n/2}}e^{-\frac{1}{2}\mathbf{z}^T \mathbf{z}}.
\end{align*}
Let $\mathbf{X} = \boldsymbol\mu + A\mathbf{Z}$ for $A\in\GL_n(\R)$. Then
\begin{align*}
 |J| = |\det (A^{-1})| = \frac{1}{|\det A|}
\end{align*}
and
\begin{align*}
\cov(\mathbf{X})=A\cov(\mathbf{Z})A^T=AIA^T=AA^T=\Sigma.
\end{align*}
So the joint pdf of $X_1,\ldots X_n$ is
\begin{align*}
  f_\mathbf{X}(\mathbf{x}) &= \frac{1}{(2\pi)^{n/2}}\frac{1}{|\det A|}\exp\left[-\frac{1}{2}\big((A^{-1}(\mathbf{x} - \boldsymbol\mu))^T(A^{-1}(\mathbf{x} - \boldsymbol\mu))\big)\right]\\
  &= \frac{1}{(2\pi)^{n/2}\sqrt{\det \Sigma}}\exp\left[-\frac{1}{2}(\mathbf{x} - \boldsymbol\mu)^T \Sigma^{-1}(\mathbf{x} - \boldsymbol\mu)\right]\\
  &= \frac{1}{(2\pi)^{n/2}\sqrt{\det \Sigma}}\exp\left[-\frac{1}{2}(\mathbf{x} - \boldsymbol\mu)^T \Sigma^{-1}(\mathbf{x} - \boldsymbol\mu)\right].
\end{align*}

Q: Linear combination of multivariate normals
A: \begin{prop}\leavevmode
If $\mathbf{X} \sim N_n(\mu, \Sigma)$, and $A$ is an $m\times n$ matrix, then $A\mathbf{X} \sim N_m (A\boldsymbol\mu, A\Sigma A^T)$.
\end{prop}
\begin{proof}
For any $\mathbf{t}\in\R^m$:
\begin{align*}
\mathbf{t}^TA\mathbf{X} = (A^T\mathbf{t})^T \mathbf{X}
\end{align*}
is normal.
\end{proof}

Q: Norm of multivariate normal
A: \begin{prop}\leavevmode
If $\mathbf{X}\sim N_n(\mathbf{0}, \sigma^2 I)$, then $\|\mathbf{X}\|^2 \sim \sigma^2 \chi_n^2$.
\end{prop}
\begin{proof}Since $X_i/\sigma_i\sim N(0,1)$ are independent
\begin{align*}
   \frac{\|\mathbf{X}\|^2}{\sigma^2} = \sum_{i=1}^n \frac{X_i^2}{\sigma ^2}\sim \chi_n^2.
\end{align*}
\end{proof}

Q: Multivariate normal: zero covariance
A: Suppose $\mathbf{X}\sim N_n(\boldsymbol\mu, \Sigma)$ where $\Sigma = \diag(\sigma_1^2, \cdots, \sigma_n^2)$. Then $X_1, \cdots, X_n$ are independent, with $X_i\sim N(\mu_i, \sigma_i^2)$.
\begin{proof}
We just observe the joint pdf factorises
\begin{align*}
  f_\mathbf{X}(\mathbf{x}) &= \frac{1}{(2\pi)^{n/2}\sqrt{\det \Sigma}}\exp\left[-\frac{1}{2}(\mathbf{x} - \boldsymbol\mu)^T \Sigma^{-1}(\mathbf{x} - \boldsymbol\mu)\right]\\
&= \prod_{i=1}^n \frac{1}{\sqrt{2\pi}\sigma_i} e^{-\frac{1}{2\sigma_i^2}(x_i - \mu_i)^2}.
\end{align*}
\end{proof}

Q: Multivariate normals split
A: \begin{prop}
  Let $\mathbf{X}\sim N_n(\mu, \Sigma)$. We split $\mathbf{X}$ up into two parts: $\mathbf{X} =
  \begin{pmatrix}
    \mathbf{X}_1\\
    \mathbf{X}_2
  \end{pmatrix}$, where $\mathbf{X}_i$ is a $n_i \times 1$ column vector and $n_1 + n_2 = n$.  Similarly write
  \begin{align*}
    \boldsymbol\mu =
    \begin{pmatrix}
      \boldsymbol\mu_1\\
      \boldsymbol\mu_2
    \end{pmatrix}
    ,\quad
    \Sigma =
    \begin{pmatrix}
      \Sigma_{11} & \Sigma_{12}\\
      \Sigma_{21} & \Sigma_{22}
    \end{pmatrix},
  \end{align*}
  where $\Sigma_{ij}$ is an $n_i\times n_j$ matrix. Then
  \begin{enumerate}
    \item $\mathbf{X}_i \sim N_{n_i}(\boldsymbol\mu_i, \Sigma_{ii})$
    \item $\mathbf{X}_1$ and $\mathbf{X}_2$ are independent iff $\Sigma_{12} = 0$.
  \end{enumerate}
\end{prop}
\begin{proof}\leavevmode
  \begin{enumerate}
    \item Given $\mathbf{t_i}\in\R^{n_i}$, let $\mathbf{t}$ be the projection in $\R^n$. Then $\mathbf{t_i}^T\mathbf{X}_i = \mathbf{t}^T\mathbf{X}$
is normal.
    \item Write $\mathbf{t} = (\mathbf{t}_1, \mathbf{t}_2)^T$. Then the mgf is
      \begin{align*}
        M_\mathbf{X}(\mathbf{t}) = \exp\Bigg( &\mathbf{t}_1^T\boldsymbol\mu_1 + \mathbf{t}_2^T\boldsymbol\mu_2 + \\
&\frac{1}{2}\mathbf{t}_1^T \Sigma_{11}\mathbf{t}_1 +\frac{1}{2}\mathbf{t}_1^T \Sigma_{12}\mathbf{t}_2 +\\
& \frac{1}{2}\mathbf{t}_2^T \Sigma_{21}\mathbf{t}_1 + \frac{1}{2}\mathbf{t}_2^T \Sigma_{22}\mathbf{t}_2 \Bigg).
      \end{align*}
 Whereas $M_{\mathbf{X}_i}(\mathbf{t}_i) = \exp(\mathbf{t}_i^T\boldsymbol\mu_i + \frac{1}{2}\mathbf{t}_i^T \Sigma_{ii}\mathbf{t}_i)$. So $M_\mathbf{X}(\mathbf{t}) = M_{\mathbf{X}_1}(\mathbf{t}_1)M_{\mathbf{X}_2}(\mathbf{t}_2)$ for all $\mathbf{t}$ if and only if $\Sigma_{12} = 0$.
  \end{enumerate}
\end{proof}

Q: Joint MGF
A: Let $\mathbf{X}=(X_1,\ldots X_n)$. Then
\begin{align*}
M_\mathbf{X}(\mathbf{t})= M_{X_1}(t_1)\ldots M_{X_n}(t_n)
\end{align*}
iff the $X_i$ are independent.
\begin{proof}
If $X_i$ are independent, then
\begin{align*}
M_\mathbf{X}(\mathbf{t})&=\E[\exp(\mathbf{t}^T\mathbf{x})] \\
&= \E[\exp(t_1x_1+\ldots t_nx_n) \\
&=\E[\exp(t_1x_1)\cdots\exp(t_nx_n)] \\
&=\E[\exp(t_1x_1)]\cdots\E[\exp(t_nx_n)] \\
&= M_{X_1}(t_1)\ldots M_{X_n}(t_n).
\end{align*}
Conversely, if the MGF factorises, then (by the above) the joint MGF is compatible with $X_i$ being independent. By uniqueness of joint MGFs, the $X_i$ must indeed be independent.
\end{proof}

Q: Covariance matrix
A: If $\Sigma = \cov(X)$, then $\Sigma$ is symmetric and is positive semi-definite.
\begin{proof}
For symmetry
\begin{align*}
\Sigma_{ij}=\cov(X_i,X_j)=\cov(X_j,X_i)=\Sigma_{ji}
\end{align*}
 and for positive semi-definiteness
\begin{align*}
  \mathbf{t}^T\Sigma \mathbf{t}=\mathbf{t}^T\cov(\mathbf{X}) \mathbf{t} = \cov(\mathbf{t}^T\mathbf{X})=\var(\mathbf{t}^T\mathbf{X}) \geq 0.
\end{align*}
\end{proof}

Q: Normal sample distribution
A: \begin{thm}
  Suppose $X_1, \cdots, X_n$ are iid $N(\mu, \sigma^2)$, $\bar X = \frac{1}{n} \sum X_i$, and $S_{XX} = \sum (X_i - \bar X)^2$. Then
  \begin{enumerate}
    \item $\bar X \sim N(\mu, \sigma^2/n)$
    \item $S_{XX}/\sigma^2 \sim \chi_{n - 1}^2$.
    \item $\bar X$ and $S_{XX}$ are independent.
  \end{enumerate}
\end{thm}
\begin{proof}
  We can write the joint density as $\mathbf{X} \sim N_n(\boldsymbol\mu, \sigma^2I)$, where $\boldsymbol\mu = (\mu, \mu, \cdots, \mu)$.
Pick $A\in O_n(\R)$ with the first row all $1/\sqrt{n}$, e.g.
  \begin{align*}
    A =
    \begin{pmatrix}
      \frac{1}{\sqrt{n}} & \frac{1}{\sqrt{n}} & \frac{1}{\sqrt{n}} & \frac{1}{\sqrt{n}} & \cdots & \frac{1}{\sqrt{n}}\\
      \frac{1}{\sqrt{2\times 1}} & \frac{-1}{\sqrt{2\times 1}} & 0 & 0 & \cdots & 0\\
      \frac{1}{\sqrt{3\times 2}} & \frac{1}{\sqrt{3\times 2}} & \frac{-2}{\sqrt{3\times 2}} & 0 \cdots & 0\\
      \vdots & \vdots & \vdots & \vdots & \ddots & \vdots \\
      \frac{1}{\sqrt{n(n - 1)}} & \frac{1}{\sqrt{n(n - 1)}} & \frac{1}{\sqrt{n(n - 1)}} & \frac{1}{\sqrt{n(n - 1)}} & \cdots & \frac{-(n - 1)}{\sqrt{n(n - 1)}} \\
    \end{pmatrix}
  \end{align*}
works since the rows are orthonormal. Now define $\mathbf{Y} = A\mathbf{X}$ such that
 \begin{align*}
    Y_1 = \frac{1}{\sqrt{n}}\sum_{i = 1}^n X_i = \sqrt{n} \bar X.
  \end{align*}
and
  \begin{align*}
    \mathbf{Y} \sim N_n(A\boldsymbol\mu, A\sigma^2IA^T) = N_n(A\boldsymbol\mu, \sigma^2 I)
  \end{align*}
where
  \begin{align*}
    A\boldsymbol\mu = (\sqrt{n}\mu, 0, \cdots, 0)^T.
  \end{align*}
  So $Y_1 \sim N(\sqrt{n}\mu, \sigma^2)$, $Y_i \sim N(0, \sigma^2)$ for $i \geq 2$ and the $Y_i$ are independent where
  \begin{align*}
    Y_2^2 + \cdots + Y_n^2 &= \mathbf{Y}^T\mathbf{Y} - Y_1^2\\
    &= \mathbf{X}^TA^TA\mathbf{X} - Y_1^2\\
    &= \mathbf{X}^T\mathbf{X} - n\bar X^2\\
    &= \sum_{i = 1}^n X_i^2 - n\bar X^2\\
    &= \sum_{i = 1}^n (X_i - \bar X)^2.
  \end{align*}
So $S_{XX}\sim \sigma^2 \chi_{n - 1}^2$ is independent of $\bar X= \frac{1}{\sqrt{n}}Y_1$.
\end{proof}

Q: Student $t$-distribution
A: Suppose that $Z\sim N(0, 1)$ and $Y\sim \chi_k^2$ are independent. Then $T = Z/\sqrt{Y/k}$ has a $t$-distribution on $k$ degrees of freedom, written $T\sim t_k$.
The density
\begin{align*}
  f_T(t) = \frac{\Gamma((k + 1)/2)}{\Gamma(k/2)} \frac{1}{\sqrt{\pi k}}\left(1 + \frac{t^2}{k}\right)^{-(k+1)/2}
\end{align*}
is symmetric, bell-shaped, and has a maximum at $t = 0$. It has a ``fatter'' tail than and, as $k \to \infty$, approaches a normal distribution.
It is the pivot for $N(\mu,\sigma^2)$.

Q: Homoscedastic
A: The error variance in the linear model is the same for each observation, i.e.
\begin{align*}
\cov (\mathbf{Y})=\cov (\boldsymbol\varepsilon)=\sigma^2 I.
\end{align*}

Q: OLS estimator
A: In a linear model $\mathbf{Y} = X\boldsymbol\beta + \boldsymbol\varepsilon$, the \emph{least squares estimator} $\hat{\boldsymbol\beta}$ of $\boldsymbol\beta$ minimizes the RSS
  \begin{align*}
    S(\boldsymbol\beta) = \|\mathbf{Y} - X\boldsymbol\beta\|^2= (Y_i - x_{ij}\beta_j)(Y_i - x_{ij}\beta_j).
  \end{align*}
To minimise it, we want
\begin{align*}
 0= \left.\frac{\partial S}{\partial \beta_k}\right|_{\boldsymbol\beta = \hat{\boldsymbol\beta}}= -2 x_{ik}(Y_i - x_{ij}\hat{\beta}_j),
\end{align*}
i.e.
\begin{align*}
  x_{ik}x_{ij}\hat{\beta_j} = x_{ik}Y_i, \quad X^TX\hat{\boldsymbol\beta} = X^T\mathbf{Y}.
\end{align*}
Since wlog $X$ is of full rank $p$, $X^TX$ is positive definite and thus invertible such that
\begin{align*}
  \hat{\boldsymbol\beta} = (X^TX)^{-1} X^T\mathbf{Y}.
\end{align*}

Q: Simple linear regression
A: If $Y_i = a + bx_i + \varepsilon_i$, then we can reparameterise to
\begin{align*}
  Y_i = a' + b(x_i - \bar x) + \varepsilon_i,
\end{align*}
where $a' = a + b\bar x$. In matrix form,
\begin{align*}
  X =
  \begin{pmatrix}
    1 & (x_1 - \bar x)\\
    \vdots & \vdots \\
    1 & (x_{n} - \bar x),
  \end{pmatrix}
\end{align*}
\begin{align*}
  (X^TX)^{-1} =
  \begin{pmatrix}
    \frac{1}{n} & 0\\
    0 & \frac{1}{S_{xx}}.
  \end{pmatrix}
\end{align*}
and
\begin{align*}
  X^TY =
  \begin{pmatrix}
    n\bar Y\\
    S_{xy}.
  \end{pmatrix}
\end{align*}
So
\begin{align*}
  \hat{\boldsymbol\beta} =
  \begin{pmatrix}
    \bar Y \\
    \frac{S_{xy}}{S_{xx}}
  \end{pmatrix}.
\end{align*}
Hence the estimated intercept is $\hat{a}' = \bar y$ (or $\hat{a}=\bar y - \hat{b}\bar x$) and the estimated gradient is
\begin{align*}
  \hat{b} = \frac{S_{xy}}{S_{xx}} = \frac{S_{xy}}{\sqrt{S_{xx}S_{yy}}} \sqrt{\frac{S_{yy}}{S_{xx}}} = r \sqrt{\frac{S_{yy}}{S_{xx}}}.
\end{align*}

Q: Gauss Markov theorem
A: In a full rank linear model, the LS estimator $\hat{\boldsymbol\beta}$ is the BLUE, i.e. for any linear unbiased $\boldsymbol\beta^*$
  \begin{align*}
    \var(\mathbf{t}^T\hat{\boldsymbol\beta}) \leq \var (\mathbf{t}^T\boldsymbol\beta^*)
  \end{align*}
  for all $\mathbf{t}\in \R^p$.
\begin{proof}
By linearity $\boldsymbol\beta^* = A\mathbf{Y}$ for some $A\in\Mat_{p\times n}(\R)$.
Since it is also unbiased $\E[\boldsymbol\beta^*] = A\E[\mathbf{Y}] = AX\boldsymbol\beta=\beta$. So $AX = I_p$. Taking covariances
  \begin{align*}
    \cov(\boldsymbol\beta^*) &= \cov(AY) \\
&=A\cov(Y)A^T \\
    &= \sigma^2 AA^T.
  \end{align*}
  Now let $\boldsymbol\beta^* - \hat{\boldsymbol\beta} = (A - (X^TX)^{-1}X^T)\mathbf{Y} = B\mathbf{Y}$, such that
  \begin{align*}
    BX = AX - (X^TX)^{-1}X^TX = I_p - I_p = 0
  \end{align*}
and hence the cross-terms vanish leaving
  \begin{align*}
    \cov(\boldsymbol\beta^*) &= \sigma^2AA^T\\
    &= \sigma^2(B + (X^TX)^{-1}X^T)(B + (X^TX)^{-1}X^T)^T\\
    &= \sigma^2(BB^T + (X^TX)^{-1})\\
    &= \sigma^2 BB^T + \cov(\hat{\boldsymbol\beta}).
  \end{align*}
 So for any $\mathbf{t}\in \R^p$
  \begin{align*}
    \var(\mathbf{t}^T\boldsymbol\beta^*) &= \mathbf{t}^T\cov(\boldsymbol\beta^*)\mathbf{t} \\
    &= \mathbf{t}^T\cov (\hat{\boldsymbol\beta})\mathbf{t} + \sigma^2\mathbf{t}^TBB^T\mathbf{t}\\
    &= \var(\mathbf{t}^T \hat{\boldsymbol\beta}) + \sigma^2\|B^T\mathbf{t}\|^2\\
    &\geq \var (\mathbf{t}^T\hat{\boldsymbol\beta}).
  \end{align*}
\end{proof}

Q: Projection matrix
A: Let $\mathbf{R} = \mathbf{Y} - \hat{\mathbf{Y}}$ be the \emph{vector of residuals}.
Then $X^T\mathbf{R}  = X^T\mathbf{Y} - X^TX\hat{\boldsymbol\beta} = 0$. So $\mathbf{R}$ is orthogonal to the column space of $X$.
Write $\hat{\mathbf{Y}} = X\hat{\boldsymbol\beta} = X(X^TX)^{-1}X^T\mathbf{Y} = P\mathbf{Y}$, where $P = X(X^TX)^{-1}X^T$ is the idempotent, symmetric projection matrix.
$P$ represents an orthogonal projection of $\R^n$ onto the column space of $X$, ie. it projects the actual data $\mathbf{Y}$ to the fitted values $\hat{\mathbf{Y}}$.

Q: Normal error MLEs
A: Let $\mathbf{Y} = X\boldsymbol\beta + \boldsymbol\varepsilon,\quad \boldsymbol\varepsilon\sim N_n(\mathbf{0}, \sigma^2 I)$.
Since $\mathbf{Y} = N_n(X\boldsymbol\beta, \sigma^2 I)$, the log-likelihood is
\begin{align*}
  \ell(\boldsymbol\beta, \sigma^2) &=\log\left( \frac{1}{(2\pi\sigma^2)^\frac{n}{2}} \exp\left(-\frac{1}{2\sigma^2} \| \mathbf{Y} - X\boldsymbol\beta \|^2 \right)\right)\\
&=-\frac{n}{2}\log 2\pi - \frac{n}{2}\log \sigma^2 - \frac{1}{2\sigma^2}S(\boldsymbol\beta).
\end{align*}
Maximising wrt $\beta$ gives the OLS estimator
  \begin{align*}
    \hat{\boldsymbol\beta} = (X^TX)^{-1}X^T\mathbf{Y}.
  \end{align*}
To obtain the MLE for $\sigma^2$, we require
\begin{align*}
 0= \left.\frac{\partial \ell}{\partial \sigma^2}\right|_{\hat{\boldsymbol\beta}, \hat{\sigma}^2}=-\frac{n}{2\hat{\sigma}^2} + \frac{S(\hat{\boldsymbol\beta})}{2 \hat{\sigma}^4}.
\end{align*}
So $\hat{\sigma}^2 = \frac{1}{n}S(\hat{\boldsymbol\beta})$.

Q: Quadric of normals
A: If $\mathbf{Z}\sim N_n(\mathbf{0}, \sigma^2 I)$ and $A$ is $n\times n$, symmetric, idempotent with rank $r$, then $\mathbf{Z}^TA\mathbf{Z} \sim \sigma^2 \chi_r^2$.
\begin{proof}\leavevmode
Since $A$ is idempotent, the eigenvalues are either 0 or 1.
      Since $A$ is also symmetric, it is orthogonally diagonalisable as
      \begin{align*}
        \Lambda = Q^TAQ = \diag(\underbrace{1, \cdots, 1}_r, \underbrace{0, \cdots, 0}_{n-r}).
      \end{align*}
Then $ Q^T\mathbf{Z}\sim N_n(\mathbf{0}, \sigma^2 I)$ and
      \begin{align*}
        \mathbf{Z}^TA\mathbf{Z} &=\mathbf{Z}^TQ\Lambda Q^T\mathbf{Z}\\
&= (Q^T\mathbf{Z})^T\Lambda Q^T\mathbf{Z}\\
&= \sum_{i=1}^r (Q^T\mathbf{Z})_i^2 \\
&\sim \sigma^2\chi_r^2.
      \end{align*}
\end{proof}

Q: Rank vs trace
A: For a real symmetric idempotent matrix $A$
\begin{align*}
\rank(A) = \tr(A).
\end{align*}
\begin{proof}\leavevmode
Simply note that $A$ has orthogonal diagonalisation
\begin{align*}
Q^TAQ=\diag(\underbrace{1, \cdots, 1}_r, \underbrace{0, \cdots, 0}_{n-r}).
\end{align*}
\end{proof}

Q: Distribution of OLS estimator
A: \begin{thm}
  For the normal linear model $\mathbf{Y}\sim N_n(X\boldsymbol\beta, \sigma^2 I)$:
\begin{align*}
\hat{\boldsymbol\beta} \sim N_p(\boldsymbol\beta, \sigma^2(X^TX)^{-1}).
\end{align*}
\end{thm}
\begin{proof}\leavevmode
We know $\hat{\boldsymbol\beta} = (X^TX)^{-1}X^T\mathbf{Y}$. Then $\hat {\boldsymbol\beta}$ has a normal distribution with mean
      \begin{align*}
        (X^TX)^{-1}X^T(X\boldsymbol\beta) = \boldsymbol\beta
      \end{align*}
      and covariance
      \begin{align*}
        (X^TX)^{-1}X^T(\sigma^2 I)[(X^TX)^{-1}X^T]^T = \sigma^2(X^TX)^{-1}.
      \end{align*}
  \end{proof}

Q: Distribution of RSS
A: For the normal linear model $\mathbf{Y}\sim N_n(X\boldsymbol\beta, \sigma^2 I)$:
\begin{align*}
\text{RSS}\sim \sigma^2 \chi_{n - p}^2, \quad \hat{\sigma}^2 \sim \frac{\sigma^2}{n}\chi_{n -p}^2.
\end{align*}
\begin{proof}\leavevmode
We know $A = (I_n - P)$ is symmetric idempotent, where $P = X(X^TX)^{-1}X^T$. Also
      \begin{align*}
        \rank(I_n - P) = \tr(I_n - P) = n - p.
      \end{align*}
Since $(I_n - P)X = \mathbf{0}$:\vspace{-0.2in}
\begin{align*}
        \boldsymbol\varepsilon^TA\boldsymbol\varepsilon &= (\mathbf{Y} - X\boldsymbol\beta)^T(I_n - P)(\mathbf{Y} - X\boldsymbol\beta)\\
&=\mathbf{Y}^T(I_n - P)\mathbf{Y}\\
&=\mathbf{Y}^T(I_n - P) (I_n - P)\mathbf{Y} \\
&=\mathbf{R}^T\mathbf{R}\\
&=\text{RSS}\\
&\sim \sigma^2\chi_{n-p}^2.
      \end{align*}
\end{proof}

Q: Independence of parameters in NLM
A: For the normal linear model $\mathbf{Y}\sim N_n(X\boldsymbol\beta, \sigma^2 I)$, $\hat{\boldsymbol\beta}$ and $\hat{\sigma}^2$ are independent.
\begin{proof}\leavevmode
Write $\hat{\boldsymbol\beta} = (X^TX)^{-1}X^T\mathbf{Y}=C\mathbf{Y}$. Then $(I_n - P)X = 0 \implies C(I_n - P) = 0$.
  Let $V = \begin{pmatrix}\hat{\boldsymbol\beta}\\\mathbf{R}\end{pmatrix} = D\mathbf{Y}$, where $D = \begin{pmatrix}C\\I_n - P\end{pmatrix}$ is a $(p + n)\times n$ matrix.
Then $V$ is MVN with
      \begin{align*}
        \cov (V) &= D\sigma^2ID^T\\
        &= \sigma^2
        \begin{pmatrix}
          CC^T & C(I_n - P)^T\\
          (I_n - P)C^T & (I_n - P)(I_n - P)^T
        \end{pmatrix}\\
        &=
        \sigma^2
        \begin{pmatrix}
          CC^T & C(I_n - P)\\
          (I_n - P)C^T & (I_n - P)
        \end{pmatrix}\\
        &= \sigma^2
        \begin{pmatrix}
          CC^T & 0\\
          0 & I_n - p.
        \end{pmatrix}
      \end{align*}
So $\hat{\boldsymbol\beta}$ and $\mathbf{R}$ are independent.
\end{proof}

Q: Residual standard error
A: Since RSS $\sim \sigma^2\chi_{n-p}^2$, we know that $\E(\mathrm{RSS}) = \sigma^2(n - p)$.
Hence the RSE on $n - p$ degrees of freedom $\tilde{\sigma}^2 = \frac{\mathrm{RSS}}{n - p}$ is an unbiased estimator of $\sigma^2$.

Q: $F$ distribution
A: If $U\sim \chi_m^2$ and $V\sim \chi_n^2$ are independent, then $X = \frac{U/m}{V/n}\sim F_{m, n}$ has an \emph{$F$-distribution} on $m$ and $n$ degrees of freedom.
$F$ is often approximately $1$ and has the property
\begin{align*}
X\sim F_{m, n} \iff 1/X\sim F_{n, m}.
\end{align*}
Also if $Y\sim t_n$, then $Y^2\sim F_{1, n}$.

Q: Pair of normal quadrics
A: Suppose $\mathbf{Z} \sim N_n(\mathbf{0}, \sigma^2 I_n)$, and $A_1$ and $A_2$ are symmetric, idempotent $n\times n$ matrices with $A_1A_2 = 0$. Then $\mathbf{Z}^T A_1 \mathbf{Z}$ and $\mathbf{Z}^TA_2 \mathbf{Z}$ are independent.
\begin{proof}
  Let
  \begin{align*}
    \mathbf{W}  = \begin{pmatrix} A_1\\ A_2\end{pmatrix}\mathbf{Z}.
  \end{align*}
  Then
  \begin{align*}
    \mathbf{W}\sim N_{2n}\left(
    \begin{pmatrix}
      \mathbf{0}\\
      \mathbf{0}
    \end{pmatrix}, \sigma^2
    \begin{pmatrix}
      A_1 & 0\\
      0 & A_2.
    \end{pmatrix}\right)
  \end{align*}
 So $A_1\mathbf{Z}$ and $A_2\mathbf{Z}$ are independent, which implies
  \begin{align*}
   (A_1\mathbf{Z})^T (A_1\mathbf{Z})  = \mathbf{Z}^T A_1 \mathbf{Z}
  \end{align*}
  and
  \begin{align*}
    (A_2\mathbf{Z})^T (A_2\mathbf{Z}) = \mathbf{Z}^T A_2 \mathbf{Z}
  \end{align*}
  are independent.
\end{proof}

Q: GLRT of NLM
A: The GLR is
\begin{align*}
  \Lambda_\mathbf{Y}(H_0, H_1) &= \frac{\left(\frac{1}{\sqrt{2\pi \hat{\sigma}^2}}\right)\exp\left(-\frac{1}{2\hat{\sigma}^2}(\|\mathbf{Y} - X\hat{\boldsymbol\beta}\|^2)\right)}{\left(\frac{1}{\sqrt{2\pi \hat{\hat{\sigma}}^2}}\right)\exp\left(-\frac{1}{2\hat{\hat{\sigma}}^2}(\|\mathbf{Y} - X\hat{\hat{\boldsymbol\beta}}_0\|^2)\right)}\\
  &= \left(\frac{\hat{\hat{\sigma}}^2}{\hat{\sigma}^2}\right)^{n/2} \\
  &= \left(\frac{\text{RSS}_0}{\mathrm{RSS}}\right)^{n/2} \\
  &= \left(1 + \frac{\mathrm{RSS}_0 - \mathrm{RSS}}{\mathrm{RSS}}\right)^{n/2} \\
  &= (1 + R^2)^{n/2}
\end{align*}
where $R^2$ is the proprtion of variance explained by $\beta_1$. We have the approximation
\begin{align*}
  2\log \Lambda = n\log\left(1 + R^2\right)\sim \chi_{p - p_0}^2.
\end{align*}
For an exact test, the numerator is
\begin{align*}
  \mathrm{RSS}_0 - \mathrm{RSS} = \mathbf{Y}^T(I_n - P_0)\mathbf{Y} - \mathbf{Y}^T(I_n - P)\mathbf{Y} = \mathbf{Y}^T(P - P_0)\mathbf{Y}
\end{align*}
where
\begin{align*}
  (I_n - P)(P - P_0) = (I_n - P)P - (I_n - P)P_0 = 0.
\end{align*}
So we have the independent rvs
\begin{align*}
  \mathrm{RSS} = \mathbf{Y}^T(I_n - P)\mathbf{Y} &\sim \chi_{n - p}^2\\
  \mathrm{RSS}_0 - \mathrm{RSS} = \mathbf{Y}^T(P - P_0)\mathbf{Y} &\sim \chi^2_{p - p_0}.
\end{align*}
So under $H_0$,
\begin{align*}
 \frac{\mathbf{Y}^T(P - P_0)\mathbf{Y}/(p - p_0)}{\mathbf{Y}^T(I_n - P)\mathbf{Y}/(n - p)} = \frac{(\mathrm{RSS}_0 - \mathrm{RSS})/(p - p_0)}{\mathrm{RSS}/(n - p)} \sim F_{p - p_0, n - p}.
\end{align*}

Q: Inference for $\boldsymbol\beta$
A: We know that $\hat{\boldsymbol\beta} \sim N_p(\boldsymbol\beta, \sigma^2(X^TX)^{-1})$. So
\begin{align*}
  \hat{\beta}_j \sim N(\beta_j, \sigma^2(X^TX)^{-1}_{jj}).
\end{align*}
The \emph{standard error} of $\hat{\beta}_j$ is defined to be
\begin{align*}
  \mathrm{SE}(\hat{\beta}_j) = \sqrt{\tilde{\sigma}^2 (X^TX)_{jj}^{-1}},
\end{align*}
where $\tilde{\sigma}^2 = \mathrm{RSS}/(n - p)$. Then
\begin{align*}
  \frac{\hat{\beta}_j - \beta_j}{\mathrm{SE}(\hat{\beta}_j)} = \frac{\hat{\beta}_j - \beta_j}{\sqrt{\tilde{\sigma}^2(X^TX)^{-1}_{jj}}} = \frac{(\hat{\beta}_j - \beta_j)/\sqrt{\sigma^2(X^TX)_{jj}^{-1}}}{\sqrt{\mathrm{RSS}/((n - p)\sigma^2)}} \sim t_{n - p}.
\end{align*}

Q: Expected response at $\mathbf{x}^*$
A: The expected response at $\mathbf{x}^*$ is $\E[\mathbf{Y}\mid \mathbf{x}^*] = \mathbf{x}^{*T}\boldsymbol\beta$. We estimate this by $\mathbf{x}^{*T}\hat{\boldsymbol\beta}$. Then we have
\begin{align*}
  \mathbf{x}^{*T}(\hat{\boldsymbol\beta} - \boldsymbol\beta)\sim N(0, \mathbf{x}^{*T} \cov(\hat{\boldsymbol\beta})\mathbf{x}^*) = N(0, \sigma^2\mathbf{x}^{*T}(X^TX)^{-1}\mathbf{x}^*).
\end{align*}
Let $\tau^2 = \mathbf{x}^{*T} (X^TX)^{-1}\mathbf{x}^*$. Then
\begin{align*}
  \frac{\mathbf{x}^{*T}(\hat{\boldsymbol\beta} - \boldsymbol\beta)}{\tilde{\sigma}\tau} = \frac{\mathbf{x}^{*T}(\hat{\boldsymbol\beta} - \boldsymbol\beta)/\sigma\tau}{\sqrt{\mathrm{RSS}/((n - p)\sigma^2)}}\sim t_{n - p} .
\end{align*}

Q: Predicted response
A: The predicted response at $\mathbf{x}^*$ is $Y^* = (\mathbf{x}^*)^T\boldsymbol\beta + \varepsilon^*$. An unbiased estimator is the predicted \emph{expected} response $\hat{Y}^* = \mathbf{x}^{*T}\boldsymbol\beta$. The variance is
\begin{align*}
  \var(\hat{Y}^* - Y^*) &= \var (\mathbf{x}^{*T}(\hat{\boldsymbol\beta})) + \var (\varepsilon^*)\\
  &= \sigma^2 \mathbf{x}^{*T}(X^TX)^{-1}\mathbf{x}^* + \sigma^2.\\
  &= \sigma^2(\tau^2 + 1).
\end{align*}
So
\begin{align*}
  \hat{Y}^* - Y^* \sim N(0, \sigma^2(\tau^2 + 1))
\end{align*}
giving the pivot
\begin{align*}
  \frac{\hat{Y}^* - Y^*}{\tilde{\sigma}\sqrt{\tau^2 + 1}}=\frac{(\hat{Y}^* - Y^*)/\sigma\sqrt{\tau^2+1}}{\sqrt{\mathrm{RSS}/((n - p)\sigma^2)}} \sim t_{n - p}.
\end{align*}