Title: Analysis II
Tags: A2.4

Q: Lipschitz equivalence
A: Let $V$ be a vector space. Two norms $\|\ph \|, \|\ph \|'$ on $V$ are \emph{Lipschitz equivalent} iff
\begin{align*}
&\exists c > 0 : \forall \mathbf{x} \in V: \|\mathbf{x}\|/c \leq \|\mathbf{x}\|' \leq c\|\mathbf{x}\| \\
\iff &\exists a,b>0: \forall \mathbf{x} \in V: a \|\mathbf{x}\| \leq \|\mathbf{x}\|' \leq b\|\mathbf{x}\| \\
\iff &\id:(V,\|\ph \|)\to V,\|\ph \|') \\
 \textrm{ and } &\id:(V,\|\ph \|')\to V,\|\ph \|) \textrm{ are both continuous}.
\end{align*}
This is an equivalence relation on the set of all norms on $V$. If two norms are Lipschitz equivalent, then they are homeomorphic.

Q: Lipschitz equivalence justification
A: Let $\|\ph \|, \|\ph \|'$ be norms on a vector space $V$. Then
\begin{align*}
 &\id: (V, \|\ph \|)\to (V, \|\ph \|') \textrm{ is continuous}\\
 \iff &\exists C>0: \forall \mathbf{v} \in V: \|\mathbf{v} \|' \leq C \|\mathbf{v}\|.
\end{align*}
\begin{proof}
($\Rightarrow$) By continuity, $\exists\delta >0$ such that
\[
B_\delta(0, \|\ph \|) \subseteq B_1(0, \|\ph \|').
\]
Pick $C=\frac{2}{\delta}$. Now, given $\vec{v}\in V$, if $\| \mathbf{v} \| =0$ then $\| \mathbf{v} \|'=0$. Otherwise we can pick $k\in\R$ such that $\| k\mathbf{v} \|=\delta / 2$. So $k\mathbf{v}\in B_1(0, \| \ph \|')$, i.e. $\| k\mathbf{v} \|' < 1 = \frac{2}{\delta} \| k\mathbf{v} \| = C \| k\mathbf{v} \|$.
($\Leftarrow$) If $\mathbf{v_n}\to \vec{v}$ wrt $\|\ph \|$, then $\|\mathbf{v_n}-\mathbf{v} \|\to 0$ as $n\to\infty$. So $\|\vec{v}_n-\mathbf{v} \|'\to 0$ by squeeze rule. So $\mathbf{v}_n\to\mathbf{v}$ wrt $\|\ph \|'$.
\end{proof}

Q: Uniform limit of continuous functions
A: \begin{thm}
  Let $f_n, f: X\to Y$ where $X,Y$ are normed spaces. If $f_n \to f$ uniformly and $f_n$ are all continuous at some point $x\in X$ for all $n$, then $f$ is also continuous at $x$.
 Equivalently, $C[a,b]$ is a closed subset of $B[a,b]$.
\end{thm}
\begin{proof}
  Let $\varepsilon > 0$. By uniform convergence $\exists N\in\N: \forall n \geq N$:
  \[
    \sup_{y\in X}|f_n(y) - f(y)| < \frac{\varepsilon}{4}.
  \]
  Since $f_N$ is continuous at $x$, $\exists\delta >0:\forall y \in X:$
  \[
    |x - y| < \delta \Rightarrow |f_N(x) - f_N(y)| < \frac{\varepsilon}{2}.
  \]
  Then for $\forall y\in B_\delta(x)\cap X$:
  \begin{align*}
    &|f(x) - f(y)| \\
\leq &|f(x) - f_N(x)| + |f_N(x) - f_N(y)| + |f_N(y) - f(y)| \\
< & \frac{\varepsilon}{4} + \frac{\varepsilon}{2} + \frac{\varepsilon}{4} \\
=&\varepsilon.
  \end{align*}
\end{proof}

Q: Integral of uniformly convergent functions
A: \begin{thm}
  Let $f_n, f: [a, b]\to \R$ be Riemann integrable, with $f_n \to f$ uniformly. Further assume $x_0\in[a,b]$ and write $F_n(x)=\int_{x_0}^x f_n(t)\;\d t, F(x)=\int_{x_0}^x f(t) \;\d t$. Then $F_n \to F$ uniformly. In particular $f_n\to f$ wrt $L^1$.
\end{thm}
\begin{proof}
Since $f_n\to f$ uniformly, $f$ is continuous and thus integrable. So $F_n$ and $F$ are well-defined.
Given $\varepsilon>0$, since $f_n\to f$ uniformly, $\exists N\in\N:\forall n\geq N: \forall x\in [a,b]:$
\[
|f_n(x)-f(x)| <\frac{\varepsilon}{b-a}.
\]
Then
  \begin{align*}
|F_n(x)-F(x)| &= \left| \int_{x_0}^x |f_n(t) - f(t)|\;\d t \right|\\
    &\leq \sup_{t\in [x_0, x]}|f_n(t) - f(t)| (x - x_0)\\
    &\leq \sup_{t\in [a, b]}|f_n(t) - f(t)| (b - a)\\
    &= \left(\frac{\varepsilon}{b-a} \right)(b-a) \\
&= \varepsilon.
  \end{align*}
So $|F_n-F|_\infty < \varepsilon$.
\end{proof}

Q: Local uniform convergence inside radius of convergence
A: Let $\sum\limits_{n = 0}^\infty a_n (z - c)^n$ have r.o.c. $R>0$. Then for $0 < r < R$, $\sum a_n (z - c)^n$ converges absolutely uniformly on $\bar{B}_r(c)$.
\begin{proof}
Fix $z_0\in\C$ such that $r<|z_0-c|<R$. Then $\sum\limits_{n = 0}^\infty a_n (z - c)^n$ converges absolutely and $0<\alpha=\left|\frac{r}{z_0-c}\right|<1$.
Given $\varepsilon >0$, $\exists N_0\in\N$ such that $\forall n\geq N_0: |a_n(z_0-c)^n|<1$ and $\alpha^{n+1} < (1-\alpha)\varepsilon$. Then $\forall z\in \bar{B}_r(c), N\geq N_0$:
\begin{align*}
&\left|\sum_{n=0}^\infty |a_n(z-c)^n|- \sum_{n=0}^N \left|a_n (z-c)^n\right| \right| \\
\leq &\sum_{n=N+1} \left|a_n (z-c)^n\right| \\
\leq &\sum_{n=N+1} |a_n(z_0-c)^n| \left|\frac{z-c}{z_0-c}\right|^n \\
\leq &\sum_{n=N+1} \left|\frac{r}{z_0-c}\right|^n \\
&= \frac{\alpha^{N+1}}{1-\alpha} \\
&<\varepsilon.
\end{align*}
\end{proof}

Q: Continuity of power series
A: $f(z)=\sum_{n=0}^\infty a_n(z-c)^n$ is a continuous map $B_R(c)\to\C$ where $R$ is the r.o.c.
\begin{proof}
The partial sums $\sum_{n=0}^N a_n(z-c)^n$ are polynomials so continuous maps $\C\to\C$.
For any $r<R$, $\sum_{n=0}^N a_n(z-c)^n\to f$ uniformly on $\bar{B}_r(c)$, so $f:\bar{B}_r(c)\to\C$ is continuous.
Given $z\in B_R(c)$, pick $r<R$ with $z\in B_r(c)$ such that $f$ is continuous at $z$. So $f:B_R(c)\to\C$ is continuous.
\end{proof}

Q: Weierstrass M-test
A: Let $f_n: A \to \R$ be a sequence of functions. Suppose there is some sequence $M_n$ such that for all $n$, we have
  \[
    \sup_{x\in A}|f_n (x)| \leq M_n.
  \]
  If $\sum M_n$ converges, then $\sum f_n$ converges absolutely uniformly in $A$.
\begin{proof}
  Let $F_N = \sum\limits_{n = 1}^N |f_n|$ be the partial sums. Then $\forall N > M >N_0, x\in A$
  \[
    |F_N(x) - F_M(x)| = \sum_{n = M + 1}^N |f_n(x)| \leq \sum_{n=M+1}^N M_n.
  \]
  Taking supremum, we have
  \[
    \sup_{x\in A}|F_N(x) - F_M(x)| \leq \sum_{n=M+1}^N M_n \to 0\text{ as }N_0\to \infty.
  \]
  So done by the Cauchy criterion.
\end{proof}

Q: Continuous maps extend to open covers
A: Suppose $f:V\to W$ and $f$ is continuous on all $U_\alpha$ for $\alpha\in T$. Then $f$ is continuous on $V=\bigcup_{\alpha\in T}V_\alpha$.
\begin{proof}
Given $\varepsilon>0$, $v\in V$, then $v\in V_{\alpha_0}$ for some $\alpha_0 \in T$. Since $V_{\alpha_0}$ is open, $\exists\delta_1>0$ such that
\[
B_{\delta_1}(v) \subseteq V_{\alpha_0}.
\]
Since $f$ is continuous on $V_{\alpha_0}$, $\exists\delta_2>0$ such that
\[
f(B_{\delta_2}(v)\cap V_{\alpha_0}) \subseteq B_\epsilon(f(v)).
\]
Let $\delta=\min(\delta_1, \delta_2)$. Then
\begin{align*}
f(B_\delta(v)\cap V) &= f(B_\delta(v)) \\
&\subseteq f(B_{\delta_2}(v) \cap V_{\alpha_0} ) \\
&\subseteq B_\epsilon(f(v)).
\end{align*}
\end{proof}

Q: Integration of power series
A: Let $f(t)=\sum_{n=0}^\infty a_n(t-c)^n$ be a real power series with r.o.c. R and $p_n(t)=\sum_{n=0}^N a_n(t-c)^n$, then $\forall x \in B_R(c):$
\[
\int_c^x f(t)\; \d t = \sum_{n=0}^\infty \frac{a_n}{n+1} (x-c)^{n+1}.
\]
\begin{proof}
Given $z\in B_R(c)$, pick $r$ with $|x-c|<r<R$. Then $p_n\to f$ uniformly on $\bar{B}_r(c)$. So $P_n\to \int_c^x f(t)\;\d t$ where
\begin{align*}
P_n&=\int_c^x p_n(t) \; \d t\\
&=\sum_{n=1}^N \frac{a_n}{n+1} (x-c)^{n+1}.
\end{align*}
\end{proof}

Q: Termwise differentiation of power series
A: If $f(z)=\sum_{n=0}^\infty a_n(z-c)^n$ converges on $B_R(c)$, then $\sum_{n=0}^\infty na_n(z_0-c)^{n-1}$ converges to $f'(z)$ on $B_R(c)$.
\begin{proof}
Pick $z_0\in B_R(c)$ such that $|z-c|<|z_0-c|$ and let $\alpha=\left| \frac{z-c}{z_0-c} \right| < 1$. Then $\exists N\in N$ such that $\forall n\geq N: |a_n(z-c)|^n<\alpha^n$ and
\[
|na_n (z-c)^{n-1}|<\frac{n}{|z-c|} \alpha^n.
\]
So by the ratio test $\sum_{n=0}^\infty a_nn(z_0-c)^{n-1}\to g(z)$ for some continuous function on $B_R(c)$. Then
\[
\int_c^z g(t)\; \d t = \sum_{n=1}^\infty a_n(z-c)^n=f(z).
\]
By FTC, $g=f'$.
\end{proof}

Q: Bolzano-Weierstrass in $\R^n$
A: Any bounded sequence in $\R^n$ (with, say, the Euclidean norm) has a convergent subsequence.
\begin{proof}  We induct on $n$. The $n = 1$ case is the usual Bolzano-Weierstrass on the real line.
  Assume true for $\R^{n - 1}$, and let $\mathbf{x}^{(k)} = (x^{(k)}_1, \cdots, x_n^{(k)})$ be a bounded sequence in $\R^n$. Define $\mathbf{y}^{(k)} = (x^{(k)}_1, \cdots, x_{n - 1}^{(k)})$. Then $\|\mathbf{y}^{(k)}\|\leq \|\mathbf{x}^{(k)}\|$ and $|x_n^{(k)}| \leq \|\mathbf{x}^{(k)}\|$ are bounded. So by the induction hypothesis, there is a subsequence $(k_j)$ of $(k)$ and some $\mathbf{y} \in \R^{n - 1}$ such that $\mathbf{y}^{(k_j)} \to \mathbf{y}$. Also, by Bolzano-Weierstrass in $\R$, there is a further subsequence $(x_n^{(k_{j_\ell})})$ of $(x_n^{(k_j)})$ that converges to, say, $y_n \in \R$. Then
  \[
    \mathbf{x}^{(k_{j_\ell})} \to (\mathbf{y}, y_n).
  \]
\end{proof}

Q: Lipschitz equivalence of $p-$norms in $\R^n$
A: For any $\mathbf{x}=(x_1,x_2,\ldots x_n)$ in $\R^n$
\begin{align*}
\| \mathbf{x}\|_\infty &= \max_{1\leq i \leq n}(|x_i|)\\
&= \left(\max_{1\leq i \leq n}(x_i^2) \right)^\frac{1}{2}\\
& \leq \left( \sum_{i=1}^n x_i^2\right)^\frac{1}{2} \\
&= \| \mathbf{x}\|_2
\end{align*}
and
\begin{align*}
\| \mathbf{x}\|_2^2 &=  \sum_{i=1}^n x_i^2 \\
&\leq \sum_{i=1}^n x_i^2 + 2\sum_{1\leq i < j \leq n} |x_i| |x_j| \\
&= \left(\sum_{i=1}^n |x_i|\right)^2 \\
&= \| \mathbf{x}\|_1^2.
\end{align*}
Finally
\begin{align*}
\| \mathbf{x}\|_1 &= \sum_{i=1}^n |x_i| \\
&\leq \sum_{i=1}^n \max_{1\leq i \leq n}(|x_i|)\\
&= n\max_{1\leq i \leq n}(|x_i|) \\
&= n\| \mathbf{x}\|_\infty.
\end{align*}
Therefore
\[
\| x_\infty \| \leq \| x_2 \| \leq \| x_1 \| \leq n \| x_\infty \|.
\]

Q: Heine-Borel Theorem
A: \begin{thm}
  Let $V$ be a metric space, $C\subseteq V$ a subset. Then
  \begin{enumerate}
    \item If $C$ is sequentially compact, then $C$ is closed and bounded.
    \item If $V$ is $\R^n$ (with, say, the Euclidean norm), then if $C$ is closed and bounded, then $C$ is sequentially compact.
  \end{enumerate}
\end{thm}
\begin{proof}\leavevmode
  \begin{enumerate}
    \item Let $C$ be sequentially compact. If $C$ is unbounded, then $\exists(\mathbf{x}_n)$ such that $\mathbf{x}_n\geq n\to \infty$. This has all subsequences unbounded and thus not convergent - contradicting sequential compactness.
     If $C$ is not closed, then $\exists \mathbf{y}_n\in C \to \mathbf{y}\not\in C$ and every subsequence converges to $\mathbf{y}$. Then by sequential compactness, there is a subsequence of $\mathbf{y}_{n_k}\to \mathbf{y}'\in C$. Contradiction.
    \item Let $C$ be closed and bounded. Let $\mathbf{x}_n$ be a sequence in $C$. Since $C$ is bounded, $(\mathbf{x}_n)$ is a bounded sequence in $\R^n$. So by higher dimensional Bolzano-Weierstrass, this has a convergent subsequence $\mathbf{x}_{n_k}$. Since $C$ is closed, $\mathbf{x}_{n_k}\to \mathbf{x}\in C$.
  \end{enumerate}
\end{proof}

Q: Continuous functions preserve sequential compactness
A: \begin{thm}
  Let $X$ and $Y$ be metric spaces, and $C$ a compact subset of $X$, and $f: X \to Y$ a continuous function. Then $f(C)$ is sequentially compact in $Y$.
\end{thm}
\begin{proof}\leavevmode
Let $(\mathbf{x}_n)$ be a sequence in $f(C)$ with $\mathbf{x}_n = f(\mathbf{y}_n)$ for some $\mathbf{y}_n \in C$. By sequential compactness of $C$, there is a subsequence $(\mathbf{y}_{k_j})$ such that $\mathbf{y}_{n_k} \to \mathbf{y}\in C$. By continuity $f(\mathbf{y}_{n_k}) \to f(\mathbf{y})$. So $\mathbf{x}_{n_k} \to f(\mathbf{y}) \in f(C)$. So $f(C)$ is compact.
\end{proof}

Q: Norms on $\R^n$ are continuous maps
A: \begin{lemma}
The map $(V, \| \ph \|) \to \R$ by $v \mapsto \| v \|$ is continuous.
\end{lemma}
\begin{proof}
Given $\varepsilon>0$, $\forall w \in V: \| w-v \| < \varepsilon \implies | \| w-v\| | <\varepsilon$.
\end{proof}
\begin{lemma}
The map $\id:(\R^n, \| \ph \|_1) \to (\R^n, \| \ph \|)$ is continuous.
\end{lemma}
\begin{proof}
Given $\mathbf{v}\in V$, write $\mathbf{v}=\sum_{i=1}^n v_i\mathbf{e}_i$. By the triangle inequality
\begin{align*}
\| \mathbf{v} \| &\leq \sum_{i=1}^n \| v_i\mathbf{e}_i \| \\
&= \sum_{i=1}^n  |v_i| \|\mathbf{e}_i \| \\
&\leq C\sum_{i=1}^n  |v_i| \\
&= C \| \mathbf{v} \|_1 \\
\end{align*}
where $C=\max_{1\leq i \leq n} \{ \| \mathbf{e}_i \| \}$. So done by Lipschitz condition.
\end{proof}
\begin{cor}
The map $f:(\R^n, \| \ph \|_1) \to \R$ given by $f(v)=\| \ph \|$ is a composition of continuous maps
\[
(\R^n, \| \ph \|_1) \xrightarrow{\id} (\R^n, \| \ph \|)\xrightarrow{\|\ph\|} \R
\]
so is continuous.
\end{cor}

Q: Uniqueness of norms on $\R^n$
A: \begin{thm}
  Any norm $\| \ph \|$ on $\R^n$ is Lipschitz equivalent to $\| \ph \|_1$.
\end{thm}
\begin{proof}
 The function $(S^{n-1}, \| \ph \|_1) \to \R$ given by $\mathbf{x} \to \| \mathbf{x} \|$ is a continuous function on a compact domain so by the Maximum Value Theorem $\exists \mathbf{a},\mathbf{b}\in  (S^{n-1}, \| \ph \|_1) $ such that $\forall \mathbf{x}\in \R^n\setminus \{ 0 \}$:
\[
m=\| \mathbf{a} \| \leq \| \frac{\mathbf{x}}{\| \mathbf{x} \|_1} \| \leq \| \mathbf{b}\| = M
\]
and hence
\[
m \| \mathbf{x} \|_1\leq \| \mathbf{x} \| \leq M\|\mathbf{x} \|_1
\]
where $\mathbf{a},\mathbf{b}\neq \mathbf{0} \implies m, M \neq 0$.
\end{proof}

Q: Compactness of the unit sphere
A: The unit sphere $S^n=\{ \mathbf{x} \in \R^{n+1} : \| \mathbf{x}\| =1 \}$ is compact.
\begin{proof}
Let $f:S^n\to \R$ be given by $\mathbf{x}\mapsto \| \mathbf{x} \|$. Then $S^n=f^{-1}(\{ 1 \})$. Since $\R^{n+1}$ is Hausdorff, $\{ 1 \}$ is closed. Hence, as $f$ is continuous, $S^n$ is closed.
Since $S$ is bounded, it is also compact.
\end{proof}

Q: Completeness of $C[0,1]$ wrt $\|\ph\|_\infty$
A: $C[0,1]$ is complete wrt $\|\ph\|_\infty$.
\begin{proof}
Given a Cauchy sequence $f_n\in C[0,1]$, $\forall\varepsilon>0: \exists N\in\N: \forall n,m \geq N: \| f_m - f_n \|_\infty < \varepsilon$.
\textbf{Pointwise Convergence}. We have $\forall x\in [0,1], \varepsilon>0, \forall n,m\geq N:$
\[
|f_n(x)-f_m(x)| \leq \| f_n - f_m\|_\infty < \varepsilon.
\]
So, for each $x\in C[0,1]$, $f_n(x)$ is a Cauchy sequence in $\R$ such that $f_n(x)\to f(x)$.
\textbf{Uniform Convergence}. Given $\varepsilon>0$, for each $ x\in C[0,1]$ define $M(x)>N$ such that $\forall n\geq M(x): |f_n(x)-f(x)|<\varepsilon / 2$. Then $\forall n\geq N, x\in [0,1]$
\begin{align*}
|f_n(x)-f(x)| &\leq |f_n(x)-f_R(x)| + |f_R(x)-f(x)| \\
&< \varepsilon / 2 + \varepsilon / 2 \\
&= \varepsilon.
\end{align*}
So $\forall n\geq N: \|f_n-f\|_\infty < \varepsilon$.
\textbf{Closed}. Since $f_n\to f$ uniformly and $f_n\in C[0,1]$, we have $f\in C[0,1]$.
\end{proof}

Q: Continuous function on a compact metric space is uniformly continuous
A: Let $f:X\to Y$ be a continuous function where $(X,d_X),(Y,d_Y)$ are metric spaces and $X$ is compact. Then $f$ is uniformly continuous.
\begin{proof}
Given $\varepsilon >0$, by continuity $\forall x \in X: \exists \delta_x >0 : f(B_{\delta_x}(x))\subseteq B_{\frac{\varepsilon}{2}}(f(x))$. Since $\{ B_{\frac{\delta_x}{2}}(x) : x \in X\}$ is an open cover of the compact space $X$, it has finite subcover $\{ B_{\frac{\delta_i}{2}}(x_i) : 1\leq i \leq n\}$ for some $n\in\N$. Pick \[
\delta= \min_{1\leq i \leq n} \delta_i / 2.
\]
Now, given $x,y \in X$ with $d_X(x,y)<\delta$, $\exists j \leq n$ such that $x\in B_{\frac{\delta_j}{2}}(x_j)$. Since $\delta \leq \frac{\delta_j}{2}$, by the triangle inequality $y\in B_{\delta_j}(x_j)$. So $f(x),f(y)\in B_{\frac{\varepsilon}{2}}(f(x_j))$ and by the triangle inequality
\[
d_Y(f(x), f(y))< \varepsilon.
\]
\end{proof}

Q: Continuous functions on a sequentially compact domain are uniformly continuous
A: Let $f:C\to V$ be a continuous function where $C,V$ are metric spaces and $C$ is sequentially compact. Then $f$ is uniformly continuous.
\begin{proof}
Suppose otherwise, then $\exists \varepsilon_0 > 0$ such that $\forall n\in \N: \exists (x_n,y_n)\in C$ where $d(x_n, y_n) < 1/n$ but $d(f(x_n), f(y_n))\geq \varepsilon_0$.
Since $x_n\in C$ which is sequentially compact, $\exists (n_k, x)$ such that $x_{n_k}\to x$.
Now, given $\varepsilon>0$, $\exists (N, K)\in \N$ such that
\begin{align*}
\forall n\geq N: d(x_n,y_n)&< \varepsilon / 2 \\
\forall k\geq K: d(x_{n_k},x)&<\varepsilon / 2.
\end{align*}
Hence $\forall k\geq \max(K,N)$:
\[
d(y_{n_k}, x) < d(y_{n_k}, x_{n_k}) + d(x_{n_k}, y_{n_k}) < \varepsilon / 2 + \varepsilon / 2 = \varepsilon.
\]
So $y_{n_k} \to x$. By continuity $f(x_{n_k}), f(y_{n_k}) \to f(x)$ so $f(x_{n_k})-f(x_{n_k}) \to 0$. Contradiction.
\end{proof}

Q: Integrability of continuous functions
A: Every $f\in C[a,b]$ is Riemann integrable.
\begin{proof}
Given $\varepsilon>0$, since $f$ is uniformly continuous on $[a,b]$, $\exists\delta >0$ such that $\forall x,y \in [a,b]$:
\[
 |x-y|<\delta\implies |f(x)-f(y)|<\displaystyle\frac{\varepsilon}{b-a}.
\]
Let the mesh of $\mathcal{D}$ be less than $\delta$ . Then
\begin{align*}
U_\mathcal{D}(f)-L_\mathcal{D}(f)&< \sum_{i=1}^n(x_i-x_{i-1})\frac{\varepsilon}{b-a} \\
&=\frac{\varepsilon}{b-a}(b-a) \\
&=\varepsilon.
\end{align*}
\end{proof}

Q: Riemann's integrability criterion
A: Let $f: [a, b] \to \R$. Then
\[
f \text{ is RI } \iff \forall\varepsilon>0: \exists \mathcal{D}: U_\mathcal{D}f - L_\mathcal{D}f < \varepsilon.
\]
\begin{proof}
  $(\Rightarrow)$ Suppose that $f$ is integrable, so there exist $\mathcal{D}_1$ and $\mathcal{D}_2$ such that
  \[
    U_{\mathcal{D}_1} < \int_a^b f(x)\;\d x + \frac{\varepsilon}{2},
  \]
  and
  \[
    L_{\mathcal{D}_2} > \int_a^b f(x)\;\d x - \frac{\varepsilon}{2}.
  \]
  Let $\mathcal{D}$ be a common refinement of $\mathcal{D}_1$ and $\mathcal{D}_2$. Then
  \[
    U_\mathcal{D} f - L_\mathcal{D} f \leq U_{\mathcal{D}_1} f- L_{\mathcal{D}_2} f < \varepsilon.
  \]
  $(\Leftarrow)$ Conversely, if there exists $\mathcal{D}$ such that
\[
U_\mathcal{D} f - L_\mathcal{D}f < \varepsilon,
\]
then
\[
\inf U_\mathcal{D} f - \sup L_\mathcal{D} f < \varepsilon
\]
and
\[
\overline{\int_a^b} f(x)\;\d x - \underline{\int_a^b} f(x)\;\d x < \varepsilon.
\]
  Since $\varepsilon > 0$ is arbitrary, this gives us that
  \[
    \overline{\int_a^b} f(x)\;\d x = \underline{\int_a^b} f(x)\;\d x.
  \]
  So $f$ is integrable.
\end{proof}

Q: Density of $P[a,b]$ in $C[a,b]$
A: If $f\in C[a,b]$, then $\exists f_n\in P[a,b]$ such that $f_n\to f$ uniformly on $[a,b]$.
\begin{proof}
Given $n\in N$, since $f$ is uniformly continuous on $[a,b]$, $\exists \delta >0 $ such that $\forall x,y \in [a,b]$:
\[
 |x-y|<\delta\implies |f(x)-f(y)|< 1/n.
\]
Then $\mathcal{D}_n$ with $|\mathcal{D}_n < \varepsilon$ satisfies $\forall x\in [a,b]$:
\[
|S_{\mathcal{D}_n}f(x) - f(x)| < 1/n.
\]
So $\| S_{\mathcal{D}_n}f - f\|_\infty < \frac{1}{n}$. and $S_{\mathcal{D}_n}f \to f$ uniformly.
\end{proof}

Q: Differentiation in $\R^n$
A: Let $U\subseteq \R^n$ be open, $\mathbf{f}: \R^n \to \R^m$. We say $\mathbf{f}$ is differentiable at a point $\mathbf{a} \in U$ if there exists a linear map $A: \R^n \to \R^m$ such that
  \[
    \lim_{\mathbf{h} \to \mathbf{0}} \frac{\mathbf{f}(\mathbf{a} + \mathbf{h}) - \mathbf{f}(\mathbf{a}) - A (\mathbf{h})}{\|\mathbf{h}\|} = 0.
  \]
  We call $A$ the \emph{derivative of $\mathbf{f}$ at $a$}. We write the derivative as $D \mathbf{f}(\mathbf{a})$.

Q: Uniqueness of derivative
A: \begin{prop}
  Derivatives are unique.
\end{prop}
\begin{proof}
  Suppose $A, B: \R^n \to \R^m$ both satisfy the condition
  \begin{align*}
    \lim_{\mathbf{h} \to \mathbf{0}} \frac{\mathbf{f}(\mathbf{a} + \mathbf{h}) - \mathbf{f}(\mathbf{a}) - A \mathbf{h}}{\|\mathbf{h}\|} &= \mathbf{0}\\
    \lim_{\mathbf{h} \to \mathbf{0}} \frac{\mathbf{f}(\mathbf{a} + \mathbf{h}) - \mathbf{f}(\mathbf{a}) - B \mathbf{h}}{\|\mathbf{h}\|} &= \mathbf{0}.
  \end{align*}
  Subtracting gives
  \[
   \lim_{\mathbf{h} \to \mathbf{0}} \frac{(B - A)\mathbf{h}}{\|\mathbf{h}\|} = 0.
  \]
We set $\mathbf{h} = t\mathbf{v}$ for fixed $\mathbf{v}\neq 0 \in\R^n$ to obtain
\begin{align*}
0 &= \lim_{t\to 0 ^+} \frac{B(t\mathbf{v})-A(t\mathbf{v})}{\|t\mathbf{v}\|} \\
&= \lim_{t\to 0 ^+} \frac{t(B(\mathbf{v})-A(\mathbf{v}))}{t \|\mathbf{v}\|} \\
&= \frac{B(\mathbf{v})-A(\mathbf{v})}{\|\mathbf{v}\|}.
\end{align*}
So $A=B$.
\end{proof}

Q: Linear maps are Lipschitz
A: \begin{lemma}
If $L:\R^n\to (W,\|\ph\|)$ is a linear map, then $L$ is Lipschitz.
\end{lemma}
\begin{proof}
Let $\mathbf{v}=(v_1,\ldots v_n)=\sum_{i=1}^n v_i\mathbf{e}_i$. Then
\begin{align*}
A(\mathbf{v}) &= \sum_{i=1}^n v_iA(\mathbf{e}_i) \\
\|A(\mathbf{v}) \|_1 &= \sum_{i=1}^n |v_i|\|A(\mathbf{e}_i)\| \\
&\leq C\|\mathbf{v} \|_1 \\
\end{align*}
where $C=\max_{1\leq i \leq n} \|A(\mathbf{e}_i)\|$. So $L$ is Lipschitz.
\end{proof}

Q: Differentiable implies continuous
A: If $f:U\to \R^m$ is differentiable at $\mathbf{a}\in U$, then $f$ is continuous at $\mathbf{a}$.
\begin{proof}
Since $f$ is differentiable at $\mathbf{a}$:
\begin{align*}
f(\mathbf{a}+\mathbf{h})- f(\mathbf{a}) &= Df(\mathbf{a})(\mathbf{h}) + o(\mathbf{h}) \\
\lim_{\mathbf{h}\to 0} [f(\mathbf{a}+\mathbf{h})- f(\mathbf{a})] &= 0.
\end{align*}
\end{proof}

Q: Directional derivative
A: Let $\mathbf{f}: U \to \R^m$ be differentiable at $\mathbf{a} \in U$ and $\mathbf{u} \in \R^n$. Then
\[
D_\mathbf{u}f|_a=Df|_a(\mathbf{u}).
\]
\begin{proof}
If $\mathbf{u}=0$, then both sides are 0. If $\mathbf{u}\neq 0$, since $t\to 0 \implies t\mathbf{u}\to 0$, then
\begin{align*}
0 &= \lim_{t \to 0} \frac{\mathbf{f}(\mathbf{a} + t\mathbf{u}) - \mathbf{f}(\mathbf{a}) - D \mathbf{f}(\mathbf{a}) (t\mathbf{u})}{\|t \mathbf{u}\|}\\
&= \lim_{t \to 0} \frac{\mathbf{f}(\mathbf{a} + t\mathbf{u}) - \mathbf{f}(\mathbf{a}) - t D \mathbf{f}(\mathbf{a}) \mathbf{u}}{|t|\|\mathbf{u}\|} \\
&= \lim_{t \to 0} \frac{\mathbf{f}(\mathbf{a} + t\mathbf{u}) - \mathbf{f}(\mathbf{a}) - t D \mathbf{f}(\mathbf{a}) \mathbf{u}}{t} \\
 D\mathbf{f}(\mathbf{a}) \mathbf{u} &= \lim_{t \to 0} \frac{\mathbf{f}(\mathbf{a} + t\mathbf{u}) - \mathbf{f}(\mathbf{a})}{t}.
\end{align*}
\end{proof}

Q: Partial derivative
A: The \emph{$j$th partial derivative} of $f: U \to \R$ at $\mathbf{a} \in U$ is
\[
D_{\mathbf{e}_j} f(\mathbf{a}) = \lim_{t \to \infty} \frac{f(\mathbf{a} + t \mathbf{e}_j) - f(\mathbf{a})}{t}
\]
when the limit exists. We often write this as
\[
D_{\mathbf{e}_j} f(\mathbf{a}) = D_j f(\mathbf{a}) = \frac{\partial f}{\partial x_j}.
\]
This is simply the rate of change with all but th $j$th parameter fixed:
\begin{align*}
D_{\mathbf{e}_j} f(\mathbf{a}) &=  \lim_{t \to \infty} \frac{f(\mathbf{a} + t \mathbf{e}_j) - f(\mathbf{a})}{t} \\
&=\left.\frac{\d}{\d t} f(\mathbf{a} + t \mathbf{e}_j)\right|_{t=0} \\
&=\left.\frac{\d}{\d t} f(a_1 + \ldots + (a_j+t) + \ldots a_n)\right|_{t=0} \\
&=\left.\frac{\d}{\d t} f(a_1 + \ldots + t + \ldots a_n)\right|_{t=a_j}.
\end{align*}

Q: Componentwise differentiation
A: Let $f:U\to \R^m$ be a function where $U\subseteq\R^n$. Then $f$ is differentiable $\iff$ $f_i$ is differentiable for $1\leq i \leq m$. Also
\[
Df|_a= (Df_1|_a, \ldots Df_m|a).
\]
\begin{proof}
Let $L(v)=(L_1(v), \ldots L_m(v))$. Then using $\|\ph\|_\infty$ on $\R^m$
\begin{align*}
&0 = \lim_{h\to 0} \frac{f(a+h)-f(a)-L(h)}{\| h \|} \\
\iff &0=\lim_{h\to 0} \frac{f_i(a+h)-f_i(a)-L_i(h)}{\| h \|}\quad  (\forall 1\leq i \leq m) \\
\iff & f_i \text{ is differentiable at }a \text{ and } Df_i|_a = L_i \quad (\forall 1\leq i \leq m).
\end{align*}
\end{proof}

Q: One-sided directional derivatives
A: For $f:U\to \R^m$ with $U$ open, we write
\[
D^+_vf=\lim_{t\to 0^+} \frac{f(x+tv)-f(x)}{t}
\]
and
\[
D^-_vf=\lim_{t\to 0^-} \frac{f(x+tv)-f(x)}{t}.
\]
Then we have that $D_v^-f|_x=-D_{-v}^+f|_x$ for any $x\in U$ for which the derivative exists.
\begin{proof}
Letting $t\mapsto -t$ and $v\mapsto -v$ we have
\begin{align*}
D^-_vf&=\lim_{t\to 0^-} \frac{f(x+tv)-f(x)}{t} \\
&=\lim_{t\to 0^+} \frac{f(x+(-t)(-v))-f(x)}{-t} \\
&=-\lim_{t\to 0^+} \frac{f(x+tv)-f(x)}{t} \\
&=-D_{-v}^+f|_x.
\end{align*}
\end{proof}

Q: Sufficient condition for differentiability
A: \begin{thm}
Let $U\subseteq \R^n$ be open, $f: U \to \R^m$ and $\mathbf{a} \in U$. If $f_i\in C_1(U)=\{g:U\to\R : D_jg|_\mathbf{x} \text{ is continuous for }1\leq j \leq n \}$  for $1\leq i \leq m$, then $f$ is differentiable at $\mathbf{a}$.
\end{thm}
\begin{proof}
By componentwise differentiation, wlog $m = 1$. For each $\mathbf{h} = (h_1, \cdots, h_n) \in \R^n$, write $\mathbf{h}^{(j)} = h_1 \mathbf{e}_1 + \cdots + h_j \mathbf{e}_j$. By telescoping
\begin{align*}
 f(\mathbf{a} + \mathbf{h}) - f(\mathbf{a}) &= \sum_{j = 1}^n f(\mathbf{a} + \mathbf{h}^{(j)}) - f(a + \mathbf{h}^{(j - 1)}) \\
 &= \sum_{j = 1}^n f(\mathbf{a} + \mathbf{h}^{(j - 1)} + h_j \mathbf{e}_j) - f(\mathbf{a} + \mathbf{h}^{(j - 1)}) \\
  &= \sum_{j = 1}^n h_j D_j f (\mathbf{a} + \mathbf{h}^{(j - 1)} + \theta_j h_j \mathbf{e}_j)\\
&= \sum_{j = 1}^n h_j D_j f(\mathbf{a}) + h_j \left(D_j f (\mathbf{a} + \mathbf{h}^{(j - 1)} + \theta_j h_j \mathbf{e}_j) - D_j f(\mathbf{a})\right) \\
&=  \sum_{j = 1}^n h_j D_jf(\mathbf{a}) +o(\mathbf{h})
 \end{align*}
where $\theta_j \in (0, 1)$ and we applied the mean value theorem of single-variable calculus to
\[
g(t) = f(\mathbf{a} + \mathbf{h}^{(j - 1)} + t \mathbf{e}_j)
\]
on the interval $t \in [0, h_j]$.
So $f$ is differentiable at $\mathbf{a}$ with
\[
D f (\mathbf{a})\mathbf{h}= \sum_{j = 1}^n h_jD_j f (\mathbf{a}).
\]
\end{proof}

Q: Chain rule
A: Let $U \subseteq \R^n$ be open, $\mathbf{a} \in U$, $\mathbf{f}: U \to \R^m$ differentiable at $\mathbf{a}$. Assume further that $V \subseteq \R^m$ is open with $\mathbf{f}(U) \subseteq V$ and $\mathbf{g}: V \to \R^p$ is differentiable at $\mathbf{f}(\mathbf{a})$. Then $\mathbf{g} \circ \mathbf{f}: U \to \R^p$ is differentiable at $\mathbf{a}$, with derivative
\[
D(\mathbf{g} \circ \mathbf{f})(\mathbf{a}) = D \mathbf{g}(\mathbf{f}(\mathbf{a}))\circ D \mathbf{f}(\mathbf{a}).
\]
\begin{proof}
Let $A = D\mathbf{f}(\mathbf{a})$ and $B = D \mathbf{g}(\mathbf{f}(\mathbf{a}))$. By differentiability of $\mathbf{f}$ and $\mathbf{g}$
\begin{align*}
\mathbf{f}(\mathbf{a} + \mathbf{h}) &= \mathbf{f}(\mathbf{a}) + A \mathbf{h} + o(\mathbf{h})\\
\mathbf{g}(\mathbf{f}(\mathbf{a}) + \mathbf{k}) &= \mathbf{g}(\mathbf{f}(\mathbf{a})) + B\mathbf{k} + o(\mathbf{k}) \\
\implies \mathbf{g}\circ \mathbf{f}(\mathbf{a} + \mathbf{h}) &= \mathbf{g}(\mathbf{f}(\mathbf{a}) + A \mathbf{h} + o(\mathbf{h}))\\
&= \mathbf{g}(\mathbf{f}(\mathbf{a})) + B(A \mathbf{h} + o(\mathbf{h})) + o(A \mathbf{h} + o(\mathbf{h}))\\
&= \mathbf{g}\circ \mathbf{f}(\mathbf{a}) + B A \mathbf{h} + B(o(\mathbf{h})) + o(A \mathbf{h} + o(\mathbf{h})) \\
&= \mathbf{g} \circ \mathbf{f}(\mathbf{a}) + BA \mathbf{h} + o(\mathbf{h}).
\end{align*}
\end{proof}

Q: Little-o notation and linear maps
A: \begin{lemma}
If $L:\R^n\to\R^m$ is a linear map defined on $B_r(0)$ for some $r>0$, then $L(o(\mathbf{h}))=o(\mathbf{h})$ and $o(L \mathbf{h})=o(\mathbf{h})$.
\end{lemma}
\begin{proof}
We have
\[
\|L(o(\mathbf{h}))\| \leq \|L\| \|o(\mathbf{h})\|=o(\mathbf{h})
\]
and
\[
\|L\mathbf{h}\|\leq \|L\| \|\mathbf{h}\| \implies o(L \mathbf{h})=o(\mathbf{h}).
\]
\end{proof}

Q: Gradient
A: Suppose $f:U\to\R$ where $U\subseteq\R^n$ is open such that $Df\in\mathcal{L}(\R^n,\R)$.
We know that $\R^n\simeq \mathcal{L}(\R^n,\R)$ by $\mathbf{x}\mapsto\psi_\mathbf{x}$ where $\psi_\mathbf{x}(\mathbf{y})=\mathbf{x}\cdot\mathbf{y}$.
Hence $\nabla f|_\mathbf{x}=(D_1f|_\mathbf{x}, \ldots D_nf\mathbf{x})$, called the \emph{gradient} of $f$ at $\mathbf{x}$, is the unique vector such that
\[
D_\mathbf{v}f|_\mathbf{x}=Df|_\mathbf{x} (\mathbf{v})=\nabla f|_\mathbf{x} \cdot \mathbf{v} \quad (\forall \mathbf{v},\mathbf{x}\in\R^n).
\]

Q: Mean Value Inequality
A: Let $U\subseteq\R^n$ be open and convex. Let $F:U\to\R$ be differentiable. If $\forall\mathbf{x}\in U: \| \nabla f|_\mathbf{x} \|\leq M$, then $\forall \mathbf{x}_0,\mathbf{x}_1\in U$:
\[
|f(\mathbf{x}_1)-f(\mathbf{x}_0)| \leq M\|\mathbf{x}_1-\mathbf{x}_0\|.
\]
\begin{proof}
Let $\gamma:[0,1]\to\R^n$ be given by $\gamma(t)=(1-t)\mathbf{x}_0 + t\mathbf{x}_1$. Then $\gamma'(t)=\mathbf{x}_1-\mathbf{x}_0$.
Let $f(t)=F(\gamma(t))$. By the Chain Rule, $f$ is differentiable and
\[
f'(t)=\nabla f|_{\gamma(t)} \cdot\gamma'(t).
\]
By Cauchy-Schwarz
\begin{align*}
f'(t)&\leq \|\nabla f|_{\gamma(t)} \| \|\gamma'(t) \| \\
&=\|\nabla f|_{\gamma(t)} \| \| \mathbf{x}_1-\mathbf{x}_0\| \\
&\leq M \| \mathbf{x}_1-\mathbf{x}_0\|.
\end{align*}
By 1-D Mean Value Theorem, $\exists c\in [0,1]$ such that
\[
F(\mathbf{x}_1)-F(\mathbf{x}_0)=f(1)-f(0)=f'(c).
\]
Hence
\[
|F(\mathbf{x}_1)-F(\mathbf{x}_0)|=|f'(c)|\leq M \| \mathbf{x}_1-\mathbf{x}_0\|.
\]
\end{proof}

Q: Zero derivative function on a convex domain
A: If $U\subseteq\R^n$ is open and convex, $F:U\to\R$ has $D_i F=0$ for $1\leq i \leq n$, then $F(\mathbf{x})=$constant.
\begin{proof}
By the Mean Value Inequality, $\forall \mathbf{x}_0, \mathbf{x}_1\in U$:
\[
|F(\mathbf{x}_1)-F(\mathbf{x}_0)|\leq 0 \cdot \|\mathbf{x}_1-\mathbf{x}_0 \| = 0.
\]
\end{proof}

Q: Zero derivative function on a differentiably path connected domain
A: If $U\subseteq\R^n$ is open and differentiably path connected and $D_iF=0$ for all $1\leq i \leq n$, then $F=$ constant on $U$.
\begin{proof}
Given $\mathbf{x_0},\mathbf{x_1}\in U$, $\exists \gamma: [0,1]\to U$ differentiable such that $\gamma(0)=\mathbf{x}_0$ and $\gamma(1)=\mathbf{x}_1$. Let $f(t)=F(\gamma(t))$. By the Chain Rule $f'(t)=DF|_{\gamma(t)} (\gamma'(t))$. So
\begin{align*}
D_iF=0 &\implies DF=0\\
&\implies f'(t)=0 \\
&\implies f = \text{ constant} \\
&\implies f(1)=F(\mathbf{x_1})=F(\mathbf{x_0})=f(0).
\end{align*}
\end{proof}

Q: Zero derivative function on a path connected domain
A: \begin{thm}
Let $U\subseteq \R^m$ be open and path-connected. Then for any differentiable $\mathbf{F}: U \to \R^m$, if $D_iF= 0$ for all $1\leq i \leq n$, then $F$ is constant on $U$.
\end{thm}
\begin{proof}
Since $D_iF=0$ for all $1\leq i \leq n$, we have $Df=0$.
Given any $\mathbf{a}, \mathbf{b} \in U, \exists \gamma: [0, 1] \to U$ continuous such that $\gamma(0)=\mathbf{a}$ and $\gamma(1)=\mathbf{b}$.
 Given $s \in (0, 1)$, since $U$ is open, $\exists\varepsilon>0$ such that $B_\varepsilon(\gamma(s)) \subseteq U$. By continuity of $\gamma$, $\exists\delta>0$ such that $(s - \delta, s + \delta) \subseteq [0, 1]$ with $\gamma((s - \delta, s + \delta)) \subseteq B_\varepsilon(\gamma(s)) \subseteq U$. By convexity, $F$ is constant on $B_\varepsilon(\gamma(s))$. So $f(t) = F \circ \gamma (t)$ is constant on $(s - \delta, s + \delta)$. In particular, $f'(s)=0$.
Therefore $f'=0$ on $(0, 1)$ and hence $f$ is constant on $[0,1]$. So $f(0)= f(\mathbf{a}) = f(\mathbf{b})= f(1)$.
\end{proof}

Q: Second derivative is a symmetric bilinear form
A: Let $U\subseteq\R^n$ be open and $f\in C^2(U,\R^m)$. Then
\[
D^2 \mathbf{f} (\mathbf{a})(\mathbf{u}, \mathbf{v}) = \sum_j \sum_i D_{ij}\mathbf{f}(\mathbf{a}) u_i v_j=\mathbf{u}^T H \mathbf{v}
\]
is a symmetric bilinear form.
\begin{proof}
Symmetry follows from Schwarz's Theorem:
\[
\mathbf{u}^T H \mathbf{v}=(\mathbf{u}^T H \mathbf{v})^T=\mathbf{v}^T H^T \mathbf{u}=\mathbf{v}^T H \mathbf{u}.
\]
By linearity of the first derivative
\[
 Df|_\mathbf{x} (\mathbf{v}_1+\lambda\mathbf{v}_2)=Df|_\mathbf{x} (\mathbf{v}_1)+\lambda Df|_\mathbf{x} (\mathbf{v}_2).
\]
Differentiating gives linearity in the first argument \vspace{-0.9cm}
\[
D^2f|_\mathbf{x} (\mathbf{v}_1+\lambda\mathbf{v}_2)=D^2f|_\mathbf{x} (\mathbf{v}_1,\mathbf{w})+\lambda D^2f|_\mathbf{x} (\mathbf{v}_2,\mathbf{w}).
\]
By linearity of the second derivative we obtain linearity in the second argument:
\begin{align*}
D^2f|_\mathbf{x} (\mathbf{v},\mathbf{w}_1+\lambda\mathbf{w}_2)&=D(Df|_\mathbf{x} (\mathbf{v}))(\mathbf{w}_1+\lambda\mathbf{w}_2)  \\
&=D(Df|_\mathbf{x} (\mathbf{v}))(\mathbf{w}_1) + \lambda D(Df|_\mathbf{x} (\mathbf{v}))(\mathbf{w}_2)) \\
&=D^2f|_\mathbf{x} (\mathbf{v},\mathbf{w}_1)+\lambda D^2f|_\mathbf{x} (\mathbf{v},\mathbf{w}_2).
\end{align*}
\end{proof}

Q: Sufficient condition for twice differentiability
A: If $F\in C^2(U)=\{F:U\to\R \ | \ D_{ij}F|_\mathbf{x}\in C^0(U)$ for all $1\leq i, j \leq n\}$, then $F$ is twice differentiable and
\[
D^2F|_\mathbf{x} (\mathbf{v},\mathbf{w})=\sum_{1\leq i, j \leq n} v_i w_j D_{ji}F(\mathbf{x})=\mathbf{w}^TH\mathbf{v}.
\]
\begin{proof}
All first order partial derivatives of $D_iF$ are continuous so $D_iF$ is differentiable.
Then $\forall\mathbf{v}\in\R^n$:
\[
DF|_\mathbf{x} (\mathbf{v}) = \sum_{1 \leq i \leq n} v_iD_iF_\mathbf{x}.
\]
So $DF|_\mathbf{x}$ is a linear combination of $D_iF$s and thus differentiable. Hence $F$ is twice differentiable and
\[
D^2F|_\mathbf{x}(\mathbf{v},\mathbf{w})=D(DF|_\mathbf{x}(\mathbf{v}))(\mathbf{w})=\sum_{1\leq j \leq n}w_j D_j(DF|_\mathbf{x}(\mathbf{v})).
\]
\end{proof}

Q: Symmetry of mixed partials
A: \begin{thm}[Schwarz]
Let $U \subseteq \R^n$ be open, $f\in C^2(U,\R^m)$ and $\mathbf{a} \in U$. Then for $i, j \in \{1, \cdots, n\}$:
\[
D_i D_j \mathbf{f}(\mathbf{a}) = D_j D_i \mathbf{f}(\mathbf{a}).
\]
\end{thm}
\begin{proof}
Wlog $m=1$ and $i\neq j$. Let $\mathbf{a}\in B_r(\mathbf{a})$ and
\[
g_{ij}(t) = f(\mathbf{a} + t \mathbf{e}_i + t \mathbf{e}_j) - f(\mathbf{a} + t \mathbf{e}_i) - f(\mathbf{a} + t \mathbf{e}_j) + f(\mathbf{a})
\]
be defined for $t<r / \sqrt{2}$. Now for each fixed $t$, define $\phi: [0, 1] \to \R$ by
\[
\phi(s) = f(\mathbf{a} + st \mathbf{e}_i + t \mathbf{e}_j) - f(\mathbf{a} + st \mathbf{e}_i)
\]
such that $g_{ij}(t) = \phi(1) - \phi(0)$. By MVT and CR, $\exists\theta \in (0, 1)$ such that
\[
g_{ij}(t)= \phi'(\theta) = t\Big(D_i(\mathbf{a} + \theta t \mathbf{e}_i + t \mathbf{e}_j) - D_i(\mathbf{a} + \theta t \mathbf{e}_i)\Big).
\]
Let $\psi(s)=D_i (\mathbf{a} + \theta t \mathbf{e}_i + st \mathbf{e}_j)$ such that $g_{ij}(t)/t = \psi(1)-\psi(0)$. By MVT, $\exists\eta \in (0, 1)$ such that
\[
g_{ij}(t)/t =\psi'(\eta) = t D_j D_i f(\mathbf{a} + \theta t \mathbf{e}_i + \eta t \mathbf{e}_j).
\]
 By continuity of the partial derivatives
\[
\lim_{t\to 0} \frac{g_{ij}(t)}{t^2} = \lim_{t\to 0} D_j D_i f(\mathbf{a} + \theta t \mathbf{e}_i + \eta t \mathbf{e}_j)=D_j D_i f(\mathbf{a}).
\]
So done by symmetry $i\leftrightarrow j$ in $g_{ij}(t)/t^2$.
\end{proof}

Q: Higher dimensional directional derivative
A: Suppose $F\in C^k(U)$ where $U\subseteq\R^n$ is open and $\mathbf{a}\in U$. Define $f(t)=F(\mathbf{a}+t\mathbf{v})$. Then $f$ is $k$-times differentiable and
\[
f^{(k)}(t)=D^kF_{\mathbf{a}+t\mathbf{v}}\underbrace{(\mathbf{v}, \ldots \mathbf{v})}_{k}.
\]
\begin{proof}
Recall that if $G\in C^1(U)$, $g(t)=G(\mathbf{a}+t\mathbf{v})$, then
\[
g'(t)=D_\mathbf{v}G|_{\mathbf{a}+t\mathbf{v}}=DG|_{\mathbf{a}+t\mathbf{v}}(\mathbf{v}).
\]
Proceed by induction on $k$. Let \[
h(t)=f^{(k-1)}(t)=D^{k-1}F|_{\mathbf{a}+t\mathbf{v}}\underbrace{(\mathbf{v},\ldots \mathbf{v})}_{k-1}=H(\mathbf{a}+t\mathbf{v})
\]
where $H(\mathbf{x})=D^{k-1}F|_\mathbf{x}\underbrace{(\mathbf{v}, \ldots \mathbf{v})}_{k-1}$. Using the $k=1$ result
\[
f^{(k)}(t)=h'(t)=DH_{\mathbf{a}+t\mathbf{v}}(\mathbf{v})=D^kF_{\mathbf{a}+t\mathbf{v}}\underbrace{(\mathbf{v},\ldots \mathbf{v})}_k.
\]
\end{proof}

Q: Taylor's Theorem
A: \begin{thm}
Let $U\in\R^n$ be open, $F\in C^k(U)$ and assume $\mathbf{a}+t\mathbf{h}\in U$ for all $t\in[0,1]$. Then $\exists\theta\in[0,1]$ such that
\[
F(\mathbf{a}+\mathbf{h})=\sum_{i=0}^{k-1} \frac{1}{i!} D^iF|_\mathbf{a}(\mathbf{h},\ldots \mathbf{h}) + \frac{1}{k!}D^kF|_{\mathbf{a}+\theta\mathbf{h}}(\mathbf{h},\ldots \mathbf{h}).
\]
\end{thm}
\begin{proof}
Apply Taylor's Theorem in a single variable to $f:[0,1]\to\R$ given by $f(t)=F(\mathbf{a}+t\mathbf{h})$. Then $\exists\theta\in[0,1]$ such that
\[
f(1)=\sum_{i=0}^{k-1} \frac{1}{i!} f^{(i)}(0)+ \frac{1}{k!}f^{(k)}(\theta).
\]
That is
\[
F(\mathbf{a}+\mathbf{h})=\sum_{i=0}^{k-1} \frac{1}{i!} D^iF|_\mathbf{a}(\mathbf{h},\ldots \mathbf{h}) + \frac{1}{k!}D^kF|_{\mathbf{a}+\theta\mathbf{h}}(\mathbf{h},\ldots \mathbf{h}).
\]
\end{proof}

Q: Lipschitz maps
A: Let $(X,d_X), (Y,d_Y)$ be metric spaces and $f:X\to Y$. We say that $f$ is $K-$\emph{Lipschitz} on $X$ if $\exists K \in [0, \infty)$ such that $\forall x_1, x_2 \in X$:
\[
d_Y(f(x_1), f(x_2)) \leq K d_X(x_1, x_2).
\]
\emph{Any} such $K$ is called a Lipschitz constant.

Q: Lipschitz vs Uniform continuity
A: Let $(X,d_X), (Y,d_Y)$ be metric spaces and $f:X\to Y$. Then $f$ Lipschitz $\implies$ uniformly continuous but not vice versa.
\begin{proof}
Suppose $f$ is Lipschitz. Given $\varepsilon>0$, pick $\delta=\frac{\varepsilon}{K}$. Then $\forall x_1,x_2\in X$:
\[
d_X(x_1,x_2)<\delta \implies d_Y(f(x_1),f(x_2)) < K \frac{\varepsilon}{K} = \varepsilon.
\]
So $f$ is uniformly continuous.
Conversely, take $X = Y = \R$, $f=\id$ and
\[
d_X(x_1, x_2) = \min\{1, |x_1 - x_2|\}, \quad d_Y(y_1, y_2) = |y_1 - y_2|.
\]
\textbf{Uniform continuity}. Given $\varepsilon>0$, pick $\delta=\min(1,\varepsilon)$. Then $\forall x_1,x_2\in X$:
\[
d_X(x_1,x_2)<\delta \implies |x_1-x_2|<\varepsilon \implies d_Y(f(x_1),f(x_2))<\varepsilon.
\]
\textbf{Lipschitz}. Given $K>0$, pick $x=K+1$. Then
\[
d_X(0,x) =1, d_Y(f(0),f(x))=x> K.
\]
So $f$ is not Lipschitz.
\end{proof}

Q: $C^1$ functions on a compact set
A: Suppose $U\subseteq\R^n$ is open, $F\in C^1(U)$ and $K=\bar{B}_r(x_0)\subseteq U$. Then $F|_K$ is Lipschitz.
\begin{proof}
Since $F\in C^1(U)$, the map
\begin{align*}
U&\to\R^n\to\R \\
\mathbf{x}&\mapsto \nabla F|_\mathbf{x}\mapsto \|\nabla F|_\mathbf{x} \|
\end{align*}
is continuous. Since $B_r(\mathbf{x}_0)$ is closed and bounded in $\R^n$, it is compact. By Maximum Value Theorem, $\exists M\in\R$ such that $\forall\mathbf{x}\in K:\|\nabla F|_\mathbf{x}\|\leq M$.
Since $K=\bar{B}_r(\mathbf{x}_0)$ is convex, by the Mean Value Inequality
\[
|F(\mathbf{x}_1)-F(\mathbf{x}_0) | \leq M\|\mathbf{x}_1-\mathbf{x}_2\|_2.
\]
\end{proof}

Q: Operator norm
A: The \emph{operator norm} on $\mathcal{L} = \mathcal{L}(\R^n, \R^m)$ is defined by
\[
\|A\| = \sup_{ \R^n\setminus \{0\}} \frac{\|A \mathbf{x}\|}{\|\mathbf{x}\|}.
\]
Since $A$ is continuous and $\{\mathbf{x} \in \R^n: \|\mathbf{x}\| = 1\}$ is compact, the $\sup$ exists and in fact
\[
\|A\| = \max_{\mathbf{x} \in \R^n: \|\mathbf{x}\| = 1} \|A \mathbf{x}\|.
\]
The fact that it is a norm is inherited from the norm on $\R^m$.
Note $\|A\|=K\iff A$ is $K-$Lipschitz.

Q: Operator norm of a composition
A: For any $A,B\in\mathcal{L}(\R^n,\R^m)$:
\[
\|B A\| \leq \|B\| \|A\|.
\]
\begin{proof} Using $\|A \mathbf{x}\| \leq \|A\| \|\mathbf{x}\|$ for $\mathbf{x}\neq 0$:
\[
\|BA\| = \sup_{\R^n\setminus \{0\}} \frac{\|BA \mathbf{x}\|}{\|\mathbf{x}\|} \leq \sup_{\R^n \setminus \{0\}} \frac{\|B\| \|A \mathbf{x}\|}{\|\mathbf{x}\|} = \|B\|\|A\|.
\]
\end{proof}
\begin{cor}
If $A$ is invertible, then \[
\|A\|_\text{op}\|A^{-1}\|_\text{op}=\|\id\|_\text{op}=1.
\]
\end{cor}

Q: Contraction mapping
A: Let $(X, d)$ be a metric space. A mapping $f: X \to X$ is a \emph{contraction} if $f$ is $K-$Lipschitz for $K<1$, i.e.
\[
d(f(x), f(y)) \leq K d(x, y)<d(x,y).
\]

Q: Contraction mapping theorem
A: \begin{thm}
If $X$ be a non-empty complete metric space, and $f: X \to X$ a contraction, then $f$ has a \emph{unique} fixed point.
\end{thm}
\begin{proof}
\textbf{Uniqueness}. By assumption, $\exists K < 1$ such that $\forall x, y \in X:$
\[
d(f(x), f(y)) \leq K d(x, y).
\]
If $x$ and $y$ are both fixed points, then this says
\[
d(x, y) = d(f(x), f(y)) \leq K d(x, y).
\]
This is possible only if $d(x, y) = 0$, ie. $x = y$.
\textbf{Existence}. Pick your favourite $x \in X$. Then $(f^n(x))$ is Cauchy: for any $m>n\geq N$
\begin{align*}
d(f^m(x),f^n(x)) &=d(f^n(x),f^n(f^{m-n}(x)))\\
&\leq K^nd(x,f^{m-n}(x) \\
&\leq \frac{K^N}{1-K} d(x,f(x)) \\
&\to 0 \text{ as } N \to \infty.
\end{align*}
Since $X$ is complete, $f^n(x)\to x^*\in X$. As $f$ is a contraction, it is continuous. Hence $f(f^n(x))\to f(x^*)$ and $f(f^n(x))=f^{n+1}(x)\to x^*$. So $x^*=f(x^*)$.
\end{proof}
\begin{cor}
More generally, if $f^{m}$ is a contraction for some $m\in\N$, then $f$ has a unique fixed point.
\end{cor}
\begin{proof}
By Theorem, $\exists ! x^* \in X$ such that $f^{(m)}(x^*) = x^*$. But then
\[
f^{(m)}(f(x^*)) = f(f^{(m)}(x^*)) = f(x^*).
\]
So $f(x)$ is also a fixed point of $f^{m}$. By uniqueness of fixed points, we must have $f(x^*) = x^*$. Since any fixed point of $f$ is also a fixed point of $f^{(m)}$, it follows that $x^*$ is the unique fixed point of $f$.
\end{proof}

Q: Error term for composition of contraction maps
A: If $f: X \to X$ is a contraction such that $f$ is $K-$Lipschitz, then $\forall x \in X$:
\[
d(f^n(x),x) \leq  \frac{1}{1 - K} d(f(x), x).
\]
\begin{proof}
For any $m\in\N$, since $f^m$ is $K^m$ Lipschitz, we have
\[
d(f^m(x),f^{m+1}(x))=d(f^m(x),f^m(f(x)))\leq K^md(x,f(x)).
\]
Hence by the triangle inequality
\begin{align*}
d(f^n(x),x) &\leq \sum_{m = 0}^{n - 1} d(f^{m + 1}(x), f^m(x))\\
&\leq \sum_{m=0}^{n - 1} K^m d(f(x), x)\\
&\leq d(f(x), x)\sum_{m=0}^{\infty} K^m \\
&= \frac{1}{1 - K} d(f(x), x).
\end{align*}
\end{proof}

Q: Newton's Method
A: Let $U\subseteq\R^n$ and $f:U\to\R^n$ be $C^1$ with $Df|_\mathbf{x}$ invertible for all $\mathbf{x}\in U$. Then fixed points of
\[
n_\mathbf{y}(\mathbf{x})=\mathbf{x}+Df|_\mathbf{x}^{-1}(\mathbf{y}-f(\mathbf{x}))
\]
are solutions to $f(\mathbf{x})=\mathbf{y}$.
\begin{proof}
We can approximate $f(\mathbf{x})$ near $\mathbf{x}_0$ as
\[
g(\mathbf{x})=f(\mathbf{x}_0)+Df|_{\mathbf{x}_0}(\mathbf{x}-\mathbf{x}_0).
\]
So the solution to $g(\mathbf{x})=\mathbf{y}$ is
\[
\mathbf{x}_1=\mathbf{x}_0+Df|_{\mathbf{x}_0}^{-1}(\mathbf{y}-f(\mathbf{x}_0)) = n_\mathbf{y}(\mathbf{x}_0).
\]
We want $(\mathbf{x}_k)=n_\mathbf{y}^k(\mathbf{x}_0)$ to converge to a solution of $f(\mathbf{x})=\mathbf{y}$. If $\mathbf{x}$ is a fixed point of $n_\mathbf{y}$, then
\[
\mathbf{x}=\mathbf{x}+Df|_\mathbf{x}^{-1}(\mathbf{y}-f(\mathbf{x})).
\]
So indeed $f(\mathbf{x})=\mathbf{y}$.
\end{proof}

Q: Modified Newton's Method
A: Suppose $f:U\to\R^n$ is $C^1$, $U\subseteq\R^n$ open, $\exists\mathbf{x}_0\in U: f(\mathbf{x}_0)=\mathbf{y}$ and $A=Df|_{\mathbf{x}_0}\in\mathcal{L}(\R^n,\R^n)$ is invertible. If $\mathbf{x}$ is a fixed point of
\[
N_\mathbf{y}(\mathbf{x})=\mathbf{x}+A^{-1}(\mathbf{y}-f(\mathbf{x})),
\]
then $\mathbf{y}=f(\mathbf{x})$. Moreover $\forall\varepsilon >0, \exists\delta>0$ s.t. $N_\mathbf{y}$ is $\varepsilon$-Lipschitz on $B_\delta(\mathbf{x}_0)$ and hence a fixed point exists.
\begin{proof}
If $N_\mathbf{y}(\mathbf{x})=\mathbf{x}$, then
\[
\mathbf{x}=\mathbf{x}+A^{-1}(\mathbf{y}-f(\mathbf{x}))
\]
so indeed $\mathbf{y}=f(\mathbf{x})$.
Now $\forall \mathbf{x},\mathbf{x'}\in U$:\vspace{-0.5cm}
\begin{align*}
N_\mathbf{y}(\mathbf{x})-N_\mathbf{y}(\mathbf{x}')&=\mathbf{x}-\mathbf{x}'+A^{-1}(f(\mathbf{x}')-f(\mathbf{x})) \\
&=A^{-1}(A(\mathbf{x}-\mathbf{x}')- [f(\mathbf{x})-f(\mathbf{x}')])\\
&=A^{-1}(h(\mathbf{x})-h(\mathbf{x}')) \\
\intertext{where $h(\mathbf{x})=A(\mathbf{x})-f(\mathbf{x})$ such that}
Dh|_\mathbf{x}&=DA|_\mathbf{x}-Df|_\mathbf{x} \\
&=A|_\mathbf{x}-Df|_\mathbf{x}\\
&=Df|_{\mathbf{x}_0} - Df|_\mathbf{x}.
\end{align*}
Given $\varepsilon>0$, since $f$ is $C^1$, $\exists \delta>0$ such that $\forall\mathbf{x}\in B_\delta(\mathbf{x}_0):\|Dh|_\mathbf{x}\|_\text{op}<\varepsilon / (\|A^{-1}\|_\text{op}\sqrt{n})$. Now for each $1\leq i \leq n$:
\[
\| \nabla h_i |_\mathbf{x} \| = \|Dh|_\mathbf{x} (\mathbf{e}_i) \| \leq \|Dh|_\mathbf{x}\|_\text{op} \|\mathbf{e}_i\|\leq \varepsilon / (\|A^{-1}\|_\text{op}\sqrt{n})
\]
and hence by MVI $\forall\mathbf{x},\mathbf{x}'\in B_\delta(\mathbf{x}_0)$:
\[
|h_i(\mathbf{x})-h_i(\mathbf{x}')| \leq \frac{ \varepsilon\|\mathbf{x}-\mathbf{x}'\|}{\|A^{-1}\|_\text{op}\sqrt{n}}
\]
such that
\[
\|h(\mathbf{x}-\mathbf{x}') \| \leq \frac{ \varepsilon\|\mathbf{x}-\mathbf{x}'\|}{\|A^{-1}\|_\text{op}}.
\]
Therefore
\begin{align*}
\|N_\mathbf{y}(\mathbf{x})-N_\mathbf{y}(\mathbf{x}')\| &= \|A^{-1}(h(\mathbf{x})-h(\mathbf{x}'))\| \\
&\leq \|A^{-1}\|_\text{op} \|h(\mathbf{x})-h(\mathbf{x}')\| \\
&\leq \|A^{-1}\|_\text{op} \frac{\varepsilon}{\|A^{-1}\|_\text{op}} \|\mathbf{x}-\mathbf{x}'\| \\
&=\varepsilon \|\mathbf{x}-\mathbf{x}'\|.
\end{align*}
\end{proof}

Q: Continuous differentiability
A: Let $U\subseteq\R^n$ be open, $F:U\to\R^m$. Then $DF|_\mathbf{x}$ is a continuous function of $\mathbf{x}$ iff $\forall 1 \leq i \leq m: F_i\in C_1(U)=\{f:U\to\R : D_jf|_\mathbf{x} \text{ is continuous for }1\leq j \leq n \} $.
\begin{proof}
Since $\mathcal{L}(\R^n,\R^m)\simeq \Mat_{m,n}(\R) \simeq \R^{mn}$, the choice of norm is irrelevant. So we choose the maximum norm on the unit ball. By componentwise differentiation wlog $m=1$.
Now
\[
\|DF|_{\mathbf{x}+\mathbf{h}}(\mathbf{e}_i) - DF|_\mathbf{x}(\mathbf{e}_i)  \| \leq \|DF|_{\mathbf{x}+\mathbf{h}} - DF|_\mathbf{x}\|_{\infty}
\]
and $\forall\mathbf{v}=\sum_{i=1}^n v_i\mathbf{e}_i\in B_1(0)$:
\[
\|(DF|_{\mathbf{x}+\mathbf{h}}-DF|_\mathbf{x})\left(\sum_{i=1}^n v_i\mathbf{e}_i\right)\|  \leq \sum_{i=1}^n (\|DF|_{\mathbf{x}+\mathbf{h}}-DF|_\mathbf{x})(\mathbf{e}_i)  \|.
\]
So
\[
\|DF|_{\mathbf{x}+\mathbf{h}}-DF|_\mathbf{x}\|_{\infty}  \leq \sum_{i=1}^n \|(DF|_{\mathbf{x}+\mathbf{h}}-DF|_\mathbf{x})(\mathbf{e}_i)  \|.
\]
Hence in the limit as $\mathbf{h}\to 0$:
\[
\|DF|_{\mathbf{x}+\mathbf{h}}-DF|_\mathbf{x}\|_{\infty} \to 0 \iff \forall i: \| (DF|_{\mathbf{x}+\mathbf{h}}-DF|_\mathbf{x})(\mathbf{e}_i)  \|.
\]
\end{proof}