\documentclass{article}

% Language setting
\usepackage[english]{babel}

% Set page size and margins
\usepackage[letterpaper,top=2cm,bottom=2cm,left=3cm,right=3cm,marginparwidth=1.75cm]{geometry}

% Useful packages
\usepackage{amsmath}
\usepackage{graphicx}
\usepackage{amsfonts}
\usepackage[colorlinks=true, allcolors=blue]{hyperref}
\usepackage{array} % For better table column definitions

\title{MATH 6601 - Project 1 Report}
\author{Zhe Yuan (yuan.1435)}
\date{Sep 2025}

\begin{document}
\maketitle

\section{Projections}

\subsection*{(a)}
The projection of vector $b$ onto the column space of a full-rank matrix $A$ is given by $p=A(A^*A)^{-1}A^*b$. A function \texttt{proj(A, b)} was implemented based on this formula (see \texttt{Project1\_Q1.py}). For $\epsilon = 1$, the projection is:
\begin{verbatim}
p = [1.85714286, 1.0, 3.14285714, 1.28571429, 3.85714286]
\end{verbatim}

\subsection*{(b)}
As $\epsilon \rightarrow 0$, the normal equation method becomes numerically unstable. A more robust SVD-based method, \texttt{proj\_SVD(A, b)}, was implemented to handle this (see \texttt{Project1\_Q1.py}). The results are summarized in Table~\ref{tab:proj_results}.

\begin{table}[h!]
\centering
\small % Use a smaller font for the table
\begin{tabular}{|c|>{\ttfamily}p{3cm}|>{\ttfamily}p{3cm}|p{3.5cm}|}
\hline
\textbf{$\epsilon$} & \textbf{Normal Equation Method \texttt{proj(A,b)}} & \textbf{SVD Method \texttt{proj\_SVD(A,b)}} & \textbf{Difference (Normal Eq - SVD)} \\
\hline
\textbf{1.0} & [1.85714286 1. 3.14285714 1.28571429 3.85714286] & [1.85714286 1. 3.14285714 1.28571429 3.85714286] & [ 2.22e-16  1.55e-15 -4.44e-16  2.22e-16 -8.88e-16] \\
\hline
\textbf{0.1} & [1.85714286 1. 3.14285714 1.28571429 3.85714286] & [1.85714286 1. 3.14285714 1.28571429 3.85714286] & [ 8.08e-14  5.87e-14 -8.88e-16  4.82e-14 -1.38e-14] \\
\hline
\textbf{0.01} & [1.85714286 1. 3.14285714 1.28571429 3.85714286] & [1.85714286 1. 3.14285714 1.28571429 3.85714286] & [-4.77e-12 -1.84e-14  5.54e-13 -4.00e-12  1.50e-12] \\
\hline
\textbf{1e-4} & [1.85714282 1. 3.14285716 1.28571427 3.85714291] & [1.85714286 1. 3.14285714 1.28571429 3.85714286] & [-3.60e-08  9.11e-13  1.94e-08 -1.20e-08  4.80e-08] \\
\hline
\textbf{1e-8} & [-1.87500007 0.99999993 -3.12499997 -2.62500007 3.49999996] & [1.85714286 1. 3.14285714 1.28571427 3.85714286] & [-3.73e+00 -7.45e-08 -6.27e+00 -3.91e+00 -3.57e-01] \\
\hline
\textbf{1e-16} & \textbf{Error:} \texttt{ValueError: Matrix A must be full rank} & [3.4 1. 1.6 1.8 1.8] & Could not compute difference due to previous error. \\
\hline
\textbf{1e-32} & \textbf{Error:} \texttt{ValueError: Matrix A must be full rank} & [3.4 1. 1.6 1.8 1.8] & Could not compute difference due to previous error. \\
\hline
\end{tabular}
\caption{Comparison of results from the normal equation and SVD-based methods.}
\label{tab:proj_results}
\end{table}

\noindent Numerical experiments show that as $\epsilon$ becomes small, the difference between the two methods increases. When $\epsilon$ becomes very small (e.g., 1e-8), the normal equation method can't give a valid result and fails when $\epsilon$ continues to decrease (e.g., 1e-16), while the SVD method continues to provide a stable solution.

\section{QR Factorisation}

\subsection*{(a, b, c)}
Four QR factorization algorithms were implemented to find an orthonormal basis $Q$ for the column space of the $n \times n$ Hilbert matrix (for $n=2$ to $20$):
\begin{enumerate}
    \item Classical Gram-Schmidt (CGS, \texttt{cgs\_q})
    \item Modified Gram-Schmidt (MGS, \texttt{mgs\_q})
    \item Householder QR (\texttt{householder\_q})
    \item Classical Gram-Schmidt Twice (CGS-Twice, \texttt{cgs\_twice\_q})
\end{enumerate}
For implementation details, please see the \texttt{Project1\_Q2.py} file.

\subsection*{(d)}
The quality of the computed $Q$ from each method was assessed by plotting the orthogonality loss, $\log_{10}(\|I - Q^T Q\|_F)$, versus the matrix size $n$, as shown in Figure~\ref{fig:ortho_loss}.

\begin{figure}[h!]
    \centering
    \includegraphics[width=0.8\textwidth]{Project1_A2d_orthogonality_loss.png}
    \caption{Orthogonality Loss vs. Matrix Size for QR Methods.}
    \label{fig:ortho_loss}
\end{figure}

A summary of the loss and computation time for $n = 4, 9, 11$ is provided in Table~\ref{tab:qr_results}.

\begin{table}[h!]
    \centering
    \begin{tabular}{|c|l|r|r|}
        \hline
        \textbf{n} & \textbf{Method} & \textbf{Loss ($log_{10}$)} & \textbf{Time (s)} \\
        \hline
        4 & Classical Gram-Schmidt & -10.311054 & 4.28e-05 \\
        4 & Modified Gram-Schmidt & -12.374458 & 5.89e-05 \\
        4 & Householder & -15.351739 & 1.42e-04 \\
        4 & Classical Gram-Schmidt Twice & -15.570756 & 8.99e-05 \\
        \hline
        9 & Classical Gram-Schmidt & 0.389425 & 2.89e-04 \\
        9 & Modified Gram-Schmidt & -5.257679 & 3.50e-04 \\
        9 & Householder & -14.715202 & 6.19e-04 \\
        9 & Classical Gram-Schmidt Twice & -8.398044 & 6.07e-04 \\
        \hline
        11 & Classical Gram-Schmidt & 0.609491 & 5.23e-04 \\
        11 & Modified Gram-Schmidt & -2.005950 & 5.77e-04 \\
        11 & Householder & -14.697055 & 8.68e-04 \\
        11 & Classical Gram-Schmidt Twice & -1.745087 & 1.07e-03 \\
        \hline
    \end{tabular}
    \caption{Loss and computation time for selected matrix sizes.}
    \label{tab:qr_results}
\end{table}

\subsection*{(e)}
The speed and accuracy of the four methods were compared based on the plots in Figure~\ref{fig:ortho_loss} and Figure~\ref{fig:time_taken}.

\begin{figure}[h!]
    \centering
    \includegraphics[width=0.8\textwidth]{Project1_A2d_time_taken.png}
    \caption{Computation Time vs. Matrix Size for QR Methods.}
    \label{fig:time_taken}
\end{figure}

\begin{itemize}
    \item \textbf{Accuracy:} Householder is the most accurate and stable. MGS is significantly better than CGS. CGS-Twice improves on CGS, achieving accuracy comparable to MGS. CGS is highly unstable and loses orthogonality quickly.
    \item \textbf{Speed:} CGS and MGS are the fastest. Householder is slightly slower, and CGS-Twice is the slowest.
\end{itemize}

\section{Least Square Problem}

\subsection*{(a)}
The problem is formulated as $Ax = b$, where $A$ is an $(N+1) \times (n+1)$ Vandermonde matrix, $x$ is the $(n+1) \times 1$ vector of unknown polynomial coefficients $[a_0, \dots, a_n]^T$, and $b$ is the $(N+1) \times 1$ vector of known function values $f(t_j)$.

\subsection*{(b)}
The least squares problem was solved using the implemented Householder QR factorization (\texttt{householder\_lstsq} in \texttt{Project1\_Q3.py}, the same function in Question 2) to find the coefficient vector $x$.

\subsection*{(c)}
The function $f(t) = 1 / (1 + t^2)$ and its polynomial approximation $p(t)$ were plotted for $N=30$ and $n=5, 15, 30$, as shown in Figure~\ref{fig:ls_approx}.

\begin{figure}[h!]
    \centering
    \includegraphics[width=0.8\textwidth]{Project1_A3c_least_squares_approximation.png}
    \caption{Least squares approximation of $f(t)=1/(1+t^2)$ for varying polynomial degrees $n$.}
    \label{fig:ls_approx}
\end{figure}

When $n=15$, the approximation is excellent. When $n=30$, the polynomial interpolates the points but exhibits wild oscillations between them.

\subsection*{(d)}
No, $p(t)$ will not converge to $f(t)$ if $N = n$ tends to infinity. Polynomial interpolation on equally spaced nodes diverges for this function, as demonstrated by the severe oscillations in the $n=30$ case shown in Figure~\ref{fig:ls_approx}.

\subsection*{(e)}
The error between the computed and true coefficients was plotted against the polynomial degree $n$, as shown in Figure~\ref{fig:coeff_error}.

\begin{figure}[h!]
    \centering
    \includegraphics[width=0.8\textwidth]{Project1_A3e_coefficient_error.png}
    \caption{Error in computed polynomial coefficients vs. polynomial degree $n$.}
    \label{fig:coeff_error}
\end{figure}

The error in the computed coefficients grows significantly as $n$ increases.

\end{document}