\documentclass[12pt,a4paper,pdftex]{exam}

\usepackage[german]{babel}
\usepackage{natbib}
\usepackage{xcolor}
\usepackage{graphicx}
\usepackage[small]{caption}
\usepackage{sidecap}
\usepackage{pslatex}
\usepackage{amsmath}
\usepackage{amssymb}
\setlength{\marginparwidth}{2cm}
\usepackage[breaklinks=true,bookmarks=true,bookmarksopen=true,pdfpagemode=UseNone,pdfstartview=FitH,colorlinks=true,citecolor=blue]{hyperref}

%%%%% text size %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\usepackage[left=20mm,right=20mm,top=25mm,bottom=25mm]{geometry}
\pagestyle{headandfoot}
\ifprintanswers
\newcommand{\stitle}{: Solutions}
\else
\newcommand{\stitle}{}
\fi
\header{{\bfseries\large Exercise 10\stitle}}{{\bfseries\large Gradient descent}}{{\bfseries\large December 17th, 2018}}
\firstpagefooter{Dr. Jan Grewe}{Phone: 29 74588}{Email:
  jan.grewe@uni-tuebingen.de}
\runningfooter{}{\thepage}{}

\setlength{\baselineskip}{15pt}
\setlength{\parindent}{0.0cm}
\setlength{\parskip}{0.3cm}
\renewcommand{\baselinestretch}{1.15}

\newcommand{\code}[1]{\texttt{#1}}
\renewcommand{\solutiontitle}{\noindent\textbf{Solution:}\par\noindent}
%%%%% listings %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\usepackage{listings}
\lstset{
  language=Matlab,
  basicstyle=\ttfamily\footnotesize,
  numbers=left,
  numberstyle=\tiny,
  title=\lstname,
  showstringspaces=false,
  commentstyle=\itshape\color{darkgray},
  breaklines=true,
  breakautoindent=true,
  columns=flexible,
  frame=single,
  xleftmargin=1em,
  xrightmargin=1em,
  aboveskip=10pt
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 
\begin{document}

\input{instructions}

\begin{questions}

  \question Implement the gradient descent for finding the parameters
  of a straigth line \[ y = mx+b \] that we want to fit to the data in
  the file \emph{lin\_regression.mat}.

  In the lecture we already prepared most of the necessary functions:
  1. the cost function (\code{lsqError()}), and 2. the gradient
  (\code{lsqGradient()}). Read chapter 8 ``Optimization and gradient
  descent'' in the script, in particular section 8.4 and exercise 8.4!

  The algorithm for the descent towards the minimum of the cost
  function is as follows:
  
  \begin{enumerate}
  \item Start with some arbitrary parameter values (intercept $b_0$
    and slope $m_0$, $\vec p_0 = (b_0, m_0)$ for the slope and the
    intercept of the straight line.
  \item \label{computegradient} Compute the gradient of the cost function
    at the current values of the parameters $\vec p_i$.
  \item If the magnitude (length) of the gradient is smaller than some
    small number, the algorithm converged close to the minimum of the
    cost function and we abort the descent.  Right at the minimum the
    magnitude of the gradient is zero.  However, since we determine
    the gradient numerically, it will never be exactly zero. This is
    why we just require the gradient to be sufficiently small
    (e.g. \code{norm(gradient) < 0.1}).
  \item \label{gradientstep} Move against the gradient by a small step
    ($\epsilon = 0.01$):
    \[\vec p_{i+1} = \vec p_i - \epsilon \cdot \nabla f_{cost}(m_i, b_i)\]
  \item Repeat steps \ref{computegradient} -- \ref{gradientstep}.
  \end{enumerate}

  \begin{parts}
    \part Implement the gradient descent in a function that returns
    the parameter values at the minimum of the cost function and a vector
    with the value of the cost function at each step of the algorithm.
    \begin{solution}
      \lstinputlisting{../code/descent.m}
    \end{solution}

    \part Plot the data and the straight line with the parameter
    values that you found with the gradient descent method.

    \part Plot the development of the costs as a function of the
    iteration step.
    \begin{solution}
      \lstinputlisting{../code/descentfit.m}
    \end{solution}

    \part Find the position of the minimum of the cost function by
    means of the \code{min()} function. Compare with the result of the
    gradient descent method. Vary the value of $\epsilon$ and the
    minimum gradient. What are good values such that the gradient
    descent gets closest to the true minimum of the cost function?
    \begin{solution}
      \lstinputlisting{../code/checkdescent.m}
    \end{solution}

    \part Use the functions \code{polyfit()} and \code{lsqcurvefit()}
    provided by matlab to find the slope and intercept of a straight
    line that fits the data.
    \begin{solution}
      \lstinputlisting{../code/linefit.m}
    \end{solution}

  \end{parts}

\end{questions}

\end{document}