Merge branch 'master' of https://whale.am28.uni-tuebingen.de/git/teaching/scientificComputing

Makefile

@@ -3,7 +3,7 @@ BASENAME=scientificcomputing-script
 SUBDIRS=programming debugging plotting codestyle statistics bootstrap regression likelihood pointprocesses designpattern 
 SUBTEXS=$(foreach subd, $(SUBDIRS), $(subd)/lecture/$(subd).tex)
 
-all : script chapters
+all : plots chapters index script exercises
 
 chapters :
 	for sd in $(SUBDIRS); do $(MAKE) -C $$sd/lecture chapter; done
@@ -37,6 +37,11 @@ watchpdf :
 watchscript :
 	while true; do ! make -s -q script && make script; sleep 0.5; done
 
+
+exercises:
+	for sd in $(SUBDIRS); do if test -d $$sd/exercises; then $(MAKE) -C $$sd/exercises pdf; fi; done
+
+
 cleanplots:
 	for sd in $(SUBDIRS); do $(MAKE) -C $$sd/lecture cleanplots; done
 
@@ -46,6 +51,7 @@ cleantex:
 
 clean : cleantex
 	for sd in $(SUBDIRS); do $(MAKE) -C $$sd/lecture clean; done
+	for sd in $(SUBDIRS); do if test -d $$sd/exercises; then $(MAKE) -C $$sd/exercises clean; fi; done
 
 cleanall : clean
 	rm -f $(BASENAME).pdf

README.fonts

@@ -1,6 +1,20 @@
 Fonts for matplotlib
 --------------------
 
+Install Humor Sans font
+```
+sudo apt install fonts-humor-sans
+```
+
+Clear matplotlib font cache:
+```
+cd ~/.cache/matplotlib/
+rm -r *
+```
+
+Older problems
+--------------
+
 Make sure the right fonts are installed:
 ```
 sudo apt-get install ttf-lyx2.0 

bootstrap/code/bootstrapsem.m

@@ -1,7 +1,7 @@
 nsamples = 100;
 nresamples = 1000;
 
-% draw a SRS (simple random sample, "Stichprobe") from the population:
+% draw a simple random sample ("Stichprobe") from the population:
 x = randn(1, nsamples);
 fprintf('%-30s  %-5s  %-5s  %-5s\n', '', 'mean', 'stdev', 'sem')
 fprintf('%30s  %5.2f  %5.2f  %5.2f\n', 'single SRS', mean(x), std(x), std(x)/sqrt(nsamples))
@@ -15,7 +15,8 @@ for i = 1:nresamples        % loop for generating the bootstraps
 end
 fprintf('%30s  %5.2f  %5.2f  -\n', 'bootstrapped distribution', mean(mus), std(mus))
 
-% many SRS (we can do that with the random number generator, but not in real life!):
+% many SRS (we can do that with the random number generator,
+% but not in real life!):
 musrs = zeros(nresamples,1);  % vector for the means of each SRS
 for i = 1:nresamples
     x = randn(1, nsamples);   % draw a new SRS

bootstrap/code/meandiffsignificance.m

@@ -0,0 +1,40 @@
+% generate two data sets:
+n = 200;
+d = 0.2;
+x = randn(n, 1);
+y = randn(n, 1) + d;
+
+% difference of sample means:
+md = mean(y) - mean(x);
+fprintf('difference of means of data d = %.2f\n', md);
+
+% distribution of null hypothesis by permutation:
+nperm = 1000;
+xy = [x; y];                         % x and y data in one column vector
+ds = zeros(nperm,1);
+for i = 1:nperm
+    xyr = xy(randperm(length(xy)));  % shuffle data
+    xr = xyr(1:length(x));           % random x-data
+    yr = xyr(length(x)+1:end);       % random y-data
+    ds(i) = mean(yr) - mean(xr);     % difference of means
+end
+[h, b] = hist(ds, 20);
+h = h/sum(h)/(b(2)-b(1));       % normalization
+
+% significance:
+dq = quantile(ds, 0.95);
+fprintf('difference of means of null hypothesis at 5%% significance = %.2f\n', dq);
+if md >= dq
+    fprintf('--> difference of means d=%.2f is significant\n', md);
+else
+    fprintf('--> d=%.2f is not a significant difference of means\n', md);
+end
+
+% plot:
+bar(b, h, 'facecolor', 'b');
+hold on;
+bar(b(b>=dq), h(b>=dq), 'facecolor', 'r');
+plot([md md], [0 4], 'r', 'linewidth', 2);
+xlabel('Difference of means');
+ylabel('Probability density of H0');
+hold off;

bootstrap/code/meandiffsignificance.out

@@ -0,0 +1,4 @@
+>> meandiffsignificance
+difference of means of data d = 0.18
+difference of means of null hypothesis at 5% significance = 0.17
+--> difference of means d=0.18 is significant

bootstrap/exercises/Makefile

@@ -1,34 +1,3 @@
-TEXFILES=$(wildcard exercises??.tex)
+TEXFILES=$(wildcard resampling-?.tex)
-EXERCISES=$(TEXFILES:.tex=.pdf)
-SOLUTIONS=$(EXERCISES:exercises%=solutions%)
 
-.PHONY: pdf exercises solutions watch watchexercises watchsolutions clean
+include ../../exercises.mk
-
-pdf : $(SOLUTIONS) $(EXERCISES)
-
-exercises : $(EXERCISES)
-
-solutions : $(SOLUTIONS)
-
-$(SOLUTIONS) : solutions%.pdf : exercises%.tex instructions.tex
-	{ echo "\\documentclass[answers,12pt,a4paper,pdftex]{exam}"; sed -e '1d' $<; } > $(patsubst %.pdf,%.tex,$@)
-	pdflatex -interaction=scrollmode $(patsubst %.pdf,%.tex,$@) | tee /dev/stderr | fgrep -q "Rerun to get cross-references right" && pdflatex -interaction=scrollmode $(patsubst %.pdf,%.tex,$@) || true
-	rm $(patsubst %.pdf,%,$@).[!p]*
-
-$(EXERCISES) : %.pdf : %.tex instructions.tex
-	pdflatex -interaction=scrollmode $< | tee /dev/stderr | fgrep -q "Rerun to get cross-references right" && pdflatex -interaction=scrollmode $< || true
-
-watch :
-	while true; do ! make -q pdf && make pdf; sleep 0.5; done
-
-watchexercises :
-	while true; do ! make -q exercises && make exercises; sleep 0.5; done
-
-watchsolutions :
-	while true; do ! make -q solutions && make solutions; sleep 0.5; done
-
-clean :
-	rm -f *~ *.aux *.log *.out
-
-cleanup : clean
-	rm -f $(SOLUTIONS) $(EXERCISES)

bootstrap/exercises/bootstrapmean.m

@@ -10,7 +10,7 @@ function [bootsem, mu] = bootstrapmean(x, resample)
     for i = 1:resample
         % resample:
         xr = x(randi(nsamples, nsamples, 1));
-        % compute statistics on sample:
+        % compute statistics of resampled sample:
         mu(i) = mean(xr);
     end
     bootsem = std(mu);

bootstrap/exercises/correlationbootstrap.m

@@ -25,8 +25,8 @@ end
 
 %% plot:
 hold on;
-bar(b, h, 'facecolor', [0.5 0.5 0.5]);
+bar(b, h, 'facecolor', [0.5 0.5 0.5]);           % permuation test
-bar(bb, hb, 'facecolor', 'b');
+bar(bb, hb, 'facecolor', 'b');                   % bootstrap
 bar(bb(bb<=rbq), hb(bb<=rbq), 'facecolor', 'r');
 plot([rd rd], [0 4], 'r', 'linewidth', 2);
 xlim([-0.25 0.75])

bootstrap/exercises/correlationsignificance.m

@@ -1,4 +1,4 @@
-%% (a) generate correlated data
+%% (a) generate correlated data:
 n = 1000;
 a = 0.2;
 x = randn(n, 1);

bootstrap/exercises/exercises01.tex

@@ -1,205 +0,0 @@
-\documentclass[12pt,a4paper,pdftex]{exam}
-
-\usepackage[english]{babel}
-\usepackage{pslatex}
-\usepackage[mediumspace,mediumqspace,Gray]{SIunits}      % \ohm, \micro
-\usepackage{xcolor}
-\usepackage{graphicx}
-\usepackage[breaklinks=true,bookmarks=true,bookmarksopen=true,pdfpagemode=UseNone,pdfstartview=FitH,colorlinks=true,citecolor=blue]{hyperref}
-
-%%%%% layout %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage[left=20mm,right=20mm,top=25mm,bottom=25mm]{geometry}
-\pagestyle{headandfoot}
-\ifprintanswers
-\newcommand{\stitle}{: Solutions}
-\else
-\newcommand{\stitle}{}
-\fi
-\header{{\bfseries\large Exercise 9\stitle}}{{\bfseries\large Bootstrap}}{{\bfseries\large December 9th, 2019}}
-\firstpagefooter{Prof. Dr. Jan Benda}{Phone: 29 74573}{Email:
-jan.benda@uni-tuebingen.de}
-\runningfooter{}{\thepage}{}
-
-\setlength{\baselineskip}{15pt}
-\setlength{\parindent}{0.0cm}
-\setlength{\parskip}{0.3cm}
-\renewcommand{\baselinestretch}{1.15}
-
-%%%%% listings %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage{listings}
-\lstset{
-  language=Matlab,
-  basicstyle=\ttfamily\footnotesize,
-  numbers=left,
-  numberstyle=\tiny,
-  title=\lstname,
-  showstringspaces=false,
-  commentstyle=\itshape\color{darkgray},
-  breaklines=true,
-  breakautoindent=true,
-  columns=flexible,
-  frame=single,
-  xleftmargin=1em,
-  xrightmargin=1em,
-  aboveskip=10pt
-}
-
-%%%%% math stuff: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage{amsmath}
-\usepackage{amssymb}
-\usepackage{bm} 
-\usepackage{dsfont}
-\newcommand{\naZ}{\mathds{N}}
-\newcommand{\gaZ}{\mathds{Z}}
-\newcommand{\raZ}{\mathds{Q}}
-\newcommand{\reZ}{\mathds{R}}
-\newcommand{\reZp}{\mathds{R^+}}
-\newcommand{\reZpN}{\mathds{R^+_0}}
-\newcommand{\koZ}{\mathds{C}}
-
-%%%%% page breaks %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\newcommand{\continue}{\ifprintanswers%
-\else
-\vfill\hspace*{\fill}$\rightarrow$\newpage%
-\fi}
-\newcommand{\continuepage}{\ifprintanswers%
-\newpage
-\else
-\vfill\hspace*{\fill}$\rightarrow$\newpage%
-\fi}
-\newcommand{\newsolutionpage}{\ifprintanswers%
-\newpage%
-\else
-\fi}
-
-%%%%% new commands %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\newcommand{\qt}[1]{\textbf{#1}\\}
-\newcommand{\pref}[1]{(\ref{#1})}
-\newcommand{\extra}{--- Zusatzaufgabe ---\ \mbox{}}
-\newcommand{\code}[1]{\texttt{#1}}
-
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\begin{document}
-
-\input{instructions}
-
-\begin{questions}
-
-\question \qt{Bootstrap the standard error of the mean}
-We want to compute the standard error of the mean of a data set by
-means of the bootstrap method and compare the result with the formula
-``standard deviation divided by the square-root of $n$''.
-\begin{parts}
-  \part Download the file \code{thymusglandweights.dat} from Ilias.
-  This is a data set of the weights of the thymus glands of 14-day old chicken embryos
-  measured in milligram.
-  \part Load the data into Matlab (\code{load} function).
-  \part Compute histogram, mean, and standard error of the mean of the first 80 data points.
-  \part Compute the standard error of the mean of the first 80 data
-  points by bootstrapping the data 500 times. Write a function that
-  bootstraps the standard error of the mean of a given data set. The
-  function should also return a vector with the bootstrapped means.
-  \part Compute the 95\,\% confidence interval for the mean from the
-  bootstrap distribution (\code{quantile()} function) --- the
-  interval that contains the true mean with 95\,\% probability.
-  \part Use the whole data set and the bootstrap method for computing
-  the dependence of the standard error of the mean from the sample
-  size $n$.
-  \part Compare your result with the formula for the standard error
-  $\sigma/\sqrt{n}$.
-\end{parts}
-\begin{solution}
-  \lstinputlisting{bootstrapmean.m}
-  \lstinputlisting{bootstraptymus.m}
-  \includegraphics[width=0.5\textwidth]{bootstraptymus-datahist}
-  \includegraphics[width=0.5\textwidth]{bootstraptymus-meanhist}
-  \includegraphics[width=0.5\textwidth]{bootstraptymus-samples}
-\end{solution}
-
-
-\question \qt{Student t-distribution}
-The distribution of Student's t, $t=\bar x/(\sigma_x/\sqrt{n})$, the
-estimated mean $\bar x$ of a data set of size $n$ divided by the
-estimated standard error of the mean $\sigma_x/\sqrt{n}$, where
-$\sigma_x$ is the estimated standard deviation, is not a normal
-distribution but a Student-t distribution.  We want to compute the
-Student-t distribution and compare it with the normal distribution.
-\begin{parts}
-\part Generate 100000 normally distributed random numbers.
-\part Draw from these data 1000 samples of size $n=3$, 5, 10, and
-50. For each sample size $n$ ...
-\part ... compute the mean $\bar x$ of the samples and plot the
-probability density of these means.
-\part ... compare the resulting probability densities with corresponding
-normal distributions.
-\part ... compute Student's $t=\bar x/(\sigma_x/\sqrt{n})$ and compare its
-distribution with the normal distribution with standard deviation of
-one. Is $t$ normally distributed? Under which conditions is $t$
-normally distributed?
-\end{parts}
-\newsolutionpage
-\begin{solution}
-  \lstinputlisting{tdistribution.m}
-  \includegraphics[width=1\textwidth]{tdistribution-n03}\\
-  \includegraphics[width=1\textwidth]{tdistribution-n05}\\
-  \includegraphics[width=1\textwidth]{tdistribution-n10}\\
-  \includegraphics[width=1\textwidth]{tdistribution-n50}
-\end{solution}
-
-
-\continue
-\question \qt{Permutation test} \label{permutationtest}
-We want to compute the significance of a correlation by means of a permutation test.
-\begin{parts}
-  \part \label{permutationtestdata} Generate 1000 correlated pairs
-  $x$, $y$ of random numbers according to:
-\begin{verbatim}
-n = 1000
-a = 0.2;
-x = randn(n, 1);
-y = randn(n, 1) + a*x;
-\end{verbatim}
-  \part Generate a scatter plot of the two variables.
-  \part Why is $y$ correlated with $x$?
-  \part Compute the correlation coefficient between $x$ and $y$.
-  \part What do you need to do in order to destroy the correlations between the $x$-$y$ pairs?
-  \part Do exactly this 1000 times and compute each time the correlation coefficient.
-  \part Compute and plot the probability density of these correlation
-  coefficients.
-  \part Is the correlation of the original data set significant?
-  \part What does ``significance of the correlation'' mean?
-%  \part Vary the sample size \code{n} and compute in the same way the
-%  significance of the correlation.
-\end{parts}
-\begin{solution}
-  \lstinputlisting{correlationsignificance.m}
-  \includegraphics[width=1\textwidth]{correlationsignificance}
-\end{solution}
-
-\question \qt{Bootstrap the correlation coefficient} 
-The permutation test generates the distribution of the null hypothesis
-of uncorrelated data and we check whether the correlation coefficient
-of the data differs significantly from this
-distribution. Alternatively we can bootstrap the data while keeping
-the pairs and determine the confidence interval of the correlation
-coefficient of the data. If this differs significantly from a
-correlation coefficient of zero we can conclude that the correlation
-coefficient of the data indeed quantifies correlated data.
-
-We take the same data set that we have generated in exercise
-\ref{permutationtest} (\ref{permutationtestdata}).
-\begin{parts}
-  \part Bootstrap 1000 times the correlation coefficient from the data.
-  \part Compute and plot the probability density of these correlation
-  coefficients.
-  \part Is the correlation of the original data set significant?
-\end{parts}
-\begin{solution}
-  \lstinputlisting{correlationbootstrap.m}
-  \includegraphics[width=1\textwidth]{correlationbootstrap}
-\end{solution}
-
-\end{questions}
-
-\end{document}

bootstrap/exercises/instructions.tex

@@ -1,6 +0,0 @@
-\vspace*{-7.8ex}
-\begin{center}
-\textbf{\Large Introduction to Scientific Computing}\\[2.3ex]
-{\large Jan Grewe, Jan Benda}\\[-3ex]
-Neuroethology Lab \hfill --- \hfill Institute for Neurobiology \hfill --- \hfill \includegraphics[width=0.28\textwidth]{UT_WBMW_Black_RGB} \\
-\end{center}

bootstrap/exercises/meandiffpermutation.m

@@ -0,0 +1,29 @@
+function [md, ds, dq] = meandiffpermutation(x, y, nperm, alpha)
+% Permutation test for difference of means of two independent samples.
+%
+% [md, ds, dq] = meandiffpermutation(x, y, nperm, alpha);
+%
+% Arguments:
+%   x: vector with the samples of the x data set.
+%   y: vector with the samples of the y data set.
+%   nperm: number of permutations run.
+%   alpha: significance level.
+%
+% Returns:
+%   md: difference of the means 
+%   ds: vector containing the differences of the means of the resampled data sets
+%   dq: difference of the means at a significance of alpha.
+
+  md = mean(x) - mean(y);             % measured difference
+  xy = [x; y];                        % merge data sets
+  % permutations:
+  ds = zeros(nperm, 1);
+  for i = 1:nperm
+      xyr = xy(randperm(length(xy))); % shuffle xy
+      xr = xyr(1:length(x));          % random x sample
+      yr = xyr(length(x)+1:end);      % random y sample
+      ds(i) = mean(xr) - mean(yr);
+  end
+  % significance:
+  dq = quantile(ds, 1.0 - alpha);
+end

bootstrap/exercises/meandiffplot.m

@@ -0,0 +1,42 @@
+function meandiffplot(x, y, md, ds, dq, k, nrows)
+% Plot histogram of data sets and of null hypothesis for differences in mean.
+%
+% meandiffplot(x, y, md, ds, dq, k, rows);
+%
+% Arguments:
+%   x: vector with the samples of the x data set.
+%   y: vector with the samples of the y data set.
+%   md: difference of means of the two data sets.
+%   ds: vector containing the differences of the means of the resampled data sets
+%   dq: minimum difference of the means considered significant.
+%   k: current row for the plot panels.
+%   nrows: number of rows of panels in the figure.
+
+  %% (b) plot histograms:
+  subplot(nrows, 2, k*2-1);
+  bmin = min([x; y]);
+  bmax = max([x; y]);
+  bins = bmin:(bmax-bmin)/20.0:bmax;
+  [xh, b] = hist(x, bins);
+  [yh, b] = hist(y, bins);
+  bar(bins, xh, 'facecolor', 'b')
+  hold on
+  bar(bins, yh, 'facecolor', 'r');
+  xlabel('x and y [mV]')
+  ylabel('counts')
+  hold off
+
+  %% (f) pdf of the differences:
+  [h, b] = hist(ds, 20);
+  h = h/sum(h)/(b(2)-b(1));           % normalization
+
+  %% plot:
+  subplot(nrows, 2, k*2)
+  bar(b, h, 'facecolor', 'b');
+  hold on;
+  bar(b(b>=dq), h(b>=dq), 'facecolor', 'r');
+  plot([md md], [0 4], 'r', 'linewidth', 2);
+  xlabel('Difference of means [mV]');
+  ylabel('pdf of H0');
+  hold off;
+end

bootstrap/exercises/meandiffsignificance.m

@@ -0,0 +1,37 @@
+n = 200;
+mx = -40.0;
+nperm = 1000;
+alpha = 0.05;
+
+%% (h) repeat for various means of the y-data set:
+mys = [-40.1, -40.2, -40.5];
+for k=1:length(mys)
+
+  %% (a) generate data:
+  my = mys(k);
+  x = randn(n, 1) + mx;
+  y = randn(n, 1) + my;
+
+  %% (d), (e) permutation test:
+  [md, ds, dq] = meandiffpermutation(x, y, nperm, alpha);
+
+  %% (c) difference of means:
+  fprintf('\nmean x = %.1fmV, mean y = %.1fmV\n', mx, my);
+  fprintf('  difference of means = %.2fmV\n', md);
+
+  %% (g) significance:
+  fprintf('  difference of means at 5%% significance = %.2fmV\n', dq);
+  if md >= dq
+      fprintf('  --> measured difference of means is significant\n');
+  else
+      fprintf('  --> measured difference of means is not significant\n');
+  end
+
+  %% (b), (f) plot histograms of data and pdf of differences:
+  meandiffplot(x, y, md, ds, dq, k, length(mys));
+
+  subplot(length(mys), 2, k*2-1);
+  title(sprintf('mx=%.1fmV, my=%.1fmV', mx, my))
+end
+
+savefigpdf(gcf, 'meandiffsignificance.pdf', 12, 10);

bootstrap/exercises/resampling-1.tex

@@ -0,0 +1,116 @@
+\documentclass[12pt,a4paper,pdftex]{exam}
+
+\newcommand{\exercisetopic}{Resampling}
+\newcommand{\exercisenum}{8}
+\newcommand{\exercisedate}{December 14th, 2020}
+
+\input{../../exercisesheader}
+
+\firstpagefooter{Prof. Dr. Jan Benda}{}{jan.benda@uni-tuebingen.de}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{document}
+
+\input{../../exercisestitle}
+
+\begin{questions}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\question \qt{Read chapter 7 of the script on ``resampling methods''!}\vspace{-3ex}
+
+\question \qt{Permutation test of correlations} \label{correlationtest}
+We want to compute the significance of a correlation by means of a permutation test.
+\begin{parts}
+  \part \label{correlationtestdata} Generate 1000 correlated pairs
+  $x$, $y$ of random numbers according to:
+\begin{verbatim}
+n = 1000
+a = 0.2;
+x = randn(n, 1);
+y = randn(n, 1) + a*x;
+\end{verbatim}
+  \part Generate a scatter plot of the two variables.
+  \part Why is $y$ correlated with $x$?
+  \part Compute the correlation coefficient between $x$ and $y$.
+  \part What do you need to do in order to destroy the correlations between the $x$-$y$ pairs?
+  \part Do exactly this 1000 times and compute each time the correlation coefficient.
+  \part Compute and plot the probability density of these correlation
+  coefficients.
+  \part Is the correlation of the original data set significant?
+  \part What does ``significance of the correlation'' mean?
+%  \part Vary the sample size \code{n} and compute in the same way the
+%  significance of the correlation.
+\end{parts}
+\begin{solution}
+  \lstinputlisting{correlationsignificance.m}
+  \includegraphics[width=1\textwidth]{correlationsignificance}
+\end{solution}
+
+\newsolutionpage
+\question \qt{Bootstrap the correlation coefficient} 
+The permutation test generates the distribution of the null hypothesis
+of uncorrelated data and we check whether the correlation coefficient
+of the data differs significantly from this
+distribution. Alternatively we can bootstrap the data while keeping
+the pairs and determine the confidence interval of the correlation
+coefficient of the data. If this differs significantly from a
+correlation coefficient of zero we can conclude that the correlation
+coefficient of the data indeed quantifies correlated data.
+
+We take the same data set that we have generated in exercise
+\ref{correlationtest} (\ref{correlationtestdata}).
+\begin{parts}
+  \part Bootstrap 1000 times the correlation coefficient from the
+  data, i.e.  generate bootstrap data by randomly resampling the
+  original data pairs with replacement. Use the \code{randi()}
+  function for generating random indices that you can use to select a
+  random sample from the original data.
+  \part Compute and plot the probability density of these correlation
+  coefficients.
+  \part Is the correlation of the original data set significant?
+\end{parts}
+\begin{solution}
+  \lstinputlisting{correlationbootstrap.m}
+  \includegraphics[width=1\textwidth]{correlationbootstrap}
+\end{solution}
+
+
+\continue 
+\question \qt{Permutation test of difference of means}
+We want to test whether two data sets come from distributions that
+differ in their mean by means of a permutation test.
+\begin{parts}
+  \part Generate two normally distributed data sets $x$ and $y$
+  containing each $n=200$ samples. Let's assume the $x$ samples are
+  measurements of the membrane potential of a mammalian photoreceptor
+  in darkness with a mean of $-40$\,mV and a standard deviation of
+  1\,mV. The $y$ values are the membrane potentials measured under dim
+  illumination and come from a distribution with the same standard
+  deviation and a mean of $-40.5$\,mV. See section 5.2 ``Scaling and
+  shifting random numbers'' in the script.
+  \part Plot histograms of the $x$ and $y$ data in a single
+  plot. Choose appropriate bins.
+  \part Compute the means of $x$ and $y$ and their difference.
+  \part The null hypothesis is that the $x$ and $y$ data come from the
+  same distribution. How can you generate new samples $x_r$ and $y_r$
+  from the original data that come from the same distribution?
+  \part Do exactly this 1000 times and compute each time the
+  difference of the means of the two resampled samples.
+  \part Compute and plot the probability density of the resulting
+  distribution of the null hypothesis.
+  \part Is the difference of the means of the original data sets significant?
+  \part Repeat this procedure for $y$ samples that are closer or
+  further apart from the mean of the $x$ data set. For this put the
+  computations of the permuation test in a function and all the plotting
+  in another function.
+\end{parts}
+\begin{solution}
+  \lstinputlisting{meandiffpermutation.m}
+  \lstinputlisting{meandiffplot.m}
+  \lstinputlisting{meandiffsignificance.m}
+  \includegraphics[width=1\textwidth]{meandiffsignificance}
+\end{solution}
+
+\end{questions}
+
+\end{document}

bootstrap/exercises/resampling-2.tex

@@ -0,0 +1,84 @@
+\documentclass[12pt,a4paper,pdftex]{exam}
+
+\newcommand{\exercisetopic}{Resampling}
+\newcommand{\exercisenum}{X2}
+\newcommand{\exercisedate}{December 14th, 2020}
+
+\input{../../exercisesheader}
+
+\firstpagefooter{Prof. Dr. Jan Benda}{}{jan.benda@uni-tuebingen.de}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{document}
+
+\input{../../exercisestitle}
+
+\begin{questions}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\question \qt{Read chapter 7 of the script on ``resampling methods''!}\vspace{-3ex}
+
+\question \qt{Bootstrap the standard error of the mean}
+We want to compute the standard error of the mean of a data set by
+means of the bootstrap method and compare the result with the formula
+``standard deviation divided by the square-root of $n$''.
+\begin{parts}
+  \part Download the file \code{thymusglandweights.dat} from Ilias.
+  This is a data set of the weights of the thymus glands of 14-day old chicken embryos
+  measured in milligram.
+  \part Load the data into Matlab (\code{load} function).
+  \part Compute histogram, mean, and standard error of the mean of the first 80 data points.
+  \part Compute the standard error of the mean of the first 80 data
+  points by bootstrapping the data 500 times. Write a function that
+  bootstraps the standard error of the mean of a given data set. The
+  function should also return a vector with the bootstrapped means.
+  \part Compute the 95\,\% confidence interval for the mean from the
+  bootstrap distribution (\code{quantile()} function) --- the
+  interval that contains the true mean with 95\,\% probability.
+  \part Use the whole data set and the bootstrap method for computing
+  the dependence of the standard error of the mean from the sample
+  size $n$.
+  \part Compare your result with the formula for the standard error
+  $\sigma/\sqrt{n}$.
+\end{parts}
+\begin{solution}
+  \lstinputlisting{bootstrapmean.m}
+  \lstinputlisting{bootstraptymus.m}
+  \includegraphics[width=0.5\textwidth]{bootstraptymus-datahist}
+  \includegraphics[width=0.5\textwidth]{bootstraptymus-meanhist}
+  \includegraphics[width=0.5\textwidth]{bootstraptymus-samples}
+\end{solution}
+
+
+\question \qt{Student t-distribution}
+The distribution of Student's t, $t=\bar x/(\sigma_x/\sqrt{n})$, the
+estimated mean $\bar x$ of a data set of size $n$ divided by the
+estimated standard error of the mean $\sigma_x/\sqrt{n}$, where
+$\sigma_x$ is the estimated standard deviation, is not a normal
+distribution but a Student-t distribution.  We want to compute the
+Student-t distribution and compare it with the normal distribution.
+\begin{parts}
+\part Generate 100000 normally distributed random numbers.
+\part Draw from these data 1000 samples of size $n=3$, 5, 10, and
+50. For each sample size $n$ ...
+\part ... compute the mean $\bar x$ of the samples and plot the
+probability density of these means.
+\part ... compare the resulting probability densities with corresponding
+normal distributions.
+\part ... compute Student's $t=\bar x/(\sigma_x/\sqrt{n})$ and compare its
+distribution with the normal distribution with standard deviation of
+one. Is $t$ normally distributed? Under which conditions is $t$
+normally distributed?
+\end{parts}
+\newsolutionpage
+\begin{solution}
+  \lstinputlisting{tdistribution.m}
+  \includegraphics[width=1\textwidth]{tdistribution-n03}\\
+  \includegraphics[width=1\textwidth]{tdistribution-n05}\\
+  \includegraphics[width=1\textwidth]{tdistribution-n10}\\
+  \includegraphics[width=1\textwidth]{tdistribution-n50}
+\end{solution}
+
+\end{questions}
+
+\end{document}

bootstrap/exercises/tdistribution.m

@@ -4,7 +4,7 @@ x=randn(n, 1);
 
 for nsamples = [3 5 10 50]
     nsamples
-    %% compute mean, standard deviation and t:
+    % compute mean, standard deviation and t:
     nmeans = 10000;
     means = zeros(nmeans, 1);
     sdevs = zeros(nmeans, 1);
@@ -26,7 +26,7 @@ for nsamples=[3 5 10 50]
     pm = exp(-0.5*(xg/msdev).^2)/sqrt(2.0*pi)/msdev;
     pt = exp(-0.5*(xg/tsdev).^2)/sqrt(2.0*pi)/tsdev;
     
-    %% plots
+    % plots:
     subplot(1, 2, 1)
     bins = xmin:0.2:xmax;
     [h,b] = hist(means, bins);

bootstrap/lecture/bootstrap-chapter.tex

@@ -10,6 +10,8 @@
 \setcounter{page}{\pagenumber}
 \setcounter{chapter}{\chapternumber}
 
+\renewcommand{\exercisesolutions}{here}  % 0: here, 1: chapter, 2: end
+
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \begin{document} 
@@ -17,10 +19,20 @@
 \include{bootstrap}
 
 \section{TODO}
+
 This chapter easily covers two lectures:
 \begin{itemize}
 \item 1. Bootstrapping with a proper introduction of of confidence intervals
-\item 2. Permutation test with a proper introduction of statistical tests (dsitribution of nullhypothesis, significance, power, etc.)
+\item 2. Permutation test with a proper introduction of statistical tests (distribution of nullhypothesis, significance, power, etc.)
+\end{itemize}
+ToDo:
+\begin{itemize}
+\item Add jacknife methods to bootstrapping
+\item Add discussion of confidence intervals to descriptive statistics chapter
+\item Have a separate chapter on statistical tests before. What is the
+  essence of a statistical test (null hypothesis distribution), power
+  analysis, and a few examples of existing functions for statistical
+  tests.
 \end{itemize}
 
 \end{document}

bootstrap/lecture/bootstrap.tex

@@ -1,20 +1,28 @@
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\chapter{Bootstrap methods}
+\chapter{Resampling methods}
 \label{bootstrapchapter}
-\exercisechapter{Bootstrap methods}
+\exercisechapter{Resampling methods}
 
-Bootstrapping methods are applied to create distributions of
+
-statistical measures via resampling of a sample. Bootstrapping offers several
+\entermde{Resampling methoden}{Resampling methods} are applied to
-advantages:
+generate distributions of statistical measures via resampling of
+existing samples. Resampling offers several advantages:
 \begin{itemize}
 \item Fewer assumptions (e.g. a measured sample does not need to be
   normally distributed).
 \item Increased precision as compared to classical methods. %such as?
-\item General applicability: the bootstrapping methods are very
+\item General applicability: the resampling methods are very
   similar for different statistics and there is no need to specialize
   the method to specific statistic measures.
 \end{itemize}
+Resampling methods can be used for both estimating the precision of
+estimated statistics (e.g. standard error of the mean, confidence
+intervals) and testing for significane.
+
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\section{Bootstrapping}
 
 \begin{figure}[tp]
   \includegraphics[width=0.8\textwidth]{2012-10-29_16-26-05_771}\\[2ex]
@@ -84,19 +92,20 @@ of the statistical population. We can use the bootstrap distribution
 to draw conclusion regarding the precision of our estimation (e.g.
 standard errors and confidence intervals).
 
-Bootstrapping methods create bootstrapped samples from a SRS by
+Bootstrapping methods generate bootstrapped samples from a SRS by
 resampling. The bootstrapped samples are used to estimate the sampling
 distribution of a statistical measure. The bootstrapped samples have
-the same size as the original sample and are created by randomly
+the same size as the original sample and are generated by randomly
 drawing with replacement. That is, each value of the original sample
-can occur once, multiple time, or not at all in a bootstrapped
+can occur once, multiple times, or not at all in a bootstrapped
 sample. This can be implemented by generating random indices into the
 data set using the \code{randi()} function.
 
 
-\section{Bootstrap of the standard error}
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\subsection{Bootstrap the standard error}
 
-Bootstrapping can be nicely illustrated at the example of the
+Bootstrapping can be nicely illustrated on the example of the
 \enterm{standard error} of the mean (\determ{Standardfehler}). The
 arithmetic mean is calculated for a simple random sample. The standard
 error of the mean is the standard deviation of the expected
@@ -116,11 +125,10 @@ population.
     the standard error of the mean.}
 \end{figure}
 
-Via bootstrapping we create a distribution of the mean values
+Via bootstrapping we generate a distribution of mean values
 (\figref{bootstrapsemfig}) and the standard deviation of this
-distribution is the standard error of the mean.
+distribution is the standard error of the sample mean.
 
-\pagebreak[4]
 \begin{exercise}{bootstrapsem.m}{bootstrapsem.out}
   Create the distribution of mean values from bootstrapped samples
   resampled from a single SRS. Use this distribution to estimate the
@@ -138,68 +146,166 @@ distribution is the standard error of the mean.
 \end{exercise}
 
 
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \section{Permutation tests}
+
 Statistical tests ask for the probability of a measured value to
 originate from a null hypothesis. Is this probability smaller than the
 desired \entermde{Signifikanz}{significance level}, the
-\entermde{Nullhypothese}{null hypothesis} may be rejected.
+\entermde{Nullhypothese}{null hypothesis} can be rejected.
 
 Traditionally, such probabilities are taken from theoretical
-distributions which are based on assumptions about the data.  Thus the
+distributions which have been derived based on some assumptions about
-applied statistical test has to be appropriate for the type of
+the data. For example, the data should be normally distributed.  Given
-data. An alternative approach is to calculate the probability density
+some data one has to find an appropriate test that matches the
-of the null hypothesis directly from the data itself. To do this, we
+properties of the data. An alternative approach is to calculate the
-need to resample the data according to the null hypothesis from the
+probability density of the null hypothesis directly from the data
-SRS. By such permutation operations we destroy the feature of interest
+themselves. To do so, we need to resample the data according to the
-while we conserve all other statistical properties of the data.
+null hypothesis from the SRS. By such permutation operations we
+destroy the feature of interest while conserving all other statistical
+properties of the data.
+
+
+\subsection{Significance of a difference in the mean}
+
+Often we would like to know whether two data sets differ in their
+mean. Whether the ears of foxes are larger in southern Europe compared
+to the ones from Scandinavia, whether a drug decreases blood pressure
+in humans, whether a sensory stimulus increases the firing rate of a
+neuron, etc. The \entermde{Nullhypothese}{null hypothesis} is that
+they do not differ in their means, i.e. that both data sets come from
+the same distribution. But even if the two data sets come from the
+same distribution, their sample means may nevertheless differ by
+chance. We need to figure out how these differences of the means are
+distributed. Only if the measured difference between the means is
+significantly larger than the ones obtained by chance we can reject
+the null hypothesis and consider the two data sets to differ
+significantly in their means.
+
+We can easily estimate the distribution of the null hypothesis by
+putting the data of both data sets in one big bag. By merging the two
+data sets we assume that all the data values come from the same
+distribution. We then randomly separate the data values into two new
+data sets. These random data sets contain data from both original data
+sets and thus come from the same distribution. From these random data
+sets we compute the difference of their sample means. This procedure
+is repeated many, say one thousand, times and each time we get a value
+for a difference of means. The distribution of these values is the
+distribution of the null hypothesis. It is the distribution of
+differences of mean values that we get by chance although the two data
+sets come from the same distribution. For a one-sided test that checks
+whether the measured difference of means is significantly larger than
+zero at a significance level of 5\,\% we compute the value of the
+95\,\% percentile of the null distribution. If the measured value is
+larger, we can reject the null hypothesis and consider the two data
+sets to differ significantly in their means.
+
+By using the original data to estimate the null hypothesis, we make no
+assumption about the properties of the data. We do not need to worry
+about the data being normally distributed. We do not need to memorize
+which test to use in which situation. And we better understand what we
+are testing, because we design the test ourselves. Nowadays, computer
+are powerful enough to iterate even ten thousand times over the data
+to compute the distribution of the null hypothesis --- with only a few
+lines of code. This is why \entermde{Permutationstest}{permutaion
+  test} are getting quite popular.
 
 \begin{figure}[tp]
-  \includegraphics[width=1\textwidth]{permutecorrelation}
+  \includegraphics[width=1\textwidth]{permuteaverage}
-  \titlecaption{\label{permutecorrelationfig}Permutation test for
+  \titlecaption{\label{permuteaverage}Permutation test for differences
-    correlations.}{Let the correlation coefficient of a dataset with
+    in means.}{We want to test whether two datasets
-    200 samples be $\rho=0.21$. The distribution of the null
+    $\left\{x_i\right\}$ (red) and $\left\{y_i\right\}$ (blue) come
-    hypothesis (yellow), optained from the correlation coefficients of
+    from different distributions by assessing the significance of the
-    permuted and therefore uncorrelated datasets is centered around
+    difference in their sample means. The data sets were generated
-    zero. The measured correlation coefficient is larger than the
+    with a difference in their population means of $d=0.7$. For
-    95\,\% percentile of the null hypothesis. The null hypothesis may
+    generating the distribution of the null hypothesis, i.e. the
-    thus be rejected and the measured correlation is considered
+    distribution of differences in the means if the two data sets come
-    statistically significant.}
+    from the same distribution, we randomly select the same number of
+    samples from both data sets (top right). This is repeated many
+    times and results in the desired distribution of differences of
+    means (bottom). The measured difference is clearly beyond the
+    95\,\% percentile of this distribution and thus indicates a
+    significant difference between the distributions of the two
+    original data sets.}
 \end{figure}
 
-A good example for the application of a
+\begin{exercise}{meandiffsignificance.m}{meandiffsignificance.out}
-\entermde{Permutationstest}{permutaion test} is the statistical
+Estimate the statistical significance of a difference in the mean of two data sets.
-assessment of \entermde[correlation]{Korrelation}{correlations}. Given
+\vspace{-1ex}
-are measured pairs of data points $(x_i, y_i)$. By calculating the
+\begin{enumerate}
+\item Generate two independent data sets, $\left\{x_i\right\}$ and
+  $\left\{y_i\right\}$, of $n=200$ samples each, by drawing random
+  numbers from a normal distribution. Add 0.2 to all the $y_i$ samples
+  to ensure the population means to differ by 0.2.
+\item Calculate the difference between the sample means of the two data sets.
+\item Estimate the distribution of the null hypothesis of no
+  difference of the means by generating new data sets with the same
+  number of samples randomly selected from both data sets.  For this
+  lump the two data sets together into a single vector. Then permute
+  the order of the elements in this vector using the function
+  \varcode{randperm()}, split it into two data sets and calculate
+  the difference of their means. Repeat this 1000 times.
+\item Read out the 95\,\% percentile from the resulting distribution
+  of the differences in the mean, the null hypothesis using the
+  \varcode{quantile()} function, and compare it with the difference of
+  means measured from the original data sets.
+\end{enumerate}
+\end{exercise}
+
+
+\subsection{Significance of correlations}
+
+Another nice example for the application of a
+\entermde{Permutationstest}{permutaion test} is testing for
+significant \entermde[correlation]{Korrelation}{correlations}
+(figure\,\ref{permutecorrelationfig}). Given are measured pairs of
+data points $(x_i, y_i)$. By calculating the
 \entermde[correlation!correlation
-coefficient]{Korrelationskoeffizient}{correlation
+coefficient]{Korrelationskoeffizient}{correlation coefficient} we can
-  coefficient} we can quantify how strongly $y$ depends on $x$. The
+quantify how strongly $y$ depends on $x$. The correlation coefficient
-correlation coefficient alone, however, does not tell whether the
+alone, however, does not tell whether the correlation is significantly
-correlation is significantly different from a random correlation. The
+different from a non-zero correlation that we might get although there
-\entermde{Nullhypothese}{null hypothesis} for such a situation is that
+is no true correlation in the data. The \entermde{Nullhypothese}{null
-$y$ does not depend on $x$. In order to perform a permutation test, we
+  hypothesis} for such a situation is that $y$ does not depend on
-need to destroy the correlation by permuting the $(x_i, y_i)$ pairs,
+$x$. In order to perform a permutation test, we need to destroy the
-i.e. we rearrange the $x_i$ and $y_i$ values in a random
+correlation between the data pairs by permuting the $(x_i, y_i)$
+pairs, i.e. we rearrange the $x_i$ and $y_i$ values in a random
 fashion. Generating many sets of random pairs and computing the
-resulting correlation coefficients yields a distribution of
+corresponding correlation coefficients yields a distribution of
-correlation coefficients that result randomly from uncorrelated
+correlation coefficients that result randomly from truly uncorrelated
 data. By comparing the actually measured correlation coefficient with
 this distribution we can directly assess the significance of the
-correlation (figure\,\ref{permutecorrelationfig}).
+correlation.
+
+\begin{figure}[tp]
+  \includegraphics[width=1\textwidth]{permutecorrelation}
+  \titlecaption{\label{permutecorrelationfig}Permutation test for
+    correlations.}{Let the correlation coefficient of a dataset with
+    200 samples be $\rho=0.21$ (top left). By shuffling the data pairs
+    we destroy any true correlation (top right). The resulting
+    distribution of the null hypothesis (bottm, yellow), optained from
+    the correlation coefficients of permuted and therefore
+    uncorrelated datasets is centered around zero. The measured
+    correlation coefficient is larger than the 95\,\% percentile of
+    the null hypothesis. The null hypothesis may thus be rejected and
+    the measured correlation is considered statistically significant.}
+\end{figure}
 
 \begin{exercise}{correlationsignificance.m}{correlationsignificance.out}
 Estimate the statistical significance of a correlation coefficient.
 \begin{enumerate}
-\item Create pairs of $(x_i, y_i)$ values. Randomly choose $x$-values
+\item Generate pairs of $(x_i, y_i)$ values. Randomly choose $x$-values
   and calculate the respective $y$-values according to $y_i =0.2 \cdot x_i + u_i$
   where $u_i$ is a random number drawn from a normal distribution.
 \item Calculate the correlation coefficient.
-\item Generate the distribution of the null hypothesis by generating
+\item Estimate the distribution of the null hypothesis by generating
   uncorrelated pairs. For this permute $x$- and $y$-values
   \matlabfun{randperm()} 1000 times and calculate for each permutation
   the correlation coefficient.
 \item Read out the 95\,\% percentile from the resulting distribution
-  of the null hypothesis and compare it with the correlation
+  of the null hypothesis using the \varcode{quantile()} function and
-  coefficient computed from the original data.
+  compare it with the correlation coefficient computed from the
+  original data.
 \end{enumerate}
 \end{exercise}
 

bootstrap/lecture/bootstrapsem.py

@@ -24,7 +24,7 @@ for i in range(nresamples) :
     musrs.append(np.mean(rng.randn(nsamples)))
 hmusrs, _ = np.histogram(musrs, bins, density=True)
 
-fig, ax = plt.subplots(figsize=cm_size(figure_width, 1.2*figure_height))
+fig, ax = plt.subplots(figsize=cm_size(figure_width, 1.05*figure_height))
 fig.subplots_adjust(**adjust_fs(left=4.0, bottom=2.7, right=1.5))
 ax.set_xlabel('Mean')
 ax.set_xlim(-0.4, 0.4)

bootstrap/lecture/permuteaverage.py

@@ -0,0 +1,103 @@
+import numpy as np
+import scipy.stats as st
+import matplotlib.pyplot as plt
+import matplotlib.gridspec as gridspec
+import matplotlib.ticker as ticker
+from plotstyle import *
+
+rng = np.random.RandomState(637281)
+
+# generate data that differ in their mein by d:
+n = 200
+d = 0.7
+x = rng.randn(n) + d;
+y = rng.randn(n);
+#x = rng.exponential(1.0, n);
+#y = rng.exponential(1.0, n) + d;
+
+# histogram of data:
+db = 0.5
+bins = np.arange(-2.5, 2.6, db)
+hx, _ = np.histogram(x, bins)
+hy, _ = np.histogram(y, bins)
+
+# Diference of means, pooled standard deviation and Cohen's d:
+ad = np.mean(x)-np.mean(y)
+s = np.sqrt(((len(x)-1)*np.var(x)+(len(y)-1)*np.var(y))/(len(x)+len(y)-2))
+cd = ad/s
+
+# permutation:
+nperm = 1000
+ads = []
+xy = np.hstack((x, y))
+for i in range(nperm) :
+    xyp = rng.permutation(xy)
+    ads.append(np.mean(xyp[:len(x)])-np.mean(xyp[len(x):]))
+
+# histogram of shuffled data:
+hxp, _ = np.histogram(xyp[:len(x)], bins)
+hyp, _ = np.histogram(xyp[len(x):], bins)
+    
+# pdf of the differences of means:
+h, b = np.histogram(ads, 20, density=True)
+
+# significance:
+dq = np.percentile(ads, 95.0)
+print('Measured difference of means = %.2f, difference at 95%% percentile of permutation = %.2f' % (ad, dq))
+da = 1.0-0.01*st.percentileofscore(ads, ad)
+print('Measured difference of means %.2f is at %.2f%% percentile of permutation' % (ad, 100.0*da))
+
+ap, at = st.ttest_ind(x, y)
+print('Measured difference of means %.2f is at %.2f%% percentile of test' % (ad, ap))
+
+
+fig = plt.figure(figsize=cm_size(figure_width, 1.8*figure_height))
+gs = gridspec.GridSpec(nrows=2, ncols=2, wspace=0.35, hspace=0.8,
+                       **adjust_fs(fig, left=5.0, right=1.5, top=1.0, bottom=2.7))
+
+ax = fig.add_subplot(gs[0,0])
+ax.bar(bins[:-1]-0.25*db, hy, 0.5*db, **fsA)
+ax.bar(bins[:-1]+0.25*db, hx, 0.5*db, **fsB)
+ax.annotate('', xy=(0.0, 45.0), xytext=(d, 45.0), arrowprops=dict(arrowstyle='<->'))
+ax.text(0.5*d, 50.0, 'd=%.1f' % d, ha='center')
+ax.set_xlim(-2.5, 2.5)
+ax.set_ylim(0.0, 50)
+ax.yaxis.set_major_locator(ticker.MultipleLocator(20.0))
+ax.set_xlabel('Original x and y values')
+ax.set_ylabel('Counts')
+
+ax = fig.add_subplot(gs[0,1])
+ax.bar(bins[:-1]-0.25*db, hyp, 0.5*db, **fsA)
+ax.bar(bins[:-1]+0.25*db, hxp, 0.5*db, **fsB)
+ax.set_xlim(-2.5, 2.5)
+ax.set_ylim(0.0, 50)
+ax.yaxis.set_major_locator(ticker.MultipleLocator(20.0))
+ax.set_xlabel('Shuffled x and y values')
+ax.set_ylabel('Counts')
+
+ax = fig.add_subplot(gs[1,:])
+ax.annotate('Measured\ndifference\nis significant!',
+            xy=(ad, 1.2), xycoords='data',
+            xytext=(ad-0.1, 2.2), textcoords='data', ha='right',
+            arrowprops=dict(arrowstyle="->", relpos=(1.0,0.5),
+                connectionstyle="angle3,angleA=-20,angleB=100") )
+ax.annotate('95% percentile',
+            xy=(0.19, 0.9), xycoords='data',
+            xytext=(0.3, 5.0), textcoords='data', ha='left',
+            arrowprops=dict(arrowstyle="->", relpos=(0.1,0.0),
+                connectionstyle="angle3,angleA=40,angleB=80") )
+ax.annotate('Distribution of\nnullhypothesis',
+            xy=(-0.08, 3.0), xycoords='data',
+            xytext=(-0.22, 4.5), textcoords='data', ha='left',
+            arrowprops=dict(arrowstyle="->", relpos=(0.2,0.0),
+                connectionstyle="angle3,angleA=60,angleB=150") )
+ax.bar(b[:-1], h, width=b[1]-b[0], **fsC)
+ax.bar(b[:-1][b[:-1]>=dq], h[b[:-1]>=dq], width=b[1]-b[0], **fsB)
+ax.plot( [ad, ad], [0, 1], **lsA)
+ax.set_xlim(-0.25, 0.85)
+ax.set_ylim(0.0, 5.0)
+ax.yaxis.set_major_locator(ticker.MultipleLocator(2.0))
+ax.set_xlabel('Difference of means')
+ax.set_ylabel('PDF of H0')
+
+plt.savefig('permuteaverage.pdf')

bootstrap/lecture/permutecorrelation.py

@@ -1,9 +1,11 @@
 import numpy as np
 import scipy.stats as st
 import matplotlib.pyplot as plt
+import matplotlib.gridspec as gridspec
+import matplotlib.ticker as ticker
 from plotstyle import *
 
-rng = np.random.RandomState(637281)
+rng = np.random.RandomState(37281)
 
 # generate correlated data:
 n = 200
@@ -22,29 +24,53 @@ for i in range(nperm) :
     xr = rng.permutation(x)
     yr = rng.permutation(y)
     rs.append(np.corrcoef(xr, yr)[0, 1])
+rr = np.corrcoef(xr, yr)[0, 1]
 
 # pdf of the correlation coefficients:
 h, b = np.histogram(rs, 20, density=True)
 
 # significance:
 rq = np.percentile(rs, 95.0)
-print('Measured correlation coefficient = %.2f, correlation coefficient at 95%% percentile of bootstrap = %.2f' % (rd, rq))
+print('Measured correlation coefficient = %.2f, correlation coefficient at 95%% percentile of permutation = %.2f' % (rd, rq))
 ra = 1.0-0.01*st.percentileofscore(rs, rd)
-print('Measured correlation coefficient %.2f is at %.4f percentile of bootstrap' % (rd, ra))
+print('Measured correlation coefficient %.2f is at %.4f percentile of permutation' % (rd, ra))
 
 rp, ra = st.pearsonr(x, y)
 print('Measured correlation coefficient %.2f is at %.4f percentile of test' % (rp, ra))
 
-fig, ax = plt.subplots(figsize=cm_size(figure_width, 1.2*figure_height))
+fig = plt.figure(figsize=cm_size(figure_width, 1.8*figure_height))
-fig.subplots_adjust(**adjust_fs(left=4.0, bottom=2.7, right=0.5, top=1.0))
+gs = gridspec.GridSpec(nrows=2, ncols=2, wspace=0.35, hspace=0.8,
+                       **adjust_fs(fig, left=5.0, right=1.5, top=1.0, bottom=2.7))
+
+ax = fig.add_subplot(gs[0,0])
+ax.text(0.0, 4.0, 'r=%.2f' % rd, ha='center')
+ax.plot(x, y, **psAm)
+ax.set_xlim(-4.0, 4.0)
+ax.set_ylim(-4.0, 4.0)
+ax.xaxis.set_major_locator(ticker.MultipleLocator(2.0))
+ax.yaxis.set_major_locator(ticker.MultipleLocator(2.0))
+ax.set_xlabel('x')
+ax.set_ylabel('y')
+
+ax = fig.add_subplot(gs[0,1])
+ax.text(0.0, 4.0, 'r=%.2f' % rr, ha='center')
+ax.plot(xr, yr, **psAm)
+ax.set_xlim(-4.0, 4.0)
+ax.set_ylim(-4.0, 4.0)
+ax.xaxis.set_major_locator(ticker.MultipleLocator(2.0))
+ax.yaxis.set_major_locator(ticker.MultipleLocator(2.0))
+ax.set_xlabel('Shuffled x')
+ax.set_ylabel('Shuffled y')
+
+ax = fig.add_subplot(gs[1,:])
 ax.annotate('Measured\ncorrelation\nis significant!',
             xy=(rd, 1.1), xycoords='data',
-            xytext=(rd, 2.2), textcoords='data', ha='left',
+            xytext=(rd-0.01, 3.0), textcoords='data', ha='left',
-            arrowprops=dict(arrowstyle="->", relpos=(0.2,0.0),
+            arrowprops=dict(arrowstyle="->", relpos=(0.3,0.0),
                 connectionstyle="angle3,angleA=10,angleB=80") )
 ax.annotate('95% percentile',
             xy=(0.14, 0.9), xycoords='data',
-            xytext=(0.18, 4.0), textcoords='data', ha='left',
+            xytext=(0.16, 6.2), textcoords='data', ha='left',
             arrowprops=dict(arrowstyle="->", relpos=(0.1,0.0),
                 connectionstyle="angle3,angleA=30,angleB=80") )
 ax.annotate('Distribution of\nuncorrelated\nsamples',
@@ -56,7 +82,9 @@ ax.bar(b[:-1], h, width=b[1]-b[0], **fsC)
 ax.bar(b[:-1][b[:-1]>=rq], h[b[:-1]>=rq], width=b[1]-b[0], **fsB)
 ax.plot( [rd, rd], [0, 1], **lsA)
 ax.set_xlim(-0.25, 0.35)
+ax.set_ylim(0.0, 6.0)
+ax.yaxis.set_major_locator(ticker.MultipleLocator(2.0))
 ax.set_xlabel('Correlation coefficient')
-ax.set_ylabel('Prob. density of H0')
+ax.set_ylabel('PDF of H0')
 
 plt.savefig('permutecorrelation.pdf')

chapter.mk

@@ -10,6 +10,8 @@ pythonplots : $(PYPDFFILES)
 
 $(PYPDFFILES) : %.pdf: %.py ../../plotstyle.py
 	PYTHONPATH="../../" python3 $<
+#ps2pdf $@ $(@:.pdf=-temp.pdf)  # strip fonts, saves only about 1.5MB
+#mv $(@:.pdf=-temp.pdf) $@
 
 cleanpythonplots :
 	rm -f $(PYPDFFILES)
@@ -42,6 +44,9 @@ $(BASENAME)-chapter.pdf : $(BASENAME)-chapter.tex $(BASENAME).tex $(wildcard $(B
 watchchapter :
 	while true; do ! make -q chapter && make chapter; sleep 0.5; done
 
+watchplots :
+	while true; do ! make -q plots && make plots; sleep 0.5; done
+
 cleantex:
 	rm -f *~ 
 	rm -f $(BASENAME).aux $(BASENAME).log *.idx

codestyle/lecture/codestyle.tex

@@ -4,6 +4,7 @@
   more months might as well have been written by someone
   else.}{Eagleson's law}
 
+\noindent
 Cultivating a good code style is not just a matter of good taste but
 rather is a key ingredient for readability and maintainability of code
 and, in the end, facilitates reproducibility of scientific
@@ -204,7 +205,7 @@ random walk\footnote{A random walk is a simple simulation of Brownian
 (listing \ref{chaoticcode}) then in cleaner way (listing
 \ref{cleancode})
 
-\begin{lstlisting}[label=chaoticcode, caption={Chaotic implementation of the random-walk.}]
+\begin{pagelisting}[label=chaoticcode, caption={Chaotic implementation of the random-walk.}]
 num_runs = 10; max_steps = 1000;
 
 positions = zeros(max_steps, num_runs);
@@ -218,17 +219,14 @@ x = randn(1);
 if x<0
 positions(step, run)= positions(step-1, run)+1;
        
-       
 elseif x>0
  positions(step,run)=positions(step-1,run)-1;
  end
 end
 end
-\end{lstlisting}
+\end{pagelisting}
 
-\pagebreak[4]
+\begin{pagelisting}[label=cleancode, caption={Clean implementation of the random-walk.}]
-
-\begin{lstlisting}[label=cleancode, caption={Clean implementation of the random-walk.}]
 num_runs = 10;
 max_steps = 1000;
 positions = zeros(max_steps, num_runs);
@@ -243,7 +241,7 @@ for run = 1:num_runs
         end
     end
 end
-\end{lstlisting}
+\end{pagelisting}
 
 \section{Using comments}
 
@@ -275,7 +273,6 @@ avoided:\\ \varcode{ x = x + 2; \% add two to x}\\
     make it even clearer.}{Steve McConnell}
 \end{important}
 
-\pagebreak[4]
 \section{Documenting functions}
 All pre-defined \matlab{} functions begin with a comment block that
 describes the purpose of the function, the required and optional
@@ -335,8 +332,7 @@ used within the context of another function \matlab{} allows to define
 within the same file. Listing \ref{localfunctions} shows an example of
 a local function definition.
 
-\pagebreak[3] 
+\pageinputlisting[label=localfunctions, caption={Example for local
-\lstinputlisting[label=localfunctions, caption={Example for local
   functions.}]{calculateSines.m}
 
 \emph{Local function} live in the same \entermde{m-File}{m-file} as

debugging/lecture/debugging.tex

@@ -59,14 +59,14 @@ every opening parenthesis must be matched by a closing one or every
 respective error messages are clear and the editor will point out and
 highlight most syntax errors.
 
-\begin{lstlisting}[label=syntaxerror, caption={Unbalanced parenthesis error.}]
+\begin{pagelisting}[label=syntaxerror, caption={Unbalanced parenthesis error.}]
 >> mean(random_numbers
                       |
 Error: Expression or statement is incorrect--possibly unbalanced (, {, or [.
 
 Did you mean:
 >>   mean(random_numbers)
-\end{lstlisting}
+\end{pagelisting}
 
 \subsection{Indexing error}\label{index_error}
 Second on the list of common errors are the
@@ -125,7 +125,7 @@ dimensions. The command in line 7 works due to the fact, that matlab
 automatically extends the matrix, if you assign values to a range
 outside its bounds.
 
-\begin{lstlisting}[label=assignmenterror, caption={Assignment errors.}]
+\begin{pagelisting}[label=assignmenterror, caption={Assignment errors.}]
 >> a = zeros(1, 100);
 >> b = 0:10;
 
@@ -136,7 +136,7 @@ outside its bounds.
 >> size(a)
 ans =
      110    1
-\end{lstlisting}
+\end{pagelisting}
 
 \subsection{Dimension mismatch error}
 Similarly, some arithmetic operations are only valid if the variables
@@ -154,7 +154,7 @@ in listing\,\ref{dimensionmismatch} does not throw an error but the
 result is something else than the expected elementwise multiplication.
 
 % XXX Some arithmetic operations make size constraints, violating them leads to dimension mismatch errors.
-\begin{lstlisting}[label=dimensionmismatch, caption={Dimension mismatch errors.}]
+\begin{pagelisting}[label=dimensionmismatch, caption={Dimension mismatch errors.}]
   >> a = randn(100, 1);
   >> b = randn(10, 1);
   >> a + b
@@ -171,7 +171,7 @@ result is something else than the expected elementwise multiplication.
   >> size(c)
   ans =
        100    10
-\end{lstlisting}
+\end{pagelisting}
 
 
 
@@ -239,7 +239,7 @@ but it requires a deep understanding of the applied functions and also
 the task at hand.
 
 % XXX Converting a series of spike times into the firing rate as a function of time. Many tasks can be solved with a single line of code. But is this readable?
-\begin{lstlisting}[label=easyvscomplicated, caption={One-liner versus readable code.}]
+\begin{pagelisting}[label=easyvscomplicated, caption={One-liner versus readable code.}]
 % the one-liner
 rate = conv(full(sparse(1, round(spike_times/dt), 1, 1, length(time))), kernel, 'same');
 
@@ -248,7 +248,7 @@ rate = zeros(size(time));
 spike_indices = round(spike_times/dt);
 rate(spike_indices) = 1;
 rate = conv(rate, kernel, 'same');
-\end{lstlisting}
+\end{pagelisting}
 
 The preferred way depends on several considerations. (i) How deep is
 your personal understanding of the programming language?  (ii) What
@@ -291,7 +291,7 @@ example given in the \matlab{} help and assume that there is a
 function \varcode{rightTriangle()} (listing\,\ref{trianglelisting}).
 
 % XXX Slightly more readable version of the example given in the \matlab{} help system. Note: The variable name for the angles have been capitalized in order to not override the matlab defined functions \code{alpha, beta,} and \code{gamma}.
-\begin{lstlisting}[label=trianglelisting, caption={Example function for unit testing.}]
+\begin{pagelisting}[label=trianglelisting, caption={Example function for unit testing.}]
 function angles = rightTriangle(length_a, length_b)
     ALPHA = atand(length_a / length_b);
     BETA = atand(length_a / length_b);
@@ -300,7 +300,7 @@ function angles = rightTriangle(length_a, length_b)
 
     angles = [ALPHA BETA GAMMA];
 end
-\end{lstlisting}
+\end{pagelisting}
 
 This function expects two input arguments that are the length of the
 sides $a$ and $b$ and assumes a right angle between them. From this
@@ -337,7 +337,7 @@ The test script for the \varcode{rightTriangle()} function
 (listing\,\ref{trianglelisting}) may look like in
 listing\,\ref{testscript}.
 
-\begin{lstlisting}[label=testscript, caption={Unit test for the \varcode{rightTriangle()} function stored in an m-file testRightTriangle.m}]
+\begin{pagelisting}[label=testscript, caption={Unit test for the \varcode{rightTriangle()} function stored in an m-file testRightTriangle.m}]
 tolerance = 1e-10;
 
 % preconditions
@@ -373,7 +373,7 @@ angles = rightTriangle(1, 1500);
 smallAngle = (pi / 180) * angles(1); % radians
 approx = sin(smallAngle);
 assert(abs(approx - smallAngle) <= tolerance, 'Problem with small angle approximation')
-\end{lstlisting}
+\end{pagelisting}
 
 In a test script we can execute any code. The actual test whether or
 not the results match our predictions is done using the
@@ -390,16 +390,16 @@ executed. We further define a \varcode{tolerance} variable that is
 used when comparing double values (Why might the test on equality of
 double values be tricky?).
 
-\begin{lstlisting}[label=runtestlisting, caption={Run the test!}]
+\begin{pagelisting}[label=runtestlisting, caption={Run the test!}]
 result = runtests('testRightTriangle')
-\end{lstlisting}
+\end{pagelisting}
 
 During the run, \matlab{} will put out error messages onto the command
 line and a summary of the test results is then stored within the
 \varcode{result} variable. These can be displayed using the function
 \code[table()]{table(result)}.
 
-\begin{lstlisting}[label=testresults, caption={The test results.}, basicstyle=\ttfamily\scriptsize]
+\begin{pagelisting}[label=testresults, caption={The test results.}, basicstyle=\ttfamily\scriptsize]
 table(result)
 ans =
   4x6 table
@@ -411,7 +411,7 @@ _________________________________   ______  ______  ___________  ________  _____
 'testR.../Test_IsoscelesTriangles'   true   false   false        0.004893   [1x1 struct]
 'testR.../Test_30_60_90Triangle'     true   false   false        0.005057   [1x1 struct]
 'testR.../Test_SmallAngleApprox'     true   false   false        0.0049     [1x1 struct]
-\end{lstlisting}
+\end{pagelisting}
 
 So far so good, all tests pass and our function appears to do what it
 is supposed to do. But tests are only as good as the programmer who
@@ -479,11 +479,11 @@ stopped in debug mode (listing\,\ref{debuggerlisting}).
 \end{figure}
 
 
-\begin{lstlisting}[label=debuggerlisting, caption={Command line when the program execution was stopped in the debugger.}]
+\begin{pagelisting}[label=debuggerlisting, caption={Command line when the program execution was stopped in the debugger.}]
 >> simplerandomwalk
 6   for run = 1:num_runs
 K>>
-\end{lstlisting}
+\end{pagelisting}
 
 When stopped in the debugger we can view and change the state of the
 program at this point, we can also issue commands to try the next

designpattern/lecture/designpattern.tex

@@ -10,7 +10,8 @@ pattern]{design pattern}.
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \section{Looping over  vector elements}
 Iterating over vector elements by means of a \mcode{for}-loop is a very commen pattern:
-\begin{lstlisting}[caption={\varcode{for}-loop for accessing vector elements by indices}]
+
+\begin{pagelisting}[caption={\varcode{for}-loop for accessing vector elements by indices}]
 x = [2:3:20];      % Some vector.
 for i=1:length(x)  % For loop over the indices of the vector.
   i                % This is the index (an integer number)
@@ -22,12 +23,14 @@ for i=1:length(x)  % For loop over the indices of the vector.
   % as an argument to a function:
   do_something(x(i));
 end
-\end{lstlisting}
+\end{pagelisting}
 
+\noindent
 If the result of the computation within the loop are single numbers
 that are to be stored in a vector, you should create this vector with
 the right size before the loop:
-\begin{lstlisting}[caption={\varcode{for}-loop for writing a vector}]
+
+\begin{pagelisting}[caption={\varcode{for}-loop for writing a vector}]
 x = [1.2 2.3 2.6 3.1];   % Some vector.
 % Create a vector for the results, as long as the number of loops:
 y = zeros(length(x),1);
@@ -38,13 +41,15 @@ for i=1:length(x)
 end
 % Now the result vector can be further processed:
 mean(y);
-\end{lstlisting}
+\end{pagelisting}
 
+\noindent
 The computation within the loop could also result in a vector of some
 length and not just a single number. If the length of this vector
 (here 10) is known beforehand, then you should create a matrix of
 appropriate size for storing the results:
-\begin{lstlisting}[caption={\varcode{for}-loop for writing rows of a matrix}]
+
+\begin{pagelisting}[caption={\varcode{for}-loop for writing rows of a matrix}]
 x = [2:3:20];             % Some vector.
 % Create space for results -
 % as many rows as loops, as many columns as needed:
@@ -56,30 +61,33 @@ for i=1:length(x)
 end
 % Process the results stored in matrix y:
 mean(y, 1)
-\end{lstlisting}
+\end{pagelisting}
 
+\noindent
 Another possibility is that the result vectors (here of unknown size)
 need to be combined into a single large vector:
-\begin{lstlisting}[caption={\varcode{for}-loop for appending vectors}]
+
+\begin{pagelisting}[caption={\varcode{for}-loop for appending vectors}]
 x = [2:3:20];        % Some vector.
 y = [];              % Empty vector for storing the results.
 for i=1:length(x)
   % The function get_something() returns a vector of unspecified size:
-  z = get_somehow_more(x(i));
+  z = get_something(x(i));
   % The content of z is appended to the result vector y:
-  y = [y; z(:)];
+  y = [y, z(:)];
   % The z(:) syntax ensures that we append column-vectors.
 end
-% Process the results stored in the vector z:
+% Process the results stored in the vector y:
-mean(y)
+mean(y, 1)
-\end{lstlisting}
+\end{pagelisting}
 
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \section{Scaling and shifting random numbers, zeros, and ones}
 Random number generators usually return random numbers of a given mean
 and standard deviation. Multiply those numbers by a factor to change their standard deviation and add a number to shift the mean.
-\begin{lstlisting}[caption={Scaling and shifting of random numbers}]
+
+\begin{pagelisting}[caption={Scaling and shifting of random numbers}]
 % 100 random numbers drawn from a normal distribution 
 % with mean 0 and standard deviation 1:
 x = randn(100, 1);
@@ -89,18 +97,20 @@ x = randn(100, 1);
 mu = 4.8;
 sigma = 2.3;
 y = randn(100, 1)*sigma + mu;
-\end{lstlisting}
+\end{pagelisting}
 
+\noindent
 The same principle can be useful for in the context of the functions
 \mcode{zeros()} or \mcode{ones()}:
-\begin{lstlisting}[caption={Scaling and shifting of \varcode{zeros()} and \varcode{ones()}}]
+
+\begin{pagelisting}[caption={Scaling and shifting of \varcode{zeros()} and \varcode{ones()}}]
 x = -1:0.01:2;      % Vector of x-values for plotting
 plot(x, exp(-x.*x));
 % Plot for the same x-values a horizontal line with y=0.8:
 plot(x, zeros(size(x))+0.8);
 % ... or a line with y=0.5:
 plot(x, ones(size(x))*0.5);
-\end{lstlisting}
+\end{pagelisting}
 
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -119,25 +129,26 @@ can plot the values of the $y$ vector against the ones of the $x$
 vector.
 
 The following scripts compute and plot the function $f(x)=e^{-x^2}$:
-\begin{lstlisting}[caption={Plotting a mathematical function --- very detailed}]
+
+\begin{pagelisting}[caption={Plotting a mathematical function --- very detailed}]
 xmin = -1.0;
 xmax = 2.0;
 dx = 0.01;        % Step size
 x = xmin:dx:xmax; % Vector with x-values.
 y = exp(-x.*x);   % No for loop! '.*' for multiplying the vector elements.
 plot(x, y);
-\end{lstlisting}
+\end{pagelisting}
 
-\begin{lstlisting}[caption={Plotting a mathematical function --- shorter}]
+\begin{pagelisting}[caption={Plotting a mathematical function --- shorter}]
 x = -1:0.01:2;
 y = exp(-x.*x);
 plot(x, y);
-\end{lstlisting}
+\end{pagelisting}
 
-\begin{lstlisting}[caption={Plotting a mathematical function --- compact}]
+\begin{pagelisting}[caption={Plotting a mathematical function --- compact}]
 x = -1:0.01:2;
 plot(x, exp(-x.*x));
-\end{lstlisting}
+\end{pagelisting}
 
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -146,28 +157,34 @@ For estimating probabilities or probability densities from histograms
 we need to normalize them appropriately.
 
 The \mcode{histogram()} function does this automatically with the appropriate arguments:
-\begin{lstlisting}[caption={Probability density with the \varcode{histogram()}-function}]
+
+\begin{pagelisting}[caption={Probability density with the \varcode{histogram()}-function}]
 x = randn(100, 1);         % Some real-valued data.
 histogram(x, 'Normalization', 'pdf');
-\end{lstlisting}
+\end{pagelisting}
-\begin{lstlisting}[caption={Probability with the \varcode{histogram()}-function}]
+
+\begin{pagelisting}[caption={Probability with the \varcode{histogram()}-function}]
 x = randi(6, 100, 1);      % Some integer-valued data.
 histogram(x, 'Normalization', 'probability');
-\end{lstlisting}
+\end{pagelisting}
+
+\noindent
 Alternatively one can normalize the histogram data as returned by the
 \code{hist()}-function manually:
-\begin{lstlisting}[caption={Probability density with the \varcode{hist()}- and \varcode{bar()}-function}]
+
+\begin{pagelisting}[caption={Probability density with the \varcode{hist()}- and \varcode{bar()}-function}]
 x = randn(100, 1);         % Some real-valued data.
 [h, b] = hist(x);          % Compute histogram.
 h = h/sum(h)/(b(2)-b(1));  % Normalization to a probability density.
 bar(b, h);                 % Plot the probability density.
-\end{lstlisting}
+\end{pagelisting}
-\begin{lstlisting}[caption={Probability with the \varcode{hist()}- and \varcode{bar()}-function}]
+
+\begin{pagelisting}[caption={Probability with the \varcode{hist()}- and \varcode{bar()}-function}]
 x = randi(6, 100, 1);      % Some integer-valued data.
 [h, b] = hist(x);          % Compute histogram.
 h = h/sum(h);              % Normalize to probability.
 bar(b, h);                 % Plot the probabilities.
-\end{lstlisting}
+\end{pagelisting}
 
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

exercises.mk

@@ -0,0 +1,33 @@
+EXERCISES=$(TEXFILES:.tex=.pdf)
+SOLUTIONS=$(EXERCISES:%.pdf=%-solutions.pdf)
+
+.PHONY: pdf exercises solutions watch watchexercises watchsolutions clean
+
+pdf : $(SOLUTIONS) $(EXERCISES)
+
+exercises : $(EXERCISES)
+
+solutions : $(SOLUTIONS)
+
+$(SOLUTIONS) : %-solutions.pdf : %.tex ../../exercisesheader.tex ../../exercisestitle.tex
+	{ echo "\\documentclass[answers,12pt,a4paper,pdftex]{exam}"; sed -e '1d' $<; } > $(patsubst %.pdf,%.tex,$@)
+	pdflatex -interaction=scrollmode $(patsubst %.pdf,%.tex,$@) | tee /dev/stderr | fgrep -q "Rerun to get cross-references right" && pdflatex -interaction=scrollmode $(patsubst %.pdf,%.tex,$@) || true
+	rm $(patsubst %.pdf,%,$@).[!p]*
+
+$(EXERCISES) : %.pdf : %.tex ../../exercisesheader.tex ../../exercisestitle.tex
+	pdflatex -interaction=scrollmode $< | tee /dev/stderr | fgrep -q "Rerun to get cross-references right" && pdflatex -interaction=scrollmode $< || true
+
+watch :
+	while true; do ! make -q pdf && make pdf; sleep 0.5; done
+
+watchexercises :
+	while true; do ! make -q exercises && make exercises; sleep 0.5; done
+
+watchsolutions :
+	while true; do ! make -q solutions && make solutions; sleep 0.5; done
+
+clean :
+	rm -f *~ *.aux *.log *.out
+
+cleanup : clean
+	rm -f $(SOLUTIONS) $(EXERCISES)

exercisesheader.tex

@@ -0,0 +1,110 @@
+\usepackage[english]{babel}
+\usepackage{pslatex}
+\usepackage{graphicx}
+\usepackage{tikz}
+\usepackage{xcolor}
+\usepackage{amsmath}
+\usepackage{amssymb}
+
+%%%%% listings %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\usepackage{listings}
+\lstset{
+  language=Matlab,
+  basicstyle=\ttfamily\footnotesize,
+  numbers=left,
+  numberstyle=\tiny,
+  title=\lstname,
+  showstringspaces=false,
+  commentstyle=\itshape\color{darkgray},
+  breaklines=true,
+  breakautoindent=true,
+  % columns=flexible,
+  frame=single,
+  xleftmargin=1.5em,
+  xrightmargin=1em,
+  aboveskip=10pt
+}
+
+%%%%% page style %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\usepackage[left=20mm,right=20mm,top=25mm,bottom=19mm,headsep=2ex,headheight=3ex,footskip=3.5ex]{geometry}
+\pagestyle{headandfoot}
+\ifprintanswers
+\newcommand{\stitle}{Solutions}
+\else
+\newcommand{\stitle}{}
+\fi
+\header{{\bfseries\large \exercisenum. \exercisetopic}}{{\bfseries\large\stitle}}{{\bfseries\large\exercisedate}}
+\firstpagefooter{Prof. Dr. Jan Benda}{}{jan.benda@uni-tuebingen.de}
+\runningfooter{}{\thepage}{}
+
+\shadedsolutions
+\definecolor{SolutionColor}{gray}{0.9}
+
+\setlength{\baselineskip}{15pt}
+\setlength{\parindent}{0.0cm}
+\setlength{\parskip}{0.3cm}
+
+\usepackage{multicol}
+
+%%%%% units %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\usepackage[mediumspace,mediumqspace,Gray,squaren]{SIunits}      % \ohm, \micro
+
+%%%%% new commands %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\newcommand{\cin}[1]{[{\rm #1}]_{\text{in}}}
+\newcommand{\cout}[1]{[{\rm #1}]_{\text{out}}}
+\newcommand{\units}[1]{\text{#1}}
+\newcommand{\K}{\text{K$^+$}}
+\newcommand{\Na}{\text{Na$^+$}}
+\newcommand{\Cl}{\text{Cl$^-$}}
+\newcommand{\Ca}{\text{Ca$^{2+}$}}
+\newcommand{\water}{$\text{H}_2\text{O}$}
+
+\usepackage{dsfont}
+\newcommand{\naZ}{\mathds{N}}
+\newcommand{\gaZ}{\mathds{Z}}
+\newcommand{\raZ}{\mathds{Q}}
+\newcommand{\reZ}{\mathds{R}}
+\newcommand{\reZp}{\mathds{R^+}}
+\newcommand{\reZpN}{\mathds{R^+_0}}
+\newcommand{\koZ}{\mathds{C}}
+\newcommand{\RT}{{\rm Re\!\ }}
+\newcommand{\IT}{{\rm Im\!\ }}
+\newcommand{\arccot}{{\rm arccot}}
+\newcommand{\sgn}{{\rm sgn}}
+\newcommand{\im}{{\rm i}}
+\newcommand{\drv}{\mathrm{d}}
+\newcommand{\divis}[2]{$^{#1}\!\!/\!_{#2}$}
+\newcommand{\vect}[2]{\left( \!\! \begin{array}{c} #1 \\ #2 \end{array} \!\! \right)}
+\newcommand{\matt}[2]{\left( \!\! \begin{array}{cc} #1 \\ #2 \end{array} \!\! \right)}
+\renewcommand{\Re}{\mathrm{Re}}
+\renewcommand{\Im}{\mathrm{Im}}
+\newcommand{\av}[1]{\left\langle #1 \right\rangle}
+
+\newcommand{\pref}[1]{(\ref{#1})}
+\newcommand{\eqnref}[1]{(\ref{#1})}
+
+\newcommand{\hr}{\par\vspace{-5ex}\noindent\rule{\textwidth}{1pt}\par\vspace{-1ex}}
+
+\newcommand{\qt}[1]{\textbf{#1}\\}
+
+\newcommand{\extra}{--- Optional ---\ \mbox{}}
+
+\newcommand{\code}[1]{\texttt{#1}}
+
+%%%%% page breaks %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\newcommand{\continue}{\ifprintanswers%
+\else
+\vfill\hspace*{\fill}$\rightarrow$\newpage%
+\fi}
+\newcommand{\continuepage}{\ifprintanswers%
+\newpage
+\else
+\vfill\hspace*{\fill}$\rightarrow$\newpage%
+\fi}
+\newcommand{\newsolutionpage}{\ifprintanswers%
+\newpage%
+\else
+\fi}
+
+%%%%% hyperref %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\usepackage[breaklinks=true,bookmarks=true,bookmarksopen=true,pdfpagemode=UseNone,pdfstartview=FitH,colorlinks=true,citecolor=blue!50!black,linkcolor=blue!50!black,urlcolor=blue!50!black]{hyperref}

exercisestitle.tex

@@ -4,3 +4,4 @@
 {\large Jan Grewe, Jan Benda}\\[-3ex]
 Neuroethology Lab \hfill --- \hfill Institute for Neurobiology \hfill --- \hfill \includegraphics[width=0.28\textwidth]{UT_WBMW_Black_RGB} \\
 \end{center}
+\hr

header.tex

@@ -2,7 +2,7 @@
 \title{\textbf{\huge\sffamily\tr{Introduction to\\[1ex] Scientific Computing}%
        {Einf\"uhrung in die\\[1ex] wissenschaftliche Datenverarbeitung}}}
 
-\author{{\LARGE Jan Grewe \& Jan Benda}\\[5ex]Abteilung Neuroethologie\\[2ex]%
+\author{{\LARGE Jan Grewe \& Jan Benda}\\[5ex]Neuroethology Lab\\[2ex]%
         \includegraphics[width=0.3\textwidth]{UT_WBMW_Rot_RGB}\vspace{3ex}}
 
 \date{WS 2020/2021\\\vfill%
@@ -77,7 +77,7 @@
 \setcounter{totalnumber}{2}
 
 % float placement fractions:
-\renewcommand{\textfraction}{0.2}
+\renewcommand{\textfraction}{0.1}
 \renewcommand{\topfraction}{0.9}
 \renewcommand{\bottomfraction}{0.0}
 \renewcommand{\floatpagefraction}{0.7}
@@ -209,6 +209,21 @@
 \let\l@lstlisting\l@figure
 \makeatother
 
+% \lstinputlisting wrapped in a minipage to avoid page breaks:
+\newcommand{\pageinputlisting}[2][]{\vspace{-2ex}\noindent\begin{minipage}[t]{1\linewidth}\lstinputlisting[#1]{#2}\end{minipage}}
+
+%%%%% listing environment: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+% usage:
+%
+% \begin{pagelisting}[label=listing1, caption={A script.}]
+%   >> x = 6
+% \end{pagelisting}
+%
+% This is the lstlisting environment but wrapped in a minipage to avoid page breaks.
+\lstnewenvironment{pagelisting}[1][]%
+  {\vspace{-2ex}\lstset{#1}\noindent\minipage[t]{1\linewidth}}%
+  {\endminipage}
+
 %%%%% english, german, code and file terms: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \usepackage{ifthen}
 
@@ -349,11 +364,12 @@
       }{%
         \immediate\write\solutions{\unexpanded{\subsection}{Exercise \thechapter.\arabic{exercisef}}}%
         \immediate\write\solutions{\unexpanded{\label}{solution\arabic{chapter}-\arabic{exercisef}}}%
-        \immediate\write\solutions{\unexpanded{\lstinputlisting[belowskip=0ex,aboveskip=0ex,%
+        \immediate\write\solutions{\unexpanded{\begin{minipage}{1\linewidth}\lstinputlisting[belowskip=0ex,aboveskip=0ex,%
-          nolol=true, title={\textbf{Source code:} \protect\StrSubstitute{#1}{_}{\_}}]}{\codepath#1}}%
+          nolol=true, title={\textbf{Source code:} \protect\StrSubstitute{#1}{_}{\_}}]}{\codepath#1}\unexpanded{\end{minipage}}}%
         \ifthenelse{\equal{#2}{}}{}%
-          {\immediate\write\solutions{\unexpanded{\lstinputlisting[language={},%
+          {\immediate\write\solutions{\unexpanded{\begin{minipage}{1\linewidth}\lstinputlisting[language={},%
-             nolol=true, title={\textbf{Output:}}, belowskip=0ex, aboveskip=1ex]}{\codepath#2}}}%
+             nolol=true, title={\textbf{Output:}}, belowskip=0ex, aboveskip=1ex]}{\codepath#2}\unexpanded{\end{minipage}}}}%
+        \immediate\write\solutions{\unexpanded{\vspace*{\fill}}}%
         \immediate\write\solutions{}%
       }%
   }%
@@ -362,14 +378,18 @@
         { \hypersetup{hypertexnames=false}%
           \ifthenelse{\equal{\exercisesource}{}}{}%
           { \addtocounter{lstlisting}{-1}%
-            \lstinputlisting[belowskip=0pt,aboveskip=1ex,nolol=true,%
+            \par\noindent\begin{minipage}[t]{1\linewidth}%
+            \lstinputlisting[belowskip=1ex,aboveskip=-1ex,nolol=true,%
                              title={\textbf{Solution:} \exercisefile}]%
                             {\exercisesource}%
+            \end{minipage}%
             \ifthenelse{\equal{\exerciseoutput}{}}{}%
               { \addtocounter{lstlisting}{-1}%
+                \par\noindent\begin{minipage}[t]{1\linewidth}%
                 \lstinputlisting[language={},title={\textbf{Output:}},%
-                                 nolol=true,belowskip=0pt]%
+                                 nolol=true,belowskip=0pt,aboveskip=-0.5ex]%
                                 {\exerciseoutput}%
+                \end{minipage}%
               }%
           }%
           \hypersetup{hypertexnames=true}%
@@ -452,12 +472,12 @@
   { \captionsetup{singlelinecheck=off,hypcap=false,labelformat={empty},%
                   labelfont={large,sf,it,bf},font={large,sf,it,bf}}
     \ifthenelse{\equal{#1}{}}%
-    { \begin{mdframed}[linecolor=importantline,linewidth=1ex,%
+    { \begin{mdframed}[nobreak=true,linecolor=importantline,linewidth=1ex,%
                        backgroundcolor=importantback,font={\sffamily}]%
         \setlength{\parindent}{0pt}%
         \setlength{\parskip}{1ex}%
     }%
-    { \begin{mdframed}[linecolor=importantline,linewidth=1ex,%
+    { \begin{mdframed}[nobreak=true,linecolor=importantline,linewidth=1ex,%
                        backgroundcolor=importantback,font={\sffamily},%
                        frametitle={\captionof{iboxf}{#1}},frametitleaboveskip=-1ex,%
                        frametitlebackgroundcolor=importantline]%

likelihood/exercises/Makefile

@@ -1,34 +1,3 @@
-TEXFILES=$(wildcard exercises??.tex)
+TEXFILES=$(wildcard likelihood-?.tex)
-EXERCISES=$(TEXFILES:.tex=.pdf)
-SOLUTIONS=$(EXERCISES:exercises%=solutions%)
 
-.PHONY: pdf exercises solutions watch watchexercises watchsolutions clean
+include ../../exercises.mk
-
-pdf : $(SOLUTIONS) $(EXERCISES)
-
-exercises : $(EXERCISES)
-
-solutions : $(SOLUTIONS)
-
-$(SOLUTIONS) : solutions%.pdf : exercises%.tex instructions.tex
-	{ echo "\\documentclass[answers,12pt,a4paper,pdftex]{exam}"; sed -e '1d' $<; } > $(patsubst %.pdf,%.tex,$@)
-	pdflatex -interaction=scrollmode $(patsubst %.pdf,%.tex,$@) | tee /dev/stderr | fgrep -q "Rerun to get cross-references right" && pdflatex -interaction=scrollmode $(patsubst %.pdf,%.tex,$@) || true
-	rm $(patsubst %.pdf,%,$@).[!p]*
-
-$(EXERCISES) : %.pdf : %.tex instructions.tex
-	pdflatex -interaction=scrollmode $< | tee /dev/stderr | fgrep -q "Rerun to get cross-references right" && pdflatex -interaction=scrollmode $< || true
-
-watch :
-	while true; do ! make -q pdf && make pdf; sleep 0.5; done
-
-watchexercises :
-	while true; do ! make -q exercises && make exercises; sleep 0.5; done
-
-watchsolutions :
-	while true; do ! make -q solutions && make solutions; sleep 0.5; done
-
-clean :
-	rm -f *~ *.aux *.log *.out
-
-cleanup : clean
-	rm -f $(SOLUTIONS) $(EXERCISES)

likelihood/exercises/instructions.tex

@@ -1,41 +0,0 @@
-\vspace*{-8ex}
-\begin{center}
-\textbf{\Large Introduction to scientific computing}\\[1ex]
-{\large Jan Grewe, Jan Benda}\\[-3ex]
-Neuroethology lab \hfill --- \hfill Institute for Neurobiology \hfill --- \hfill \includegraphics[width=0.28\textwidth]{UT_WBMW_Black_RGB} \\
-\end{center}
-
-% \ifprintanswers%
-% \else
-
-% % Die folgenden Aufgaben dienen der Wiederholung, \"Ubung und
-% % Selbstkontrolle und sollten eigenst\"andig bearbeitet und gel\"ost
-% % werden. Die L\"osung soll in Form eines einzelnen Skriptes (m-files)
-% % im ILIAS hochgeladen werden. Jede Aufgabe sollte in einer eigenen
-% % ``Zelle'' gel\"ost sein. Die Zellen \textbf{m\"ussen} unabh\"angig
-% % voneinander ausf\"uhrbar sein. Das Skript sollte nach dem Muster:
-% % ``variablen\_datentypen\_\{nachname\}.m'' benannt werden
-% % (z.B. variablen\_datentypen\_mueller.m).
-
-% \begin{itemize}
-% \item \"Uberzeuge dich von jeder einzelnen Zeile deines Codes, dass
-%   sie auch wirklich das macht, was sie machen soll! Teste dies mit
-%   kleinen Beispielen direkt in der Kommandozeile.
-% \item Versuche die L\"osungen der Aufgaben m\"oglichst in
-%   sinnvolle kleine Funktionen herunterzubrechen.
-%   Sobald etwas \"ahnliches mehr als einmal berechnet werden soll,
-%   lohnt es sich eine Funktion daraus zu schreiben!
-% \item Teste rechenintensive \code{for} Schleifen, Vektoren, Matrizen
-%   zuerst mit einer kleinen Anzahl von Wiederholungen oder kleiner
-%   Gr\"o{\ss}e, und benutze erst am Ende, wenn alles \"uberpr\"uft
-%   ist, eine gro{\ss}e Anzahl von Wiederholungen oder Elementen, um eine gute
-%   Statistik zu bekommen.
-% \item Benutze die Hilfsfunktion von \code{matlab} (\code{help
-%     commando} oder \code{doc commando}) und das Internet, um
-%   herauszufinden, wie bestimmte \code{matlab} Funktionen zu verwenden
-%   sind und was f\"ur M\"oglichkeiten sie bieten.
-%   Auch zu inhaltlichen Konzepten bietet das Internet oft viele
-%   Antworten!
-% \end{itemize}
-
-% \fi

likelihood/exercises/likelihood-1.tex

@@ -1,90 +1,17 @@
 \documentclass[12pt,a4paper,pdftex]{exam}
 
-\usepackage[german]{babel}
+\newcommand{\exercisetopic}{Maximum Likelihood}
-\usepackage{pslatex}
+\newcommand{\exercisenum}{10}
-\usepackage[mediumspace,mediumqspace,Gray]{SIunits}      % \ohm, \micro
+\newcommand{\exercisedate}{January 12th, 2021}
-\usepackage{xcolor}
-\usepackage{graphicx}
-\usepackage[breaklinks=true,bookmarks=true,bookmarksopen=true,pdfpagemode=UseNone,pdfstartview=FitH,colorlinks=true,citecolor=blue]{hyperref}
-
-%%%%% layout %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage[left=20mm,right=20mm,top=25mm,bottom=25mm]{geometry}
-\pagestyle{headandfoot}
-\ifprintanswers
-\newcommand{\stitle}{: Solutions}
-\else
-\newcommand{\stitle}{}
-\fi
-\header{{\bfseries\large Exercise 11\stitle}}{{\bfseries\large Maximum likelihood}}{{\bfseries\large January 7th, 2020}}
-\firstpagefooter{Prof. Dr. Jan Benda}{Phone: 29 74573}{Email:
-jan.benda@uni-tuebingen.de}
-\runningfooter{}{\thepage}{}
-
-\setlength{\baselineskip}{15pt}
-\setlength{\parindent}{0.0cm}
-\setlength{\parskip}{0.3cm}
-\renewcommand{\baselinestretch}{1.15}
-
-%%%%% listings %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage{listings}
-\lstset{
-  language=Matlab,
-  basicstyle=\ttfamily\footnotesize,
-  numbers=left,
-  numberstyle=\tiny,
-  title=\lstname,
-  showstringspaces=false,
-  commentstyle=\itshape\color{darkgray},
-  breaklines=true,
-  breakautoindent=true,
-  columns=flexible,
-  frame=single,
-  xleftmargin=1em,
-  xrightmargin=1em,
-  aboveskip=10pt
-}
-
-%%%%% math stuff: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage{amsmath}
-\usepackage{amssymb}
-\usepackage{bm} 
-\usepackage{dsfont}
-\newcommand{\naZ}{\mathds{N}}
-\newcommand{\gaZ}{\mathds{Z}}
-\newcommand{\raZ}{\mathds{Q}}
-\newcommand{\reZ}{\mathds{R}}
-\newcommand{\reZp}{\mathds{R^+}}
-\newcommand{\reZpN}{\mathds{R^+_0}}
-\newcommand{\koZ}{\mathds{C}}
-
-%%%%% page breaks %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\newcommand{\continue}{\ifprintanswers%
-\else
-\vfill\hspace*{\fill}$\rightarrow$\newpage%
-\fi}
-\newcommand{\continuepage}{\ifprintanswers%
-\newpage
-\else
-\vfill\hspace*{\fill}$\rightarrow$\newpage%
-\fi}
-\newcommand{\newsolutionpage}{\ifprintanswers%
-\newpage%
-\else
-\fi}
-
-%%%%% new commands %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\newcommand{\qt}[1]{\textbf{#1}\\}
-\newcommand{\pref}[1]{(\ref{#1})}
-\newcommand{\extra}{--- Zusatzaufgabe ---\ \mbox{}}
-\newcommand{\code}[1]{\texttt{#1}}
 
+\input{../../exercisesheader}
+
+\firstpagefooter{Prof. Dr. Jan Benda}{}{jan.benda@uni-tuebingen.de}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \begin{document}
 
-\input{instructions}
+\input{../../exercisestitle}
-
 
 \begin{questions}
 

plotstyle.py

@@ -33,17 +33,18 @@ colors['white'] = '#FFFFFF'
 # general settings for plot styles:
 lwthick = 3.0
 lwthin = 1.8
+edgewidth = 0.0 if xkcd_style else 1.0
 mainline = {'linestyle': '-', 'linewidth': lwthick}
 minorline = {'linestyle': '-', 'linewidth': lwthin}
-largemarker = {'marker': 'o', 'markersize': 9, 'markeredgecolor': colors['white'], 'markeredgewidth': 1}
+largemarker = {'marker': 'o', 'markersize': 9, 'markeredgecolor': colors['white'], 'markeredgewidth': edgewidth}
-smallmarker = {'marker': 'o', 'markersize': 6, 'markeredgecolor': colors['white'], 'markeredgewidth': 1}
+smallmarker = {'marker': 'o', 'markersize': 6, 'markeredgecolor': colors['white'], 'markeredgewidth': edgewidth}
 largelinepoints = {'linestyle': '-', 'linewidth': lwthick, 'marker': 'o', 'markersize': 10, 'markeredgecolor': colors['white'], 'markeredgewidth': 1}
 smalllinepoints = {'linestyle': '-', 'linewidth': 1.4, 'marker': 'o', 'markersize': 7, 'markeredgecolor': colors['white'], 'markeredgewidth': 1}
-filllw = 1.0
+filllw = edgewidth
 fillec = colors['white']
 fillalpha = 0.4
 filledge = {'linewidth': filllw, 'joinstyle': 'round'}
-if int(mpl.__version__.split('.')[0]) < 2:
+if mpl_major < 2:
     del filledge['joinstyle']
 
 # helper lines:

plotting/exercises/Makefile

@@ -0,0 +1,3 @@
+TEXFILES=$(wildcard plotting-?.tex)
+
+include ../../exercises.mk

plotting/exercises/instructions.tex

@@ -1,17 +1,11 @@
-\vspace*{-8ex}
+\ifprintanswers%
-\begin{center}
+\else
-\textbf{\Large Introduction to scientific computing}\\[1ex]
-{\large Jan Grewe, Jan Benda}\\[-3ex]
-Neuroethology lab \hfill --- \hfill Institute for Neurobiology \hfill --- \hfill \includegraphics[width=0.28\textwidth]{UT_WBMW_Black_RGB} \\
-\end{center}
 
 The exercises are meant for self-monitoring and revision of the
 lecture. You should try to solve them on your own. In contrast
 to previous exercises, the solutions can not be saved in a single file. Combine the files into a
 single zip archive and submit it via ILIAS. Name the archive according
 to the pattern: ``plotting\_\{surname\}.zip''.
-% \ifprintanswers%
-% \else
 
 % % Die folgenden Aufgaben dienen der Wiederholung, \"Ubung und
 % % Selbstkontrolle und sollten eigenst\"andig bearbeitet und gel\"ost
@@ -43,4 +37,4 @@ to the pattern: ``plotting\_\{surname\}.zip''.
 %   Antworten!
 % \end{itemize}
 
-% \fi
+\fi

plotting/exercises/plotting-1.tex

@@ -1,88 +1,18 @@
 \documentclass[12pt,a4paper,pdftex]{exam}
 
-\usepackage[german]{babel}
+\newcommand{\exercisetopic}{Plotting}
-\usepackage{pslatex}
+\newcommand{\exercisenum}{X}
-\usepackage[mediumspace,mediumqspace]{SIunits}      % \ohm, \micro
+\newcommand{\exercisedate}{December 14th, 2020}
-\usepackage{xcolor}
-\usepackage{graphicx}
-\usepackage[breaklinks=true,bookmarks=true,bookmarksopen=true,pdfpagemode=UseNone,pdfstartview=FitH,colorlinks=true,citecolor=blue]{hyperref}
 
-%%%%% layout %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\input{../../exercisesheader}
-\usepackage[left=20mm,right=20mm,top=25mm,bottom=25mm]{geometry}
-\pagestyle{headandfoot}
-\ifprintanswers
-\newcommand{\stitle}{: Solutions}
-\else
-\newcommand{\stitle}{}
-\fi
-\header{{\bfseries\large Exercise 7\stitle}}{{\bfseries\large Plotting}}{{\bfseries\large November 20, 2019}}
-\firstpagefooter{Dr. Jan Grewe}{Phone: 29 74588}{Email:
-jan.grewe@uni-tuebingen.de}
-\runningfooter{}{\thepage}{}
 
-\setlength{\baselineskip}{15pt}
+\firstpagefooter{Dr. Jan Grewe}{}{jan.grewe@uni-tuebingen.de}
-\setlength{\parindent}{0.0cm}
-\setlength{\parskip}{0.3cm}
-\renewcommand{\baselinestretch}{1.15}
 
-%%%%% listings %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage{listings}
-\lstset{
-  language=Matlab,
-  basicstyle=\ttfamily\footnotesize,
-  numbers=left,
-  numberstyle=\tiny,
-  title=\lstname,
-  showstringspaces=false,
-  commentstyle=\itshape\color{darkgray},
-  breaklines=true,
-  breakautoindent=true,
-  columns=flexible,
-  frame=single,
-  xleftmargin=1em,
-  xrightmargin=1em,
-  aboveskip=10pt
-}
-
-%%%%% math stuff: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage{amsmath}
-\usepackage{amssymb}
-\usepackage{bm} 
-\usepackage{dsfont}
-\newcommand{\naZ}{\mathds{N}}
-\newcommand{\gaZ}{\mathds{Z}}
-\newcommand{\raZ}{\mathds{Q}}
-\newcommand{\reZ}{\mathds{R}}
-\newcommand{\reZp}{\mathds{R^+}}
-\newcommand{\reZpN}{\mathds{R^+_0}}
-\newcommand{\koZ}{\mathds{C}}
-
-%%%%% page breaks %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\newcommand{\continue}{\ifprintanswers%
-\else
-\vfill\hspace*{\fill}$\rightarrow$\newpage%
-\fi}
-\newcommand{\continuepage}{\ifprintanswers%
-\newpage
-\else
-\vfill\hspace*{\fill}$\rightarrow$\newpage%
-\fi}
-\newcommand{\newsolutionpage}{\ifprintanswers%
-\newpage%
-\else
-\fi}
-
-%%%%% new commands %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\newcommand{\qt}[1]{\textbf{#1}\\}
-\newcommand{\pref}[1]{(\ref{#1})}
-\newcommand{\extra}{--- Zusatzaufgabe ---\ \mbox{}}
-\newcommand{\code}[1]{\texttt{#1}}
-
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \begin{document}
 
+\input{../../exercisestitle}
+
 \input{instructions}
 
 \begin{questions}

plotting/lecture/plotting-chapter.tex

@@ -3,7 +3,7 @@
 \input{../../header}
 
 \lstset{inputpath=../code}
-\graphicspath{{images/}}
+\graphicspath{{figures/}}
 
 \typein[\pagenumber]{Number of first page}
 \typein[\chapternumber]{Chapter number}

plotting/lecture/plotting.tex

@@ -74,20 +74,20 @@ the data was changed or the same kind of plot has to be created for a
 number of datasets.
 
 \begin{important}[Why manual editing should be avoided.]
-  On first glance the manual editing of a figure using tools such as
+  On first glance manual editing of a figure using tools such as
-  Corel draw, Illustrator, etc.\,appears much more convenient and less
+  inkscape, Corel draw, Illustrator, etc.\,appears much more
-  complex than coding everything into the analysis scripts. This,
+  convenient and less complex than coding everything into the analysis
-  however, is not entirely true. What if the figure has to be re-drawn
+  scripts. This, however, is not entirely true. What if the figure has
-  or updated? Then the editing work starts all over again. Rather,
+  to be re-drawn or updated, because, for example, you got more data?
-  there is a great risk associated with the manual editing
+  Then the editing work starts all over again. In addition, there is a
-  approach. Axes may be shifted, fonts have not been embedded into the
+  great risk associated with the manual editing approach. Axes may be
-  final document, annotations have been copy pasted between figures
+  shifted, fonts have not been embedded into the final document,
-  and are not valid. All of these mistakes can be found in
+  annotations have been copy-pasted between figures and are not
-  publications and then require an erratum, which is not
+  valid. All of these mistakes can be found in publications and then
-  desirable. Even if it appears more cumbersome in the beginning one
+  require an erratum, which is not desirable. Even if it appears more
-  should always try to create publication-ready figures directly from
+  cumbersome in the beginning, one should always try to generate
-  the data analysis tool using scripts or functions to properly layout
+  publication-ready figures directly from the data analysis tool using
-  the plot.
+  scripts or functions to properly layout and annotate the plot.
 \end{important}
 
 \subsection{Simple plotting}
@@ -148,7 +148,7 @@ or the color. For additional options consult the help.
 
 The following listing shows a simple line plot with axis labeling and a title
 
-\lstinputlisting[caption={A simple plot showing a sinewave.},
+\pageinputlisting[caption={A simple plot showing a sinewave.},
   label=simpleplotlisting]{simple_plot.m}
 
 
@@ -162,10 +162,10 @@ chosen, and star marker symbols is used. Finally, the name of the
 curve is set to \emph{plot 1} which will be displayed in a legend, if
 chosen.
 
-\begin{lstlisting}[label=settinglineprops, caption={Setting line properties when calling \varcode{plot}.}]
+\begin{pagelisting}[label=settinglineprops, caption={Setting line properties when calling \varcode{plot}.}]
   x = 0:0.1:2*pi; y = sin(x); plot( x, y, 'color', 'r', 'linestyle',
   ':', 'marker', '*', 'linewidth', 1.5, 'displayname', 'plot 1')
-\end{lstlisting}
+\end{pagelisting}
 
 \begin{important}[Choosing the right color.]
   Choosing the perfect color goes a little bit beyond personal
@@ -277,7 +277,7 @@ the last one defines the output format (box\,\ref{graphicsformatbox}).
     listing\,\ref{niceplotlisting}.}\label{spikedetectionfig}
 \end{figure}
 
-\begin{ibox}[t]{\label{graphicsformatbox}File formats for digital artwork.}
+\begin{ibox}[tp]{\label{graphicsformatbox}File formats for digital artwork.}
   There are two fundamentally different types of formats for digital artwork:
   \begin{enumerate}
   \item \enterm[bitmap]{Bitmaps} (\determ{Rastergrafik})
@@ -322,7 +322,7 @@ the last one defines the output format (box\,\ref{graphicsformatbox}).
   efficient.
 \end{ibox}
 
-\lstinputlisting[caption={Script for creating the plot shown in
+\pageinputlisting[caption={Script for creating the plot shown in
     \figref{spikedetectionfig}.},
   label=niceplotlisting]{automatic_plot.m}
 
@@ -380,7 +380,7 @@ draw the data. In the example we also provide further arguments to set
 the size, color of the dots and specify that they are filled
 (listing\,\ref{scatterlisting1}).
 
-\lstinputlisting[caption={Creating a scatter plot with red filled dots.},
+\pageinputlisting[caption={Creating a scatter plot with red filled dots.},
   label=scatterlisting1, firstline=9, lastline=9]{scatterplot.m}
 
 We could have used plot for this purpose and set the marker to
@@ -395,8 +395,7 @@ manipulate the color we need to specify a length(x)-by-3 matrix. For
 each dot we provide an individual color (i.e. the RGB triplet in each
 row of the color matrix, lines 2-4 in listing\,\ref{scatterlisting2})
 
-
+\pageinputlisting[caption={Creating a scatter plot with size and color
-\lstinputlisting[caption={Creating a scatter plot with size and color
     variations. The RGB triplets define the respective color intensity
     in a range 0:1. Here, we modify only the red color channel.},
   label=scatterlisting2, linerange={15-15, 21-23}]{scatterplot.m}
@@ -431,7 +430,7 @@ figures\,\ref{regularsubplotsfig}, \ref{irregularsubplotsfig}).
     also below).}\label{regularsubplotsfig}
 \end{figure}
 
-\lstinputlisting[caption={Script for creating subplots in a regular
+\pageinputlisting[caption={Script for creating subplots in a regular
     grid \figref{regularsubplotsfig}.}, label=regularsubplotlisting,
   basicstyle=\ttfamily\scriptsize]{regular_subplot.m}
 
@@ -458,7 +457,7 @@ create a grid with larger numbers of columns and rows, and specify the
 used cells of the grid by passing a vector as the third argument to
 \code{subplot()}.
 
-\lstinputlisting[caption={Script for creating subplots of different
+\pageinputlisting[caption={Script for creating subplots of different
     sizes \figref{irregularsubplotsfig}.},
   label=irregularsubplotslisting,
   basicstyle=\ttfamily\scriptsize]{irregular_subplot.m}
@@ -516,7 +515,7 @@ its properties. See the \matlab{} help for more information.
     listing\,\ref{errorbarlisting} for A and C and listing\,\ref{errorbarlisting2} }\label{errorbarplot}
 \end{figure}
 
-\lstinputlisting[caption={Illustrating estimation errors using error bars. Script that
+\pageinputlisting[caption={Illustrating estimation errors using error bars. Script that
     creates \figref{errorbarplot}. A, B},
   label=errorbarlisting, firstline=13, lastline=31,
   basicstyle=\ttfamily\scriptsize]{errorbarplot.m}
@@ -550,7 +549,7 @@ leading to invisibility and a value of one to complete
 opaqueness. Finally, we use the normal plot command to draw a line
 connecting the average values (line 12).
 
-\lstinputlisting[caption={Illustrating estimation errors using a shaded area. Script that
+\pageinputlisting[caption={Illustrating estimation errors using a shaded area. Script that
     creates \figref{errorbarplot} C.}, label=errorbarlisting2,
   firstline=33,
   basicstyle=\ttfamily\scriptsize]{errorbarplot.m}
@@ -575,7 +574,7 @@ listing\,\ref{annotationsplotlisting}. For more options consult the
     listing\,\ref{annotationsplotlisting}}\label{annotationsplot}
 \end{figure}
 
-\lstinputlisting[caption={Adding annotations to figures. Script that
+\pageinputlisting[caption={Adding annotations to figures. Script that
     creates \figref{annotationsplot}.},
   label=annotationsplotlisting,
   basicstyle=\ttfamily\scriptsize]{annotations.m}
@@ -632,7 +631,7 @@ Lissajous figure. The basic steps are:
 \item Finally, close the file (line 31).
 \end{enumerate}
 
-\lstinputlisting[caption={Making animations and saving them as a
+\pageinputlisting[caption={Making animations and saving them as a
     movie.}, label=animationlisting, firstline=16, lastline=36,
   basicstyle=\ttfamily\scriptsize]{movie_example.m}
 

pointprocesses/code/convolutionRate.m

@@ -10,14 +10,13 @@ function [time, rate] = convolution_rate(spikes, sigma, dt, t_max)
 %   t_max : the trial duration in seconds.
 %
 % Returns:
-    two vectors containing the time and the rate.
+%   two vectors containing the time and the rate.
 
     time = 0:dt:t_max - dt;
     rate = zeros(size(time));
     spike_indices = round(spikes / dt);
     rate(spike_indices) = 1;
     kernel = gaussKernel(sigma, dt);
-
     rate = conv(rate, kernel, 'same');
 end
 

pointprocesses/exercises/Makefile

@@ -1,35 +1,3 @@
-BASENAME=pointprocesses
+TEXFILES=$(wildcard pointprocesses-?.tex)
-TEXFILES=$(wildcard $(BASENAME)??.tex)
-EXERCISES=$(TEXFILES:.tex=.pdf)
-SOLUTIONS=$(EXERCISES:pointprocesses%=pointprocesses-solutions%)
 
-.PHONY: pdf exercises solutions watch watchexercises watchsolutions clean
+include ../../exercises.mk
-
-pdf : $(SOLUTIONS) $(EXERCISES)
-
-exercises : $(EXERCISES)
-
-solutions : $(SOLUTIONS)
-
-$(SOLUTIONS) : pointprocesses-solutions%.pdf : pointprocesses%.tex instructions.tex
-	{ echo "\\documentclass[answers,12pt,a4paper,pdftex]{exam}"; sed -e '1d' $<; } > $(patsubst %.pdf,%.tex,$@)
-	pdflatex -interaction=scrollmode $(patsubst %.pdf,%.tex,$@) | tee /dev/stderr | fgrep -q "Rerun to get cross-references right" && pdflatex -interaction=scrollmode $(patsubst %.pdf,%.tex,$@) || true
-	rm $(patsubst %.pdf,%,$@).[!p]*
-
-$(EXERCISES) : %.pdf : %.tex instructions.tex
-	pdflatex -interaction=scrollmode $< | tee /dev/stderr | fgrep -q "Rerun to get cross-references right" && pdflatex -interaction=scrollmode $< || true
-
-watch :
-	while true; do ! make -q pdf && make pdf; sleep 0.5; done
-
-watchexercises :
-	while true; do ! make -q exercises && make exercises; sleep 0.5; done
-
-watchsolutions :
-	while true; do ! make -q solutions && make solutions; sleep 0.5; done
-
-clean :
-	rm -f *~ *.aux *.log *.out
-
-cleanup : clean
-	rm -f $(SOLUTIONS) $(EXERCISES)

pointprocesses/exercises/pointprocesses-1.tex

@@ -1,90 +1,17 @@
 \documentclass[12pt,a4paper,pdftex]{exam}
 
-\usepackage[german]{babel}
+\newcommand{\exercisetopic}{Point Processes}
-\usepackage{pslatex}
+\newcommand{\exercisenum}{11}
-\usepackage[mediumspace,mediumqspace,Gray]{SIunits}      % \ohm, \micro
+\newcommand{\exercisedate}{January 19th, 2021}
-\usepackage{xcolor}
-\usepackage{graphicx}
-\usepackage[breaklinks=true,bookmarks=true,bookmarksopen=true,pdfpagemode=UseNone,pdfstartview=FitH,colorlinks=true,citecolor=blue]{hyperref}
-
-%%%%% layout %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage[left=20mm,right=20mm,top=25mm,bottom=25mm]{geometry}
-\pagestyle{headandfoot}
-\ifprintanswers
-\newcommand{\stitle}{: Solutions}
-\else
-\newcommand{\stitle}{}
-\fi
-\header{{\bfseries\large Exercise 12\stitle}}{{\bfseries\large Point processes}}{{\bfseries\large January 14th, 2020}}
-\firstpagefooter{Prof. Dr. Jan Benda}{Phone: 29 74573}{Email:
-jan.benda@uni-tuebingen.de}
-\runningfooter{}{\thepage}{}
-
-\setlength{\baselineskip}{15pt}
-\setlength{\parindent}{0.0cm}
-\setlength{\parskip}{0.3cm}
-\renewcommand{\baselinestretch}{1.15}
-
-%%%%% listings %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage{listings}
-\lstset{
-  language=Matlab,
-  basicstyle=\ttfamily\footnotesize,
-  numbers=left,
-  numberstyle=\tiny,
-  title=\lstname,
-  showstringspaces=false,
-  commentstyle=\itshape\color{darkgray},
-  breaklines=true,
-  breakautoindent=true,
-  columns=flexible,
-  frame=single,
-  xleftmargin=1em,
-  xrightmargin=1em,
-  aboveskip=10pt
-}
-
-%%%%% math stuff: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage{amsmath}
-\usepackage{amssymb}
-\usepackage{bm} 
-\usepackage{dsfont}
-\newcommand{\naZ}{\mathds{N}}
-\newcommand{\gaZ}{\mathds{Z}}
-\newcommand{\raZ}{\mathds{Q}}
-\newcommand{\reZ}{\mathds{R}}
-\newcommand{\reZp}{\mathds{R^+}}
-\newcommand{\reZpN}{\mathds{R^+_0}}
-\newcommand{\koZ}{\mathds{C}}
-
-%%%%% page breaks %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\newcommand{\continue}{\ifprintanswers%
-\else
-\vfill\hspace*{\fill}$\rightarrow$\newpage%
-\fi}
-\newcommand{\continuepage}{\ifprintanswers%
-\newpage
-\else
-\vfill\hspace*{\fill}$\rightarrow$\newpage%
-\fi}
-\newcommand{\newsolutionpage}{\ifprintanswers%
-\newpage%
-\else
-\fi}
-
-%%%%% new commands %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\newcommand{\qt}[1]{\textbf{#1}\\}
-\newcommand{\pref}[1]{(\ref{#1})}
-\newcommand{\extra}{--- Zusatzaufgabe ---\ \mbox{}}
-\newcommand{\code}[1]{\texttt{#1}}
 
+\input{../../exercisesheader}
+
+\firstpagefooter{Prof. Dr. Jan Benda}{}{jan.benda@uni-tuebingen.de}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \begin{document}
 
-\input{instructions}
+\input{../../exercisestitle}
-
 
 \begin{questions}
 

pointprocesses/exercises/pointprocesses-2.tex

@@ -1,90 +1,17 @@
 \documentclass[12pt,a4paper,pdftex]{exam}
 
-\usepackage[german]{babel}
+\newcommand{\exercisetopic}{Point Processes}
-\usepackage{pslatex}
+\newcommand{\exercisenum}{X2}
-\usepackage[mediumspace,mediumqspace,Gray]{SIunits}      % \ohm, \micro
+\newcommand{\exercisedate}{January 19th, 2021}
-\usepackage{xcolor}
-\usepackage{graphicx}
-\usepackage[breaklinks=true,bookmarks=true,bookmarksopen=true,pdfpagemode=UseNone,pdfstartview=FitH,colorlinks=true,citecolor=blue]{hyperref}
-
-%%%%% layout %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage[left=20mm,right=20mm,top=25mm,bottom=25mm]{geometry}
-\pagestyle{headandfoot}
-\ifprintanswers
-\newcommand{\stitle}{L\"osungen}
-\else
-\newcommand{\stitle}{\"Ubung}
-\fi
-\header{{\bfseries\large \stitle}}{{\bfseries\large Punktprozesse 2}}{{\bfseries\large 27. Oktober, 2015}}
-\firstpagefooter{Prof. Dr. Jan Benda}{Phone: 29 74573}{Email:
-jan.benda@uni-tuebingen.de}
-\runningfooter{}{\thepage}{}
-
-\setlength{\baselineskip}{15pt}
-\setlength{\parindent}{0.0cm}
-\setlength{\parskip}{0.3cm}
-\renewcommand{\baselinestretch}{1.15}
-
-%%%%% listings %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage{listings}
-\lstset{
-  language=Matlab,
-  basicstyle=\ttfamily\footnotesize,
-  numbers=left,
-  numberstyle=\tiny,
-  title=\lstname,
-  showstringspaces=false,
-  commentstyle=\itshape\color{darkgray},
-  breaklines=true,
-  breakautoindent=true,
-  columns=flexible,
-  frame=single,
-  xleftmargin=1em,
-  xrightmargin=1em,
-  aboveskip=10pt
-}
-
-%%%%% math stuff: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage{amsmath}
-\usepackage{amssymb}
-\usepackage{bm} 
-\usepackage{dsfont}
-\newcommand{\naZ}{\mathds{N}}
-\newcommand{\gaZ}{\mathds{Z}}
-\newcommand{\raZ}{\mathds{Q}}
-\newcommand{\reZ}{\mathds{R}}
-\newcommand{\reZp}{\mathds{R^+}}
-\newcommand{\reZpN}{\mathds{R^+_0}}
-\newcommand{\koZ}{\mathds{C}}
-
-%%%%% page breaks %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\newcommand{\continue}{\ifprintanswers%
-\else
-\vfill\hspace*{\fill}$\rightarrow$\newpage%
-\fi}
-\newcommand{\continuepage}{\ifprintanswers%
-\newpage
-\else
-\vfill\hspace*{\fill}$\rightarrow$\newpage%
-\fi}
-\newcommand{\newsolutionpage}{\ifprintanswers%
-\newpage%
-\else
-\fi}
-
-%%%%% new commands %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\newcommand{\qt}[1]{\textbf{#1}\\}
-\newcommand{\pref}[1]{(\ref{#1})}
-\newcommand{\extra}{--- Zusatzaufgabe ---\ \mbox{}}
-\newcommand{\code}[1]{\texttt{#1}}
 
+\input{../../exercisesheader}
+
+\firstpagefooter{Prof. Dr. Jan Benda}{}{jan.benda@uni-tuebingen.de}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \begin{document}
 
-\input{instructions}
+\input{../../exercisestitle}
-
 
 \begin{questions}
 

pointprocesses/exercises/pointprocesses-3.tex

@@ -1,90 +1,17 @@
 \documentclass[12pt,a4paper,pdftex]{exam}
 
-\usepackage[german]{babel}
+\newcommand{\exercisetopic}{Point Processes}
-\usepackage{pslatex}
+\newcommand{\exercisenum}{X3}
-\usepackage[mediumspace,mediumqspace,Gray]{SIunits}      % \ohm, \micro
+\newcommand{\exercisedate}{January 19th, 2021}
-\usepackage{xcolor}
-\usepackage{graphicx}
-\usepackage[breaklinks=true,bookmarks=true,bookmarksopen=true,pdfpagemode=UseNone,pdfstartview=FitH,colorlinks=true,citecolor=blue]{hyperref}
-
-%%%%% layout %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage[left=20mm,right=20mm,top=25mm,bottom=25mm]{geometry}
-\pagestyle{headandfoot}
-\ifprintanswers
-\newcommand{\stitle}{: L\"osungen}
-\else
-\newcommand{\stitle}{}
-\fi
-\header{{\bfseries\large \"Ubung 8\stitle}}{{\bfseries\large Spiketrain Analyse}}{{\bfseries\large 6. Dezember, 2016}}
-\firstpagefooter{Prof. Dr. Jan Benda}{Phone: 29 74573}{Email:
-jan.benda@uni-tuebingen.de}
-\runningfooter{}{\thepage}{}
-
-\setlength{\baselineskip}{15pt}
-\setlength{\parindent}{0.0cm}
-\setlength{\parskip}{0.3cm}
-\renewcommand{\baselinestretch}{1.15}
-
-%%%%% listings %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage{listings}
-\lstset{
-  language=Matlab,
-  basicstyle=\ttfamily\footnotesize,
-  numbers=left,
-  numberstyle=\tiny,
-  title=\lstname,
-  showstringspaces=false,
-  commentstyle=\itshape\color{darkgray},
-  breaklines=true,
-  breakautoindent=true,
-  columns=flexible,
-  frame=single,
-  xleftmargin=1em,
-  xrightmargin=1em,
-  aboveskip=10pt
-}
-
-%%%%% math stuff: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage{amsmath}
-\usepackage{amssymb}
-\usepackage{bm} 
-\usepackage{dsfont}
-\newcommand{\naZ}{\mathds{N}}
-\newcommand{\gaZ}{\mathds{Z}}
-\newcommand{\raZ}{\mathds{Q}}
-\newcommand{\reZ}{\mathds{R}}
-\newcommand{\reZp}{\mathds{R^+}}
-\newcommand{\reZpN}{\mathds{R^+_0}}
-\newcommand{\koZ}{\mathds{C}}
-
-%%%%% page breaks %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\newcommand{\continue}{\ifprintanswers%
-\else
-\vfill\hspace*{\fill}$\rightarrow$\newpage%
-\fi}
-\newcommand{\continuepage}{\ifprintanswers%
-\newpage
-\else
-\vfill\hspace*{\fill}$\rightarrow$\newpage%
-\fi}
-\newcommand{\newsolutionpage}{\ifprintanswers%
-\newpage%
-\else
-\fi}
-
-%%%%% new commands %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\newcommand{\qt}[1]{\textbf{#1}\\}
-\newcommand{\pref}[1]{(\ref{#1})}
-\newcommand{\extra}{--- Zusatzaufgabe ---\ \mbox{}}
-\newcommand{\code}[1]{\texttt{#1}}
 
+\input{../../exercisesheader}
+
+\firstpagefooter{Prof. Dr. Jan Benda}{}{jan.benda@uni-tuebingen.de}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \begin{document}
 
-\input{instructions}
+\input{../../exercisestitle}
-
 
 \begin{questions}
   

pointprocesses/lecture/pointprocesses-chapter.tex

@@ -25,9 +25,9 @@
 \item Multitrial firing rates
 \item Better explain difference between ISI method and PSTHes. The
   latter is dependent on precision of spike times the former not.
-\item Choice of bin width for PSTH, kernel width, also in relation sto
+\item Choice of bin width for PSTH, kernel width, also in relation to
   stimulus time scale
-\item Kernle firing rate: discuss different kernel shapes, in
+\item Kernel firing rate: discuss different kernel shapes, in
   particular causal kernels (gamma, exponential), relation to synaptic
   potentials
 \end{itemize}

pointprocesses/lecture/pointprocessscetchA.eps

@@ -1,7 +1,7 @@
 %!PS-Adobe-2.0 EPSF-2.0
 %%Title: pointprocessscetchA.tex
 %%Creator: gnuplot 4.6 patchlevel 4
-%%CreationDate: Tue Oct 27 23:58:04 2020
+%%CreationDate: Mon Dec 14 23:59:13 2020
 %%DocumentFonts: 
 %%BoundingBox: 50 50 373 135
 %%EndComments
@@ -433,7 +433,7 @@ SDict begin [
   /Author (jan)
 %  /Producer (gnuplot)
 %  /Keywords ()
-  /CreationDate (Tue Oct 27 23:58:04 2020)
+  /CreationDate (Mon Dec 14 23:59:13 2020)
   /DOCINFO pdfmark
 end
 } ifelse

pointprocesses/lecture/pointprocessscetchB.eps

@@ -1,7 +1,7 @@
 %!PS-Adobe-2.0 EPSF-2.0
 %%Title: pointprocessscetchB.tex
 %%Creator: gnuplot 4.6 patchlevel 4
-%%CreationDate: Tue Oct 27 23:58:04 2020
+%%CreationDate: Mon Dec 14 23:59:13 2020
 %%DocumentFonts: 
 %%BoundingBox: 50 50 373 237
 %%EndComments
@@ -433,7 +433,7 @@ SDict begin [
   /Author (jan)
 %  /Producer (gnuplot)
 %  /Keywords ()
-  /CreationDate (Tue Oct 27 23:58:04 2020)
+  /CreationDate (Mon Dec 14 23:59:13 2020)
   /DOCINFO pdfmark
 end
 } ifelse

programming/code/vectorsize.m

@@ -1,4 +1,5 @@
 a = [2 4 6 8 10];   % row vector with five elements
 s = size(a)  % store the return value of size() in a new variable
-s(2)        % get the second element of s, i.e. the length along the 2nd dimension
+s(2)         % get the second element of s, 
+             % i.e. the length along the 2nd dimension
 size(a, 2)   % the shortcut

programming/code/vectorsize.out

@@ -1,3 +1,3 @@
-s =  1    10
+s =  1    5
-ans = 10
+ans = 5
-ans = 10
+ans = 5

programming/exercises/Makefile

@@ -0,0 +1,11 @@
+TEXFILES= \
+  boolean_logical_indexing.tex \
+  control_flow.tex \
+  matrices.tex \
+  scripts_functions.tex \
+  structs_cells.tex \
+  variables_types.tex \
+  vectors_matrices.tex \
+  vectors.tex
+
+include ../../exercises.mk

programming/exercises/boolean_logical_indexing.tex

@@ -1,39 +1,17 @@
 \documentclass[12pt,a4paper,pdftex]{exam}
 
-\usepackage[german]{babel}
+\newcommand{\exercisetopic}{Logical indexing}
-\usepackage{natbib}
+\newcommand{\exercisenum}{4}
-\usepackage{graphicx}
+\newcommand{\exercisedate}{17. November, 2020}
-\usepackage[small]{caption}
+
-\usepackage{sidecap}
+\input{../../exercisesheader}
-\usepackage{pslatex}
+
-\usepackage{amsmath}
+\firstpagefooter{Dr. Jan Grewe}{}{jan.grewe@uni-tuebingen.de}
-\usepackage{amssymb}
-\setlength{\marginparwidth}{2cm}
-\usepackage[breaklinks=true,bookmarks=true,bookmarksopen=true,pdfpagemode=UseNone,pdfstartview=FitH,colorlinks=true,citecolor=blue]{hyperref}
-
-%%%%% text size %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage[left=20mm,right=20mm,top=25mm,bottom=25mm]{geometry}
-\pagestyle{headandfoot} \header{{\bfseries\large Exercise 4
-    }}{{\bfseries\large Boolean expressions \& logical indexing}}{{\bfseries\large 17. November, 2020}}
-\firstpagefooter{Dr. Jan Grewe}{Phone: 29 74588}{Email:
-  jan.grewe@uni-tuebingen.de} \runningfooter{}{\thepage}{}
-
-\setlength{\baselineskip}{15pt}
-\setlength{\parindent}{0.0cm}
-\setlength{\parskip}{0.3cm}
-\renewcommand{\baselinestretch}{1.15}
-
-\newcommand{\code}[1]{\texttt{#1}}
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \begin{document}
 
-\vspace*{-6.5ex}
+\input{../../exercisestitle}
-\begin{center}
-  \textbf{\Large Introduction to scientific computing}\\[1ex]
-  {\large Jan Grewe, Jan Benda}\\[-3ex]
-  Abteilung Neuroethologie \hfill --- \hfill Institut f\"ur Neurobiologie \hfill --- \hfill \includegraphics[width=0.28\textwidth]{UT_WBMW_Black_RGB} \\
-\end{center}
 
 The exercises are meant for self-monitoring and revision of the
 lecture. You should try to solve them on your own. Your solution

programming/exercises/control_flow.tex

@@ -1,40 +1,17 @@
 \documentclass[12pt,a4paper,pdftex]{exam}
 
-\usepackage[german]{babel}
+\newcommand{\exercisetopic}{Control flow}
-\usepackage{natbib}
+\newcommand{\exercisenum}{5}
-\usepackage{graphicx}
+\newcommand{\exercisedate}{24. November, 2020}
-\usepackage[small]{caption}
-\usepackage{sidecap}
-\usepackage{pslatex}
-\usepackage{amsmath}
-\usepackage{amssymb}
-\setlength{\marginparwidth}{2cm}
-\usepackage[breaklinks=true,bookmarks=true,bookmarksopen=true,pdfpagemode=UseNone,pdfstartview=FitH,colorlinks=true,citecolor=blue]{hyperref}
 
-%%%%% text size %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\input{../../exercisesheader}
-\usepackage[left=20mm,right=20mm,top=25mm,bottom=25mm]{geometry}
-\pagestyle{headandfoot}
-\header{{\bfseries\large Exercise 5}}{{\bfseries\large Control Flow}}{{\bfseries\large 24. November, 2020}}
-\firstpagefooter{Dr. Jan Grewe}{Phone: 29 74588}{Email:
-  jan.grewe@uni-tuebingen.de}
-\runningfooter{}{\thepage}{}
 
-\setlength{\baselineskip}{15pt}
+\firstpagefooter{Dr. Jan Grewe}{}{jan.grewe@uni-tuebingen.de}
-\setlength{\parindent}{0.0cm}
-\setlength{\parskip}{0.3cm}
-\renewcommand{\baselinestretch}{1.15}
-
-\newcommand{\code}[1]{\texttt{#1}}
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \begin{document}
-\vspace*{-6.5ex}
 
-\begin{center}
+\input{../../exercisestitle}
-  \textbf{\Large Introduction to scientific computing}\\[1ex]
-  {\large Jan Grewe, Jan Benda}\\[-3ex]
-  Neuroethologie \hfill --- \hfill Institute for Neurobiology \hfill --- \hfill \includegraphics[width=0.28\textwidth]{UT_WBMW_Black_RGB} \\
-\end{center}
 
 The exercises are meant for self-monitoring and revision of the
 lecture. You should try to solve them on your own. Your solution

programming/exercises/matrices.tex

@@ -1,40 +1,17 @@
 \documentclass[12pt,a4paper,pdftex]{exam}
 
-\usepackage[german]{babel}
+\newcommand{\exercisetopic}{Matrices}
-\usepackage{natbib}
+\newcommand{\exercisenum}{3}
-\usepackage{graphicx}
+\newcommand{\exercisedate}{22. October, 2019}
-\usepackage[small]{caption}
+
-\usepackage{sidecap}
+\input{../../exercisesheader}
-\usepackage{pslatex}
+
-\usepackage{amsmath}
+\firstpagefooter{Dr. Jan Grewe}{}{jan.grewe@uni-tuebingen.de}
-\usepackage{amssymb}
-\setlength{\marginparwidth}{2cm}
-\usepackage[breaklinks=true,bookmarks=true,bookmarksopen=true,pdfpagemode=UseNone,pdfstartview=FitH,colorlinks=true,citecolor=blue]{hyperref}
-
-%%%%% text size %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage[left=20mm,right=20mm,top=25mm,bottom=25mm]{geometry}
-\pagestyle{headandfoot}
-\header{{\bfseries\large Exercise 3}}{{\bfseries\large Matrices}}{{\bfseries\large 10. Oktober, 2020}}
-\firstpagefooter{Dr. Jan Grewe}{Phone: 29 74588}{Email:
-  jan.grewe@uni-tuebingen.de}
-\runningfooter{}{\thepage}{}
-
-\setlength{\baselineskip}{15pt}
-\setlength{\parindent}{0.0cm}
-\setlength{\parskip}{0.3cm}
-\renewcommand{\baselinestretch}{1.15}
-
-\newcommand{\code}[1]{\texttt{#1}}
-\renewcommand{\solutiontitle}{\noindent\textbf{Solutions:}\par\noindent}
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \begin{document}
-\vspace*{-6.5ex}
+
-\begin{center}
+\input{../../exercisestitle}
-  \textbf{\Large Introduction to Scientific Computing}\\[1ex]
-  {\large Jan Grewe, Jan Benda}\\[-3ex]
-  Neuroethology \hfill --- \hfill Institute for Neurobiology \hfill --- \hfill \includegraphics[width=0.28\textwidth]{UT_WBMW_Black_RGB} \\
-\end{center}
 
 The exercises are meant for self-monitoring and revision of the
 lecture. You should try to solve them on your own. Your solution

programming/exercises/scripts_functions.tex

@@ -1,62 +1,17 @@
 \documentclass[12pt,a4paper,pdftex]{exam}
-%\documentclass[answers,12pt,a4paper,pdftex]{exam}
+
-
+\newcommand{\exercisetopic}{Scripts and Functions}
-\usepackage[german]{babel}
+\newcommand{\exercisenum}{6}
-\usepackage{natbib}
+\newcommand{\exercisedate}{1. December, 2020}
-\usepackage{xcolor}
+
-\usepackage{graphicx}
+\input{../../exercisesheader}
-\usepackage[small]{caption}
+
-\usepackage{sidecap}
+\firstpagefooter{Dr. Jan Grewe}{}{jan.grewe@uni-tuebingen.de}
-\usepackage{pslatex}
-\usepackage{amsmath}
-\usepackage{amssymb}
-\setlength{\marginparwidth}{2cm}
-\usepackage[breaklinks=true,bookmarks=true,bookmarksopen=true,pdfpagemode=UseNone,pdfstartview=FitH,colorlinks=true,citecolor=blue]{hyperref}
-
-%%%%% text size %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage[left=20mm,right=20mm,top=25mm,bottom=25mm]{geometry}
-\pagestyle{headandfoot}
-\header{{\bfseries\large Exercise 6}}{{\bfseries\large Scripts and functions}}{{\bfseries\large 01. December, 2020}}
-\firstpagefooter{Dr. Jan Grewe}{Phone: 29 74 588}{Email:
-  jan.grewe@uni-tuebingen.de}
-\runningfooter{}{\thepage}{}
-
-\setlength{\baselineskip}{15pt}
-\setlength{\parindent}{0.0cm}
-\setlength{\parskip}{0.3cm}
-\renewcommand{\baselinestretch}{1.15}
-
-
-%%%%% listings %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage{listings}
-\lstset{
-  language=Matlab,
-  basicstyle=\ttfamily\footnotesize,
-  numbers=left,
-  numberstyle=\tiny,
-  title=\lstname,
-  showstringspaces=false,
-  commentstyle=\itshape\color{darkgray},
-  breaklines=true,
-  breakautoindent=true,
-  columns=flexible,
-  frame=single,
-  xleftmargin=1em,
-  xrightmargin=1em,
-  aboveskip=10pt
-}
-
-\newcommand{\code}[1]{\texttt{#1}}
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \begin{document}
 
-\vspace*{-6.5ex}
+\input{../../exercisestitle}
-\begin{center}
-  \textbf{\Large Introduction to Scientific Computing}\\[1ex]
-         {\large Jan Grewe, Jan Benda}\\[-3ex]
-         Neuroethology \hfill --- \hfill Institute for Neurobiology \hfill --- \hfill \includegraphics[width=0.28\textwidth]{UT_WBMW_Black_RGB} \\
-\end{center}
 
 The exercises are meant for self-monitoring and revision of the
 lecture. You should try to solve them on your own. In contrast

programming/exercises/structs_cells.tex

@@ -1,40 +1,17 @@
 \documentclass[12pt,a4paper,pdftex]{exam}
 
-\usepackage[german]{babel}
+\newcommand{\exercisetopic}{Structs and Cell Arrays}
-\usepackage{natbib}
+\newcommand{\exercisenum}{6}
-\usepackage{graphicx}
+\newcommand{\exercisedate}{16. October, 2015}
-\usepackage[small]{caption}
-\usepackage{sidecap}
-\usepackage{pslatex}
-\usepackage{amsmath}
-\usepackage{amssymb}
-\setlength{\marginparwidth}{2cm}
-\usepackage[breaklinks=true,bookmarks=true,bookmarksopen=true,pdfpagemode=UseNone,pdfstartview=FitH,colorlinks=true,citecolor=blue]{hyperref}
 
-%%%%% text size %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\input{../../exercisesheader}
-\usepackage[left=20mm,right=20mm,top=25mm,bottom=25mm]{geometry}
-\pagestyle{headandfoot}
-\header{{\bfseries\large \"Ubung 6}}{{\bfseries\large Strukturen und Cell Arrays}}{{\bfseries\large 16. Oktober, 2015}}
-\firstpagefooter{Dr. Jan Grewe}{Phone: 29 74588}{Email:
-  jan.grewe@uni-tuebingen.de}
-\runningfooter{}{\thepage}{}
 
-\setlength{\baselineskip}{15pt}
+\firstpagefooter{Dr. Jan Grewe}{}{jan.grewe@uni-tuebingen.de}
-\setlength{\parindent}{0.0cm}
-\setlength{\parskip}{0.3cm}
-\renewcommand{\baselinestretch}{1.15}
-
-\newcommand{\code}[1]{\texttt{#1}}
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \begin{document}
 
-\vspace*{-6.5ex}
+\input{../../exercisestitle}
-\begin{center}
-  \textbf{\Large Einf\"uhrung in die wissenschaftliche Datenverarbeitung}\\[1ex]
-  {\large Jan Grewe, Jan Benda}\\[-3ex]
-  Abteilung Neuroethologie \hfill --- \hfill Institut f\"ur Neurobiologie \hfill --- \hfill \includegraphics[width=0.28\textwidth]{UT_WBMW_Black_RGB} \\
-\end{center}
 
 Die folgenden Aufgaben dienen der Wiederholung, \"Ubung und
 Selbstkontrolle und sollten eigenst\"andig bearbeitet und gel\"ost

programming/exercises/variables_types.tex

@@ -1,41 +1,17 @@
 \documentclass[12pt,a4paper,pdftex]{exam}
 
-\usepackage[english]{babel}
+\newcommand{\exercisetopic}{Variables und Datatypes}
-\usepackage{natbib}
+\newcommand{\exercisenum}{1}
-\usepackage{graphicx}
+\newcommand{\exercisedate}{3. November, 2020}
-\usepackage[small]{caption}
+
-\usepackage{sidecap}
+\input{../../exercisesheader}
-\usepackage{pslatex}
+
-\usepackage{amsmath}
+\firstpagefooter{Dr. Jan Grewe}{}{jan.grewe@uni-tuebingen.de}
-\usepackage{amssymb}
+
-\usepackage{lipsum}
-\setlength{\marginparwidth}{2cm}
-\usepackage[breaklinks=true,bookmarks=true,bookmarksopen=true,pdfpagemode=UseNone,pdfstartview=FitH,colorlinks=true,citecolor=blue]{hyperref}
-
-%%%%% text size %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage[left=20mm,right=20mm,top=25mm,bottom=25mm]{geometry}
-\pagestyle{headandfoot}
-\header{{\bfseries\large Exercise 1}}{{\bfseries\large Variables und Datatypes}}{{\bfseries\large 03. November, 2020}}
-\firstpagefooter{Dr. Jan Grewe}{Phone: 29 74588}{Email:
-  jan.grewe@uni-tuebingen.de}
-\runningfooter{}{\thepage}{}
-
-\setlength{\baselineskip}{15pt}
-\setlength{\parindent}{0.0cm}
-\setlength{\parskip}{0.3cm}
-\renewcommand{\baselinestretch}{1.15}
-
-\newcommand{\code}[1]{\texttt{#1}}
-\renewcommand{\solutiontitle}{\noindent\textbf{Solutions:}\par\noindent}
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \begin{document}
 
-\vspace*{-6.5ex}
+\input{../../exercisestitle}
-\begin{center}
-  \textbf{\Large Introduction to Scientific Computing}\\[1ex]
-  {\large Jan Grewe, Jan Benda}\\[-3ex]
-  Neuroethology \hfill --- \hfill Institute for Neurobiology \hfill --- \hfill \includegraphics[width=0.28\textwidth]{UT_WBMW_Black_RGB} \\
-\end{center}
 
 The exercises are meant for self-monitoring and revision of the
 lecture. You should try to solve them on your own. Your solution

programming/exercises/vectors.tex

@@ -1,40 +1,17 @@
 \documentclass[12pt,a4paper,pdftex]{exam}
 
-\usepackage[german]{babel}
+\newcommand{\exercisetopic}{Vectors}
-\usepackage{natbib}
+\newcommand{\exercisenum}{2}
-\usepackage{graphicx}
+\newcommand{\exercisedate}{3. November, 2020}
-\usepackage[small]{caption}
-\usepackage{sidecap}
-\usepackage{pslatex}
-\usepackage{amsmath}
-\usepackage{amssymb}
-\setlength{\marginparwidth}{2cm}
-\usepackage[breaklinks=true,bookmarks=true,bookmarksopen=true,pdfpagemode=UseNone,pdfstartview=FitH,colorlinks=true,citecolor=blue]{hyperref}
 
-%%%%% text size %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\input{../../exercisesheader}
-\usepackage[left=20mm,right=20mm,top=25mm,bottom=25mm]{geometry}
-\pagestyle{headandfoot}
-\header{{\bfseries\large Exercise 2}}{{\bfseries\large Vectors}}{{\bfseries\large 03. November, 2020}}
-\firstpagefooter{Dr. Jan Grewe}{Phone: 29 74588}{Email:
-  jan.grewe@uni-tuebingen.de}
-\runningfooter{}{\thepage}{}
 
-\setlength{\baselineskip}{15pt}
+\firstpagefooter{Dr. Jan Grewe}{}{jan.grewe@uni-tuebingen.de}
-\setlength{\parindent}{0.0cm}
-\setlength{\parskip}{0.3cm}
-\renewcommand{\baselinestretch}{1.15}
-
-\newcommand{\code}[1]{\texttt{#1}}
-\renewcommand{\solutiontitle}{\noindent\textbf{Solutions:}\par\noindent}
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \begin{document}
-\vspace*{-6.5ex}
+
-\begin{center}
+\input{../../exercisestitle}
-  \textbf{\Large Introduction to Scientific Computing}\\[1ex]
-         {\large Jan Grewe, Jan Benda}\\[-3ex]
-         Neuroethology \hfill --- \hfill Institute for Neurobiology \hfill --- \hfill \includegraphics[width=0.28\textwidth]{UT_WBMW_Black_RGB} \\
-\end{center}
 
 The exercises are meant for self-monitoring and revision of the
 lecture. You should try to solve them on your own. Your solution

programming/exercises/vectors_matrices.tex

@@ -1,40 +1,17 @@
-\documentclass[12pt,a4paper,pdftex, answers]{exam}
+\documentclass[12pt,a4paper,pdftex]{exam}
 
-\usepackage[german]{babel}
+\newcommand{\exercisetopic}{Vectors}
-\usepackage{natbib}
+\newcommand{\exercisenum}{2}
-\usepackage{graphicx}
+\newcommand{\exercisedate}{3. November, 2020}
-\usepackage[small]{caption}
-\usepackage{sidecap}
-\usepackage{pslatex}
-\usepackage{amsmath}
-\usepackage{amssymb}
-\setlength{\marginparwidth}{2cm}
-\usepackage[breaklinks=true,bookmarks=true,bookmarksopen=true,pdfpagemode=UseNone,pdfstartview=FitH,colorlinks=true,citecolor=blue]{hyperref}
 
-%%%%% text size %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\input{../../exercisesheader}
-\usepackage[left=20mm,right=20mm,top=25mm,bottom=25mm]{geometry}
-\pagestyle{headandfoot}
-\header{{\bfseries\large Exercise 2}}{{\bfseries\large Vectors}}{{\bfseries\large 03. November, 2020}}
-\firstpagefooter{Dr. Jan Grewe}{Phone: 29 74588}{Email:
-  jan.grewe@uni-tuebingen.de}
-\runningfooter{}{\thepage}{}
 
-\setlength{\baselineskip}{15pt}
+\firstpagefooter{Dr. Jan Grewe}{}{jan.grewe@uni-tuebingen.de}
-\setlength{\parindent}{0.0cm}
-\setlength{\parskip}{0.3cm}
-\renewcommand{\baselinestretch}{1.15}
-
-\newcommand{\code}[1]{\texttt{#1}}
-\renewcommand{\solutiontitle}{\noindent\textbf{Solutions:}\par\noindent}
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \begin{document}
-\vspace*{-6.5ex}
+
-\begin{center}
+\input{../../exercisestitle}
-  \textbf{\Large Introduction to Scientific Computing}\\[1ex]
-         {\large Jan Grewe, Jan Benda}\\[-3ex]
-         Neuroethology \hfill --- \hfill Institute for Neurobiology \hfill --- \hfill \includegraphics[width=0.28\textwidth]{UT_WBMW_Black_RGB} \\
-\end{center}
 
 The exercises are meant for self-monitoring and revision of the lecture
 topic. You should try to solve them on your own. Your solution should

programming/lecture/programming.tex

@@ -60,7 +60,8 @@ variable.
 In \matlab{} variables can be created at any time on the command line
 or any place in a script or function. Listing~\ref{varListing1} shows
 three different ways of creating a variable:
-\begin{lstlisting}[label=varListing1, caption={Creating variables.}]
+
+\begin{pagelisting}[label=varListing1, caption={Creating variables.}]
 >> x = 38
 x =
     38
@@ -72,7 +73,7 @@ y =
 >> z = 'A'
 z =
     A
-\end{lstlisting}
+\end{pagelisting}
 
 Line 1 can be read like: ``Create a variable with the name \varcode{x}
 and assign the value 38''. The equality sign is the so called
@@ -93,8 +94,7 @@ information but it is not suited to be used in programs (see
 also the \code{who} function that returns a list of all defined
 variables, listing~\ref{varListing2}).
 
-\newpage
+\begin{pagelisting}[label=varListing2, caption={Requesting information about defined variables and their types.}]
-\begin{lstlisting}[label=varListing2, caption={Requesting information about defined variables and their types.}]
 >>class(x)
 ans =
       double
@@ -110,7 +110,7 @@ x  y  z
   x         1x1                 8  double              
   y         0x0                 0  double              
   z         1x1                 2  char                
-\end{lstlisting}
+\end{pagelisting}
 
 \begin{important}[Naming conventions]
   There are a few rules regarding variable names. \matlab{} is
@@ -120,7 +120,6 @@ x  y  z
   variable names.
 \end{important}
 
-\pagebreak[4]
 \subsection{Working with variables}
 We can certainly work, i.e. do calculations, with variables. \matlab{}
 knows all basic \entermde[Operator!arithmetic]{Operator!arithmetischer}{arithmetic operators}
@@ -131,7 +130,7 @@ such as \code[Operator!arithmetic!1add@+]{+},
 \code[Operator!arithmetic!5pow@\^{}]{\^{}}. Listing~\ref{varListing3}
 shows their use.
 
-\begin{lstlisting}[label=varListing3, caption={Working with variables.}]
+\begin{pagelisting}[label=varListing3, caption={Working with variables.}]
 >> x = 1;
 >> x + 10
 ans = 
@@ -149,13 +148,12 @@ ans =
 z =
     3
 
->> z = z * 5;
+>> z = z * 5
->> z 
 z = 
     15
 
 >> clear z      % deleting a variable
-\end{lstlisting}
+\end{pagelisting}
 
 Note: in lines 2 and 10 the variables have been used without changing
 their values. Whenever the value of a variable should change, the
@@ -272,8 +270,8 @@ step-sizes unequal to 1. Line 5 can be read like: ``Create a variable
 \varcode{b} and assign the values from 0 to 9 in increasing steps of
 1.''. Line 9 reads: ``Create a variable \varcode{c} and assign the
 values from 0 to 10 in steps of 2''.
-\pagebreak
+
-\begin{lstlisting}[label=generatevectorslisting, caption={Creating simple row-vectors.}]
+\begin{pagelisting}[label=generatevectorslisting, caption={Creating simple row-vectors.}]
 >> a = [0 1 2 3 4 5 6 7 8 9]  % Creating a row-vector
 a =
     0  1   2   3   4   5   6   7   8   9
@@ -285,7 +283,7 @@ b =
 >> c = (0:2:10)                   
 c = 
     0  2  4  6  8  10
-\end{lstlisting}
+\end{pagelisting}
 
 The length of a vector, that is the number of elements, can be
 requested using the \code{length()} or \code{numel()}
@@ -293,14 +291,14 @@ functions. \code{size()} provides the same information in a slightly,
 yet more powerful way (listing~\ref{vectorsizeslisting}). The above
 used vector \varcode{a} has the following size:
 
-\begin{lstlisting}[label=vectorsizeslisting, caption={Size of a vector.}]
+\begin{pagelisting}[label=vectorsizeslisting, caption={Size of a vector.}]
 >> length(a)
 ans =
       10
 >> size(a)
 ans =
       1  10
-\end{lstlisting}
+\end{pagelisting}
 
 The answer provided by the \code{size()} function demonstrates that
 vectors are nothing else but 2-dimensional matrices in which one
@@ -311,7 +309,7 @@ create a column-vector and how the \code[Operator!Matrix!']{'} ---
 operator is used to transpose the column-vector into a row-vector
 (lines 14 and following).
 
-\begin{lstlisting}[label=columnvectorlisting, caption={Column-vectors.}]
+\begin{pagelisting}[label=columnvectorlisting, caption={Column-vectors.}]
 >> b = [1; 2; 3; 4; 5; 6; 7; 8; 9; 10]  % Creating a column-vector
 b =
      1
@@ -334,7 +332,7 @@ b =
 >> size(b)
 ans =
       1  10
-\end{lstlisting}
+\end{pagelisting}
 
 
 \subsubsection{Accessing elements of a vector}
@@ -355,7 +353,7 @@ number of elements irrespective of the type of vector.
   Elements of a vector are accessed via their index. This process is
   called \entermde{Indizierung}{indexing}.
 
-  In \matlab{} the first element has the index one.
+  In \matlab{} the first element in a vector has the index one.
   
   The last element's index equals the length of the vector.
 \end{important}
@@ -366,8 +364,8 @@ individual values by providing a single index or use the
 \code[Operator!Matrix!:]{:} operator to access multiple values with a
 single command (see also the info box below \ref{important:colon_operator}).
 
-\begin{lstlisting}[label=vectorelementslisting, caption={Access to individual elements of a vector.}]
+\begin{pagelisting}[label=vectorelementslisting, caption={Access to individual elements of a vector.}]
->> a = (11:20)
+>> a = (11:20)   % generate a vector
 a = 
     11    12    13    14    15    16    17    18    19    20
 
@@ -377,14 +375,14 @@ ans = 11
 ans = 15
 >> a(end)        % the last element
 ans = 20
-\end{lstlisting}
+\end{pagelisting}
 
-\begin{lstlisting}[caption={Access to multiple elements.}, label=vectorrangelisting]
+\begin{pagelisting}[caption={Access to multiple elements.}, label=vectorrangelisting]
 >> a([1 3 5])   % 1., 3. and 5. element
 ans =
       11  13  15
 
->> a(2:4)       % all elements with the indices 2 to 4
+>> a(2:4)       % elements at indices 2 to 4
 ans =
       12  13  14 
 
@@ -395,7 +393,7 @@ ans =
 >> a(:)         % all elements as row-vector
 ans = 
       11  12  13  14  15  16  17  18  19  20
-\end{lstlisting}
+\end{pagelisting}
 
 \begin{exercise}{vectorsize.m}{vectorsize.out}
   Create a row-vector \varcode{a} with 5 elements. The return value of
@@ -425,7 +423,7 @@ how vectors and scalars can be combined with the operators \code[Operator!arithm
 \code[Operator!arithmetic!4div@/]{/}
 \code[Operator!arithmetic!5powe@.\^{}]{.\^}.
 
-\begin{lstlisting}[caption={Calculating with vectors and scalars.},label=vectorscalarlisting]
+\begin{pagelisting}[caption={Calculations with vectors and scalars.},label=vectorscalarlisting]
 >> a = (0:2:8)
 a =
       0   2   4   6   8
@@ -449,7 +447,7 @@ ans =
 >> a .^ 2           % exponentiation
 ans =
       0   4  16  36  64
-\end{lstlisting}
+\end{pagelisting}
 
 When doing calculations with scalars and vectors the same mathematical
 operation is done to each element of the vector. In case of, e.g. an
@@ -462,7 +460,7 @@ element-wise operations of two vectors, e.g. each element of vector
 layout (row- or column vectors). Addition and subtraction are always
 element-wise (listing~\ref{vectoradditionlisting}).
 
-\begin{lstlisting}[caption={Element-wise addition and subtraction of two vectors.},label=vectoradditionlisting]
+\begin{pagelisting}[caption={Element-wise addition and subtraction of two vectors.},label=vectoradditionlisting]
 >> a = [4   9  12];
 >> b = [4   3   2];
 >> a + b           % addition 
@@ -481,7 +479,7 @@ Matrix dimensions must agree.
 >> a + d           % both vectors must have the same layout!
 Error using +
 Matrix dimensions must agree.
-\end{lstlisting}
+\end{pagelisting}
 
 Element-wise multiplication, division, or raising a vector to a given power requires a
 different operator with a preceding '.'.  \matlab{} defines the
@@ -491,7 +489,7 @@ following operators for element-wise operations on vectors
 \code[Operator!arithmetic!5powe@.\^{}]{.\^{}}
 (listing~\ref{vectorelemmultiplicationlisting}).
 
-\begin{lstlisting}[caption={Element-wise multiplication, division and
+\begin{pagelisting}[caption={Element-wise multiplication, division and
     exponentiation of two vectors.},label=vectorelemmultiplicationlisting]
 >> a .* b           % element-wise multiplication
 ans = 
@@ -511,7 +509,7 @@ Matrix dimensions must agree.
 >> a .* d           % Both vectors must have the same layout!
 Error using .*
 Matrix dimensions must agree.
-\end{lstlisting}
+\end{pagelisting}
 
 The simple operators \code[Operator!arithmetic!3mul@*]{*},
 \code[Operator!arithmetic!4div@/]{/} and
@@ -521,7 +519,7 @@ matrix-operations known from linear algebra (Box~
 of a row-vectors $\vec a$ with a column-vector $\vec b$ the
 scalar-poduct (or dot-product) $\sum_i = a_i b_i$.
 
-\begin{lstlisting}[caption={Multiplication of vectors.},label=vectormultiplicationlisting]
+\begin{pagelisting}[caption={Multiplication of vectors.},label=vectormultiplicationlisting]
 >> a * b            % multiplication of two vectors
 Error using  * 
 Inner matrix dimensions must agree.
@@ -538,14 +536,13 @@ ans =
     16    12     8
     36    27    18
     48    36    24
-\end{lstlisting}
+\end{pagelisting}
-
-\pagebreak[4] 
 
 To remove elements from a vector an empty value
 (\code[Operator!Matrix!{[]}]{[]}) is assigned to the respective
 elements:
-\begin{lstlisting}[label=vectoreraselisting, caption={Deleting elements of a vector.}]
+
+\begin{pagelisting}[label=vectoreraselisting, caption={Deleting elements of a vector.}]
 >> a = (0:2:8);
 >> length(a) 
 ans = 5
@@ -558,7 +555,7 @@ a = 4  8
 
 >> length(a)
 ans = 2
-\end{lstlisting}
+\end{pagelisting}
 
 In addition to deleting of vector elements one also add new elements
 or concatenate two vectors. When performing a concatenation the two
@@ -720,7 +717,7 @@ remapping, but can be really helpful
     rows in each column and so on.}\label{matrixlinearindexingfig}
 \end{figure}
 
-\begin{lstlisting}[label=matrixLinearIndexing, caption={Lineares indexing in matrices.}]
+\begin{pagelisting}[label=matrixLinearIndexing, caption={Lineares indexing in matrices.}]
 >> x = randi(100, [3, 4, 5]); % 3-D matrix filled with random numbers 
 >> size(x)
 ans =
@@ -737,17 +734,17 @@ ans =
 >> min(x(:))  % or even simpler
 ans =
 4
-\end{lstlisting}
+\end{pagelisting}
 
 \matlab{} defines functions that convert subscript indices to linear indices and back (\code{sub2ind()} and \code{ind2sub()}).
 
-\begin{ibox}[tp]{\label{matrixmultiplication} The matrix--multiplication.}
+\begin{ibox}[t]{\label{matrixmultiplication} The matrix--multiplication.}
   The matrix--multiplication from linear algebra is \textbf{not} an
   element--wise multiplication of each element in a matrix \varcode{A}
   and the respective element of matrix \varcode{B}. It is something
   completely different. Confusing element--wise and
   matrix--multiplication is one of the most common mistakes in
-  \matlab{}. \linebreak 
+  \matlab{}.
 
   The matrix--multiplication of two 2-D matrices is only possible if
   the number of columns in the first matrix agrees with the number of
@@ -797,8 +794,7 @@ box~\ref{matrixmultiplication}). To do a matrix-multiplication the
 inner dimensions of the matrices must agree
 (box~\ref{matrixmultiplication}).
 
-\pagebreak[4]
+\begin{pagelisting}[label=matrixOperations, caption={Two kinds of multiplications of matrices.}]
-\begin{lstlisting}[label=matrixOperations, caption={Two kinds of multiplications of matrices.}]
 >> A = randi(5, [2, 3])   % 2-D matrix 
 A =
      1     5     3
@@ -824,7 +820,7 @@ ans =
     10    15    20
     24    23    35
     16    17    25
-\end{lstlisting}
+\end{pagelisting}
 
 \section{Boolean expressions}
 
@@ -856,7 +852,7 @@ synonymous for the logical values 1 and
 0. Listing~\ref{logicaldatatype} exemplifies the use of the logical
 data type.
 
-\begin{lstlisting}[caption={The logical data type. Please note that the actual \matlab{} output looks a little different.}, label=logicaldatatype]
+\begin{pagelisting}[caption={The logical data type. Please note that the actual \matlab{} output looks a little different.}, label=logicaldatatype]
 >> true
 ans = 1
 >> false
@@ -873,7 +869,7 @@ ans = 0
 ans = 1   1   1   1
 >> logical([1 2 3 4 0 0 10])
 ans = 1   1   1   1   0   0   1
-\end{lstlisting}
+\end{pagelisting}
 
 \varcode{true} and \varcode{false} are reserved keywords that evaluate
 to the logical values 1 and 0, respectively. If you want to create a
@@ -886,7 +882,7 @@ code of each character in ``test'' is non-zero value, thus, the result
 of casting it to logical is a vector of logicals. A similar thing
 happens upon casting a vector (or matrix) of numbers to logical. Each
 value is converted to logical and the result is true for all non-zero
-values (line 21).
+values (line 15).
 
 Knowing how to represent true and false values in \matlab{} using the
 logical data type allows us to take a step towards more complex
@@ -924,8 +920,8 @@ stored in variable \varcode{b}?''.
 
 The result of such questions is then given as a logical
 value. Listing~\ref{relationaloperationslisting} shows examples using relational operators.
-\pagebreak
+
-\begin{lstlisting}[caption={Relational Boolean expressions.}, label=relationaloperationslisting]
+\begin{pagelisting}[caption={Relational Boolean expressions.}, label=relationaloperationslisting]
 >> true == logical(1)
 ans = 1
 >> false ~= logical(1)
@@ -943,7 +939,7 @@ ans = 0  0  1  0  0
 ans = 0  1  0  0  1 
 >> [2  0  0  5  0] >= [1  0  3  2  0]
 ans = 1  1  0  1  1 
-\end{lstlisting}
+\end{pagelisting}
 
 Testing the relations between numbers and scalar variables is straight
 forward. When comparing vectors, the relational operator will be
@@ -1040,7 +1036,7 @@ for implementing such
 expressions. Listing~\ref{logicaloperatorlisting} shows a few
 examples and respective illustrations are shown in figure~\ref{logicaloperationsfig}.
 
-\begin{lstlisting}[caption={Boolean expressions.}, label=logicaloperatorlisting]
+\begin{pagelisting}[caption={Boolean expressions.}, label=logicaloperatorlisting]
 >> x = rand(1)   % create a single random number in the range [0, 1]
 x = 0.3452
 >> x > 0.25 & x < 0.75
@@ -1051,7 +1047,7 @@ x = 0.4920, 0.9106, 0.7218, 0.8749, 0.1574, 0.0201, 0.9107, 0.8357, 0.0357, 0.47
 ans = 1  0  1  0  0  0  0  0  0  1
 >> x < 0.25 | x > 0.75  
 ans = 0  1  0  1  1  1  1  1  1  0
-\end{lstlisting}
+\end{pagelisting}
 
 \begin{figure}[ht]
   \includegraphics[]{logical_operations}
@@ -1066,7 +1062,6 @@ ans = 0  1  0  1  1  1  1  1  1  0
     data.}\label{logicaloperationsfig}
 \end{figure}
 
-\pagebreak
 \begin{important}[Assignment and equality operators]
   The assignment operator \code[Operator!Assignment!=]{=} and the
   relational equality operator \code[Operator!relational!==]{==} are
@@ -1095,7 +1090,7 @@ elements of \varcode{x} where the Boolean expression \varcode{x < 0}
 evaluates to true and store the result in the variable
 \varcode{x\_smaller\_zero}''.
 
-\begin{lstlisting}[caption={Logical indexing.}, label=logicalindexing1]
+\begin{pagelisting}[caption={Logical indexing.}, label=logicalindexing1]
 >> x = randn(1, 6)   % a vector with 6 random numbers
 x =
    -1.4023   -1.4224    0.4882   -0.1774   -0.1961    1.4193
@@ -1112,7 +1107,7 @@ elements_smaller_zero =
 >> elements_smaller_zero = x(x < 0)
 elements_smaller_zero =
    -1.4023   -1.4224   -0.1774   -0.1961
-\end{lstlisting}
+\end{pagelisting}
 
 \begin{exercise}{logicalVector.m}{logicalVector.out}
   Create a vector \varcode{x} containing the values 0--10.
@@ -1121,8 +1116,7 @@ elements_smaller_zero =
   \item Display the content of \varcode{y} in the command window.
   \item  What is the data type of \varcode{y}?
   \item  Return only those elements \varcode{x} that are less than 5.
-  \end{enumerate}
+  \end{enumerate}\vspace{-1ex}
-  \pagebreak[4]
 \end{exercise}
 
 \begin{figure}[t]
@@ -1144,7 +1138,6 @@ segment of data of a certain time span (the stimulus was on,
 \begin{exercise}{logicalIndexingTime.m}{}
   Assume that measurements have been made for a certain time. Usually
   measured values and the time are stored in two vectors.
-  
   \begin{itemize}
   \item Create a vector that represents the recording time \varcode{t
     = 0:0.001:10;}.
@@ -1152,9 +1145,9 @@ segment of data of a certain time span (the stimulus was on,
     that has the same length as \varcode{t}. The values stored in
     \varcode{x} represent the measured data at the times in
     \varcode{t}.
-  \item Use logical indexing to select those values that have been
+  \item Use logical indexing to select values that have been
-    recorded in the time span from 5--6\,s.
+    recorded in the time span 5--6\,s.
-  \end{itemize}
+  \end{itemize}\vspace{-1ex}
 \end{exercise}
 
 \begin{ibox}[!ht]{\label{advancedtypesbox}Advanced data types}
@@ -1276,7 +1269,7 @@ As the name already suggests loops are used to execute the same parts
 of the code repeatedly. In one of the earlier exercises the factorial of
 five has been calculated as depicted in listing~\ref{facultylisting}.
 
-\begin{lstlisting}[caption={Calculation of the factorial of 5 in five steps}, label=facultylisting]
+\begin{pagelisting}[caption={Calculation of the factorial of 5 in five steps}, label=facultylisting]
 >> x = 1;
 >> x = x * 2;
 >> x = x * 3;  
@@ -1285,7 +1278,7 @@ five has been calculated as depicted in listing~\ref{facultylisting}.
 >> x
 x =
     120
-\end{lstlisting}
+\end{pagelisting}
 
 This kind of program solves the taks but it is rather repetitive. The only
 thing that changes is the increasing factor. The repetition of such
@@ -1325,7 +1318,7 @@ a certain purpose. The \varcode{for}-loop is closed with the keyword
 \code{end}. Listing~\ref{looplisting} shows a simple version of such a
 for-loop.
 
-\begin{lstlisting}[caption={Example of a \varcode{for}-loop.}, label=looplisting]
+\begin{pagelisting}[caption={Example of a \varcode{for}-loop.}, label=looplisting]
 >> for x = 1:3 % head
      disp(x)   % body
    end
@@ -1334,7 +1327,7 @@ for-loop.
      1
      2
      3
-\end{lstlisting}
+\end{pagelisting}
 
 
 \begin{exercise}{factorialLoop.m}{factorialLoop.out}
@@ -1354,11 +1347,11 @@ keyword \code{while} that is followed by a Boolean expression. If this
 can be evaluated to true, the code in the body is executed. The loop
 is closed with an \code{end}.
 
-\begin{lstlisting}[caption={Basic structure of a \varcode{while} loop.}, label=whileloop]
+\begin{pagelisting}[caption={Basic structure of a \varcode{while} loop.}, label=whileloop]
 while x == true   % head with a Boolean expression
    % execute this code if the expression yields true
 end
-\end{lstlisting}
+\end{pagelisting}
 
 \begin{exercise}{factorialWhileLoop.m}{}
   Implement the factorial of a number \varcode{n} using a \varcode{while}-loop.
@@ -1413,7 +1406,7 @@ expression to provide a default case. The last body of the
 \varcode{if} - \varcode{elseif} - \varcode{else} statement has to be
 finished with the \code{end} (listing~\ref{ifelselisting}).
   
-\begin{lstlisting}[label=ifelselisting, caption={Structure of an \varcode{if} statement.}]
+\begin{pagelisting}[label=ifelselisting, caption={Structure of an \varcode{if} statement.}]
 if x < y % head
    % body I, executed only if x < y
 elseif x > y
@@ -1421,7 +1414,7 @@ elseif x > y
 else
    % body III, executed only if the previous conditions did not match
 end  
-  \end{lstlisting}
+  \end{pagelisting}
 
 \begin{exercise}{ifelse.m}{}
   Draw a random number and check with an appropriate \varcode{if}
@@ -1449,7 +1442,7 @@ that were not explicitly stated above (listing~\ref{switchlisting}).
 As usual the \code{switch} statement needs to be closed with an
 \code{end}.
 
-\begin{lstlisting}[label=switchlisting, caption={Structure of a \varcode{switch} statement.}]
+\begin{pagelisting}[label=switchlisting, caption={Structure of a \varcode{switch} statement.}]
 mynumber = input('Enter a number:');
 switch mynumber
   case -1
@@ -1459,7 +1452,7 @@ switch mynumber
   otherwise
       disp('something else');
 end
-\end{lstlisting}
+\end{pagelisting}
 
 
 \subsubsection{Comparison \varcode{if}  and \varcode{switch} -- statements}
@@ -1483,7 +1476,7 @@ skip the execution of the body under certain circumstances, one can
 use the keywords \code{break} and \code{continue}
 (listings~\ref{continuelisting} and \ref{continuelisting}).
 
-\begin{lstlisting}[caption={Stop the execution of a loop using \varcode{break}.}, label=breaklisting]
+\begin{pagelisting}[caption={Stop the execution of a loop using \varcode{break}.}, label=breaklisting]
 >> x = 1;
    while true
      if (x > 3)
@@ -1496,9 +1489,9 @@ use the keywords \code{break} and \code{continue}
      1
      2
      3
-\end{lstlisting}
+\end{pagelisting}
 
-\begin{lstlisting}[caption={Skipping iterations using \varcode{continue}.}, label=continuelisting]
+\begin{pagelisting}[caption={Skipping iterations using \varcode{continue}.}, label=continuelisting]
 for x = 1:5 
   if(x > 2 & x < 5)
     continue;
@@ -1509,7 +1502,7 @@ end
      1
      2
      5
-\end{lstlisting}
+\end{pagelisting}
 
 \begin{exercise}{logicalIndexingBenchmark.m}{logicalIndexingBenchmark.out}
   Above we claimed that logical indexing is faster and much more
@@ -1581,11 +1574,11 @@ has one \entermde{Argument}{argument} $x$ that is transformed into the
 function's output value $y$. In \matlab{} the syntax of a function
 declaration is very similar (listing~\ref{functiondefinitionlisting}).
 
-\begin{lstlisting}[caption={Declaration of a function in \matlab{}}, label=functiondefinitionlisting]
+\begin{pagelisting}[caption={Declaration of a function in \matlab{}}, label=functiondefinitionlisting]
 function [y] = functionName(arg_1, arg_2)
 %         ^                  ^       ^
 % return value        argument_1, argument_2
-\end{lstlisting}
+\end{pagelisting}
 
 The keyword \code{function} is followed by the return value(s) (it can
 be a list \varcode{[]} of values), the function name and the
@@ -1618,7 +1611,7 @@ The following listing (\ref{badsinewavelisting}) shows a function that
 calculates and displays a bunch of sine waves with different amplitudes.
 
 
-\begin{lstlisting}[caption={Bad example of a function that displays a series of sine waves.},label=badsinewavelisting]
+\begin{pagelisting}[caption={Bad example of a function that displays a series of sine waves.},label=badsinewavelisting]
 function myFirstFunction() % function head
    t = (0:0.01:2); 
    frequency = 1.0;
@@ -1629,7 +1622,7 @@ function myFirstFunction() % function head
        hold on;
    end 
 end
-\end{lstlisting}
+\end{pagelisting}
 
 \varcode{myFirstFunction} (listing~\ref{badsinewavelisting}) is a
 prime-example of a bad function. There are several issues with it's
@@ -1690,7 +1683,7 @@ define (i) how to name the function, (ii) which information it needs
 Having defined this we can start coding
 (listing~\ref{sinefunctionlisting}).
 
-\begin{lstlisting}[caption={Function that calculates a sine wave.}, label=sinefunctionlisting]
+\begin{pagelisting}[caption={Function that calculates a sine wave.}, label=sinefunctionlisting]
 function [time, sine] = sinewave(frequency, amplitude, t_max, t_step)
 % Calculate a sinewave of a given frequency, amplitude, 
 % duration and temporal resolution. 
@@ -1708,7 +1701,7 @@ function [time, sine] = sinewave(frequency, amplitude, t_max, t_step)
    time = (0:t_step:t_max);
    sine = sin(frequency .* time .* 2 .* pi) .* amplitude;
 end
-\end{lstlisting}
+\end{pagelisting}
   
 
 \paragraph{II. Plotting a single sine wave}
@@ -1735,7 +1728,7 @@ specification of the function:
 With this specification we can start to implement the function
 (listing~\ref{sineplotfunctionlisting}).
 
-\begin{lstlisting}[caption={Function for the graphical display of data.}, label=sineplotfunctionlisting]
+\begin{pagelisting}[caption={Function for the graphical display of data.}, label=sineplotfunctionlisting]
 function plotFunction(x_data, y_data, name)
 % Plots x-data against y-data and sets the display name.
 %
@@ -1747,7 +1740,7 @@ function plotFunction(x_data, y_data, name)
 %   name  : the displayname
   plot(x_data, y_data, 'displayname', name) 
 end
-\end{lstlisting}
+\end{pagelisting}
 
 \begin{ibox}[!ht]{\label{box:sprintf}\enterm[Format strings]{Formatted strings} with \mcode{sprintf}.}
     We can use \varcode{disp} to dispaly the content of a variable on the command line. But how can we embed variable contents nicely into a text and store it in a variable or show it on the command line? \varcode{sprintf} is the standard function to achieve this. We introduce only a fraction of its real might here. For a more complete description visit the \matlab{} help or e.g. \url{https://en.wikipedia.org/wiki/Printf_format_string}. 
@@ -1820,11 +1813,11 @@ for i = 1:length(amplitudes)
 end
 hold off
 legend('show')
-\end{lstlisting}
+\end{pagelisting}
 
+\pagebreak[4]
 \begin{exercise}{plotMultipleSinewaves.m}{}
   Extend the program to plot also a range of frequencies.
-  \pagebreak[4]
 \end{exercise}
 
 

regression/code/gradientDescent.m

@@ -1,32 +1,42 @@
-% x, y from exercise 8.3
+function [p, ps, mses] = gradientDescent(x, y, func, p0, epsilon, threshold)
+% Gradient descent for fitting a function to data pairs.
+%
+% Arguments: x, vector of the x-data values.
+%            y, vector of the corresponding y-data values.
+%            func, function handle func(x, p)
+%            p0, vector with initial parameter values
+%            epsilon: factor multiplying the gradient.
+%            threshold: minimum value for gradient
+%
+% Returns:   p, vector with the final parameter values.
+%            ps: 2D-vector with all the parameter vectors traversed.
+%            mses: vector with the corresponding mean squared errors
 
-% some arbitrary values for the slope and the intercept to start with:
+  p = p0;
-position = [-2.0, 10.0];  
+  gradient = ones(1, length(p0)) * 1000.0;
+  ps = [];
+  mses = [];
+  while norm(gradient) > threshold
+      ps = [ps, p(:)];
+      mses = [mses, meanSquaredError(x, y, func, p)];
+      gradient = meanSquaredGradient(x, y, func, p);
+      p = p - epsilon * gradient;
+  end
+end
 
-% gradient descent:
+function mse = meanSquaredError(x, y, func, p)
-gradient = [];
+    mse = mean((y - func(x, p)).^2);
-errors = [];
+end
-count = 1;
+
-eps = 0.0001;
+function gradmse = meanSquaredGradient(x, y, func, p)
-while isempty(gradient) || norm(gradient) > 0.1
+  gradmse = zeros(size(p, 1), size(p, 2));
-    gradient = meanSquaredGradient(x, y, position);
+  h = 1e-7;               % stepsize for derivatives
-    errors(count) = meanSquaredError(x, y, position);
+  mse = meanSquaredError(x, y, func, p);
-    position = position - eps .* gradient; 
+  for i = 1:length(p)     % for each coordinate ...
-    count = count + 1;
+    pi = p;
+    pi(i) = pi(i) + h;    %   displace i-th parameter
+    msepi = meanSquaredError(x, y, func, pi);
+    gradmse(i) = (msepi - mse)/h;
+  end
 end
 
-figure()
-subplot(2,1,1)
-hold on
-scatter(x, y, 'displayname', 'data')
-xx = min(x):0.01:max(x);
-yy = position(1).*xx + position(2);
-plot(xx, yy, 'displayname', 'fit')
-xlabel('Input')
-ylabel('Output')
-grid on
-legend show
-subplot(2,1,2)
-plot(errors)
-xlabel('optimization steps')
-ylabel('error')

regression/code/gradientDescentCubic.m

@@ -0,0 +1,25 @@
+function [c, cs, mses] = gradientDescentCubic(x, y, c0, epsilon, threshold)
+% Gradient descent for fitting a cubic relation.
+%
+% Arguments: x, vector of the x-data values.
+%            y, vector of the corresponding y-data values.
+%            c0, initial value for the parameter c. 
+%            epsilon: factor multiplying the gradient.
+%            threshold: minimum value for gradient
+%
+% Returns:   c, the final value of the c-parameter.
+%            cs: vector with all the c-values traversed.
+%            mses: vector with the corresponding mean squared errors
+  c = c0;
+  gradient = 1000.0;
+  cs = [];
+  mses = [];
+  count = 1;
+  while abs(gradient) > threshold
+      cs(count) = c;
+      mses(count) = meanSquaredErrorCubic(x, y, c);
+      gradient = meanSquaredGradientCubic(x, y, c);
+      c = c - epsilon * gradient; 
+      count = count + 1;
+  end
+end

regression/code/meanSquaredError.m

@@ -1,12 +0,0 @@
-function mse = meanSquaredError(x, y, parameter)
-% Mean squared error between a straight line and data pairs.
-%
-% Arguments: x, vector of the input values
-%            y, vector of the corresponding measured output values
-%            parameter, vector containing slope and intercept 
-%                       as the 1st and 2nd element, respectively. 
-%
-% Returns:   mse, the mean-squared-error.
-
-  mse = mean((y - x * parameter(1) - parameter(2)).^2);
-end

regression/code/meanSquaredErrorCubic.m

@@ -0,0 +1,11 @@
+function mse = meanSquaredErrorCubic(x, y, c)
+% Mean squared error between data pairs and a cubic relation.
+%
+% Arguments: x, vector of the x-data values
+%            y, vector of the corresponding y-data values
+%            c, the factor for the cubic relation. 
+%
+% Returns:   mse, the mean-squared-error.
+
+  mse = mean((y - c*x.^3).^2);
+end

regression/code/meanSquaredErrorLine.m

@@ -1 +0,0 @@
-mse = mean((y - y_est).^2);

regression/code/meanSquaredGradientCubic.m

@@ -0,0 +1,14 @@
+function dmsedc = meanSquaredGradientCubic(x, y, c)
+% The gradient of the mean squared error for a cubic relation.
+%
+% Arguments: x, vector of the x-data values
+%            y, vector of the corresponding y-data values
+%            c, the factor for the cubic relation. 
+%
+% Returns: the derivative of the mean squared error at c.
+
+  h = 1e-7;   % stepsize for derivatives
+  mse = meanSquaredErrorCubic(x, y, c);
+  mseh = meanSquaredErrorCubic(x, y, c+h);
+  dmsedc = (mseh - mse)/h;
+end

regression/code/meansquarederrorline.m

@@ -0,0 +1,15 @@
+n = 40;
+xmin = 2.2;
+xmax = 3.9;
+c = 6.0;
+noise = 50.0;
+
+% generate data:
+x = rand(n, 1) * (xmax-xmin) + xmin;
+yest = c * x.^3;
+y = yest + noise*randn(n, 1);
+
+% compute mean squared error:
+mse = mean((y - yest).^2);
+
+fprintf('the mean squared error is %.0f kg^2\n', mse)

regression/code/plotcubiccosts.m

@@ -0,0 +1,9 @@
+cs = 2.0:0.1:8.0;
+mses = zeros(length(cs));
+for i = 1:length(cs)
+    mses(i) = meanSquaredErrorCubic(x, y, cs(i));
+end
+
+plot(cs, mses)
+xlabel('c')
+ylabel('mean squared error')

regression/code/plotcubicgradient.m

@@ -0,0 +1,11 @@
+meansquarederrorline;               % generate data
+
+cs = 2.0:0.1:8.0;
+mseg = zeros(length(cs));
+for i = 1:length(cs)
+    mseg(i) = meanSquaredGradientCubic(x, y, cs(i));
+end
+
+plot(cs, mseg)
+xlabel('c')
+ylabel('gradient')

regression/code/plotgradientdescentcubic.m

@@ -0,0 +1,28 @@
+meansquarederrorline;               % generate data
+
+c0 = 2.0;
+eps = 0.00001;
+thresh = 1.0;
+[cest, cs, mses] = gradientDescentCubic(x, y, c0, eps, thresh);
+
+subplot(2, 2, 1);                   % top left panel
+hold on;
+plot(cs, '-o');
+plot([1, length(cs)], [c, c], 'k'); % line indicating true c value
+hold off;
+xlabel('Iteration');
+ylabel('C');
+subplot(2, 2, 3);                   % bottom left panel
+plot(mses, '-o');
+xlabel('Iteration steps');
+ylabel('MSE');
+subplot(1, 2, 2);                   % right panel
+hold on;
+% generate x-values for plottig the fit:
+xx = min(x):0.01:max(x);
+yy = cest * xx.^3;
+plot(xx, yy);
+plot(x, y, 'o');                    % plot original data
+xlabel('Size [m]');
+ylabel('Weight [kg]');
+legend('fit', 'data', 'location', 'northwest');

regression/code/plotgradientdescentpower.m

@@ -0,0 +1,30 @@
+meansquarederrorline;               % generate data
+
+p0 = [2.0, 1.0];
+eps = 0.00001;
+thresh = 1.0;
+[pest, ps, mses] = gradientDescent(x, y, @powerLaw, p0, eps, thresh);
+pest
+
+subplot(2, 2, 1);                   % top left panel
+hold on;
+plot(ps(1,:), ps(2,:), '.');
+plot(ps(1,end), ps(2,end), 'og');
+plot(c, 3.0, 'or'); % dot indicating true parameter values
+hold off;
+xlabel('Iteration');
+ylabel('C');
+subplot(2, 2, 3);                   % bottom left panel
+plot(mses, '-o');
+xlabel('Iteration steps');
+ylabel('MSE');
+subplot(1, 2, 2);                   % right panel
+hold on;
+% generate x-values for plottig the fit:
+xx = min(x):0.01:max(x);
+yy = powerLaw(xx, pest);
+plot(xx, yy);
+plot(x, y, 'o');                    % plot original data
+xlabel('Size [m]');
+ylabel('Weight [kg]');
+legend('fit', 'data', 'location', 'northwest');

regression/exercises/Makefile

@@ -1,34 +1,4 @@
-TEXFILES=$(wildcard exercises??.tex)
+TEXFILES=$(wildcard gradientdescent-?.tex)
-EXERCISES=$(TEXFILES:.tex=.pdf)
-SOLUTIONS=$(EXERCISES:exercises%=solutions%)
 
-.PHONY: pdf exercises solutions watch watchexercises watchsolutions clean
+include ../../exercises.mk
 
-pdf : $(SOLUTIONS) $(EXERCISES)
-
-exercises : $(EXERCISES)
-
-solutions : $(SOLUTIONS)
-
-$(SOLUTIONS) : solutions%.pdf : exercises%.tex instructions.tex
-	{ echo "\\documentclass[answers,12pt,a4paper,pdftex]{exam}"; sed -e '1d' $<; } > $(patsubst %.pdf,%.tex,$@)
-	pdflatex -interaction=scrollmode $(patsubst %.pdf,%.tex,$@) | tee /dev/stderr | fgrep -q "Rerun to get cross-references right" && pdflatex -interaction=scrollmode $(patsubst %.pdf,%.tex,$@) || true
-	rm $(patsubst %.pdf,%,$@).[!p]*
-
-$(EXERCISES) : %.pdf : %.tex instructions.tex
-	pdflatex -interaction=scrollmode $< | tee /dev/stderr | fgrep -q "Rerun to get cross-references right" && pdflatex -interaction=scrollmode $< || true
-
-watch :
-	while true; do ! make -q pdf && make pdf; sleep 0.5; done
-
-watchexercises :
-	while true; do ! make -q exercises && make exercises; sleep 0.5; done
-
-watchsolutions :
-	while true; do ! make -q solutions && make solutions; sleep 0.5; done
-
-clean :
-	rm -f *~ *.aux *.log *.out
-
-cleanup : clean
-	rm -f $(SOLUTIONS) $(EXERCISES)

regression/exercises/gradientdescent-1.tex

@@ -1,60 +1,17 @@
 \documentclass[12pt,a4paper,pdftex]{exam}
 
-\usepackage[german]{babel}
+\newcommand{\exercisetopic}{Resampling}
-\usepackage{natbib}
+\newcommand{\exercisenum}{9}
-\usepackage{xcolor}
+\newcommand{\exercisedate}{December 22th, 2020}
-\usepackage{graphicx}
+
-\usepackage[small]{caption}
+\input{../../exercisesheader}
-\usepackage{sidecap}
+
-\usepackage{pslatex}
+\firstpagefooter{Prof. Dr. Jan Benda}{}{jan.benda@uni-tuebingen.de}
-\usepackage{amsmath}
-\usepackage{amssymb}
-\setlength{\marginparwidth}{2cm}
-\usepackage[breaklinks=true,bookmarks=true,bookmarksopen=true,pdfpagemode=UseNone,pdfstartview=FitH,colorlinks=true,citecolor=blue]{hyperref}
-
-%%%%% text size %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage[left=20mm,right=20mm,top=25mm,bottom=25mm]{geometry}
-\pagestyle{headandfoot}
-\ifprintanswers
-\newcommand{\stitle}{: Solutions}
-\else
-\newcommand{\stitle}{}
-\fi
-\header{{\bfseries\large Exercise 10\stitle}}{{\bfseries\large Gradient descent}}{{\bfseries\large December 16th, 2019}}
-\firstpagefooter{Dr. Jan Grewe}{Phone: 29 74588}{Email:
-  jan.grewe@uni-tuebingen.de}
-\runningfooter{}{\thepage}{}
-
-\setlength{\baselineskip}{15pt}
-\setlength{\parindent}{0.0cm}
-\setlength{\parskip}{0.3cm}
-\renewcommand{\baselinestretch}{1.15}
-
-\newcommand{\code}[1]{\texttt{#1}}
-\renewcommand{\solutiontitle}{\noindent\textbf{Solution:}\par\noindent}
-%%%%% listings %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage{listings}
-\lstset{
-  language=Matlab,
-  basicstyle=\ttfamily\footnotesize,
-  numbers=left,
-  numberstyle=\tiny,
-  title=\lstname,
-  showstringspaces=false,
-  commentstyle=\itshape\color{darkgray},
-  breaklines=true,
-  breakautoindent=true,
-  columns=flexible,
-  frame=single,
-  xleftmargin=1em,
-  xrightmargin=1em,
-  aboveskip=10pt
-}
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \begin{document}
 
-\input{instructions}
+\input{../../exercisestitle}
 
 \begin{questions}
 
@@ -96,7 +53,7 @@
     the parameter values at the minimum of the cost function and a vector
     with the value of the cost function at each step of the algorithm.
     \begin{solution}
-      \lstinputlisting{../code/descent.m}
+      \lstinputlisting{descent.m}
     \end{solution}
 
     \part Plot the data and the straight line with the parameter
@@ -105,7 +62,7 @@
     \part Plot the development of the costs as a function of the
     iteration step.
     \begin{solution}
-      \lstinputlisting{../code/descentfit.m}
+      \lstinputlisting{descentfit.m}
     \end{solution}
 
     \part For checking the gradient descend method from (a) compare
@@ -116,7 +73,7 @@
     minimum gradient. What are good values such that the gradient
     descent gets closest to the true minimum of the cost function?
     \begin{solution}
-      \lstinputlisting{../code/checkdescent.m}
+      \lstinputlisting{checkdescent.m}
     \end{solution}
 
     \part Use the functions \code{polyfit()} and \code{lsqcurvefit()}
@@ -124,7 +81,7 @@
     line that fits the data. Compare the resulting fit parameters of
     those functions with the ones of your gradient descent algorithm.
     \begin{solution}
-      \lstinputlisting{../code/linefit.m}
+      \lstinputlisting{linefit.m}
     \end{solution}
 
   \end{parts}

regression/exercises/instructions.tex

@@ -1,6 +0,0 @@
-\vspace*{-7.8ex}
-\begin{center}
-\textbf{\Large Introduction to Scientific Computing}\\[2.3ex]
-{\large Jan Grewe, Jan Benda}\\[-3ex]
-Neuroethology Lab \hfill --- \hfill Institute for Neurobiology \hfill --- \hfill \includegraphics[width=0.28\textwidth]{UT_WBMW_Black_RGB} \\
-\end{center}

regression/lecture/cubiccost.py

@@ -0,0 +1,82 @@
+import numpy as np
+import matplotlib.pyplot as plt
+import matplotlib.ticker as mt
+from plotstyle import *
+
+def create_data():
+    # wikipedia:
+    # Generally, males vary in total length from 250 to 390 cm and
+    # weigh between 90 and 306 kg
+    c = 6
+    x = np.arange(2.2, 3.9, 0.05)
+    y = c * x**3.0
+    rng = np.random.RandomState(32281)
+    noise = rng.randn(len(x))*50
+    y += noise
+    return x, y, c
+
+    
+def plot_mse(ax, x, y, c):
+    ccs = np.linspace(0.5, 10.0, 200)
+    mses = np.zeros(len(ccs))
+    for i, cc in enumerate(ccs):
+        mses[i] = np.mean((y-(cc*x**3.0))**2.0)
+    imin = np.argmin(mses)
+        
+    ax.plot(ccs, mses, **lsAm)
+    ax.plot(c, 500.0, **psB)
+    ax.plot(ccs[imin], mses[imin], **psC)
+    ax.annotate('Minimum of\ncost\nfunction',
+                xy=(ccs[imin], mses[imin]*1.2), xycoords='data',
+                xytext=(4, 7000), textcoords='data', ha='left',
+                arrowprops=dict(arrowstyle="->", relpos=(0.2,0.0),
+                connectionstyle="angle3,angleA=10,angleB=90") )
+    ax.text(2.2, 500, 'True\nparameter\nvalue')
+    ax.annotate('', xy=(c-0.2, 500), xycoords='data',
+                xytext=(4.1, 700), textcoords='data', ha='left',
+                arrowprops=dict(arrowstyle="->", relpos=(1.0,0.0),
+                connectionstyle="angle3,angleA=-10,angleB=0") )
+    ax.set_xlabel('c')
+    ax.set_ylabel('Mean squared error')
+    ax.set_xlim(2, 8.2)
+    ax.set_ylim(0, 10000)
+    ax.set_xticks(np.arange(2.0, 8.1, 2.0))
+    ax.set_yticks(np.arange(0, 10001, 5000))
+
+    
+def plot_mse_min(ax, x, y, c):
+    ccs = np.arange(0.5, 10.0, 0.05)
+    mses = np.zeros(len(ccs))
+    for i, cc in enumerate(ccs):
+        mses[i] = np.mean((y-(cc*x**3.0))**2.0)
+    imin = np.argmin(mses)
+    di = 25
+    i0 = 16
+    dimin = np.argmin(mses[i0::di])*di + i0
+        
+    ax.plot(c, 500.0, **psB)
+    ax.plot(ccs, mses, **lsAm)
+    ax.plot(ccs[i0::di], mses[i0::di], **psAm)
+    ax.plot(ccs[dimin], mses[dimin], **psD)
+    #ax.plot(ccs[imin], mses[imin], **psCm)
+    ax.annotate('Estimated\nminimum of\ncost\nfunction',
+                xy=(ccs[dimin], mses[dimin]*1.2), xycoords='data',
+                xytext=(4, 6700), textcoords='data', ha='left',
+                arrowprops=dict(arrowstyle="->", relpos=(0.8,0.0),
+                connectionstyle="angle3,angleA=0,angleB=85") )
+    ax.set_xlabel('c')
+    ax.set_xlim(2, 8.2)
+    ax.set_ylim(0, 10000)
+    ax.set_xticks(np.arange(2.0, 8.1, 2.0))
+    ax.set_yticks(np.arange(0, 10001, 5000))
+    ax.yaxis.set_major_formatter(mt.NullFormatter())
+
+
+if __name__ == "__main__":
+    x, y, c = create_data()
+    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=cm_size(figure_width, 1.1*figure_height))
+    fig.subplots_adjust(**adjust_fs(left=8.0, right=1.2))
+    plot_mse(ax1, x, y, c)
+    plot_mse_min(ax2, x, y, c)
+    fig.savefig("cubiccost.pdf")
+    plt.close()

regression/lecture/cubicerrors.py

@@ -15,28 +15,13 @@ def create_data():
     return x, y, c
 
     
-def plot_data(ax, x, y, c):
-    ax.plot(x, y, zorder=10, **psAm)
-    xx = np.linspace(2.1, 3.9, 100)
-    ax.plot(xx, c*xx**3.0, zorder=5, **lsBm)
-    for cc in [0.25*c, 0.5*c, 2.0*c, 4.0*c]:
-        ax.plot(xx, cc*xx**3.0, zorder=5, **lsDm)
-    ax.set_xlabel('Size x', 'm')
-    ax.set_ylabel('Weight y', 'kg')
-    ax.set_xlim(2, 4)
-    ax.set_ylim(0, 400)
-    ax.set_xticks(np.arange(2.0, 4.1, 0.5))
-    ax.set_yticks(np.arange(0, 401, 100))
-    
-    
 def plot_data_errors(ax, x, y, c):
     ax.set_xlabel('Size x', 'm')
-    #ax.set_ylabel('Weight y', 'kg')
+    ax.set_ylabel('Weight y', 'kg')
     ax.set_xlim(2, 4)
     ax.set_ylim(0, 400)
     ax.set_xticks(np.arange(2.0, 4.1, 0.5))
     ax.set_yticks(np.arange(0, 401, 100))
-    ax.set_yticklabels([])
     ax.annotate('Error',
                 xy=(x[28]+0.05, y[28]+60), xycoords='data',
                 xytext=(3.4, 70), textcoords='data', ha='left',
@@ -52,31 +37,30 @@ def plot_data_errors(ax, x, y, c):
         yy = [c*x[i]**3.0, y[i]]
         ax.plot(xx, yy, zorder=5, **lsDm)
 
+        
 def plot_error_hist(ax, x, y, c):
     ax.set_xlabel('Squared error')
     ax.set_ylabel('Frequency')
-    bins = np.arange(0.0, 1250.0, 100)
+    bins = np.arange(0.0, 11000.0, 750)
     ax.set_xlim(bins[0], bins[-1])
-    #ax.set_ylim(0, 35)
+    ax.set_ylim(0, 15)
-    ax.set_xticks(np.arange(bins[0], bins[-1], 200))
+    ax.set_xticks(np.arange(bins[0], bins[-1], 5000))
-    #ax.set_yticks(np.arange(0, 36, 10))
+    ax.set_yticks(np.arange(0, 16, 5))
     errors = (y-(c*x**3.0))**2.0
     mls = np.mean(errors)
     ax.annotate('Mean\nsquared\nerror',
                 xy=(mls, 0.5), xycoords='data',
-                xytext=(800, 3), textcoords='data', ha='left',
+                xytext=(4500, 6), textcoords='data', ha='left',
                 arrowprops=dict(arrowstyle="->", relpos=(0.0,0.2),
                 connectionstyle="angle3,angleA=10,angleB=90") )
     ax.hist(errors, bins, **fsC)
 
 
-
 if __name__ == "__main__":
     x, y, c = create_data()
-    fig, (ax1, ax2) = plt.subplots(1, 2)
+    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=cm_size(figure_width, 0.9*figure_height))
-    fig.subplots_adjust(wspace=0.2, **adjust_fs(left=6.0, right=1.2))
+    fig.subplots_adjust(wspace=0.5, **adjust_fs(left=6.0, right=1.2))
-    plot_data(ax1, x, y, c)
+    plot_data_errors(ax1, x, y, c)
-    plot_data_errors(ax2, x, y, c)
+    plot_error_hist(ax2, x, y, c)
-    #plot_error_hist(ax2, x, y, c)
     fig.savefig("cubicerrors.pdf")
     plt.close()

regression/lecture/cubicfunc.py

@@ -2,7 +2,7 @@ import matplotlib.pyplot as plt
 import numpy as np
 from plotstyle import *
 
-if __name__ == "__main__":
+def create_data():
     # wikipedia:
     # Generally, males vary in total length from 250 to 390 cm and
     # weigh between 90 and 306 kg
@@ -12,10 +12,20 @@ if __name__ == "__main__":
     rng = np.random.RandomState(32281)
     noise = rng.randn(len(x))*50
     y += noise
+    return x, y, c
 
-    fig, ax = plt.subplots(figsize=cm_size(figure_width, 1.4*figure_height))
-    fig.subplots_adjust(**adjust_fs(left=6.0, right=1.2))
     
+def plot_data(ax, x, y):
+    ax.plot(x, y, **psA)
+    ax.set_xlabel('Size x', 'm')
+    ax.set_ylabel('Weight y', 'kg')
+    ax.set_xlim(2, 4)
+    ax.set_ylim(0, 400)
+    ax.set_xticks(np.arange(2.0, 4.1, 0.5))
+    ax.set_yticks(np.arange(0, 401, 100))
+
+        
+def plot_data_fac(ax, x, y, c):
     ax.plot(x, y, zorder=10, **psA)
     xx = np.linspace(2.1, 3.9, 100)
     ax.plot(xx, c*xx**3.0, zorder=5, **lsB)
@@ -28,5 +38,13 @@ if __name__ == "__main__":
     ax.set_xticks(np.arange(2.0, 4.1, 0.5))
     ax.set_yticks(np.arange(0, 401, 100))
 
+    
+if __name__ == "__main__":
+    x, y, c = create_data()
+    print('n=%d' % len(x))
+    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=cm_size(figure_width, 0.9*figure_height))
+    fig.subplots_adjust(wspace=0.5, **adjust_fs(fig, left=6.0, right=1.2))
+    plot_data(ax1, x, y)
+    plot_data_fac(ax2, x, y, c)
     fig.savefig("cubicfunc.pdf")
     plt.close()

regression/lecture/cubicmse.py

@@ -14,6 +14,7 @@ def create_data():
     y += noise
     return x, y, c
 
+
 def gradient_descent(x, y):
     n = 20
     dc = 0.01
@@ -30,6 +31,7 @@ def gradient_descent(x, y):
         cc -= eps*dmdc
     return cs, mses
 
+    
 def plot_mse(ax, x, y, c, cs):
     ms = np.zeros(len(cs))
     for i, cc in enumerate(cs):
@@ -55,6 +57,7 @@ def plot_mse(ax, x, y, c, cs):
     ax.set_xticks(np.arange(0.0, 10.1, 2.0))
     ax.set_yticks(np.arange(0, 30001, 10000))
 
+        
 def plot_descent(ax, cs, mses):
     ax.plot(np.arange(len(mses))+1, mses, **lpsBm)
     ax.set_xlabel('Iteration')
@@ -69,8 +72,8 @@ def plot_descent(ax, cs, mses):
 if __name__ == "__main__":
     x, y, c = create_data()
     cs, mses = gradient_descent(x, y)
-    fig, (ax1, ax2) = plt.subplots(1, 2)
+    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=cm_size(figure_width, 1.1*figure_height))
-    fig.subplots_adjust(wspace=0.2, **adjust_fs(left=8.0, right=0.5))
+    fig.subplots_adjust(**adjust_fs(left=8.0, right=1.2))
     plot_mse(ax1, x, y, c, cs)
     plot_descent(ax2, cs, mses)
     fig.savefig("cubicmse.pdf")

regression/lecture/error_surface.py

@@ -1,56 +0,0 @@
-import numpy as np
-from mpl_toolkits.mplot3d import Axes3D
-import matplotlib.pyplot as plt
-import matplotlib.cm as cm
-from plotstyle import *
-
-def create_data():
-    m = 0.75
-    n=  -40
-    x = np.arange(10.,110., 2.5)
-    y = m * x + n;
-    rng = np.random.RandomState(37281)
-    noise = rng.randn(len(x))*15
-    y += noise
-    return x, y, m, n
-    
-
-def plot_error_plane(ax, x, y, m, n):
-    ax.set_xlabel('Slope m')
-    ax.set_ylabel('Intercept b')
-    ax.set_zlabel('Mean squared error')
-    ax.set_xlim(-4.5, 5.0)
-    ax.set_ylim(-60.0, -20.0)
-    ax.set_zlim(0.0, 700.0)
-    ax.set_xticks(np.arange(-4, 5, 2))
-    ax.set_yticks(np.arange(-60, -19, 10))
-    ax.set_zticks(np.arange(0, 700, 200))
-    ax.grid(True)
-    ax.w_xaxis.set_pane_color((1.0, 1.0, 1.0, 1.0))
-    ax.w_yaxis.set_pane_color((1.0, 1.0, 1.0, 1.0))
-    ax.w_zaxis.set_pane_color((1.0, 1.0, 1.0, 1.0))
-    ax.invert_xaxis()
-    ax.view_init(25, 40)
-    slopes = np.linspace(-4.5, 5, 40)
-    intercepts = np.linspace(-60, -20, 40)
-    x, y = np.meshgrid(slopes, intercepts)
-    error_surf = np.zeros(x.shape)
-    for i, s in enumerate(slopes) :
-        for j, b in enumerate(intercepts) :
-                error_surf[j,i] = np.mean((y-s*x-b)**2.0)
-    ax.plot_surface(x, y, error_surf, rstride=1, cstride=1, cmap=cm.coolwarm,
-                    linewidth=0, shade=True)
-    # Minimum:
-    mini = np.unravel_index(np.argmin(error_surf), error_surf.shape)
-    ax.scatter(slopes[mini[1]], intercepts[mini[0]], [0.0], color='#cc0000', s=60)
-
-
-if __name__ == "__main__":
-    x, y, m, n = create_data()
-    fig = plt.figure()
-    ax = fig.add_subplot(1, 1, 1, projection='3d')
-    plot_error_plane(ax, x, y, m, n)
-    fig.set_size_inches(7., 5.)
-    fig.subplots_adjust(**adjust_fs(fig, 1.0, 0.0, 0.0, 0.0))
-    fig.savefig("error_surface.pdf")
-    plt.close()

regression/lecture/lin_regress.py

@@ -1,60 +0,0 @@
-import numpy as np
-import matplotlib.pyplot as plt
-from plotstyle import *
-
-def create_data():
-    m = 0.75
-    n=  -40
-    x = np.arange(10.,110., 2.5)
-    y = m * x + n;
-    rng = np.random.RandomState(37281)
-    noise = rng.randn(len(x))*15
-    y += noise
-    return x, y, m, n
-
-    
-def plot_data(ax, x, y):
-    ax.plot(x, y, **psA)
-    ax.set_xlabel('Input x')
-    ax.set_ylabel('Output y')
-    ax.set_xlim(0, 120)
-    ax.set_ylim(-80, 80)
-    ax.set_xticks(np.arange(0,121, 40))
-    ax.set_yticks(np.arange(-80,81, 40))
-    
-
-def plot_data_slopes(ax, x, y, m, n):
-    ax.plot(x, y, **psA)
-    xx = np.asarray([2, 118])
-    for i in np.linspace(0.3*m, 2.0*m, 5):
-        ax.plot(xx, i*xx+n, **lsBm)
-    ax.set_xlabel('Input x')
-    #ax.set_ylabel('Output y')
-    ax.set_xlim(0, 120)
-    ax.set_ylim(-80, 80)
-    ax.set_xticks(np.arange(0,121, 40))
-    ax.set_yticks(np.arange(-80,81, 40))
-
-
-def plot_data_intercepts(ax, x, y, m, n):
-    ax.plot(x, y, **psA)
-    xx = np.asarray([2, 118])
-    for i in np.linspace(n-1*n, n+1*n, 5):
-        ax.plot(xx, m*xx + i, **lsBm)
-    ax.set_xlabel('Input x')
-    #ax.set_ylabel('Output y')
-    ax.set_xlim(0, 120)
-    ax.set_ylim(-80, 80)
-    ax.set_xticks(np.arange(0,121, 40))
-    ax.set_yticks(np.arange(-80,81, 40))
-
-
-if __name__ == "__main__":
-    x, y, m, n = create_data()
-    fig, axs = plt.subplots(1, 3)
-    fig.subplots_adjust(wspace=0.5, **adjust_fs(fig, left=6.0, right=1.5))
-    plot_data(axs[0], x, y)
-    plot_data_slopes(axs[1], x, y, m, n)
-    plot_data_intercepts(axs[2], x, y, m, n)
-    fig.savefig("lin_regress.pdf")
-    plt.close()

regression/lecture/linear_least_squares.py

@@ -1,65 +0,0 @@
-import numpy as np
-import matplotlib.pyplot as plt
-from plotstyle import *
-
-def create_data():
-    m = 0.75
-    n=  -40
-    x = np.concatenate( (np.arange(10.,110., 2.5), np.arange(0.,120., 2.0)) )
-    y = m * x + n;
-    rng = np.random.RandomState(37281)
-    noise = rng.randn(len(x))*15
-    y += noise
-    return x, y, m, n
-
-    
-def plot_data(ax, x, y, m, n):
-    ax.set_xlabel('Input x')
-    ax.set_ylabel('Output y')
-    ax.set_xlim(0, 120)
-    ax.set_ylim(-80, 80)
-    ax.set_xticks(np.arange(0,121, 40))
-    ax.set_yticks(np.arange(-80,81, 40))
-    ax.annotate('Error',
-                xy=(x[34]+1, y[34]+15), xycoords='data',
-                xytext=(80, -50), textcoords='data', ha='left',
-                arrowprops=dict(arrowstyle="->", relpos=(0.9,1.0),
-                connectionstyle="angle3,angleA=50,angleB=-30") )
-    ax.plot(x[:40], y[:40], zorder=0, **psAm)
-    inxs = [3, 13, 16, 19, 25, 34, 36]
-    ax.plot(x[inxs], y[inxs], zorder=10, **psA)
-    xx = np.asarray([2, 118])
-    ax.plot(xx, m*xx+n, **lsBm)
-    for i in inxs :
-        xx = [x[i], x[i]]
-        yy = [m*x[i]+n, y[i]]
-        ax.plot(xx, yy, zorder=5, **lsDm)
-    
-
-def plot_error_hist(ax, x, y, m, n):
-    ax.set_xlabel('Squared error')
-    ax.set_ylabel('Frequency')
-    bins = np.arange(0.0, 602.0, 50.0)
-    ax.set_xlim(bins[0], bins[-1])
-    ax.set_ylim(0, 35)
-    ax.set_xticks(np.arange(bins[0], bins[-1], 100))
-    ax.set_yticks(np.arange(0, 36, 10))
-    errors = (y-(m*x+n))**2.0
-    mls = np.mean(errors)
-    ax.annotate('Mean\nsquared\nerror',
-                xy=(mls, 0.5), xycoords='data',
-                xytext=(350, 20), textcoords='data', ha='left',
-                arrowprops=dict(arrowstyle="->", relpos=(0.0,0.2),
-                connectionstyle="angle3,angleA=10,angleB=90") )
-    ax.hist(errors, bins, **fsD)
-
-
-
-if __name__ == "__main__":
-    x, y, m, n = create_data()
-    fig, axs = plt.subplots(1, 2)
-    fig.subplots_adjust(**adjust_fs(fig, left=6.0))
-    plot_data(axs[0], x, y, m, n)
-    plot_error_hist(axs[1], x, y, m, n)
-    fig.savefig("linear_least_squares.pdf")
-    plt.close()

regression/lecture/regression-chapter.tex

@@ -23,39 +23,18 @@
 \item Fig 8.2 right: this should be a chi-squared distribution with one degree of freedom!
 \end{itemize}
 
-\subsection{Start with one-dimensional problem!}
-\begin{itemize}
-\item Let's fit a cubic function $y=cx^3$ (weight versus length of a tiger)\\
-\includegraphics[width=0.8\textwidth]{cubicfunc}
-\item Introduce the problem, $c$ is density and form factor
-\item How to generate an artificial data set (refer to simulation chapter)
-\item How to plot a function (do not use the data x values!)
-\item Just the mean square error as a function of the factor c\\
-\includegraphics[width=0.8\textwidth]{cubicerrors}
-\item Also mention the cost function for a straight line
-\item 1-d gradient, NO quiver plot (it is a nightmare to get this right)\\
-\includegraphics[width=0.8\textwidth]{cubicmse}
-\item 1-d gradient descend
-\item Describe in words the n-d problem.
-\item Homework is to do the 2d problem with the straight line!
-\end{itemize}
-
 \subsection{Linear fits}
 \begin{itemize}
 \item Polyfit is easy: unique solution! $c x^2$ is also a linear fit.
 \item Example for overfitting with polyfit of a high order (=number of data points)
 \end{itemize}
 
-\section{Fitting in practice}
-
-Fit with matlab functions lsqcurvefit, polyfit
-
 
 \subsection{Non-linear fits}
 \begin{itemize}
 \item Example that illustrates the Nebenminima Problem (with error surface)
-\item You need got initial values for the parameter!
+\item You need initial values for the parameter!
-\item Example that fitting gets harder the more parameter yuo have.
+\item Example that fitting gets harder the more parameter you have.
 \item Try to fix as many parameter before doing the fit.
 \item How to test the quality of a fit? Residuals. $\chi^2$ test. Run-test.
 \end{itemize}

regression/lecture/regression.tex

@@ -2,195 +2,197 @@
 \exercisechapter{Optimization and gradient descent}
 
 Optimization problems arise in many different contexts. For example,
-to understand the behavior of a given system, the system is probed
+to understand the behavior of a given neuronal system, the system is
-with a range of input signals and then the resulting responses are
+probed with a range of input signals and then the resulting responses
-measured. This input-output relation can be described by a model. Such
+are measured. This input-output relation can be described by a
-a model can be a simple function that maps the input signals to
+model. Such a model can be a simple function that maps the input
-corresponding responses, it can be a filter, or a system of
+signals to corresponding responses, it can be a filter, or a system of
-differential equations. In any case, the model has some parameters that
+differential equations. In any case, the model has some parameters
-specify how input and output signals are related.  Which combination
+that specify how input and output signals are related.  Which
-of parameter values are best suited to describe the input-output
+combination of parameter values are best suited to describe the
-relation? The process of finding the best parameter values is an
+input-output relation? The process of finding the best parameter
-optimization problem. For a simple parameterized function that maps
+values is an optimization problem. For a simple parameterized function
-input to output values, this is the special case of a \enterm{curve
+that maps input to output values, this is the special case of a
-  fitting} problem, where the average distance between the curve and
+\enterm{curve fitting} problem, where the average distance between the
-the response values is minimized. One basic numerical method used for
+curve and the response values is minimized. One basic numerical method
-such optimization problems is the so called gradient descent, which is
+used for such optimization problems is the so called gradient descent,
-introduced in this chapter.
+which is introduced in this chapter.
-
-%%% Weiteres einfaches verbales Beispiel? Eventuell aus der Populationsoekologie?
 
 \begin{figure}[t]
-  \includegraphics[width=1\textwidth]{lin_regress}\hfill
+  \includegraphics{cubicfunc}
-  \titlecaption{Example data suggesting a linear relation.}{A set of
+  \titlecaption{Example data suggesting a cubic relation.}{The length
-    input signals $x$, e.g. stimulus intensities, were used to probe a
+    $x$ and weight $y$ of $n=34$ male tigers (blue, left). Assuming a
-    system. The system's output $y$ to the inputs are noted
+    cubic relation between size and weight leaves us with a single
-    (left). Assuming a linear relation between $x$ and $y$ leaves us
+    free parameters, a scaling factor. The cubic relation is shown for
-    with 2 parameters, the slope (center) and the intercept with the
+    a few values of this scaling factor (orange and red,
-    y-axis (right panel).}\label{linregressiondatafig}
+    right).}\label{cubicdatafig}
 \end{figure}
 
-The data plotted in \figref{linregressiondatafig} suggest a linear
+For demonstrating the curve-fitting problem let's take the simple
-relation between input and output of the system. We thus assume that a
+example of weights and sizes measured for a number of male tigers
-straight line
+(\figref{cubicdatafig}). Weight $y$ is proportional to volume
+$V$ via the density $\rho$. The volume $V$ of any object is
+proportional to its length $x$ cubed. The factor $\alpha$ relating
+volume and size cubed depends on the shape of the object and we do not
+know this factor for tigers. For the data set we thus expect a cubic
+relation between weight and length
 \begin{equation}
-  \label{straightline}
+  \label{cubicfunc}
-  y = f(x; m, b) = m\cdot x + b
+  y = f(x; c) = c\cdot x^3
 \end{equation}
-is an appropriate model to describe the system.  The line has two free
+where $c=\rho\alpha$, the product of a tiger's density and form
-parameter, the slope $m$ and the $y$-intercept $b$. We need to find
+factor, is the only free parameter in the relation. We would like to
-values for the slope and the intercept that best describe the measured
+find out which value of $c$ best describes the measured data.  In the
-data.  In this chapter we use this example to illustrate the gradient
+following we use this example to illustrate the gradient descent as a
-descent and how this methods can be used to find a combination of
+basic method for finding such an optimal parameter.
-slope and intercept that best describes the system.
 
 
 \section{The error function --- mean squared error}
 
-Before the optimization can be done we need to specify what exactly is
+Before we go ahead finding the optimal parameter value we need to
-considered an optimal fit. In our example we search the parameter
+specify what exactly we consider as an optimal fit. In our example we
-combination that describe the relation of $x$ and $y$ best. What is
+search the parameter that describes the relation of $x$ and $y$
-meant by this? Each input $x_i$ leads to an measured output $y_i$ and
+best. What is meant by this? The length $x_i$ of each tiger is
-for each $x_i$ there is a \emph{prediction} or \emph{estimation}
+associated with a weight $y_i$ and for each $x_i$ we have a
-$y^{est}(x_i)$ of the output value by the model. At each $x_i$
+\emph{prediction} or \emph{estimation} $y^{est}(x_i)$ of the weight by
-estimation and measurement have a distance or error $y_i -
+the model \eqnref{cubicfunc} for a specific value of the parameter
-y^{est}(x_i)$. In our example the estimation is given by the equation
+$c$. Prediction and actual data value ideally match (in a perfect
-$y^{est}(x_i) = f(x_i;m,b)$. The best fitting model with parameters
+noise-free world), but in general the estimate and measurement are
-$m$ and $b$ is the one that minimizes the distances between
+separated by some distance or error $y_i - y^{est}(x_i)$. In our
-observation $y_i$ and estimation $y^{est}(x_i)$
+example the estimate of the weight for the length $x_i$ is given by
-(\figref{leastsquareerrorfig}).
+equation \eqref{cubicfunc} $y^{est}(x_i) = f(x_i;c)$. The best fitting
-
+model with parameter $c$ is the one that somehow minimizes the
-As a first guess we could simply minimize the sum $\sum_{i=1}^N y_i -
+distances between observations $y_i$ and corresponding estimations
-y^{est}(x_i)$. This approach, however, will not work since a minimal sum
+$y^{est}(x_i)$ (\figref{cubicerrorsfig}).
-can also be achieved if half of the measurements is above and the
+
-other half below the predicted line. Positive and negative errors
+As a first guess we could simply minimize the sum of the distances,
-would cancel out and then sum up to values close to zero. A better
+$\sum_{i=1}^N y_i - y^{est}(x_i)$. This, however, does not work
-approach is to sum over the absolute values of the distances:
+because positive and negative errors would cancel out, no matter how
-$\sum_{i=1}^N |y_i - y^{est}(x_i)|$. This sum can only be small if all
+large they are, and sum up to values close to zero. Better is to sum
-deviations are indeed small no matter if they are above or below the
+over absolute distances: $\sum_{i=1}^N |y_i - y^{est}(x_i)|$. This sum
-predicted line. Instead of the sum we could also take the average
+can only be small if all deviations are indeed small no matter if they
-\begin{equation}
+are above or below the prediction.  The sum of the squared distances,
-  \label{meanabserror}
+$\sum_{i=1}^N (y_i - y^{est}(x_i))^2$, turns out to be an even better
-  f_{dist}(\{(x_i, y_i)\}|\{y^{est}(x_i)\}) = \frac{1}{N} \sum_{i=1}^N |y_i - y^{est}(x_i)|
+choice. Instead of the sum we could also minimize the average distance
-\end{equation}
-Instead of the averaged absolute errors, the \enterm[mean squared
-error]{mean squared error} (\determ[quadratischer
-Fehler!mittlerer]{mittlerer quadratischer Fehler})
 \begin{equation}
   \label{meansquarederror}
   f_{mse}(\{(x_i, y_i)\}|\{y^{est}(x_i)\}) = \frac{1}{N} \sum_{i=1}^N (y_i - y^{est}(x_i))^2
 \end{equation}
-is commonly used (\figref{leastsquareerrorfig}). Similar to the
+This is known as the \enterm[mean squared error]{mean squared error}
-absolute distance, the square of the errors, $(y_i - y^{est}(x_i))^2$, is
+(\determ[quadratischer Fehler!mittlerer]{mittlerer quadratischer
-always positive and thus positive and negative error values do not
+  Fehler}). Similar to the absolute distance, the square of the errors
+is always positive and thus positive and negative error values do not
 cancel each other out. In addition, the square punishes large
 deviations over small deviations. In
 chapter~\ref{maximumlikelihoodchapter} we show that minimizing the
-mean square error is equivalent to maximizing the likelihood that the
+mean squared error is equivalent to maximizing the likelihood that the
 observations originate from the model, if the data are normally
 distributed around the model prediction.
 
-\begin{exercise}{meanSquaredErrorLine.m}{}\label{mseexercise}%
+\begin{exercise}{meansquarederrorline.m}{}\label{mseexercise}
-  Given a vector of observations \varcode{y} and a vector with the
+  Simulate $n=40$ tigers ranging from 2.2 to 3.9\,m in size and store
-  corresponding predictions \varcode{y\_est}, compute the \emph{mean
+  these sizes in a vector \varcode{x}. Compute the corresponding
-    square error} between \varcode{y} and \varcode{y\_est} in a single
+  predicted weights \varcode{yest} for each tiger according to
-  line of code.
+  \eqnref{cubicfunc} with $c=6$\,\kilo\gram\per\meter\cubed. From the
+  predictions generate simulated measurements of the tiger's weights
+  \varcode{y}, by adding normally distributed random numbers to the
+  predictions scaled to a standard deviation of 50\,\kilo\gram.
+
+  Compute the \emph{mean squared error} between \varcode{y} and
+  \varcode{yest} in a single line of code.
 \end{exercise}
 
 
-\section{Objective function}
+\section{Cost function}
 
 The mean squared error is a so called \enterm{objective function} or
 \enterm{cost function} (\determ{Kostenfunktion}). A cost function
 assigns to a model prediction $\{y^{est}(x_i)\}$ for a given data set
 $\{(x_i, y_i)\}$ a single scalar value that we want to minimize.  Here
-we aim to adapt the model parameters to minimize the mean squared
+we aim to adapt the model parameter to minimize the mean squared error
-error \eqref{meansquarederror}. In general, the \enterm{cost function}
+\eqref{meansquarederror}. In general, the \enterm{cost function} can
-can be any function that describes the quality of the fit by mapping
+be any function that describes the quality of a fit by mapping the
-the data and the predictions to a single scalar value.
+data and the predictions to a single scalar value.
 
 \begin{figure}[t]
-  \includegraphics[width=1\textwidth]{linear_least_squares}
+  \includegraphics{cubicerrors}
-  \titlecaption{Estimating the \emph{mean square error}.}  {The
+  \titlecaption{Estimating the \emph{mean squared error}.}  {The
-    deviation error, orange) between the prediction (red
+    deviation error (orange) between the prediction (red line) and the
-    line) and the observations (blue dots) is calculated for each data
+    observations (blue dots) is calculated for each data point
-    point (left). Then the deviations are squared and the aveage is
+    (left). Then the deviations are squared and the average is
     calculated (right).}
-  \label{leastsquareerrorfig}
+  \label{cubicerrorsfig}
 \end{figure}
 
 Replacing $y^{est}$ in the mean squared error \eqref{meansquarederror}
-with our model, the straight line \eqref{straightline}, the cost
+with our cubic model \eqref{cubicfunc}, the cost function reads
-function reads
 \begin{eqnarray}
-  f_{cost}(m,b|\{(x_i, y_i)\}) & = & \frac{1}{N} \sum_{i=1}^N (y_i - f(x_i;m,b))^2 \label{msefunc} \\
+  f_{cost}(c|\{(x_i, y_i)\}) & = & \frac{1}{N} \sum_{i=1}^N (y_i - f(x_i;c))^2 \label{msefunc} \\
-  & = & \frac{1}{N} \sum_{i=1}^N (y_i - m x_i - b)^2 \label{mseline}
+  & = & \frac{1}{N} \sum_{i=1}^N (y_i - c x_i^3)^2 \label{msecube}
 \end{eqnarray}
-The optimization process tries to find the slope $m$ and the intercept
+The optimization process tries to find a value for the factor $c$ such
-$b$ such that the cost function is minimized. With the mean squared
+that the cost function is minimized. With the mean squared error as
-error as the cost function this optimization process is also called
+the cost function this optimization process is also called method of
-method of the \enterm{least square error} (\determ[quadratischer
+ \enterm{least squares} (\determ[quadratischer
 Fehler!kleinster]{Methode der kleinsten Quadrate}).
 
-\begin{exercise}{meanSquaredError.m}{}
+\begin{exercise}{meanSquaredErrorCubic.m}{}
-  Implement the objective function \eqref{mseline} as a function
+  Implement the objective function \eqref{msecube} as a function
-  \varcode{meanSquaredError()}.  The function takes three
+  \varcode{meanSquaredErrorCubic()}.  The function takes three
   arguments. The first is a vector of $x$-values and the second
   contains the measurements $y$ for each value of $x$. The third
-  argument is a 2-element vector that contains the values of
+  argument is the value of the factor $c$. The function returns the
-  parameters \varcode{m} and \varcode{b}. The function returns the
+  mean squared error.
-  mean square error.
 \end{exercise}
 
 
-\section{Error surface}
+\section{Graph of the cost function}
-For each combination of the two parameters $m$ and $b$ of the model we
+For each value of the parameter $c$ of the model we can use
-can use \eqnref{mseline} to calculate the corresponding value of the
+\eqnref{msecube} to calculate the corresponding value of the cost
-cost function. The cost function $f_{cost}(m,b|\{(x_i, y_i)\}|)$ is a
+function. The cost function $f_{cost}(c|\{(x_i, y_i)\}|)$ is a
-function $f_{cost}(m,b)$, that maps the parameter values $m$ and $b$
+function $f_{cost}(c)$ that maps the parameter value $c$ to a scalar
-to a scalar error value.  The error values describe a landscape over the
+error value. For a given data set we thus can simply plot the cost
-$m$-$b$ plane, the error surface, that can be illustrated graphically
+function as a function of the parameter $c$ (\figref{cubiccostfig}).
-using a 3-d surface-plot. $m$ and $b$ are plotted on the $x$- and $y$-
+
-axis while the third dimension indicates the error value
+\begin{exercise}{plotcubiccosts.m}{}
-(\figref{errorsurfacefig}).
+  Calculate the mean squared error between the data and the cubic
+  function for a range of values of the factor $c$ using the
+  \varcode{meanSquaredErrorCubic()} function from the previous
+  exercise. Plot the graph of the cost function.
+\end{exercise}
 
 \begin{figure}[t]
-  \includegraphics[width=0.75\textwidth]{error_surface}
+  \includegraphics{cubiccost}
-  \titlecaption{Error surface.}{The two model parameters $m$ and $b$
+  \titlecaption{Minimum of the cost function.}{For a given data set
-    define the base area of the surface plot. For each parameter
+    the cost function, the mean squared error \eqnref{msecube}, as a
-    combination of slope and intercept the error is calculated. The
+    function of the unknown parameter $c$ has a minimum close to the
-    resulting surface has a minimum which indicates the parameter
+    true value of $c$ that was used to generate the data
-    combination that best fits the data.}\label{errorsurfacefig}
+    (left). Simply taking the absolute minimum of the cost function
+    computed for a pre-set range of values for the parameter $c$, has
+    the disadvantage to be limited in precision (right) and the
+    possibility to entirely miss the global minimum if it is outside
+    the computed range.}\label{cubiccostfig}
 \end{figure}
 
-\begin{exercise}{errorSurface.m}{}\label{errorsurfaceexercise}
+By looking at the plot of the cost function we can visually identify
-  Generate 20 data pairs $(x_i|y_i)$ that are linearly related with
+the position of the minimum and thus estimate the optimal value for
-  slope $m=0.75$ and intercept $b=-40$, using \varcode{rand()} for
+the parameter $c$. How can we use the cost function to guide an
-  drawing $x$ values between 0 and 120 and \varcode{randn()} for
+automatic optimization process?
-  jittering the $y$ values with a standard deviation of 15.  Then
+
-  calculate the mean squared error between the data and straight lines
+The obvious approach would be to calculate the mean squared error for
-  for a range of slopes and intercepts using the
+a range of parameter values and then find the position of the minimum
-  \varcode{meanSquaredError()} function from the previous exercise.
+using the \code{min()} function. This approach, however has several
-  Illustrates the error surface using the \code{surface()} function.
+disadvantages: (i) The accuracy of the estimation of the best
-  Consult the documentation to find out how to use \code{surface()}.
+parameter is limited by the resolution used to sample the parameter
-\end{exercise}
+space. The coarser the parameters are sampled the less precise is the
-
+obtained position of the minimum (\figref{cubiccostfig}, right).  (ii)
-By looking at the error surface we can directly see the position of
+The range of parameter values might not include the absolute minimum.
-the minimum and thus estimate the optimal parameter combination. How
+(iii) In particular for functions with more than a single free
-can we use the error surface to guide an automatic optimization
+parameter it is computationally expensive to calculate the cost
-process?
+function for each parameter combination at a sufficient
-
+resolution. The number of combinations increases exponentially with
-The obvious approach would be to calculate the error surface for any
+the number of free parameters. This is known as the \enterm{curse of
-combination of slope and intercept values and then find the position
+  dimensionality}.
-of the minimum using the \code{min} function. This approach, however
-has several disadvantages: (i) it is computationally very expensive to
-calculate the error for each parameter combination. The number of
-combinations increases exponentially with the number of free
-parameters (also known as the ``curse of dimensionality''). (ii) the
-accuracy with which the best parameters can be estimated is limited by
-the resolution used to sample the parameter space. The coarser the
-parameters are sampled the less precise is the obtained position of
-the minimum.
 
 So we need a different approach. We want a procedure that finds the
 minimum of the cost function with a minimal number of computations and
@@ -206,34 +208,242 @@ to arbitrary precision.
       m = \frac{f(x + \Delta x) - f(x)}{\Delta x}
     \end{equation}
     of a function $y = f(x)$ is the slope of the secant (red) defined
-    by the points $(x,f(x))$ and $(x+\Delta x,f(x+\Delta x))$ with the
+    by the points $(x,f(x))$ and $(x+\Delta x,f(x+\Delta x))$ at
     distance $\Delta x$.
 
-    The slope of the function $y=f(x)$ at the position $x$ (yellow) is
+    The slope of the function $y=f(x)$ at the position $x$ (orange) is
     given by the derivative $f'(x)$ of the function at that position.
     It is defined by the difference quotient in the limit of
-    infinitesimally (orange) small distances $\Delta x$:
+    infinitesimally (red and yellow) small distances $\Delta x$:
     \begin{equation}
       \label{derivative}
       f'(x) = \frac{{\rm d} f(x)}{{\rm d}x} = \lim\limits_{\Delta x \to 0} \frac{f(x + \Delta x) - f(x)}{\Delta x} \end{equation}
   \end{minipage}\vspace{2ex} 
 
-  It is not possible to calculate the derivative, \eqnref{derivative},
+  It is not possible to calculate the exact value of the derivative
-  numerically. The derivative can only be estimated using the
+  \eqref{derivative} numerically. The derivative can only be estimated
-  difference quotient, \eqnref{difffrac}, by using sufficiently small
+  by computing the difference quotient \eqref{difffrac} using
-  $\Delta x$.
+  sufficiently small $\Delta x$.
 \end{ibox}
 
-\begin{ibox}[tp]{\label{partialderivativebox}Partial derivative and gradient}
+\section{Gradient}
-  Some functions that depend on more than a single variable:
+
+Imagine to place a ball at some point on the cost function
+\figref{cubiccostfig}. Naturally, it would roll down the slope and
+eventually stop at the minimum of the error surface (if it had no
+inertia). We will use this analogy to develop an algorithm to find our
+way to the minimum of the cost function. The ball always follows the
+steepest slope. Thus we need to figure out the direction of the slope
+at the position of the ball.
+
+\begin{figure}[t]
+  \includegraphics{cubicgradient}
+  \titlecaption{Derivative of the cost function.}{The gradient, the
+    derivative \eqref{costderivative} of the cost function, is
+    negative to the left of the minimum (vertical line) of the cost
+    function, zero (horizontal line) at, and positive to the right of
+    the minimum (left). For each value of the parameter $c$ the
+    negative gradient (arrows) points towards the minimum of the cost
+    function (right).} \label{gradientcubicfig}
+\end{figure}
+
+In our one-dimensional example of a single free parameter the slope is
+simply the derivative of the cost function with respect to the
+parameter $c$ (\figref{gradientcubicfig}, left). This derivative is called
+the \entermde{Gradient}{gradient} of the cost function:
+\begin{equation}
+  \label{costderivative}
+  \nabla f_{cost}(c) = \frac{{\rm d} f_{cost}(c)}{{\rm d} c}
+\end{equation}
+There is no need to calculate this derivative analytically, because it
+can be approximated numerically by the difference quotient
+(Box~\ref{differentialquotientbox}) for small steps $\Delta c$:
+\begin{equation}
+  \frac{{\rm d} f_{cost}(c)}{{\rm d} c} =
+    \lim\limits_{\Delta c \to 0} \frac{f_{cost}(c + \Delta c) - f_{cost}(c)}{\Delta c}
+    \approx \frac{f_{cost}(c + \Delta c) - f_{cost}(c)}{\Delta c}
+\end{equation}
+Choose, for example, $\Delta c = 10^{-7}$.  The derivative is positive
+for positive slopes. Since want to go down the hill, we choose the
+opposite direction (\figref{gradientcubicfig}, right).
+
+\begin{exercise}{meanSquaredGradientCubic.m}{}\label{gradientcubic}
+  Implement a function \varcode{meanSquaredGradientCubic()}, that
+  takes the $x$- and $y$-data and the parameter $c$ as input
+  arguments. The function should return the derivative of the mean
+  squared error $f_{cost}(c)$ with respect to $c$ at the position
+  $c$.
+\end{exercise}
+
+\begin{exercise}{plotcubicgradient.m}{}
+  Using the \varcode{meanSquaredGradientCubic()} function from the
+  previous exercise, plot the derivative of the cost function as a
+  function of $c$.
+\end{exercise}
+
+\section{Gradient descent}
+
+\begin{figure}[t]
+  \includegraphics{cubicmse}
+  \titlecaption{Gradient descent.}{The algorithm starts at an
+    arbitrary position. At each point the gradient is estimated and
+    the position is updated as long as the length of the gradient is
+    sufficiently large. The dots show the positions after each
+    iteration of the algorithm.} \label{gradientdescentcubicfig}
+\end{figure}
+
+Finally, we are able to implement the optimization itself. By now it
+should be obvious why it is called the gradient descent method. From a
+starting position on we iteratively walk down the slope of the cost
+function against its gradient. All ingredients necessary for this
+algorithm are already there. We need: (i) the cost function
+(\varcode{meanSquaredErrorCubic()}), and (ii) the gradient
+(\varcode{meanSquaredGradientCubic()}). The algorithm of the gradient
+descent works as follows:
+\begin{enumerate}
+\item Start with some arbitrary value $p_0$ for the parameter $c$.
+\item \label{computegradient} Compute the gradient
+  \eqnref{costderivative} at the current position $p_i$.
+\item If the length of the gradient, the absolute value of the
+  derivative \eqnref{costderivative}, is smaller than some threshold
+  value, we assume to have reached the minimum and stop the search.
+  We return the current parameter value $p_i$ as our best estimate of
+  the parameter $c$ that minimizes the cost function.
+
+  We are actually looking for the point at which the derivative of the
+  cost function equals zero, but this is impossible because of
+  numerical imprecision. We thus apply a threshold below which we are
+  sufficiently close to zero.
+\item \label{gradientstep} If the length of the gradient exceeds the
+  threshold we take a small step into the opposite direction:
+  \begin{equation}
+    \label{gradientdescent}
+    p_{i+1} = p_i - \epsilon \cdot \nabla f_{cost}(p_i)
+  \end{equation}
+  where $\epsilon$ is a factor linking the gradient to
+  appropriate steps in the parameter space.
+\item Repeat steps \ref{computegradient} -- \ref{gradientstep}.
+\end{enumerate}
+
+\Figref{gradientdescentcubicfig} illustrates the gradient descent ---
+the path the imaginary ball has chosen to reach the minimum. We walk
+along the parameter axis against the gradient as long as the gradient
+differs sufficiently from zero.  At steep slopes we take large steps
+(the distance between the red dots in \figref{gradientdescentcubicfig}
+is large).
+
+\begin{exercise}{gradientDescentCubic.m}{}\label{gradientdescentexercise}
+  Implement the gradient descent algorithm for the problem of fitting
+  a cubic function \eqref{cubicfunc} to some measured data pairs $x$
+  and $y$ as a function \varcode{gradientDescentCubic()} that returns
+  the estimated best fitting parameter value $c$ as well as two
+  vectors with all the parameter values and the corresponding values
+  of the cost function that the algorithm iterated trough. As
+  additional arguments that function takes the initial value for the
+  parameter $c$, the factor $\epsilon$ connecting the gradient with
+  iteration steps in \eqnref{gradientdescent}, and the threshold value
+  for the absolute value of the gradient terminating the algorithm.
+\end{exercise}
+
+\begin{exercise}{plotgradientdescentcubic.m}{}\label{plotgradientdescentexercise}
+  Use the function \varcode{gradientDescentCubic()} to fit the
+  simulated data from exercise~\ref{mseexercise}. Plot the returned
+  values of the parameter $c$ as as well as the corresponding mean
+  squared errors as a function of iteration step (two plots). Compare
+  the result of the gradient descent method with the true value of $c$
+  used to simulate the data. Inspect the plots and adapt $\epsilon$
+  and the threshold to make the algorithm behave as intended. Finally
+  plot the data together with the best fitting cubic relation
+  \eqref{cubicfunc}.
+\end{exercise}
+
+The $\epsilon$ parameter in \eqnref{gradientdescent} is critical. If
+too large, the algorithm does not converge to the minimum of the cost
+function (try it!). At medium values it oscillates around the minimum
+but might nevertheless converge. Only for sufficiently small values
+(here $\epsilon = 0.00001$) does the algorithm follow the slope
+downwards towards the minimum.
+
+The terminating condition on the absolute value of the gradient
+influences how often the cost function is evaluated. The smaller the
+threshold value the more often the cost is computed and the more
+precisely the fit parameter is estimated. If it is too small, however,
+the increase in precision is negligible, in particular in comparison
+to the increased computational effort. Have a look at the derivatives
+that we plotted in exercise~\ref{gradientcubic} and decide on a
+sensible value for the threshold. Run the gradient descent algorithm
+and check how the resulting $c$ parameter values converge and how many
+iterations were needed. Then reduce the threshold (by factors of ten)
+and check how this changes the results.
+
+Many modern algorithms for finding the minimum of a function are based
+on the basic idea of the gradient descent. Luckily these algorithms
+choose $\epsilon$ in a smart adaptive way and they also come up with
+sensible default values for the termination condition. On the other
+hand, these algorithm often take optional arguments that let you
+control how they behave. Now you know what this is all about.
+
+\section{N-dimensional minimization problems}
+
+So far we were concerned about finding the right value for a single
+parameter that minimizes a cost function.  Often we deal with
+functions that have more than a single parameter, in general $n$
+parameters. We then need to find the minimum in an $n$ dimensional
+parameter space.
+
+For our tiger problem, we could have also fitted the exponent $a$ of
+the power-law relation between size and weight, instead of assuming a
+cubic relation with $a=3$:
+\begin{equation}
+  \label{powerfunc}
+  y = f(x; c, a) = f(x; \vec p) = c\cdot x^a
+\end{equation}
+We then could check whether the resulting estimate of the exponent
+$a$ indeed is close to the expected power of three.  The
+power-law \eqref{powerfunc} has two free parameters $c$ and $a$.
+Instead of a single parameter we are now dealing with a vector $\vec
+p$ containing $n$ parameter values. Here, $\vec p = (c, a)$. All
+the concepts we introduced on the example of the one dimensional
+problem of tiger weights generalize to $n$-dimensional problems. We
+only need to adapt a few things. The cost function for the mean
+squared error reads
+\begin{equation}
+  \label{ndimcostfunc}
+  f_{cost}(\vec p|\{(x_i, y_i)\}) = \frac{1}{N} \sum_{i=1}^N (y_i - f(x_i;\vec p))^2
+\end{equation}
+
+\begin{figure}[t]
+  \includegraphics{powergradientdescent}
+  \titlecaption{Gradient descent on an error surface.}{Contour plot of
+    the cost function \eqref{ndimcostfunc} for the fit of a power law
+    \eqref{powerfunc} to some data. Here the cost function is a long
+    and narrow valley on the plane spanned by the two parameters $c$
+    and $a$ of the power law. The red line marks the path of the
+    gradient descent algorithm. The gradient is always perpendicular
+    to the contour lines. The algorithm quickly descends into the
+    valley and then slowly creeps on the shallow bottom of the valley
+    towards the global minimum where it terminates (yellow circle).
+  } \label{powergradientdescentfig}
+\end{figure}
+
+For two-dimensional problems the graph of the cost function is an
+\enterm{error surface} (\determ{{Fehlerfl\"ache}}). The two parameters
+span a two-dimensional plane. The cost function assigns to each
+parameter combination on this plane a single value. This results in a
+landscape over the parameter plane with mountains and valleys and we
+are searching for the position of the bottom of the deepest valley
+(\figref{powergradientdescentfig}).
+
+\begin{ibox}[tp]{\label{partialderivativebox}Partial derivatives and gradient}
+  Some functions depend on more than a single variable. For example, the function
   \[ z = f(x,y) \]
-  for example depends on $x$ and $y$. Using the partial derivative
+  depends on both $x$ and $y$. Using the partial derivatives
   \[ \frac{\partial f(x,y)}{\partial x} = \lim\limits_{\Delta x \to 0} \frac{f(x + \Delta x,y) - f(x,y)}{\Delta x} \]
   and
   \[ \frac{\partial f(x,y)}{\partial y} = \lim\limits_{\Delta y \to 0} \frac{f(x, y + \Delta y) - f(x,y)}{\Delta y} \]
-  one can estimate the slope in the direction of the variables
+  one can calculate the slopes in the directions of each of the
-  individually by using the respective difference quotient
+  variables by means of the respective difference quotient
-  (Box~\ref{differentialquotientbox}).  \vspace{1ex}
+  (see box~\ref{differentialquotientbox}).  \vspace{1ex}
 
   \begin{minipage}[t]{0.5\textwidth}
     \mbox{}\\[-2ex]
@@ -242,145 +452,129 @@ to arbitrary precision.
   \hfill
   \begin{minipage}[t]{0.46\textwidth}
     For example, the partial derivatives of
-    \[ f(x,y) = x^2+y^2 \] are 
+    \[ f(x,y) = x^2+y^2 \vspace{-1ex} \] are 
     \[ \frac{\partial f(x,y)}{\partial x} = 2x \; , \quad \frac{\partial f(x,y)}{\partial y} = 2y \; .\]
 
-    The gradient is a vector that is constructed from the partial derivatives:
+    The gradient is a vector with the partial derivatives as coordinates:
     \[ \nabla f(x,y) = \left( \begin{array}{c} \frac{\partial f(x,y)}{\partial x} \\[1ex] \frac{\partial f(x,y)}{\partial y} \end{array} \right) \]
     This vector points into the direction of the strongest ascend of
     $f(x,y)$.
   \end{minipage}
 
-  \vspace{0.5ex} The figure shows the contour lines of a bi-variate
+  \vspace{0.5ex} The figure shows the contour lines of a bivariate
   Gaussian $f(x,y) = \exp(-(x^2+y^2)/2)$ and the gradient (thick
-  arrows) and the corresponding two partial derivatives (thin arrows)
+  arrows) together with the corresponding partial derivatives (thin
-  for three different locations.
+  arrows) for three different locations.
 \end{ibox}
 
-
+When we place a ball somewhere on the slopes of a hill, it rolls
-\section{Gradient}
+downwards and eventually stops at the bottom. The ball always rolls in
-Imagine to place a small ball at some point on the error surface
+the direction of the steepest slope. For a two-dimensional parameter
-\figref{errorsurfacefig}. Naturally, it would roll down the steepest
+space of our example, the \entermde{Gradient}{gradient}
-slope and eventually stop at the minimum of the error surface (if it had no
+(box~\ref{partialderivativebox}) of the cost function is a vector
-inertia). We will use this picture to develop an algorithm to find our
+\begin{equation}
-way to the minimum of the objective function. The ball will always
+  \label{gradientpowerlaw}
-follow the steepest slope. Thus we need to figure out the direction of
+  \nabla f_{cost}(c, a) = \left( \frac{\partial f_{cost}(c, a)}{\partial c},
-the steepest slope at the position of the ball.
+    \frac{\partial f_{cost}(c, a)}{\partial a} \right)
-
+\end{equation}
-The \entermde{Gradient}{gradient} (Box~\ref{partialderivativebox}) of the
+that points into the direction of the strongest ascend of the cost
-objective function is the vector
+function.  The gradient is given by the partial derivatives
+(box~\ref{partialderivativebox}) of the mean squared error with
+respect to the parameters $c$ and $a$ of the power law
+relation. In general, for $n$-dimensional problems, the gradient is an
+$n-$ dimensional vector containing for each of the $n$ parameters
+$p_j$ the respective partial derivatives as coordinates:
 \begin{equation}
   \label{gradient}
-  \nabla f_{cost}(m,b) = \left( \frac{\partial f(m,b)}{\partial m},
+  \nabla f_{cost}(\vec p) = \left( \frac{\partial f_{cost}(\vec p)}{\partial p_j} \right)
-    \frac{\partial f(m,b)}{\partial b} \right)
 \end{equation}
-that points to the strongest ascend of the objective function.  The
+
-gradient is given by partial derivatives
+The iterative equation \eqref{gradientdescent} of the gradient descent
-(Box~\ref{partialderivativebox}) of the mean squared error with
+stays exactly the same, with the only difference that the current
-respect to the parameters $m$ and $b$ of the straight line. There is
+parameter value $p_i$ becomes a vector $\vec p_i$ of parameter values:
-no need to calculate it analytically because it can be estimated from
-the partial derivatives using the difference quotient
-(Box~\ref{differentialquotientbox}) for small steps $\Delta m$ and
-$\Delta b$. For example, the partial derivative with respect to $m$
-can be computed as
 \begin{equation}
-  \frac{\partial f_{cost}(m,b)}{\partial m} = \lim\limits_{\Delta m \to
+  \label{ndimgradientdescent}
-  0} \frac{f_{cost}(m + \Delta m, b) - f_{cost}(m,b)}{\Delta m}
+  \vec p_{i+1} = \vec p_i - \epsilon \cdot \nabla f_{cost}(\vec p_i)
-\approx \frac{f_{cost}(m + \Delta m, b) - f_{cost}(m,b)}{\Delta m} \; . 
 \end{equation}
-The length of the gradient indicates the steepness of the slope
+The algorithm proceeds along the negative gradient
-(\figref{gradientquiverfig}). Since want to go down the hill, we
+(\figref{powergradientdescentfig}).
-choose the opposite direction.
+
+For the termination condition we need the length of the gradient. In
+one dimension it was just the absolute value of the derivative. For
+$n$ dimensions this is according to the \enterm{Pythagorean theorem}
+(\determ[Pythagoras!Satz des]{Satz des Pythagoras}) the square root of
+the sum of the squared partial derivatives:
+\begin{equation}
+  \label{ndimabsgradient}
+  |\nabla f_{cost}(\vec p_i)| = \sqrt{\sum_{i=1}^n \left(\frac{\partial f_{cost}(\vec p)}{\partial p_i}\right)^2}
+\end{equation}
+The \code{norm()} function implements this given a vector with the
+partial derivatives.
 
+\subsection{Passing a function as an argument to another function}
 
-\begin{figure}[t]
+So far, all our code for the gradient descent algorithm was tailored
-  \includegraphics[width=0.75\textwidth]{error_gradient}
+to a specific function, the cubic relation \eqref{cubicfunc}. It would
-  \titlecaption{Gradient of the error surface.}  {Each arrow points
+be much better if we could pass an arbitrary function to our gradient
-    into the direction of the greatest ascend at different positions
+algorithm. Then we would not need to rewrite it every time anew.
-    of the error surface shown in \figref{errorsurfacefig}. The
-    contour lines in the background illustrate the error surface. Warm
-    colors indicate high errors, colder colors low error values. Each
-    contour line connects points of equal
-    error.}\label{gradientquiverfig}
-\end{figure}
 
-\begin{exercise}{meanSquaredGradient.m}{}\label{gradientexercise}%
+This is possible. We can indeed pass a function as an argument to
-  Implement a function \varcode{meanSquaredGradient()}, that takes the
+another function. For this use the \code{@}-operator. As an example
-  $x$- and $y$-data and the set of parameters $(m, b)$ of a straight
+let's define a function that produces a standard plot for a function:
-  line as a two-element vector as input arguments. The function should
-  return the gradient at the position $(m, b)$ as a vector with two
-  elements.
-\end{exercise}
 
-\begin{exercise}{errorGradient.m}{}
+\pageinputlisting[caption={Example function taking a function as argument.}]{funcPlotter.m}
-  Extend the script of exercises~\ref{errorsurfaceexercise} to plot
-  both the error surface and gradients using the
-  \varcode{meanSquaredGradient()} function from
-  exercise~\ref{gradientexercise}. Vectors in space can be easily
-  plotted using the function \code{quiver()}. Use \code{contour()}
-  instead of \code{surface()} to plot the error surface.
-\end{exercise}
 
+This function can then be used as follows for plotting a sine wave. We
+pass the built in \varcode{sin()} function as \varcode{@sin} as an
+argument to our function:
 
-\section{Gradient descent}
+\pageinputlisting[caption={Passing a function handle as an argument to a function.}]{funcplotterexamples.m}
-Finally, we are able to implement the optimization itself. By now it
-should be obvious why it is called the gradient descent method. All
-ingredients are already there. We need: (i) the cost function
-(\varcode{meanSquaredError()}), and (ii) the gradient
-(\varcode{meanSquaredGradient()}). The algorithm of the gradient
-descent works as follows:
-\begin{enumerate}
-\item Start with some given combination of the parameters $m$ and $b$
-  ($p_0 = (m_0, b_0)$).
-\item \label{computegradient} Calculate the gradient at the current
-  position $p_i$.
-\item If the length of the gradient falls below a certain value, we
-  assume to have reached the minimum and stop the search. We are
-  actually looking for the point at which the length of the gradient
-  is zero, but finding zero is impossible because of numerical
-  imprecision. We thus apply a threshold below which we are
-  sufficiently close to zero (e.g. \varcode{norm(gradient) < 0.1}).
-\item \label{gradientstep} If the length of the gradient exceeds the
-  threshold we take a small step into the opposite direction:
-  \begin{equation}
-    p_{i+1} = p_i - \epsilon \cdot \nabla f_{cost}(m_i, b_i)
-  \end{equation}
-  where $\epsilon = 0.01$ is a factor linking the gradient to
-  appropriate steps in the parameter space.
-\item Repeat steps \ref{computegradient} -- \ref{gradientstep}.
-\end{enumerate}
 
-\Figref{gradientdescentfig} illustrates the gradient descent --- the
-path the imaginary ball has chosen to reach the minimum. Starting at
-an arbitrary position on the error surface we change the position as
-long as the gradient at that position is larger than a certain
-threshold. If the slope is very steep, the change in the position (the
-distance between the red dots in \figref{gradientdescentfig}) is
-large.
 
-\begin{figure}[t]
+\subsection{Gradient descent algorithm for arbitrary functions}
-  \includegraphics[width=0.45\textwidth]{gradient_descent}
+
-  \titlecaption{Gradient descent.}{The algorithm starts at an
+Now we are ready to adapt the gradient descent algorithm from
-    arbitrary position. At each point the gradient is estimated and
+exercise~\ref{gradientdescentexercise} to arbitrary functions with $n$
-    the position is updated as long as the length of the gradient is
+parameters that we want to fit to some data.
-    sufficiently large.The dots show the positions after each
-    iteration of the algorithm.} \label{gradientdescentfig}
-\end{figure}
 
 \begin{exercise}{gradientDescent.m}{}
-  Implement the gradient descent for the problem of fitting a straight
+  Adapt the function \varcode{gradientDescentCubic()} from
-  line to some measured data. Reuse the data generated in
+  exercise~\ref{gradientdescentexercise} to implement the gradient
-  exercise~\ref{errorsurfaceexercise}.
+  descent algorithm for any function \varcode{func(x, p)} that takes
-  \begin{enumerate}
+  as first argument the $x$-data values and as second argument a
-  \item Store for each iteration the error value.
+  vector with parameter values.  The new function takes a vector $\vec
-  \item Plot the error values as a function of the iterations, the
+  p_0$ for the initial parameter values and also returns the best
-    number of optimization steps.
+  parameter combination as a vector. Use a \varcode{for} loop over the
-  \item Plot the measured data together with the best fitting straight line.
+  two dimensions for computing the gradient.
-  \end{enumerate}\vspace{-4.5ex}
 \end{exercise}
 
+For testing our new function we need to implement the power law
+\eqref{powerfunc}:
 
-\section{Summary}
+\begin{exercise}{powerLaw.m}{}
+  Write a function that implements \eqref{powerfunc}. The function
+  gets as arguments a vector $x$ containing the $x$-data values and
+  another vector containing as elements the parameters for the power
+  law, i.e. the factor $c$ and the power $a$. It returns a vector
+  with the computed $y$ values for each $x$ value. 
+\end{exercise}
+
+Now let's use the new gradient descent function to fit a power law to
+our tiger data-set (\figref{powergradientdescentfig}):
+
+\begin{exercise}{plotgradientdescentpower.m}{}
+  Use the function \varcode{gradientDescent()} to fit the
+  \varcode{powerLaw()} function to the simulated data from
+  exercise~\ref{mseexercise}. Plot the returned values of the two
+  parameters against each other. Compare the result of the gradient
+  descent method with the true values of $c$ and $a$ used to simulate
+  the data. Observe the norm of the gradient and inspect the plots to
+  adapt $\epsilon$ and the threshold if necessary. Finally plot the
+  data together with the best fitting power-law \eqref{powerfunc}.
+\end{exercise}
+
+
+\section{Fitting non-linear functions to data}
 
 The gradient descent is an important numerical method for solving
 optimization problems. It is used to find the global minimum of an
@@ -389,33 +583,44 @@ objective function.
 Curve fitting is a common application for the gradient descent method.
 For the case of fitting straight lines to data pairs, the error
 surface (using the mean squared error) has exactly one clearly defined
-global minimum. In fact, the position of the minimum can be analytically
+global minimum. In fact, the position of the minimum can be
-calculated as shown in the next chapter.
+analytically calculated as shown in the next chapter. For linear
-
+fitting problems numerical methods like the gradient descent are not
-Problems that involve nonlinear computations on parameters, e.g. the
+needed.
-rate $\lambda$ in an exponential function $f(x;\lambda) = e^{\lambda
+
-  x}$, do not have an analytical solution for the least squares. To
+Fitting problems that involve nonlinear functions of the parameters,
-find the least squares for such functions numerical methods such as
+e.g. the power law \eqref{powerfunc} or the exponential function
-the gradient descent have to be applied.
+$f(x;\lambda) = e^{\lambda x}$, do not have an analytical solution for
-
+the least squares. To find the least squares for such functions
-The suggested gradient descent algorithm can be improved in multiple
+numerical methods such as the gradient descent have to be applied.
-ways to converge faster.  For example one could adapt the step size to
+
-the length of the gradient. These numerical tricks have already been
+The suggested gradient descent algorithm is quite fragile and requires
-implemented in pre-defined functions. Generic optimization functions
+manually tuned values for $\epsilon$ and the threshold for terminating
-such as \matlabfun{fminsearch()} have been implemented for arbitrary
+the iteration.  The algorithm can be improved in multiple ways to
-objective functions, while the more specialized function
+converge more robustly and faster.  For example one could adapt the
-\matlabfun{lsqcurvefit()} i specifically designed for optimizations in
+step size to the length of the gradient. These numerical tricks have
-the least square error sense.
+already been implemented in pre-defined functions. Generic
-
+optimization functions such as \mcode{fminsearch()} have been
-%\newpage
+implemented for arbitrary objective functions, while the more
+specialized function \mcode{lsqcurvefit()} is specifically designed
+for optimizations in the least square error sense.
+
+\begin{exercise}{plotlsqcurvefitpower.m}{}
+  Use the \matlab-function \varcode{lsqcurvefit()} instead of
+  \varcode{gradientDescent()} to fit the \varcode{powerLaw()} function
+  to the simulated data from exercise~\ref{mseexercise}. Plot the data
+  and the resulting best fitting power law function.
+\end{exercise}
+
+
 \begin{important}[Beware of secondary minima!]
   Finding the absolute minimum is not always as easy as in the case of
-  fitting a straight line. Often, the error surface has secondary or
+  fitting a straight line. Often, the cost function has secondary or
   local minima in which the gradient descent stops even though there
   is a more optimal solution, a global minimum that is lower than the
   local minimum. Starting from good initial positions is a good
   approach to avoid getting stuck in local minima. Also keep in mind
-  that error surfaces tend to be simpler (less local minima) the fewer
+  that cost functions tend to be simpler (less local minima) the fewer
   parameters are fitted from the data. Each additional parameter
   increases complexity and is computationally more expensive.
 \end{important}