Merge branch 'master' of https://whale.am28.uni-tuebingen.de/git/teaching/scientificComputing

Makefile

@@ -3,7 +3,7 @@ BASENAME=scientificcomputing-script
 SUBDIRS=programming debugging plotting codestyle statistics bootstrap regression likelihood pointprocesses designpattern 
 SUBTEXS=$(foreach subd, $(SUBDIRS), $(subd)/lecture/$(subd).tex)
 
-all : script chapters
+all : plots chapters index script exercises
 
 chapters :
 	for sd in $(SUBDIRS); do $(MAKE) -C $$sd/lecture chapter; done
@@ -37,6 +37,11 @@ watchpdf :
 watchscript :
 	while true; do ! make -s -q script && make script; sleep 0.5; done
 
+
+exercises:
+	for sd in $(SUBDIRS); do if test -d $$sd/exercises; then $(MAKE) -C $$sd/exercises pdf; fi; done
+
+
 cleanplots:
 	for sd in $(SUBDIRS); do $(MAKE) -C $$sd/lecture cleanplots; done
 
@@ -46,6 +51,7 @@ cleantex:
 
 clean : cleantex
 	for sd in $(SUBDIRS); do $(MAKE) -C $$sd/lecture clean; done
+	for sd in $(SUBDIRS); do if test -d $$sd/exercises; then $(MAKE) -C $$sd/exercises clean; fi; done
 
 cleanall : clean
 	rm -f $(BASENAME).pdf

README.fonts

@@ -1,6 +1,20 @@
 Fonts for matplotlib
 --------------------
 
+Install Humor Sans font
+```
+sudo apt install fonts-humor-sans
+```
+
+Clear matplotlib font cache:
+```
+cd ~/.cache/matplotlib/
+rm -r *
+```
+
+Older problems
+--------------
+
 Make sure the right fonts are installed:
 ```
 sudo apt-get install ttf-lyx2.0 

bootstrap/code/bootstrapsem.m

@@ -1,24 +1,25 @@
 nsamples = 100;
 nresamples = 1000;
 
-% draw a SRS (simple random sample, "Stichprobe") from the population:
-x = randn( 1, nsamples );
-fprintf('%-30s  %-5s  %-5s  %-5s\n', '', 'mean', 'stdev', 'sem' )
-fprintf('%30s  %5.2f  %5.2f  %5.2f\n', 'single SRS', mean( x ), std( x ), std( x )/sqrt(nsamples) )
+% draw a simple random sample ("Stichprobe") from the population:
+x = randn(1, nsamples);
+fprintf('%-30s  %-5s  %-5s  %-5s\n', '', 'mean', 'stdev', 'sem')
+fprintf('%30s  %5.2f  %5.2f  %5.2f\n', 'single SRS', mean(x), std(x), std(x)/sqrt(nsamples))
 
 % bootstrap the mean:
-mus = zeros(nresamples,1);  % vector for storing the means
-for i = 1:nresamples        % loop for generating the bootstraps
+mus = zeros(nresamples,1);    % vector for storing the means
+for i = 1:nresamples          % loop for generating the bootstraps
     inx = randi(nsamples, 1, nsamples);  % range, 1D-vector, number
-    xr = x(inx);            % resample the original SRS
-    mus(i) = mean(xr);      % compute statistic of the resampled SRS
+    xr = x(inx);              % resample the original SRS
+    mus(i) = mean(xr);        % compute statistic of the resampled SRS
 end
-fprintf('%30s  %5.2f  %5.2f  -\n', 'bootstrapped distribution', mean( mus ), std( mus ) )
+fprintf('%30s  %5.2f  %5.2f  -\n', 'bootstrapped distribution', mean(mus), std(mus))
 
-% many SRS (we can do that with the random number generator, but not in real life!):
-musrs = zeros(nresamples,1);      % vector for the means of each SRS
+% many SRS (we can do that with the random number generator,
+% but not in real life!):
+musrs = zeros(nresamples,1);  % vector for the means of each SRS
 for i = 1:nresamples
-    x = randn( 1, nsamples );     % draw a new SRS
-    musrs(i) = mean( x );         % compute its mean
+    x = randn(1, nsamples);   % draw a new SRS
+    musrs(i) = mean(x);       % compute its mean
 end
-fprintf('%30s  %5.2f  %5.2f  -\n', 'sampling distribution', mean( musrs ), std( musrs ) )
+fprintf('%30s  %5.2f  %5.2f  -\n', 'sampling distribution', mean(musrs), std(musrs))

bootstrap/code/correlationsignificance.m

@@ -1,27 +1,27 @@
 % generate correlated data:
-n=200;
-a=0.2;
+n = 200;
+a = 0.2;
 x = randn(n, 1);
 y = randn(n, 1) + a*x;
 
 % correlation coefficient:
 rd = corr(x, y);
-fprintf('correlation coefficient of data r = %.2f\n', rd );
+fprintf('correlation coefficient of data r = %.2f\n', rd);
 
 % distribution of null hypothesis by permutation:
 nperm = 1000;
 rs = zeros(nperm,1);
-for i=1:nperm
-    xr=x(randperm(length(x)));  % shuffle x
-    yr=y(randperm(length(y)));  % shuffle y
+for i = 1:nperm
+    xr = x(randperm(length(x)));  % shuffle x
+    yr = y(randperm(length(y)));  % shuffle y
     rs(i) = corr(xr, yr);
 end
-[h,b] = hist(rs, 20 );
-h = h/sum(h)/(b(2)-b(1));  % normalization
+[h, b] = hist(rs, 20);
+h = h/sum(h)/(b(2)-b(1));         % normalization
 
 % significance:
 rq = quantile(rs, 0.95);
-fprintf('correlation coefficient of null hypothesis at 5%% significance = %.2f\n', rq );
+fprintf('correlation coefficient of null hypothesis at 5%% significance = %.2f\n', rq);
 if rd >= rq
     fprintf('--> correlation r=%.2f is significant\n', rd);
 else
@@ -32,7 +32,7 @@ end
 bar(b, h, 'facecolor', 'b');
 hold on;
 bar(b(b>=rq), h(b>=rq), 'facecolor', 'r');
-plot( [rd rd], [0 4], 'r', 'linewidth', 2 );
+plot([rd rd], [0 4], 'r', 'linewidth', 2);
 xlabel('Correlation coefficient');
 ylabel('Probability density of H0');
 hold off;

bootstrap/code/meandiffsignificance.m

@@ -0,0 +1,40 @@
+% generate two data sets:
+n = 200;
+d = 0.2;
+x = randn(n, 1);
+y = randn(n, 1) + d;
+
+% difference of sample means:
+md = mean(y) - mean(x);
+fprintf('difference of means of data d = %.2f\n', md);
+
+% distribution of null hypothesis by permutation:
+nperm = 1000;
+xy = [x; y];                         % x and y data in one column vector
+ds = zeros(nperm,1);
+for i = 1:nperm
+    xyr = xy(randperm(length(xy)));  % shuffle data
+    xr = xyr(1:length(x));           % random x-data
+    yr = xyr(length(x)+1:end);       % random y-data
+    ds(i) = mean(yr) - mean(xr);     % difference of means
+end
+[h, b] = hist(ds, 20);
+h = h/sum(h)/(b(2)-b(1));       % normalization
+
+% significance:
+dq = quantile(ds, 0.95);
+fprintf('difference of means of null hypothesis at 5%% significance = %.2f\n', dq);
+if md >= dq
+    fprintf('--> difference of means d=%.2f is significant\n', md);
+else
+    fprintf('--> d=%.2f is not a significant difference of means\n', md);
+end
+
+% plot:
+bar(b, h, 'facecolor', 'b');
+hold on;
+bar(b(b>=dq), h(b>=dq), 'facecolor', 'r');
+plot([md md], [0 4], 'r', 'linewidth', 2);
+xlabel('Difference of means');
+ylabel('Probability density of H0');
+hold off;

bootstrap/code/meandiffsignificance.out

@@ -0,0 +1,4 @@
+>> meandiffsignificance
+difference of means of data d = 0.18
+difference of means of null hypothesis at 5% significance = 0.17
+--> difference of means d=0.18 is significant

bootstrap/exercises/Makefile

@@ -1,34 +1,3 @@
-TEXFILES=$(wildcard exercises??.tex)
-EXERCISES=$(TEXFILES:.tex=.pdf)
-SOLUTIONS=$(EXERCISES:exercises%=solutions%)
+TEXFILES=$(wildcard resampling-?.tex)
 
-.PHONY: pdf exercises solutions watch watchexercises watchsolutions clean
-
-pdf : $(SOLUTIONS) $(EXERCISES)
-
-exercises : $(EXERCISES)
-
-solutions : $(SOLUTIONS)
-
-$(SOLUTIONS) : solutions%.pdf : exercises%.tex instructions.tex
-	{ echo "\\documentclass[answers,12pt,a4paper,pdftex]{exam}"; sed -e '1d' $<; } > $(patsubst %.pdf,%.tex,$@)
-	pdflatex -interaction=scrollmode $(patsubst %.pdf,%.tex,$@) | tee /dev/stderr | fgrep -q "Rerun to get cross-references right" && pdflatex -interaction=scrollmode $(patsubst %.pdf,%.tex,$@) || true
-	rm $(patsubst %.pdf,%,$@).[!p]*
-
-$(EXERCISES) : %.pdf : %.tex instructions.tex
-	pdflatex -interaction=scrollmode $< | tee /dev/stderr | fgrep -q "Rerun to get cross-references right" && pdflatex -interaction=scrollmode $< || true
-
-watch :
-	while true; do ! make -q pdf && make pdf; sleep 0.5; done
-
-watchexercises :
-	while true; do ! make -q exercises && make exercises; sleep 0.5; done
-
-watchsolutions :
-	while true; do ! make -q solutions && make solutions; sleep 0.5; done
-
-clean :
-	rm -f *~ *.aux *.log *.out
-
-cleanup : clean
-	rm -f $(SOLUTIONS) $(EXERCISES)
+include ../../exercises.mk

bootstrap/exercises/bootstrapmean.m

@@ -10,7 +10,7 @@ function [bootsem, mu] = bootstrapmean(x, resample)
     for i = 1:resample
         % resample:
         xr = x(randi(nsamples, nsamples, 1));
-        % compute statistics on sample:
+        % compute statistics of resampled sample:
         mu(i) = mean(xr);
     end
     bootsem = std(mu);

bootstrap/exercises/correlationbootstrap.m

@@ -1,18 +1,18 @@
 %% (a) bootstrap:
 nperm = 1000;
-rb = zeros(nperm,1);
+rb = zeros(nperm, 1);
 for i=1:nperm
     % indices for resampling the data:
     inx = randi(length(x), length(x), 1);
     % resampled data pairs:
-    xb=x(inx);
-    yb=y(inx);
+    xb = x(inx);
+    yb = y(inx);
     rb(i) = corr(xb, yb);
 end
 
 %% (b) pdf of the correlation coefficients:
-[hb,bb] = hist(rb, 20);
-hb = hb/sum(hb)/(bb(2)-bb(1));  % normalization
+[hb, bb] = hist(rb, 20);
+hb = hb/sum(hb)/(bb(2)-bb(1));      % normalization
 
 %% (c) significance:
 rbq = quantile(rb, 0.05);
@@ -25,8 +25,8 @@ end
 
 %% plot:
 hold on;
-bar(b, h, 'facecolor', [0.5 0.5 0.5]);
-bar(bb, hb, 'facecolor', 'b');
+bar(b, h, 'facecolor', [0.5 0.5 0.5]);           % permuation test
+bar(bb, hb, 'facecolor', 'b');                   % bootstrap
 bar(bb(bb<=rbq), hb(bb<=rbq), 'facecolor', 'r');
 plot([rd rd], [0 4], 'r', 'linewidth', 2);
 xlim([-0.25 0.75])

bootstrap/exercises/correlationsignificance.m

@@ -1,4 +1,4 @@
-%% (a) generate correlated data
+%% (a) generate correlated data:
 n = 1000;
 a = 0.2;
 x = randn(n, 1);
@@ -8,7 +8,7 @@ y = randn(n, 1) + a*x;
 subplot(1, 2, 1);
 plot(x, a*x, 'r', 'linewidth', 3);
 hold on
-%scatter(x, y );   % either scatter ...
+%scatter(x, y );                    % either scatter ...
 plot(x, y, 'o', 'markersize', 2 );  % ... or plot - same plot.
 xlim([-4 4])
 ylim([-4 4])
@@ -20,20 +20,20 @@ hold off
 %c = corrcoef(x, y);  % returns correlation matrix
 %rd = c(1, 2);
 rd = corr(x, y);
-fprintf('correlation coefficient = %.2f\n', rd );
+fprintf('correlation coefficient = %.2f\n', rd);
 
 %% (e) permutation:
 nperm = 1000;
-rs = zeros(nperm,1);
-for i=1:nperm
-    xr=x(randperm(length(x)));  % shuffle x
-    yr=y(randperm(length(y)));  % shuffle y
+rs = zeros(nperm, 1);
+for i = 1:nperm
+    xr = x(randperm(length(x)));      % shuffle x
+    yr = y(randperm(length(y)));      % shuffle y
     rs(i) = corr(xr, yr);
 end
 
 %% (g) pdf of the correlation coefficients:
-[h,b] = hist(rs, 20);
-h = h/sum(h)/(b(2)-b(1));  % normalization
+[h, b] = hist(rs, 20);
+h = h/sum(h)/(b(2)-b(1));           % normalization
 
 %% (h) significance:
 rq = quantile(rs, 0.95);
@@ -49,7 +49,7 @@ subplot(1, 2, 2)
 hold on;
 bar(b, h, 'facecolor', 'b');
 bar(b(b>=rq), h(b>=rq), 'facecolor', 'r');
-plot( [rd rd], [0 4], 'r', 'linewidth', 2);
+plot([rd rd], [0 4], 'r', 'linewidth', 2);
 xlim([-0.25 0.25])
 xlabel('Correlation coefficient');
 ylabel('Probability density of H0');

bootstrap/exercises/exercises01.tex

@@ -1,205 +0,0 @@
-\documentclass[12pt,a4paper,pdftex]{exam}
-
-\usepackage[english]{babel}
-\usepackage{pslatex}
-\usepackage[mediumspace,mediumqspace,Gray]{SIunits}      % \ohm, \micro
-\usepackage{xcolor}
-\usepackage{graphicx}
-\usepackage[breaklinks=true,bookmarks=true,bookmarksopen=true,pdfpagemode=UseNone,pdfstartview=FitH,colorlinks=true,citecolor=blue]{hyperref}
-
-%%%%% layout %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage[left=20mm,right=20mm,top=25mm,bottom=25mm]{geometry}
-\pagestyle{headandfoot}
-\ifprintanswers
-\newcommand{\stitle}{: Solutions}
-\else
-\newcommand{\stitle}{}
-\fi
-\header{{\bfseries\large Exercise 9\stitle}}{{\bfseries\large Bootstrap}}{{\bfseries\large December 9th, 2019}}
-\firstpagefooter{Prof. Dr. Jan Benda}{Phone: 29 74573}{Email:
-jan.benda@uni-tuebingen.de}
-\runningfooter{}{\thepage}{}
-
-\setlength{\baselineskip}{15pt}
-\setlength{\parindent}{0.0cm}
-\setlength{\parskip}{0.3cm}
-\renewcommand{\baselinestretch}{1.15}
-
-%%%%% listings %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage{listings}
-\lstset{
-  language=Matlab,
-  basicstyle=\ttfamily\footnotesize,
-  numbers=left,
-  numberstyle=\tiny,
-  title=\lstname,
-  showstringspaces=false,
-  commentstyle=\itshape\color{darkgray},
-  breaklines=true,
-  breakautoindent=true,
-  columns=flexible,
-  frame=single,
-  xleftmargin=1em,
-  xrightmargin=1em,
-  aboveskip=10pt
-}
-
-%%%%% math stuff: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage{amsmath}
-\usepackage{amssymb}
-\usepackage{bm} 
-\usepackage{dsfont}
-\newcommand{\naZ}{\mathds{N}}
-\newcommand{\gaZ}{\mathds{Z}}
-\newcommand{\raZ}{\mathds{Q}}
-\newcommand{\reZ}{\mathds{R}}
-\newcommand{\reZp}{\mathds{R^+}}
-\newcommand{\reZpN}{\mathds{R^+_0}}
-\newcommand{\koZ}{\mathds{C}}
-
-%%%%% page breaks %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\newcommand{\continue}{\ifprintanswers%
-\else
-\vfill\hspace*{\fill}$\rightarrow$\newpage%
-\fi}
-\newcommand{\continuepage}{\ifprintanswers%
-\newpage
-\else
-\vfill\hspace*{\fill}$\rightarrow$\newpage%
-\fi}
-\newcommand{\newsolutionpage}{\ifprintanswers%
-\newpage%
-\else
-\fi}
-
-%%%%% new commands %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\newcommand{\qt}[1]{\textbf{#1}\\}
-\newcommand{\pref}[1]{(\ref{#1})}
-\newcommand{\extra}{--- Zusatzaufgabe ---\ \mbox{}}
-\newcommand{\code}[1]{\texttt{#1}}
-
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\begin{document}
-
-\input{instructions}
-
-\begin{questions}
-
-\question \qt{Bootstrap the standard error of the mean}
-We want to compute the standard error of the mean of a data set by
-means of the bootstrap method and compare the result with the formula
-``standard deviation divided by the square-root of $n$''.
-\begin{parts}
-  \part Download the file \code{thymusglandweights.dat} from Ilias.
-  This is a data set of the weights of the thymus glands of 14-day old chicken embryos
-  measured in milligram.
-  \part Load the data into Matlab (\code{load} function).
-  \part Compute histogram, mean, and standard error of the mean of the first 80 data points.
-  \part Compute the standard error of the mean of the first 80 data
-  points by bootstrapping the data 500 times. Write a function that
-  bootstraps the standard error of the mean of a given data set. The
-  function should also return a vector with the bootstrapped means.
-  \part Compute the 95\,\% confidence interval for the mean from the
-  bootstrap distribution (\code{quantile()} function) --- the
-  interval that contains the true mean with 95\,\% probability.
-  \part Use the whole data set and the bootstrap method for computing
-  the dependence of the standard error of the mean from the sample
-  size $n$.
-  \part Compare your result with the formula for the standard error
-  $\sigma/\sqrt{n}$.
-\end{parts}
-\begin{solution}
-  \lstinputlisting{bootstrapmean.m}
-  \lstinputlisting{bootstraptymus.m}
-  \includegraphics[width=0.5\textwidth]{bootstraptymus-datahist}
-  \includegraphics[width=0.5\textwidth]{bootstraptymus-meanhist}
-  \includegraphics[width=0.5\textwidth]{bootstraptymus-samples}
-\end{solution}
-
-
-\question \qt{Student t-distribution}
-The distribution of Student's t, $t=\bar x/(\sigma_x/\sqrt{n})$, the
-estimated mean $\bar x$ of a data set of size $n$ divided by the
-estimated standard error of the mean $\sigma_x/\sqrt{n}$, where
-$\sigma_x$ is the estimated standard deviation, is not a normal
-distribution but a Student-t distribution.  We want to compute the
-Student-t distribution and compare it with the normal distribution.
-\begin{parts}
-\part Generate 100000 normally distributed random numbers.
-\part Draw from these data 1000 samples of size $n=3$, 5, 10, and
-50. For each sample size $n$ ...
-\part ... compute the mean $\bar x$ of the samples and plot the
-probability density of these means.
-\part ... compare the resulting probability densities with corresponding
-normal distributions.
-\part ... compute Student's $t=\bar x/(\sigma_x/\sqrt{n})$ and compare its
-distribution with the normal distribution with standard deviation of
-one. Is $t$ normally distributed? Under which conditions is $t$
-normally distributed?
-\end{parts}
-\newsolutionpage
-\begin{solution}
-  \lstinputlisting{tdistribution.m}
-  \includegraphics[width=1\textwidth]{tdistribution-n03}\\
-  \includegraphics[width=1\textwidth]{tdistribution-n05}\\
-  \includegraphics[width=1\textwidth]{tdistribution-n10}\\
-  \includegraphics[width=1\textwidth]{tdistribution-n50}
-\end{solution}
-
-
-\continue
-\question \qt{Permutation test} \label{permutationtest}
-We want to compute the significance of a correlation by means of a permutation test.
-\begin{parts}
-  \part \label{permutationtestdata} Generate 1000 correlated pairs
-  $x$, $y$ of random numbers according to:
-\begin{verbatim}
-n = 1000
-a = 0.2;
-x = randn(n, 1);
-y = randn(n, 1) + a*x;
-\end{verbatim}
-  \part Generate a scatter plot of the two variables.
-  \part Why is $y$ correlated with $x$?
-  \part Compute the correlation coefficient between $x$ and $y$.
-  \part What do you need to do in order to destroy the correlations between the $x$-$y$ pairs?
-  \part Do exactly this 1000 times and compute each time the correlation coefficient.
-  \part Compute and plot the probability density of these correlation
-  coefficients.
-  \part Is the correlation of the original data set significant?
-  \part What does ``significance of the correlation'' mean?
-%  \part Vary the sample size \code{n} and compute in the same way the
-%  significance of the correlation.
-\end{parts}
-\begin{solution}
-  \lstinputlisting{correlationsignificance.m}
-  \includegraphics[width=1\textwidth]{correlationsignificance}
-\end{solution}
-
-\question \qt{Bootstrap the correlation coefficient} 
-The permutation test generates the distribution of the null hypothesis
-of uncorrelated data and we check whether the correlation coefficient
-of the data differs significantly from this
-distribution. Alternatively we can bootstrap the data while keeping
-the pairs and determine the confidence interval of the correlation
-coefficient of the data. If this differs significantly from a
-correlation coefficient of zero we can conclude that the correlation
-coefficient of the data indeed quantifies correlated data.
-
-We take the same data set that we have generated in exercise
-\ref{permutationtest} (\ref{permutationtestdata}).
-\begin{parts}
-  \part Bootstrap 1000 times the correlation coefficient from the data.
-  \part Compute and plot the probability density of these correlation
-  coefficients.
-  \part Is the correlation of the original data set significant?
-\end{parts}
-\begin{solution}
-  \lstinputlisting{correlationbootstrap.m}
-  \includegraphics[width=1\textwidth]{correlationbootstrap}
-\end{solution}
-
-\end{questions}
-
-\end{document}

bootstrap/exercises/instructions.tex

@@ -1,6 +0,0 @@
-\vspace*{-7.8ex}
-\begin{center}
-\textbf{\Large Introduction to Scientific Computing}\\[2.3ex]
-{\large Jan Grewe, Jan Benda}\\[-3ex]
-Neuroethology Lab \hfill --- \hfill Institute for Neurobiology \hfill --- \hfill \includegraphics[width=0.28\textwidth]{UT_WBMW_Black_RGB} \\
-\end{center}

bootstrap/exercises/meandiffpermutation.m

@@ -0,0 +1,29 @@
+function [md, ds, dq] = meandiffpermutation(x, y, nperm, alpha)
+% Permutation test for difference of means of two independent samples.
+%
+% [md, ds, dq] = meandiffpermutation(x, y, nperm, alpha);
+%
+% Arguments:
+%   x: vector with the samples of the x data set.
+%   y: vector with the samples of the y data set.
+%   nperm: number of permutations run.
+%   alpha: significance level.
+%
+% Returns:
+%   md: difference of the means 
+%   ds: vector containing the differences of the means of the resampled data sets
+%   dq: difference of the means at a significance of alpha.
+
+  md = mean(x) - mean(y);             % measured difference
+  xy = [x; y];                        % merge data sets
+  % permutations:
+  ds = zeros(nperm, 1);
+  for i = 1:nperm
+      xyr = xy(randperm(length(xy))); % shuffle xy
+      xr = xyr(1:length(x));          % random x sample
+      yr = xyr(length(x)+1:end);      % random y sample
+      ds(i) = mean(xr) - mean(yr);
+  end
+  % significance:
+  dq = quantile(ds, 1.0 - alpha);
+end

bootstrap/exercises/meandiffplot.m

@@ -0,0 +1,42 @@
+function meandiffplot(x, y, md, ds, dq, k, nrows)
+% Plot histogram of data sets and of null hypothesis for differences in mean.
+%
+% meandiffplot(x, y, md, ds, dq, k, rows);
+%
+% Arguments:
+%   x: vector with the samples of the x data set.
+%   y: vector with the samples of the y data set.
+%   md: difference of means of the two data sets.
+%   ds: vector containing the differences of the means of the resampled data sets
+%   dq: minimum difference of the means considered significant.
+%   k: current row for the plot panels.
+%   nrows: number of rows of panels in the figure.
+
+  %% (b) plot histograms:
+  subplot(nrows, 2, k*2-1);
+  bmin = min([x; y]);
+  bmax = max([x; y]);
+  bins = bmin:(bmax-bmin)/20.0:bmax;
+  [xh, b] = hist(x, bins);
+  [yh, b] = hist(y, bins);
+  bar(bins, xh, 'facecolor', 'b')
+  hold on
+  bar(bins, yh, 'facecolor', 'r');
+  xlabel('x and y [mV]')
+  ylabel('counts')
+  hold off
+
+  %% (f) pdf of the differences:
+  [h, b] = hist(ds, 20);
+  h = h/sum(h)/(b(2)-b(1));           % normalization
+
+  %% plot:
+  subplot(nrows, 2, k*2)
+  bar(b, h, 'facecolor', 'b');
+  hold on;
+  bar(b(b>=dq), h(b>=dq), 'facecolor', 'r');
+  plot([md md], [0 4], 'r', 'linewidth', 2);
+  xlabel('Difference of means [mV]');
+  ylabel('pdf of H0');
+  hold off;
+end

bootstrap/exercises/meandiffsignificance.m

@@ -0,0 +1,37 @@
+n = 200;
+mx = -40.0;
+nperm = 1000;
+alpha = 0.05;
+
+%% (h) repeat for various means of the y-data set:
+mys = [-40.1, -40.2, -40.5];
+for k=1:length(mys)
+
+  %% (a) generate data:
+  my = mys(k);
+  x = randn(n, 1) + mx;
+  y = randn(n, 1) + my;
+
+  %% (d), (e) permutation test:
+  [md, ds, dq] = meandiffpermutation(x, y, nperm, alpha);
+
+  %% (c) difference of means:
+  fprintf('\nmean x = %.1fmV, mean y = %.1fmV\n', mx, my);
+  fprintf('  difference of means = %.2fmV\n', md);
+
+  %% (g) significance:
+  fprintf('  difference of means at 5%% significance = %.2fmV\n', dq);
+  if md >= dq
+      fprintf('  --> measured difference of means is significant\n');
+  else
+      fprintf('  --> measured difference of means is not significant\n');
+  end
+
+  %% (b), (f) plot histograms of data and pdf of differences:
+  meandiffplot(x, y, md, ds, dq, k, length(mys));
+
+  subplot(length(mys), 2, k*2-1);
+  title(sprintf('mx=%.1fmV, my=%.1fmV', mx, my))
+end
+
+savefigpdf(gcf, 'meandiffsignificance.pdf', 12, 10);

bootstrap/exercises/resampling-1.tex

@@ -0,0 +1,116 @@
+\documentclass[12pt,a4paper,pdftex]{exam}
+
+\newcommand{\exercisetopic}{Resampling}
+\newcommand{\exercisenum}{8}
+\newcommand{\exercisedate}{December 14th, 2020}
+
+\input{../../exercisesheader}
+
+\firstpagefooter{Prof. Dr. Jan Benda}{}{jan.benda@uni-tuebingen.de}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{document}
+
+\input{../../exercisestitle}
+
+\begin{questions}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\question \qt{Read chapter 7 of the script on ``resampling methods''!}\vspace{-3ex}
+
+\question \qt{Permutation test of correlations} \label{correlationtest}
+We want to compute the significance of a correlation by means of a permutation test.
+\begin{parts}
+  \part \label{correlationtestdata} Generate 1000 correlated pairs
+  $x$, $y$ of random numbers according to:
+\begin{verbatim}
+n = 1000
+a = 0.2;
+x = randn(n, 1);
+y = randn(n, 1) + a*x;
+\end{verbatim}
+  \part Generate a scatter plot of the two variables.
+  \part Why is $y$ correlated with $x$?
+  \part Compute the correlation coefficient between $x$ and $y$.
+  \part What do you need to do in order to destroy the correlations between the $x$-$y$ pairs?
+  \part Do exactly this 1000 times and compute each time the correlation coefficient.
+  \part Compute and plot the probability density of these correlation
+  coefficients.
+  \part Is the correlation of the original data set significant?
+  \part What does ``significance of the correlation'' mean?
+%  \part Vary the sample size \code{n} and compute in the same way the
+%  significance of the correlation.
+\end{parts}
+\begin{solution}
+  \lstinputlisting{correlationsignificance.m}
+  \includegraphics[width=1\textwidth]{correlationsignificance}
+\end{solution}
+
+\newsolutionpage
+\question \qt{Bootstrap the correlation coefficient} 
+The permutation test generates the distribution of the null hypothesis
+of uncorrelated data and we check whether the correlation coefficient
+of the data differs significantly from this
+distribution. Alternatively we can bootstrap the data while keeping
+the pairs and determine the confidence interval of the correlation
+coefficient of the data. If this differs significantly from a
+correlation coefficient of zero we can conclude that the correlation
+coefficient of the data indeed quantifies correlated data.
+
+We take the same data set that we have generated in exercise
+\ref{correlationtest} (\ref{correlationtestdata}).
+\begin{parts}
+  \part Bootstrap 1000 times the correlation coefficient from the
+  data, i.e.  generate bootstrap data by randomly resampling the
+  original data pairs with replacement. Use the \code{randi()}
+  function for generating random indices that you can use to select a
+  random sample from the original data.
+  \part Compute and plot the probability density of these correlation
+  coefficients.
+  \part Is the correlation of the original data set significant?
+\end{parts}
+\begin{solution}
+  \lstinputlisting{correlationbootstrap.m}
+  \includegraphics[width=1\textwidth]{correlationbootstrap}
+\end{solution}
+
+
+\continue 
+\question \qt{Permutation test of difference of means}
+We want to test whether two data sets come from distributions that
+differ in their mean by means of a permutation test.
+\begin{parts}
+  \part Generate two normally distributed data sets $x$ and $y$
+  containing each $n=200$ samples. Let's assume the $x$ samples are
+  measurements of the membrane potential of a mammalian photoreceptor
+  in darkness with a mean of $-40$\,mV and a standard deviation of
+  1\,mV. The $y$ values are the membrane potentials measured under dim
+  illumination and come from a distribution with the same standard
+  deviation and a mean of $-40.5$\,mV. See section 5.2 ``Scaling and
+  shifting random numbers'' in the script.
+  \part Plot histograms of the $x$ and $y$ data in a single
+  plot. Choose appropriate bins.
+  \part Compute the means of $x$ and $y$ and their difference.
+  \part The null hypothesis is that the $x$ and $y$ data come from the
+  same distribution. How can you generate new samples $x_r$ and $y_r$
+  from the original data that come from the same distribution?
+  \part Do exactly this 1000 times and compute each time the
+  difference of the means of the two resampled samples.
+  \part Compute and plot the probability density of the resulting
+  distribution of the null hypothesis.
+  \part Is the difference of the means of the original data sets significant?
+  \part Repeat this procedure for $y$ samples that are closer or
+  further apart from the mean of the $x$ data set. For this put the
+  computations of the permuation test in a function and all the plotting
+  in another function.
+\end{parts}
+\begin{solution}
+  \lstinputlisting{meandiffpermutation.m}
+  \lstinputlisting{meandiffplot.m}
+  \lstinputlisting{meandiffsignificance.m}
+  \includegraphics[width=1\textwidth]{meandiffsignificance}
+\end{solution}
+
+\end{questions}
+
+\end{document}

bootstrap/exercises/resampling-2.tex

@@ -0,0 +1,84 @@
+\documentclass[12pt,a4paper,pdftex]{exam}
+
+\newcommand{\exercisetopic}{Resampling}
+\newcommand{\exercisenum}{X2}
+\newcommand{\exercisedate}{December 14th, 2020}
+
+\input{../../exercisesheader}
+
+\firstpagefooter{Prof. Dr. Jan Benda}{}{jan.benda@uni-tuebingen.de}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{document}
+
+\input{../../exercisestitle}
+
+\begin{questions}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\question \qt{Read chapter 7 of the script on ``resampling methods''!}\vspace{-3ex}
+
+\question \qt{Bootstrap the standard error of the mean}
+We want to compute the standard error of the mean of a data set by
+means of the bootstrap method and compare the result with the formula
+``standard deviation divided by the square-root of $n$''.
+\begin{parts}
+  \part Download the file \code{thymusglandweights.dat} from Ilias.
+  This is a data set of the weights of the thymus glands of 14-day old chicken embryos
+  measured in milligram.
+  \part Load the data into Matlab (\code{load} function).
+  \part Compute histogram, mean, and standard error of the mean of the first 80 data points.
+  \part Compute the standard error of the mean of the first 80 data
+  points by bootstrapping the data 500 times. Write a function that
+  bootstraps the standard error of the mean of a given data set. The
+  function should also return a vector with the bootstrapped means.
+  \part Compute the 95\,\% confidence interval for the mean from the
+  bootstrap distribution (\code{quantile()} function) --- the
+  interval that contains the true mean with 95\,\% probability.
+  \part Use the whole data set and the bootstrap method for computing
+  the dependence of the standard error of the mean from the sample
+  size $n$.
+  \part Compare your result with the formula for the standard error
+  $\sigma/\sqrt{n}$.
+\end{parts}
+\begin{solution}
+  \lstinputlisting{bootstrapmean.m}
+  \lstinputlisting{bootstraptymus.m}
+  \includegraphics[width=0.5\textwidth]{bootstraptymus-datahist}
+  \includegraphics[width=0.5\textwidth]{bootstraptymus-meanhist}
+  \includegraphics[width=0.5\textwidth]{bootstraptymus-samples}
+\end{solution}
+
+
+\question \qt{Student t-distribution}
+The distribution of Student's t, $t=\bar x/(\sigma_x/\sqrt{n})$, the
+estimated mean $\bar x$ of a data set of size $n$ divided by the
+estimated standard error of the mean $\sigma_x/\sqrt{n}$, where
+$\sigma_x$ is the estimated standard deviation, is not a normal
+distribution but a Student-t distribution.  We want to compute the
+Student-t distribution and compare it with the normal distribution.
+\begin{parts}
+\part Generate 100000 normally distributed random numbers.
+\part Draw from these data 1000 samples of size $n=3$, 5, 10, and
+50. For each sample size $n$ ...
+\part ... compute the mean $\bar x$ of the samples and plot the
+probability density of these means.
+\part ... compare the resulting probability densities with corresponding
+normal distributions.
+\part ... compute Student's $t=\bar x/(\sigma_x/\sqrt{n})$ and compare its
+distribution with the normal distribution with standard deviation of
+one. Is $t$ normally distributed? Under which conditions is $t$
+normally distributed?
+\end{parts}
+\newsolutionpage
+\begin{solution}
+  \lstinputlisting{tdistribution.m}
+  \includegraphics[width=1\textwidth]{tdistribution-n03}\\
+  \includegraphics[width=1\textwidth]{tdistribution-n05}\\
+  \includegraphics[width=1\textwidth]{tdistribution-n10}\\
+  \includegraphics[width=1\textwidth]{tdistribution-n50}
+\end{solution}
+
+\end{questions}
+
+\end{document}

bootstrap/exercises/savefigpdf.m

@@ -1,28 +1,28 @@
-function savefigpdf( fig, name, width, height )
+function savefigpdf(fig, name, width, height)
 % Saves figure fig in pdf file name.pdf with appropriately set page size
 % and fonts
 
-% default width:
-if nargin < 3
-    width = 11.7;
-end
-% default height:
-if nargin < 4
-    height = 9.0;
-end
+  % default width:
+  if nargin < 3
+      width = 11.7;
+  end
+  % default height:
+  if nargin < 4
+      height = 9.0;
+  end
 
-% paper:
-set( fig, 'PaperUnits', 'centimeters' );
-set( fig, 'PaperSize', [width height] );
-set( fig, 'PaperPosition', [0.0 0.0 width height] );
-set( fig, 'Color', 'white')
+  % paper:
+  set(fig, 'PaperUnits', 'centimeters');
+  set(fig, 'PaperSize', [width height]);
+  set(fig, 'PaperPosition', [0.0 0.0 width height]);
+  set(fig, 'Color', 'white')
 
-% font:
-set( findall( fig, 'type', 'axes' ), 'FontSize', 12 )
-set( findall( fig, 'type', 'text' ), 'FontSize', 12 )
+  % font:
+  set(findall(fig, 'type', 'axes'), 'FontSize', 12)
+  set(findall(fig, 'type', 'text'), 'FontSize', 12)
 
-% save:
-saveas( fig, name, 'pdf' )
+  % save:
+  saveas(fig, name, 'pdf')
 
 end
 

bootstrap/exercises/tdistribution.m

@@ -1,10 +1,10 @@
 %% (a) generate random numbers:
 n = 100000;
-x=randn(n, 1);
+x = randn(n, 1);
 
-for nsamples=[3 5 10 50]
+for nsamples = [3 5 10 50]
     nsamples
-    %% compute mean, standard deviation and t:
+    % compute mean, standard deviation and t:
     nmeans = 10000;
     means = zeros(nmeans, 1);
     sdevs = zeros(nmeans, 1);
@@ -19,14 +19,14 @@ for nsamples=[3 5 10 50]
     % Gaussian pdfs:
     msdev = std(means);
     tsdev = 1.0;
-    dxg=0.01;
+    dxg = 0.01;
     xmax = 10.0;
     xmin = -xmax;
     xg = [xmin:dxg:xmax];
     pm = exp(-0.5*(xg/msdev).^2)/sqrt(2.0*pi)/msdev;
     pt = exp(-0.5*(xg/tsdev).^2)/sqrt(2.0*pi)/tsdev;
     
-    %% plots
+    % plots:
     subplot(1, 2, 1)
     bins = xmin:0.2:xmax;
     [h,b] = hist(means, bins);
@@ -35,7 +35,7 @@ for nsamples=[3 5 10 50]
     hold on
     plot(xg, pm, 'r', 'linewidth', 2)
     title(sprintf('sample size = %d', nsamples));
-    xlim( [-3, 3] );
+    xlim([-3, 3]);
     xlabel('Mean');
     ylabel('pdf');
     hold off;
@@ -48,11 +48,11 @@ for nsamples=[3 5 10 50]
     hold on
     plot(xg, pt, 'r', 'linewidth', 2)
     title(sprintf('sample size = %d', nsamples));
-    xlim( [-8, 8] );
+    xlim([-8, 8]);
     xlabel('Student-t');
     ylabel('pdf');
     hold off;
     
     savefigpdf(gcf, sprintf('tdistribution-n%02d.pdf', nsamples), 14, 5);
-    pause( 3.0 )
+    pause(3.0)
 end

bootstrap/lecture/bootstrap-chapter.tex

@@ -10,6 +10,8 @@
 \setcounter{page}{\pagenumber}
 \setcounter{chapter}{\chapternumber}
 
+\renewcommand{\exercisesolutions}{here}  % 0: here, 1: chapter, 2: end
+
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \begin{document} 
@@ -17,10 +19,20 @@
 \include{bootstrap}
 
 \section{TODO}
+
 This chapter easily covers two lectures:
 \begin{itemize}
 \item 1. Bootstrapping with a proper introduction of of confidence intervals
-\item 2. Permutation test with a proper introduction of statistical tests (dsitribution of nullhypothesis, significance, power, etc.)
+\item 2. Permutation test with a proper introduction of statistical tests (distribution of nullhypothesis, significance, power, etc.)
+\end{itemize}
+ToDo:
+\begin{itemize}
+\item Add jacknife methods to bootstrapping
+\item Add discussion of confidence intervals to descriptive statistics chapter
+\item Have a separate chapter on statistical tests before. What is the
+  essence of a statistical test (null hypothesis distribution), power
+  analysis, and a few examples of existing functions for statistical
+  tests.
 \end{itemize}
 
 \end{document}

bootstrap/lecture/bootstrap.tex

@@ -1,20 +1,28 @@
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\chapter{Bootstrap methods}
+\chapter{Resampling methods}
 \label{bootstrapchapter}
-\exercisechapter{Bootstrap methods}
+\exercisechapter{Resampling methods}
 
-Bootstrapping methods are applied to create distributions of
-statistical measures via resampling of a sample. Bootstrapping offers several
-advantages:
+
+\entermde{Resampling methoden}{Resampling methods} are applied to
+generate distributions of statistical measures via resampling of
+existing samples. Resampling offers several advantages:
 \begin{itemize}
 \item Fewer assumptions (e.g. a measured sample does not need to be
   normally distributed).
 \item Increased precision as compared to classical methods. %such as?
-\item General applicability: the bootstrapping methods are very
+\item General applicability: the resampling methods are very
   similar for different statistics and there is no need to specialize
   the method to specific statistic measures.
 \end{itemize}
+Resampling methods can be used for both estimating the precision of
+estimated statistics (e.g. standard error of the mean, confidence
+intervals) and testing for significane.
+
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\section{Bootstrapping}
 
 \begin{figure}[tp]
   \includegraphics[width=0.8\textwidth]{2012-10-29_16-26-05_771}\\[2ex]
@@ -84,19 +92,20 @@ of the statistical population. We can use the bootstrap distribution
 to draw conclusion regarding the precision of our estimation (e.g.
 standard errors and confidence intervals).
 
-Bootstrapping methods create bootstrapped samples from a SRS by
+Bootstrapping methods generate bootstrapped samples from a SRS by
 resampling. The bootstrapped samples are used to estimate the sampling
 distribution of a statistical measure. The bootstrapped samples have
-the same size as the original sample and are created by randomly
+the same size as the original sample and are generated by randomly
 drawing with replacement. That is, each value of the original sample
-can occur once, multiple time, or not at all in a bootstrapped
+can occur once, multiple times, or not at all in a bootstrapped
 sample. This can be implemented by generating random indices into the
 data set using the \code{randi()} function.
 
 
-\section{Bootstrap of the standard error}
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\subsection{Bootstrap the standard error}
 
-Bootstrapping can be nicely illustrated at the example of the
+Bootstrapping can be nicely illustrated on the example of the
 \enterm{standard error} of the mean (\determ{Standardfehler}). The
 arithmetic mean is calculated for a simple random sample. The standard
 error of the mean is the standard deviation of the expected
@@ -116,11 +125,10 @@ population.
     the standard error of the mean.}
 \end{figure}
 
-Via bootstrapping we create a distribution of the mean values
+Via bootstrapping we generate a distribution of mean values
 (\figref{bootstrapsemfig}) and the standard deviation of this
-distribution is the standard error of the mean.
+distribution is the standard error of the sample mean.
 
-\pagebreak[4]
 \begin{exercise}{bootstrapsem.m}{bootstrapsem.out}
   Create the distribution of mean values from bootstrapped samples
   resampled from a single SRS. Use this distribution to estimate the
@@ -138,68 +146,166 @@ distribution is the standard error of the mean.
 \end{exercise}
 
 
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \section{Permutation tests}
+
 Statistical tests ask for the probability of a measured value to
 originate from a null hypothesis. Is this probability smaller than the
 desired \entermde{Signifikanz}{significance level}, the
-\entermde{Nullhypothese}{null hypothesis} may be rejected.
+\entermde{Nullhypothese}{null hypothesis} can be rejected.
 
 Traditionally, such probabilities are taken from theoretical
-distributions which are based on assumptions about the data.  Thus the
-applied statistical test has to be appropriate for the type of
-data. An alternative approach is to calculate the probability density
-of the null hypothesis directly from the data itself. To do this, we
-need to resample the data according to the null hypothesis from the
-SRS. By such permutation operations we destroy the feature of interest
-while we conserve all other statistical properties of the data.
+distributions which have been derived based on some assumptions about
+the data. For example, the data should be normally distributed.  Given
+some data one has to find an appropriate test that matches the
+properties of the data. An alternative approach is to calculate the
+probability density of the null hypothesis directly from the data
+themselves. To do so, we need to resample the data according to the
+null hypothesis from the SRS. By such permutation operations we
+destroy the feature of interest while conserving all other statistical
+properties of the data.
+
+
+\subsection{Significance of a difference in the mean}
+
+Often we would like to know whether two data sets differ in their
+mean. Whether the ears of foxes are larger in southern Europe compared
+to the ones from Scandinavia, whether a drug decreases blood pressure
+in humans, whether a sensory stimulus increases the firing rate of a
+neuron, etc. The \entermde{Nullhypothese}{null hypothesis} is that
+they do not differ in their means, i.e. that both data sets come from
+the same distribution. But even if the two data sets come from the
+same distribution, their sample means may nevertheless differ by
+chance. We need to figure out how these differences of the means are
+distributed. Only if the measured difference between the means is
+significantly larger than the ones obtained by chance we can reject
+the null hypothesis and consider the two data sets to differ
+significantly in their means.
+
+We can easily estimate the distribution of the null hypothesis by
+putting the data of both data sets in one big bag. By merging the two
+data sets we assume that all the data values come from the same
+distribution. We then randomly separate the data values into two new
+data sets. These random data sets contain data from both original data
+sets and thus come from the same distribution. From these random data
+sets we compute the difference of their sample means. This procedure
+is repeated many, say one thousand, times and each time we get a value
+for a difference of means. The distribution of these values is the
+distribution of the null hypothesis. It is the distribution of
+differences of mean values that we get by chance although the two data
+sets come from the same distribution. For a one-sided test that checks
+whether the measured difference of means is significantly larger than
+zero at a significance level of 5\,\% we compute the value of the
+95\,\% percentile of the null distribution. If the measured value is
+larger, we can reject the null hypothesis and consider the two data
+sets to differ significantly in their means.
+
+By using the original data to estimate the null hypothesis, we make no
+assumption about the properties of the data. We do not need to worry
+about the data being normally distributed. We do not need to memorize
+which test to use in which situation. And we better understand what we
+are testing, because we design the test ourselves. Nowadays, computer
+are powerful enough to iterate even ten thousand times over the data
+to compute the distribution of the null hypothesis --- with only a few
+lines of code. This is why \entermde{Permutationstest}{permutaion
+  test} are getting quite popular.
 
 \begin{figure}[tp]
-  \includegraphics[width=1\textwidth]{permutecorrelation}
-  \titlecaption{\label{permutecorrelationfig}Permutation test for
-    correlations.}{Let the correlation coefficient of a dataset with
-    200 samples be $\rho=0.21$. The distribution of the null
-    hypothesis (yellow), optained from the correlation coefficients of
-    permuted and therefore uncorrelated datasets is centered around
-    zero. The measured correlation coefficient is larger than the
-    95\,\% percentile of the null hypothesis. The null hypothesis may
-    thus be rejected and the measured correlation is considered
-    statistically significant.}
+  \includegraphics[width=1\textwidth]{permuteaverage}
+  \titlecaption{\label{permuteaverage}Permutation test for differences
+    in means.}{We want to test whether two datasets
+    $\left\{x_i\right\}$ (red) and $\left\{y_i\right\}$ (blue) come
+    from different distributions by assessing the significance of the
+    difference in their sample means. The data sets were generated
+    with a difference in their population means of $d=0.7$. For
+    generating the distribution of the null hypothesis, i.e. the
+    distribution of differences in the means if the two data sets come
+    from the same distribution, we randomly select the same number of
+    samples from both data sets (top right). This is repeated many
+    times and results in the desired distribution of differences of
+    means (bottom). The measured difference is clearly beyond the
+    95\,\% percentile of this distribution and thus indicates a
+    significant difference between the distributions of the two
+    original data sets.}
 \end{figure}
 
-A good example for the application of a
-\entermde{Permutationstest}{permutaion test} is the statistical
-assessment of \entermde[correlation]{Korrelation}{correlations}. Given
-are measured pairs of data points $(x_i, y_i)$. By calculating the
+\begin{exercise}{meandiffsignificance.m}{meandiffsignificance.out}
+Estimate the statistical significance of a difference in the mean of two data sets.
+\vspace{-1ex}
+\begin{enumerate}
+\item Generate two independent data sets, $\left\{x_i\right\}$ and
+  $\left\{y_i\right\}$, of $n=200$ samples each, by drawing random
+  numbers from a normal distribution. Add 0.2 to all the $y_i$ samples
+  to ensure the population means to differ by 0.2.
+\item Calculate the difference between the sample means of the two data sets.
+\item Estimate the distribution of the null hypothesis of no
+  difference of the means by generating new data sets with the same
+  number of samples randomly selected from both data sets.  For this
+  lump the two data sets together into a single vector. Then permute
+  the order of the elements in this vector using the function
+  \varcode{randperm()}, split it into two data sets and calculate
+  the difference of their means. Repeat this 1000 times.
+\item Read out the 95\,\% percentile from the resulting distribution
+  of the differences in the mean, the null hypothesis using the
+  \varcode{quantile()} function, and compare it with the difference of
+  means measured from the original data sets.
+\end{enumerate}
+\end{exercise}
+
+
+\subsection{Significance of correlations}
+
+Another nice example for the application of a
+\entermde{Permutationstest}{permutaion test} is testing for
+significant \entermde[correlation]{Korrelation}{correlations}
+(figure\,\ref{permutecorrelationfig}). Given are measured pairs of
+data points $(x_i, y_i)$. By calculating the
 \entermde[correlation!correlation
-coefficient]{Korrelationskoeffizient}{correlation
-  coefficient} we can quantify how strongly $y$ depends on $x$. The
-correlation coefficient alone, however, does not tell whether the
-correlation is significantly different from a random correlation. The
-\entermde{Nullhypothese}{null hypothesis} for such a situation is that
-$y$ does not depend on $x$. In order to perform a permutation test, we
-need to destroy the correlation by permuting the $(x_i, y_i)$ pairs,
-i.e. we rearrange the $x_i$ and $y_i$ values in a random
+coefficient]{Korrelationskoeffizient}{correlation coefficient} we can
+quantify how strongly $y$ depends on $x$. The correlation coefficient
+alone, however, does not tell whether the correlation is significantly
+different from a non-zero correlation that we might get although there
+is no true correlation in the data. The \entermde{Nullhypothese}{null
+  hypothesis} for such a situation is that $y$ does not depend on
+$x$. In order to perform a permutation test, we need to destroy the
+correlation between the data pairs by permuting the $(x_i, y_i)$
+pairs, i.e. we rearrange the $x_i$ and $y_i$ values in a random
 fashion. Generating many sets of random pairs and computing the
-resulting correlation coefficients yields a distribution of
-correlation coefficients that result randomly from uncorrelated
+corresponding correlation coefficients yields a distribution of
+correlation coefficients that result randomly from truly uncorrelated
 data. By comparing the actually measured correlation coefficient with
 this distribution we can directly assess the significance of the
-correlation (figure\,\ref{permutecorrelationfig}).
+correlation.
+
+\begin{figure}[tp]
+  \includegraphics[width=1\textwidth]{permutecorrelation}
+  \titlecaption{\label{permutecorrelationfig}Permutation test for
+    correlations.}{Let the correlation coefficient of a dataset with
+    200 samples be $\rho=0.21$ (top left). By shuffling the data pairs
+    we destroy any true correlation (top right). The resulting
+    distribution of the null hypothesis (bottm, yellow), optained from
+    the correlation coefficients of permuted and therefore
+    uncorrelated datasets is centered around zero. The measured
+    correlation coefficient is larger than the 95\,\% percentile of
+    the null hypothesis. The null hypothesis may thus be rejected and
+    the measured correlation is considered statistically significant.}
+\end{figure}
 
 \begin{exercise}{correlationsignificance.m}{correlationsignificance.out}
 Estimate the statistical significance of a correlation coefficient.
 \begin{enumerate}
-\item Create pairs of $(x_i, y_i)$ values. Randomly choose $x$-values
+\item Generate pairs of $(x_i, y_i)$ values. Randomly choose $x$-values
   and calculate the respective $y$-values according to $y_i =0.2 \cdot x_i + u_i$
   where $u_i$ is a random number drawn from a normal distribution.
 \item Calculate the correlation coefficient.
-\item Generate the distribution of the null hypothesis by generating
+\item Estimate the distribution of the null hypothesis by generating
   uncorrelated pairs. For this permute $x$- and $y$-values
   \matlabfun{randperm()} 1000 times and calculate for each permutation
   the correlation coefficient.
 \item Read out the 95\,\% percentile from the resulting distribution
-  of the null hypothesis and compare it with the correlation
-  coefficient computed from the original data.
+  of the null hypothesis using the \varcode{quantile()} function and
+  compare it with the correlation coefficient computed from the
+  original data.
 \end{enumerate}
 \end{exercise}
 

bootstrap/lecture/bootstrapsem.py

@@ -24,7 +24,7 @@ for i in range(nresamples) :
     musrs.append(np.mean(rng.randn(nsamples)))
 hmusrs, _ = np.histogram(musrs, bins, density=True)
 
-fig, ax = plt.subplots(figsize=cm_size(figure_width, 1.2*figure_height))
+fig, ax = plt.subplots(figsize=cm_size(figure_width, 1.05*figure_height))
 fig.subplots_adjust(**adjust_fs(left=4.0, bottom=2.7, right=1.5))
 ax.set_xlabel('Mean')
 ax.set_xlim(-0.4, 0.4)

bootstrap/lecture/permuteaverage.py

@@ -0,0 +1,103 @@
+import numpy as np
+import scipy.stats as st
+import matplotlib.pyplot as plt
+import matplotlib.gridspec as gridspec
+import matplotlib.ticker as ticker
+from plotstyle import *
+
+rng = np.random.RandomState(637281)
+
+# generate data that differ in their mein by d:
+n = 200
+d = 0.7
+x = rng.randn(n) + d;
+y = rng.randn(n);
+#x = rng.exponential(1.0, n);
+#y = rng.exponential(1.0, n) + d;
+
+# histogram of data:
+db = 0.5
+bins = np.arange(-2.5, 2.6, db)
+hx, _ = np.histogram(x, bins)
+hy, _ = np.histogram(y, bins)
+
+# Diference of means, pooled standard deviation and Cohen's d:
+ad = np.mean(x)-np.mean(y)
+s = np.sqrt(((len(x)-1)*np.var(x)+(len(y)-1)*np.var(y))/(len(x)+len(y)-2))
+cd = ad/s
+
+# permutation:
+nperm = 1000
+ads = []
+xy = np.hstack((x, y))
+for i in range(nperm) :
+    xyp = rng.permutation(xy)
+    ads.append(np.mean(xyp[:len(x)])-np.mean(xyp[len(x):]))
+
+# histogram of shuffled data:
+hxp, _ = np.histogram(xyp[:len(x)], bins)
+hyp, _ = np.histogram(xyp[len(x):], bins)
+    
+# pdf of the differences of means:
+h, b = np.histogram(ads, 20, density=True)
+
+# significance:
+dq = np.percentile(ads, 95.0)
+print('Measured difference of means = %.2f, difference at 95%% percentile of permutation = %.2f' % (ad, dq))
+da = 1.0-0.01*st.percentileofscore(ads, ad)
+print('Measured difference of means %.2f is at %.2f%% percentile of permutation' % (ad, 100.0*da))
+
+ap, at = st.ttest_ind(x, y)
+print('Measured difference of means %.2f is at %.2f%% percentile of test' % (ad, ap))
+
+
+fig = plt.figure(figsize=cm_size(figure_width, 1.8*figure_height))
+gs = gridspec.GridSpec(nrows=2, ncols=2, wspace=0.35, hspace=0.8,
+                       **adjust_fs(fig, left=5.0, right=1.5, top=1.0, bottom=2.7))
+
+ax = fig.add_subplot(gs[0,0])
+ax.bar(bins[:-1]-0.25*db, hy, 0.5*db, **fsA)
+ax.bar(bins[:-1]+0.25*db, hx, 0.5*db, **fsB)
+ax.annotate('', xy=(0.0, 45.0), xytext=(d, 45.0), arrowprops=dict(arrowstyle='<->'))
+ax.text(0.5*d, 50.0, 'd=%.1f' % d, ha='center')
+ax.set_xlim(-2.5, 2.5)
+ax.set_ylim(0.0, 50)
+ax.yaxis.set_major_locator(ticker.MultipleLocator(20.0))
+ax.set_xlabel('Original x and y values')
+ax.set_ylabel('Counts')
+
+ax = fig.add_subplot(gs[0,1])
+ax.bar(bins[:-1]-0.25*db, hyp, 0.5*db, **fsA)
+ax.bar(bins[:-1]+0.25*db, hxp, 0.5*db, **fsB)
+ax.set_xlim(-2.5, 2.5)
+ax.set_ylim(0.0, 50)
+ax.yaxis.set_major_locator(ticker.MultipleLocator(20.0))
+ax.set_xlabel('Shuffled x and y values')
+ax.set_ylabel('Counts')
+
+ax = fig.add_subplot(gs[1,:])
+ax.annotate('Measured\ndifference\nis significant!',
+            xy=(ad, 1.2), xycoords='data',
+            xytext=(ad-0.1, 2.2), textcoords='data', ha='right',
+            arrowprops=dict(arrowstyle="->", relpos=(1.0,0.5),
+                connectionstyle="angle3,angleA=-20,angleB=100") )
+ax.annotate('95% percentile',
+            xy=(0.19, 0.9), xycoords='data',
+            xytext=(0.3, 5.0), textcoords='data', ha='left',
+            arrowprops=dict(arrowstyle="->", relpos=(0.1,0.0),
+                connectionstyle="angle3,angleA=40,angleB=80") )
+ax.annotate('Distribution of\nnullhypothesis',
+            xy=(-0.08, 3.0), xycoords='data',
+            xytext=(-0.22, 4.5), textcoords='data', ha='left',
+            arrowprops=dict(arrowstyle="->", relpos=(0.2,0.0),
+                connectionstyle="angle3,angleA=60,angleB=150") )
+ax.bar(b[:-1], h, width=b[1]-b[0], **fsC)
+ax.bar(b[:-1][b[:-1]>=dq], h[b[:-1]>=dq], width=b[1]-b[0], **fsB)
+ax.plot( [ad, ad], [0, 1], **lsA)
+ax.set_xlim(-0.25, 0.85)
+ax.set_ylim(0.0, 5.0)
+ax.yaxis.set_major_locator(ticker.MultipleLocator(2.0))
+ax.set_xlabel('Difference of means')
+ax.set_ylabel('PDF of H0')
+
+plt.savefig('permuteaverage.pdf')

bootstrap/lecture/permutecorrelation.py

@@ -1,9 +1,11 @@
 import numpy as np
 import scipy.stats as st
 import matplotlib.pyplot as plt
+import matplotlib.gridspec as gridspec
+import matplotlib.ticker as ticker
 from plotstyle import *
 
-rng = np.random.RandomState(637281)
+rng = np.random.RandomState(37281)
 
 # generate correlated data:
 n = 200
@@ -19,32 +21,56 @@ rd = np.corrcoef(x, y)[0, 1]
 nperm = 1000
 rs = []
 for i in range(nperm) :
-    xr=rng.permutation(x)
-    yr=rng.permutation(y)
-    rs.append( np.corrcoef(xr, yr)[0, 1] )
+    xr = rng.permutation(x)
+    yr = rng.permutation(y)
+    rs.append(np.corrcoef(xr, yr)[0, 1])
+rr = np.corrcoef(xr, yr)[0, 1]
 
 # pdf of the correlation coefficients:
 h, b = np.histogram(rs, 20, density=True)
 
 # significance:
 rq = np.percentile(rs, 95.0)
-print('Measured correlation coefficient = %.2f, correlation coefficient at 95%% percentile of bootstrap = %.2f' % (rd, rq))
+print('Measured correlation coefficient = %.2f, correlation coefficient at 95%% percentile of permutation = %.2f' % (rd, rq))
 ra = 1.0-0.01*st.percentileofscore(rs, rd)
-print('Measured correlation coefficient %.2f is at %.4f percentile of bootstrap' % (rd, ra))
+print('Measured correlation coefficient %.2f is at %.4f percentile of permutation' % (rd, ra))
 
 rp, ra = st.pearsonr(x, y)
 print('Measured correlation coefficient %.2f is at %.4f percentile of test' % (rp, ra))
 
-fig, ax = plt.subplots(figsize=cm_size(figure_width, 1.2*figure_height))
-fig.subplots_adjust(**adjust_fs(left=4.0, bottom=2.7, right=0.5, top=1.0))
+fig = plt.figure(figsize=cm_size(figure_width, 1.8*figure_height))
+gs = gridspec.GridSpec(nrows=2, ncols=2, wspace=0.35, hspace=0.8,
+                       **adjust_fs(fig, left=5.0, right=1.5, top=1.0, bottom=2.7))
+
+ax = fig.add_subplot(gs[0,0])
+ax.text(0.0, 4.0, 'r=%.2f' % rd, ha='center')
+ax.plot(x, y, **psAm)
+ax.set_xlim(-4.0, 4.0)
+ax.set_ylim(-4.0, 4.0)
+ax.xaxis.set_major_locator(ticker.MultipleLocator(2.0))
+ax.yaxis.set_major_locator(ticker.MultipleLocator(2.0))
+ax.set_xlabel('x')
+ax.set_ylabel('y')
+
+ax = fig.add_subplot(gs[0,1])
+ax.text(0.0, 4.0, 'r=%.2f' % rr, ha='center')
+ax.plot(xr, yr, **psAm)
+ax.set_xlim(-4.0, 4.0)
+ax.set_ylim(-4.0, 4.0)
+ax.xaxis.set_major_locator(ticker.MultipleLocator(2.0))
+ax.yaxis.set_major_locator(ticker.MultipleLocator(2.0))
+ax.set_xlabel('Shuffled x')
+ax.set_ylabel('Shuffled y')
+
+ax = fig.add_subplot(gs[1,:])
 ax.annotate('Measured\ncorrelation\nis significant!',
             xy=(rd, 1.1), xycoords='data',
-            xytext=(rd, 2.2), textcoords='data', ha='left',
-            arrowprops=dict(arrowstyle="->", relpos=(0.2,0.0),
+            xytext=(rd-0.01, 3.0), textcoords='data', ha='left',
+            arrowprops=dict(arrowstyle="->", relpos=(0.3,0.0),
                 connectionstyle="angle3,angleA=10,angleB=80") )
 ax.annotate('95% percentile',
             xy=(0.14, 0.9), xycoords='data',
-            xytext=(0.18, 4.0), textcoords='data', ha='left',
+            xytext=(0.16, 6.2), textcoords='data', ha='left',
             arrowprops=dict(arrowstyle="->", relpos=(0.1,0.0),
                 connectionstyle="angle3,angleA=30,angleB=80") )
 ax.annotate('Distribution of\nuncorrelated\nsamples',
@@ -56,7 +82,9 @@ ax.bar(b[:-1], h, width=b[1]-b[0], **fsC)
 ax.bar(b[:-1][b[:-1]>=rq], h[b[:-1]>=rq], width=b[1]-b[0], **fsB)
 ax.plot( [rd, rd], [0, 1], **lsA)
 ax.set_xlim(-0.25, 0.35)
+ax.set_ylim(0.0, 6.0)
+ax.yaxis.set_major_locator(ticker.MultipleLocator(2.0))
 ax.set_xlabel('Correlation coefficient')
-ax.set_ylabel('Prob. density of H0')
+ax.set_ylabel('PDF of H0')
 
 plt.savefig('permutecorrelation.pdf')

chapter.mk

@@ -10,6 +10,8 @@ pythonplots : $(PYPDFFILES)
 
 $(PYPDFFILES) : %.pdf: %.py ../../plotstyle.py
 	PYTHONPATH="../../" python3 $<
+#ps2pdf $@ $(@:.pdf=-temp.pdf)  # strip fonts, saves only about 1.5MB
+#mv $(@:.pdf=-temp.pdf) $@
 
 cleanpythonplots :
 	rm -f $(PYPDFFILES)
@@ -42,6 +44,9 @@ $(BASENAME)-chapter.pdf : $(BASENAME)-chapter.tex $(BASENAME).tex $(wildcard $(B
 watchchapter :
 	while true; do ! make -q chapter && make chapter; sleep 0.5; done
 
+watchplots :
+	while true; do ! make -q plots && make plots; sleep 0.5; done
+
 cleantex:
 	rm -f *~ 
 	rm -f $(BASENAME).aux $(BASENAME).log *.idx

codestyle/lecture/codestyle.tex

@@ -4,6 +4,7 @@
   more months might as well have been written by someone
   else.}{Eagleson's law}
 
+\noindent
 Cultivating a good code style is not just a matter of good taste but
 rather is a key ingredient for readability and maintainability of code
 and, in the end, facilitates reproducibility of scientific
@@ -204,7 +205,7 @@ random walk\footnote{A random walk is a simple simulation of Brownian
 (listing \ref{chaoticcode}) then in cleaner way (listing
 \ref{cleancode})
 
-\begin{lstlisting}[label=chaoticcode, caption={Chaotic implementation of the random-walk.}]
+\begin{pagelisting}[label=chaoticcode, caption={Chaotic implementation of the random-walk.}]
 num_runs = 10; max_steps = 1000;
 
 positions = zeros(max_steps, num_runs);
@@ -218,17 +219,14 @@ x = randn(1);
 if x<0
 positions(step, run)= positions(step-1, run)+1;
        
-       
 elseif x>0
  positions(step,run)=positions(step-1,run)-1;
  end
 end
 end
-\end{lstlisting}
-
-\pagebreak[4]
+\end{pagelisting}
 
-\begin{lstlisting}[label=cleancode, caption={Clean implementation of the random-walk.}]
+\begin{pagelisting}[label=cleancode, caption={Clean implementation of the random-walk.}]
 num_runs = 10;
 max_steps = 1000;
 positions = zeros(max_steps, num_runs);
@@ -243,7 +241,7 @@ for run = 1:num_runs
         end
     end
 end
-\end{lstlisting}
+\end{pagelisting}
 
 \section{Using comments}
 
@@ -275,7 +273,6 @@ avoided:\\ \varcode{ x = x + 2; \% add two to x}\\
     make it even clearer.}{Steve McConnell}
 \end{important}
 
-\pagebreak[4]
 \section{Documenting functions}
 All pre-defined \matlab{} functions begin with a comment block that
 describes the purpose of the function, the required and optional
@@ -335,8 +332,7 @@ used within the context of another function \matlab{} allows to define
 within the same file. Listing \ref{localfunctions} shows an example of
 a local function definition.
 
-\pagebreak[3] 
-\lstinputlisting[label=localfunctions, caption={Example for local
+\pageinputlisting[label=localfunctions, caption={Example for local
   functions.}]{calculateSines.m}
 
 \emph{Local function} live in the same \entermde{m-File}{m-file} as

debugging/lecture/debugging.tex

@@ -59,14 +59,14 @@ every opening parenthesis must be matched by a closing one or every
 respective error messages are clear and the editor will point out and
 highlight most syntax errors.
 
-\begin{lstlisting}[label=syntaxerror, caption={Unbalanced parenthesis error.}]
+\begin{pagelisting}[label=syntaxerror, caption={Unbalanced parenthesis error.}]
 >> mean(random_numbers
                       |
 Error: Expression or statement is incorrect--possibly unbalanced (, {, or [.
 
 Did you mean:
 >>   mean(random_numbers)
-\end{lstlisting}
+\end{pagelisting}
 
 \subsection{Indexing error}\label{index_error}
 Second on the list of common errors are the
@@ -125,7 +125,7 @@ dimensions. The command in line 7 works due to the fact, that matlab
 automatically extends the matrix, if you assign values to a range
 outside its bounds.
 
-\begin{lstlisting}[label=assignmenterror, caption={Assignment errors.}]
+\begin{pagelisting}[label=assignmenterror, caption={Assignment errors.}]
 >> a = zeros(1, 100);
 >> b = 0:10;
 
@@ -136,7 +136,7 @@ outside its bounds.
 >> size(a)
 ans =
      110    1
-\end{lstlisting}
+\end{pagelisting}
 
 \subsection{Dimension mismatch error}
 Similarly, some arithmetic operations are only valid if the variables
@@ -154,7 +154,7 @@ in listing\,\ref{dimensionmismatch} does not throw an error but the
 result is something else than the expected elementwise multiplication.
 
 % XXX Some arithmetic operations make size constraints, violating them leads to dimension mismatch errors.
-\begin{lstlisting}[label=dimensionmismatch, caption={Dimension mismatch errors.}]
+\begin{pagelisting}[label=dimensionmismatch, caption={Dimension mismatch errors.}]
   >> a = randn(100, 1);
   >> b = randn(10, 1);
   >> a + b
@@ -171,7 +171,7 @@ result is something else than the expected elementwise multiplication.
   >> size(c)
   ans =
        100    10
-\end{lstlisting}
+\end{pagelisting}
 
 
 
@@ -239,7 +239,7 @@ but it requires a deep understanding of the applied functions and also
 the task at hand.
 
 % XXX Converting a series of spike times into the firing rate as a function of time. Many tasks can be solved with a single line of code. But is this readable?
-\begin{lstlisting}[label=easyvscomplicated, caption={One-liner versus readable code.}]
+\begin{pagelisting}[label=easyvscomplicated, caption={One-liner versus readable code.}]
 % the one-liner
 rate = conv(full(sparse(1, round(spike_times/dt), 1, 1, length(time))), kernel, 'same');
 
@@ -248,7 +248,7 @@ rate = zeros(size(time));
 spike_indices = round(spike_times/dt);
 rate(spike_indices) = 1;
 rate = conv(rate, kernel, 'same');
-\end{lstlisting}
+\end{pagelisting}
 
 The preferred way depends on several considerations. (i) How deep is
 your personal understanding of the programming language?  (ii) What
@@ -291,7 +291,7 @@ example given in the \matlab{} help and assume that there is a
 function \varcode{rightTriangle()} (listing\,\ref{trianglelisting}).
 
 % XXX Slightly more readable version of the example given in the \matlab{} help system. Note: The variable name for the angles have been capitalized in order to not override the matlab defined functions \code{alpha, beta,} and \code{gamma}.
-\begin{lstlisting}[label=trianglelisting, caption={Example function for unit testing.}]
+\begin{pagelisting}[label=trianglelisting, caption={Example function for unit testing.}]
 function angles = rightTriangle(length_a, length_b)
     ALPHA = atand(length_a / length_b);
     BETA = atand(length_a / length_b);
@@ -300,7 +300,7 @@ function angles = rightTriangle(length_a, length_b)
 
     angles = [ALPHA BETA GAMMA];
 end
-\end{lstlisting}
+\end{pagelisting}
 
 This function expects two input arguments that are the length of the
 sides $a$ and $b$ and assumes a right angle between them. From this
@@ -337,7 +337,7 @@ The test script for the \varcode{rightTriangle()} function
 (listing\,\ref{trianglelisting}) may look like in
 listing\,\ref{testscript}.
 
-\begin{lstlisting}[label=testscript, caption={Unit test for the \varcode{rightTriangle()} function stored in an m-file testRightTriangle.m}]
+\begin{pagelisting}[label=testscript, caption={Unit test for the \varcode{rightTriangle()} function stored in an m-file testRightTriangle.m}]
 tolerance = 1e-10;
 
 % preconditions
@@ -373,7 +373,7 @@ angles = rightTriangle(1, 1500);
 smallAngle = (pi / 180) * angles(1); % radians
 approx = sin(smallAngle);
 assert(abs(approx - smallAngle) <= tolerance, 'Problem with small angle approximation')
-\end{lstlisting}
+\end{pagelisting}
 
 In a test script we can execute any code. The actual test whether or
 not the results match our predictions is done using the
@@ -390,16 +390,16 @@ executed. We further define a \varcode{tolerance} variable that is
 used when comparing double values (Why might the test on equality of
 double values be tricky?).
 
-\begin{lstlisting}[label=runtestlisting, caption={Run the test!}]
+\begin{pagelisting}[label=runtestlisting, caption={Run the test!}]
 result = runtests('testRightTriangle')
-\end{lstlisting}
+\end{pagelisting}
 
 During the run, \matlab{} will put out error messages onto the command
 line and a summary of the test results is then stored within the
 \varcode{result} variable. These can be displayed using the function
 \code[table()]{table(result)}.
 
-\begin{lstlisting}[label=testresults, caption={The test results.}, basicstyle=\ttfamily\scriptsize]
+\begin{pagelisting}[label=testresults, caption={The test results.}, basicstyle=\ttfamily\scriptsize]
 table(result)
 ans =
   4x6 table
@@ -411,7 +411,7 @@ _________________________________   ______  ______  ___________  ________  _____
 'testR.../Test_IsoscelesTriangles'   true   false   false        0.004893   [1x1 struct]
 'testR.../Test_30_60_90Triangle'     true   false   false        0.005057   [1x1 struct]
 'testR.../Test_SmallAngleApprox'     true   false   false        0.0049     [1x1 struct]
-\end{lstlisting}
+\end{pagelisting}
 
 So far so good, all tests pass and our function appears to do what it
 is supposed to do. But tests are only as good as the programmer who
@@ -479,11 +479,11 @@ stopped in debug mode (listing\,\ref{debuggerlisting}).
 \end{figure}
 
 
-\begin{lstlisting}[label=debuggerlisting, caption={Command line when the program execution was stopped in the debugger.}]
+\begin{pagelisting}[label=debuggerlisting, caption={Command line when the program execution was stopped in the debugger.}]
 >> simplerandomwalk
 6   for run = 1:num_runs
 K>>
-\end{lstlisting}
+\end{pagelisting}
 
 When stopped in the debugger we can view and change the state of the
 program at this point, we can also issue commands to try the next

designpattern/lecture/designpattern.tex

@@ -10,7 +10,8 @@ pattern]{design pattern}.
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \section{Looping over  vector elements}
 Iterating over vector elements by means of a \mcode{for}-loop is a very commen pattern:
-\begin{lstlisting}[caption={\varcode{for}-loop for accessing vector elements by indices}]
+
+\begin{pagelisting}[caption={\varcode{for}-loop for accessing vector elements by indices}]
 x = [2:3:20];      % Some vector.
 for i=1:length(x)  % For loop over the indices of the vector.
   i                % This is the index (an integer number)
@@ -22,12 +23,14 @@ for i=1:length(x)  % For loop over the indices of the vector.
   % as an argument to a function:
   do_something(x(i));
 end
-\end{lstlisting}
+\end{pagelisting}
 
+\noindent
 If the result of the computation within the loop are single numbers
 that are to be stored in a vector, you should create this vector with
 the right size before the loop:
-\begin{lstlisting}[caption={\varcode{for}-loop for writing a vector}]
+
+\begin{pagelisting}[caption={\varcode{for}-loop for writing a vector}]
 x = [1.2 2.3 2.6 3.1];   % Some vector.
 % Create a vector for the results, as long as the number of loops:
 y = zeros(length(x),1);
@@ -38,13 +41,15 @@ for i=1:length(x)
 end
 % Now the result vector can be further processed:
 mean(y);
-\end{lstlisting}
+\end{pagelisting}
 
+\noindent
 The computation within the loop could also result in a vector of some
 length and not just a single number. If the length of this vector
 (here 10) is known beforehand, then you should create a matrix of
 appropriate size for storing the results:
-\begin{lstlisting}[caption={\varcode{for}-loop for writing rows of a matrix}]
+
+\begin{pagelisting}[caption={\varcode{for}-loop for writing rows of a matrix}]
 x = [2:3:20];             % Some vector.
 % Create space for results -
 % as many rows as loops, as many columns as needed:
@@ -56,30 +61,33 @@ for i=1:length(x)
 end
 % Process the results stored in matrix y:
 mean(y, 1)
-\end{lstlisting}
+\end{pagelisting}
 
+\noindent
 Another possibility is that the result vectors (here of unknown size)
 need to be combined into a single large vector:
-\begin{lstlisting}[caption={\varcode{for}-loop for appending vectors}]
+
+\begin{pagelisting}[caption={\varcode{for}-loop for appending vectors}]
 x = [2:3:20];        % Some vector.
 y = [];              % Empty vector for storing the results.
 for i=1:length(x)
   % The function get_something() returns a vector of unspecified size:
-  z = get_somehow_more(x(i));
+  z = get_something(x(i));
   % The content of z is appended to the result vector y:
-  y = [y; z(:)];
+  y = [y, z(:)];
   % The z(:) syntax ensures that we append column-vectors.
 end
-% Process the results stored in the vector z:
-mean(y)
-\end{lstlisting}
+% Process the results stored in the vector y:
+mean(y, 1)
+\end{pagelisting}
 
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \section{Scaling and shifting random numbers, zeros, and ones}
 Random number generators usually return random numbers of a given mean
 and standard deviation. Multiply those numbers by a factor to change their standard deviation and add a number to shift the mean.
-\begin{lstlisting}[caption={Scaling and shifting of random numbers}]
+
+\begin{pagelisting}[caption={Scaling and shifting of random numbers}]
 % 100 random numbers drawn from a normal distribution 
 % with mean 0 and standard deviation 1:
 x = randn(100, 1);
@@ -89,18 +97,20 @@ x = randn(100, 1);
 mu = 4.8;
 sigma = 2.3;
 y = randn(100, 1)*sigma + mu;
-\end{lstlisting}
+\end{pagelisting}
 
+\noindent
 The same principle can be useful for in the context of the functions
 \mcode{zeros()} or \mcode{ones()}:
-\begin{lstlisting}[caption={Scaling and shifting of \varcode{zeros()} and \varcode{ones()}}]
+
+\begin{pagelisting}[caption={Scaling and shifting of \varcode{zeros()} and \varcode{ones()}}]
 x = -1:0.01:2;      % Vector of x-values for plotting
 plot(x, exp(-x.*x));
 % Plot for the same x-values a horizontal line with y=0.8:
 plot(x, zeros(size(x))+0.8);
 % ... or a line with y=0.5:
 plot(x, ones(size(x))*0.5);
-\end{lstlisting}
+\end{pagelisting}
 
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -119,25 +129,26 @@ can plot the values of the $y$ vector against the ones of the $x$
 vector.
 
 The following scripts compute and plot the function $f(x)=e^{-x^2}$:
-\begin{lstlisting}[caption={Plotting a mathematical function --- very detailed}]
+
+\begin{pagelisting}[caption={Plotting a mathematical function --- very detailed}]
 xmin = -1.0;
 xmax = 2.0;
 dx = 0.01;        % Step size
 x = xmin:dx:xmax; % Vector with x-values.
 y = exp(-x.*x);   % No for loop! '.*' for multiplying the vector elements.
 plot(x, y);
-\end{lstlisting}
+\end{pagelisting}
 
-\begin{lstlisting}[caption={Plotting a mathematical function --- shorter}]
+\begin{pagelisting}[caption={Plotting a mathematical function --- shorter}]
 x = -1:0.01:2;
 y = exp(-x.*x);
 plot(x, y);
-\end{lstlisting}
+\end{pagelisting}
 
-\begin{lstlisting}[caption={Plotting a mathematical function --- compact}]
+\begin{pagelisting}[caption={Plotting a mathematical function --- compact}]
 x = -1:0.01:2;
 plot(x, exp(-x.*x));
-\end{lstlisting}
+\end{pagelisting}
 
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -146,28 +157,34 @@ For estimating probabilities or probability densities from histograms
 we need to normalize them appropriately.
 
 The \mcode{histogram()} function does this automatically with the appropriate arguments:
-\begin{lstlisting}[caption={Probability density with the \varcode{histogram()}-function}]
+
+\begin{pagelisting}[caption={Probability density with the \varcode{histogram()}-function}]
 x = randn(100, 1);         % Some real-valued data.
 histogram(x, 'Normalization', 'pdf');
-\end{lstlisting}
-\begin{lstlisting}[caption={Probability with the \varcode{histogram()}-function}]
+\end{pagelisting}
+
+\begin{pagelisting}[caption={Probability with the \varcode{histogram()}-function}]
 x = randi(6, 100, 1);      % Some integer-valued data.
 histogram(x, 'Normalization', 'probability');
-\end{lstlisting}
+\end{pagelisting}
+
+\noindent
 Alternatively one can normalize the histogram data as returned by the
 \code{hist()}-function manually:
-\begin{lstlisting}[caption={Probability density with the \varcode{hist()}- and \varcode{bar()}-function}]
+
+\begin{pagelisting}[caption={Probability density with the \varcode{hist()}- and \varcode{bar()}-function}]
 x = randn(100, 1);         % Some real-valued data.
 [h, b] = hist(x);          % Compute histogram.
 h = h/sum(h)/(b(2)-b(1));  % Normalization to a probability density.
 bar(b, h);                 % Plot the probability density.
-\end{lstlisting}
-\begin{lstlisting}[caption={Probability with the \varcode{hist()}- and \varcode{bar()}-function}]
+\end{pagelisting}
+
+\begin{pagelisting}[caption={Probability with the \varcode{hist()}- and \varcode{bar()}-function}]
 x = randi(6, 100, 1);      % Some integer-valued data.
 [h, b] = hist(x);          % Compute histogram.
 h = h/sum(h);              % Normalize to probability.
 bar(b, h);                 % Plot the probabilities.
-\end{lstlisting}
+\end{pagelisting}
 
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

exercises.mk

@@ -0,0 +1,33 @@
+EXERCISES=$(TEXFILES:.tex=.pdf)
+SOLUTIONS=$(EXERCISES:%.pdf=%-solutions.pdf)
+
+.PHONY: pdf exercises solutions watch watchexercises watchsolutions clean
+
+pdf : $(SOLUTIONS) $(EXERCISES)
+
+exercises : $(EXERCISES)
+
+solutions : $(SOLUTIONS)
+
+$(SOLUTIONS) : %-solutions.pdf : %.tex ../../exercisesheader.tex ../../exercisestitle.tex
+	{ echo "\\documentclass[answers,12pt,a4paper,pdftex]{exam}"; sed -e '1d' $<; } > $(patsubst %.pdf,%.tex,$@)
+	pdflatex -interaction=scrollmode $(patsubst %.pdf,%.tex,$@) | tee /dev/stderr | fgrep -q "Rerun to get cross-references right" && pdflatex -interaction=scrollmode $(patsubst %.pdf,%.tex,$@) || true
+	rm $(patsubst %.pdf,%,$@).[!p]*
+
+$(EXERCISES) : %.pdf : %.tex ../../exercisesheader.tex ../../exercisestitle.tex
+	pdflatex -interaction=scrollmode $< | tee /dev/stderr | fgrep -q "Rerun to get cross-references right" && pdflatex -interaction=scrollmode $< || true
+
+watch :
+	while true; do ! make -q pdf && make pdf; sleep 0.5; done
+
+watchexercises :
+	while true; do ! make -q exercises && make exercises; sleep 0.5; done
+
+watchsolutions :
+	while true; do ! make -q solutions && make solutions; sleep 0.5; done
+
+clean :
+	rm -f *~ *.aux *.log *.out
+
+cleanup : clean
+	rm -f $(SOLUTIONS) $(EXERCISES)

exercisesheader.tex

@@ -0,0 +1,110 @@
+\usepackage[english]{babel}
+\usepackage{pslatex}
+\usepackage{graphicx}
+\usepackage{tikz}
+\usepackage{xcolor}
+\usepackage{amsmath}
+\usepackage{amssymb}
+
+%%%%% listings %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\usepackage{listings}
+\lstset{
+  language=Matlab,
+  basicstyle=\ttfamily\footnotesize,
+  numbers=left,
+  numberstyle=\tiny,
+  title=\lstname,
+  showstringspaces=false,
+  commentstyle=\itshape\color{darkgray},
+  breaklines=true,
+  breakautoindent=true,
+  % columns=flexible,
+  frame=single,
+  xleftmargin=1.5em,
+  xrightmargin=1em,
+  aboveskip=10pt
+}
+
+%%%%% page style %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\usepackage[left=20mm,right=20mm,top=25mm,bottom=19mm,headsep=2ex,headheight=3ex,footskip=3.5ex]{geometry}
+\pagestyle{headandfoot}
+\ifprintanswers
+\newcommand{\stitle}{Solutions}
+\else
+\newcommand{\stitle}{}
+\fi
+\header{{\bfseries\large \exercisenum. \exercisetopic}}{{\bfseries\large\stitle}}{{\bfseries\large\exercisedate}}
+\firstpagefooter{Prof. Dr. Jan Benda}{}{jan.benda@uni-tuebingen.de}
+\runningfooter{}{\thepage}{}
+
+\shadedsolutions
+\definecolor{SolutionColor}{gray}{0.9}
+
+\setlength{\baselineskip}{15pt}
+\setlength{\parindent}{0.0cm}
+\setlength{\parskip}{0.3cm}
+
+\usepackage{multicol}
+
+%%%%% units %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\usepackage[mediumspace,mediumqspace,Gray,squaren]{SIunits}      % \ohm, \micro
+
+%%%%% new commands %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\newcommand{\cin}[1]{[{\rm #1}]_{\text{in}}}
+\newcommand{\cout}[1]{[{\rm #1}]_{\text{out}}}
+\newcommand{\units}[1]{\text{#1}}
+\newcommand{\K}{\text{K$^+$}}
+\newcommand{\Na}{\text{Na$^+$}}
+\newcommand{\Cl}{\text{Cl$^-$}}
+\newcommand{\Ca}{\text{Ca$^{2+}$}}
+\newcommand{\water}{$\text{H}_2\text{O}$}
+
+\usepackage{dsfont}
+\newcommand{\naZ}{\mathds{N}}
+\newcommand{\gaZ}{\mathds{Z}}
+\newcommand{\raZ}{\mathds{Q}}
+\newcommand{\reZ}{\mathds{R}}
+\newcommand{\reZp}{\mathds{R^+}}
+\newcommand{\reZpN}{\mathds{R^+_0}}
+\newcommand{\koZ}{\mathds{C}}
+\newcommand{\RT}{{\rm Re\!\ }}
+\newcommand{\IT}{{\rm Im\!\ }}
+\newcommand{\arccot}{{\rm arccot}}
+\newcommand{\sgn}{{\rm sgn}}
+\newcommand{\im}{{\rm i}}
+\newcommand{\drv}{\mathrm{d}}
+\newcommand{\divis}[2]{$^{#1}\!\!/\!_{#2}$}
+\newcommand{\vect}[2]{\left( \!\! \begin{array}{c} #1 \\ #2 \end{array} \!\! \right)}
+\newcommand{\matt}[2]{\left( \!\! \begin{array}{cc} #1 \\ #2 \end{array} \!\! \right)}
+\renewcommand{\Re}{\mathrm{Re}}
+\renewcommand{\Im}{\mathrm{Im}}
+\newcommand{\av}[1]{\left\langle #1 \right\rangle}
+
+\newcommand{\pref}[1]{(\ref{#1})}
+\newcommand{\eqnref}[1]{(\ref{#1})}
+
+\newcommand{\hr}{\par\vspace{-5ex}\noindent\rule{\textwidth}{1pt}\par\vspace{-1ex}}
+
+\newcommand{\qt}[1]{\textbf{#1}\\}
+
+\newcommand{\extra}{--- Optional ---\ \mbox{}}
+
+\newcommand{\code}[1]{\texttt{#1}}
+
+%%%%% page breaks %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\newcommand{\continue}{\ifprintanswers%
+\else
+\vfill\hspace*{\fill}$\rightarrow$\newpage%
+\fi}
+\newcommand{\continuepage}{\ifprintanswers%
+\newpage
+\else
+\vfill\hspace*{\fill}$\rightarrow$\newpage%
+\fi}
+\newcommand{\newsolutionpage}{\ifprintanswers%
+\newpage%
+\else
+\fi}
+
+%%%%% hyperref %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\usepackage[breaklinks=true,bookmarks=true,bookmarksopen=true,pdfpagemode=UseNone,pdfstartview=FitH,colorlinks=true,citecolor=blue!50!black,linkcolor=blue!50!black,urlcolor=blue!50!black]{hyperref}

exercisestitle.tex

@@ -4,3 +4,4 @@
 {\large Jan Grewe, Jan Benda}\\[-3ex]
 Neuroethology Lab \hfill --- \hfill Institute for Neurobiology \hfill --- \hfill \includegraphics[width=0.28\textwidth]{UT_WBMW_Black_RGB} \\
 \end{center}
+\hr

header.tex

@@ -2,7 +2,7 @@
 \title{\textbf{\huge\sffamily\tr{Introduction to\\[1ex] Scientific Computing}%
        {Einf\"uhrung in die\\[1ex] wissenschaftliche Datenverarbeitung}}}
 
-\author{{\LARGE Jan Grewe \& Jan Benda}\\[5ex]Abteilung Neuroethologie\\[2ex]%
+\author{{\LARGE Jan Grewe \& Jan Benda}\\[5ex]Neuroethology Lab\\[2ex]%
         \includegraphics[width=0.3\textwidth]{UT_WBMW_Rot_RGB}\vspace{3ex}}
 
 \date{WS 2020/2021\\\vfill%
@@ -77,7 +77,7 @@
 \setcounter{totalnumber}{2}
 
 % float placement fractions:
-\renewcommand{\textfraction}{0.2}
+\renewcommand{\textfraction}{0.1}
 \renewcommand{\topfraction}{0.9}
 \renewcommand{\bottomfraction}{0.0}
 \renewcommand{\floatpagefraction}{0.7}
@@ -209,6 +209,21 @@
 \let\l@lstlisting\l@figure
 \makeatother
 
+% \lstinputlisting wrapped in a minipage to avoid page breaks:
+\newcommand{\pageinputlisting}[2][]{\vspace{-2ex}\noindent\begin{minipage}[t]{1\linewidth}\lstinputlisting[#1]{#2}\end{minipage}}
+
+%%%%% listing environment: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+% usage:
+%
+% \begin{pagelisting}[label=listing1, caption={A script.}]
+%   >> x = 6
+% \end{pagelisting}
+%
+% This is the lstlisting environment but wrapped in a minipage to avoid page breaks.
+\lstnewenvironment{pagelisting}[1][]%
+  {\vspace{-2ex}\lstset{#1}\noindent\minipage[t]{1\linewidth}}%
+  {\endminipage}
+
 %%%%% english, german, code and file terms: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \usepackage{ifthen}
 
@@ -349,11 +364,12 @@
       }{%
         \immediate\write\solutions{\unexpanded{\subsection}{Exercise \thechapter.\arabic{exercisef}}}%
         \immediate\write\solutions{\unexpanded{\label}{solution\arabic{chapter}-\arabic{exercisef}}}%
-        \immediate\write\solutions{\unexpanded{\lstinputlisting[belowskip=0ex,aboveskip=0ex,%
-          nolol=true, title={\textbf{Source code:} \protect\StrSubstitute{#1}{_}{\_}}]}{\codepath#1}}%
+        \immediate\write\solutions{\unexpanded{\begin{minipage}{1\linewidth}\lstinputlisting[belowskip=0ex,aboveskip=0ex,%
+          nolol=true, title={\textbf{Source code:} \protect\StrSubstitute{#1}{_}{\_}}]}{\codepath#1}\unexpanded{\end{minipage}}}%
         \ifthenelse{\equal{#2}{}}{}%
-          {\immediate\write\solutions{\unexpanded{\lstinputlisting[language={},%
-             nolol=true, title={\textbf{Output:}}, belowskip=0ex, aboveskip=1ex]}{\codepath#2}}}%
+          {\immediate\write\solutions{\unexpanded{\begin{minipage}{1\linewidth}\lstinputlisting[language={},%
+             nolol=true, title={\textbf{Output:}}, belowskip=0ex, aboveskip=1ex]}{\codepath#2}\unexpanded{\end{minipage}}}}%
+        \immediate\write\solutions{\unexpanded{\vspace*{\fill}}}%
         \immediate\write\solutions{}%
       }%
   }%
@@ -362,14 +378,18 @@
         { \hypersetup{hypertexnames=false}%
           \ifthenelse{\equal{\exercisesource}{}}{}%
           { \addtocounter{lstlisting}{-1}%
-            \lstinputlisting[belowskip=0pt,aboveskip=1ex,nolol=true,%
+            \par\noindent\begin{minipage}[t]{1\linewidth}%
+            \lstinputlisting[belowskip=1ex,aboveskip=-1ex,nolol=true,%
                              title={\textbf{Solution:} \exercisefile}]%
                             {\exercisesource}%
+            \end{minipage}%
             \ifthenelse{\equal{\exerciseoutput}{}}{}%
               { \addtocounter{lstlisting}{-1}%
+                \par\noindent\begin{minipage}[t]{1\linewidth}%
                 \lstinputlisting[language={},title={\textbf{Output:}},%
-                                 nolol=true,belowskip=0pt]%
+                                 nolol=true,belowskip=0pt,aboveskip=-0.5ex]%
                                 {\exerciseoutput}%
+                \end{minipage}%
               }%
           }%
           \hypersetup{hypertexnames=true}%
@@ -452,12 +472,12 @@
   { \captionsetup{singlelinecheck=off,hypcap=false,labelformat={empty},%
                   labelfont={large,sf,it,bf},font={large,sf,it,bf}}
     \ifthenelse{\equal{#1}{}}%
-    { \begin{mdframed}[linecolor=importantline,linewidth=1ex,%
+    { \begin{mdframed}[nobreak=true,linecolor=importantline,linewidth=1ex,%
                        backgroundcolor=importantback,font={\sffamily}]%
         \setlength{\parindent}{0pt}%
         \setlength{\parskip}{1ex}%
     }%
-    { \begin{mdframed}[linecolor=importantline,linewidth=1ex,%
+    { \begin{mdframed}[nobreak=true,linecolor=importantline,linewidth=1ex,%
                        backgroundcolor=importantback,font={\sffamily},%
                        frametitle={\captionof{iboxf}{#1}},frametitleaboveskip=-1ex,%
                        frametitlebackgroundcolor=importantline]%

likelihood/exercises/Makefile

@@ -1,34 +1,3 @@
-TEXFILES=$(wildcard exercises??.tex)
-EXERCISES=$(TEXFILES:.tex=.pdf)
-SOLUTIONS=$(EXERCISES:exercises%=solutions%)
+TEXFILES=$(wildcard likelihood-?.tex)
 
-.PHONY: pdf exercises solutions watch watchexercises watchsolutions clean
-
-pdf : $(SOLUTIONS) $(EXERCISES)
-
-exercises : $(EXERCISES)
-
-solutions : $(SOLUTIONS)
-
-$(SOLUTIONS) : solutions%.pdf : exercises%.tex instructions.tex
-	{ echo "\\documentclass[answers,12pt,a4paper,pdftex]{exam}"; sed -e '1d' $<; } > $(patsubst %.pdf,%.tex,$@)
-	pdflatex -interaction=scrollmode $(patsubst %.pdf,%.tex,$@) | tee /dev/stderr | fgrep -q "Rerun to get cross-references right" && pdflatex -interaction=scrollmode $(patsubst %.pdf,%.tex,$@) || true
-	rm $(patsubst %.pdf,%,$@).[!p]*
-
-$(EXERCISES) : %.pdf : %.tex instructions.tex
-	pdflatex -interaction=scrollmode $< | tee /dev/stderr | fgrep -q "Rerun to get cross-references right" && pdflatex -interaction=scrollmode $< || true
-
-watch :
-	while true; do ! make -q pdf && make pdf; sleep 0.5; done
-
-watchexercises :
-	while true; do ! make -q exercises && make exercises; sleep 0.5; done
-
-watchsolutions :
-	while true; do ! make -q solutions && make solutions; sleep 0.5; done
-
-clean :
-	rm -f *~ *.aux *.log *.out
-
-cleanup : clean
-	rm -f $(SOLUTIONS) $(EXERCISES)
+include ../../exercises.mk

likelihood/exercises/instructions.tex

@@ -1,41 +0,0 @@
-\vspace*{-8ex}
-\begin{center}
-\textbf{\Large Introduction to scientific computing}\\[1ex]
-{\large Jan Grewe, Jan Benda}\\[-3ex]
-Neuroethology lab \hfill --- \hfill Institute for Neurobiology \hfill --- \hfill \includegraphics[width=0.28\textwidth]{UT_WBMW_Black_RGB} \\
-\end{center}
-
-% \ifprintanswers%
-% \else
-
-% % Die folgenden Aufgaben dienen der Wiederholung, \"Ubung und
-% % Selbstkontrolle und sollten eigenst\"andig bearbeitet und gel\"ost
-% % werden. Die L\"osung soll in Form eines einzelnen Skriptes (m-files)
-% % im ILIAS hochgeladen werden. Jede Aufgabe sollte in einer eigenen
-% % ``Zelle'' gel\"ost sein. Die Zellen \textbf{m\"ussen} unabh\"angig
-% % voneinander ausf\"uhrbar sein. Das Skript sollte nach dem Muster:
-% % ``variablen\_datentypen\_\{nachname\}.m'' benannt werden
-% % (z.B. variablen\_datentypen\_mueller.m).
-
-% \begin{itemize}
-% \item \"Uberzeuge dich von jeder einzelnen Zeile deines Codes, dass
-%   sie auch wirklich das macht, was sie machen soll! Teste dies mit
-%   kleinen Beispielen direkt in der Kommandozeile.
-% \item Versuche die L\"osungen der Aufgaben m\"oglichst in
-%   sinnvolle kleine Funktionen herunterzubrechen.
-%   Sobald etwas \"ahnliches mehr als einmal berechnet werden soll,
-%   lohnt es sich eine Funktion daraus zu schreiben!
-% \item Teste rechenintensive \code{for} Schleifen, Vektoren, Matrizen
-%   zuerst mit einer kleinen Anzahl von Wiederholungen oder kleiner
-%   Gr\"o{\ss}e, und benutze erst am Ende, wenn alles \"uberpr\"uft
-%   ist, eine gro{\ss}e Anzahl von Wiederholungen oder Elementen, um eine gute
-%   Statistik zu bekommen.
-% \item Benutze die Hilfsfunktion von \code{matlab} (\code{help
-%     commando} oder \code{doc commando}) und das Internet, um
-%   herauszufinden, wie bestimmte \code{matlab} Funktionen zu verwenden
-%   sind und was f\"ur M\"oglichkeiten sie bieten.
-%   Auch zu inhaltlichen Konzepten bietet das Internet oft viele
-%   Antworten!
-% \end{itemize}
-
-% \fi

likelihood/exercises/likelihood-1.tex

@@ -1,90 +1,17 @@
 \documentclass[12pt,a4paper,pdftex]{exam}
 
-\usepackage[german]{babel}
-\usepackage{pslatex}
-\usepackage[mediumspace,mediumqspace,Gray]{SIunits}      % \ohm, \micro
-\usepackage{xcolor}
-\usepackage{graphicx}
-\usepackage[breaklinks=true,bookmarks=true,bookmarksopen=true,pdfpagemode=UseNone,pdfstartview=FitH,colorlinks=true,citecolor=blue]{hyperref}
-
-%%%%% layout %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage[left=20mm,right=20mm,top=25mm,bottom=25mm]{geometry}
-\pagestyle{headandfoot}
-\ifprintanswers
-\newcommand{\stitle}{: Solutions}
-\else
-\newcommand{\stitle}{}
-\fi
-\header{{\bfseries\large Exercise 11\stitle}}{{\bfseries\large Maximum likelihood}}{{\bfseries\large January 7th, 2020}}
-\firstpagefooter{Prof. Dr. Jan Benda}{Phone: 29 74573}{Email:
-jan.benda@uni-tuebingen.de}
-\runningfooter{}{\thepage}{}
-
-\setlength{\baselineskip}{15pt}
-\setlength{\parindent}{0.0cm}
-\setlength{\parskip}{0.3cm}
-\renewcommand{\baselinestretch}{1.15}
-
-%%%%% listings %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage{listings}
-\lstset{
-  language=Matlab,
-  basicstyle=\ttfamily\footnotesize,
-  numbers=left,
-  numberstyle=\tiny,
-  title=\lstname,
-  showstringspaces=false,
-  commentstyle=\itshape\color{darkgray},
-  breaklines=true,
-  breakautoindent=true,
-  columns=flexible,
-  frame=single,
-  xleftmargin=1em,
-  xrightmargin=1em,
-  aboveskip=10pt
-}
-
-%%%%% math stuff: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage{amsmath}
-\usepackage{amssymb}
-\usepackage{bm} 
-\usepackage{dsfont}
-\newcommand{\naZ}{\mathds{N}}
-\newcommand{\gaZ}{\mathds{Z}}
-\newcommand{\raZ}{\mathds{Q}}
-\newcommand{\reZ}{\mathds{R}}
-\newcommand{\reZp}{\mathds{R^+}}
-\newcommand{\reZpN}{\mathds{R^+_0}}
-\newcommand{\koZ}{\mathds{C}}
-
-%%%%% page breaks %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\newcommand{\continue}{\ifprintanswers%
-\else
-\vfill\hspace*{\fill}$\rightarrow$\newpage%
-\fi}
-\newcommand{\continuepage}{\ifprintanswers%
-\newpage
-\else
-\vfill\hspace*{\fill}$\rightarrow$\newpage%
-\fi}
-\newcommand{\newsolutionpage}{\ifprintanswers%
-\newpage%
-\else
-\fi}
-
-%%%%% new commands %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\newcommand{\qt}[1]{\textbf{#1}\\}
-\newcommand{\pref}[1]{(\ref{#1})}
-\newcommand{\extra}{--- Zusatzaufgabe ---\ \mbox{}}
-\newcommand{\code}[1]{\texttt{#1}}
+\newcommand{\exercisetopic}{Maximum Likelihood}
+\newcommand{\exercisenum}{10}
+\newcommand{\exercisedate}{January 12th, 2021}
 
+\input{../../exercisesheader}
+
+\firstpagefooter{Prof. Dr. Jan Benda}{}{jan.benda@uni-tuebingen.de}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \begin{document}
 
-\input{instructions}
-
+\input{../../exercisestitle}
 
 \begin{questions}
 

plotstyle.py

@@ -33,17 +33,18 @@ colors['white'] = '#FFFFFF'
 # general settings for plot styles:
 lwthick = 3.0
 lwthin = 1.8
+edgewidth = 0.0 if xkcd_style else 1.0
 mainline = {'linestyle': '-', 'linewidth': lwthick}
 minorline = {'linestyle': '-', 'linewidth': lwthin}
-largemarker = {'marker': 'o', 'markersize': 9, 'markeredgecolor': colors['white'], 'markeredgewidth': 1}
-smallmarker = {'marker': 'o', 'markersize': 6, 'markeredgecolor': colors['white'], 'markeredgewidth': 1}
+largemarker = {'marker': 'o', 'markersize': 9, 'markeredgecolor': colors['white'], 'markeredgewidth': edgewidth}
+smallmarker = {'marker': 'o', 'markersize': 6, 'markeredgecolor': colors['white'], 'markeredgewidth': edgewidth}
 largelinepoints = {'linestyle': '-', 'linewidth': lwthick, 'marker': 'o', 'markersize': 10, 'markeredgecolor': colors['white'], 'markeredgewidth': 1}
 smalllinepoints = {'linestyle': '-', 'linewidth': 1.4, 'marker': 'o', 'markersize': 7, 'markeredgecolor': colors['white'], 'markeredgewidth': 1}
-filllw = 1.0
+filllw = edgewidth
 fillec = colors['white']
 fillalpha = 0.4
 filledge = {'linewidth': filllw, 'joinstyle': 'round'}
-if int(mpl.__version__.split('.')[0]) < 2:
+if mpl_major < 2:
     del filledge['joinstyle']
 
 # helper lines:

plotting/exercises/Makefile

@@ -0,0 +1,3 @@
+TEXFILES=$(wildcard plotting-?.tex)
+
+include ../../exercises.mk

plotting/exercises/instructions.tex

@@ -1,17 +1,11 @@
-\vspace*{-8ex}
-\begin{center}
-\textbf{\Large Introduction to scientific computing}\\[1ex]
-{\large Jan Grewe, Jan Benda}\\[-3ex]
-Neuroethology lab \hfill --- \hfill Institute for Neurobiology \hfill --- \hfill \includegraphics[width=0.28\textwidth]{UT_WBMW_Black_RGB} \\
-\end{center}
+\ifprintanswers%
+\else
 
 The exercises are meant for self-monitoring and revision of the
 lecture. You should try to solve them on your own. In contrast
 to previous exercises, the solutions can not be saved in a single file. Combine the files into a
 single zip archive and submit it via ILIAS. Name the archive according
 to the pattern: ``plotting\_\{surname\}.zip''.
-% \ifprintanswers%
-% \else
 
 % % Die folgenden Aufgaben dienen der Wiederholung, \"Ubung und
 % % Selbstkontrolle und sollten eigenst\"andig bearbeitet und gel\"ost
@@ -43,4 +37,4 @@ to the pattern: ``plotting\_\{surname\}.zip''.
 %   Antworten!
 % \end{itemize}
 
-% \fi
+\fi

plotting/exercises/plotting-1.tex

@@ -1,88 +1,18 @@
 \documentclass[12pt,a4paper,pdftex]{exam}
 
-\usepackage[german]{babel}
-\usepackage{pslatex}
-\usepackage[mediumspace,mediumqspace]{SIunits}      % \ohm, \micro
-\usepackage{xcolor}
-\usepackage{graphicx}
-\usepackage[breaklinks=true,bookmarks=true,bookmarksopen=true,pdfpagemode=UseNone,pdfstartview=FitH,colorlinks=true,citecolor=blue]{hyperref}
+\newcommand{\exercisetopic}{Plotting}
+\newcommand{\exercisenum}{X}
+\newcommand{\exercisedate}{December 14th, 2020}
 
-%%%%% layout %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage[left=20mm,right=20mm,top=25mm,bottom=25mm]{geometry}
-\pagestyle{headandfoot}
-\ifprintanswers
-\newcommand{\stitle}{: Solutions}
-\else
-\newcommand{\stitle}{}
-\fi
-\header{{\bfseries\large Exercise 7\stitle}}{{\bfseries\large Plotting}}{{\bfseries\large November 20, 2019}}
-\firstpagefooter{Dr. Jan Grewe}{Phone: 29 74588}{Email:
-jan.grewe@uni-tuebingen.de}
-\runningfooter{}{\thepage}{}
+\input{../../exercisesheader}
 
-\setlength{\baselineskip}{15pt}
-\setlength{\parindent}{0.0cm}
-\setlength{\parskip}{0.3cm}
-\renewcommand{\baselinestretch}{1.15}
+\firstpagefooter{Dr. Jan Grewe}{}{jan.grewe@uni-tuebingen.de}
 
-%%%%% listings %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage{listings}
-\lstset{
-  language=Matlab,
-  basicstyle=\ttfamily\footnotesize,
-  numbers=left,
-  numberstyle=\tiny,
-  title=\lstname,
-  showstringspaces=false,
-  commentstyle=\itshape\color{darkgray},
-  breaklines=true,
-  breakautoindent=true,
-  columns=flexible,
-  frame=single,
-  xleftmargin=1em,
-  xrightmargin=1em,
-  aboveskip=10pt
-}
-
-%%%%% math stuff: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage{amsmath}
-\usepackage{amssymb}
-\usepackage{bm} 
-\usepackage{dsfont}
-\newcommand{\naZ}{\mathds{N}}
-\newcommand{\gaZ}{\mathds{Z}}
-\newcommand{\raZ}{\mathds{Q}}
-\newcommand{\reZ}{\mathds{R}}
-\newcommand{\reZp}{\mathds{R^+}}
-\newcommand{\reZpN}{\mathds{R^+_0}}
-\newcommand{\koZ}{\mathds{C}}
-
-%%%%% page breaks %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\newcommand{\continue}{\ifprintanswers%
-\else
-\vfill\hspace*{\fill}$\rightarrow$\newpage%
-\fi}
-\newcommand{\continuepage}{\ifprintanswers%
-\newpage
-\else
-\vfill\hspace*{\fill}$\rightarrow$\newpage%
-\fi}
-\newcommand{\newsolutionpage}{\ifprintanswers%
-\newpage%
-\else
-\fi}
-
-%%%%% new commands %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\newcommand{\qt}[1]{\textbf{#1}\\}
-\newcommand{\pref}[1]{(\ref{#1})}
-\newcommand{\extra}{--- Zusatzaufgabe ---\ \mbox{}}
-\newcommand{\code}[1]{\texttt{#1}}
-
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \begin{document}
 
+\input{../../exercisestitle}
+
 \input{instructions}
 
 \begin{questions}

plotting/lecture/plotting-chapter.tex

@@ -3,7 +3,7 @@
 \input{../../header}
 
 \lstset{inputpath=../code}
-\graphicspath{{images/}}
+\graphicspath{{figures/}}
 
 \typein[\pagenumber]{Number of first page}
 \typein[\chapternumber]{Chapter number}

plotting/lecture/plotting.tex

@@ -74,20 +74,20 @@ the data was changed or the same kind of plot has to be created for a
 number of datasets.
 
 \begin{important}[Why manual editing should be avoided.]
-  On first glance the manual editing of a figure using tools such as
-  Corel draw, Illustrator, etc.\,appears much more convenient and less
-  complex than coding everything into the analysis scripts. This,
-  however, is not entirely true. What if the figure has to be re-drawn
-  or updated? Then the editing work starts all over again. Rather,
-  there is a great risk associated with the manual editing
-  approach. Axes may be shifted, fonts have not been embedded into the
-  final document, annotations have been copy pasted between figures
-  and are not valid. All of these mistakes can be found in
-  publications and then require an erratum, which is not
-  desirable. Even if it appears more cumbersome in the beginning one
-  should always try to create publication-ready figures directly from
-  the data analysis tool using scripts or functions to properly layout
-  the plot.
+  On first glance manual editing of a figure using tools such as
+  inkscape, Corel draw, Illustrator, etc.\,appears much more
+  convenient and less complex than coding everything into the analysis
+  scripts. This, however, is not entirely true. What if the figure has
+  to be re-drawn or updated, because, for example, you got more data?
+  Then the editing work starts all over again. In addition, there is a
+  great risk associated with the manual editing approach. Axes may be
+  shifted, fonts have not been embedded into the final document,
+  annotations have been copy-pasted between figures and are not
+  valid. All of these mistakes can be found in publications and then
+  require an erratum, which is not desirable. Even if it appears more
+  cumbersome in the beginning, one should always try to generate
+  publication-ready figures directly from the data analysis tool using
+  scripts or functions to properly layout and annotate the plot.
 \end{important}
 
 \subsection{Simple plotting}
@@ -148,7 +148,7 @@ or the color. For additional options consult the help.
 
 The following listing shows a simple line plot with axis labeling and a title
 
-\lstinputlisting[caption={A simple plot showing a sinewave.},
+\pageinputlisting[caption={A simple plot showing a sinewave.},
   label=simpleplotlisting]{simple_plot.m}
 
 
@@ -162,10 +162,10 @@ chosen, and star marker symbols is used. Finally, the name of the
 curve is set to \emph{plot 1} which will be displayed in a legend, if
 chosen.
 
-\begin{lstlisting}[label=settinglineprops, caption={Setting line properties when calling \varcode{plot}.}]
+\begin{pagelisting}[label=settinglineprops, caption={Setting line properties when calling \varcode{plot}.}]
   x = 0:0.1:2*pi; y = sin(x); plot( x, y, 'color', 'r', 'linestyle',
   ':', 'marker', '*', 'linewidth', 1.5, 'displayname', 'plot 1')
-\end{lstlisting}
+\end{pagelisting}
 
 \begin{important}[Choosing the right color.]
   Choosing the perfect color goes a little bit beyond personal
@@ -277,7 +277,7 @@ the last one defines the output format (box\,\ref{graphicsformatbox}).
     listing\,\ref{niceplotlisting}.}\label{spikedetectionfig}
 \end{figure}
 
-\begin{ibox}[t]{\label{graphicsformatbox}File formats for digital artwork.}
+\begin{ibox}[tp]{\label{graphicsformatbox}File formats for digital artwork.}
   There are two fundamentally different types of formats for digital artwork:
   \begin{enumerate}
   \item \enterm[bitmap]{Bitmaps} (\determ{Rastergrafik})
@@ -322,7 +322,7 @@ the last one defines the output format (box\,\ref{graphicsformatbox}).
   efficient.
 \end{ibox}
 
-\lstinputlisting[caption={Script for creating the plot shown in
+\pageinputlisting[caption={Script for creating the plot shown in
     \figref{spikedetectionfig}.},
   label=niceplotlisting]{automatic_plot.m}
 
@@ -380,7 +380,7 @@ draw the data. In the example we also provide further arguments to set
 the size, color of the dots and specify that they are filled
 (listing\,\ref{scatterlisting1}).
 
-\lstinputlisting[caption={Creating a scatter plot with red filled dots.},
+\pageinputlisting[caption={Creating a scatter plot with red filled dots.},
   label=scatterlisting1, firstline=9, lastline=9]{scatterplot.m}
 
 We could have used plot for this purpose and set the marker to
@@ -395,8 +395,7 @@ manipulate the color we need to specify a length(x)-by-3 matrix. For
 each dot we provide an individual color (i.e. the RGB triplet in each
 row of the color matrix, lines 2-4 in listing\,\ref{scatterlisting2})
 
-
-\lstinputlisting[caption={Creating a scatter plot with size and color
+\pageinputlisting[caption={Creating a scatter plot with size and color
     variations. The RGB triplets define the respective color intensity
     in a range 0:1. Here, we modify only the red color channel.},
   label=scatterlisting2, linerange={15-15, 21-23}]{scatterplot.m}
@@ -431,7 +430,7 @@ figures\,\ref{regularsubplotsfig}, \ref{irregularsubplotsfig}).
     also below).}\label{regularsubplotsfig}
 \end{figure}
 
-\lstinputlisting[caption={Script for creating subplots in a regular
+\pageinputlisting[caption={Script for creating subplots in a regular
     grid \figref{regularsubplotsfig}.}, label=regularsubplotlisting,
   basicstyle=\ttfamily\scriptsize]{regular_subplot.m}
 
@@ -458,7 +457,7 @@ create a grid with larger numbers of columns and rows, and specify the
 used cells of the grid by passing a vector as the third argument to
 \code{subplot()}.
 
-\lstinputlisting[caption={Script for creating subplots of different
+\pageinputlisting[caption={Script for creating subplots of different
     sizes \figref{irregularsubplotsfig}.},
   label=irregularsubplotslisting,
   basicstyle=\ttfamily\scriptsize]{irregular_subplot.m}
@@ -516,7 +515,7 @@ its properties. See the \matlab{} help for more information.
     listing\,\ref{errorbarlisting} for A and C and listing\,\ref{errorbarlisting2} }\label{errorbarplot}
 \end{figure}
 
-\lstinputlisting[caption={Illustrating estimation errors using error bars. Script that
+\pageinputlisting[caption={Illustrating estimation errors using error bars. Script that
     creates \figref{errorbarplot}. A, B},
   label=errorbarlisting, firstline=13, lastline=31,
   basicstyle=\ttfamily\scriptsize]{errorbarplot.m}
@@ -550,7 +549,7 @@ leading to invisibility and a value of one to complete
 opaqueness. Finally, we use the normal plot command to draw a line
 connecting the average values (line 12).
 
-\lstinputlisting[caption={Illustrating estimation errors using a shaded area. Script that
+\pageinputlisting[caption={Illustrating estimation errors using a shaded area. Script that
     creates \figref{errorbarplot} C.}, label=errorbarlisting2,
   firstline=33,
   basicstyle=\ttfamily\scriptsize]{errorbarplot.m}
@@ -575,7 +574,7 @@ listing\,\ref{annotationsplotlisting}. For more options consult the
     listing\,\ref{annotationsplotlisting}}\label{annotationsplot}
 \end{figure}
 
-\lstinputlisting[caption={Adding annotations to figures. Script that
+\pageinputlisting[caption={Adding annotations to figures. Script that
     creates \figref{annotationsplot}.},
   label=annotationsplotlisting,
   basicstyle=\ttfamily\scriptsize]{annotations.m}
@@ -632,7 +631,7 @@ Lissajous figure. The basic steps are:
 \item Finally, close the file (line 31).
 \end{enumerate}
 
-\lstinputlisting[caption={Making animations and saving them as a
+\pageinputlisting[caption={Making animations and saving them as a
     movie.}, label=animationlisting, firstline=16, lastline=36,
   basicstyle=\ttfamily\scriptsize]{movie_example.m}
 

pointprocesses/code/binnedRate.m

@@ -18,7 +18,7 @@ function [time, rate] = binned_rate(spikes, bin_width, dt, t_max)
     rate = zeros(size(time));
     h = hist(spikes, bins) ./ bin_width;
     for i = 2:length(bins)
-        rate(round(bins(i - 1) / dt) + 1:round(bins(i) / dt)) = h(i);
+        rate(round(bins(i-1)/dt) + 1:round(bins(i)/dt)) = h(i);
     end
 end
 

pointprocesses/code/convolutionRate.m

@@ -10,19 +10,18 @@ function [time, rate] = convolution_rate(spikes, sigma, dt, t_max)
 %   t_max : the trial duration in seconds.
 %
 % Returns:
-    two vectors containing the time and the rate.
+%   two vectors containing the time and the rate.
 
-time = 0:dt:t_max - dt;
-rate = zeros(size(time));
-spike_indices = round(spikes / dt);
-rate(spike_indices) = 1;
-kernel = gaussKernel(sigma, dt);
-
-rate = conv(rate, kernel, 'same');
+    time = 0:dt:t_max - dt;
+    rate = zeros(size(time));
+    spike_indices = round(spikes / dt);
+    rate(spike_indices) = 1;
+    kernel = gaussKernel(sigma, dt);
+    rate = conv(rate, kernel, 'same');
 end
 
 
 function y = gaussKernel(s, step)
     x = -4 * s:step:4 * s;
-    y = exp(-0.5 .* (x ./ s) .^ 2) ./ sqrt(2 * pi) / s;
+    y = exp(-0.5 .* (x ./ s).^ 2) ./ sqrt(2 * pi) / s;
 end

pointprocesses/code/counthist.m

@@ -37,8 +37,8 @@ function [counts, bins] = counthist(spikes, w)
 
     % plot:
     if nargout == 0
-        bar( bins, counts );
-        xlabel( 'counts k' );
-        ylabel( 'P(k)' );
+        bar(bins, counts);
+        xlabel('counts k');
+        ylabel('P(k)');
     end
 end

pointprocesses/code/instantaneousRate.m

@@ -19,6 +19,6 @@ function [time, rate] = instantaneous_rate(spikes, dt, t_max)
     spike_indices = [1 round(spikes ./ dt)];
 
     for i = 2:length(spike_indices)
-        rate(spike_indices(i - 1):spike_indices(i)) = inst_rate(i - 1);
+        rate(spike_indices(i-1):spike_indices(i)) = inst_rate(i-1);
     end
 end

pointprocesses/code/isihist.m

@@ -13,7 +13,7 @@ function [pdf, centers] = isihist(isis, binwidth)
 
     if nargin < 2
         % compute good binwidth:
-        nperbin = 200;        % average number of data points per bin
+        nperbin = 200;     % average number of data points per bin
         bins = length(isis)/nperbin;   % number of bins
         binwidth = max(isis)/bins;
         if binwidth < 5e-4             % half a millisecond

pointprocesses/code/rasterplot.m

@@ -5,25 +5,25 @@ function rasterplot(spikes, tmax)
 %   spikes: a cell array of vectors of spike times in seconds
 %   tmax: plot spike raster upto tmax seconds
 
-ntrials = length(spikes);
-for k = 1:ntrials
-    times = spikes{k};
-    times = times(times<tmax);
-    if tmax < 1.5
-        times = 1000.0*times; % conversion to ms
+    ntrials = length(spikes);
+    for k = 1:ntrials
+        times = spikes{k};
+        times = times(times<tmax);
+        if tmax < 1.5
+            times = 1000.0*times; % conversion to ms
+        end
+        for i = 1:length( times )
+            line([times(i) times(i)],[k-0.4 k+0.4], 'Color', 'k'); 
+        end
     end
-    for i = 1:length( times )
-      line([times(i) times(i)],[k-0.4 k+0.4], 'Color', 'k'); 
+    if tmax < 1.5
+        xlabel('Time [ms]');
+        xlim([0.0 1000.0*tmax]);
+    else
+        xlabel('Time [s]');
+        xlim([0.0 tmax]);
     end
-end
-if tmax < 1.5
-    xlabel('Time [ms]');
-    xlim([0.0 1000.0*tmax]);
-else
-    xlabel('Time [s]');
-    xlim([0.0 tmax]);
-end
-ylabel('Trials');
-ylim([0.3 ntrials+0.7 ]);
+    ylabel('Trials');
+    ylim([0.3 ntrials+0.7]);
 end
 

pointprocesses/exercises/Makefile

@@ -1,35 +1,3 @@
-BASENAME=pointprocesses
-TEXFILES=$(wildcard $(BASENAME)??.tex)
-EXERCISES=$(TEXFILES:.tex=.pdf)
-SOLUTIONS=$(EXERCISES:pointprocesses%=pointprocesses-solutions%)
+TEXFILES=$(wildcard pointprocesses-?.tex)
 
-.PHONY: pdf exercises solutions watch watchexercises watchsolutions clean
-
-pdf : $(SOLUTIONS) $(EXERCISES)
-
-exercises : $(EXERCISES)
-
-solutions : $(SOLUTIONS)
-
-$(SOLUTIONS) : pointprocesses-solutions%.pdf : pointprocesses%.tex instructions.tex
-	{ echo "\\documentclass[answers,12pt,a4paper,pdftex]{exam}"; sed -e '1d' $<; } > $(patsubst %.pdf,%.tex,$@)
-	pdflatex -interaction=scrollmode $(patsubst %.pdf,%.tex,$@) | tee /dev/stderr | fgrep -q "Rerun to get cross-references right" && pdflatex -interaction=scrollmode $(patsubst %.pdf,%.tex,$@) || true
-	rm $(patsubst %.pdf,%,$@).[!p]*
-
-$(EXERCISES) : %.pdf : %.tex instructions.tex
-	pdflatex -interaction=scrollmode $< | tee /dev/stderr | fgrep -q "Rerun to get cross-references right" && pdflatex -interaction=scrollmode $< || true
-
-watch :
-	while true; do ! make -q pdf && make pdf; sleep 0.5; done
-
-watchexercises :
-	while true; do ! make -q exercises && make exercises; sleep 0.5; done
-
-watchsolutions :
-	while true; do ! make -q solutions && make solutions; sleep 0.5; done
-
-clean :
-	rm -f *~ *.aux *.log *.out
-
-cleanup : clean
-	rm -f $(SOLUTIONS) $(EXERCISES)
+include ../../exercises.mk

pointprocesses/exercises/pointprocesses-1.tex

@@ -1,90 +1,17 @@
 \documentclass[12pt,a4paper,pdftex]{exam}
 
-\usepackage[german]{babel}
-\usepackage{pslatex}
-\usepackage[mediumspace,mediumqspace,Gray]{SIunits}      % \ohm, \micro
-\usepackage{xcolor}
-\usepackage{graphicx}
-\usepackage[breaklinks=true,bookmarks=true,bookmarksopen=true,pdfpagemode=UseNone,pdfstartview=FitH,colorlinks=true,citecolor=blue]{hyperref}
-
-%%%%% layout %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage[left=20mm,right=20mm,top=25mm,bottom=25mm]{geometry}
-\pagestyle{headandfoot}
-\ifprintanswers
-\newcommand{\stitle}{: Solutions}
-\else
-\newcommand{\stitle}{}
-\fi
-\header{{\bfseries\large Exercise 12\stitle}}{{\bfseries\large Point processes}}{{\bfseries\large January 14th, 2020}}
-\firstpagefooter{Prof. Dr. Jan Benda}{Phone: 29 74573}{Email:
-jan.benda@uni-tuebingen.de}
-\runningfooter{}{\thepage}{}
-
-\setlength{\baselineskip}{15pt}
-\setlength{\parindent}{0.0cm}
-\setlength{\parskip}{0.3cm}
-\renewcommand{\baselinestretch}{1.15}
-
-%%%%% listings %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage{listings}
-\lstset{
-  language=Matlab,
-  basicstyle=\ttfamily\footnotesize,
-  numbers=left,
-  numberstyle=\tiny,
-  title=\lstname,
-  showstringspaces=false,
-  commentstyle=\itshape\color{darkgray},
-  breaklines=true,
-  breakautoindent=true,
-  columns=flexible,
-  frame=single,
-  xleftmargin=1em,
-  xrightmargin=1em,
-  aboveskip=10pt
-}
-
-%%%%% math stuff: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage{amsmath}
-\usepackage{amssymb}
-\usepackage{bm} 
-\usepackage{dsfont}
-\newcommand{\naZ}{\mathds{N}}
-\newcommand{\gaZ}{\mathds{Z}}
-\newcommand{\raZ}{\mathds{Q}}
-\newcommand{\reZ}{\mathds{R}}
-\newcommand{\reZp}{\mathds{R^+}}
-\newcommand{\reZpN}{\mathds{R^+_0}}
-\newcommand{\koZ}{\mathds{C}}
-
-%%%%% page breaks %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\newcommand{\continue}{\ifprintanswers%
-\else
-\vfill\hspace*{\fill}$\rightarrow$\newpage%
-\fi}
-\newcommand{\continuepage}{\ifprintanswers%
-\newpage
-\else
-\vfill\hspace*{\fill}$\rightarrow$\newpage%
-\fi}
-\newcommand{\newsolutionpage}{\ifprintanswers%
-\newpage%
-\else
-\fi}
-
-%%%%% new commands %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\newcommand{\qt}[1]{\textbf{#1}\\}
-\newcommand{\pref}[1]{(\ref{#1})}
-\newcommand{\extra}{--- Zusatzaufgabe ---\ \mbox{}}
-\newcommand{\code}[1]{\texttt{#1}}
+\newcommand{\exercisetopic}{Point Processes}
+\newcommand{\exercisenum}{11}
+\newcommand{\exercisedate}{January 19th, 2021}
 
+\input{../../exercisesheader}
+
+\firstpagefooter{Prof. Dr. Jan Benda}{}{jan.benda@uni-tuebingen.de}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \begin{document}
 
-\input{instructions}
-
+\input{../../exercisestitle}
 
 \begin{questions}
 

pointprocesses/exercises/pointprocesses-2.tex

@@ -1,90 +1,17 @@
 \documentclass[12pt,a4paper,pdftex]{exam}
 
-\usepackage[german]{babel}
-\usepackage{pslatex}
-\usepackage[mediumspace,mediumqspace,Gray]{SIunits}      % \ohm, \micro
-\usepackage{xcolor}
-\usepackage{graphicx}
-\usepackage[breaklinks=true,bookmarks=true,bookmarksopen=true,pdfpagemode=UseNone,pdfstartview=FitH,colorlinks=true,citecolor=blue]{hyperref}
-
-%%%%% layout %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage[left=20mm,right=20mm,top=25mm,bottom=25mm]{geometry}
-\pagestyle{headandfoot}
-\ifprintanswers
-\newcommand{\stitle}{L\"osungen}
-\else
-\newcommand{\stitle}{\"Ubung}
-\fi
-\header{{\bfseries\large \stitle}}{{\bfseries\large Punktprozesse 2}}{{\bfseries\large 27. Oktober, 2015}}
-\firstpagefooter{Prof. Dr. Jan Benda}{Phone: 29 74573}{Email:
-jan.benda@uni-tuebingen.de}
-\runningfooter{}{\thepage}{}
-
-\setlength{\baselineskip}{15pt}
-\setlength{\parindent}{0.0cm}
-\setlength{\parskip}{0.3cm}
-\renewcommand{\baselinestretch}{1.15}
-
-%%%%% listings %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage{listings}
-\lstset{
-  language=Matlab,
-  basicstyle=\ttfamily\footnotesize,
-  numbers=left,
-  numberstyle=\tiny,
-  title=\lstname,
-  showstringspaces=false,
-  commentstyle=\itshape\color{darkgray},
-  breaklines=true,
-  breakautoindent=true,
-  columns=flexible,
-  frame=single,
-  xleftmargin=1em,
-  xrightmargin=1em,
-  aboveskip=10pt
-}
-
-%%%%% math stuff: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage{amsmath}
-\usepackage{amssymb}
-\usepackage{bm} 
-\usepackage{dsfont}
-\newcommand{\naZ}{\mathds{N}}
-\newcommand{\gaZ}{\mathds{Z}}
-\newcommand{\raZ}{\mathds{Q}}
-\newcommand{\reZ}{\mathds{R}}
-\newcommand{\reZp}{\mathds{R^+}}
-\newcommand{\reZpN}{\mathds{R^+_0}}
-\newcommand{\koZ}{\mathds{C}}
-
-%%%%% page breaks %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\newcommand{\continue}{\ifprintanswers%
-\else
-\vfill\hspace*{\fill}$\rightarrow$\newpage%
-\fi}
-\newcommand{\continuepage}{\ifprintanswers%
-\newpage
-\else
-\vfill\hspace*{\fill}$\rightarrow$\newpage%
-\fi}
-\newcommand{\newsolutionpage}{\ifprintanswers%
-\newpage%
-\else
-\fi}
-
-%%%%% new commands %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\newcommand{\qt}[1]{\textbf{#1}\\}
-\newcommand{\pref}[1]{(\ref{#1})}
-\newcommand{\extra}{--- Zusatzaufgabe ---\ \mbox{}}
-\newcommand{\code}[1]{\texttt{#1}}
+\newcommand{\exercisetopic}{Point Processes}
+\newcommand{\exercisenum}{X2}
+\newcommand{\exercisedate}{January 19th, 2021}
 
+\input{../../exercisesheader}
+
+\firstpagefooter{Prof. Dr. Jan Benda}{}{jan.benda@uni-tuebingen.de}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \begin{document}
 
-\input{instructions}
-
+\input{../../exercisestitle}
 
 \begin{questions}
 

pointprocesses/exercises/pointprocesses-3.tex

@@ -1,90 +1,17 @@
 \documentclass[12pt,a4paper,pdftex]{exam}
 
-\usepackage[german]{babel}
-\usepackage{pslatex}
-\usepackage[mediumspace,mediumqspace,Gray]{SIunits}      % \ohm, \micro
-\usepackage{xcolor}
-\usepackage{graphicx}
-\usepackage[breaklinks=true,bookmarks=true,bookmarksopen=true,pdfpagemode=UseNone,pdfstartview=FitH,colorlinks=true,citecolor=blue]{hyperref}
-
-%%%%% layout %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage[left=20mm,right=20mm,top=25mm,bottom=25mm]{geometry}
-\pagestyle{headandfoot}
-\ifprintanswers
-\newcommand{\stitle}{: L\"osungen}
-\else
-\newcommand{\stitle}{}
-\fi
-\header{{\bfseries\large \"Ubung 8\stitle}}{{\bfseries\large Spiketrain Analyse}}{{\bfseries\large 6. Dezember, 2016}}
-\firstpagefooter{Prof. Dr. Jan Benda}{Phone: 29 74573}{Email:
-jan.benda@uni-tuebingen.de}
-\runningfooter{}{\thepage}{}
-
-\setlength{\baselineskip}{15pt}
-\setlength{\parindent}{0.0cm}
-\setlength{\parskip}{0.3cm}
-\renewcommand{\baselinestretch}{1.15}
-
-%%%%% listings %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage{listings}
-\lstset{
-  language=Matlab,
-  basicstyle=\ttfamily\footnotesize,
-  numbers=left,
-  numberstyle=\tiny,
-  title=\lstname,
-  showstringspaces=false,
-  commentstyle=\itshape\color{darkgray},
-  breaklines=true,
-  breakautoindent=true,
-  columns=flexible,
-  frame=single,
-  xleftmargin=1em,
-  xrightmargin=1em,
-  aboveskip=10pt
-}
-
-%%%%% math stuff: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage{amsmath}
-\usepackage{amssymb}
-\usepackage{bm} 
-\usepackage{dsfont}
-\newcommand{\naZ}{\mathds{N}}
-\newcommand{\gaZ}{\mathds{Z}}
-\newcommand{\raZ}{\mathds{Q}}
-\newcommand{\reZ}{\mathds{R}}
-\newcommand{\reZp}{\mathds{R^+}}
-\newcommand{\reZpN}{\mathds{R^+_0}}
-\newcommand{\koZ}{\mathds{C}}
-
-%%%%% page breaks %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\newcommand{\continue}{\ifprintanswers%
-\else
-\vfill\hspace*{\fill}$\rightarrow$\newpage%
-\fi}
-\newcommand{\continuepage}{\ifprintanswers%
-\newpage
-\else
-\vfill\hspace*{\fill}$\rightarrow$\newpage%
-\fi}
-\newcommand{\newsolutionpage}{\ifprintanswers%
-\newpage%
-\else
-\fi}
-
-%%%%% new commands %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\newcommand{\qt}[1]{\textbf{#1}\\}
-\newcommand{\pref}[1]{(\ref{#1})}
-\newcommand{\extra}{--- Zusatzaufgabe ---\ \mbox{}}
-\newcommand{\code}[1]{\texttt{#1}}
+\newcommand{\exercisetopic}{Point Processes}
+\newcommand{\exercisenum}{X3}
+\newcommand{\exercisedate}{January 19th, 2021}
 
+\input{../../exercisesheader}
+
+\firstpagefooter{Prof. Dr. Jan Benda}{}{jan.benda@uni-tuebingen.de}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \begin{document}
 
-\input{instructions}
-
+\input{../../exercisestitle}
 
 \begin{questions}
   

pointprocesses/lecture/pointprocesses-chapter.tex

@@ -25,9 +25,9 @@
 \item Multitrial firing rates
 \item Better explain difference between ISI method and PSTHes. The
   latter is dependent on precision of spike times the former not.
-\item Choice of bin width for PSTH, kernel width, also in relation sto
+\item Choice of bin width for PSTH, kernel width, also in relation to
   stimulus time scale
-\item Kernle firing rate: discuss different kernel shapes, in
+\item Kernel firing rate: discuss different kernel shapes, in
   particular causal kernels (gamma, exponential), relation to synaptic
   potentials
 \end{itemize}

pointprocesses/lecture/pointprocessscetchA.eps

@@ -1,7 +1,7 @@
 %!PS-Adobe-2.0 EPSF-2.0
 %%Title: pointprocessscetchA.tex
 %%Creator: gnuplot 4.6 patchlevel 4
-%%CreationDate: Tue Oct 27 23:58:04 2020
+%%CreationDate: Mon Dec 14 23:59:13 2020
 %%DocumentFonts: 
 %%BoundingBox: 50 50 373 135
 %%EndComments
@@ -433,7 +433,7 @@ SDict begin [
   /Author (jan)
 %  /Producer (gnuplot)
 %  /Keywords ()
-  /CreationDate (Tue Oct 27 23:58:04 2020)
+  /CreationDate (Mon Dec 14 23:59:13 2020)
   /DOCINFO pdfmark
 end
 } ifelse

pointprocesses/lecture/pointprocessscetchB.eps

@@ -1,7 +1,7 @@
 %!PS-Adobe-2.0 EPSF-2.0
 %%Title: pointprocessscetchB.tex
 %%Creator: gnuplot 4.6 patchlevel 4
-%%CreationDate: Tue Oct 27 23:58:04 2020
+%%CreationDate: Mon Dec 14 23:59:13 2020
 %%DocumentFonts: 
 %%BoundingBox: 50 50 373 237
 %%EndComments
@@ -433,7 +433,7 @@ SDict begin [
   /Author (jan)
 %  /Producer (gnuplot)
 %  /Keywords ()
-  /CreationDate (Tue Oct 27 23:58:04 2020)
+  /CreationDate (Mon Dec 14 23:59:13 2020)
   /DOCINFO pdfmark
 end
 } ifelse

programming/code/vectorsize.m

@@ -1,4 +1,5 @@
-a = [2 4 6 8  10];   % row vector with five elements
-s = size(a) % store the return value of size() in a new variable
-s(2)        % get the second element of s, i.e. the length along the 2nd dimension
-size(a,2)   % the shortcut
+a = [2 4 6 8 10];   % row vector with five elements
+s = size(a)  % store the return value of size() in a new variable
+s(2)         % get the second element of s, 
+             % i.e. the length along the 2nd dimension
+size(a, 2)   % the shortcut

programming/code/vectorsize.out

@@ -1,3 +1,3 @@
-s =  1    10
-ans = 10
-ans = 10
+s =  1    5
+ans = 5
+ans = 5

programming/exercises/Makefile

@@ -0,0 +1,11 @@
+TEXFILES= \
+  boolean_logical_indexing.tex \
+  control_flow.tex \
+  matrices.tex \
+  scripts_functions.tex \
+  structs_cells.tex \
+  variables_types.tex \
+  vectors_matrices.tex \
+  vectors.tex
+
+include ../../exercises.mk

programming/exercises/boolean_logical_indexing.tex

@@ -1,39 +1,17 @@
-\documentclass[12pt, a4paper, pdftex]{exam}
-
-\usepackage[german]{babel}
-\usepackage{natbib}
-\usepackage{graphicx}
-\usepackage[small]{caption}
-\usepackage{sidecap}
-\usepackage{pslatex}
-\usepackage{amsmath}
-\usepackage{amssymb}
-\setlength{\marginparwidth}{2cm}
-\usepackage[breaklinks=true,bookmarks=true,bookmarksopen=true,pdfpagemode=UseNone,pdfstartview=FitH,colorlinks=true,citecolor=blue]{hyperref}
-
-%%%%% text size %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage[left=20mm,right=20mm,top=25mm,bottom=25mm]{geometry}
-\pagestyle{headandfoot} \header{{\bfseries\large Exercise 4
-    }}{{\bfseries\large Boolean expressions \& logical indexing}}{{\bfseries\large 17. November, 2020}}
-\firstpagefooter{Dr. Jan Grewe}{Phone: 29 74588}{Email:
-  jan.grewe@uni-tuebingen.de} \runningfooter{}{\thepage}{}
-
-\setlength{\baselineskip}{15pt}
-\setlength{\parindent}{0.0cm}
-\setlength{\parskip}{0.3cm}
-\renewcommand{\baselinestretch}{1.15}
-
-\newcommand{\code}[1]{\texttt{#1}}
+\documentclass[12pt,a4paper,pdftex]{exam}
+
+\newcommand{\exercisetopic}{Logical indexing}
+\newcommand{\exercisenum}{4}
+\newcommand{\exercisedate}{17. November, 2020}
+
+\input{../../exercisesheader}
+
+\firstpagefooter{Dr. Jan Grewe}{}{jan.grewe@uni-tuebingen.de}
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \begin{document}
 
-\vspace*{-6.5ex}
-\begin{center}
-  \textbf{\Large Introduction to scientific computing}\\[1ex]
-  {\large Jan Grewe, Jan Benda}\\[-3ex]
-  Abteilung Neuroethologie \hfill --- \hfill Institut f\"ur Neurobiologie \hfill --- \hfill \includegraphics[width=0.28\textwidth]{UT_WBMW_Black_RGB} \\
-\end{center}
+\input{../../exercisestitle}
 
 The exercises are meant for self-monitoring and revision of the
 lecture. You should try to solve them on your own. Your solution

programming/exercises/control_flow.tex

@@ -1,40 +1,17 @@
 \documentclass[12pt,a4paper,pdftex]{exam}
 
-\usepackage[german]{babel}
-\usepackage{natbib}
-\usepackage{graphicx}
-\usepackage[small]{caption}
-\usepackage{sidecap}
-\usepackage{pslatex}
-\usepackage{amsmath}
-\usepackage{amssymb}
-\setlength{\marginparwidth}{2cm}
-\usepackage[breaklinks=true,bookmarks=true,bookmarksopen=true,pdfpagemode=UseNone,pdfstartview=FitH,colorlinks=true,citecolor=blue]{hyperref}
+\newcommand{\exercisetopic}{Control flow}
+\newcommand{\exercisenum}{5}
+\newcommand{\exercisedate}{24. November, 2020}
 
-%%%%% text size %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage[left=20mm,right=20mm,top=25mm,bottom=25mm]{geometry}
-\pagestyle{headandfoot}
-\header{{\bfseries\large Exercise 5}}{{\bfseries\large Control Flow}}{{\bfseries\large 24. November, 2020}}
-\firstpagefooter{Dr. Jan Grewe}{Phone: 29 74588}{Email:
-  jan.grewe@uni-tuebingen.de}
-\runningfooter{}{\thepage}{}
+\input{../../exercisesheader}
 
-\setlength{\baselineskip}{15pt}
-\setlength{\parindent}{0.0cm}
-\setlength{\parskip}{0.3cm}
-\renewcommand{\baselinestretch}{1.15}
-
-\newcommand{\code}[1]{\texttt{#1}}
+\firstpagefooter{Dr. Jan Grewe}{}{jan.grewe@uni-tuebingen.de}
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \begin{document}
-\vspace*{-6.5ex}
 
-\begin{center}
-  \textbf{\Large Introduction to scientific computing}\\[1ex]
-  {\large Jan Grewe, Jan Benda}\\[-3ex]
-  Neuroethologie \hfill --- \hfill Institute for Neurobiology \hfill --- \hfill \includegraphics[width=0.28\textwidth]{UT_WBMW_Black_RGB} \\
-\end{center}
+\input{../../exercisestitle}
 
 The exercises are meant for self-monitoring and revision of the
 lecture. You should try to solve them on your own. Your solution

programming/exercises/matrices.tex

@@ -1,40 +1,17 @@
 \documentclass[12pt,a4paper,pdftex]{exam}
 
-\usepackage[german]{babel}
-\usepackage{natbib}
-\usepackage{graphicx}
-\usepackage[small]{caption}
-\usepackage{sidecap}
-\usepackage{pslatex}
-\usepackage{amsmath}
-\usepackage{amssymb}
-\setlength{\marginparwidth}{2cm}
-\usepackage[breaklinks=true,bookmarks=true,bookmarksopen=true,pdfpagemode=UseNone,pdfstartview=FitH,colorlinks=true,citecolor=blue]{hyperref}
-
-%%%%% text size %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage[left=20mm,right=20mm,top=25mm,bottom=25mm]{geometry}
-\pagestyle{headandfoot}
-\header{{\bfseries\large Exercise 3}}{{\bfseries\large Matrices}}{{\bfseries\large 10. Oktober, 2020}}
-\firstpagefooter{Dr. Jan Grewe}{Phone: 29 74588}{Email:
-  jan.grewe@uni-tuebingen.de}
-\runningfooter{}{\thepage}{}
-
-\setlength{\baselineskip}{15pt}
-\setlength{\parindent}{0.0cm}
-\setlength{\parskip}{0.3cm}
-\renewcommand{\baselinestretch}{1.15}
-
-\newcommand{\code}[1]{\texttt{#1}}
-\renewcommand{\solutiontitle}{\noindent\textbf{Solutions:}\par\noindent}
+\newcommand{\exercisetopic}{Matrices}
+\newcommand{\exercisenum}{3}
+\newcommand{\exercisedate}{22. October, 2019}
+
+\input{../../exercisesheader}
+
+\firstpagefooter{Dr. Jan Grewe}{}{jan.grewe@uni-tuebingen.de}
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \begin{document}
-\vspace*{-6.5ex}
-\begin{center}
-  \textbf{\Large Introduction to Scientific Computing}\\[1ex]
-  {\large Jan Grewe, Jan Benda}\\[-3ex]
-  Neuroethology \hfill --- \hfill Institute for Neurobiology \hfill --- \hfill \includegraphics[width=0.28\textwidth]{UT_WBMW_Black_RGB} \\
-\end{center}
+
+\input{../../exercisestitle}
 
 The exercises are meant for self-monitoring and revision of the
 lecture. You should try to solve them on your own. Your solution

programming/exercises/scripts_functions.tex

@@ -1,62 +1,17 @@
 \documentclass[12pt,a4paper,pdftex]{exam}
-%\documentclass[answers,12pt,a4paper,pdftex]{exam}
-
-\usepackage[german]{babel}
-\usepackage{natbib}
-\usepackage{xcolor}
-\usepackage{graphicx}
-\usepackage[small]{caption}
-\usepackage{sidecap}
-\usepackage{pslatex}
-\usepackage{amsmath}
-\usepackage{amssymb}
-\setlength{\marginparwidth}{2cm}
-\usepackage[breaklinks=true,bookmarks=true,bookmarksopen=true,pdfpagemode=UseNone,pdfstartview=FitH,colorlinks=true,citecolor=blue]{hyperref}
-
-%%%%% text size %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage[left=20mm,right=20mm,top=25mm,bottom=25mm]{geometry}
-\pagestyle{headandfoot}
-\header{{\bfseries\large Exercise 6}}{{\bfseries\large Scripts and functions}}{{\bfseries\large 01. December, 2020}}
-\firstpagefooter{Dr. Jan Grewe}{Phone: 29 74 588}{Email:
-  jan.grewe@uni-tuebingen.de}
-\runningfooter{}{\thepage}{}
-
-\setlength{\baselineskip}{15pt}
-\setlength{\parindent}{0.0cm}
-\setlength{\parskip}{0.3cm}
-\renewcommand{\baselinestretch}{1.15}
-
-
-%%%%% listings %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage{listings}
-\lstset{
-  language=Matlab,
-  basicstyle=\ttfamily\footnotesize,
-  numbers=left,
-  numberstyle=\tiny,
-  title=\lstname,
-  showstringspaces=false,
-  commentstyle=\itshape\color{darkgray},
-  breaklines=true,
-  breakautoindent=true,
-  columns=flexible,
-  frame=single,
-  xleftmargin=1em,
-  xrightmargin=1em,
-  aboveskip=10pt
-}
-
-\newcommand{\code}[1]{\texttt{#1}}
+
+\newcommand{\exercisetopic}{Scripts and Functions}
+\newcommand{\exercisenum}{6}
+\newcommand{\exercisedate}{1. December, 2020}
+
+\input{../../exercisesheader}
+
+\firstpagefooter{Dr. Jan Grewe}{}{jan.grewe@uni-tuebingen.de}
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \begin{document}
 
-\vspace*{-6.5ex}
-\begin{center}
-  \textbf{\Large Introduction to Scientific Computing}\\[1ex]
-         {\large Jan Grewe, Jan Benda}\\[-3ex]
-         Neuroethology \hfill --- \hfill Institute for Neurobiology \hfill --- \hfill \includegraphics[width=0.28\textwidth]{UT_WBMW_Black_RGB} \\
-\end{center}
+\input{../../exercisestitle}
 
 The exercises are meant for self-monitoring and revision of the
 lecture. You should try to solve them on your own. In contrast

programming/exercises/structs_cells.tex

@@ -1,40 +1,17 @@
 \documentclass[12pt,a4paper,pdftex]{exam}
 
-\usepackage[german]{babel}
-\usepackage{natbib}
-\usepackage{graphicx}
-\usepackage[small]{caption}
-\usepackage{sidecap}
-\usepackage{pslatex}
-\usepackage{amsmath}
-\usepackage{amssymb}
-\setlength{\marginparwidth}{2cm}
-\usepackage[breaklinks=true,bookmarks=true,bookmarksopen=true,pdfpagemode=UseNone,pdfstartview=FitH,colorlinks=true,citecolor=blue]{hyperref}
+\newcommand{\exercisetopic}{Structs and Cell Arrays}
+\newcommand{\exercisenum}{6}
+\newcommand{\exercisedate}{16. October, 2015}
 
-%%%%% text size %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage[left=20mm,right=20mm,top=25mm,bottom=25mm]{geometry}
-\pagestyle{headandfoot}
-\header{{\bfseries\large \"Ubung 6}}{{\bfseries\large Strukturen und Cell Arrays}}{{\bfseries\large 16. Oktober, 2015}}
-\firstpagefooter{Dr. Jan Grewe}{Phone: 29 74588}{Email:
-  jan.grewe@uni-tuebingen.de}
-\runningfooter{}{\thepage}{}
+\input{../../exercisesheader}
 
-\setlength{\baselineskip}{15pt}
-\setlength{\parindent}{0.0cm}
-\setlength{\parskip}{0.3cm}
-\renewcommand{\baselinestretch}{1.15}
-
-\newcommand{\code}[1]{\texttt{#1}}
+\firstpagefooter{Dr. Jan Grewe}{}{jan.grewe@uni-tuebingen.de}
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \begin{document}
 
-\vspace*{-6.5ex}
-\begin{center}
-  \textbf{\Large Einf\"uhrung in die wissenschaftliche Datenverarbeitung}\\[1ex]
-  {\large Jan Grewe, Jan Benda}\\[-3ex]
-  Abteilung Neuroethologie \hfill --- \hfill Institut f\"ur Neurobiologie \hfill --- \hfill \includegraphics[width=0.28\textwidth]{UT_WBMW_Black_RGB} \\
-\end{center}
+\input{../../exercisestitle}
 
 Die folgenden Aufgaben dienen der Wiederholung, \"Ubung und
 Selbstkontrolle und sollten eigenst\"andig bearbeitet und gel\"ost

programming/exercises/variables_types.tex

@@ -1,41 +1,17 @@
 \documentclass[12pt,a4paper,pdftex]{exam}
 
-\usepackage[english]{babel}
-\usepackage{natbib}
-\usepackage{graphicx}
-\usepackage[small]{caption}
-\usepackage{sidecap}
-\usepackage{pslatex}
-\usepackage{amsmath}
-\usepackage{amssymb}
-\usepackage{lipsum}
-\setlength{\marginparwidth}{2cm}
-\usepackage[breaklinks=true,bookmarks=true,bookmarksopen=true,pdfpagemode=UseNone,pdfstartview=FitH,colorlinks=true,citecolor=blue]{hyperref}
-
-%%%%% text size %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage[left=20mm,right=20mm,top=25mm,bottom=25mm]{geometry}
-\pagestyle{headandfoot}
-\header{{\bfseries\large Exercise 1}}{{\bfseries\large Variables und Datatypes}}{{\bfseries\large 03. November, 2020}}
-\firstpagefooter{Dr. Jan Grewe}{Phone: 29 74588}{Email:
-  jan.grewe@uni-tuebingen.de}
-\runningfooter{}{\thepage}{}
-
-\setlength{\baselineskip}{15pt}
-\setlength{\parindent}{0.0cm}
-\setlength{\parskip}{0.3cm}
-\renewcommand{\baselinestretch}{1.15}
-
-\newcommand{\code}[1]{\texttt{#1}}
-\renewcommand{\solutiontitle}{\noindent\textbf{Solutions:}\par\noindent}
+\newcommand{\exercisetopic}{Variables und Datatypes}
+\newcommand{\exercisenum}{1}
+\newcommand{\exercisedate}{3. November, 2020}
+
+\input{../../exercisesheader}
+
+\firstpagefooter{Dr. Jan Grewe}{}{jan.grewe@uni-tuebingen.de}
+
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \begin{document}
 
-\vspace*{-6.5ex}
-\begin{center}
-  \textbf{\Large Introduction to Scientific Computing}\\[1ex]
-  {\large Jan Grewe, Jan Benda}\\[-3ex]
-  Neuroethology \hfill --- \hfill Institute for Neurobiology \hfill --- \hfill \includegraphics[width=0.28\textwidth]{UT_WBMW_Black_RGB} \\
-\end{center}
+\input{../../exercisestitle}
 
 The exercises are meant for self-monitoring and revision of the
 lecture. You should try to solve them on your own. Your solution

programming/exercises/vectors.tex

@@ -1,40 +1,17 @@
 \documentclass[12pt,a4paper,pdftex]{exam}
 
-\usepackage[german]{babel}
-\usepackage{natbib}
-\usepackage{graphicx}
-\usepackage[small]{caption}
-\usepackage{sidecap}
-\usepackage{pslatex}
-\usepackage{amsmath}
-\usepackage{amssymb}
-\setlength{\marginparwidth}{2cm}
-\usepackage[breaklinks=true,bookmarks=true,bookmarksopen=true,pdfpagemode=UseNone,pdfstartview=FitH,colorlinks=true,citecolor=blue]{hyperref}
+\newcommand{\exercisetopic}{Vectors}
+\newcommand{\exercisenum}{2}
+\newcommand{\exercisedate}{3. November, 2020}
 
-%%%%% text size %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage[left=20mm,right=20mm,top=25mm,bottom=25mm]{geometry}
-\pagestyle{headandfoot}
-\header{{\bfseries\large Exercise 2}}{{\bfseries\large Vectors}}{{\bfseries\large 03. November, 2020}}
-\firstpagefooter{Dr. Jan Grewe}{Phone: 29 74588}{Email:
-  jan.grewe@uni-tuebingen.de}
-\runningfooter{}{\thepage}{}
+\input{../../exercisesheader}
 
-\setlength{\baselineskip}{15pt}
-\setlength{\parindent}{0.0cm}
-\setlength{\parskip}{0.3cm}
-\renewcommand{\baselinestretch}{1.15}
-
-\newcommand{\code}[1]{\texttt{#1}}
-\renewcommand{\solutiontitle}{\noindent\textbf{Solutions:}\par\noindent}
+\firstpagefooter{Dr. Jan Grewe}{}{jan.grewe@uni-tuebingen.de}
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \begin{document}
-\vspace*{-6.5ex}
-\begin{center}
-  \textbf{\Large Introduction to Scientific Computing}\\[1ex]
-         {\large Jan Grewe, Jan Benda}\\[-3ex]
-         Neuroethology \hfill --- \hfill Institute for Neurobiology \hfill --- \hfill \includegraphics[width=0.28\textwidth]{UT_WBMW_Black_RGB} \\
-\end{center}
+
+\input{../../exercisestitle}
 
 The exercises are meant for self-monitoring and revision of the
 lecture. You should try to solve them on your own. Your solution

programming/exercises/vectors_matrices.tex

@@ -1,40 +1,17 @@
-\documentclass[12pt,a4paper,pdftex, answers]{exam}
+\documentclass[12pt,a4paper,pdftex]{exam}
 
-\usepackage[german]{babel}
-\usepackage{natbib}
-\usepackage{graphicx}
-\usepackage[small]{caption}
-\usepackage{sidecap}
-\usepackage{pslatex}
-\usepackage{amsmath}
-\usepackage{amssymb}
-\setlength{\marginparwidth}{2cm}
-\usepackage[breaklinks=true,bookmarks=true,bookmarksopen=true,pdfpagemode=UseNone,pdfstartview=FitH,colorlinks=true,citecolor=blue]{hyperref}
+\newcommand{\exercisetopic}{Vectors}
+\newcommand{\exercisenum}{2}
+\newcommand{\exercisedate}{3. November, 2020}
 
-%%%%% text size %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage[left=20mm,right=20mm,top=25mm,bottom=25mm]{geometry}
-\pagestyle{headandfoot}
-\header{{\bfseries\large Exercise 2}}{{\bfseries\large Vectors}}{{\bfseries\large 03. November, 2020}}
-\firstpagefooter{Dr. Jan Grewe}{Phone: 29 74588}{Email:
-  jan.grewe@uni-tuebingen.de}
-\runningfooter{}{\thepage}{}
+\input{../../exercisesheader}
 
-\setlength{\baselineskip}{15pt}
-\setlength{\parindent}{0.0cm}
-\setlength{\parskip}{0.3cm}
-\renewcommand{\baselinestretch}{1.15}
-
-\newcommand{\code}[1]{\texttt{#1}}
-\renewcommand{\solutiontitle}{\noindent\textbf{Solutions:}\par\noindent}
+\firstpagefooter{Dr. Jan Grewe}{}{jan.grewe@uni-tuebingen.de}
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \begin{document}
-\vspace*{-6.5ex}
-\begin{center}
-  \textbf{\Large Introduction to Scientific Computing}\\[1ex]
-         {\large Jan Grewe, Jan Benda}\\[-3ex]
-         Neuroethology \hfill --- \hfill Institute for Neurobiology \hfill --- \hfill \includegraphics[width=0.28\textwidth]{UT_WBMW_Black_RGB} \\
-\end{center}
+
+\input{../../exercisestitle}
 
 The exercises are meant for self-monitoring and revision of the lecture
 topic. You should try to solve them on your own. Your solution should

programming/lecture/programming.tex

@@ -60,7 +60,8 @@ variable.
 In \matlab{} variables can be created at any time on the command line
 or any place in a script or function. Listing~\ref{varListing1} shows
 three different ways of creating a variable:
-\begin{lstlisting}[label=varListing1, caption={Creating variables.}]
+
+\begin{pagelisting}[label=varListing1, caption={Creating variables.}]
 >> x = 38
 x =
     38
@@ -72,7 +73,7 @@ y =
 >> z = 'A'
 z =
     A
-\end{lstlisting}
+\end{pagelisting}
 
 Line 1 can be read like: ``Create a variable with the name \varcode{x}
 and assign the value 38''. The equality sign is the so called
@@ -93,8 +94,7 @@ information but it is not suited to be used in programs (see
 also the \code{who} function that returns a list of all defined
 variables, listing~\ref{varListing2}).
 
-\newpage
-\begin{lstlisting}[label=varListing2, caption={Requesting information about defined variables and their types.}]
+\begin{pagelisting}[label=varListing2, caption={Requesting information about defined variables and their types.}]
 >>class(x)
 ans =
       double
@@ -110,7 +110,7 @@ x  y  z
   x         1x1                 8  double              
   y         0x0                 0  double              
   z         1x1                 2  char                
-\end{lstlisting}
+\end{pagelisting}
 
 \begin{important}[Naming conventions]
   There are a few rules regarding variable names. \matlab{} is
@@ -120,7 +120,6 @@ x  y  z
   variable names.
 \end{important}
 
-\pagebreak[4]
 \subsection{Working with variables}
 We can certainly work, i.e. do calculations, with variables. \matlab{}
 knows all basic \entermde[Operator!arithmetic]{Operator!arithmetischer}{arithmetic operators}
@@ -131,7 +130,7 @@ such as \code[Operator!arithmetic!1add@+]{+},
 \code[Operator!arithmetic!5pow@\^{}]{\^{}}. Listing~\ref{varListing3}
 shows their use.
 
-\begin{lstlisting}[label=varListing3, caption={Working with variables.}]
+\begin{pagelisting}[label=varListing3, caption={Working with variables.}]
 >> x = 1;
 >> x + 10
 ans = 
@@ -149,13 +148,12 @@ ans =
 z =
     3
 
->> z = z * 5;
->> z 
+>> z = z * 5
 z = 
     15
 
 >> clear z      % deleting a variable
-\end{lstlisting}
+\end{pagelisting}
 
 Note: in lines 2 and 10 the variables have been used without changing
 their values. Whenever the value of a variable should change, the
@@ -272,8 +270,8 @@ step-sizes unequal to 1. Line 5 can be read like: ``Create a variable
 \varcode{b} and assign the values from 0 to 9 in increasing steps of
 1.''. Line 9 reads: ``Create a variable \varcode{c} and assign the
 values from 0 to 10 in steps of 2''.
-\pagebreak
-\begin{lstlisting}[label=generatevectorslisting, caption={Creating simple row-vectors.}]
+
+\begin{pagelisting}[label=generatevectorslisting, caption={Creating simple row-vectors.}]
 >> a = [0 1 2 3 4 5 6 7 8 9]  % Creating a row-vector
 a =
     0  1   2   3   4   5   6   7   8   9
@@ -285,7 +283,7 @@ b =
 >> c = (0:2:10)                   
 c = 
     0  2  4  6  8  10
-\end{lstlisting}
+\end{pagelisting}
 
 The length of a vector, that is the number of elements, can be
 requested using the \code{length()} or \code{numel()}
@@ -293,14 +291,14 @@ functions. \code{size()} provides the same information in a slightly,
 yet more powerful way (listing~\ref{vectorsizeslisting}). The above
 used vector \varcode{a} has the following size:
 
-\begin{lstlisting}[label=vectorsizeslisting, caption={Size of a vector.}]
+\begin{pagelisting}[label=vectorsizeslisting, caption={Size of a vector.}]
 >> length(a)
 ans =
       10
 >> size(a)
 ans =
       1  10
-\end{lstlisting}
+\end{pagelisting}
 
 The answer provided by the \code{size()} function demonstrates that
 vectors are nothing else but 2-dimensional matrices in which one
@@ -311,7 +309,7 @@ create a column-vector and how the \code[Operator!Matrix!']{'} ---
 operator is used to transpose the column-vector into a row-vector
 (lines 14 and following).
 
-\begin{lstlisting}[label=columnvectorlisting, caption={Column-vectors.}]
+\begin{pagelisting}[label=columnvectorlisting, caption={Column-vectors.}]
 >> b = [1; 2; 3; 4; 5; 6; 7; 8; 9; 10]  % Creating a column-vector
 b =
      1
@@ -334,7 +332,7 @@ b =
 >> size(b)
 ans =
       1  10
-\end{lstlisting}
+\end{pagelisting}
 
 
 \subsubsection{Accessing elements of a vector}
@@ -355,7 +353,7 @@ number of elements irrespective of the type of vector.
   Elements of a vector are accessed via their index. This process is
   called \entermde{Indizierung}{indexing}.
 
-  In \matlab{} the first element has the index one.
+  In \matlab{} the first element in a vector has the index one.
   
   The last element's index equals the length of the vector.
 \end{important}
@@ -366,25 +364,25 @@ individual values by providing a single index or use the
 \code[Operator!Matrix!:]{:} operator to access multiple values with a
 single command (see also the info box below \ref{important:colon_operator}).
 
-\begin{lstlisting}[label=vectorelementslisting, caption={Access to individual elements of a vector.}]
->> a = (11:20)
+\begin{pagelisting}[label=vectorelementslisting, caption={Access to individual elements of a vector.}]
+>> a = (11:20)   % generate a vector
 a = 
     11    12    13    14    15    16    17    18    19    20
 
->> a(1)     % the 1. element
+>> a(1)          % the 1. element
 ans = 11 
->> a(5)     % the 5. element
+>> a(5)          % the 5. element
 ans = 15
->> a(end)   % the last element
+>> a(end)        % the last element
 ans = 20
-\end{lstlisting}
+\end{pagelisting}
 
-\begin{lstlisting}[caption={Access to multiple elements.}, label=vectorrangelisting]
+\begin{pagelisting}[caption={Access to multiple elements.}, label=vectorrangelisting]
 >> a([1 3 5])   % 1., 3. and 5. element
 ans =
       11  13  15
 
->> a(2:4)       % all elements with the indices 2 to 4
+>> a(2:4)       % elements at indices 2 to 4
 ans =
       12  13  14 
 
@@ -395,7 +393,7 @@ ans =
 >> a(:)         % all elements as row-vector
 ans = 
       11  12  13  14  15  16  17  18  19  20
-\end{lstlisting}
+\end{pagelisting}
 
 \begin{exercise}{vectorsize.m}{vectorsize.out}
   Create a row-vector \varcode{a} with 5 elements. The return value of
@@ -425,7 +423,7 @@ how vectors and scalars can be combined with the operators \code[Operator!arithm
 \code[Operator!arithmetic!4div@/]{/}
 \code[Operator!arithmetic!5powe@.\^{}]{.\^}.
 
-\begin{lstlisting}[caption={Calculating with vectors and scalars.},label=vectorscalarlisting]
+\begin{pagelisting}[caption={Calculations with vectors and scalars.},label=vectorscalarlisting]
 >> a = (0:2:8)
 a =
       0   2   4   6   8
@@ -449,7 +447,7 @@ ans =
 >> a .^ 2           % exponentiation
 ans =
       0   4  16  36  64
-\end{lstlisting}
+\end{pagelisting}
 
 When doing calculations with scalars and vectors the same mathematical
 operation is done to each element of the vector. In case of, e.g. an
@@ -462,7 +460,7 @@ element-wise operations of two vectors, e.g. each element of vector
 layout (row- or column vectors). Addition and subtraction are always
 element-wise (listing~\ref{vectoradditionlisting}).
 
-\begin{lstlisting}[caption={Element-wise addition and subtraction of two vectors.},label=vectoradditionlisting]
+\begin{pagelisting}[caption={Element-wise addition and subtraction of two vectors.},label=vectoradditionlisting]
 >> a = [4   9  12];
 >> b = [4   3   2];
 >> a + b           % addition 
@@ -481,7 +479,7 @@ Matrix dimensions must agree.
 >> a + d           % both vectors must have the same layout!
 Error using +
 Matrix dimensions must agree.
-\end{lstlisting}
+\end{pagelisting}
 
 Element-wise multiplication, division, or raising a vector to a given power requires a
 different operator with a preceding '.'.  \matlab{} defines the
@@ -491,7 +489,7 @@ following operators for element-wise operations on vectors
 \code[Operator!arithmetic!5powe@.\^{}]{.\^{}}
 (listing~\ref{vectorelemmultiplicationlisting}).
 
-\begin{lstlisting}[caption={Element-wise multiplication, division and
+\begin{pagelisting}[caption={Element-wise multiplication, division and
     exponentiation of two vectors.},label=vectorelemmultiplicationlisting]
 >> a .* b           % element-wise multiplication
 ans = 
@@ -511,7 +509,7 @@ Matrix dimensions must agree.
 >> a .* d           % Both vectors must have the same layout!
 Error using .*
 Matrix dimensions must agree.
-\end{lstlisting}
+\end{pagelisting}
 
 The simple operators \code[Operator!arithmetic!3mul@*]{*},
 \code[Operator!arithmetic!4div@/]{/} and
@@ -521,7 +519,7 @@ matrix-operations known from linear algebra (Box~
 of a row-vectors $\vec a$ with a column-vector $\vec b$ the
 scalar-poduct (or dot-product) $\sum_i = a_i b_i$.
 
-\begin{lstlisting}[caption={Multiplication of vectors.},label=vectormultiplicationlisting]
+\begin{pagelisting}[caption={Multiplication of vectors.},label=vectormultiplicationlisting]
 >> a * b            % multiplication of two vectors
 Error using  * 
 Inner matrix dimensions must agree.
@@ -538,14 +536,13 @@ ans =
     16    12     8
     36    27    18
     48    36    24
-\end{lstlisting}
-
-\pagebreak[4] 
+\end{pagelisting}
 
 To remove elements from a vector an empty value
 (\code[Operator!Matrix!{[]}]{[]}) is assigned to the respective
 elements:
-\begin{lstlisting}[label=vectoreraselisting, caption={Deleting elements of a vector.}]
+
+\begin{pagelisting}[label=vectoreraselisting, caption={Deleting elements of a vector.}]
 >> a = (0:2:8);
 >> length(a) 
 ans = 5
@@ -558,7 +555,7 @@ a = 4  8
 
 >> length(a)
 ans = 2
-\end{lstlisting}
+\end{pagelisting}
 
 In addition to deleting of vector elements one also add new elements
 or concatenate two vectors. When performing a concatenation the two
@@ -720,7 +717,7 @@ remapping, but can be really helpful
     rows in each column and so on.}\label{matrixlinearindexingfig}
 \end{figure}
 
-\begin{lstlisting}[label=matrixLinearIndexing, caption={Lineares indexing in matrices.}]
+\begin{pagelisting}[label=matrixLinearIndexing, caption={Lineares indexing in matrices.}]
 >> x = randi(100, [3, 4, 5]); % 3-D matrix filled with random numbers 
 >> size(x)
 ans =
@@ -737,17 +734,17 @@ ans =
 >> min(x(:))  % or even simpler
 ans =
 4
-\end{lstlisting}
+\end{pagelisting}
 
 \matlab{} defines functions that convert subscript indices to linear indices and back (\code{sub2ind()} and \code{ind2sub()}).
 
-\begin{ibox}[tp]{\label{matrixmultiplication} The matrix--multiplication.}
+\begin{ibox}[t]{\label{matrixmultiplication} The matrix--multiplication.}
   The matrix--multiplication from linear algebra is \textbf{not} an
   element--wise multiplication of each element in a matrix \varcode{A}
   and the respective element of matrix \varcode{B}. It is something
   completely different. Confusing element--wise and
   matrix--multiplication is one of the most common mistakes in
-  \matlab{}. \linebreak 
+  \matlab{}.
 
   The matrix--multiplication of two 2-D matrices is only possible if
   the number of columns in the first matrix agrees with the number of
@@ -797,8 +794,7 @@ box~\ref{matrixmultiplication}). To do a matrix-multiplication the
 inner dimensions of the matrices must agree
 (box~\ref{matrixmultiplication}).
 
-\pagebreak[4]
-\begin{lstlisting}[label=matrixOperations, caption={Two kinds of multiplications of matrices.}]
+\begin{pagelisting}[label=matrixOperations, caption={Two kinds of multiplications of matrices.}]
 >> A = randi(5, [2, 3])   % 2-D matrix 
 A =
      1     5     3
@@ -824,7 +820,7 @@ ans =
     10    15    20
     24    23    35
     16    17    25
-\end{lstlisting}
+\end{pagelisting}
 
 \section{Boolean expressions}
 
@@ -856,7 +852,7 @@ synonymous for the logical values 1 and
 0. Listing~\ref{logicaldatatype} exemplifies the use of the logical
 data type.
 
-\begin{lstlisting}[caption={The logical data type. Please note that the actual \matlab{} output looks a little different.}, label=logicaldatatype]
+\begin{pagelisting}[caption={The logical data type. Please note that the actual \matlab{} output looks a little different.}, label=logicaldatatype]
 >> true
 ans = 1
 >> false
@@ -873,7 +869,7 @@ ans = 0
 ans = 1   1   1   1
 >> logical([1 2 3 4 0 0 10])
 ans = 1   1   1   1   0   0   1
-\end{lstlisting}
+\end{pagelisting}
 
 \varcode{true} and \varcode{false} are reserved keywords that evaluate
 to the logical values 1 and 0, respectively. If you want to create a
@@ -886,7 +882,7 @@ code of each character in ``test'' is non-zero value, thus, the result
 of casting it to logical is a vector of logicals. A similar thing
 happens upon casting a vector (or matrix) of numbers to logical. Each
 value is converted to logical and the result is true for all non-zero
-values (line 21).
+values (line 15).
 
 Knowing how to represent true and false values in \matlab{} using the
 logical data type allows us to take a step towards more complex
@@ -924,8 +920,8 @@ stored in variable \varcode{b}?''.
 
 The result of such questions is then given as a logical
 value. Listing~\ref{relationaloperationslisting} shows examples using relational operators.
-\pagebreak
-\begin{lstlisting}[caption={Relational Boolean expressions.}, label=relationaloperationslisting]
+
+\begin{pagelisting}[caption={Relational Boolean expressions.}, label=relationaloperationslisting]
 >> true == logical(1)
 ans = 1
 >> false ~= logical(1)
@@ -943,7 +939,7 @@ ans = 0  0  1  0  0
 ans = 0  1  0  0  1 
 >> [2  0  0  5  0] >= [1  0  3  2  0]
 ans = 1  1  0  1  1 
-\end{lstlisting}
+\end{pagelisting}
 
 Testing the relations between numbers and scalar variables is straight
 forward. When comparing vectors, the relational operator will be
@@ -1040,7 +1036,7 @@ for implementing such
 expressions. Listing~\ref{logicaloperatorlisting} shows a few
 examples and respective illustrations are shown in figure~\ref{logicaloperationsfig}.
 
-\begin{lstlisting}[caption={Boolean expressions.}, label=logicaloperatorlisting]
+\begin{pagelisting}[caption={Boolean expressions.}, label=logicaloperatorlisting]
 >> x = rand(1)   % create a single random number in the range [0, 1]
 x = 0.3452
 >> x > 0.25 & x < 0.75
@@ -1051,7 +1047,7 @@ x = 0.4920, 0.9106, 0.7218, 0.8749, 0.1574, 0.0201, 0.9107, 0.8357, 0.0357, 0.47
 ans = 1  0  1  0  0  0  0  0  0  1
 >> x < 0.25 | x > 0.75  
 ans = 0  1  0  1  1  1  1  1  1  0
-\end{lstlisting}
+\end{pagelisting}
 
 \begin{figure}[ht]
   \includegraphics[]{logical_operations}
@@ -1066,7 +1062,6 @@ ans = 0  1  0  1  1  1  1  1  1  0
     data.}\label{logicaloperationsfig}
 \end{figure}
 
-\pagebreak
 \begin{important}[Assignment and equality operators]
   The assignment operator \code[Operator!Assignment!=]{=} and the
   relational equality operator \code[Operator!relational!==]{==} are
@@ -1095,7 +1090,7 @@ elements of \varcode{x} where the Boolean expression \varcode{x < 0}
 evaluates to true and store the result in the variable
 \varcode{x\_smaller\_zero}''.
 
-\begin{lstlisting}[caption={Logical indexing.}, label=logicalindexing1]
+\begin{pagelisting}[caption={Logical indexing.}, label=logicalindexing1]
 >> x = randn(1, 6)   % a vector with 6 random numbers
 x =
    -1.4023   -1.4224    0.4882   -0.1774   -0.1961    1.4193
@@ -1112,7 +1107,7 @@ elements_smaller_zero =
 >> elements_smaller_zero = x(x < 0)
 elements_smaller_zero =
    -1.4023   -1.4224   -0.1774   -0.1961
-\end{lstlisting}
+\end{pagelisting}
 
 \begin{exercise}{logicalVector.m}{logicalVector.out}
   Create a vector \varcode{x} containing the values 0--10.
@@ -1121,8 +1116,7 @@ elements_smaller_zero =
   \item Display the content of \varcode{y} in the command window.
   \item  What is the data type of \varcode{y}?
   \item  Return only those elements \varcode{x} that are less than 5.
-  \end{enumerate}
-  \pagebreak[4]
+  \end{enumerate}\vspace{-1ex}
 \end{exercise}
 
 \begin{figure}[t]
@@ -1144,7 +1138,6 @@ segment of data of a certain time span (the stimulus was on,
 \begin{exercise}{logicalIndexingTime.m}{}
   Assume that measurements have been made for a certain time. Usually
   measured values and the time are stored in two vectors.
-  
   \begin{itemize}
   \item Create a vector that represents the recording time \varcode{t
     = 0:0.001:10;}.
@@ -1152,9 +1145,9 @@ segment of data of a certain time span (the stimulus was on,
     that has the same length as \varcode{t}. The values stored in
     \varcode{x} represent the measured data at the times in
     \varcode{t}.
-  \item Use logical indexing to select those values that have been
-    recorded in the time span from 5--6\,s.
-  \end{itemize}
+  \item Use logical indexing to select values that have been
+    recorded in the time span 5--6\,s.
+  \end{itemize}\vspace{-1ex}
 \end{exercise}
 
 \begin{ibox}[!ht]{\label{advancedtypesbox}Advanced data types}
@@ -1276,7 +1269,7 @@ As the name already suggests loops are used to execute the same parts
 of the code repeatedly. In one of the earlier exercises the factorial of
 five has been calculated as depicted in listing~\ref{facultylisting}.
 
-\begin{lstlisting}[caption={Calculation of the factorial of 5 in five steps}, label=facultylisting]
+\begin{pagelisting}[caption={Calculation of the factorial of 5 in five steps}, label=facultylisting]
 >> x = 1;
 >> x = x * 2;
 >> x = x * 3;  
@@ -1285,7 +1278,7 @@ five has been calculated as depicted in listing~\ref{facultylisting}.
 >> x
 x =
     120
-\end{lstlisting}
+\end{pagelisting}
 
 This kind of program solves the taks but it is rather repetitive. The only
 thing that changes is the increasing factor. The repetition of such
@@ -1325,7 +1318,7 @@ a certain purpose. The \varcode{for}-loop is closed with the keyword
 \code{end}. Listing~\ref{looplisting} shows a simple version of such a
 for-loop.
 
-\begin{lstlisting}[caption={Example of a \varcode{for}-loop.}, label=looplisting]
+\begin{pagelisting}[caption={Example of a \varcode{for}-loop.}, label=looplisting]
 >> for x = 1:3 % head
      disp(x)   % body
    end
@@ -1334,7 +1327,7 @@ for-loop.
      1
      2
      3
-\end{lstlisting}
+\end{pagelisting}
 
 
 \begin{exercise}{factorialLoop.m}{factorialLoop.out}
@@ -1354,11 +1347,11 @@ keyword \code{while} that is followed by a Boolean expression. If this
 can be evaluated to true, the code in the body is executed. The loop
 is closed with an \code{end}.
 
-\begin{lstlisting}[caption={Basic structure of a \varcode{while} loop.}, label=whileloop]
-while x == true % head with a Boolean expression
+\begin{pagelisting}[caption={Basic structure of a \varcode{while} loop.}, label=whileloop]
+while x == true   % head with a Boolean expression
    % execute this code if the expression yields true
 end
-\end{lstlisting}
+\end{pagelisting}
 
 \begin{exercise}{factorialWhileLoop.m}{}
   Implement the factorial of a number \varcode{n} using a \varcode{while}-loop.
@@ -1413,7 +1406,7 @@ expression to provide a default case. The last body of the
 \varcode{if} - \varcode{elseif} - \varcode{else} statement has to be
 finished with the \code{end} (listing~\ref{ifelselisting}).
   
-\begin{lstlisting}[label=ifelselisting, caption={Structure of an \varcode{if} statement.}]
+\begin{pagelisting}[label=ifelselisting, caption={Structure of an \varcode{if} statement.}]
 if x < y % head
    % body I, executed only if x < y
 elseif x > y
@@ -1421,7 +1414,7 @@ elseif x > y
 else
    % body III, executed only if the previous conditions did not match
 end  
-  \end{lstlisting}
+  \end{pagelisting}
 
 \begin{exercise}{ifelse.m}{}
   Draw a random number and check with an appropriate \varcode{if}
@@ -1449,7 +1442,7 @@ that were not explicitly stated above (listing~\ref{switchlisting}).
 As usual the \code{switch} statement needs to be closed with an
 \code{end}.
 
-\begin{lstlisting}[label=switchlisting, caption={Structure of a \varcode{switch} statement.}]
+\begin{pagelisting}[label=switchlisting, caption={Structure of a \varcode{switch} statement.}]
 mynumber = input('Enter a number:');
 switch mynumber
   case -1
@@ -1459,7 +1452,7 @@ switch mynumber
   otherwise
       disp('something else');
 end
-\end{lstlisting}
+\end{pagelisting}
 
 
 \subsubsection{Comparison \varcode{if}  and \varcode{switch} -- statements}
@@ -1483,7 +1476,7 @@ skip the execution of the body under certain circumstances, one can
 use the keywords \code{break} and \code{continue}
 (listings~\ref{continuelisting} and \ref{continuelisting}).
 
-\begin{lstlisting}[caption={Stop the execution of a loop using \varcode{break}.}, label=breaklisting]
+\begin{pagelisting}[caption={Stop the execution of a loop using \varcode{break}.}, label=breaklisting]
 >> x = 1;
    while true
      if (x > 3)
@@ -1496,9 +1489,9 @@ use the keywords \code{break} and \code{continue}
      1
      2
      3
-\end{lstlisting}
+\end{pagelisting}
 
-\begin{lstlisting}[caption={Skipping iterations using \varcode{continue}.}, label=continuelisting]
+\begin{pagelisting}[caption={Skipping iterations using \varcode{continue}.}, label=continuelisting]
 for x = 1:5 
   if(x > 2 & x < 5)
     continue;
@@ -1509,7 +1502,7 @@ end
      1
      2
      5
-\end{lstlisting}
+\end{pagelisting}
 
 \begin{exercise}{logicalIndexingBenchmark.m}{logicalIndexingBenchmark.out}
   Above we claimed that logical indexing is faster and much more
@@ -1581,11 +1574,11 @@ has one \entermde{Argument}{argument} $x$ that is transformed into the
 function's output value $y$. In \matlab{} the syntax of a function
 declaration is very similar (listing~\ref{functiondefinitionlisting}).
 
-\begin{lstlisting}[caption={Declaration of a function in \matlab{}}, label=functiondefinitionlisting]
+\begin{pagelisting}[caption={Declaration of a function in \matlab{}}, label=functiondefinitionlisting]
 function [y] = functionName(arg_1, arg_2)
 %         ^                  ^       ^
 % return value        argument_1, argument_2
-\end{lstlisting}
+\end{pagelisting}
 
 The keyword \code{function} is followed by the return value(s) (it can
 be a list \varcode{[]} of values), the function name and the
@@ -1618,7 +1611,7 @@ The following listing (\ref{badsinewavelisting}) shows a function that
 calculates and displays a bunch of sine waves with different amplitudes.
 
 
-\begin{lstlisting}[caption={Bad example of a function that displays a series of sine waves.},label=badsinewavelisting]
+\begin{pagelisting}[caption={Bad example of a function that displays a series of sine waves.},label=badsinewavelisting]
 function myFirstFunction() % function head
    t = (0:0.01:2); 
    frequency = 1.0;
@@ -1629,7 +1622,7 @@ function myFirstFunction() % function head
        hold on;
    end 
 end
-\end{lstlisting}
+\end{pagelisting}
 
 \varcode{myFirstFunction} (listing~\ref{badsinewavelisting}) is a
 prime-example of a bad function. There are several issues with it's
@@ -1690,7 +1683,7 @@ define (i) how to name the function, (ii) which information it needs
 Having defined this we can start coding
 (listing~\ref{sinefunctionlisting}).
 
-\begin{lstlisting}[caption={Function that calculates a sine wave.}, label=sinefunctionlisting]
+\begin{pagelisting}[caption={Function that calculates a sine wave.}, label=sinefunctionlisting]
 function [time, sine] = sinewave(frequency, amplitude, t_max, t_step)
 % Calculate a sinewave of a given frequency, amplitude, 
 % duration and temporal resolution. 
@@ -1708,7 +1701,7 @@ function [time, sine] = sinewave(frequency, amplitude, t_max, t_step)
    time = (0:t_step:t_max);
    sine = sin(frequency .* time .* 2 .* pi) .* amplitude;
 end
-\end{lstlisting}
+\end{pagelisting}
   
 
 \paragraph{II. Plotting a single sine wave}
@@ -1735,7 +1728,7 @@ specification of the function:
 With this specification we can start to implement the function
 (listing~\ref{sineplotfunctionlisting}).
 
-\begin{lstlisting}[caption={Function for the graphical display of data.}, label=sineplotfunctionlisting]
+\begin{pagelisting}[caption={Function for the graphical display of data.}, label=sineplotfunctionlisting]
 function plotFunction(x_data, y_data, name)
 % Plots x-data against y-data and sets the display name.
 %
@@ -1747,7 +1740,7 @@ function plotFunction(x_data, y_data, name)
 %   name  : the displayname
   plot(x_data, y_data, 'displayname', name) 
 end
-\end{lstlisting}
+\end{pagelisting}
 
 \begin{ibox}[!ht]{\label{box:sprintf}\enterm[Format strings]{Formatted strings} with \mcode{sprintf}.}
     We can use \varcode{disp} to dispaly the content of a variable on the command line. But how can we embed variable contents nicely into a text and store it in a variable or show it on the command line? \varcode{sprintf} is the standard function to achieve this. We introduce only a fraction of its real might here. For a more complete description visit the \matlab{} help or e.g. \url{https://en.wikipedia.org/wiki/Printf_format_string}. 
@@ -1820,11 +1813,11 @@ for i = 1:length(amplitudes)
 end
 hold off
 legend('show')
-\end{lstlisting}
+\end{pagelisting}
 
+\pagebreak[4]
 \begin{exercise}{plotMultipleSinewaves.m}{}
   Extend the program to plot also a range of frequencies.
-  \pagebreak[4]
 \end{exercise}
 
 

regression/code/gradientDescent.m

@@ -1,32 +1,42 @@
-% x, y from exercise 8.3
+function [p, ps, mses] = gradientDescent(x, y, func, p0, epsilon, threshold)
+% Gradient descent for fitting a function to data pairs.
+%
+% Arguments: x, vector of the x-data values.
+%            y, vector of the corresponding y-data values.
+%            func, function handle func(x, p)
+%            p0, vector with initial parameter values
+%            epsilon: factor multiplying the gradient.
+%            threshold: minimum value for gradient
+%
+% Returns:   p, vector with the final parameter values.
+%            ps: 2D-vector with all the parameter vectors traversed.
+%            mses: vector with the corresponding mean squared errors
 
-% some arbitrary values for the slope and the intercept to start with:
-position = [-2.0, 10.0];  
+  p = p0;
+  gradient = ones(1, length(p0)) * 1000.0;
+  ps = [];
+  mses = [];
+  while norm(gradient) > threshold
+      ps = [ps, p(:)];
+      mses = [mses, meanSquaredError(x, y, func, p)];
+      gradient = meanSquaredGradient(x, y, func, p);
+      p = p - epsilon * gradient;
+  end
+end
+
+function mse = meanSquaredError(x, y, func, p)
+    mse = mean((y - func(x, p)).^2);
+end
 
-% gradient descent:
-gradient = [];
-errors = [];
-count = 1;
-eps = 0.0001;
-while isempty(gradient) || norm(gradient) > 0.1
-    gradient = meanSquaredGradient(x, y, position);
-    errors(count) = meanSquaredError(x, y, position);
-    position = position - eps .* gradient; 
-    count = count + 1;
+function gradmse = meanSquaredGradient(x, y, func, p)
+  gradmse = zeros(size(p, 1), size(p, 2));
+  h = 1e-7;               % stepsize for derivatives
+  mse = meanSquaredError(x, y, func, p);
+  for i = 1:length(p)     % for each coordinate ...
+    pi = p;
+    pi(i) = pi(i) + h;    %   displace i-th parameter
+    msepi = meanSquaredError(x, y, func, pi);
+    gradmse(i) = (msepi - mse)/h;
+  end
 end
 
-figure()
-subplot(2,1,1)
-hold on
-scatter(x, y, 'displayname', 'data')
-xx = min(x):0.01:max(x);
-yy = position(1).*xx + position(2);
-plot(xx, yy, 'displayname', 'fit')
-xlabel('Input')
-ylabel('Output')
-grid on
-legend show
-subplot(2,1,2)
-plot(errors)
-xlabel('optimization steps')
-ylabel('error')

regression/code/gradientDescentCubic.m

@@ -0,0 +1,25 @@
+function [c, cs, mses] = gradientDescentCubic(x, y, c0, epsilon, threshold)
+% Gradient descent for fitting a cubic relation.
+%
+% Arguments: x, vector of the x-data values.
+%            y, vector of the corresponding y-data values.
+%            c0, initial value for the parameter c. 
+%            epsilon: factor multiplying the gradient.
+%            threshold: minimum value for gradient
+%
+% Returns:   c, the final value of the c-parameter.
+%            cs: vector with all the c-values traversed.
+%            mses: vector with the corresponding mean squared errors
+  c = c0;
+  gradient = 1000.0;
+  cs = [];
+  mses = [];
+  count = 1;
+  while abs(gradient) > threshold
+      cs(count) = c;
+      mses(count) = meanSquaredErrorCubic(x, y, c);
+      gradient = meanSquaredGradientCubic(x, y, c);
+      c = c - epsilon * gradient; 
+      count = count + 1;
+  end
+end

regression/code/meanSquaredError.m

@@ -1,12 +0,0 @@
-function mse = meanSquaredError(x, y, parameter)
-% Mean squared error between a straight line and data pairs.
-%
-% Arguments: x, vector of the input values
-%            y, vector of the corresponding measured output values
-%            parameter, vector containing slope and intercept 
-%                       as the 1st and 2nd element, respectively. 
-%
-% Returns:   mse, the mean-squared-error.
-
-  mse = mean((y - x * parameter(1) - parameter(2)).^2);
-end

regression/code/meanSquaredErrorCubic.m

@@ -0,0 +1,11 @@
+function mse = meanSquaredErrorCubic(x, y, c)
+% Mean squared error between data pairs and a cubic relation.
+%
+% Arguments: x, vector of the x-data values
+%            y, vector of the corresponding y-data values
+%            c, the factor for the cubic relation. 
+%
+% Returns:   mse, the mean-squared-error.
+
+  mse = mean((y - c*x.^3).^2);
+end

regression/code/meanSquaredErrorLine.m

@@ -1 +0,0 @@
-mse = mean((y - y_est).^2);

regression/code/meanSquaredGradientCubic.m

@@ -0,0 +1,14 @@
+function dmsedc = meanSquaredGradientCubic(x, y, c)
+% The gradient of the mean squared error for a cubic relation.
+%
+% Arguments: x, vector of the x-data values
+%            y, vector of the corresponding y-data values
+%            c, the factor for the cubic relation. 
+%
+% Returns: the derivative of the mean squared error at c.
+
+  h = 1e-7;   % stepsize for derivatives
+  mse = meanSquaredErrorCubic(x, y, c);
+  mseh = meanSquaredErrorCubic(x, y, c+h);
+  dmsedc = (mseh - mse)/h;
+end

regression/code/meansquarederrorline.m

@@ -0,0 +1,15 @@
+n = 40;
+xmin = 2.2;
+xmax = 3.9;
+c = 6.0;
+noise = 50.0;
+
+% generate data:
+x = rand(n, 1) * (xmax-xmin) + xmin;
+yest = c * x.^3;
+y = yest + noise*randn(n, 1);
+
+% compute mean squared error:
+mse = mean((y - yest).^2);
+
+fprintf('the mean squared error is %.0f kg^2\n', mse)

regression/code/plotcubiccosts.m

@@ -0,0 +1,9 @@
+cs = 2.0:0.1:8.0;
+mses = zeros(length(cs));
+for i = 1:length(cs)
+    mses(i) = meanSquaredErrorCubic(x, y, cs(i));
+end
+
+plot(cs, mses)
+xlabel('c')
+ylabel('mean squared error')

regression/code/plotcubicgradient.m

@@ -0,0 +1,11 @@
+meansquarederrorline;               % generate data
+
+cs = 2.0:0.1:8.0;
+mseg = zeros(length(cs));
+for i = 1:length(cs)
+    mseg(i) = meanSquaredGradientCubic(x, y, cs(i));
+end
+
+plot(cs, mseg)
+xlabel('c')
+ylabel('gradient')

regression/code/plotgradientdescentcubic.m

@@ -0,0 +1,28 @@
+meansquarederrorline;               % generate data
+
+c0 = 2.0;
+eps = 0.00001;
+thresh = 1.0;
+[cest, cs, mses] = gradientDescentCubic(x, y, c0, eps, thresh);
+
+subplot(2, 2, 1);                   % top left panel
+hold on;
+plot(cs, '-o');
+plot([1, length(cs)], [c, c], 'k'); % line indicating true c value
+hold off;
+xlabel('Iteration');
+ylabel('C');
+subplot(2, 2, 3);                   % bottom left panel
+plot(mses, '-o');
+xlabel('Iteration steps');
+ylabel('MSE');
+subplot(1, 2, 2);                   % right panel
+hold on;
+% generate x-values for plottig the fit:
+xx = min(x):0.01:max(x);
+yy = cest * xx.^3;
+plot(xx, yy);
+plot(x, y, 'o');                    % plot original data
+xlabel('Size [m]');
+ylabel('Weight [kg]');
+legend('fit', 'data', 'location', 'northwest');

regression/code/plotgradientdescentpower.m

@@ -0,0 +1,30 @@
+meansquarederrorline;               % generate data
+
+p0 = [2.0, 1.0];
+eps = 0.00001;
+thresh = 1.0;
+[pest, ps, mses] = gradientDescent(x, y, @powerLaw, p0, eps, thresh);
+pest
+
+subplot(2, 2, 1);                   % top left panel
+hold on;
+plot(ps(1,:), ps(2,:), '.');
+plot(ps(1,end), ps(2,end), 'og');
+plot(c, 3.0, 'or'); % dot indicating true parameter values
+hold off;
+xlabel('Iteration');
+ylabel('C');
+subplot(2, 2, 3);                   % bottom left panel
+plot(mses, '-o');
+xlabel('Iteration steps');
+ylabel('MSE');
+subplot(1, 2, 2);                   % right panel
+hold on;
+% generate x-values for plottig the fit:
+xx = min(x):0.01:max(x);
+yy = powerLaw(xx, pest);
+plot(xx, yy);
+plot(x, y, 'o');                    % plot original data
+xlabel('Size [m]');
+ylabel('Weight [kg]');
+legend('fit', 'data', 'location', 'northwest');

regression/exercises/Makefile

@@ -1,34 +1,4 @@
-TEXFILES=$(wildcard exercises??.tex)
-EXERCISES=$(TEXFILES:.tex=.pdf)
-SOLUTIONS=$(EXERCISES:exercises%=solutions%)
+TEXFILES=$(wildcard gradientdescent-?.tex)
 
-.PHONY: pdf exercises solutions watch watchexercises watchsolutions clean
+include ../../exercises.mk
 
-pdf : $(SOLUTIONS) $(EXERCISES)
-
-exercises : $(EXERCISES)
-
-solutions : $(SOLUTIONS)
-
-$(SOLUTIONS) : solutions%.pdf : exercises%.tex instructions.tex
-	{ echo "\\documentclass[answers,12pt,a4paper,pdftex]{exam}"; sed -e '1d' $<; } > $(patsubst %.pdf,%.tex,$@)
-	pdflatex -interaction=scrollmode $(patsubst %.pdf,%.tex,$@) | tee /dev/stderr | fgrep -q "Rerun to get cross-references right" && pdflatex -interaction=scrollmode $(patsubst %.pdf,%.tex,$@) || true
-	rm $(patsubst %.pdf,%,$@).[!p]*
-
-$(EXERCISES) : %.pdf : %.tex instructions.tex
-	pdflatex -interaction=scrollmode $< | tee /dev/stderr | fgrep -q "Rerun to get cross-references right" && pdflatex -interaction=scrollmode $< || true
-
-watch :
-	while true; do ! make -q pdf && make pdf; sleep 0.5; done
-
-watchexercises :
-	while true; do ! make -q exercises && make exercises; sleep 0.5; done
-
-watchsolutions :
-	while true; do ! make -q solutions && make solutions; sleep 0.5; done
-
-clean :
-	rm -f *~ *.aux *.log *.out
-
-cleanup : clean
-	rm -f $(SOLUTIONS) $(EXERCISES)

regression/exercises/gradientdescent-1.tex

@@ -1,60 +1,17 @@
 \documentclass[12pt,a4paper,pdftex]{exam}
 
-\usepackage[german]{babel}
-\usepackage{natbib}
-\usepackage{xcolor}
-\usepackage{graphicx}
-\usepackage[small]{caption}
-\usepackage{sidecap}
-\usepackage{pslatex}
-\usepackage{amsmath}
-\usepackage{amssymb}
-\setlength{\marginparwidth}{2cm}
-\usepackage[breaklinks=true,bookmarks=true,bookmarksopen=true,pdfpagemode=UseNone,pdfstartview=FitH,colorlinks=true,citecolor=blue]{hyperref}
-
-%%%%% text size %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage[left=20mm,right=20mm,top=25mm,bottom=25mm]{geometry}
-\pagestyle{headandfoot}
-\ifprintanswers
-\newcommand{\stitle}{: Solutions}
-\else
-\newcommand{\stitle}{}
-\fi
-\header{{\bfseries\large Exercise 10\stitle}}{{\bfseries\large Gradient descent}}{{\bfseries\large December 16th, 2019}}
-\firstpagefooter{Dr. Jan Grewe}{Phone: 29 74588}{Email:
-  jan.grewe@uni-tuebingen.de}
-\runningfooter{}{\thepage}{}
-
-\setlength{\baselineskip}{15pt}
-\setlength{\parindent}{0.0cm}
-\setlength{\parskip}{0.3cm}
-\renewcommand{\baselinestretch}{1.15}
-
-\newcommand{\code}[1]{\texttt{#1}}
-\renewcommand{\solutiontitle}{\noindent\textbf{Solution:}\par\noindent}
-%%%%% listings %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\usepackage{listings}
-\lstset{
-  language=Matlab,
-  basicstyle=\ttfamily\footnotesize,
-  numbers=left,
-  numberstyle=\tiny,
-  title=\lstname,
-  showstringspaces=false,
-  commentstyle=\itshape\color{darkgray},
-  breaklines=true,
-  breakautoindent=true,
-  columns=flexible,
-  frame=single,
-  xleftmargin=1em,
-  xrightmargin=1em,
-  aboveskip=10pt
-}
+\newcommand{\exercisetopic}{Resampling}
+\newcommand{\exercisenum}{9}
+\newcommand{\exercisedate}{December 22th, 2020}
+
+\input{../../exercisesheader}
+
+\firstpagefooter{Prof. Dr. Jan Benda}{}{jan.benda@uni-tuebingen.de}
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \begin{document}
 
-\input{instructions}
+\input{../../exercisestitle}
 
 \begin{questions}
 
@@ -96,7 +53,7 @@
     the parameter values at the minimum of the cost function and a vector
     with the value of the cost function at each step of the algorithm.
     \begin{solution}
-      \lstinputlisting{../code/descent.m}
+      \lstinputlisting{descent.m}
     \end{solution}
 
     \part Plot the data and the straight line with the parameter
@@ -105,7 +62,7 @@
     \part Plot the development of the costs as a function of the
     iteration step.
     \begin{solution}
-      \lstinputlisting{../code/descentfit.m}
+      \lstinputlisting{descentfit.m}
     \end{solution}
 
     \part For checking the gradient descend method from (a) compare
@@ -116,7 +73,7 @@
     minimum gradient. What are good values such that the gradient
     descent gets closest to the true minimum of the cost function?
     \begin{solution}
-      \lstinputlisting{../code/checkdescent.m}
+      \lstinputlisting{checkdescent.m}
     \end{solution}
 
     \part Use the functions \code{polyfit()} and \code{lsqcurvefit()}
@@ -124,7 +81,7 @@
     line that fits the data. Compare the resulting fit parameters of
     those functions with the ones of your gradient descent algorithm.
     \begin{solution}
-      \lstinputlisting{../code/linefit.m}
+      \lstinputlisting{linefit.m}
     \end{solution}
 
   \end{parts}

regression/exercises/instructions.tex

@@ -1,6 +0,0 @@
-\vspace*{-7.8ex}
-\begin{center}
-\textbf{\Large Introduction to Scientific Computing}\\[2.3ex]
-{\large Jan Grewe, Jan Benda}\\[-3ex]
-Neuroethology Lab \hfill --- \hfill Institute for Neurobiology \hfill --- \hfill \includegraphics[width=0.28\textwidth]{UT_WBMW_Black_RGB} \\
-\end{center}

regression/lecture/cubiccost.py

@@ -0,0 +1,82 @@
+import numpy as np
+import matplotlib.pyplot as plt
+import matplotlib.ticker as mt
+from plotstyle import *
+
+def create_data():
+    # wikipedia:
+    # Generally, males vary in total length from 250 to 390 cm and
+    # weigh between 90 and 306 kg
+    c = 6
+    x = np.arange(2.2, 3.9, 0.05)
+    y = c * x**3.0
+    rng = np.random.RandomState(32281)
+    noise = rng.randn(len(x))*50
+    y += noise
+    return x, y, c
+
+    
+def plot_mse(ax, x, y, c):
+    ccs = np.linspace(0.5, 10.0, 200)
+    mses = np.zeros(len(ccs))
+    for i, cc in enumerate(ccs):
+        mses[i] = np.mean((y-(cc*x**3.0))**2.0)
+    imin = np.argmin(mses)
+        
+    ax.plot(ccs, mses, **lsAm)
+    ax.plot(c, 500.0, **psB)
+    ax.plot(ccs[imin], mses[imin], **psC)
+    ax.annotate('Minimum of\ncost\nfunction',
+                xy=(ccs[imin], mses[imin]*1.2), xycoords='data',
+                xytext=(4, 7000), textcoords='data', ha='left',
+                arrowprops=dict(arrowstyle="->", relpos=(0.2,0.0),
+                connectionstyle="angle3,angleA=10,angleB=90") )
+    ax.text(2.2, 500, 'True\nparameter\nvalue')
+    ax.annotate('', xy=(c-0.2, 500), xycoords='data',
+                xytext=(4.1, 700), textcoords='data', ha='left',
+                arrowprops=dict(arrowstyle="->", relpos=(1.0,0.0),
+                connectionstyle="angle3,angleA=-10,angleB=0") )
+    ax.set_xlabel('c')
+    ax.set_ylabel('Mean squared error')
+    ax.set_xlim(2, 8.2)
+    ax.set_ylim(0, 10000)
+    ax.set_xticks(np.arange(2.0, 8.1, 2.0))
+    ax.set_yticks(np.arange(0, 10001, 5000))
+
+    
+def plot_mse_min(ax, x, y, c):
+    ccs = np.arange(0.5, 10.0, 0.05)
+    mses = np.zeros(len(ccs))
+    for i, cc in enumerate(ccs):
+        mses[i] = np.mean((y-(cc*x**3.0))**2.0)
+    imin = np.argmin(mses)
+    di = 25
+    i0 = 16
+    dimin = np.argmin(mses[i0::di])*di + i0
+        
+    ax.plot(c, 500.0, **psB)
+    ax.plot(ccs, mses, **lsAm)
+    ax.plot(ccs[i0::di], mses[i0::di], **psAm)
+    ax.plot(ccs[dimin], mses[dimin], **psD)
+    #ax.plot(ccs[imin], mses[imin], **psCm)
+    ax.annotate('Estimated\nminimum of\ncost\nfunction',
+                xy=(ccs[dimin], mses[dimin]*1.2), xycoords='data',
+                xytext=(4, 6700), textcoords='data', ha='left',
+                arrowprops=dict(arrowstyle="->", relpos=(0.8,0.0),
+                connectionstyle="angle3,angleA=0,angleB=85") )
+    ax.set_xlabel('c')
+    ax.set_xlim(2, 8.2)
+    ax.set_ylim(0, 10000)
+    ax.set_xticks(np.arange(2.0, 8.1, 2.0))
+    ax.set_yticks(np.arange(0, 10001, 5000))
+    ax.yaxis.set_major_formatter(mt.NullFormatter())
+
+
+if __name__ == "__main__":
+    x, y, c = create_data()
+    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=cm_size(figure_width, 1.1*figure_height))
+    fig.subplots_adjust(**adjust_fs(left=8.0, right=1.2))
+    plot_mse(ax1, x, y, c)
+    plot_mse_min(ax2, x, y, c)
+    fig.savefig("cubiccost.pdf")
+    plt.close()

regression/lecture/cubicerrors.py

@@ -15,28 +15,13 @@ def create_data():
     return x, y, c
 
     
-def plot_data(ax, x, y, c):
-    ax.plot(x, y, zorder=10, **psAm)
-    xx = np.linspace(2.1, 3.9, 100)
-    ax.plot(xx, c*xx**3.0, zorder=5, **lsBm)
-    for cc in [0.25*c, 0.5*c, 2.0*c, 4.0*c]:
-        ax.plot(xx, cc*xx**3.0, zorder=5, **lsDm)
-    ax.set_xlabel('Size x', 'm')
-    ax.set_ylabel('Weight y', 'kg')
-    ax.set_xlim(2, 4)
-    ax.set_ylim(0, 400)
-    ax.set_xticks(np.arange(2.0, 4.1, 0.5))
-    ax.set_yticks(np.arange(0, 401, 100))
-    
-    
 def plot_data_errors(ax, x, y, c):
     ax.set_xlabel('Size x', 'm')
-    #ax.set_ylabel('Weight y', 'kg')
+    ax.set_ylabel('Weight y', 'kg')
     ax.set_xlim(2, 4)
     ax.set_ylim(0, 400)
     ax.set_xticks(np.arange(2.0, 4.1, 0.5))
     ax.set_yticks(np.arange(0, 401, 100))
-    ax.set_yticklabels([])
     ax.annotate('Error',
                 xy=(x[28]+0.05, y[28]+60), xycoords='data',
                 xytext=(3.4, 70), textcoords='data', ha='left',
@@ -52,31 +37,30 @@ def plot_data_errors(ax, x, y, c):
         yy = [c*x[i]**3.0, y[i]]
         ax.plot(xx, yy, zorder=5, **lsDm)
 
+        
 def plot_error_hist(ax, x, y, c):
     ax.set_xlabel('Squared error')
     ax.set_ylabel('Frequency')
-    bins = np.arange(0.0, 1250.0, 100)
+    bins = np.arange(0.0, 11000.0, 750)
     ax.set_xlim(bins[0], bins[-1])
-    #ax.set_ylim(0, 35)
-    ax.set_xticks(np.arange(bins[0], bins[-1], 200))
-    #ax.set_yticks(np.arange(0, 36, 10))
+    ax.set_ylim(0, 15)
+    ax.set_xticks(np.arange(bins[0], bins[-1], 5000))
+    ax.set_yticks(np.arange(0, 16, 5))
     errors = (y-(c*x**3.0))**2.0
     mls = np.mean(errors)
     ax.annotate('Mean\nsquared\nerror',
                 xy=(mls, 0.5), xycoords='data',
-                xytext=(800, 3), textcoords='data', ha='left',
+                xytext=(4500, 6), textcoords='data', ha='left',
                 arrowprops=dict(arrowstyle="->", relpos=(0.0,0.2),
                 connectionstyle="angle3,angleA=10,angleB=90") )
     ax.hist(errors, bins, **fsC)
 
 
-
 if __name__ == "__main__":
     x, y, c = create_data()
-    fig, (ax1, ax2) = plt.subplots(1, 2)
-    fig.subplots_adjust(wspace=0.2, **adjust_fs(left=6.0, right=1.2))
-    plot_data(ax1, x, y, c)
-    plot_data_errors(ax2, x, y, c)
-    #plot_error_hist(ax2, x, y, c)
+    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=cm_size(figure_width, 0.9*figure_height))
+    fig.subplots_adjust(wspace=0.5, **adjust_fs(left=6.0, right=1.2))
+    plot_data_errors(ax1, x, y, c)
+    plot_error_hist(ax2, x, y, c)
     fig.savefig("cubicerrors.pdf")
     plt.close()

regression/lecture/cubicfunc.py

@@ -2,7 +2,7 @@ import matplotlib.pyplot as plt
 import numpy as np
 from plotstyle import *
 
-if __name__ == "__main__":
+def create_data():
     # wikipedia:
     # Generally, males vary in total length from 250 to 390 cm and
     # weigh between 90 and 306 kg
@@ -12,10 +12,20 @@ if __name__ == "__main__":
     rng = np.random.RandomState(32281)
     noise = rng.randn(len(x))*50
     y += noise
+    return x, y, c
 
-    fig, ax = plt.subplots(figsize=cm_size(figure_width, 1.4*figure_height))
-    fig.subplots_adjust(**adjust_fs(left=6.0, right=1.2))
     
+def plot_data(ax, x, y):
+    ax.plot(x, y, **psA)
+    ax.set_xlabel('Size x', 'm')
+    ax.set_ylabel('Weight y', 'kg')
+    ax.set_xlim(2, 4)
+    ax.set_ylim(0, 400)
+    ax.set_xticks(np.arange(2.0, 4.1, 0.5))
+    ax.set_yticks(np.arange(0, 401, 100))
+
+        
+def plot_data_fac(ax, x, y, c):
     ax.plot(x, y, zorder=10, **psA)
     xx = np.linspace(2.1, 3.9, 100)
     ax.plot(xx, c*xx**3.0, zorder=5, **lsB)
@@ -28,5 +38,13 @@ if __name__ == "__main__":
     ax.set_xticks(np.arange(2.0, 4.1, 0.5))
     ax.set_yticks(np.arange(0, 401, 100))
 
+    
+if __name__ == "__main__":
+    x, y, c = create_data()
+    print('n=%d' % len(x))
+    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=cm_size(figure_width, 0.9*figure_height))
+    fig.subplots_adjust(wspace=0.5, **adjust_fs(fig, left=6.0, right=1.2))
+    plot_data(ax1, x, y)
+    plot_data_fac(ax2, x, y, c)
     fig.savefig("cubicfunc.pdf")
     plt.close()

regression/lecture/cubicmse.py

@@ -14,6 +14,7 @@ def create_data():
     y += noise
     return x, y, c
 
+
 def gradient_descent(x, y):
     n = 20
     dc = 0.01
@@ -30,6 +31,7 @@ def gradient_descent(x, y):
         cc -= eps*dmdc
     return cs, mses
 
+    
 def plot_mse(ax, x, y, c, cs):
     ms = np.zeros(len(cs))
     for i, cc in enumerate(cs):
@@ -55,6 +57,7 @@ def plot_mse(ax, x, y, c, cs):
     ax.set_xticks(np.arange(0.0, 10.1, 2.0))
     ax.set_yticks(np.arange(0, 30001, 10000))
 
+        
 def plot_descent(ax, cs, mses):
     ax.plot(np.arange(len(mses))+1, mses, **lpsBm)
     ax.set_xlabel('Iteration')
@@ -69,8 +72,8 @@ def plot_descent(ax, cs, mses):
 if __name__ == "__main__":
     x, y, c = create_data()
     cs, mses = gradient_descent(x, y)
-    fig, (ax1, ax2) = plt.subplots(1, 2)
-    fig.subplots_adjust(wspace=0.2, **adjust_fs(left=8.0, right=0.5))
+    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=cm_size(figure_width, 1.1*figure_height))
+    fig.subplots_adjust(**adjust_fs(left=8.0, right=1.2))
     plot_mse(ax1, x, y, c, cs)
     plot_descent(ax2, cs, mses)
     fig.savefig("cubicmse.pdf")

regression/lecture/error_surface.py

@@ -1,56 +0,0 @@
-import numpy as np
-from mpl_toolkits.mplot3d import Axes3D
-import matplotlib.pyplot as plt
-import matplotlib.cm as cm
-from plotstyle import *
-
-def create_data():
-    m = 0.75
-    n=  -40
-    x = np.arange(10.,110., 2.5)
-    y = m * x + n;
-    rng = np.random.RandomState(37281)
-    noise = rng.randn(len(x))*15
-    y += noise
-    return x, y, m, n
-    
-
-def plot_error_plane(ax, x, y, m, n):
-    ax.set_xlabel('Slope m')
-    ax.set_ylabel('Intercept b')
-    ax.set_zlabel('Mean squared error')
-    ax.set_xlim(-4.5, 5.0)
-    ax.set_ylim(-60.0, -20.0)
-    ax.set_zlim(0.0, 700.0)
-    ax.set_xticks(np.arange(-4, 5, 2))
-    ax.set_yticks(np.arange(-60, -19, 10))
-    ax.set_zticks(np.arange(0, 700, 200))
-    ax.grid(True)
-    ax.w_xaxis.set_pane_color((1.0, 1.0, 1.0, 1.0))
-    ax.w_yaxis.set_pane_color((1.0, 1.0, 1.0, 1.0))
-    ax.w_zaxis.set_pane_color((1.0, 1.0, 1.0, 1.0))
-    ax.invert_xaxis()
-    ax.view_init(25, 40)
-    slopes = np.linspace(-4.5, 5, 40)
-    intercepts = np.linspace(-60, -20, 40)
-    x, y = np.meshgrid(slopes, intercepts)
-    error_surf = np.zeros(x.shape)
-    for i, s in enumerate(slopes) :
-        for j, b in enumerate(intercepts) :
-                error_surf[j,i] = np.mean((y-s*x-b)**2.0)
-    ax.plot_surface(x, y, error_surf, rstride=1, cstride=1, cmap=cm.coolwarm,
-                    linewidth=0, shade=True)
-    # Minimum:
-    mini = np.unravel_index(np.argmin(error_surf), error_surf.shape)
-    ax.scatter(slopes[mini[1]], intercepts[mini[0]], [0.0], color='#cc0000', s=60)
-
-
-if __name__ == "__main__":
-    x, y, m, n = create_data()
-    fig = plt.figure()
-    ax = fig.add_subplot(1, 1, 1, projection='3d')
-    plot_error_plane(ax, x, y, m, n)
-    fig.set_size_inches(7., 5.)
-    fig.subplots_adjust(**adjust_fs(fig, 1.0, 0.0, 0.0, 0.0))
-    fig.savefig("error_surface.pdf")
-    plt.close()

regression/lecture/lin_regress.py

@@ -1,60 +0,0 @@
-import numpy as np
-import matplotlib.pyplot as plt
-from plotstyle import *
-
-def create_data():
-    m = 0.75
-    n=  -40
-    x = np.arange(10.,110., 2.5)
-    y = m * x + n;
-    rng = np.random.RandomState(37281)
-    noise = rng.randn(len(x))*15
-    y += noise
-    return x, y, m, n
-
-    
-def plot_data(ax, x, y):
-    ax.plot(x, y, **psA)
-    ax.set_xlabel('Input x')
-    ax.set_ylabel('Output y')
-    ax.set_xlim(0, 120)
-    ax.set_ylim(-80, 80)
-    ax.set_xticks(np.arange(0,121, 40))
-    ax.set_yticks(np.arange(-80,81, 40))
-    
-
-def plot_data_slopes(ax, x, y, m, n):
-    ax.plot(x, y, **psA)
-    xx = np.asarray([2, 118])
-    for i in np.linspace(0.3*m, 2.0*m, 5):
-        ax.plot(xx, i*xx+n, **lsBm)
-    ax.set_xlabel('Input x')
-    #ax.set_ylabel('Output y')
-    ax.set_xlim(0, 120)
-    ax.set_ylim(-80, 80)
-    ax.set_xticks(np.arange(0,121, 40))
-    ax.set_yticks(np.arange(-80,81, 40))
-
-
-def plot_data_intercepts(ax, x, y, m, n):
-    ax.plot(x, y, **psA)
-    xx = np.asarray([2, 118])
-    for i in np.linspace(n-1*n, n+1*n, 5):
-        ax.plot(xx, m*xx + i, **lsBm)
-    ax.set_xlabel('Input x')
-    #ax.set_ylabel('Output y')
-    ax.set_xlim(0, 120)
-    ax.set_ylim(-80, 80)
-    ax.set_xticks(np.arange(0,121, 40))
-    ax.set_yticks(np.arange(-80,81, 40))
-
-
-if __name__ == "__main__":
-    x, y, m, n = create_data()
-    fig, axs = plt.subplots(1, 3)
-    fig.subplots_adjust(wspace=0.5, **adjust_fs(fig, left=6.0, right=1.5))
-    plot_data(axs[0], x, y)
-    plot_data_slopes(axs[1], x, y, m, n)
-    plot_data_intercepts(axs[2], x, y, m, n)
-    fig.savefig("lin_regress.pdf")
-    plt.close()

regression/lecture/linear_least_squares.py

@@ -1,65 +0,0 @@
-import numpy as np
-import matplotlib.pyplot as plt
-from plotstyle import *
-
-def create_data():
-    m = 0.75
-    n=  -40
-    x = np.concatenate( (np.arange(10.,110., 2.5), np.arange(0.,120., 2.0)) )
-    y = m * x + n;
-    rng = np.random.RandomState(37281)
-    noise = rng.randn(len(x))*15
-    y += noise
-    return x, y, m, n
-
-    
-def plot_data(ax, x, y, m, n):
-    ax.set_xlabel('Input x')
-    ax.set_ylabel('Output y')
-    ax.set_xlim(0, 120)
-    ax.set_ylim(-80, 80)
-    ax.set_xticks(np.arange(0,121, 40))
-    ax.set_yticks(np.arange(-80,81, 40))
-    ax.annotate('Error',
-                xy=(x[34]+1, y[34]+15), xycoords='data',
-                xytext=(80, -50), textcoords='data', ha='left',
-                arrowprops=dict(arrowstyle="->", relpos=(0.9,1.0),
-                connectionstyle="angle3,angleA=50,angleB=-30") )
-    ax.plot(x[:40], y[:40], zorder=0, **psAm)
-    inxs = [3, 13, 16, 19, 25, 34, 36]
-    ax.plot(x[inxs], y[inxs], zorder=10, **psA)
-    xx = np.asarray([2, 118])
-    ax.plot(xx, m*xx+n, **lsBm)
-    for i in inxs :
-        xx = [x[i], x[i]]
-        yy = [m*x[i]+n, y[i]]
-        ax.plot(xx, yy, zorder=5, **lsDm)
-    
-
-def plot_error_hist(ax, x, y, m, n):
-    ax.set_xlabel('Squared error')
-    ax.set_ylabel('Frequency')
-    bins = np.arange(0.0, 602.0, 50.0)
-    ax.set_xlim(bins[0], bins[-1])
-    ax.set_ylim(0, 35)
-    ax.set_xticks(np.arange(bins[0], bins[-1], 100))
-    ax.set_yticks(np.arange(0, 36, 10))
-    errors = (y-(m*x+n))**2.0
-    mls = np.mean(errors)
-    ax.annotate('Mean\nsquared\nerror',
-                xy=(mls, 0.5), xycoords='data',
-                xytext=(350, 20), textcoords='data', ha='left',
-                arrowprops=dict(arrowstyle="->", relpos=(0.0,0.2),
-                connectionstyle="angle3,angleA=10,angleB=90") )
-    ax.hist(errors, bins, **fsD)
-
-
-
-if __name__ == "__main__":
-    x, y, m, n = create_data()
-    fig, axs = plt.subplots(1, 2)
-    fig.subplots_adjust(**adjust_fs(fig, left=6.0))
-    plot_data(axs[0], x, y, m, n)
-    plot_error_hist(axs[1], x, y, m, n)
-    fig.savefig("linear_least_squares.pdf")
-    plt.close()

regression/lecture/regression-chapter.tex

@@ -23,39 +23,18 @@
 \item Fig 8.2 right: this should be a chi-squared distribution with one degree of freedom!
 \end{itemize}
 
-\subsection{Start with one-dimensional problem!}
-\begin{itemize}
-\item Let's fit a cubic function $y=cx^3$ (weight versus length of a tiger)\\
-\includegraphics[width=0.8\textwidth]{cubicfunc}
-\item Introduce the problem, $c$ is density and form factor
-\item How to generate an artificial data set (refer to simulation chapter)
-\item How to plot a function (do not use the data x values!)
-\item Just the mean square error as a function of the factor c\\
-\includegraphics[width=0.8\textwidth]{cubicerrors}
-\item Also mention the cost function for a straight line
-\item 1-d gradient, NO quiver plot (it is a nightmare to get this right)\\
-\includegraphics[width=0.8\textwidth]{cubicmse}
-\item 1-d gradient descend
-\item Describe in words the n-d problem.
-\item Homework is to do the 2d problem with the straight line!
-\end{itemize}
-
 \subsection{Linear fits}
 \begin{itemize}
 \item Polyfit is easy: unique solution! $c x^2$ is also a linear fit.
 \item Example for overfitting with polyfit of a high order (=number of data points)
 \end{itemize}
 
-\section{Fitting in practice}
-
-Fit with matlab functions lsqcurvefit, polyfit
-
 
 \subsection{Non-linear fits}
 \begin{itemize}
 \item Example that illustrates the Nebenminima Problem (with error surface)
-\item You need got initial values for the parameter!
-\item Example that fitting gets harder the more parameter yuo have.
+\item You need initial values for the parameter!
+\item Example that fitting gets harder the more parameter you have.
 \item Try to fix as many parameter before doing the fit.
 \item How to test the quality of a fit? Residuals. $\chi^2$ test. Run-test.
 \end{itemize}

regression/lecture/regression.tex

@@ -2,195 +2,197 @@
 \exercisechapter{Optimization and gradient descent}
 
 Optimization problems arise in many different contexts. For example,
-to understand the behavior of a given system, the system is probed
-with a range of input signals and then the resulting responses are
-measured. This input-output relation can be described by a model. Such
-a model can be a simple function that maps the input signals to
-corresponding responses, it can be a filter, or a system of
-differential equations. In any case, the model has some parameters that
-specify how input and output signals are related.  Which combination
-of parameter values are best suited to describe the input-output
-relation? The process of finding the best parameter values is an
-optimization problem. For a simple parameterized function that maps
-input to output values, this is the special case of a \enterm{curve
-  fitting} problem, where the average distance between the curve and
-the response values is minimized. One basic numerical method used for
-such optimization problems is the so called gradient descent, which is
-introduced in this chapter.
-
-%%% Weiteres einfaches verbales Beispiel? Eventuell aus der Populationsoekologie?
+to understand the behavior of a given neuronal system, the system is
+probed with a range of input signals and then the resulting responses
+are measured. This input-output relation can be described by a
+model. Such a model can be a simple function that maps the input
+signals to corresponding responses, it can be a filter, or a system of
+differential equations. In any case, the model has some parameters
+that specify how input and output signals are related.  Which
+combination of parameter values are best suited to describe the
+input-output relation? The process of finding the best parameter
+values is an optimization problem. For a simple parameterized function
+that maps input to output values, this is the special case of a
+\enterm{curve fitting} problem, where the average distance between the
+curve and the response values is minimized. One basic numerical method
+used for such optimization problems is the so called gradient descent,
+which is introduced in this chapter.
 
 \begin{figure}[t]
-  \includegraphics[width=1\textwidth]{lin_regress}\hfill
-  \titlecaption{Example data suggesting a linear relation.}{A set of
-    input signals $x$, e.g. stimulus intensities, were used to probe a
-    system. The system's output $y$ to the inputs are noted
-    (left). Assuming a linear relation between $x$ and $y$ leaves us
-    with 2 parameters, the slope (center) and the intercept with the
-    y-axis (right panel).}\label{linregressiondatafig}
+  \includegraphics{cubicfunc}
+  \titlecaption{Example data suggesting a cubic relation.}{The length
+    $x$ and weight $y$ of $n=34$ male tigers (blue, left). Assuming a
+    cubic relation between size and weight leaves us with a single
+    free parameters, a scaling factor. The cubic relation is shown for
+    a few values of this scaling factor (orange and red,
+    right).}\label{cubicdatafig}
 \end{figure}
 
-The data plotted in \figref{linregressiondatafig} suggest a linear
-relation between input and output of the system. We thus assume that a
-straight line
+For demonstrating the curve-fitting problem let's take the simple
+example of weights and sizes measured for a number of male tigers
+(\figref{cubicdatafig}). Weight $y$ is proportional to volume
+$V$ via the density $\rho$. The volume $V$ of any object is
+proportional to its length $x$ cubed. The factor $\alpha$ relating
+volume and size cubed depends on the shape of the object and we do not
+know this factor for tigers. For the data set we thus expect a cubic
+relation between weight and length
 \begin{equation}
-  \label{straightline}
-  y = f(x; m, b) = m\cdot x + b
+  \label{cubicfunc}
+  y = f(x; c) = c\cdot x^3
 \end{equation}
-is an appropriate model to describe the system.  The line has two free
-parameter, the slope $m$ and the $y$-intercept $b$. We need to find
-values for the slope and the intercept that best describe the measured
-data.  In this chapter we use this example to illustrate the gradient
-descent and how this methods can be used to find a combination of
-slope and intercept that best describes the system.
+where $c=\rho\alpha$, the product of a tiger's density and form
+factor, is the only free parameter in the relation. We would like to
+find out which value of $c$ best describes the measured data.  In the
+following we use this example to illustrate the gradient descent as a
+basic method for finding such an optimal parameter.
 
 
 \section{The error function --- mean squared error}
 
-Before the optimization can be done we need to specify what exactly is
-considered an optimal fit. In our example we search the parameter
-combination that describe the relation of $x$ and $y$ best. What is
-meant by this? Each input $x_i$ leads to an measured output $y_i$ and
-for each $x_i$ there is a \emph{prediction} or \emph{estimation}
-$y^{est}(x_i)$ of the output value by the model. At each $x_i$
-estimation and measurement have a distance or error $y_i -
-y^{est}(x_i)$. In our example the estimation is given by the equation
-$y^{est}(x_i) = f(x_i;m,b)$. The best fitting model with parameters
-$m$ and $b$ is the one that minimizes the distances between
-observation $y_i$ and estimation $y^{est}(x_i)$
-(\figref{leastsquareerrorfig}).
-
-As a first guess we could simply minimize the sum $\sum_{i=1}^N y_i -
-y^{est}(x_i)$. This approach, however, will not work since a minimal sum
-can also be achieved if half of the measurements is above and the
-other half below the predicted line. Positive and negative errors
-would cancel out and then sum up to values close to zero. A better
-approach is to sum over the absolute values of the distances:
-$\sum_{i=1}^N |y_i - y^{est}(x_i)|$. This sum can only be small if all
-deviations are indeed small no matter if they are above or below the
-predicted line. Instead of the sum we could also take the average
-\begin{equation}
-  \label{meanabserror}
-  f_{dist}(\{(x_i, y_i)\}|\{y^{est}(x_i)\}) = \frac{1}{N} \sum_{i=1}^N |y_i - y^{est}(x_i)|
-\end{equation}
-Instead of the averaged absolute errors, the \enterm[mean squared
-error]{mean squared error} (\determ[quadratischer
-Fehler!mittlerer]{mittlerer quadratischer Fehler})
+Before we go ahead finding the optimal parameter value we need to
+specify what exactly we consider as an optimal fit. In our example we
+search the parameter that describes the relation of $x$ and $y$
+best. What is meant by this? The length $x_i$ of each tiger is
+associated with a weight $y_i$ and for each $x_i$ we have a
+\emph{prediction} or \emph{estimation} $y^{est}(x_i)$ of the weight by
+the model \eqnref{cubicfunc} for a specific value of the parameter
+$c$. Prediction and actual data value ideally match (in a perfect
+noise-free world), but in general the estimate and measurement are
+separated by some distance or error $y_i - y^{est}(x_i)$. In our
+example the estimate of the weight for the length $x_i$ is given by
+equation \eqref{cubicfunc} $y^{est}(x_i) = f(x_i;c)$. The best fitting
+model with parameter $c$ is the one that somehow minimizes the
+distances between observations $y_i$ and corresponding estimations
+$y^{est}(x_i)$ (\figref{cubicerrorsfig}).
+
+As a first guess we could simply minimize the sum of the distances,
+$\sum_{i=1}^N y_i - y^{est}(x_i)$. This, however, does not work
+because positive and negative errors would cancel out, no matter how
+large they are, and sum up to values close to zero. Better is to sum
+over absolute distances: $\sum_{i=1}^N |y_i - y^{est}(x_i)|$. This sum
+can only be small if all deviations are indeed small no matter if they
+are above or below the prediction.  The sum of the squared distances,
+$\sum_{i=1}^N (y_i - y^{est}(x_i))^2$, turns out to be an even better
+choice. Instead of the sum we could also minimize the average distance
 \begin{equation}
   \label{meansquarederror}
   f_{mse}(\{(x_i, y_i)\}|\{y^{est}(x_i)\}) = \frac{1}{N} \sum_{i=1}^N (y_i - y^{est}(x_i))^2
 \end{equation}
-is commonly used (\figref{leastsquareerrorfig}). Similar to the
-absolute distance, the square of the errors, $(y_i - y^{est}(x_i))^2$, is
-always positive and thus positive and negative error values do not
+This is known as the \enterm[mean squared error]{mean squared error}
+(\determ[quadratischer Fehler!mittlerer]{mittlerer quadratischer
+  Fehler}). Similar to the absolute distance, the square of the errors
+is always positive and thus positive and negative error values do not
 cancel each other out. In addition, the square punishes large
 deviations over small deviations. In
 chapter~\ref{maximumlikelihoodchapter} we show that minimizing the
-mean square error is equivalent to maximizing the likelihood that the
+mean squared error is equivalent to maximizing the likelihood that the
 observations originate from the model, if the data are normally
 distributed around the model prediction.
 
-\begin{exercise}{meanSquaredErrorLine.m}{}\label{mseexercise}%
-  Given a vector of observations \varcode{y} and a vector with the
-  corresponding predictions \varcode{y\_est}, compute the \emph{mean
-    square error} between \varcode{y} and \varcode{y\_est} in a single
-  line of code.
+\begin{exercise}{meansquarederrorline.m}{}\label{mseexercise}
+  Simulate $n=40$ tigers ranging from 2.2 to 3.9\,m in size and store
+  these sizes in a vector \varcode{x}. Compute the corresponding
+  predicted weights \varcode{yest} for each tiger according to
+  \eqnref{cubicfunc} with $c=6$\,\kilo\gram\per\meter\cubed. From the
+  predictions generate simulated measurements of the tiger's weights
+  \varcode{y}, by adding normally distributed random numbers to the
+  predictions scaled to a standard deviation of 50\,\kilo\gram.
+
+  Compute the \emph{mean squared error} between \varcode{y} and
+  \varcode{yest} in a single line of code.
 \end{exercise}
 
 
-\section{Objective function}
+\section{Cost function}
 
 The mean squared error is a so called \enterm{objective function} or
 \enterm{cost function} (\determ{Kostenfunktion}). A cost function
 assigns to a model prediction $\{y^{est}(x_i)\}$ for a given data set
 $\{(x_i, y_i)\}$ a single scalar value that we want to minimize.  Here
-we aim to adapt the model parameters to minimize the mean squared
-error \eqref{meansquarederror}. In general, the \enterm{cost function}
-can be any function that describes the quality of the fit by mapping
-the data and the predictions to a single scalar value.
+we aim to adapt the model parameter to minimize the mean squared error
+\eqref{meansquarederror}. In general, the \enterm{cost function} can
+be any function that describes the quality of a fit by mapping the
+data and the predictions to a single scalar value.
 
 \begin{figure}[t]
-  \includegraphics[width=1\textwidth]{linear_least_squares}
-  \titlecaption{Estimating the \emph{mean square error}.}  {The
-    deviation error, orange) between the prediction (red
-    line) and the observations (blue dots) is calculated for each data
-    point (left). Then the deviations are squared and the aveage is
+  \includegraphics{cubicerrors}
+  \titlecaption{Estimating the \emph{mean squared error}.}  {The
+    deviation error (orange) between the prediction (red line) and the
+    observations (blue dots) is calculated for each data point
+    (left). Then the deviations are squared and the average is
     calculated (right).}
-  \label{leastsquareerrorfig}
+  \label{cubicerrorsfig}
 \end{figure}
 
 Replacing $y^{est}$ in the mean squared error \eqref{meansquarederror}
-with our model, the straight line \eqref{straightline}, the cost
-function reads
+with our cubic model \eqref{cubicfunc}, the cost function reads
 \begin{eqnarray}
-  f_{cost}(m,b|\{(x_i, y_i)\}) & = & \frac{1}{N} \sum_{i=1}^N (y_i - f(x_i;m,b))^2 \label{msefunc} \\
-  & = & \frac{1}{N} \sum_{i=1}^N (y_i - m x_i - b)^2 \label{mseline}
+  f_{cost}(c|\{(x_i, y_i)\}) & = & \frac{1}{N} \sum_{i=1}^N (y_i - f(x_i;c))^2 \label{msefunc} \\
+  & = & \frac{1}{N} \sum_{i=1}^N (y_i - c x_i^3)^2 \label{msecube}
 \end{eqnarray}
-The optimization process tries to find the slope $m$ and the intercept
-$b$ such that the cost function is minimized. With the mean squared
-error as the cost function this optimization process is also called
-method of the \enterm{least square error} (\determ[quadratischer
+The optimization process tries to find a value for the factor $c$ such
+that the cost function is minimized. With the mean squared error as
+the cost function this optimization process is also called method of
+ \enterm{least squares} (\determ[quadratischer
 Fehler!kleinster]{Methode der kleinsten Quadrate}).
 
-\begin{exercise}{meanSquaredError.m}{}
-  Implement the objective function \eqref{mseline} as a function
-  \varcode{meanSquaredError()}.  The function takes three
+\begin{exercise}{meanSquaredErrorCubic.m}{}
+  Implement the objective function \eqref{msecube} as a function
+  \varcode{meanSquaredErrorCubic()}.  The function takes three
   arguments. The first is a vector of $x$-values and the second
   contains the measurements $y$ for each value of $x$. The third
-  argument is a 2-element vector that contains the values of
-  parameters \varcode{m} and \varcode{b}. The function returns the
-  mean square error.
+  argument is the value of the factor $c$. The function returns the
+  mean squared error.
 \end{exercise}
 
 
-\section{Error surface}
-For each combination of the two parameters $m$ and $b$ of the model we
-can use \eqnref{mseline} to calculate the corresponding value of the
-cost function. The cost function $f_{cost}(m,b|\{(x_i, y_i)\}|)$ is a
-function $f_{cost}(m,b)$, that maps the parameter values $m$ and $b$
-to a scalar error value.  The error values describe a landscape over the
-$m$-$b$ plane, the error surface, that can be illustrated graphically
-using a 3-d surface-plot. $m$ and $b$ are plotted on the $x$- and $y$-
-axis while the third dimension indicates the error value
-(\figref{errorsurfacefig}).
+\section{Graph of the cost function}
+For each value of the parameter $c$ of the model we can use
+\eqnref{msecube} to calculate the corresponding value of the cost
+function. The cost function $f_{cost}(c|\{(x_i, y_i)\}|)$ is a
+function $f_{cost}(c)$ that maps the parameter value $c$ to a scalar
+error value. For a given data set we thus can simply plot the cost
+function as a function of the parameter $c$ (\figref{cubiccostfig}).
+
+\begin{exercise}{plotcubiccosts.m}{}
+  Calculate the mean squared error between the data and the cubic
+  function for a range of values of the factor $c$ using the
+  \varcode{meanSquaredErrorCubic()} function from the previous
+  exercise. Plot the graph of the cost function.
+\end{exercise}
 
 \begin{figure}[t]
-  \includegraphics[width=0.75\textwidth]{error_surface}
-  \titlecaption{Error surface.}{The two model parameters $m$ and $b$
-    define the base area of the surface plot. For each parameter
-    combination of slope and intercept the error is calculated. The
-    resulting surface has a minimum which indicates the parameter
-    combination that best fits the data.}\label{errorsurfacefig}
+  \includegraphics{cubiccost}
+  \titlecaption{Minimum of the cost function.}{For a given data set
+    the cost function, the mean squared error \eqnref{msecube}, as a
+    function of the unknown parameter $c$ has a minimum close to the
+    true value of $c$ that was used to generate the data
+    (left). Simply taking the absolute minimum of the cost function
+    computed for a pre-set range of values for the parameter $c$, has
+    the disadvantage to be limited in precision (right) and the
+    possibility to entirely miss the global minimum if it is outside
+    the computed range.}\label{cubiccostfig}
 \end{figure}
 
-\begin{exercise}{errorSurface.m}{}\label{errorsurfaceexercise}
-  Generate 20 data pairs $(x_i|y_i)$ that are linearly related with
-  slope $m=0.75$ and intercept $b=-40$, using \varcode{rand()} for
-  drawing $x$ values between 0 and 120 and \varcode{randn()} for
-  jittering the $y$ values with a standard deviation of 15.  Then
-  calculate the mean squared error between the data and straight lines
-  for a range of slopes and intercepts using the
-  \varcode{meanSquaredError()} function from the previous exercise.
-  Illustrates the error surface using the \code{surface()} function.
-  Consult the documentation to find out how to use \code{surface()}.
-\end{exercise}
-
-By looking at the error surface we can directly see the position of
-the minimum and thus estimate the optimal parameter combination. How
-can we use the error surface to guide an automatic optimization
-process?
-
-The obvious approach would be to calculate the error surface for any
-combination of slope and intercept values and then find the position
-of the minimum using the \code{min} function. This approach, however
-has several disadvantages: (i) it is computationally very expensive to
-calculate the error for each parameter combination. The number of
-combinations increases exponentially with the number of free
-parameters (also known as the ``curse of dimensionality''). (ii) the
-accuracy with which the best parameters can be estimated is limited by
-the resolution used to sample the parameter space. The coarser the
-parameters are sampled the less precise is the obtained position of
-the minimum.
+By looking at the plot of the cost function we can visually identify
+the position of the minimum and thus estimate the optimal value for
+the parameter $c$. How can we use the cost function to guide an
+automatic optimization process?
+
+The obvious approach would be to calculate the mean squared error for
+a range of parameter values and then find the position of the minimum
+using the \code{min()} function. This approach, however has several
+disadvantages: (i) The accuracy of the estimation of the best
+parameter is limited by the resolution used to sample the parameter
+space. The coarser the parameters are sampled the less precise is the
+obtained position of the minimum (\figref{cubiccostfig}, right).  (ii)
+The range of parameter values might not include the absolute minimum.
+(iii) In particular for functions with more than a single free
+parameter it is computationally expensive to calculate the cost
+function for each parameter combination at a sufficient
+resolution. The number of combinations increases exponentially with
+the number of free parameters. This is known as the \enterm{curse of
+  dimensionality}.
 
 So we need a different approach. We want a procedure that finds the
 minimum of the cost function with a minimal number of computations and
@@ -206,34 +208,242 @@ to arbitrary precision.
       m = \frac{f(x + \Delta x) - f(x)}{\Delta x}
     \end{equation}
     of a function $y = f(x)$ is the slope of the secant (red) defined
-    by the points $(x,f(x))$ and $(x+\Delta x,f(x+\Delta x))$ with the
+    by the points $(x,f(x))$ and $(x+\Delta x,f(x+\Delta x))$ at
     distance $\Delta x$.
 
-    The slope of the function $y=f(x)$ at the position $x$ (yellow) is
+    The slope of the function $y=f(x)$ at the position $x$ (orange) is
     given by the derivative $f'(x)$ of the function at that position.
     It is defined by the difference quotient in the limit of
-    infinitesimally (orange) small distances $\Delta x$:
+    infinitesimally (red and yellow) small distances $\Delta x$:
     \begin{equation}
       \label{derivative}
       f'(x) = \frac{{\rm d} f(x)}{{\rm d}x} = \lim\limits_{\Delta x \to 0} \frac{f(x + \Delta x) - f(x)}{\Delta x} \end{equation}
   \end{minipage}\vspace{2ex} 
 
-  It is not possible to calculate the derivative, \eqnref{derivative},
-  numerically. The derivative can only be estimated using the
-  difference quotient, \eqnref{difffrac}, by using sufficiently small
-  $\Delta x$.
+  It is not possible to calculate the exact value of the derivative
+  \eqref{derivative} numerically. The derivative can only be estimated
+  by computing the difference quotient \eqref{difffrac} using
+  sufficiently small $\Delta x$.
 \end{ibox}
 
-\begin{ibox}[tp]{\label{partialderivativebox}Partial derivative and gradient}
-  Some functions that depend on more than a single variable:
+\section{Gradient}
+
+Imagine to place a ball at some point on the cost function
+\figref{cubiccostfig}. Naturally, it would roll down the slope and
+eventually stop at the minimum of the error surface (if it had no
+inertia). We will use this analogy to develop an algorithm to find our
+way to the minimum of the cost function. The ball always follows the
+steepest slope. Thus we need to figure out the direction of the slope
+at the position of the ball.
+
+\begin{figure}[t]
+  \includegraphics{cubicgradient}
+  \titlecaption{Derivative of the cost function.}{The gradient, the
+    derivative \eqref{costderivative} of the cost function, is
+    negative to the left of the minimum (vertical line) of the cost
+    function, zero (horizontal line) at, and positive to the right of
+    the minimum (left). For each value of the parameter $c$ the
+    negative gradient (arrows) points towards the minimum of the cost
+    function (right).} \label{gradientcubicfig}
+\end{figure}
+
+In our one-dimensional example of a single free parameter the slope is
+simply the derivative of the cost function with respect to the
+parameter $c$ (\figref{gradientcubicfig}, left). This derivative is called
+the \entermde{Gradient}{gradient} of the cost function:
+\begin{equation}
+  \label{costderivative}
+  \nabla f_{cost}(c) = \frac{{\rm d} f_{cost}(c)}{{\rm d} c}
+\end{equation}
+There is no need to calculate this derivative analytically, because it
+can be approximated numerically by the difference quotient
+(Box~\ref{differentialquotientbox}) for small steps $\Delta c$:
+\begin{equation}
+  \frac{{\rm d} f_{cost}(c)}{{\rm d} c} =
+    \lim\limits_{\Delta c \to 0} \frac{f_{cost}(c + \Delta c) - f_{cost}(c)}{\Delta c}
+    \approx \frac{f_{cost}(c + \Delta c) - f_{cost}(c)}{\Delta c}
+\end{equation}
+Choose, for example, $\Delta c = 10^{-7}$.  The derivative is positive
+for positive slopes. Since want to go down the hill, we choose the
+opposite direction (\figref{gradientcubicfig}, right).
+
+\begin{exercise}{meanSquaredGradientCubic.m}{}\label{gradientcubic}
+  Implement a function \varcode{meanSquaredGradientCubic()}, that
+  takes the $x$- and $y$-data and the parameter $c$ as input
+  arguments. The function should return the derivative of the mean
+  squared error $f_{cost}(c)$ with respect to $c$ at the position
+  $c$.
+\end{exercise}
+
+\begin{exercise}{plotcubicgradient.m}{}
+  Using the \varcode{meanSquaredGradientCubic()} function from the
+  previous exercise, plot the derivative of the cost function as a
+  function of $c$.
+\end{exercise}
+
+\section{Gradient descent}
+
+\begin{figure}[t]
+  \includegraphics{cubicmse}
+  \titlecaption{Gradient descent.}{The algorithm starts at an
+    arbitrary position. At each point the gradient is estimated and
+    the position is updated as long as the length of the gradient is
+    sufficiently large. The dots show the positions after each
+    iteration of the algorithm.} \label{gradientdescentcubicfig}
+\end{figure}
+
+Finally, we are able to implement the optimization itself. By now it
+should be obvious why it is called the gradient descent method. From a
+starting position on we iteratively walk down the slope of the cost
+function against its gradient. All ingredients necessary for this
+algorithm are already there. We need: (i) the cost function
+(\varcode{meanSquaredErrorCubic()}), and (ii) the gradient
+(\varcode{meanSquaredGradientCubic()}). The algorithm of the gradient
+descent works as follows:
+\begin{enumerate}
+\item Start with some arbitrary value $p_0$ for the parameter $c$.
+\item \label{computegradient} Compute the gradient
+  \eqnref{costderivative} at the current position $p_i$.
+\item If the length of the gradient, the absolute value of the
+  derivative \eqnref{costderivative}, is smaller than some threshold
+  value, we assume to have reached the minimum and stop the search.
+  We return the current parameter value $p_i$ as our best estimate of
+  the parameter $c$ that minimizes the cost function.
+
+  We are actually looking for the point at which the derivative of the
+  cost function equals zero, but this is impossible because of
+  numerical imprecision. We thus apply a threshold below which we are
+  sufficiently close to zero.
+\item \label{gradientstep} If the length of the gradient exceeds the
+  threshold we take a small step into the opposite direction:
+  \begin{equation}
+    \label{gradientdescent}
+    p_{i+1} = p_i - \epsilon \cdot \nabla f_{cost}(p_i)
+  \end{equation}
+  where $\epsilon$ is a factor linking the gradient to
+  appropriate steps in the parameter space.
+\item Repeat steps \ref{computegradient} -- \ref{gradientstep}.
+\end{enumerate}
+
+\Figref{gradientdescentcubicfig} illustrates the gradient descent ---
+the path the imaginary ball has chosen to reach the minimum. We walk
+along the parameter axis against the gradient as long as the gradient
+differs sufficiently from zero.  At steep slopes we take large steps
+(the distance between the red dots in \figref{gradientdescentcubicfig}
+is large).
+
+\begin{exercise}{gradientDescentCubic.m}{}\label{gradientdescentexercise}
+  Implement the gradient descent algorithm for the problem of fitting
+  a cubic function \eqref{cubicfunc} to some measured data pairs $x$
+  and $y$ as a function \varcode{gradientDescentCubic()} that returns
+  the estimated best fitting parameter value $c$ as well as two
+  vectors with all the parameter values and the corresponding values
+  of the cost function that the algorithm iterated trough. As
+  additional arguments that function takes the initial value for the
+  parameter $c$, the factor $\epsilon$ connecting the gradient with
+  iteration steps in \eqnref{gradientdescent}, and the threshold value
+  for the absolute value of the gradient terminating the algorithm.
+\end{exercise}
+
+\begin{exercise}{plotgradientdescentcubic.m}{}\label{plotgradientdescentexercise}
+  Use the function \varcode{gradientDescentCubic()} to fit the
+  simulated data from exercise~\ref{mseexercise}. Plot the returned
+  values of the parameter $c$ as as well as the corresponding mean
+  squared errors as a function of iteration step (two plots). Compare
+  the result of the gradient descent method with the true value of $c$
+  used to simulate the data. Inspect the plots and adapt $\epsilon$
+  and the threshold to make the algorithm behave as intended. Finally
+  plot the data together with the best fitting cubic relation
+  \eqref{cubicfunc}.
+\end{exercise}
+
+The $\epsilon$ parameter in \eqnref{gradientdescent} is critical. If
+too large, the algorithm does not converge to the minimum of the cost
+function (try it!). At medium values it oscillates around the minimum
+but might nevertheless converge. Only for sufficiently small values
+(here $\epsilon = 0.00001$) does the algorithm follow the slope
+downwards towards the minimum.
+
+The terminating condition on the absolute value of the gradient
+influences how often the cost function is evaluated. The smaller the
+threshold value the more often the cost is computed and the more
+precisely the fit parameter is estimated. If it is too small, however,
+the increase in precision is negligible, in particular in comparison
+to the increased computational effort. Have a look at the derivatives
+that we plotted in exercise~\ref{gradientcubic} and decide on a
+sensible value for the threshold. Run the gradient descent algorithm
+and check how the resulting $c$ parameter values converge and how many
+iterations were needed. Then reduce the threshold (by factors of ten)
+and check how this changes the results.
+
+Many modern algorithms for finding the minimum of a function are based
+on the basic idea of the gradient descent. Luckily these algorithms
+choose $\epsilon$ in a smart adaptive way and they also come up with
+sensible default values for the termination condition. On the other
+hand, these algorithm often take optional arguments that let you
+control how they behave. Now you know what this is all about.
+
+\section{N-dimensional minimization problems}
+
+So far we were concerned about finding the right value for a single
+parameter that minimizes a cost function.  Often we deal with
+functions that have more than a single parameter, in general $n$
+parameters. We then need to find the minimum in an $n$ dimensional
+parameter space.
+
+For our tiger problem, we could have also fitted the exponent $a$ of
+the power-law relation between size and weight, instead of assuming a
+cubic relation with $a=3$:
+\begin{equation}
+  \label{powerfunc}
+  y = f(x; c, a) = f(x; \vec p) = c\cdot x^a
+\end{equation}
+We then could check whether the resulting estimate of the exponent
+$a$ indeed is close to the expected power of three.  The
+power-law \eqref{powerfunc} has two free parameters $c$ and $a$.
+Instead of a single parameter we are now dealing with a vector $\vec
+p$ containing $n$ parameter values. Here, $\vec p = (c, a)$. All
+the concepts we introduced on the example of the one dimensional
+problem of tiger weights generalize to $n$-dimensional problems. We
+only need to adapt a few things. The cost function for the mean
+squared error reads
+\begin{equation}
+  \label{ndimcostfunc}
+  f_{cost}(\vec p|\{(x_i, y_i)\}) = \frac{1}{N} \sum_{i=1}^N (y_i - f(x_i;\vec p))^2
+\end{equation}
+
+\begin{figure}[t]
+  \includegraphics{powergradientdescent}
+  \titlecaption{Gradient descent on an error surface.}{Contour plot of
+    the cost function \eqref{ndimcostfunc} for the fit of a power law
+    \eqref{powerfunc} to some data. Here the cost function is a long
+    and narrow valley on the plane spanned by the two parameters $c$
+    and $a$ of the power law. The red line marks the path of the
+    gradient descent algorithm. The gradient is always perpendicular
+    to the contour lines. The algorithm quickly descends into the
+    valley and then slowly creeps on the shallow bottom of the valley
+    towards the global minimum where it terminates (yellow circle).
+  } \label{powergradientdescentfig}
+\end{figure}
+
+For two-dimensional problems the graph of the cost function is an
+\enterm{error surface} (\determ{{Fehlerfl\"ache}}). The two parameters
+span a two-dimensional plane. The cost function assigns to each
+parameter combination on this plane a single value. This results in a
+landscape over the parameter plane with mountains and valleys and we
+are searching for the position of the bottom of the deepest valley
+(\figref{powergradientdescentfig}).
+
+\begin{ibox}[tp]{\label{partialderivativebox}Partial derivatives and gradient}
+  Some functions depend on more than a single variable. For example, the function
   \[ z = f(x,y) \]
-  for example depends on $x$ and $y$. Using the partial derivative
+  depends on both $x$ and $y$. Using the partial derivatives
   \[ \frac{\partial f(x,y)}{\partial x} = \lim\limits_{\Delta x \to 0} \frac{f(x + \Delta x,y) - f(x,y)}{\Delta x} \]
   and
   \[ \frac{\partial f(x,y)}{\partial y} = \lim\limits_{\Delta y \to 0} \frac{f(x, y + \Delta y) - f(x,y)}{\Delta y} \]
-  one can estimate the slope in the direction of the variables
-  individually by using the respective difference quotient
-  (Box~\ref{differentialquotientbox}).  \vspace{1ex}
+  one can calculate the slopes in the directions of each of the
+  variables by means of the respective difference quotient
+  (see box~\ref{differentialquotientbox}).  \vspace{1ex}
 
   \begin{minipage}[t]{0.5\textwidth}
     \mbox{}\\[-2ex]
@@ -242,145 +452,129 @@ to arbitrary precision.
   \hfill
   \begin{minipage}[t]{0.46\textwidth}
     For example, the partial derivatives of
-    \[ f(x,y) = x^2+y^2 \] are 
+    \[ f(x,y) = x^2+y^2 \vspace{-1ex} \] are 
     \[ \frac{\partial f(x,y)}{\partial x} = 2x \; , \quad \frac{\partial f(x,y)}{\partial y} = 2y \; .\]
 
-    The gradient is a vector that is constructed from the partial derivatives:
+    The gradient is a vector with the partial derivatives as coordinates:
     \[ \nabla f(x,y) = \left( \begin{array}{c} \frac{\partial f(x,y)}{\partial x} \\[1ex] \frac{\partial f(x,y)}{\partial y} \end{array} \right) \]
     This vector points into the direction of the strongest ascend of
     $f(x,y)$.
   \end{minipage}
 
-  \vspace{0.5ex} The figure shows the contour lines of a bi-variate
+  \vspace{0.5ex} The figure shows the contour lines of a bivariate
   Gaussian $f(x,y) = \exp(-(x^2+y^2)/2)$ and the gradient (thick
-  arrows) and the corresponding two partial derivatives (thin arrows)
-  for three different locations.
+  arrows) together with the corresponding partial derivatives (thin
+  arrows) for three different locations.
 \end{ibox}
 
-
-\section{Gradient}
-Imagine to place a small ball at some point on the error surface
-\figref{errorsurfacefig}. Naturally, it would roll down the steepest
-slope and eventually stop at the minimum of the error surface (if it had no
-inertia). We will use this picture to develop an algorithm to find our
-way to the minimum of the objective function. The ball will always
-follow the steepest slope. Thus we need to figure out the direction of
-the steepest slope at the position of the ball.
-
-The \entermde{Gradient}{gradient} (Box~\ref{partialderivativebox}) of the
-objective function is the vector
+When we place a ball somewhere on the slopes of a hill, it rolls
+downwards and eventually stops at the bottom. The ball always rolls in
+the direction of the steepest slope. For a two-dimensional parameter
+space of our example, the \entermde{Gradient}{gradient}
+(box~\ref{partialderivativebox}) of the cost function is a vector
+\begin{equation}
+  \label{gradientpowerlaw}
+  \nabla f_{cost}(c, a) = \left( \frac{\partial f_{cost}(c, a)}{\partial c},
+    \frac{\partial f_{cost}(c, a)}{\partial a} \right)
+\end{equation}
+that points into the direction of the strongest ascend of the cost
+function.  The gradient is given by the partial derivatives
+(box~\ref{partialderivativebox}) of the mean squared error with
+respect to the parameters $c$ and $a$ of the power law
+relation. In general, for $n$-dimensional problems, the gradient is an
+$n-$ dimensional vector containing for each of the $n$ parameters
+$p_j$ the respective partial derivatives as coordinates:
 \begin{equation}
   \label{gradient}
-  \nabla f_{cost}(m,b) = \left( \frac{\partial f(m,b)}{\partial m},
-    \frac{\partial f(m,b)}{\partial b} \right)
+  \nabla f_{cost}(\vec p) = \left( \frac{\partial f_{cost}(\vec p)}{\partial p_j} \right)
+\end{equation}
+
+The iterative equation \eqref{gradientdescent} of the gradient descent
+stays exactly the same, with the only difference that the current
+parameter value $p_i$ becomes a vector $\vec p_i$ of parameter values:
+\begin{equation}
+  \label{ndimgradientdescent}
+  \vec p_{i+1} = \vec p_i - \epsilon \cdot \nabla f_{cost}(\vec p_i)
 \end{equation}
-that points to the strongest ascend of the objective function.  The
-gradient is given by partial derivatives
-(Box~\ref{partialderivativebox}) of the mean squared error with
-respect to the parameters $m$ and $b$ of the straight line. There is
-no need to calculate it analytically because it can be estimated from
-the partial derivatives using the difference quotient
-(Box~\ref{differentialquotientbox}) for small steps $\Delta m$ and
-$\Delta b$. For example, the partial derivative with respect to $m$
-can be computed as
+The algorithm proceeds along the negative gradient
+(\figref{powergradientdescentfig}).
+
+For the termination condition we need the length of the gradient. In
+one dimension it was just the absolute value of the derivative. For
+$n$ dimensions this is according to the \enterm{Pythagorean theorem}
+(\determ[Pythagoras!Satz des]{Satz des Pythagoras}) the square root of
+the sum of the squared partial derivatives:
 \begin{equation}
-  \frac{\partial f_{cost}(m,b)}{\partial m} = \lim\limits_{\Delta m \to
-  0} \frac{f_{cost}(m + \Delta m, b) - f_{cost}(m,b)}{\Delta m}
-\approx \frac{f_{cost}(m + \Delta m, b) - f_{cost}(m,b)}{\Delta m} \; . 
+  \label{ndimabsgradient}
+  |\nabla f_{cost}(\vec p_i)| = \sqrt{\sum_{i=1}^n \left(\frac{\partial f_{cost}(\vec p)}{\partial p_i}\right)^2}
 \end{equation}
-The length of the gradient indicates the steepness of the slope
-(\figref{gradientquiverfig}). Since want to go down the hill, we
-choose the opposite direction.
+The \code{norm()} function implements this given a vector with the
+partial derivatives.
 
+\subsection{Passing a function as an argument to another function}
 
-\begin{figure}[t]
-  \includegraphics[width=0.75\textwidth]{error_gradient}
-  \titlecaption{Gradient of the error surface.}  {Each arrow points
-    into the direction of the greatest ascend at different positions
-    of the error surface shown in \figref{errorsurfacefig}. The
-    contour lines in the background illustrate the error surface. Warm
-    colors indicate high errors, colder colors low error values. Each
-    contour line connects points of equal
-    error.}\label{gradientquiverfig}
-\end{figure}
+So far, all our code for the gradient descent algorithm was tailored
+to a specific function, the cubic relation \eqref{cubicfunc}. It would
+be much better if we could pass an arbitrary function to our gradient
+algorithm. Then we would not need to rewrite it every time anew.
 
-\begin{exercise}{meanSquaredGradient.m}{}\label{gradientexercise}%
-  Implement a function \varcode{meanSquaredGradient()}, that takes the
-  $x$- and $y$-data and the set of parameters $(m, b)$ of a straight
-  line as a two-element vector as input arguments. The function should
-  return the gradient at the position $(m, b)$ as a vector with two
-  elements.
-\end{exercise}
+This is possible. We can indeed pass a function as an argument to
+another function. For this use the \code{@}-operator. As an example
+let's define a function that produces a standard plot for a function:
 
-\begin{exercise}{errorGradient.m}{}
-  Extend the script of exercises~\ref{errorsurfaceexercise} to plot
-  both the error surface and gradients using the
-  \varcode{meanSquaredGradient()} function from
-  exercise~\ref{gradientexercise}. Vectors in space can be easily
-  plotted using the function \code{quiver()}. Use \code{contour()}
-  instead of \code{surface()} to plot the error surface.
-\end{exercise}
+\pageinputlisting[caption={Example function taking a function as argument.}]{funcPlotter.m}
 
+This function can then be used as follows for plotting a sine wave. We
+pass the built in \varcode{sin()} function as \varcode{@sin} as an
+argument to our function:
 
-\section{Gradient descent}
-Finally, we are able to implement the optimization itself. By now it
-should be obvious why it is called the gradient descent method. All
-ingredients are already there. We need: (i) the cost function
-(\varcode{meanSquaredError()}), and (ii) the gradient
-(\varcode{meanSquaredGradient()}). The algorithm of the gradient
-descent works as follows:
-\begin{enumerate}
-\item Start with some given combination of the parameters $m$ and $b$
-  ($p_0 = (m_0, b_0)$).
-\item \label{computegradient} Calculate the gradient at the current
-  position $p_i$.
-\item If the length of the gradient falls below a certain value, we
-  assume to have reached the minimum and stop the search. We are
-  actually looking for the point at which the length of the gradient
-  is zero, but finding zero is impossible because of numerical
-  imprecision. We thus apply a threshold below which we are
-  sufficiently close to zero (e.g. \varcode{norm(gradient) < 0.1}).
-\item \label{gradientstep} If the length of the gradient exceeds the
-  threshold we take a small step into the opposite direction:
-  \begin{equation}
-    p_{i+1} = p_i - \epsilon \cdot \nabla f_{cost}(m_i, b_i)
-  \end{equation}
-  where $\epsilon = 0.01$ is a factor linking the gradient to
-  appropriate steps in the parameter space.
-\item Repeat steps \ref{computegradient} -- \ref{gradientstep}.
-\end{enumerate}
+\pageinputlisting[caption={Passing a function handle as an argument to a function.}]{funcplotterexamples.m}
 
-\Figref{gradientdescentfig} illustrates the gradient descent --- the
-path the imaginary ball has chosen to reach the minimum. Starting at
-an arbitrary position on the error surface we change the position as
-long as the gradient at that position is larger than a certain
-threshold. If the slope is very steep, the change in the position (the
-distance between the red dots in \figref{gradientdescentfig}) is
-large.
 
-\begin{figure}[t]
-  \includegraphics[width=0.45\textwidth]{gradient_descent}
-  \titlecaption{Gradient descent.}{The algorithm starts at an
-    arbitrary position. At each point the gradient is estimated and
-    the position is updated as long as the length of the gradient is
-    sufficiently large.The dots show the positions after each
-    iteration of the algorithm.} \label{gradientdescentfig}
-\end{figure}
+\subsection{Gradient descent algorithm for arbitrary functions}
+
+Now we are ready to adapt the gradient descent algorithm from
+exercise~\ref{gradientdescentexercise} to arbitrary functions with $n$
+parameters that we want to fit to some data.
 
 \begin{exercise}{gradientDescent.m}{}
-  Implement the gradient descent for the problem of fitting a straight
-  line to some measured data. Reuse the data generated in
-  exercise~\ref{errorsurfaceexercise}.
-  \begin{enumerate}
-  \item Store for each iteration the error value.
-  \item Plot the error values as a function of the iterations, the
-    number of optimization steps.
-  \item Plot the measured data together with the best fitting straight line.
-  \end{enumerate}\vspace{-4.5ex}
+  Adapt the function \varcode{gradientDescentCubic()} from
+  exercise~\ref{gradientdescentexercise} to implement the gradient
+  descent algorithm for any function \varcode{func(x, p)} that takes
+  as first argument the $x$-data values and as second argument a
+  vector with parameter values.  The new function takes a vector $\vec
+  p_0$ for the initial parameter values and also returns the best
+  parameter combination as a vector. Use a \varcode{for} loop over the
+  two dimensions for computing the gradient.
 \end{exercise}
 
+For testing our new function we need to implement the power law
+\eqref{powerfunc}:
 
-\section{Summary}
+\begin{exercise}{powerLaw.m}{}
+  Write a function that implements \eqref{powerfunc}. The function
+  gets as arguments a vector $x$ containing the $x$-data values and
+  another vector containing as elements the parameters for the power
+  law, i.e. the factor $c$ and the power $a$. It returns a vector
+  with the computed $y$ values for each $x$ value. 
+\end{exercise}
+
+Now let's use the new gradient descent function to fit a power law to
+our tiger data-set (\figref{powergradientdescentfig}):
+
+\begin{exercise}{plotgradientdescentpower.m}{}
+  Use the function \varcode{gradientDescent()} to fit the
+  \varcode{powerLaw()} function to the simulated data from
+  exercise~\ref{mseexercise}. Plot the returned values of the two
+  parameters against each other. Compare the result of the gradient
+  descent method with the true values of $c$ and $a$ used to simulate
+  the data. Observe the norm of the gradient and inspect the plots to
+  adapt $\epsilon$ and the threshold if necessary. Finally plot the
+  data together with the best fitting power-law \eqref{powerfunc}.
+\end{exercise}
+
+
+\section{Fitting non-linear functions to data}
 
 The gradient descent is an important numerical method for solving
 optimization problems. It is used to find the global minimum of an
@@ -389,33 +583,44 @@ objective function.
 Curve fitting is a common application for the gradient descent method.
 For the case of fitting straight lines to data pairs, the error
 surface (using the mean squared error) has exactly one clearly defined
-global minimum. In fact, the position of the minimum can be analytically
-calculated as shown in the next chapter.
-
-Problems that involve nonlinear computations on parameters, e.g. the
-rate $\lambda$ in an exponential function $f(x;\lambda) = e^{\lambda
-  x}$, do not have an analytical solution for the least squares. To
-find the least squares for such functions numerical methods such as
-the gradient descent have to be applied.
-
-The suggested gradient descent algorithm can be improved in multiple
-ways to converge faster.  For example one could adapt the step size to
-the length of the gradient. These numerical tricks have already been
-implemented in pre-defined functions. Generic optimization functions
-such as \matlabfun{fminsearch()} have been implemented for arbitrary
-objective functions, while the more specialized function
-\matlabfun{lsqcurvefit()} i specifically designed for optimizations in
-the least square error sense.
-
-%\newpage
+global minimum. In fact, the position of the minimum can be
+analytically calculated as shown in the next chapter. For linear
+fitting problems numerical methods like the gradient descent are not
+needed.
+
+Fitting problems that involve nonlinear functions of the parameters,
+e.g. the power law \eqref{powerfunc} or the exponential function
+$f(x;\lambda) = e^{\lambda x}$, do not have an analytical solution for
+the least squares. To find the least squares for such functions
+numerical methods such as the gradient descent have to be applied.
+
+The suggested gradient descent algorithm is quite fragile and requires
+manually tuned values for $\epsilon$ and the threshold for terminating
+the iteration.  The algorithm can be improved in multiple ways to
+converge more robustly and faster.  For example one could adapt the
+step size to the length of the gradient. These numerical tricks have
+already been implemented in pre-defined functions. Generic
+optimization functions such as \mcode{fminsearch()} have been
+implemented for arbitrary objective functions, while the more
+specialized function \mcode{lsqcurvefit()} is specifically designed
+for optimizations in the least square error sense.
+
+\begin{exercise}{plotlsqcurvefitpower.m}{}
+  Use the \matlab-function \varcode{lsqcurvefit()} instead of
+  \varcode{gradientDescent()} to fit the \varcode{powerLaw()} function
+  to the simulated data from exercise~\ref{mseexercise}. Plot the data
+  and the resulting best fitting power law function.
+\end{exercise}
+
+
 \begin{important}[Beware of secondary minima!]
   Finding the absolute minimum is not always as easy as in the case of
-  fitting a straight line. Often, the error surface has secondary or
+  fitting a straight line. Often, the cost function has secondary or
   local minima in which the gradient descent stops even though there
   is a more optimal solution, a global minimum that is lower than the
   local minimum. Starting from good initial positions is a good
   approach to avoid getting stuck in local minima. Also keep in mind
-  that error surfaces tend to be simpler (less local minima) the fewer
+  that cost functions tend to be simpler (less local minima) the fewer
   parameters are fitted from the data. Each additional parameter
   increases complexity and is computationally more expensive.
 \end{important}