[regression] improved the chapter

regression/code/errorGradient.m

@@ -1,26 +1,33 @@
-load('lin_regression.mat')
-ms = -1:0.5:5;
-ns = -10:1:10;
+% x, y, slopes, and intercepts from exercise 8.3
 
-error_surf = zeros(length(ms), length(ns));
-gradient_m = zeros(size(error_surf));
-gradient_n = zeros(size(error_surf));
+slopes = -5:0.25:5;
+intercepts = -30:1:30;
+error_surface = zeros(length(slopes), length(intercepts));
+for i = 1:length(slopes)
+    for j = 1:length(intercepts)
+	      error_surf(i,j) = meanSquaredError([slopes(i), intercepts(j)], x, y);
+    end
+end
 
-for i = 1:length(ms)
-    for j = 1:length(ns)
-        error_surf(i,j) = lsqError([ms(i), ns(j)], x, y);
-        grad = lsqGradient([ms(i), ns(j)], x, y);
+error_surface = zeros(length(slopes), length(intercepts));
+gradient_m = zeros(size(error_surface));
+gradient_b = zeros(size(error_surface));
+
+for i = 1:length(slopes)
+    for j = 1:length(intercepts)
+        error_surface(i,j) = meanSquaredError([slopes(i), intercepts(j)], x, y);
+        grad = meanSquaredGradient([slopes(i), intercepts(j)], x, y);
         gradient_m(i,j) = grad(1);
-        gradient_n(i,j) = grad(2);
+        gradient_b(i,j) = grad(2);
     end
 end
 
 figure()
 hold on
-[N, M] = meshgrid(ns, ms);
-%surface(M,N, error_surf, 'FaceAlpha', 0.5);
-contour(M,N, error_surf, 50);
-quiver(M,N, gradient_m, gradient_n)
+[N, M] = meshgrid(intercepts, slopes);
+%surface(M, N, error_surface, 'FaceAlpha', 0.5);
+contour(M, N, error_surface, 50);
+quiver(M, N, gradient_m, gradient_b)
 xlabel('Slope m')
 ylabel('Intercept b')
 zlabel('Mean squared error')

regression/code/errorSurface.m

@@ -1,19 +1,24 @@
-load('lin_regression.mat');
+% generate data:
+m = 0.75;
+b = -40.0;
+n = 20;
+x = 120.0*rand(n, 1);
+y = m*x + b + 15.0*randn(n, 1);
 
 % compute mean squared error for a range of slopes and intercepts:
 slopes = -5:0.25:5;
 intercepts = -30:1:30;
-error_surf = zeros(length(slopes), length(intercepts));
+error_surface = zeros(length(slopes), length(intercepts));
 for i = 1:length(slopes)
     for j = 1:length(intercepts)
-	      error_surf(i,j) = lsqError([slopes(i), intercepts(j)], x, y);
+	      error_surf(i,j) = meanSquaredError([slopes(i), intercepts(j)], x, y);
     end
 end
 
 % plot the error surface:
 figure()
 [N,M] = meshgrid(intercepts, slopes);
-s = surface(M, N, error_surf);
+surface(M, N, error_surface);
 xlabel('slope', 'rotation', 7.5)
 ylabel('intercept', 'rotation', -22.5)
 zlabel('error')

regression/code/gradientDescent.m

@@ -1,26 +1,27 @@
-clear 
-close all
-load('lin_regression.mat')
+% x, y from exercise 8.3
 
-position = [-2. 10.];
+% some arbitrary values for the slope and the intercept to start with:
+position = [-2. 10.];  
+
+% gradient descent:
 gradient = [];
 errors = [];
 count = 1;
 eps = 0.01;
-
 while isempty(gradient) || norm(gradient) > 0.1
-    gradient = lsqGradient(position, x,y);
-    errors(count) = lsqError(position, x, y);
+    gradient = meanSquaredGradient(position, x, y);
+    errors(count) = meanSquaredError(position, x, y);
     position = position - eps .* gradient; 
     count = count + 1;
 end
+
 figure()
 subplot(2,1,1)
 hold on
-scatter(x,y, 'displayname', 'data')
-xaxis = min(x):0.01:max(x);
-f_x = position(1).*xaxis + position(2);
-plot(xaxis, f_x, 'displayname', 'fit')
+scatter(x, y, 'displayname', 'data')
+xx = min(x):0.01:max(x);
+yy = position(1).*xx + position(2);
+plot(xx, yy, 'displayname', 'fit')
 xlabel('Input')
 ylabel('Output')
 grid on
@@ -28,4 +29,4 @@ legend show
 subplot(2,1,2)
 plot(errors)
 xlabel('optimization steps')
-ylabel('error')
+ylabel('error')

regression/code/lsqError.m

@@ -1,13 +0,0 @@
-function error = lsqError(parameter, x, y)
-% Objective function for fitting a linear equation to data.
-%
-% Arguments: parameter, vector containing slope and intercept 
-%                       as the 1st and 2nd element 
-%            x, vector of the input values
-%            y, vector of the corresponding measured output values
-%
-% Returns: the estimation error in terms of the mean sqaure error
-
-  y_est = x .* parameter(1) + parameter(2);
-  error = meanSquareError(y, y_est);
-end

regression/code/meanSquareError.m

@@ -1,10 +0,0 @@
-function error = meanSquareError(y, y_est)
-% Mean squared error between observed and predicted values.
-%
-% Arguments:    y, vector of observed values.
-%               y_est, vector of predicted values.
-%
-% Returns:      the error in the mean-squared-deviation sense.
-
-  error = mean((y - y_est).^2);
-end

regression/code/meanSquaredError.m

@@ -0,0 +1,12 @@
+function mse = meanSquaredError(parameter, x, y)
+% Mean squared error between a straight line and data pairs.
+%
+% Arguments: parameter, vector containing slope and intercept 
+  %                       as the 1st and 2nd element, respectively. 
+%            x, vector of the input values
+%            y, vector of the corresponding measured output values
+%
+% Returns:   mse, the mean-squared-error.
+
+  mse = mean((y - x * parameter(1) - parameter(2)).^2);
+end

regression/code/meanSquaredErrorLine.m

@@ -0,0 +1 @@
+mse = mean((y - y_est).^2);

regression/code/meanSquaredGradient.m

@@ -1,5 +1,5 @@
-function gradient = lsqGradient(parameter, x, y)
-% The gradient of the least square error
+function gradient = meanSquaredGradient(parameter, x, y)
+% The gradient of the mean squared error
 %
 % Arguments: parameter, vector containing slope and intercept 
 %                       as the 1st and 2nd element 
@@ -7,8 +7,10 @@ function gradient = lsqGradient(parameter, x, y)
 %            y, vector of the corresponding measured output values
 %
 % Returns: the gradient as a vector with two elements
+
   h = 1e-6;   % stepsize for derivatives
-  partial_m = (lsqError([parameter(1)+h, parameter(2)], x, y) - lsqError(parameter, x, y))/ h;
-  partial_n = (lsqError([parameter(1), parameter(2)+h], x, y) - lsqError(parameter, x, y))/ h;
+  mse = meanSquaredError(parameter, x, y);
+  partial_m = (meanSquaredError([parameter(1)+h, parameter(2)], x, y) - mse)/h;
+  partial_n = (meanSquaredError([parameter(1), parameter(2)+h], x, y) - mse)/h;
   gradient = [partial_m, partial_n];
 end