%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% CSC411: Machine Learning and Data Mining 
%     Tutorial 3 (Febuary 2nd): Neural Network Example - XOR problem
%  
% Code is referred to : http://www.yov408.com/html/codespot.php?gg=57
% by Naresh Bansal
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

clear all;
close all;

display('This is a sample code for XOR problem');

% TIC Start a stopwatch timer.
tic;

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%                                  Variable declaration begins                                             %
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Input = [0 0; 0 1; 1 0; 1 1];
Output = [0 ; 1 ; 1 ; 0];
%Weight(1:2,:) is the intial weight for hidden layer 
%Weight(3,:) is the initial weight for the output layer
Weight=[1.9 1.9; 17 17;-20 20];
 
%Bias (1:2,:) is the Bias initial weight for the
%hidden layer and Bias (3,:) is the Bias initial weight for the output
%layer.
Bias =[0.5 -1 -1];
 
%A learning constant to update the weight
alpha = 0.3;

%Initial Mean Square Error
MSE=10;
Delta_error=[0 0 0];
 % error threshold 
threshold = 0.25;

%index to retrieve each row of the Input
idx =1; 
%number of loop 
loop = 1;

%disp('Neural Network is running.');

 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%                               Variable declaration ends                                            %
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

while MSE > threshold
    
    %Step 1: Calculate on each node in the hidden layer and output layer
    %based on the back propogation 
    
    %--- results at the hidden layer
    %input(mod(idx,4),:) : each row from input matrix 
     x1 = Weight(1:2,:)*Input(idx,:)'+  Bias(1:2)';
    result_hidden=sigmf(x1,[0.5 0]);    
    
     %results at the output layer 
    x2=DOT(Weight(3,:)',result_hidden)+Bias(3);
    result_output=sigmf(x2,[0.5 0]);
    
    %Step 2: update the Delta_error
    %delta error at the output layer
    Delta_error(3)=result_output*(1-result_output)*(Output(idx)-result_output);
    %delta error at the hidden layer
    Delta_error(1:2)=result_hidden.*(1-result_hidden).*(Delta_error(3).*Weight(3,:)');
    
    %Step 3: update the output weight
    Weight(3,:)=Weight(3,:)+(alpha*Delta_error(3)).*result_hidden';
    Weight(1:2,:)=Weight(1:2,:)+alpha.*(Dot(Delta_error(1:2),Input(idx,:)));
    
    %Step 4: update Error  and MSE 
    Error(loop)=abs(Output(idx)-result_output);
    %update MSE
    if(mod(loop,4)==0) % Each entry in the Error has been updated
         MSE= sqrt(sum(Error(loop-3:loop).^2))
      %  disp('Loop is '); disp(loop);  
    end
    loop=loop+1;
    %idx ranges from 1 to 4, which refers to each row of the input matrix
    idx = mod(idx,4)+1; 
    
    if(MSE>0.7 & loop>60)  pause(10000);
    end
end  %End the back propogation calculation

disp('--------------------End of Calculation--------------------------');

p=1;
for i=1:4:loop-5
    K(p)=sqrt(sum(Error(i:i+4).^2));
    p=p+1;
end
plot(K);
hold on
grid on;
xlabel('Number of Iterations');
ylabel('Mean Square Error');
title('Convergence of BackPropagation for XOR problem');

Total_Time = toc 

save myXOR