clear all
close all 
load toydata
load toydata_test 

test_err=[];
test_crerr=[];
train_err=[];
train_crerr=[];

[numcases numdims numbatches]=size(batchdata);
N=numcases; 

w1=0.1*randn(785,10);
w2=0.1*randn(11,10);
w4=0.1*randn(11,10);
l1=size(w1,1)-1;
l2=size(w2,1)-1;
l4=size(w4,1)-1;
l5=10;

w_class = 0.1*randn(11,10); 
l10 = 10; 


for epoch = 1:100

%%%% TEST STATS 
%%%% Error rates 
[testnumcases testnumdims testnumbatches]=size(testbatchdata);
N=testnumcases;
err=0;
err_cr=0;
counter=0;  
for batch = 1:testnumbatches
  data = [testbatchdata(:,:,batch)];
  target = [testbatchtargets(:,:,batch)]; 
  data = [data ones(N,1)];
  w1probs = 1./(1 + exp(-data*w1)); w1probs = [w1probs  ones(N,1)];
  w2probs = 1./(1 + exp(-w1probs*w2)); w2probs = [w2probs ones(N,1)];
  w4probs = 1./(1 + exp(-w2probs*w4)); w4probs = [w4probs  ones(N,1)];
  targetout = exp(w4probs*w_class);
  targetout = targetout./repmat(sum(targetout,2),1,10);

 [I J]=max(targetout,[],2); 
 [I1 J1]=max(target,[],2); 
 counter=counter+length(find(J==J1));  

 err_cr = err_cr- sum(sum( target(:,1:end).*log(targetout))) ;

  end
 test_err(epoch)=(1000-counter)/10;
 test_crerr(epoch)=err_cr/testnumbatches;
 fprintf(1,'epoch %d testerr %f test crerr %f \n',epoch,test_err(epoch),test_crerr(epoch));
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%% TRAINING STATS
%%%% Error rates
[numcases numdims numbatches]=size(batchdata);
N=numcases;
err=0;
err_cr=0;
counter=0;
for batch = 1:numbatches
  data = [batchdata(:,:,batch)];
  target = [batchtargets(:,:,batch)];
  data = [data ones(N,1)];
  w1probs = 1./(1 + exp(-data*w1)); w1probs = [w1probs  ones(N,1)];
  w2probs = 1./(1 + exp(-w1probs*w2)); w2probs = [w2probs ones(N,1)];
  w4probs = 1./(1 + exp(-w2probs*w4)); w4probs = [w4probs  ones(N,1)];
  targetout = exp(w4probs*w_class);
  targetout = targetout./repmat(sum(targetout,2),1,10);

 [I J]=max(targetout,[],2);
 [I1 J1]=max(target,[],2);
 counter=counter+length(find(J==J1));

 err_cr = err_cr- sum(sum( target(:,1:end).*log(targetout))) ;

  end
 train_err(epoch)=(5000-counter)/50;
 train_crerr(epoch)=err_cr/numbatches;
 fprintf(1,'epoch %d trainerr %f train crerr %f \n',epoch, train_err(epoch),train_crerr(epoch));
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

figure(1)
plot(train_err,'k-');
hold on 
plot(test_err,'r-');
drawnow 

%save testclassrbmbackprop_2 w1 w2 w4 w_class
%save testclasserrors_2 test_err test_crerr train_err train_crerr 

 errsum=0;
 tt=0;
 for batch = 1:numbatches/10
 fprintf(1,'epoch %d batch %d\r',epoch,batch);

 tt=tt+1; 
 data = [batchdata(:,:,(tt-1)*10+1)
   batchdata(:,:,(tt-1)*10+2)
   batchdata(:,:,(tt-1)*10+3)
   batchdata(:,:,(tt-1)*10+4)
   batchdata(:,:,(tt-1)*10+5)
   batchdata(:,:,(tt-1)*10+6)
   batchdata(:,:,(tt-1)*10+7)
   batchdata(:,:,(tt-1)*10+8) 
   batchdata(:,:,(tt-1)*10+9)
   batchdata(:,:,(tt-1)*10+10)
 ];

 targets = [batchtargets(:,:,(tt-1)*10+1)
   batchtargets(:,:,(tt-1)*10+2)
   batchtargets(:,:,(tt-1)*10+3)
   batchtargets(:,:,(tt-1)*10+4)
   batchtargets(:,:,(tt-1)*10+5)
   batchtargets(:,:,(tt-1)*10+6)
   batchtargets(:,:,(tt-1)*10+7)
   batchtargets(:,:,(tt-1)*10+8)
   batchtargets(:,:,(tt-1)*10+9)
   batchtargets(:,:,(tt-1)*10+10)
 ];



%%% DO CG for 3 linesearches 

max_iter=3;

VV = [w1(:)' w2(:)' w4(:)' w_class(:)']';
Dim = [l1; l2; l4; l5; l10];

% result = checkgrad('ECG',VV,10^-5,Dim,data(1:100,:),targets(1:100,:));

[X, fX, i, fIter] = minimize(VV,'ECG',max_iter,Dim,data,targets);
 w1 = reshape(X(1:(l1+1)*l2),l1+1,l2);
 xxx = (l1+1)*l2;
 w2 = reshape(X(xxx+1:xxx+(l2+1)*l4),l2+1,l4);
 xxx = xxx+(l2+1)*l4;
 w4 = reshape(X(xxx+1:xxx+(l4+1)*l5),l4+1,l5);
 xxx = xxx+(l4+1)*l5;
 w_class = reshape(X(xxx+1:xxx+(l5+1)*l10),l5+1,l10);

end

end