%CLEVAL Classifier evaluation (learning curve)
%
%   E = CLEVAL(A,CLASSF,TRAINSIZES,NREPS,S,TESTFUN)
%
% INPUT
%   A          Training dataset
%   CLASSF     Classifier to evaluate
%   TRAINSIZES Vector of training set sizes, used to generate subsets of A
%              (default [2,3,5,7,10,15,20,30,50,70,100]). TRAINSIZE is per
%              class unless A has no priors set or has soft labels.
%   NREPS      Number of repetitions (default 1)
%   S          Test dataset (default [], use remaining samples in A)
%   TESTFUN    Mapping,evaluation function (default classification error)
%
% OUTPUT
%   E          Error structure (see PLOTE) containing training and test
%              errors
%
% DESCRIPTION
% Generates at random, for all class sizes defined in TRAINSIZES, training
% sets out of the dataset A and uses these for training the untrained
% classifier CLASSF. CLASSF may also be a cell array of untrained
% classifiers; in this case the routine will be run for all of them. The
% resulting trained classifiers are tested on the training objects and
% on the left-over test objects. This procedure is then repeated NREPS
% times. The default test routine is classification error estimation by
% TESTC([],'crisp'). 
%
% Training set generation is done such that for each run the larger
% training sets include the smaller ones and that for all classifiers the
% same training sets are used.
%
% If CLASSF is fully deterministic, this function uses the RAND random
% generator and thereby reproduces if its seed is reset (see RAND).
% If CLASSF uses RANDN, its seed may have to be set as well.
%
% Per default both the true error (error on the test set) and the
% apparent error (error on the training set) are computed. They will be
% visible when the curves are plotted using PLOTE.
%
% EXAMPLE
% See PREX_CLEVAL
%
% SEE ALSO (<a href="http://37steps.com/prtools">PRTools Guide</a>)
% MAPPINGS, DATASETS, CLEVALB, TESTC, PLOTE

% Copyright: D.M.J. Tax, R.P.W. Duin, r.p.w.duin@37steps.com
% Faculty EWI, Delft University of Technology
% P.O. Box 5031, 2600 GA Delft, The Netherlands

function e = cleval(a,classf,learnsizes,nreps,t,testfun)

  defaulttest = false;
  if (nargin < 6) || isempty(testfun)
    testfun = testc([],getlabtype(a)); 
    defaulttest = true;
  end;
  if (nargin < 5) || isempty(t)
    t = [];
  end;
  if (nargin < 4) || isempty(nreps);
    nreps = 1;
  end;
  if (nargin < 3) || isempty(learnsizes);
    learnsizes = [2,3,5,7,10,15,20,30,50,70,100];
  end;

  % Correct for old argument order.

  if (isdataset(classf)) && (ismapping(a))
    tmp = a; a = classf; classf = {tmp};
  end
  if (isdataset(classf)) && (iscell(a)) && (ismapping(a{1}))
    tmp = a; a = classf; classf = tmp;
  end
  if ~iscell(classf), classf = {classf}; end

  % Assert that all is right.
  isdataset(a); ismapping(classf{1}); if (~isempty(t)), isdataset(t); end

  % Remove requested class sizes that are larger than the size of the
  % smallest class.

  [m,k,c] = getsize(a);
  if ~isempty(a,'prior') && islabtype(a,'crisp')
    classs = true;
    mc = classsizes(a);
    toolarge = find(learnsizes >= min(mc));
    if (~isempty(toolarge))
      prwarning(3,['training set class sizes ' num2str(learnsizes(toolarge)) ...
                   ' larger than the minimal class size; removed them']);
      learnsizes(toolarge) = [];
    end
  else
    if islabtype(a,'crisp') && isempty(a,'prior')
      prwarning(1,['No priors found in dataset, class frequencies are used. ' ....
                   'Training set sizes hold for entire dataset']);
    end
    if defaulttest
      testfun = testd;
    end
    classs = false;
    toolarge = find(learnsizes >= m);
    if (~isempty(toolarge))
      prwarning(3,['training set sizes ' num2str(learnsizes(toolarge)) ...
                   ' larger than number of objects; removed them']);
      learnsizes(toolarge) = [];
    end
    
  end
  learnsizes = learnsizes(:)';
    

  % Fill the error structure.

  nw = length(classf(:));
  datname = getname(a);

  e.n       = nreps;
  e.error   = zeros(nw,length(learnsizes));
  e.std     = zeros(nw,length(learnsizes));
  e.apperror   = zeros(nw,length(learnsizes));
  e.appstd     = zeros(nw,length(learnsizes));
  e.xvalues = learnsizes(:)';
  if classs
    e.xlabel   = 'Training set size per class';
  else
    e.xlabel   = 'Training set size';
  end
  e.names   = [];
  if (nreps > 1)
    e.ylabel= ['Averaged error (' num2str(nreps) ' experiments)'];
  elseif (nreps == 1)
    e.ylabel = 'Error';
  else
    error('Number of repetitions NREPS should be >= 1.');
  end;
  if (~isempty(datname))
    e.title = ['Learning curve on ' datname];
  end
  if (learnsizes(end)/learnsizes(1) > 20)
    e.plot = 'semilogx';        % If range too large, use a log-plot for X.
  else
    e.plot = 'plot';
  end

  % Report progress.
  
  s1 = sprintf('cleval: %i classifiers: ',nw);
  prwaitbar(nw,s1);

  % Store the seed, to reset the random generator later for different
  % classifiers.

  seed = randreset;

  % Loop over all classifiers (with index WI).

  for wi = 1:nw
    
    if (~isuntrained(classf{wi}))
      error('Classifiers should be untrained.')
    end
    name = getname(classf{wi});
    e.names = char(e.names,name);
    prwaitbar(nw,wi,[s1 name]);

    % E1 will contain the error estimates.

    e1 = zeros(nreps,length(learnsizes));
    e0 = zeros(nreps,length(learnsizes));

    % Take care that classifiers use same training set.

    randreset(seed); seed2 = seed;

    % For NREPS repetitions...
    
    s2 = sprintf('cleval: %i repetitions: ',nreps);
    prwaitbar(nreps,s2);

    for i = 1:nreps
  
      prwaitbar(nreps,i,[s2 int2str(i)]);
      % Store the randomly permuted indices of samples of class CI to use in
      % this training set in JR(CI,:).
      
      if classs

        JR = zeros(c,max(learnsizes));
      
        for ci = 1:c

          JC = findnlab(a,ci);

          % Necessary for reproducable training sets: set the seed and store
          % it after generation, so that next time we will use the previous one.
          randreset(seed2);

          JD = JC(randperm(mc(ci)));
          JR(ci,:) = JD(1:max(learnsizes))';
          seed2 = randreset; 
        end
        
      elseif islabtype(a,'crisp')
        
        randreset(seed2); % get seed for reproducable training sets
%         % generate indices for the entire dataset taking care that in
%         % the first 2c objects we have 2 objects for every class
%         [a1,a2,I1,I2] = gendat(a,2*ones(1,c));
%         JD = randperm(m-2*c);
%         JR = [I1;I2(JD)];
        % don't do this anymore as classifiers can stand too small class
        % sizes
        JR = randperm(m);
        seed2 = randreset; % save seed for reproducable training sets
        
      else  % soft labels
        
        randreset(seed2); % get seed for reproducable training sets
        JR = randperm(m);
        seed2 = randreset; % save seed for reproducable training sets
        
      end

      li = 0;                    % Index of training set.
      
      nlearns = length(learnsizes);
      s3 = sprintf('cleval: %i sizes: ',nlearns);
      prwaitbar(nreps,s3);
      
      for j = 1:nlearns 
        
        nj = learnsizes(j);
        
        prwaitbar(nlearns,j,[s3 int2str(j) ' (' int2str(nj) ')']);
        li = li + 1;

        % J will contain the indices for this training set.

        J = [];
        
        if classs
          for ci = 1:c
            J = [J;JR(ci,1:nj)'];
          end;
        else
          J = JR(1:nj);
        end

        prwaitbar(3,'cleval: training');
        prwaitbar(3,1,'cleval: training');
        %trainset = a(J,:)*remclass*seldat;
        trainset = a(J,:)*seldat;
        trainset = setprior(trainset,getprior(trainset,0));
        w = trainset*classf{wi};           % Use right classifier.
        prwaitbar(3,2,'cleval: test trainset');
        e0(i,li) = trainset*w*testfun;
        prwaitbar(3,3,'cleval: test testset');
        if (isempty(t))
          Jt = ones(m,1);
          Jt(J) = zeros(size(J));
          Jt = find(Jt);                 % Don't use training set for testing.
          testset = a(Jt,:);
          testset = setprior(testset,getprior(testset,0));
          e1(i,li) = testset*w*testfun;
        else
          testset = setprior(t,getprior(t,0));
          e1(i,li) = testset*w*testfun;
        end
        prwaitbar(0)

      end
      prwaitbar(0);

    end
    prwaitbar(0);

    % Calculate average error and standard deviation for this classifier
    % (or set the latter to zero if there's been just 1 repetition).

    e.error(wi,:) = mean(e1,1);
    e.apperror(wi,:) = mean(e0,1);
    if (nreps == 1)
      e.std(wi,:) = zeros(1,size(e.std,2));
      e.appstd(wi,:) = zeros(1,size(e.appstd,2));
    else
      e.std(wi,:) = std(e1)/sqrt(nreps);
      e.appstd(wi,:) = std(e0)/sqrt(nreps);
    end
  end
  prwaitbar(0);

  % The first element is the empty string [], remove it.
  e.names(1,:) = [];

return

