Logo Search packages:      
Sourcecode: libneuralnet version File versions  Download package

neuralnet.cc

//
// neuralnet.cc
//
// Made by Guillaume Stordeur
// Login   <kami@GrayArea.Masaq>
//
// Started on  Thu Aug  1 19:17:35 2002 Guillaume Stordeur
// Last update Mon May  5 22:11:33 2003 Guillaume Stordeur
//

#include <math.h>
#include <iostream>
#include <vector>
#include <sys/types.h>
#include <signal.h>
#include <unistd.h>
#include <exception/exception.hh>
#include "neuralnet.hh"
#include "nnfile.hh"
#include "libpgm.h"
#include "algo.hh"


volatile static sig_atomic_t  _sig;

static void _handleSigInt(int sig)
{
  _sig = 1;
}

namespace NeuralNet
{

//
// Main     Constructor
//
NeuralNet::NeuralNet(const std::vector<unsigned int> &nb,// nb of neurons per layer
                 ActivationFunctionType type,    // activation function type
                 ActivationFunctionType typeOut)
{
  _init(nb, type, typeOut);
}

//
// Load from file constructor
//
NeuralNet::NeuralNet(const std::string file)
{
  NNFile    nnfile;
  nnfile.inputFile(file);

  // Init with layers and type
  _init(nnfile.getLayers(), nnfile.getType(), nnfile.getTypeOut());

  // Add connections
  std::vector<std::vector<float> > connects = nnfile.getConnects();
  for (unsigned int i = 0; i < connects.size(); i++)
    addConnection((int)connects[i][0],
              (int)connects[i][1],
              (int)connects[i][2],
              (int)connects[i][3],
              connects[i][4]);
  // Add thresholds
  std::vector<std::vector<float> > thresholds = nnfile.getThresholds();
  for (unsigned int i = 0; i < thresholds.size(); i++)
    (_layers[(int)thresholds[i][0]])[(int)thresholds[i][1]]->setWeight(0, thresholds[i][2]);

  // Add fixed neurons
  std::vector<std::vector<unsigned int> > fixed = nnfile.getFixed();
  for (unsigned int i = 0; i < fixed.size(); i++)
    (_layers[fixed[i][0]])[fixed[i][1]]->setFixed(true);

  // Add timeLagged neurons
  std::vector<std::vector<unsigned int> > timeLagged = nnfile.getTimeLagged();
  for (unsigned int i = 0; i < timeLagged.size(); i++)
    (_layers[fixed[i][0]])[fixed[i][1]]->setTimeLagged(true);
}

// Dump the network to file
void  NeuralNet::outputFile(const std::string file) const
{
  NNFile    nnfile;
  nnfile.outputFile(file, _type, _typeOut, _layers);
}

// Load a neural net description file
void  NeuralNet::inputFile(const std::string file)
{
  NNFile    nnfile;
  nnfile.inputFile(file);
}

//
// Setup function called by the constructors
//
void  NeuralNet::_init(const std::vector<unsigned int> &nb,// nb of neurons per layer
                   ActivationFunctionType type,        // activation for hidden layers
                   ActivationFunctionType typeOut)// activation for output layer
{
  assert(nb.size() > 1); // at least 1 input layer and 1 output layer
  
  // Signal Handling
  struct sigaction      usr_action;
  sigset_t        block_mask;

  sigfillset(&block_mask);
  usr_action.sa_handler = _handleSigInt;
  usr_action.sa_mask = block_mask;
  usr_action.sa_flags = 0;
  sigaction(SIGINT, &usr_action, NULL);
  _sig = 0;
  
  // BackProp parameters
  _lRate = 0.6;
  _moment = 0.7;

  // Rprop Parameters
  _nPlus = 1.2;
  _nMinus = 0.5;
  _deltaMin = 0.00001;
  _deltaMax = 50;
  _tri0 = 0.05;

  // Quickprop Parameters
  _mu = 1.75;
  _wdecay = 0.0;

  // Convergence parameters
  _skip = 1;
  _epsilon = 0.001;
  _maxIter = 10000;
  _changeIter = 0;

  _gaussNewton = 0;
  
  // Activation Derivative fudge term
  _fudge = 0;

  // Error function
  _errfunc = ERR_DIFF;

  _type = type;
  _typeOut = typeOut;
  _thresholdNeuron = new ThresholdNeuron();
  _layers = std::vector<t_layer>(nb.size());
  // Input and Hidden layers
  for (unsigned int i = 0; i < nb.size() - 1; i++)
    {
      assert(nb[i] > 0);
      for (unsigned int j = 0; j < nb[i]; j++)
      {
        Neuron *neuron = (i == 0) ? new InputNeuron()
          : newNeuron(_type, _thresholdNeuron, 0.5);
        _layers[i].push_back(neuron);
      }
    }
  // Output layer
  assert(nb[nb.size() - 1] > 0);
  for (unsigned int j = 0; j < nb[nb.size() - 1]; j++)
    {
      Neuron *neuron = newNeuron(_typeOut, _thresholdNeuron, 0.5);
      _layers[nb.size() - 1].push_back(neuron);
    }
}

//
// Destructor, deletes the neurons
//
NeuralNet::~NeuralNet()
{
  delete _thresholdNeuron;
  for (unsigned int i = 0; i < _layers.size(); i++)
    for (unsigned int j = 0; j < _layers[i].size(); j++)
      delete (_layers[i])[j];
}

// (speedblue)
// Set a weight of the neuralnet
void  NeuralNet::setWeight(unsigned int layer, unsigned int neuron,
                       unsigned int weight, float value)
{
  _changeWeight(layer, neuron, weight, value);
}

// (speedblue)
// Return a weight of a neuron of the last layer
float NeuralNet::getLastLayerWeight(unsigned int neuron,
                            unsigned int weight) const
{
  if (neuron >= _layers[_layers.size() - 1].size())
    throw Exception::Exception("Invalid neuron", HERE);
  return _layers[_layers.size() - 1][neuron]->getWeight(weight);
}

// (speedblue)
// Init Neuralnetwork with a pcaalgorithm
//
void  NeuralNet::PCAinit(const std::string algo,
                   const std::vector<std::vector<float> >& inputs,
                   const std::vector<std::vector<float> >& desired)
{
  if (_layers.size() != 3)
    throw Exception::Exception("PCA need a 3 layers Perceptron", HERE);
  if (inputs.size() < 1 || inputs[0].size() < 1)
    throw Exception::Exception("Wrong vectors size", HERE);

  //First construct the matrix with new dimension for threshold
  Matrix<float> mat(inputs.size(), inputs[0].size() + 1);

  for (unsigned int i = 0; i < inputs.size(); i++)
    {
      // Set the first dimension to 1 for the threshold
      mat.set_data(i, 0, 1);
      for (unsigned int j = 1; j < inputs[0].size() + 1; j++)
      mat.set_data(i, j, inputs[i][j - 1]);
    }

  PCAAlgo<float> pca(mat);
  Matrix<float> res = pca.random_init(inputs[0].size() + 1, inputs[0].size() + 1);
  unsigned max_init = 0, nb = 0;
  float moyenne = 0, et = 0;
  unsigned int i = 0;
  float trace = mat.trace_of_varcov();
  float lambda_sum = 0.0;

  if (algo != "gha" && algo != "GHA" && algo != "ala" && algo != "ALA")
    throw Exception::Exception("Unknown PCA Algorithm", HERE);
  do
     {
      if (algo == "gha" || algo == "GHA")
      lambda_sum += pca.gha(res, i);
      else
      lambda_sum += pca.ala(res, i);
      max_init = i;
      //_layers[1][i]->setFixed(true);
      for (unsigned int j = 0; j < _layers[1][i]->getNBInputs(); j++)
      if (i < inputs.size() * inputs.size() &&
          j < inputs.size() * inputs.size())
        {
          _changeWeight(1, i, j, res(j, i));
          moyenne += res(i, j);
          nb++;
        }
      i++;
      std::cout << "Quality: " << (lambda_sum / trace) << std::endl;
    }
  // Stop when quality of representation is greater than 80% of the
  // original representation
  while ((i < _layers[1].size()) && ((lambda_sum / trace) < 0.8));

  // compute moy and ecart-type
  moyenne /= nb;
  for (unsigned  i = 0; i <= max_init; i++)
    for (unsigned  j = 0; j < _layers[1][i]->getNBInputs() - 1; j++)
      {
      float delta = (_layers[1][i]->getWeight(1 + i) - moyenne) *
        (_layers[1][i]->getWeight(1 + i) - moyenne);
      et += delta;
      }
  et = sqrt(et / (float)nb);
  // Init other weights with random values (between moy - ecart-type
  // and moy + ecart type)
  for (unsigned i = max_init + 1; i < _layers[1].size(); i++)
    for (unsigned j = 0; j < _layers[1][i]->getNBInputs() - 1; j++)
      {
      float rnd = mrand(moyenne - et, moyenne + et);
      _changeWeight(1, i, j + 1, rnd);
      }
  // Gauss-Newton Here to init last layer
  // Compute first layer only
  std::vector<std::vector<float> > inter_exemple;

  for (int i = 0; i < mat.get_nb_lines(); i++)   // get_nb_lines is an int...
    inter_exemple.push_back(this->output(inputs[i], 1));
  /*
  ** Having some problem : do not output normed vector of weights
  */
  if (_gaussNewton)
    {
      std::cout << "Beginning Gauss-Newton algorithm" << std::endl;
      _trainGaussNewton(inputs, inter_exemple, desired);
      std::cout << "Gauss-Newton ended" << std::endl;
    }

// Not yet working
//   std::cout << "Beginning Newton algorithm" << std::endl;
//   _trainNewton(inputs, inter_exemple, desired);
//   std::cout << "Newton ended" << std::endl;
}

//
// Add a weighted connection between 2 neurons
//
void  NeuralNet::addConnection(unsigned int srcLayer,
                         unsigned int src,
                         unsigned int dstLayer,
                         unsigned int dst,
                         float weight)
{
  assert(srcLayer < (_layers.size()));
  assert(dstLayer < (_layers.size()));
  assert(src < _layers[srcLayer].size());
  assert(dst < _layers[dstLayer].size());

  Neuron *dstN, *srcN;
  srcN = (_layers[srcLayer])[src];
  dstN = (_layers[dstLayer])[dst];
  assert(dstN && srcN);
  dstN->addConnection(srcN, weight);
  if (dstLayer <= srcLayer)
    dstN->setRecurrent(true);
}

//
// Add a random-weighted connection between 2 neurons
//
void  NeuralNet::addConnection(unsigned int srcLayer,
                         unsigned int src,
                         unsigned int dstLayer,
                         unsigned int dst)
{

  assert(srcLayer < (_layers.size()));
  assert(dstLayer < (_layers.size()));
  assert(src < _layers[srcLayer].size());
  assert(dst < _layers[dstLayer].size());

  Neuron *dstN, *srcN;
  srcN = (_layers[srcLayer])[src];
  dstN = (_layers[dstLayer])[dst];
  assert(dstN && srcN);
  dstN->addConnection(srcN);
  if (dstLayer <= srcLayer)
    dstN->setRecurrent(true);
}



//
// Get the network output for clamped input
//
std::vector<float>      NeuralNet::output(const std::vector<float> &inputs, int layer)
{
  assert(inputs.size() == getNbInputNeurons());
  assert(layer < (int)_layers.size());
  InputNeuron           *inNeuron;
  Neuron          *neuron;
  std::vector<float>    res;
  unsigned int olayer = layer < 0 ? (_layers.size() - 1) : layer;

  // Propagate inputs to hidden layers and to output
  for (unsigned int l = 0; l < _layers.size(); l++)
    for (unsigned int n = 0; n < _layers[l].size(); n++)
      {
      neuron = (_layers[l])[n];
      if (l == 0 && !neuron->getRecurrent())
        {
          inNeuron = dynamic_cast<InputNeuron *>(neuron);
          inNeuron->setInput(inputs[n - getNbInputRecurrentNeurons()]);
        }
      else
        neuron->refreshOutput();
      if (l == olayer)
        res.push_back(neuron->getOutput());
      }
  return res;
}

//
// Display
//
void  NeuralNet::display() const
{
  for (unsigned int l = 1; l < _layers.size(); l++)
    {
      std::cout << "Layer " << l << "\n";
      for (unsigned int n = 0; n < _layers[l].size(); n++)
      {
        std::cout << "  Neuron " << n << "\n";
        Neuron *neuron = (_layers[l])[n];
        neuron->display();
      }
    }
}

float NeuralNet::_errFunction(float dif)
{
  switch (_errfunc)
    {
    case ERR_DIFF:
      return dif;
    case ERR_SCALED_DIFF:
      return 10000 * dif;
    case ERR_TRIMMED_DIFF:
      if ((-0.1 < dif) && (dif < 0.1))
      return 0.0;
      else
      return dif;
    case ERR_ATANH:
      if (dif < -0.9) // .9999999
      return -17.0;
      else if (dif > 0.9)
      return 17.0;
      else
      return log((1.0 + dif) / (1.0 - dif));
    default:
      std::cerr << "Unsupported error function." << std::endl;
      return 0;
    }
}


//
// backward pass, calculates the deltae and changes weights if in
// stochastic mode
//
float NeuralNet::_backwardsPass(const std::vector<float> &desiredRes,
                          bool batch,
                          TrainingAlgorithm algorithm)
{
  assert(desiredRes.size() == (_layers.back()).size());

  Neuron    *neuron;
  float           d_y, delta, error = 0;

  _clear(DWSUM);
  // first calculate learning error
  for (unsigned int n = 0; n < (_layers.back()).size(); n++)
    {
      neuron = (_layers.back())[n];
      d_y = desiredRes[n] - neuron->getOutput();
      error += d_y * d_y;
      neuron->incDwsum(_errFunction(d_y));
    }
  // now process the layers down to the 1st
  for (unsigned int l = _layers.size() - 1; l > 0; l--)
    for (unsigned int n = 0; n < _layers[l].size(); n++)
      {
      neuron = (_layers[l])[n];
      delta = (neuron->getFPrime() + _fudge) * neuron->getDwsum();
      if (batch)
        neuron->updateBatch(delta);
      else
        switch (algorithm)
          {
          case ALG_BACKPROP:
            neuron->updateBackpropStochastic(_lRate, _moment, delta);
            break;
          default:
            std::cerr << "Unsupported algorithm reached backwardsPass!" << std::endl;
            break;
          }
      }
  return error * 0.5;//error / (float)(_layers.back()).size();
}

//
// Clears neuron parameters to ready for training
//
void  NeuralNet::_clear(int flags)
{
  for (unsigned int l = 0; l < _layers.size(); l++)
    for (unsigned int n = 0; n < _layers[l].size(); n++)
      {
      Neuron *neuron = (_layers[l])[n];
      neuron->clearset(flags, _wdecay, _tri0);
      }
}

//
// Outputs a reponse graph for 2d input and 1d output
//
void  NeuralNet::graph2d1output(const std::string file)
{
  assert(getNbInputNeurons() == 2 && _layers.back().size() == 1);
  std::vector<float>    input;
  float                 out;
  unsigned char         *data;

  data = new unsigned char[256*256];

  for (int i = 0; i < 256; i++)
    for (int j = 0; j < 256; j++)
      {
      input.clear();
      input.push_back(i / 255.0f);
      input.push_back(j / 255.0f);
      //input.push_back(((i / 255.0f) - 0.5) * 10);
      //input.push_back(((j / 255.0f) - 0.5) * 10);
      out = (output(input))[0];
      switch(_typeOut)
        {
        case ACT_TANH:
        case ACT_TANH_APPROX:
          out = out * 0.5 + 0.5;
          break;
        case ACT_LINEAR:
          out /= 40000;
          if (out >= 1.0)
            out = 1.0;
          break;
        default:
          break;
        }
      data[i * 256 + j] = (int) (out * 255);
      }
  store_pixmap(file.c_str(), data, 256, 256);
  delete data;
}

//
// Calculate total MSE / nbinputs
//
float NeuralNet::_computeError(const std::vector<std::vector<float> > &inputs,
                         const std::vector<std::vector<float> > &desired)
{
  float mse, totalerr = 0, d_y;
  std::vector<float> out;
  Neuron *neuron;
  unsigned int processed = 0;
  for (unsigned int i = 0; i < inputs.size(); i++)
    {
      if (i % _skip)
      continue;
      processed++; // FIXME: could just calc nbinputs
      out = output(inputs[i]);
      mse = 0;
      for (unsigned int n = 0; n < (_layers.back()).size(); n++)
      {
        neuron = (_layers.back())[n];
        d_y = desired[i][n] - neuron->getOutput();
        mse += d_y * d_y;
      }
      mse *= 0.5;
      //mse /= (float)(_layers.back()).size();
      totalerr += mse;
    }
  return totalerr / (float)processed;
}

//
// Train the network on input/desired vectors
//
float NeuralNet::_trainStochastic(const std::vector<std::vector<float> > &inputs,
                            const std::vector<std::vector<float> > &desired,
                            bool rand_permutations,
                            TrainingAlgorithm algorithm)
{
  assert(inputs.size() == desired.size());
  float     err = 0.0; 
  unsigned int iter = 0;
  std::vector<float> res;
  _clear(MOMENTUM);
  if (rand_permutations) // test factored out
    while ((err = _computeError(inputs, desired)) > _epsilon &&
         iter < _maxIter &&
         _sig == 0)
      {
      _displayError(iter++, err);
      // make a random permutation
      std::vector<unsigned int> indices = _make_permutation(inputs.size());
      for (unsigned int i = 0; i < inputs.size(); i++)
        {
          if (indices[i] % _skip)
             continue;
          res = output(inputs[indices[i]]);
          _backwardsPass(desired[indices[i]], false, algorithm);
        }
      }
  else
    while ((err = _computeError(inputs, desired)) > _epsilon &&
         iter < _maxIter &&
         _sig == 0)
      {
      _displayError(iter++, err);
      for (unsigned int i = 0; i < inputs.size(); i++)
        {
          if (i % _skip)
            continue;
          res = output(inputs[i]);
          _backwardsPass(desired[i], false, algorithm);
        }
      }
  if (_sig)
    {
      std::cerr << "Training interrupted." << std::endl;
      _sig = 0;
    }
  _displayError(iter, err);
  return err;
}

//
// Train the network on input/desired vectors
//
float NeuralNet::_trainBatch(const std::vector<std::vector<float> > &inputs,
                         const std::vector<std::vector<float> > &desired,
                         TrainingAlgorithm algorithm)
{
  assert(inputs.size() == desired.size());
  float     err = 0.0, changeErr = 100000, lastErr = 1000000;
  unsigned int iter = 0;
  std::vector<float> res;
  _clear(MOMENTUM | OLD_DELTAE | TRI);
  do
    {
      if (_wdecay)
      _clear(DECAY_DELTAE | OUTPUT);
      else  
      _clear(DELTAE | OUTPUT);
      unsigned int processed = 0;
      for (unsigned int i = 0; i < inputs.size(); i++)
      {
        if (i % _skip)
          continue;
        processed++; // FIXME: could just calc nbinputs
        res = output(inputs[i]);
        err += _backwardsPass(desired[i], true, algorithm);
      }
      err /= (float)processed;
      _displayError(++iter, err);
      if (err <= _epsilon)
      break;
      if (_changeIter && (iter % _changeIter == 0))
      {
        if (err >= changeErr)
          {
            std::cerr << "Halting training due to changeiter." << std::endl;
            break;
          }
        changeErr = err;
      }
      _updateWeights(algorithm, err, lastErr);
      lastErr = err;
    }
  while (iter < _maxIter && (_sig == 0));
  if (_sig)
    {
      std::cerr << "Training interrupted." << std::endl;
      _sig = 0;
    }
  return err;
}

Matrix<double>    NeuralNet::_makeJ(Neuron *neuron, 
                          Matrix<double>& weights, 
                          Matrix<double>& exemple)
{
  assert(weights.get_nb_cols() == 1);
  assert(exemple.get_nb_cols() == weights.get_nb_lines());
  Matrix<double> J(exemple.get_nb_lines(), exemple.get_nb_cols());

  for (int i = 0; i < exemple.get_nb_lines(); i++)
    {
      double activation = (weights.transpose() * exemple.get_line(i).transpose())(0, 0);
      double div = 1 + exp(-activation);
      double fprime = (1 / div) - (1 / (div * div));
      
      for (int j = 0; j < exemple.get_nb_cols(); j++)
      J.set_data(i, j, fprime * exemple(i, j));
    }
  return J;
}


float NeuralNet::_trainGaussNewton(const std::vector<std::vector<float> > &inputs,
                             const std::vector<std::vector<float> > &inter_exemple,
                             const std::vector<std::vector<float> > &desired)
{
  // We want at least one output desired
  assert(desired.size() > 0);
  Neuron    *neuron;
  float           err = 0.0, save_err = 0.0;
  // Warning: too many copy into matrix
//   Matrix<float> minputs(inter_exemple.size(), inter_exemple[0].size() + 1);
//   minputs.insert(0, 1, inter_exemple);
  std::vector<std::vector<double> > inter_exemple2(inter_exemple.size(), inter_exemple[0].size());
  for (unsigned int i = 0; i < inter_exemple.size(); i++)
    for (unsigned int j = 0; j < inter_exemple[0].size(); j++)
      inter_exemple2[i][j] = inter_exemple[i][j];

  std::vector<std::vector<double> > inputs2(inputs.size(), inputs[0].size());
  for (unsigned int i = 0; i < inputs.size(); i++)
    for (unsigned int j = 0; j < inputs[0].size(); j++)
      inputs2[i][j] = inputs[i][j];

  std::vector<std::vector<double> > desired2(desired.size(), desired[0].size());
  for (unsigned int i = 0; i < desired.size(); i++)
    for (unsigned int j = 0; j < desired[0].size(); j++)
      desired2[i][j] = desired[i][j];

  Matrix<double> minputs(inter_exemple2.size(), inter_exemple2[0].size() + 1);
  
//     neuron = (this->_layers[_layers.size() - 1])[0];
  minputs.insert(0, 1, inter_exemple2);
  for (unsigned i = 0; i < inter_exemple.size(); i++)
    minputs.set_data(i, 0, 1);
  
//   Matrix<float> tmp = minputs.center();
//   minputs.reduce(tmp);
  // add "threshold" input
  //Matrix<double> inverse = minputs.inverse();
  Matrix<double> mdesired(desired2);
  assert(minputs.get_nb_lines() == mdesired.get_nb_lines());
  Matrix<double> error(mdesired.get_nb_lines(), mdesired.get_nb_cols());
  Matrix<double> output(mdesired.get_nb_lines(), mdesired.get_nb_cols());
  float epsilon = 1.0;
//     std::cout << "minputs:" << minputs << std::endl
//           << "inverse:" << inverse << std::endl
//           << "II+I - I:" << (minputs * inverse * minputs) - minputs << std::endl
//           << "I+II+ - I+:" << (inverse * minputs * inverse) - inverse << std::endl;

  for (int i = 0; i < minputs.get_nb_lines(); i++)
    {
      std::vector<float> out = this->output(inputs[i]);
      for (unsigned int j = 0; j < out.size(); j++)
        output.set_data(i, j, out[j]);
    }
  error = output - mdesired;
  for (int i = 0; i < error.get_nb_cols(); i++)
    err += error.get_col(i).norme();
  unsigned int iter = 0;
  do
    {
      save_err = err;
      _displayError(iter, err * 0.5);
      for (unsigned int i = 0; i < _layers[_layers.size() - 1].size(); i++)
            {
        neuron = (_layers[_layers.size() - 1])[i];
        Matrix<double> weights = neuron->getWeights();
        Matrix<double> J = _makeJ(neuron, weights, minputs);
//      std::cout << "J: " << J << std::endl
//              << "II+I - I" << (J * J.inverse() * J) - J << std::endl
//              << "I+II+ -I+" << (J.inverse() * J * J.inverse()) - J.inverse() << std::endl;
        Matrix<double> new_weights = weights
          - J.inverse() * error.get_col(i) * epsilon;
        neuron->setWeights(new_weights);
        //    neuron->setFixed(true);
            }

      for (int i = 0; i < minputs.get_nb_lines(); i++)
      {     
        std::vector<float> out = this->output(inputs[i]);
        for (unsigned int j = 0; j < out.size(); j++)
          output.set_data(i, j, out[j]);
      }
      error = output - mdesired;
      err = 0.0;
      for (int i = 0; i < error.get_nb_cols(); i++)
      err += error.get_col(i).norme();

      iter++;
    }
  // delete fabs to refuse augmentation of err (save_err - err < 0
  while ((save_err - err) > 1e-6);
  _displayError(iter, err * 0.5);
  return err;
}

float NeuralNet::_trainNewton(const std::vector<std::vector<float> > &inputs,
                        const std::vector<std::vector<float> > &inter_exemple,
                        const std::vector<std::vector<float> > &desired)
{
  // We want at least one output desired
  assert(desired.size() > 0);
  Neuron    *neuron;
  float           err = 0, save_err = 0.0;
  // Warning: too many copy into matrix
  std::vector<std::vector<double> > inter_exemple2(inter_exemple.size(), inter_exemple[0].size());
  for (unsigned int i = 0; i < inter_exemple.size(); i++)
    for (unsigned int j = 0; j < inter_exemple[0].size(); j++)
      inter_exemple2[i][j] = inter_exemple[i][j];

  std::vector<std::vector<double> > inputs2(inputs.size(), inputs[0].size());
  for (unsigned int i = 0; i < inputs.size(); i++)
    for (unsigned int j = 0; j < inputs[0].size(); j++)
      inputs2[i][j] = inputs[i][j];

  std::vector<std::vector<double> > desired2(desired.size(), desired[0].size());
  for (unsigned int i = 0; i < desired.size(); i++)
    for (unsigned int j = 0; j < desired[0].size(); j++)
      desired2[i][j] = desired[i][j];

  Matrix<double> minputs(inter_exemple2.size(), inter_exemple2[0].size() + 1);
  minputs.insert(0, 1, inter_exemple2);
  // add "threshold" input
  for (unsigned i = 0; i < inter_exemple2.size(); i++)
    minputs.set_data(i, 0, 1);
//   Matrix<float> inverse = minputs.inverse();
  Matrix<double> mdesired(desired2);
  assert(minputs.get_nb_lines() == mdesired.get_nb_lines());
  Matrix<double> error(mdesired.get_nb_lines(), mdesired.get_nb_cols());
  Matrix<double> output(mdesired.get_nb_lines(), mdesired.get_nb_cols());

  for (int i = 0; i < minputs.get_nb_lines(); i++)
    {
      std::vector<float> out = this->output(inputs[i]);
      for (unsigned int j = 0; j < out.size(); j++)
        output.set_data(i, j, out[j]);
    }
  error = output - mdesired;
  for (int i = 0; i < error.get_nb_cols(); i++)
    err += error.get_col(i).norme();
  unsigned int iter = 0;
  std::cout << "minputs:" << minputs << std::endl;
  do
    {
      save_err = err;
      _displayError(iter, err * 0.5);
      for (unsigned int i = 0; i < 1 /* _layers[_layers.size() -
                              1].size() */; i++)
      {
        neuron = (_layers[_layers.size() - 1])[i];
        Matrix<double> hessian(minputs.get_nb_cols(), minputs.get_nb_cols());
        Matrix<double> exemple(1, minputs.get_nb_cols());
        Matrix<double> delta(minputs.get_nb_cols(), 1);
        for (int i = 0; i < minputs.get_nb_lines(); i++)
          {
//          Matrix<double> exemple = minputs.get_line(i);
            exemple += minputs.get_line(i) * error(i, 0);
            std::cout << "exemple:" << exemple << std::endl;
//          Matrix<double> hessian = exemple.transpose() * exemple;
            hessian += minputs.get_line(i).transpose() * minputs.get_line(i);
            std::cout << "hessian:" << hessian << std::endl;
//          delta += (hessian * exemple.transpose() * ;
          }
        delta = hessian * exemple.transpose();
        std::cout << "delta:" << delta << std::endl;
        Matrix<double> new_weights = neuron->getWeights() - delta;
        std::cout << "new_weights:" << new_weights << std::endl;
//      Matrix<float> tmp1 = new_weights.center();
//      new_weights.reduce(tmp1);
        neuron->setWeights(new_weights);
      }

      for (int i = 0; i < minputs.get_nb_lines(); i++)
      {     
        std::vector<float> out = this->output(inputs[i]);
        for (unsigned int j = 0; j < out.size(); j++)
          output.set_data(i, j, out[j]);
      }
      error = output - mdesired;
      err = 0.0;
      for (int i = 0; i < error.get_nb_cols(); i++)
      err += error.get_col(i).norme();
      iter++;
    }
  while ((fabs(save_err - err)) > 1e-6);
  _displayError(iter, err * 0.5);
  return err;
}

void  NeuralNet::_updateWeights(TrainingAlgorithm algorithm, float err, float lastErr)
{
  Neuron    *neuron;
  bool            errUp;
  // now process the layers down to the 1st
  for (unsigned int l = _layers.size() - 1; l > 0; l--)
    for (unsigned int n = 0; n < _layers[l].size(); n++)
      {
      neuron = (_layers[l])[n];
      if (neuron->getFixed())
        continue;
      switch (algorithm)
        {
        case ALG_BACKPROP:
          neuron->updateWeights(_lRate, _moment);
          break;
        case ALG_QUICKPROP:
          neuron->updateWeights(_lRate, _moment, _mu);
          break;
        case ALG_RPROP:
          errUp = (err > lastErr);
          neuron->updateWeights(_nPlus, _nMinus, _deltaMin, _deltaMax, errUp);
          break;
        default:
          std::cerr << "Don't know how to update the weights!" << std::endl;
          break;
        }
      }
}

//
// Make a random permutation vector
//
std::vector<unsigned int>     NeuralNet::_make_permutation(unsigned int size)
{
  std::vector<unsigned int>   indices;
  for (unsigned int i = 0; i < size; i++)
    indices.push_back(i);
  for (unsigned int i = size - 1; i > 0; --i)
    {
      unsigned int index = (unsigned int) ((rand() / (RAND_MAX + 1.0)) * i);
      unsigned int temp = indices[index];
      indices[index] = indices[i];
      indices[i] = temp;
    }
  return indices;
}

// Main training function
float NeuralNet::trainNetwork(const std::vector<std::vector<float> > &inputs,
                        const std::vector<std::vector<float> > &desired,
                        bool batch,
                        TrainingAlgorithm algorithm)
{
  // Gauss-Newton only works on the perceptron
  if (algorithm == ALG_GAUSS_NEWTON)
    {
      if (_layers.size() == 2)
      return _trainGaussNewton(inputs, inputs, desired);
      else
      {
        std::cerr << "Gauss-Newton only works on perceptrons (2 layers)" << std::endl;
        return -1;
      }
    }
  if (!batch) // stochastic mode
    {
      switch (algorithm)
      {
      case ALG_BACKPROP:
        return _trainStochastic(inputs, desired, true, algorithm);
      default:
        std::cerr << "Unsupported algorithm type for stochastic mode." << std::endl;
        std::cerr << "Currently only BACKPROP is supported." << std::endl;
        return -1;
      }
    }
  else // batch mode
    {
      switch (algorithm)
      {
      case ALG_BACKPROP:
      case ALG_RPROP:
      case ALG_QUICKPROP:
        return _trainBatch(inputs, desired, algorithm);
      default:
        std::cerr << "Unsupported algorithm type for batch mode." << std::endl;
        std::cerr << "Currently only BACKPROP, QUICKPROP and RPROP are supported." << std::endl;
        return -1;
      }
    }
}

void  NeuralNet::unsetFixed()
{
  Neuron    *neuron;

  std::cout << "Unfixing neurons" << std::endl;
  //   for (unsigned int l = _layers.size() - 1; l > 0; l--)
  //     for (unsigned int n = 0; n < _layers[l].size(); n++)
  for (unsigned int n = 0; n < _layers[_layers.size() - 1].size(); n++)
    {
      neuron = (_layers[_layers.size() - 1])[n];
      neuron->setFixed(false);
    }
}


} // end NeuralNet namespace

Generated by  Doxygen 1.6.0   Back to index