Logo Search packages:      
Sourcecode: libneuralnet version File versions  Download package

timelaggednetwork.cc

/*
** timelaggednetwork.cc
** Login : <nico@Altarion.marmotte.ath.cx>
** Started on  Sun Jul 20 10:56:29 2003 Nicolas
** $Id: timelaggednetwork.cc,v 1.22 2003/09/23 13:42:46 nico Exp $
**
** Copyright (C) 2003 Nicolas
** This program is free software; you can redistribute it and/or modify
** it under the terms of the GNU Lesser General Public License as published by
** the Free Software Foundation; either version 2 of the License, or
** (at your option) any later version.
**
** This program is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
** GNU Lesser General Public License for more details.
**
** You should have received a copy of the GNU Lesser General Public License
** along with this program; if not, write to the Free Software
** Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/

#include <fstream>
#include <iostream>
#include "timelaggednetwork.hh"


namespace NeuralNet
{

  TimeLaggedNeuralNet::TimeLaggedNeuralNet(const std::vector<unsigned int>& nb,
                                 const std::vector<unsigned>& winsize,
                                 ActivationFunctionType type,
                                 ActivationFunctionType typeout)
  {

    _init_variables();
    std::cerr << "Building TimeLagged Neural Network: " << std::endl;
    _init_network(nb, winsize, type, typeout);
    _init_connections();
  }

  TimeLaggedNeuralNet::TimeLaggedNeuralNet(const std::string& file)
  {
    std::cerr << "Building TimeLagged Neural Network from file" << std::endl;
    inputFile(file);
  }

  float TimeLaggedNeuralNet::trainNetwork(const NNdata_t& inputs,
                                bool batch, TrainingAlgorithm algo)
  {
    std::cerr << "Training TimeLagged Neural Network" << std::endl;
    //    return  NeuralNet::trainNetwork(inputs, desired, batch,
    //    algo);
    assert(inputs.size() > 0);

    std::vector<float> o = output(inputs[0], _layers.size());
    std::cout <<   " Output: " << std::endl;
    for (unsigned i = 0; i < o.size(); i ++)
      std::cout << "        "<< i << ": " << o[i] << std::endl;

    _train(inputs, batch, algo);
    o = output(inputs[0], _layers.size());
    std::cout <<   " Output: " << std::endl;
    for (unsigned i = 0; i < o.size(); i ++)
      std::cout << "        "<< i << ": " << o[i] << std::endl;

    return 0;
  }

  //
  // Get the network output for clamped input
  //
  std::vector<float> TimeLaggedNeuralNet::output(const std::vector<float> &inputs,
                                     unsigned int layer)
  {
    assert(layer == _layers.size());
    Neuron*             neuron;
    TimeLaggedInputNeuron*    inneuron;
    std::vector<float>  res;
    CIter it = inputs.begin();
    CIter it2 = inputs.begin();
    advance(it2, _winsize[0]);

    // considering input neurons
    for (unsigned n = 0; n < _inputlayer.size(); ++ n)
      {
      inneuron = _inputlayer[n];
      inneuron->setInput(it, it2);
      }


    --layer; // vectors begins at 0;
    for (unsigned int l = 0; l < _layers.size(); l++)
      for (unsigned int n = 0; n < _layers[l].size(); n++)
      {
        neuron = (_layers[l])[n];
        neuron->refreshOutput();
        if (l == layer)
          res.push_back(neuron->getOutput());
      }

    return res;
  }

  //
  // Get the network output for clamped input
  //
  std::vector<float> TimeLaggedNeuralNet::outputlag(const std::vector<float> &inputs,
                                        unsigned lag)
  {
    Neuron*             neuron;
    TimeLaggedInputNeuron*    inneuron;
    std::vector<float>  res;
    CIter it = inputs.begin();
    advance(it, lag);

    // considering input neurons
    for (unsigned n = 0; n < _inputlayer.size(); ++ n)
      {
      inneuron = _inputlayer[n];
      inneuron->setInput(it);
      }

    for (unsigned int l = 0; l < _layers.size(); l++)
      for (unsigned int n = 0; n < _layers[l].size(); n++)
      {
        neuron = (_layers[l])[n];
        neuron->refreshOutput();
        if (l == _layers.size() - 1)
          res.push_back(neuron->getOutput());
      }

    return res;
  }

  void      TimeLaggedNeuralNet::_init_variables()
  {
    // BackProp parameters
    _lRate = 0.6;
    _moment = 0.7;

    // Rprop Parameters
    _nPlus = 1.2;
    _nMinus = 0.5;
    _deltaMin = 0.00001;
    _deltaMax = 50;
    _tri0 = 0.05;

    // Quickprop Parameters
    _mu = 1.75;
    _wdecay = 0.0;

    // Convergence parameters
    _epsilon = 0.001;
    _maxIter = 10000;
    _changeIter = 0;

    // Activation Derivative fudge term
    _fudge = 0;

    // Error function
    _errfunc = ERR_DIFF;
  }

  void TimeLaggedNeuralNet::_init_network(const std::vector<unsigned int> &nb,
                                const std::vector<unsigned>& winsize,
                                ActivationFunctionType type,
                                ActivationFunctionType typeout)
  {
    assert(nb[0] == winsize.size());
    _winsize = winsize;

    _thresholdNeuron = new ThresholdNeuron();
    _layers = std::vector<t_layer>(nb.size() - 1);

    for (unsigned j = 0; j < nb[0]; ++j)
      {
      TimeLaggedInputNeuron* neuron = new TimeLaggedInputNeuron(winsize[j]);
      _inputlayer.push_back(neuron);
      }

    // Hidden layers
    for (unsigned int i = 1; i < nb.size() - 1; i++)
      {
      assert(nb[i] > 0);
      for (unsigned int j = 0; j < nb[i]; j++)
        {
          Neuron *neuron = (i == 1) ? new TimeLaggedNeuron()
            : newNeuron(type, _thresholdNeuron, 0.5);
          _layers[i - 1].push_back(neuron);
        }
      }
    
    TimeLaggedInputNeuron* n = new TimeLaggedTresholdInputNeuron(winsize[0]);
    _inputlayer.push_back(n);

    // Output layer
    assert(nb[nb.size() - 1] > 0);
    for (unsigned int j = 0; j < nb[nb.size() - 1]; j++)
      {
      Neuron *neuron = newNeuron(typeout, _thresholdNeuron, 0.5);
      _layers[nb.size() - 2].push_back(neuron);
      }
  }

  void      TimeLaggedNeuralNet::_init_connections()
  {

    // Connection of special input neurons to the first layer
    std::vector<Neuron*>::iterator iter = _layers[0].begin();
    for (;iter != _layers[0].end(); ++iter)
      for (std::vector<TimeLaggedInputNeuron*>::iterator it = _inputlayer.begin();
         it != _inputlayer.end();
         ++it)
      {
        TimeLaggedNeuron* neuron = dynamic_cast<TimeLaggedNeuron *>(*iter);
        assert(neuron != 0x0);
        neuron->addConnection(*it);
      }
    // Connection of the rest ...
    for (unsigned i = 1; i < _layers.size(); ++i)
      {
      iter = _layers[i].begin();
      for (;iter != _layers[i].end(); ++iter)
        for (std::vector<Neuron*>::iterator it = _layers[i - 1].begin();
             it != _layers[i - 1].end(); ++it)
          (*iter)->addConnection(*it);
      }
  }


  void      TimeLaggedNeuralNet::display() const
  {
    for (unsigned int l = 0; l < _layers.size(); l++)
      {
      std::cout << "Layer " << l << "\n";
      for (unsigned int n = 0; n < _layers[l].size(); n++)
        {
          std::cout << "  Neuron " << n << "\n";
          Neuron *neuron = (_layers[l])[n];
          neuron->display();
        }
      }
  }

  // Dump the network to file
  void      TimeLaggedNeuralNet::outputFile(const std::string file) const
  {
    std::ofstream ofs(file.c_str());
    if (!ofs.is_open())
      {
      std::cerr << "Error opening file" << file << std::endl;
      exit (1);
      }
    ofs << "# TLNN description file" << std::endl;
    ofs << "window = " << _winsize[0] << std::endl;
    ofs << "inputlayersize = " << _inputlayer.size() << std::endl;
    // Dump the First layer -> TimeLaggedNeuron
    ofs << "- start layer 0" << std::endl;
    for (unsigned int n = 0; n < _layers[0].size(); n++)
      {
      // for each neuron
      ofs << " - start neuron " << n << std::endl;
      for (unsigned int i = 0; i < _inputlayer.size(); i++)
        {
          // We dump all the timelaggedweights
          ofs << "  connexion " << i;
          std::vector<float> v = dynamic_cast<TimeLaggedNeuron *>
            (_layers[0][n])->getWeight(i);
          for (unsigned int j = 0; j < v.size(); j++)
            ofs << ":" << v[j];
          ofs << std::endl;
        }
      ofs << " - end neuron " << n << std::endl;
      }
    ofs << "- end layer 0" << std::endl;
    // Dump all other layer.
    for (unsigned int l = 1; l < _layers.size(); l++)
      {
      ofs << "- start layer " << l << std::endl;
      for (unsigned int n = 0; n < _layers[l].size(); n++)
        {
          // For each neuron, we dump all the weights of the
          // connection with neurons of the previous layer
          ofs << "  Neuron " << n;
          Neuron *neuron = (_layers[l])[n];
          Matrix<double> m = neuron->getWeights();
          for (int i = 0; i < m.get_nb_lines(); i++)
            ofs << ":" << m(i, 0), 0;
          ofs << std::endl;
        }
      ofs << "- end layer " << l << std::endl;
      }
    ofs.close();
  }

  // This function is used to split the vector of weights when we read
  // it
  void       TimeLaggedNeuralNet::split(const std::string& s,
                              std::vector<std::string>& v,
                              unsigned char sep)
  {
    int old_pos = -2;
    int pos = -1;

    v.clear();
    while (old_pos != pos)
      {
        old_pos = pos;
        pos = s.find(sep, pos + 1);
        if (pos < 0)
          {
            v.push_back(s.substr(old_pos + 1, s.size() - old_pos - 1));
            old_pos = pos;
          }
        else
          v.push_back(s.substr(old_pos + 1, pos - old_pos - 1));
      }
  }


  // Load a neural net description file
  void      TimeLaggedNeuralNet::inputFile(const std::string file)
  {
    std::ifstream ifs(file.c_str());
    if (!ifs.is_open())
      {
      std::cerr << "Error opening file" << file << std::endl;
      exit (1);
      }
    // Delete the existing network
    for (std::vector<t_layer>::iterator i = _layers.begin();
       i != _layers.end();
       i++)
      for (std::vector<Neuron*>::iterator j = i->begin();
         j != i->end();
         j++)
      delete (*j);
    _layers.clear();
    // Delete the input layer
    for (std::vector<TimeLaggedInputNeuron *>::iterator i = _inputlayer.begin();
       i != _inputlayer.end();
       i++)
      delete (*i);
    _inputlayer.clear();
    _winsize.clear();
    // Buffer used to read data
    std::string line;
    // The layer under construction
    t_layer tmplayer;
    // Specify if we are inside a layer or note
    bool inlayer = false;
    // Specify the number of the layer
    unsigned layernum = 0;
    //
    TimeLaggedNeuron *tmptimelaggedneuron = NULL;
    // Used to assigned the vector of weights between the good
    // timelaggedinputneuron and the current "normal" neuron
    std::vector<TimeLaggedInputNeuron*>::iterator ittlin;
    //
    bool endinputsize = false;
    while (std::getline(ifs, line))
      {
      if (line.c_str()[0] == '#') // This is a comment
        continue;
      // A new layer is started
      else if (!line.compare(0, 14, "- start layer "))
        {
          inlayer = true;
          tmplayer.clear();
        }
      // End of a layer
      else if (!line.compare(0, 12, "- end layer "))
        {
          if (inlayer == false)
            {
            std::cerr << "Error in file : " << file
                    << " bad data format" << std::endl;
            }
          inlayer = false;
          layernum++;
          // We add this new layer to the _layers member.
          _layers.push_back(tmplayer);
        }
      // Definition of the window size
      else if (!line.compare(0, 9, "window = "))
        {
          char *nb = strndup(&line.c_str()[9], line.size() - 9);
          _winsize.push_back(atoi(nb));
          free(nb);
        }
      // definition of the input layer vector size
      else if (!line.compare(0, 17, "inputlayersize = "))
        {
          char *nb = strndup(&line.c_str()[17], line.size() - 17);
          unsigned inputsize = atoi(nb) - 1;
          free(nb);
          // Now, we know the number of input neuron, so we can
          // construct the vector.
          for (unsigned int i = 0; i < inputsize; i++)
            _inputlayer.push_back(new TimeLaggedInputNeuron(_winsize[0]));
          _inputlayer.push_back(new TimeLaggedTresholdInputNeuron(_winsize[0]));
        }
      // We are in a layer, we add a normal neuron
      else if ((inlayer) && (!line.compare(0, 9, "  Neuron ")) && (layernum != 0))
        {
          std::vector<std::string> v;
          std::vector<float> w;
          split(line, v, ':');
          for (unsigned int i = 1; i < v.size(); i++)
            w.push_back(atof(v[i].c_str()));
          Neuron *tmpneuron = newNeuron(ACT_SIGMOID,
                                new ThresholdNeuron(),
                                w[0]);
          unsigned i = 1;
          // We add connection with this neuron and all neuron of
          // the layer before it.
          for (std::vector<Neuron*>::iterator it =
               _layers[layernum - 1].begin();
             it != _layers[layernum - 1].end();
             ++it)
            tmpneuron->addConnection(*it, w[i++]);
          tmplayer.push_back(tmpneuron);
        }
      // Special Case for the first layer
      else if (layernum == 0)
        {
          // Start a new timelaggedneuron
          if (!line.compare(0, 16, " - start neuron "))
            {
            tmptimelaggedneuron = new TimeLaggedNeuron();
            ittlin = _inputlayer.begin();
            endinputsize =  (ittlin == _inputlayer.end());
            }
          // End of a timelaggedneuron
          else if (!line.compare(0, 14, " - end neuron "))
            {
            tmplayer.push_back(tmptimelaggedneuron);
            }
          // We create the differents connections
          else if (!line.compare(0, 12, "  connexion "))
            {
            assert(!endinputsize);
            std::vector<std::string> v;
            std::vector<float> w;
            split(line, v, ':');
            for (unsigned int i = 1; i < v.size(); i++)
              w.push_back(atof(v[i].c_str()));
            tmptimelaggedneuron->addConnection(*ittlin, w);
            ittlin++;
            endinputsize =  (ittlin == _inputlayer.end());
            }
        }
      line = "";
      }
    ifs.close();
  }


  //-----------------------------------------------
  // Backprop

  float TimeLaggedNeuralNet::_train(const NNdata_t& inputs,
                            bool batch, TrainingAlgorithm algo)
  {
    // Batch Training, still a question how to resolve data attachment
    // Which neuron gets which data...?
    // Should add a way to know the desired output value, from the
    // inputs series...

    float   err, changeErr = 1000000, lastErr = 1000000;
    unsigned      iter = 0;
    std::vector<float>  res, desired(1);
    int size =  (_winsize[0] + 1);
    _clear(MOMENTUM | OLD_DELTAE | TRI);
    do
      {
      _clear(DELTAE);
      err = 0;

      for (unsigned i = 0; i < inputs[0].size() - size; i++)
        {
          res = outputlag(inputs[0], i);
          desired[0] = inputs[0][i + size];
          err += _backwardsPass(desired, batch, algo);
        }

      _displayError(++iter, err);
      if (err <= _epsilon)
        break;
      if (_changeIter && (iter % _changeIter == 0))
        {
          if (err >= changeErr)
            {
            std::cerr << "Halting training due to changeiter." << std::endl;
            break;
            }
          changeErr = err;
        }
      _updateWeights(ALG_BACKPROP, err, lastErr);
      lastErr = err;
      }
    while (iter < _maxIter);

    return err;
  }

  float TimeLaggedNeuralNet::_backwardsPass(const std::vector<float>& desiredRes,
                                  bool batch,
                                  TrainingAlgorithm algo)
  {
    Neuron  *neuron;
    //    TimeLaggedInputNeuron* inneuron;
    float         d_y, delta, error = 0;

    _clear(DWSUM);
    // first calculate learning error
    for (unsigned int n = 0; n < (_layers.back()).size(); n++)
      {
      neuron = (_layers.back())[n];
      d_y = desiredRes[n] - neuron->getOutput();
      error += d_y * d_y * 0.5;
      neuron->incDwsum(_errFunction(d_y));
      }
    for (int l = _layers.size() - 1; l >= 0; l--)
      for (unsigned int n = 0; n < _layers[l].size(); n++)
      {
        neuron = (_layers[l])[n];
        delta = (neuron->getFPrime() + _fudge) * neuron->getDwsum();
        if (batch)
          neuron->updateBatch(delta);
        else
          {
            assert(algo == ALG_BACKPROP);
            neuron->updateBackpropStochastic(_lRate, _moment, delta);
          }
      }
    return error;
  }

  void      TimeLaggedNeuralNet::_updateWeights(TrainingAlgorithm algorithm, float err, float lastErr)
  {
    Neuron  *neuron;
    bool          errUp;

    // now process the layers down to the 1st
    for (int l = _layers.size() - 1; l >= 0; l--)
      for (unsigned int n = 0; n < _layers[l].size(); n++)
      {
        neuron = (_layers[l])[n];
        if (neuron->getFixed())
          continue;
        switch (algorithm)
          {
          case ALG_BACKPROP:
            neuron->updateWeights(_lRate, _moment);
            break;
          case ALG_QUICKPROP:
            neuron->updateWeights(_lRate, _moment, _mu);
            break;
          case ALG_RPROP:
            errUp = (err > lastErr);
            neuron->updateWeights(_nPlus, _nMinus,
                            _deltaMin, _deltaMax, errUp);
            break;
          default:
            std::cerr << "Don't know how to update the weights!" << std::endl;
            break;
          }
      }
  }


}// ! namespace NeuralNet

Generated by  Doxygen 1.6.0   Back to index