Logo Search packages:      
Sourcecode: libneuralnet version File versions  Download package

neuron.cc

//
// neuron.cc
//
// Made by Guillaume Stordeur
// Login   <kami@GrayArea.Masaq>
//
// Started on  Thu Aug  1 05:11:01 2002 Guillaume Stordeur
// Last update Mon May  5 22:08:23 2003 Guillaume Stordeur
//

#include <stdlib.h>
#include <iostream>
#include <math.h>
#include <vector>
#include "neuron.hh"
#include "utils.hh"

// Neuron related functions
//==================================================

namespace NeuralNet
{

  //
  // Add a weighted connection to neuron's input.
  //
  void      Neuron::addConnection(Neuron *src, float weight)
  {
    assert(src != NULL);
    _inputNeurons.push_back(src);
    _weights.push_back(weight);
    _momentum.push_back(0.0);
    _oldDeltae.push_back(0.0);
    _tri.push_back(0.05);
    _deltae.push_back(0);
  }

  //
  // Add a random-weighted connection to neuron's input.
  //
  void      Neuron::addConnection(Neuron *src)
  {
    assert(src != NULL);
    _inputNeurons.push_back(src);
    _weights.push_back(randWeight());
    _momentum.push_back(0.0);
    _oldDeltae.push_back(0.0);
    _tri.push_back(0.05);
    _deltae.push_back(0);
  }

  void      Neuron::setWeights(Matrix<double>& m)
  {
    assert(m.get_nb_lines() == _weights.size());
    assert(m.get_nb_cols() == 1);

    for (int i = 0; i < m.get_nb_lines(); i++)
      _weights[i] = m(i, 0);
  }

  // Change the weights for Rprop batch
  void      Neuron::updateWeights(float nPlus, float nMinus, float deltaMin, float deltaMax, bool errUp)
  {
    for (unsigned int i = 0; i < _inputNeurons.size(); i++)
      {
      //std::cout << "OldDelta: " << _oldDeltae[i] << ", delta: " << _deltae[i] << " ";
      float wd = 0;
      float tri;
      if ((_oldDeltae[i] > 0 && _deltae[i] > 0) ||
          (_oldDeltae[i] < 0 && _deltae[i] < 0))
        {
          tri = MIN(_tri[i] * nPlus, deltaMax);
          _tri[i] = tri;
          wd = (_deltae[i] > 0) ? _tri[i] : -_tri[i];
          _oldDeltae[i] = _deltae[i];
          //std::cout << "(+)wd = " << wd << " ";
        }
      else if ((_oldDeltae[i] > 0 && _deltae[i] < 0) ||
             (_oldDeltae[i] < 0 && _deltae[i] > 0))
        {
          tri = MAX(_tri[i] * nMinus, deltaMin);
          _tri[i] = tri;
          if (errUp)
            wd = -_momentum[i];
          _oldDeltae[i] = 0;
          //std::cout << "(-)wd = " << wd << " ";
        }
      else
        {
          wd = (_deltae[i] > 0) ? _tri[i] : -_tri[i];
          _oldDeltae[i] = _deltae[i];
          //std::cout << "(0)wd = " << wd << " ";
        }
      //std::cout << "tri: " << _tri[i] << " ";
      _momentum[i] = wd;
      _weights[i] += wd;
      }
    //std::cout << std::endl;
  }

  // Change the weights for backprop (batch mode)
  void      Neuron::updateWeights(float lRate, float moment)
  {
    for (unsigned int i = 0; i < _inputNeurons.size(); i++)
      {
      float wd = lRate * _deltae[i] + moment * _momentum[i];
      _momentum[i] = wd;
      _weights[i] += wd;
      }
  }

  // Quickprop update
  void      Neuron::updateWeights(float lRate, float moment, float mu)
  {
    float shrink = mu / (1.0 + mu);
    for (unsigned int i = 0; i < _inputNeurons.size(); i++)
      {
      float wd = 0;
      /*
      if (_momentum[i] > EPSILON)
        {
          // Add epsilon if positive slope
          if (_deltae[i] > 0.0)
            wd = (lRate * _deltae[i]);// / (float) _inputNeurons.size();

          // If slope > (or close to) prev slope, take max size step
          if (_deltae[i] > (shrink * _oldDeltae[i]))
            wd += mu * _momentum[i];
          else
            wd += (_deltae[i] / (_oldDeltae[i] - _deltae[i])) * _momentum[i];
          //std::cout << "(+)";
        }
      else if (_momentum[i] < -EPSILON)
        {
          // Add in epsilon if negative slope
          if (_deltae[i] < 0.0 )
            wd = (lRate * _deltae[i]);// / (float) _inputNeurons.size();

          // If slope < (or close to) prev slope, take max size step
          if (_deltae[i] < (shrink * _oldDeltae[i]))
            wd += mu * _momentum[i];
          else
            wd += (_deltae[i] / (_oldDeltae[i] - _deltae[i])) * _momentum[i];
          //std::cout << "(-)";
        }
      else
        {
          wd = (lRate * _deltae[i]) // / (float) _inputNeurons.size()
            + (moment * _momentum[i]);
          //std::cout << "(0)";
          }*/
      _deltae[i] *= -1.0f;
      float denom = _oldDeltae[i] - _deltae[i];
      float alpha;
      if (fabs(denom) < EPSILON)
        alpha = mu;
      else
        {
          float alpha2 = _deltae[i] / denom;
          alpha = (alpha2 > mu || alpha2 * _momentum[i] * _deltae[i] > 0) ? mu : alpha2;
        }
      wd = alpha * _momentum[i];
      if ((_momentum[i] < 0 && _deltae[i] > 0) || 
          (_momentum[i] > 0 && _deltae[i] < 0) ||
          (fabs(_momentum[i]) < EPSILON))
          wd -= lRate * _deltae[i];

      //std::cout << " oldDeltae: " << _oldDeltae[i] << ", deltae: " << _deltae[i] << ", wd: " << wd << std::endl;
      _oldDeltae[i] = _deltae[i];
      _momentum[i] = wd;
      _weights[i] += wd;
      }
  }

  //
  // Update a neuron's weights (stochastic mode)
  //
  void      Neuron::updateBackpropStochastic(float    lRate,
                               float      moment,
                               float      delta)
  {
    for (unsigned int i = 0; i < _inputNeurons.size(); i++)
      {
      if (!_fixed)
        {
          float wd = lRate * delta * _inputNeurons[i]->getOutput()
            + moment * _momentum[i];
          _momentum[i] = wd;
          _weights[i] += wd;
        }
      _inputNeurons[i]->incDwsum(delta * _weights[i]);
      }
  }

  //
  // Update a neuron's deltae (batch mode)
  //
  void      Neuron::updateBatch(float delta)
  {
    for (unsigned int i = 0; i < _inputNeurons.size(); i++)
      {
      _deltae[i] += delta * _inputNeurons[i]->getOutput();
      _inputNeurons[i]->incDwsum(delta * _weights[i]);
      }
  }

  //
  // Returns the index of n in the _inputNeurons vector
  // -1 is returned if not present.
  //
  int Neuron::isInputNeuron(Neuron *n)
  {
    for (unsigned int i = 0; i < _inputNeurons.size(); i++)
      if (_inputNeurons[i] == n)
      return i;
    return -1;
  }

  //
  // Calc weighted sum
  //
  void      Neuron::_calcWeightedSum(void)
  {
    assert(_inputNeurons.size() == _weights.size());
    std::vector<Neuron*>::const_iterator i = _inputNeurons.begin();
    std::vector<float>::const_iterator w = _weights.begin();
    _s = 0;
    for (; i != _inputNeurons.end(); ++i, ++w)
      _s += (*i)->getOutput() * *w;
  }

  // GaussNeuron related functions
  //==================================================

  //
  // Gauss activation function  y = [0..1]
  //
  inline float    GaussNeuron::_gauss(float s)
  {
    return (expf(-s * s));
  }

  //
  // Process inputs and calculate activation (output)
  //
  float     GaussNeuron::refreshOutput(void)
  {
    _calcWeightedSum();
    _output = _gauss(_s);
    return (_output);
  }

  //
  // Display
  //
  void      GaussNeuron::display(void) const
  {
    std::cout << "    Activation function: Gauss\n";
    std::cout << "    Threshold: " << _weights[0] << "\n";
    for (unsigned int i = 1; i < _weights.size(); i++)
      std::cout << "    Weight " << i << ": " << _weights[i] << "\n";
  }

  // TanhNeuron related functions
  //==================================================

  //
  // Tanh activation function  y = [-1..1]
  //
  inline float    TanhNeuron::_tanh(float s)
  {
    return (2 / (1 + expf(-2 * s)) - 1);
  }

  //
  // Process inputs and calculate activation (output)
  //
  float     TanhNeuron::refreshOutput(void)
  {
    _calcWeightedSum();
    _output = _tanh(_s);
    return (_output);
  }

  //
  // Display
  //
  void      TanhNeuron::display(void) const
  {
    std::cout << "    Activation function: Tanh\n";
    std::cout << "    Threshold: " << _weights[0] << "\n";
    for (unsigned int i = 1; i < _weights.size(); i++)
      std::cout << "    Weight " << i << ": " << _weights[i] << "\n";
  }

  // ApproxTanhNeuron related functions
  //==================================================

  //
  // ApproxTanh activation function y = [-1..1]
  //
  inline float    ApproxTanhNeuron::_tanh_approx(float s)
  {
    if (s > 1.92033)
      return 0.96016;
    else if (s > 0 && s <= 1.92033)
      return (0.96016 - 0.26037 * (s - 1.92033) * (s - 1.92033));
    else if (s > -1.92033 && s <= 0)
      return (0.26037 * (s + 1.92033) * (s + 1.92033) - 0.96016);
    else
      return (-0.96016);
  }

  //
  // Process inputs and calculate activation (output)
  //
  float     ApproxTanhNeuron::refreshOutput(void)
  {
    _calcWeightedSum();
    _output = _tanh_approx(_s);
    return (_output);
  }

  //
  // Display
  //
  void      ApproxTanhNeuron::display(void) const
  {
    std::cout << "    Activation function: Approximated Tanh\n";
    std::cout << "    Threshold: " << _weights[0] << "\n";
    for (unsigned int i = 1; i < _weights.size(); i++)
      std::cout << "    Weight " << i << ": " << _weights[i] << "\n";
  }

  // SigmoidNeuron related functions
  //==================================================

  //
  // Sigmoid activation function  y = [0..1]
  //
  inline float    SigmoidNeuron::_sigmoid(float s)
  {
    // avoid overflow
    if (s > 45)
      return 1;
    else if (s < -45)
      return 0;
    else
      return (1 / (1 + expf(-s)));
  }

  //
  // Process inputs and calculate activation (output)
  //
  float     SigmoidNeuron::refreshOutput(void)
  {
    _calcWeightedSum();
    _output = _sigmoid(_s);
    return (_output);
  }

  //
  // Display
  //
  void      SigmoidNeuron::display(void) const
  {
    std::cout << "    Activation function: Sigmoid\n";
    std::cout << "    Threshold: " << _weights[0] << "\n";
    for (unsigned int i = 1; i < _weights.size(); i++)
      std::cout << "    Weight " << i << ": " << _weights[i] << "\n";
  }

  // ApproxSigmoidNeuron related functions
  //==================================================

  //
  // Approximated Sigmoid activation function  y = [0..1]
  //
  inline float    ApproxSigmoidNeuron::_sigmoid_approx(float s)
  {
    s /= 4.1;
    if (s >= 1)
      return 1;
    else if (s <= -1)
      return 0;
    else
      return (0.5 + s * (1 - fabs(s) / 2));
  }

  //
  // Process inputs and calculate activation (output)
  //
  float     ApproxSigmoidNeuron::refreshOutput(void)
  {
    _calcWeightedSum();
    _output = _sigmoid_approx(_s);
    return (_output);
  }

  //
  // Display
  //
  void      ApproxSigmoidNeuron::display(void) const
  {
    std::cout << "    Activation function: Approximated Sigmoid\n";
    std::cout << "    Threshold: " << _weights[0] << "\n";
    for (unsigned int i = 1; i < _weights.size(); i++)
      std::cout << "    Weight " << i << ": " << _weights[i] << "\n";
  }


  // SgnNeuron related functions
  //==================================================

  //
  // Process inputs and calculate activation (output)  y = [0..1]
  //
  float     SgnNeuron::refreshOutput(void)
  {
    _calcWeightedSum();
    _output = (_s > EPSILON) ? 1.0 : 0.0;
    return (_output);
  }

  //
  // Display
  //
  void      SgnNeuron::display(void) const
  {
    std::cout << "    Activation function: Sign\n";
    std::cout << "    Threshold: " << _weights[0] << "\n";
    for (unsigned int i = 1; i < _weights.size(); i++)
      std::cout << "    Weight " << i << ": " << _weights[i] << "\n";
  }


  // LinearNeuron related functions
  //==================================================

  //
  // Process inputs and calculate activation (output)
  //
  float     LinearNeuron::refreshOutput(void)
  {
    _calcWeightedSum();
    _output = _s;
    return (_output);
  }

  //
  // Display
  //
  void      LinearNeuron::display(void) const
  {
    std::cout << "    Activation function: Linear\n";
    std::cout << "    Threshold: " << _weights[0] << "\n";
    for (unsigned int i = 1; i < _weights.size(); i++)
      std::cout << "    Weight " << i << ": " << _weights[i] << "\n";
  }

  //
  //  Alloc a new neuron
  //
  Neuron    *newNeuron(ActivationFunctionType   type,
                     ThresholdNeuron            *tneuron,
                     float                threshold)
  {
    switch(type)
      {
      case ACT_SGN:
      return new SgnNeuron(tneuron, threshold);
      case ACT_LINEAR:
      return new LinearNeuron(tneuron, threshold);
      case ACT_SIGMOID:
      return new SigmoidNeuron(tneuron, threshold);
      case ACT_SIGMOID_APPROX:
      return new ApproxSigmoidNeuron(tneuron, threshold);
      case ACT_TANH:
      return new TanhNeuron(tneuron, threshold);
      case ACT_TANH_APPROX:
      return new ApproxTanhNeuron(tneuron, threshold);
      case ACT_GAUSS:
      return new GaussNeuron(tneuron, threshold);
      default:
      return NULL;
      }
  }

} // end namespace

Generated by  Doxygen 1.6.0   Back to index