#include <neural_net.h>

Classes
struct	Node
struct	WeightedNode
Public Member Functions
	NeuralNet ()
virtual	~NeuralNet ()
template<typename Type >
bool	FeedForward (const Type inputs, Type outputs)
template<typename Type >
bool	GetNetOutput (const Type inputs, int output_id, Type output)
int	in_cnt () const
int	out_cnt () const
Static Public Member Functions
static NeuralNet *	FromFile (const string file_name)
static NeuralNet *	FromInputBuffer (InputFileBuffer *ib)
Protected Member Functions
void	Init ()
void	Clear ()
template<class ReadBuffType >
bool	ReadBinary (ReadBuffType *input_buff)
bool	SetConnection (int from, int to)
bool	CreateFastNet ()
float *	AllocWgt (int wgt_cnt)
template<typename Type >
bool	FastFeedForward (const Type inputs, Type outputs)
template<typename Type >
bool	FastGetNetOutput (const Type inputs, int output_id, Type output)
Protected Attributes
bool	read_only_
int	in_cnt_
int	out_cnt_
int	neuron_cnt_
int	wts_cnt_
Neuron *	neurons_
int	alloc_wgt_cnt_
vector< vector< float > * >	wts_vec_
bool	auto_encoder_
vector< float >	inputs_max_
vector< float >	inputs_min_
vector< float >	inputs_mean_
vector< float >	inputs_std_dev_
vector< Node >	fast_nodes_
Static Protected Attributes
static const int	kWgtChunkSize = 0x10000
static const unsigned int	kNetSignature = 0xFEFEABD0

Detailed Description

Definition at line 22 of file neural_net.h.

Constructor & Destructor Documentation

tesseract::NeuralNet::NeuralNet ( )

Definition at line 15 of file neural_net.cpp.

                     {
  Init();
}

tesseract::NeuralNet::~NeuralNet ( ) [virtual]

Definition at line 19 of file neural_net.cpp.

                      {
  // clean up the wts chunks vector
  for(int vec = 0; vec < wts_vec_.size(); vec++) {
    delete wts_vec_[vec];
  }
  // clean up neurons
  delete []neurons_;
  // clean up nodes
  for (int node_idx = 0; node_idx < neuron_cnt_; node_idx++) {
    delete []fast_nodes_[node_idx].inputs;
  }

}

Member Function Documentation

float * tesseract::NeuralNet::AllocWgt ( int wgt_cnt ) [protected]

Definition at line 189 of file neural_net.cpp.

                                       {
  // see if need to allocate a new chunk of wts
  if (wts_vec_.size() == 0 || (alloc_wgt_cnt_ + wgt_cnt) > kWgtChunkSize) {
    // add the new chunck to the wts_chunks vector
    wts_vec_.push_back(new vector<float> (kWgtChunkSize));
    alloc_wgt_cnt_ = 0;
  }
  float *ret_ptr = &((*wts_vec_.back())[alloc_wgt_cnt_]);
  // incr usage counts
  alloc_wgt_cnt_ += wgt_cnt;
  wts_cnt_ += wgt_cnt;
  return ret_ptr;
}

void tesseract::NeuralNet::Clear ( ) [inline, protected]

Definition at line 100 of file neural_net.h.

                 {
      for (int node = 0; node < neuron_cnt_; node++) {
        neurons_[node].Clear();
      }
    }

bool tesseract::NeuralNet::CreateFastNet ( ) [protected]

Definition at line 124 of file neural_net.cpp.

                              {
  fast_nodes_.resize(neuron_cnt_);
  // build the node structures
  int wts_cnt = 0;
  for (int node_idx = 0; node_idx < neuron_cnt_; node_idx++) {
    Node *node = &fast_nodes_[node_idx];
    if (neurons_[node_idx].node_type() == Neuron::Input) {
      // Input neurons have no fan-in
      node->fan_in_cnt = 0;
      node->inputs = NULL;
      // Input bias is the normalization offset computed from
      // training input stats
      if (fabs(inputs_max_[node_idx] - inputs_min_[node_idx]) <
          kMinInputRange) {
        // if the range approaches zero, the stdev is not defined,
        // this indicates that this input does not change.
        // Set the bias to zero
        node->bias = 0.0f;
      } else {
        node->bias = inputs_min_[node_idx] + (inputs_mean_[node_idx] *
            (inputs_max_[node_idx] - inputs_min_[node_idx]));
      }
    } else {
      node->bias = neurons_[node_idx].bias();
      node->fan_in_cnt = neurons_[node_idx].fan_in_cnt();
      // allocate memory for fan-in nodes
      node->inputs = new WeightedNode[node->fan_in_cnt];
      if (node->inputs == NULL) {
        return false;
      }
      for (int fan_in = 0; fan_in < node->fan_in_cnt; fan_in++) {
        // identify fan-in neuron
        const int id = neurons_[node_idx].fan_in(fan_in)->id();
        // Feedback connections are not allowed and should never happen
        if (id >= node_idx) {
          return false;
        }
        // add the the fan-in neuron and its wgt
        node->inputs[fan_in].input_node = &fast_nodes_[id];
        float wgt_val = neurons_[node_idx].fan_in_wts(fan_in);
        // for input neurons normalize the wgt by the input scaling
        // values to save time during feedforward
        if (neurons_[node_idx].fan_in(fan_in)->node_type() == Neuron::Input) {
          // if the range approaches zero, the stdev is not defined,
          // this indicates that this input does not change.
          // Set the weight to zero
          if (fabs(inputs_max_[id] - inputs_min_[id]) < kMinInputRange) {
            wgt_val = 0.0f;
          } else {
            wgt_val /= ((inputs_max_[id] - inputs_min_[id]) *
                inputs_std_dev_[id]);
          }
        }
        node->inputs[fan_in].input_weight = wgt_val;
      }
      // incr wgt count to validate against at the end
      wts_cnt += node->fan_in_cnt;
    }
  }
  // sanity check
  return wts_cnt_ == wts_cnt;
}

template<typename Type >

template bool tesseract::NeuralNet::FastFeedForward	(	const Type *	inputs,
		Type *	outputs
	)		`[protected]`

Definition at line 52 of file neural_net.cpp.

                                                                        {
  int node_idx = 0;
  Node *node = &fast_nodes_[0];
  // feed inputs in and offset them by the pre-computed bias
  for (node_idx = 0; node_idx < in_cnt_; node_idx++, node++) {
    node->out = inputs[node_idx] - node->bias;
  }
  // compute nodes activations and outputs
  for (;node_idx < neuron_cnt_; node_idx++, node++) {
    double activation = -node->bias;
    for (int fan_in_idx = 0; fan_in_idx < node->fan_in_cnt; fan_in_idx++) {
      activation += (node->inputs[fan_in_idx].input_weight *
                     node->inputs[fan_in_idx].input_node->out);
    }
    node->out = Neuron::Sigmoid(activation);
  }
  // copy the outputs to the output buffers
  node = &fast_nodes_[neuron_cnt_ - out_cnt_];
  for (node_idx = 0; node_idx < out_cnt_; node_idx++, node++) {
    outputs[node_idx] = node->out;
  }
  return true;
}

template<typename Type >

template bool tesseract::NeuralNet::FastGetNetOutput	(	const Type *	inputs,
		int	output_id,
		Type *	output
	)		`[protected]`

Definition at line 231 of file neural_net.cpp.

                                                                        {
  // feed inputs in and offset them by the pre-computed bias
  int node_idx = 0;
  Node *node = &fast_nodes_[0];
  for (node_idx = 0; node_idx < in_cnt_; node_idx++, node++) {
    node->out = inputs[node_idx] - node->bias;
  }

  // compute nodes' activations and outputs for hidden nodes if any
  int hidden_node_cnt = neuron_cnt_ - out_cnt_;
  for (;node_idx < hidden_node_cnt; node_idx++, node++) {
    double activation = -node->bias;
    for (int fan_in_idx = 0; fan_in_idx < node->fan_in_cnt; fan_in_idx++) {
      activation += (node->inputs[fan_in_idx].input_weight *
                     node->inputs[fan_in_idx].input_node->out);
    }
    node->out = Neuron::Sigmoid(activation);
  }

  // compute the output of the required output node
  node += output_id;
  double activation = -node->bias;
  for (int fan_in_idx = 0; fan_in_idx < node->fan_in_cnt; fan_in_idx++) {
    activation += (node->inputs[fan_in_idx].input_weight *
                   node->inputs[fan_in_idx].input_node->out);
  }
  (*output) = Neuron::Sigmoid(activation);
  return true;
}

template<typename Type >

bool tesseract::NeuralNet::FeedForward	(	const Type *	inputs,
		Type *	outputs
	)

Definition at line 79 of file neural_net.cpp.

                                                                    {
  // call the fast version in case of readonly nets
  if (read_only_) {
    return FastFeedForward(inputs, outputs);
  }
  // clear all neurons
  Clear();
  // for auto encoders, apply no input normalization
  if (auto_encoder_) {
    for (int in = 0; in < in_cnt_; in++) {
      neurons_[in].set_output(inputs[in]);
    }
  } else {
    // Input normalization : subtract mean and divide by stddev
    for (int in = 0; in < in_cnt_; in++) {
      neurons_[in].set_output((inputs[in] - inputs_min_[in]) /
                              (inputs_max_[in] - inputs_min_[in]));
      neurons_[in].set_output((neurons_[in].output() - inputs_mean_[in]) /
                              inputs_std_dev_[in]);
    }
  }
  // compute the net outputs: follow a pull model each output pulls the
  // outputs of its input nodes and so on
  for (int out = neuron_cnt_ - out_cnt_; out < neuron_cnt_; out++) {
    neurons_[out].FeedForward();
    // copy the values to the output buffer
    outputs[out] = neurons_[out].output();
  }
  return true;
}

NeuralNet * tesseract::NeuralNet::FromFile ( const string file_name ) [static]

Definition at line 204 of file neural_net.cpp.

                                                     {
  // open the file
  InputFileBuffer   input_buff(file_name);
  // create a new net object using input buffer
  NeuralNet *net_obj = FromInputBuffer(&input_buff);
  return net_obj;
}

NeuralNet * tesseract::NeuralNet::FromInputBuffer ( InputFileBuffer * ib ) [static]

Definition at line 213 of file neural_net.cpp.

                                                         {
      // create a new net object
  NeuralNet *net_obj = new NeuralNet();
  if (net_obj == NULL) {
    return NULL;
  }
      // load the net
  if (!net_obj->ReadBinary(ib)) {
    delete net_obj;
    net_obj = NULL;
  }
  return net_obj;
}

template<typename Type >

template bool tesseract::NeuralNet::GetNetOutput	(	const Type *	inputs,
		int	output_id,
		Type *	output
	)

Definition at line 265 of file neural_net.cpp.

                                                                    {
  // validate output id
  if (output_id < 0 || output_id >= out_cnt_) {
    return false;
  }

  // call the fast version in case of readonly nets
  if (read_only_) {
    return FastGetNetOutput(inputs, output_id, output);
  }

  // For the slow version, we'll just call FeedForward and return the
  // appropriate output
  vector<Type> outputs(out_cnt_);
  if (!FeedForward(inputs, &outputs[0])) {
    return false;
  }
  (*output) = outputs[output_id];

  return true;
}

int tesseract::NeuralNet::in_cnt ( ) const [inline]

Definition at line 40 of file neural_net.h.

{ return in_cnt_; }

void tesseract::NeuralNet::Init ( ) [protected]

Definition at line 34 of file neural_net.cpp.

                     {
  read_only_ = true;
  auto_encoder_ = false;
  alloc_wgt_cnt_ = 0;
  wts_cnt_ = 0;
  neuron_cnt_ = 0;
  in_cnt_ = 0;
  out_cnt_ = 0;
  wts_vec_.clear();
  neurons_ = NULL;
  inputs_mean_.clear();
  inputs_std_dev_.clear();
  inputs_min_.clear();
  inputs_max_.clear();
}

int tesseract::NeuralNet::out_cnt ( ) const [inline]

Definition at line 41 of file neural_net.h.

{ return out_cnt_; }

template<class ReadBuffType >

bool tesseract::NeuralNet::ReadBinary ( ReadBuffType * input_buff ) [inline, protected]

Definition at line 106 of file neural_net.h.

                                                                           {
      // Init vars
      Init();
      // is this an autoencoder
      unsigned int read_val;
      unsigned int auto_encode;
      // read and verify signature
      if (input_buff->Read(&read_val, sizeof(read_val)) != sizeof(read_val)) {
        return false;
      }
      if (read_val != kNetSignature) {
        return false;
      }
      if (input_buff->Read(&auto_encode, sizeof(auto_encode)) !=
          sizeof(auto_encode)) {
        return false;
      }
      auto_encoder_ = auto_encode;
      // read and validate total # of nodes
      if (input_buff->Read(&read_val, sizeof(read_val)) != sizeof(read_val)) {
        return false;
      }
      neuron_cnt_ = read_val;
      if (neuron_cnt_ <= 0) {
        return false;
      }
      // set the size of the neurons vector
      neurons_ = new Neuron[neuron_cnt_];
      if (neurons_ == NULL) {
        return false;
      }
      // read & validate inputs
      if (input_buff->Read(&read_val, sizeof(read_val)) != sizeof(read_val)) {
        return false;
      }
      in_cnt_ = read_val;
      if (in_cnt_ <= 0) {
        return false;
      }
      // read outputs
      if (input_buff->Read(&read_val, sizeof(read_val)) != sizeof(read_val)) {
        return false;
      }
      out_cnt_ = read_val;
      if (out_cnt_ <= 0) {
        return false;
      }
      // set neuron ids and types
      for (int idx = 0; idx < neuron_cnt_; idx++) {
        neurons_[idx].set_id(idx);
        // input type
        if (idx < in_cnt_) {
          neurons_[idx].set_node_type(Neuron::Input);
        } else if (idx >= (neuron_cnt_ - out_cnt_)) {
          neurons_[idx].set_node_type(Neuron::Output);
        } else {
          neurons_[idx].set_node_type(Neuron::Hidden);
        }
      }
      // read the connections
      for (int node_idx = 0; node_idx < neuron_cnt_; node_idx++) {
        // read fanout
        if (input_buff->Read(&read_val, sizeof(read_val)) != sizeof(read_val)) {
          return false;
        }
        // read the neuron's info
        int fan_out_cnt = read_val;
        for (int fan_out_idx = 0; fan_out_idx < fan_out_cnt; fan_out_idx++) {
          // read the neuron id
          if (input_buff->Read(&read_val, sizeof(read_val)) != sizeof(read_val)) {
            return false;
          }
          // create the connection
          if (!SetConnection(node_idx, read_val)) {
            return false;
          }
        }
      }
      // read all the neurons' fan-in connections
      for (int node_idx = 0; node_idx < neuron_cnt_; node_idx++) {
        // read
        if (!neurons_[node_idx].ReadBinary(input_buff)) {
          return false;
        }
      }
      // size input stats vector to expected input size
      inputs_mean_.resize(in_cnt_);
      inputs_std_dev_.resize(in_cnt_);
      inputs_min_.resize(in_cnt_);
      inputs_max_.resize(in_cnt_);
      // read stats
      if (input_buff->Read(&(inputs_mean_.front()),
          sizeof(inputs_mean_[0]) * in_cnt_) !=
          sizeof(inputs_mean_[0]) * in_cnt_) {
        return false;
      }
      if (input_buff->Read(&(inputs_std_dev_.front()),
          sizeof(inputs_std_dev_[0]) * in_cnt_) !=
          sizeof(inputs_std_dev_[0]) * in_cnt_) {
        return false;
      }
      if (input_buff->Read(&(inputs_min_.front()),
          sizeof(inputs_min_[0]) * in_cnt_) !=
          sizeof(inputs_min_[0]) * in_cnt_) {
        return false;
      }
      if (input_buff->Read(&(inputs_max_.front()),
          sizeof(inputs_max_[0]) * in_cnt_) !=
          sizeof(inputs_max_[0]) * in_cnt_) {
        return false;
      }
      // create a readonly version for fast feedforward
      if (read_only_) {
        return CreateFastNet();
      }
      return true;
    }

bool tesseract::NeuralNet::SetConnection	(	int	from,
		int	to
	)		`[protected]`

Definition at line 112 of file neural_net.cpp.

                                              {
  // allocate the wgt
  float *wts  =  AllocWgt(1);
  if (wts == NULL) {
    return false;
  }
  // register the connection
  neurons_[to].AddFromConnection(neurons_ + from, wts, 1);
  return true;
}