content: module AI.Neuron
         ( Neuron(..)
  
         , ActivationFunction
         , ActivationFunction'
         , sigmoidNeuron
         , tanhNeuron
         , recluNeuron
         , l2Neuron
  
         , sigmoid
         , sigmoid'
         , tanh
         , tanh'
         , reclu
         , reclu'
         ) where
  
  -- | Using this structure allows users of the library to create their own
  --   neurons by creating two functions - an activation function and its
  --   derivative - and packaging them up into a neuron type.
  data Neuron = Neuron { activation  :: ActivationFunction
                       , activation' :: ActivationFunction'
                       , description :: String
                       }
  
  instance Show Neuron where
    show = description
  
  type ActivationFunction = Double -> Double
  type ActivationFunction' = Double -> Double
  
  -- | Our provided neuron types: sigmoid, tanh, reclu
  sigmoidNeuron :: Neuron
  sigmoidNeuron = Neuron sigmoid sigmoid' "sigmoid"
  
  tanhNeuron :: Neuron
  tanhNeuron = Neuron tanh tanh' "tanh"
  
  recluNeuron :: Neuron
  recluNeuron = Neuron reclu reclu' "reclu"
  
  l2Neuron :: Neuron
  l2Neuron = Neuron (^2) id "L2"
  
  -- | The sigmoid activation function, a standard activation function defined
  --   on the range (0, 1).
  sigmoid :: Double -> Double
  sigmoid t = 1 / (1 + exp (-1 * t))
  
  -- | The derivative of the sigmoid function conveniently can be computed in
  --   terms of the sigmoid function.
  sigmoid' :: Double -> Double
  sigmoid' t = s * (1 - s)
    where s = sigmoid t
  
  -- | The hyperbolic tangent activation function is provided in Prelude. Here
  --   we provide the derivative. As with the sigmoid function, the derivative
  --   of tanh can be computed in terms of tanh.
  tanh' :: Double -> Double
  tanh' t = 1 - s ^ 2
    where s = tanh t
  
  -- | The rectified linear activation function. This is a more "biologically
  --   accurate" activation function that still retains differentiability.
  reclu :: Double -> Double
  reclu t = log (1 + exp t)
  
  -- | The derivative of the rectified linear activation function is just the
  --   sigmoid.
  reclu' :: Double -> Double
  reclu' = sigmoid
  