# Three-layered NN with ReLU

# Set up inputs
input<-matrix(c(1,1,
                1,0,
                0,1,
                0,0),4,2,byrow=T)
# Set up targets
target<-matrix(c(0,1,1,0),1,4)

# Set up associative weights between input and hidden layers
set.seed(123)
W<-matrix(runif(6,-0.5,0.5),3,2)
# Set up associative weights between hidden and output layers
Z<-matrix(runif(3,-0.5,0.5),1,3)

# Set up number of iteration
nit<-100

# Set up learning rate
eta<-0.1

# Set up error vector
E<-rep(-1,nit)

# Start training
for(i in 1:nit){
  # Feed forward
  h<-W%*%t(input)
  # ReLU transformation
  H<-pmax(h,0)
  # Get the derivative of ReLU function
  dH<-ifelse(h>0,1,0)
  # Generate output
  o<-Z%*%H
  # Compute error
  e<-target-o
  E[i]<-0.5*sum(e^2)
  # Gradient of associative weights of Z
  dZ<--eta*e%*%t(H)
  # Compute error signal for nodes at hidden layer
  es<-t(e)%*%Z
  # Gradient of associative weights of W
  dW<--eta*(t(es)*dH)%*%input
  # Update associative weights
  W<-W-dW
  Z<-Z-dZ
}
plot(1:nit,E,type='l')

cat('Final output: ',round(o,2))
cat('Target: ',target)