# Shepard et al. (1961) six types of categorization problem
# Set up inputs
inputs<-matrix(c(0,0,0,
                 0,0,1,
                 0,1,0,
                 0,1,1,
                 1,0,0,
                 1,0,1,
                 1,1,0,
                 1,1,1),3,8,byrow=F)
#inputs<-rbind(inputs,-1)
# Set up targets
target1<-matrix(c(0,0,0,0,1,1,1,1),1,8)
target2<-matrix(c(0,0,1,1,1,1,0,0),1,8)
target3<-matrix(c(0,0,0,1,1,0,1,1),1,8)
target4<-matrix(c(0,0,0,1,0,1,1,1),1,8)
target5<-matrix(c(0,0,0,1,1,1,1,0),1,8)
target6<-matrix(c(0,1,1,0,1,0,0,1),1,8)

ReLU<-function(inputs,targets,nH=4,eta=0.1,nit=50){
  # Initialize variables
  nI<-nrow(inputs)
  nO<-nrow(targets)
  set.seed(123)
  # Associative weights between input and hidden layers
  W<-matrix(runif(nH*nI,-0.1,0.1),nH,nI)
  # Associative weights between hidden and output layers
  #Z<-matrix(runif(nO*nH,-0.5,0.5),nO,nH)
  Z<-matrix(runif(nO*nH,-0.1,0.1),nO,nH)
  # Set up error 
  E<-rep(-1,nit)
  
  # Start training
  for(i in 1:nit){
    # Feed forward
    # Generate activation of hidden nodes
    h<-W%*%inputs
    # ReLU transformation
    H<-ifelse(h<0,0,h)
    # Derivative of hidden node's activation
    dH<-ifelse(H>0,1,0)
    # Generate activation of output nodes
    O<-Z%*%H
    #o<-Z%*%H
    #O<-ifelse(o<0,0,o)
    #dO<-ifelse(O>0,1,0)
    # Get errors
    e<-targets-O
    E[i]<-0.5*sum(e^2)
    # Gradient descent by backpropagation
    dZ<--eta*e%*%t(H)
    #dZ<--eta*(e*dO)%*%t(H)
    es<-t(e)%*%Z
    dW<--eta*(t(es)*dH)%*%t(inputs)
    # Update associative weights
    Z<-Z-dZ
    W<-W-dW
  }
  return(list(E=E,O=O,Target=targets,W=W,Z=Z))
}

# Train this model to learn the six types of categorization problem
R<-sapply(1:6,function(i){
  targets<-switch(as.character(i),
         '1'=target1,
         '2'=target2,
         '3'=target3,
         '4'=target4,
         '5'=target5,
         '6'=target6)
  results<-ReLU(inputs,targets,4,0.1,200)
  return(results$E)
})
Res<-sapply(1:6,function(i){
  targets<-switch(as.character(i),
                  '1'=target1,
                  '2'=target2,
                  '3'=target3,
                  '4'=target4,
                  '5'=target5,
                  '6'=target6)
  results<-ReLU(inputs,targets,4,0.1,1000)
  return(results$E)
})
library(ggplot2)
library(dplyr)
tibble(Iteration=rep(1:1000,6),problem=as.character(rep(1:6,each=1000)),
       E=c(Res)) %>%
  ggplot(aes(Iteration,E,color=problem))+theme_bw()+
  theme(panel.grid.major=element_blank(),
        panel.grid.minor=element_blank())+
  geom_line()

results<-ReLU(inputs,target6,4,0.1,1000)
results$O
results$Target