@ Override public void applyUpdater ( INDArray gradient , int iteration , int epoch ) { if ( msg == null || msdx == null ) throw new IllegalStateException ( "Updater has not been initialized with view state" ) ; double rho = config . getRho ( ) ; double epsilon = config . getEpsilon ( ) ; //Line 4 of Algorithm 1: https://arxiv.org/pdf/1212.5701v1.pdf //E[g^2]_t = rho * E[g^2]_{t−1} + (1-rho)*g^2_t msg . muli ( rho ) . addi ( gradient . mul ( gradient ) . muli ( 1 - rho ) ) ; //Calculate update: //dX = - g * RMS[delta x]_{t-1} / RMS[g]_t //Note: negative is applied in the DL4J step function: params -= update rather than params += update INDArray rmsdx_t1 = Transforms . sqrt ( msdx . add ( epsilon ) , false ) ; INDArray rmsg_t = Transforms . sqrt ( msg . add ( epsilon ) , false ) ; INDArray update = gradient . muli ( rmsdx_t1 . divi ( rmsg_t ) ) ; //Accumulate gradients: E[delta x^2]_t = rho * E[delta x^2]_{t-1} + (1-rho)* (delta x_t)^2 msdx . muli ( rho ) . addi ( update . mul ( update ) . muli ( 1 - rho ) ) ; }
Get the updated gradient for the given gradient and also update the state of ada delta .