<commit_before>######################################################
# Developed by <NAME> Nov 19th 2012 at MIT #
######################################################
# Least-Squares Policy Iteration [Lagoudakis and Parr 2003]
# This version recalculates the policy every <sample_window>. Samples are obtained using the recent version of the policy  
from Agent import *
class LSPI(Agent):
    lspi_iterations = 0         # Number of LSPI iterations
    sample_window   = 0         # Number of samples to be used to calculate the LSTD solution
    samples_count   = 0         # Counter for the sample count
    epsilon         = 0         # Minimum l_2 change required to continue iterations in LSPI
    #Store Data in separate matrixes
    data_s          = []        # 
    data_a          = []        # 
    data_r          = []        #
    data_ns         = []        # 
    data_na         = []        # 
        
    def __init__(self,representation,policy,domain,logger, lspi_iterations = 5, sample_window = 100, epsilon = 1e-3):
        self.samples_count      = 0
        self.sample_window      = sample_window
        self.epsilon            = epsilon
        self.lspi_iterations    = lspi_iterations
        self.phi_sa_size        = domain.actions_num * representation.features_num
        self.data_s             = zeros((sample_window, domain.state_space_dims))
        self.data_ns            = zeros((sample_window, domain.state_space_dims))
        self.data_a             = zeros(sample_window,dtype=uint16)
        self.data_na            = zeros(sample_window,dtype=uint16)
        self.data_r             = zeros(sample_window)
        super(LSPI, self).__init__(representation, policy, domain,logger)
        if logger:
                logger.log('Max LSPI Iterations:\t%d' % lspi_iterations)
                logger.log('Data Size:\t\t%d' % sample_window)
                logger.log('Weight Difference tol.:\t%0.3f' % epsilon)
    def learn(self,s,a,r,ns,na,terminal):
        
        self.storeData(s,a,r,ns,na)        
        
        if self.samples_count == self.sample_window: #zero based hence the -1

            # Run LSTD for first solution
            A,b, all_phi_s, all_phi_s_a, all_phi_ns = self.LSTD()
            # Run Policy Iteration to change a_prime and recalculate theta
            self.policyIteration(b,all_phi_s_a, all_phi_ns)
    def policyIteration(self,b,all_phi_s_a,all_phi_ns):
            # Update the policy by recalculating A based on new na
            # Returns the TD error for each sample based on the latest weights and next actions
            # b is passed as an input because it remains unchanged during policy iteration.
            phi_sa_size     = self.domain.actions_num*self.representation.features_num
            gamma           = self.domain.gamma
            td_errors       = empty((self.sample_window)) # holds the TD_errors for all samples
            
            #Begin updating the policy in LSPI loop
            weight_diff     = self.epsilon + 1 # So that the loop starts
            lspi_iteration  = 0
            self.logger.log('Running LSPI:')
            while lspi_iteration < self.lspi_iterations and weight_diff > self.epsilon:
                if phi_sa_size != 0: A = sp.csr_matrix((phi_sa_size,phi_sa_size)) # Reset the A matrix
                for i in arange(self.sample_window):
                    ns              = self.data_ns[i,:]
                    if phi_sa_size != 0:
                        phi_s_a         = sp.csr_matrix(all_phi_s_a[i,:],dtype=all_phi_s_a.dtype)
                        phi_ns          = all_phi_ns[i,:]
                        new_na          = self.representation.bestAction(ns,phi_ns)
                        phi_ns_new_na   = sp.csr_matrix(self.representation.phi_sa(ns,new_na,phi_ns),dtype=all_phi_s_a.dtype)
                        d               = phi_s_a-gamma*phi_ns_new_na
                        A               = A + phi_s_a.T*d
                        td_errors[i]    = self.data_r[i]+sp_dot_array(-d,self.representation.theta)
                    else:
                        td_errors[i]    = self.data_r[i]
                #Calculate theta
                if phi_sa_size != 0:
                    new_theta                   = solveLinear(A,b)
                    weight_diff                 = linalg.norm(self.representation.theta - new_theta)
                    if weight_diff > self.epsilon: self.representation.theta   = new_theta
                    self.logger.log("%d: L2_norm of weight difference = %0.3f, Density of A: %0.2f%%" % (lspi_iteration+1,weight_diff, count_nonzero(A)/(prod(A.shape)*1.)*100))
                else:
                    print "No features, hence no more iterations is necessary!"
                    weight_diff = 0
                lspi_iteration +=1
            return td_errors
    def LSTD(self): 
        if self.sample_window == 0:
            print 'Window Size for LSPI should not be 0!'
            return 
            
        #No features means empty matrices
        if self.representation.features_num == 0:
            return array([]), array([]), array([]), array([]), array([])
         
        self.samples_count  = 0
       
        # Make one sample phi to decide about the type of A matrix:
        phi_s               = self.representation.phi(self.data_s[0])

        # Calculate the A and b matrixes in LSTD
        phi_sa_size     = self.domain.actions_num*self.representation.features_num
        A               = sp.csr_matrix((phi_sa_size,phi_sa_size)) # A matrix is in general float
        b               = zeros(phi_sa_size)
        all_phi_s       = zeros((self.sample_window,self.representation.features_num),dtype=phi_s.dtype) #phi_s will be saved for batch iFDD
        all_phi_s_a     = zeros((self.sample_window,phi_sa_size),dtype=phi_s.dtype) #phi_sa will be fixed during iterations
        all_phi_ns      = zeros((self.sample_window,self.representation.features_num),dtype=phi_s.dtype) #phi_ns_na will change according to na so we only cache the phi_na which remains the same
        
        #print "Making A,b"
        gamma               = self.representation.domain.gamma
        for i in arange(self.sample_window):
            s                   = self.data_s[i]
            ns                  = self.data_ns[i]
            a                   = self.data_a[i]
            na                  = self.data_na[i]
            r                   = self.data_r[i]
            phi_s               = self.representation.phi(s)
            phi_s_a             = self.representation.phi_sa(s,a,phi_s)
            phi_ns              = self.representation.phi(ns)
            phi_ns_na           = self.representation.phi_sa(ns,na,phi_ns)
            all_phi_s[i,:]      = phi_s
            all_phi_s_a[i,:]    = phi_s_a
            all_phi_ns[i,:]     = phi_ns
            b                   = b + r*phi_s_a
            phi_s_a             = sp.csr_matrix(phi_s_a,dtype=phi_s_a.dtype)
            phi_ns_na           = sp.csr_matrix(phi_s_a,dtype=phi_ns_na.dtype)
            d                   = phi_s_a-gamma*phi_ns_na<commit_msg>fixed bug in LSPI which didn't update phi_ns_na<commit_after>######################################################
# Developed by <NAME> Nov 19th 2012 at MIT #
######################################################
# Least-Squares Policy Iteration [Lagoudakis and Parr 2003]
# This version recalculates the policy every <sample_window>. Samples are obtained using the recent version of the policy  
from Agent import *
class LSPI(Agent):
    lspi_iterations = 0         # Number of LSPI iterations
    sample_window   = 0         # Number of samples to be used to calculate the LSTD solution
    samples_count   = 0         # Counter for the sample count
    epsilon         = 0         # Minimum l_2 change required to continue iterations in LSPI
    #Store Data in separate matrixes
    data_s          = []        # 
    data_a          = []        # 
    data_r          = []        #
    data_ns         = []        # 
    data_na         = []        # 
        
    def __init__(self,representation,policy,domain,logger, lspi_iterations = 5, sample_window = 100, epsilon = 1e-3):
        self.samples_count      = 0
        self.sample_window      = sample_window
        self.epsilon            = epsilon
        self.lspi_iterations    = lspi_iterations
        self.phi_sa_size        = domain.actions_num * representation.features_num
        self.data_s             = zeros((sample_window, domain.state_space_dims))
        self.data_ns            = zeros((sample_window, domain.state_space_dims))
        self.data_a             = zeros(sample_window,dtype=uint16)
        self.data_na            = zeros(sample_window,dtype=uint16)
        self.data_r             = zeros(sample_window)
        super(LSPI, self).__init__(representation, policy, domain,logger)
        if logger:
                logger.log('Max LSPI Iterations:\t%d' % lspi_iterations)
                logger.log('Data Size:\t\t%d' % sample_window)
                logger.log('Weight Difference tol.:\t%0.3f' % epsilon)
    def learn(self,s,a,r,ns,na,terminal):
        
        self.storeData(s,a,r,ns,na)        
        
        if self.samples_count == self.sample_window: #zero based hence the -1

            # Run LSTD for first solution
            A,b, all_phi_s, all_phi_s_a, all_phi_ns = self.LSTD()
            # Run Policy Iteration to change a_prime and recalculate theta
            self.policyIteration(b,all_phi_s_a, all_phi_ns)
    def policyIteration(self,b,all_phi_s_a,all_phi_ns):
            # Update the policy by recalculating A based on new na
            # Returns the TD error for each sample based on the latest weights and next actions
            # b is passed as an input because it remains unchanged during policy iteration.
            phi_sa_size     = self.domain.actions_num*self.representation.features_num
            gamma           = self.domain.gamma
            td_errors       = empty((self.sample_window)) # holds the TD_errors for all samples
            
            #Begin updating the policy in LSPI loop
            weight_diff     = self.epsilon + 1 # So that the loop starts
            lspi_iteration  = 0
            self.logger.log('Running LSPI:')
            while lspi_iteration < self.lspi_iterations and weight_diff > self.epsilon:
                if phi_sa_size != 0: A = sp.csr_matrix((phi_sa_size,phi_sa_size)) # Reset the A matrix
                for i in arange(self.sample_window):
                    ns              = self.data_ns[i,:]
                    if phi_sa_size != 0:
                        phi_s_a         = sp.csr_matrix(all_phi_s_a[i,:],dtype=all_phi_s_a.dtype)
                        phi_ns          = all_phi_ns[i,:]
                        new_na          = self.representation.bestAction(ns,phi_ns)
                        phi_ns_new_na   = sp.csr_matrix(self.representation.phi_sa(ns,new_na,phi_ns),dtype=all_phi_s_a.dtype)
                        d               = phi_s_a-gamma*phi_ns_new_na
                        A               = A + phi_s_a.T*d
                        td_errors[i]    = self.data_r[i]+sp_dot_array(-d,self.representation.theta)
                    else:
                        td_errors[i]    = self.data_r[i]
                #Calculate theta
                if phi_sa_size != 0:
                    new_theta                   = solveLinear(A,b)
                    weight_diff                 = linalg.norm(self.representation.theta - new_theta)
                    if weight_diff > self.epsilon: self.representation.theta   = new_theta
                    self.logger.log("%d: L2_norm of weight difference = %0.3f, Density of A: %0.2f%%" % (lspi_iteration+1,weight_diff, count_nonzero(A)/(prod(A.shape)*1.)*100))
                else:
                    print "No features, hence no more iterations is necessary!"
                    weight_diff = 0
                lspi_iteration +=1
            return td_errors
    def LSTD(self): 
        if self.sample_window == 0:
            print 'Window Size for LSPI should not be 0!'
            return 
            
        #No features means empty matrices
        if self.representation.features_num == 0:
            return array([]), array([]), array([]), array([]), array([])
         
        self.samples_count  = 0
       
        # Make one sample phi to decide about the type of A matrix:
        phi_s               = self.representation.phi(self.data_s[0])

        # Calculate the A and b matrixes in LSTD
        phi_sa_size     = self.domain.actions_num*self.representation.features_num
        A               = sp.csr_matrix((phi_sa_size,phi_sa_size)) # A matrix is in general float
        b               = zeros(phi_sa_size)
        all_phi_s       = zeros((self.sample_window,self.representation.features_num),dtype=phi_s.dtype) #phi_s will be saved for batch iFDD
        all_phi_s_a     = zeros((self.sample_window,phi_sa_size),dtype=phi_s.dtype) #phi_sa will be fixed during iterations
        all_phi_ns      = zeros((self.sample_window,self.representation.features_num),dtype=phi_s.dtype) #phi_ns_na will change according to na so we only cache the phi_na which remains the same
        
        #print "Making A,b"
        gamma               = self.representation.domain.gamma
        for i in arange(self.sample_window):
            s                   = self.data_s[i]
            ns                  = self.data_ns[i]
            a                   = self.data_a[i]
            na                  = self.data_na[i]
            r                   = self.data_r[i]
            phi_s               = self.representation.phi(s)
            phi_s_a             = self.representation.phi_sa(s,a,phi_s)
            phi_ns              = self.representation.phi(ns)
            phi_ns_na           = self.representation.phi_sa(ns,na,phi_ns)
            all_phi_s[i,:]      = phi_s
            all_phi_s_a[i,:]    = phi_s_a
            all_phi_ns[i,:]     = phi_ns
            b                   = b + r*phi_s_a
            phi_s_a             = sp.csr_matrix(phi_s_a,dtype=phi_s_a.dtype)
            phi_ns_na           = sp.csr_matrix(phi_ns_na,dtype=phi_ns_na.dtype)
            d                   = phi_s_a-gamma*phi_ns_na
{'fraction_non_alphanumeric': 0.09537411174585482, 'fraction_numerical': 0.007106033161488087, 'mean_word_length': 1.6932457786116322, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4841856', 'n_tokens_mistral': 4406, 'n_tokens_neox': 4132, 'n_words': 932}