<commit_before>#See http://acl.mit.edu/RLPy for documentation and future code updates

#Copyright (c) 2013, <NAME>, <NAME>, and <NAME>. How
#All rights reserved.

#Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:

#Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.

#Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.

#Neither the name of ACL nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.

#THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

######################################################
# Developed by <NAME> Oct 30th 2012 at MIT #
######################################################
# This object encodes fixed policies for some of the domains.
# 1. InvertedPendulum
from Policy import *
class FixedPolicy(Policy):
    supportedDomains = ['Pendulum_InvertedBalance','BlocksWorld','IntruderMonitoring','SystemAdministrator','MountainCar','PST']
    def pi(self,s):
        if not className(self.representation.domain) in self.supportedDomains:
            print "ERROR: There is no fixed policy defined for %s" % className(self.representation.domain)
            return None
        if className(self.representation.domain) == 'Pendulum_InvertedBalance':
            # Fixed policy rotate the pendulum in the opposite direction of the thetadot
            theta, thetadot = s
            if thetadot > 0:
                return 2
            else:
                return 0
        if className(self.representation.domain) == 'BlocksWorld':
            # Fixed policy rotate the blocksworld = Optimal Policy (Always pick the next piece of the tower and move it to the tower
            # Policy: Identify the top of the tower.
            # move the next piece on the tower with 95% chance 5% take a random action
            domain = self.representation.domain
            
            #Random Action with some probability
            if random.rand() < .3 or domain.isTerminal(s):
                return randSet(domain.possibleActions(s))

            #non-Random Policy
            #next_block is the block that should be stacked on the top of the tower
            #wrong_block is the highest block stacked on the top of the next_block    
            #Wrong_tower_block is the highest stacked on the top of the tower
            blocks = domain.blocks
            correct_tower_size = 0 # Length of the tower assumed to be built correctly.
            while True:
                # Check the next block
                block = correct_tower_size
                if (block == 0 and domain.on_table(block,s)) or domain.on(block,block-1,s):
                    #This block is on the right position, check the next block
                    correct_tower_size += 1
                else: 
                    #print s
                    #print "Incorrect block:", block
                    # The block is on the wrong place.
                    # 1. Check if the tower is empty => If not take one block from the tower and put it on the table
                    # 2. check to see if this wrong block is empty => If not put one block from its stack and put on the table
                    # 3. Otherwise move this block on the tower
                    
                    ###################
                    #1
                    ###################
                    if block != 0: # If the first block is in the wrong place, then the tower top which is table is empty by definition  
                        ideal_tower_top     = block - 1
                        tower_top = domain.towerTop(ideal_tower_top,s)
                        if tower_top != ideal_tower_top:
                            # There is a wrong block there hence we should put it on the table first
                            return domain.getActionPutAonTable(tower_top) #put the top of the tower on the table since it is not correct
                    ###################
                    #2
                    ###################
                    block_top = domain.towerTop(block,s)
                    if block_top != block:
                        # The target block to be stacked is not empty
                        return domain.getActionPutAonTable(block_top)
                    ###################
                    #3
                    ###################
                    if block == 0:
                        return domain.getActionPutAonTable(block)
                    else:
                        return domain.getActionPutAonB(block,block-1)
        if className(self.representation.domain) == 'IntruderMonitoring':
            # Each UAV assign themselves to a target
            # Each UAV finds the closest danger zone to its target and go towards there. 
            # If UAVs_num > Target, the rest will hold position
            #Move all agents based on the taken action
            domain  = self.representation.domain
            agents  = array(s[:domain.NUMBER_OF_AGENTS*2].reshape(-1,2))
            targets = array(s[domain.NUMBER_OF_AGENTS*2:].reshape(-1,2))
            zones   = domain.danger_zone_locations
            actions = ones(len(agents),dtype=integer)*4 # Default action is hold
            planned_agents_num = min(len(agents),len(targets))
            for i in arange(planned_agents_num): 
                #Find cloasest zone (manhattan) to the corresponding target
                target          = targets[i,:]
                distances       = sum(abs(tile(target,(len(zones),1)) - zones),axis=1)
                z_row,z_col     = zones[argmin(distances),:]
                # find the valid action
                a_row,a_col     = agents[i,:]
                a = 4 # hold as a default action
                if a_row > z_row:
                    a = 0 # up
                if a_row < z_row:
                    a = 1 # down
                if a_col > z_col:
                    a = 2 # left
                if a_col < z_col:
                    a = 3 # right
                actions[i] = a
#                print "Agent=", agents[i,:]
#                print "Target", target
#                print "Zone", zones[argmin(distances),:]
#                print "Action", a
#                print '============'
            return vec2id(actions,ones(len(agents),dtype=integer)*5)
        if className(self.representation.domain) == 'SystemAdministrator':
            # Select a broken computer and reset it
            brokenComputers = where(s==0)[0]
            if len(brokenComputers):
                return randSet(brokenComputers)
            else:
                return self.representation.domain.computers_num
        if className(self.representation.domain) == 'MountainCar':
            # Accelerate in the direction of the valley
            # WORK IN PROGRESS
            x,xdot = s
            if xdot > 0:
                return 2
            else:
                return 0
        if className(self.representation.domain) == 'PST':
            # One stays at comm, n-1 stay at target area. Whenever fuel is lower than reaching the base the move back
            domain = self.representation.domain
            print s
            s       = domain.state2Struct(s)
            uavs    = domain.NUM_UAV
            print s
            return vec2id(zeros(uavs),ones(uavs)*3)<commit_msg>added fixed policies for several domains<commit_after>#See http://acl.mit.edu/RLPy for documentation and future code updates

#Copyright (c) 2013, <NAME>, <NAME>, and <NAME>
#All rights reserved.

#Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:

#Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.

#Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.

#Neither the name of ACL nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.

#THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

######################################################
# Developed by <NAME> Oct 30th 2012 at MIT #
######################################################
# This object encodes fixed policies for some of the domains.
# 1. InvertedPendulum
from Policy import *

#from GridWorldPolicies import *

class FixedPolicy(Policy):
    
    policyName  = '' # The name of the desired policy, where applicable. Otherwise ignored.
    tableOfValues = None
    
    gridWorldPolicyNames = ['cw_circle', 'ccw_circle']
    aircraftStormPolicyNames = ['L_TO_R','random_policy']
    
    def __init__(self, representation, logger, policyName = 'MISSINGNO', tableOfValues=None):
        self.policyName = policyName
        self.tableOfValues = tableOfValues
        super(FixedPolicy, self).__init__(representation,logger)
    
    supportedDomains = ['Pendulum_InvertedBalance','BlocksWorld','IntruderMonitoring',\
                        'SystemAdministrator','MountainCar','PST','GridWorld',\
                        'AircraftStorm']
    def pi(self,s):
        if self.tableOfValues:
            return self.tableOfValues[(s)]
        return self.pi2(s)
    
    def pi2(self,s):
        domain = self.representation.domain
        if not className(domain) in self.supportedDomains:
            print "ERROR: There is no fixed policy defined for %s" % className(domain)
            return None

        if className(domain) == 'AircraftStorm':
            if not self.policyName in self.aircraftStormPolicyNames:
                print "Error: There is no AircraftStorm policy with name %s" % self.policyName
                return None
            
            if self.policyName == 'L_TO_R':
                return 3        # Right action, always
        
            if self.policyName == 'random_policy':
                return randSet(domain.possibleActions(s))
            
            
            
                
        if className(domain) == 'GridWorld':
            # Actions are Up, Down, Left, Right
            if not self.policyName in self.gridWorldPolicyNames:
                print "Error: There is no GridWorld policy with name %s" % self.policyName
                return None
            
            if self.policyName == 'cw_circle':
                # Cycle through actions, starting with 0, causing agent to go in loop
                if not hasattr(self, "curAction"):
                    self.curAction = 0  # it doesn't exist yet, so initialize it [immediately incremented]
                while (not(self.curAction in domain.possibleActions(s))):
                    # We can't do something simple because of the order in which actions are defined
                    # must do switch statement
                    if self.curAction == 0: #up
                        self.curAction = 3
                    elif self.curAction == 3: #right
                        self.curAction = 1
                    elif self.curAction == 1: #down
                        self.curAction = 2
                    elif self.curAction == 2: # left
                        self.curAction = 0
                    else: print 'Something terrible happened...got an invalid action on GridWorld Fixed Policy'
    #                 self.curAction = self.curAction % domain.actions_num
            elif self.policyName == 'ccw_circle':
                # Cycle through actions, starting with 0, causing agent to go in loop
                if not hasattr(self, "curAction"):
                    self.curAction = 1  # it doesn't exist yet, so initialize it
                while (not(self.curAction in domain.possibleActions(s))):
                    # We can't do something simple because of the order in which actions are defined
                    # must do switch statement
                    if self.curAction == 3: #right
                        self.curAction = 0
                    elif self.curAction == 0: #up
                        self.curAction = 2
                    elif self.curAction == 2: #left
                        self.curAction = 1
                    elif self.curAction == 1: # down
                        self.curAction = 3
                    else: print 'Something terrible happened...got an invalid action on GridWorld Fixed Policy'
    #                 self.curAction = self.curAction % domain.actions_num
            
            else:
                print "Error: No policy defined with name %s, but listed in gridWorldPolicyNames" % self.policyName
                print "You need to create a switch statement for the policy name above, or remove it from gridWorldPolicyNames"
                return None
            return self.curAction
            
#             # Cycle through actions, starting with 0, causing agent to go in other direction
#             if not hasattr(pi, "curAction"):
#                 pi.curAction = domain.actions_num-1  # it doesn't exist yet, so initialize it
#             if not(pi.curAction in domain.possibleActions(s)):
#                 pi.curAction -= 1
#                 if pi.curAction < 0: pi.curAction = domain.actions_num-1
                    
            
            
            
        if className(domain) == 'Pendulum_InvertedBalance':
            # Fixed policy rotate the pendulum in the opposite direction of the thetadot
            theta, thetadot = s
            if thetadot > 0:
                return 2
            else:
                return 0
        if className(domain) == 'BlocksWorld':
            # Fixed policy rotate the blocksworld = Optimal Policy (Always pick the next piece of the tower and move it to the tower
            # Policy: Identify the top of the tower.
            # move the next piece on the tower with 95% chance 5% take a random action
            
            #Random Action with some probability
            if random.rand() < .3 or domain.isTerminal(s):
                return randSet(domain.possibleActions(s))

            #non-Random Policy
            #next_block is the block that should be stacked on the top of the tower
            #wrong_block is the highest block stacked on the top of the next_block    
            #Wrong_tower_block is the highest stacked on the top of the tower
            blocks = domain.blocks
            correct_tower_size = 0 # Length of the tower assumed to be built correctly.
            while True:
                # Check the next block
                block = correct_tower_size
                if (block == 0 and domain.on_table(block,s)) or domain.on(block,block-1,s):
                    #This block is on the right position, check the next block
                    correct_tower_size += 1
                else: 
                    #print s
                    #print "Incorrect block:", block
                    # The block is on the wrong place.
                    # 1. Check if the tower is empty => If not take one block from the tower and put it on the table
                    # 2. check to see if this wrong block is empty => If not put one block from its stack and put on the table
                    # 3. Otherwise move this block on the tower
                    
                    ###################
                    #1
                    ###################
                    if block != 0: # If the first block is in the wrong place, then the tower top which is table is empty by definition  
                        ideal_tower_top     = block - 1
                        tower_top = domain.towerTop(ideal_tower_top,s)
                        if tower_top != ideal_tower_top:
                            # There is a wrong block there hence we should put it on the table first
                            return domain.getActionPutAonTable(tower_top) #put the top of the tower on the table since it is not correct
                    ###################
                    #2
                    ###################
                    block_top = domain.towerTop(block,s)
                    if block_top != block:
                        # The target block to be stacked is not empty
                        return domain.getActionPutAonTable(block_top)
                    ###################
                    #3
                    ###################
                    if block == 0:
                        return domain.getActionPutAonTable(block)
                    else:
                        return domain.getActionPutAonB(block,block-1)
        if className(domain) == 'IntruderMonitoring':
            # Each UAV assign themselves to a target
            # Each UAV finds the closest danger zone to its target and go towards there. 
            # If UAVs_num > Target, the rest will hold position
            #Move all agents based on the taken action
            agents  = array(s[:domain.NUMBER_OF_AGENTS*2].reshape(-1,2))
            targets = array(s[domain.NUMBER_OF_AGENTS*2:].reshape(-1,2))
            zones   = domain.danger_zone_locations
            actions = ones(len(agents),dtype=integer)*4 # Default action is hold
            planned_agents_num = min(len(agents),len(targets))
            for i in arange(planned_agents_num): 
                #Find cloasest zone (manhattan) to the corresponding target
                target          = targets[i,:]
                distances       = sum(abs(tile(target,(len(zones),1)) - zones),axis=1)
                z_row,z_col     = zones[argmin(distances),:]
                # find the valid action
                a_row,a_col     = agents[i,:]
                a = 4 # hold as a default action
                if a_row > z_row:
                    a = 0 # up
                if a_row < z_row:
                    a = 1 # down
                if a_col > z_col:
                    a = 2 # left
                if a_col < z_col:
                    a = 3 # right
                actions[i] = a
#                print "Agent=", agents[i,:]
#                print "Target", target
#                print "Zone", zones[argmin(distances),:]
#                print "Action", a
#                print '============'
            return vec2id(actions,ones(len(agents),dtype=integer)*5)
        if className(domain) == 'SystemAdministrator':
            # Select a broken computer and reset it
            brokenComputers = where(s==0)[0]
            if len(brokenComputers):
                return randSet(brokenComputers)
            else:
                return domain.computers_num
        if className(domain) == 'MountainCar':
            # Accelerate in the direction of the valley
            # WORK IN PROGRESS
            x,xdot = s
            if xdot > 0:
                return 2
            else:
                return 0
        if className(domain) == 'PST':
            # One stays at comm, n-1 stay at target area. Whenever fuel is lower than reaching the base the move back
            print s
            s       = domain.state2Struct(s)
            uavs    = domain.NUM_UAV
            print s
            return vec2id(zeros(uavs),ones(uavs)*3)
{'fraction_non_alphanumeric': 0.08435225272372615, 'fraction_numerical': 0.006755792378324374, 'mean_word_length': 1.7818951826363154, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4001969', 'n_tokens_mistral': 5327, 'n_tokens_neox': 4865, 'n_words': 1965}