<commit_before>#!/usr/bin/lua

--[[--------------------------------------------------------------------------
-- Licensed to Qualys, Inc. (QUALYS) under one or more
-- contributor license agreements.  See the NOTICE file distributed with
-- this work for additional information regarding copyright ownership.
-- QUALYS licenses this file to You under the Apache License, Version 2.0
-- (the "License"); you may not use this file except in compliance with
-- the License.  You may obtain a copy of the License at
--
--     http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
--]]--------------------------------------------------------------------------

-- ###########################################################################
-- Planner
-- ###########################################################################
--
-- Define a planner class. This holds the logic for taking
-- a SignatureDatabase and producing an execution plan.
--
-- The planner interface is only planner:plan(db). Everything else is
-- optional.
--
-- The planner borrows a C++ism by prefixing its data fields with m_
-- for "member" where functions have no such prefix. This it avoid
-- name collisions with fields like "plan" which are the verb and the noun,
-- or the function and the resultant data.
--
local Planner = {}
Planner.__index = Planner
Planner.type = "planner"
Planner.new = function(self)
    local p = {
        -- Table of rule_ids that have been put in the m_plan list.
        m_inplan = {},

        -- Lists of list rule IDs. This represents the final execution plan.
        -- The sub-lists, composed of rule ids, represent rule chains.
        -- Most lists will be of size 1, in which case there is no rule chain
        -- in use.
        m_plan = {},

        -- Table indexed by rule ids indicating that the rule may not
        -- be scheduled yet because we are currently trying to schedule
        -- a rule that must come before it. The value is the number
        -- of rules blocking the key from being scheduled.
        m_notyet = {},

        -- Table of rules that are in the planning stage.
        m_inplanning = {}

    }
    return setmetatable(p, self)
end

-- Insert or increment the rule_id in the m_notyet table to prevent
-- planning the rule.
--
-- If a rule is attempted to be planned that is in the m_notyet table
-- then a loop in the rule dependency graph has been detected and it is
-- a fatal error to the plan.
Planner.block_planning = function(self, rule_id)
    if self.m_notyet[rule_id] == nil then
        self.m_notyet[rule_id] = 1
    else
        self.m_notyet[rule_id] = self.m_notyet[rule_id] + 1
    end
end

-- Perform the reverse of block_planning, allowing a rule to be planned.
Planner.unblock_planning = function(self, rule_id)
    if self.m_notyet[rule_id] == 1 then
        self.m_notyet[rule_id] = nil
    elseif self.m_notyet[rule_id] ~= nil then
        self.m_notyet[rule_id] = self.m_notyet[rule_id] - 1
    end
end

-- Is a rule planned? 
-- @param[in] self Self.
--
-- @param[in] rule_id String representing the rule id.
--
-- @returns true or false
Planner.is_planned = function(self, rule_id)
    return self.m_inplan[rule_id] ~= nil
end

-- Take a list of rule_ids and tags and build a list of rule ids only.
--
-- @param[in] self The planner.
-- @param[in] list List of tags and rule ids.
-- @param[in] db The database of signatures (rules) to use for
--            conversion.
-- @returns List of rule ids, only. This list may have
--          duplicates in it, which is OK.
Planner.to_rule_ids = function(self, list, db)
    local r = {}

    for _, value in ipairs(list) do
        if db.db[value] ~= nil then
            table.insert(r, value)
        elseif db.tag_db[value] ~= nil then
            for rule_id, _ in pairs(db.tag_db[value]) do
                table.insert(r, rule_id)
            end
        end
    end

    return r
end

-- Recursively insert rules that rule follows into the follow list.
--
-- Tables structured { rule = rule_id, result = true|false } will be
-- inserted into the parameter "list" in the order they should
-- execute leading up to the execution of rule.
--
-- This function will check for duplicates.
-- 
local insert_follows
insert_follows = function(list, rule, db, seen)

    -- Check if the given rule is in it.
    if seen[rule.data.id] ~= nil then
        error( {
            sig_id = rule.data.id,
            sig_rev = rule.data.version,
            msg = string.format("Rule %s attempted to follow itself.", rule.data.id)
        }, 1)
    end

    -- Mark this rule as already followed.
    seen[rule.data.id] = 1

    for _, rule_link in ipairs(rule.data.follows) do
        insert_follows(list, db.db[rule_link.rule], db, seen)
        table.insert(list, rule_link)
    end

    -- Unmark this rule.
    seen[rule.data.id] = nil
end

-- Plan a single rule and all its dependencies.
-- Returns true on success and exits with error({sig_id, sig_rev, msg}, 1) on error.
Planner.plan_rule = function(self, rule, db)
    -- Block before rules.
    for _, before_rule_id in ipairs(self:to_rule_ids(rule.data.before, db)) do
        self:block_planning(before_rule_id)<commit_msg>waggle: Adding improved error messages when following missing rules.<commit_after>#!/usr/bin/lua

--[[--------------------------------------------------------------------------
-- Licensed to Qualys, Inc. (QUALYS) under one or more
-- contributor license agreements.  See the NOTICE file distributed with
-- this work for additional information regarding copyright ownership.
-- QUALYS licenses this file to You under the Apache License, Version 2.0
-- (the "License"); you may not use this file except in compliance with
-- the License.  You may obtain a copy of the License at
--
--     http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
--]]--------------------------------------------------------------------------

-- ###########################################################################
-- Planner
-- ###########################################################################
--
-- Define a planner class. This holds the logic for taking
-- a SignatureDatabase and producing an execution plan.
--
-- The planner interface is only planner:plan(db). Everything else is
-- optional.
--
-- The planner borrows a C++ism by prefixing its data fields with m_
-- for "member" where functions have no such prefix. This it avoid
-- name collisions with fields like "plan" which are the verb and the noun,
-- or the function and the resultant data.
--
local Planner = {}
Planner.__index = Planner
Planner.type = "planner"
Planner.new = function(self)
    local p = {
        -- Table of rule_ids that have been put in the m_plan list.
        m_inplan = {},

        -- Lists of list rule IDs. This represents the final execution plan.
        -- The sub-lists, composed of rule ids, represent rule chains.
        -- Most lists will be of size 1, in which case there is no rule chain
        -- in use.
        m_plan = {},

        -- Table indexed by rule ids indicating that the rule may not
        -- be scheduled yet because we are currently trying to schedule
        -- a rule that must come before it. The value is the number
        -- of rules blocking the key from being scheduled.
        m_notyet = {},

        -- Table of rules that are in the planning stage.
        m_inplanning = {}

    }
    return setmetatable(p, self)
end

-- Insert or increment the rule_id in the m_notyet table to prevent
-- planning the rule.
--
-- If a rule is attempted to be planned that is in the m_notyet table
-- then a loop in the rule dependency graph has been detected and it is
-- a fatal error to the plan.
Planner.block_planning = function(self, rule_id)
    if self.m_notyet[rule_id] == nil then
        self.m_notyet[rule_id] = 1
    else
        self.m_notyet[rule_id] = self.m_notyet[rule_id] + 1
    end
end

-- Perform the reverse of block_planning, allowing a rule to be planned.
Planner.unblock_planning = function(self, rule_id)
    if self.m_notyet[rule_id] == 1 then
        self.m_notyet[rule_id] = nil
    elseif self.m_notyet[rule_id] ~= nil then
        self.m_notyet[rule_id] = self.m_notyet[rule_id] - 1
    end
end

-- Is a rule planned? 
-- @param[in] self Self.
--
-- @param[in] rule_id String representing the rule id.
--
-- @returns true or false
Planner.is_planned = function(self, rule_id)
    return self.m_inplan[rule_id] ~= nil
end

-- Take a list of rule_ids and tags and build a list of rule ids only.
--
-- @param[in] self The planner.
-- @param[in] list List of tags and rule ids.
-- @param[in] db The database of signatures (rules) to use for
--            conversion.
-- @returns List of rule ids, only. This list may have
--          duplicates in it, which is OK.
Planner.to_rule_ids = function(self, list, db)
    local r = {}

    for _, value in ipairs(list) do
        if db.db[value] ~= nil then
            table.insert(r, value)
        elseif db.tag_db[value] ~= nil then
            for rule_id, _ in pairs(db.tag_db[value]) do
                table.insert(r, rule_id)
            end
        end
    end

    return r
end

-- Recursively insert rules that rule follows into the follow list.
--
-- Tables structured { rule = rule_id, result = true|false } will be
-- inserted into the parameter "list" in the order they should
-- execute leading up to the execution of rule.
--
-- This function will check for duplicates.
-- 
local insert_follows
insert_follows = function(list, rule, db, seen)

    -- Check if the given rule is in it.
    if seen[rule.data.id] ~= nil then
        error( {
            sig_id = rule.data.id,
            sig_rev = rule.data.version,
            msg = string.format("Rule %s attempted to follow itself.", rule.data.id)
        }, 1)
    end

    -- Mark this rule as already followed.
    seen[rule.data.id] = 1

    for _, rule_link in ipairs(rule.data.follows) do
        local next_rule = db.db[rule_link.rule]
        if next_rule == nil then
            error({
                sig_id = rule.data.id,
                sig_rev = rule.data.version,
                msg = string.format("Attempting to follow non-existant rule %s.", rule_link.rule)
            })
        end
        insert_follows(list, next_rule, db, seen)
        table.insert(list, rule_link)
    end

    -- Unmark this rule.
    seen[rule.data.id] = nil
end

-- Plan a single rule and all its dependencies.
-- Returns true on success and exits with error({sig_id, sig_rev, msg}, 1) on error.
Planner.plan_rule = function(self, rule, db)
    -- Block before rules.
    for _, before_rule_id in ipairs(self:to_rule_ids(rule.data.before, db)) do
        self:block_planning(before_rule_id)
{'fraction_non_alphanumeric': 0.14123647876176237, 'fraction_numerical': 0.0021106323102629498, 'mean_word_length': 2.387548406315162, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7033944', 'n_tokens_mistral': 3273, 'n_tokens_neox': 3180, 'n_words': 1306}