<commit_before>-- This Source Code Form is subject to the terms of the Mozilla Public
-- License, v. 2.0. If a copy of the MPL was not distributed with this
-- file, You can obtain one at http://mozilla.org/MPL/2.0/.

--[=[
Extracts data from message fields in messages generated by a
and generates JSON suitable for use with the Librato
's `HTTP API <http://dev.librato.com/v1>`_.

Config:

- ignore_fields (string, optional, defaults to "")
    A `,` seperated list of Fields that will be omitted when creating the metric name
    eg. "extra,hostname" will ignore fields called `extra` and `hostname` from the metric name

*Example Heka Configuration*

.. code-block:: ini

    [librato-encoder]
    type = "SandboxEncoder"
    filename = "lua_encoders/schema_librato.lua"
       [librato-encoder.config]
       ignore_fields = "level,hostname"

    [librato]
    type = "HttpOutput"
    message_matcher = "Type == 'heka.statmetric'"
    address = "https://metrics-api.librato.com/v1/metrics"
    encoder = "librato-encoder"
    username = "librato_username"
    password = "<PASSWORD>"

*Example Output*

.. code-block:: json

    {"gauges":{ "time_taken.video_encoding.mpeg" : {"value":441267,"measure_time":1417413408,"source":"0a5a7339e657"}}}

--]=]

require "cjson"
require "table"
require "string"


local stat_types = {
    gauge = "gauges",
    gauges = "gauges",
    counter = "counters",
    counters = "counters"
}
local ignore_fields_config = read_config("ignore_fields") or 'level'

local ignore_fields = { value = true, type = true}
for f in string.gmatch(ignore_fields_config, '([^,]+)') do
    if f ~= "" and f ~= nil then
        ignore_fields[f] = true
    end
end


function metric_name()
    local field_names = {}

    while true do
        typ, name, value, representation, count = read_next_field()
        if not typ then break end
        if ignore_fields[name] == nil then
            table.insert(field_names, name)
        end
    end
    table.sort(field_names)

    local names = {}
    for _, field in pairs(field_names) do
        local val = read_message("Fields[".. field .."]")
        if val then
            table.insert(names, val)
        end
    end

    return table.concat(names, '.')
end

function process_message()
    local ts = read_message("Timestamp") / 1e9
    local hostname = read_message("Hostname")

    local stat_type = stat_types[read_message("Fields[type]")]
    local value = read_message("Fields[value]")

    if not ts or not value then
        return -1
    end
    if not stat_type then return -1 end
    -- only process name if everything looks good
    local name = metric_name()
    if not name or name == "" then return -1 end<commit_msg>librato: user can specify fields to start with<commit_after>-- This Source Code Form is subject to the terms of the Mozilla Public
-- License, v. 2.0. If a copy of the MPL was not distributed with this
-- file, You can obtain one at http://mozilla.org/MPL/2.0/.

--[=[
Extracts data from message fields in messages generated by a
and generates JSON suitable for use with the Librato
's `HTTP API <http://dev.librato.com/v1>`_.

Config:

- ignore_fields (string, optional, defaults to "level")
    A `,` seperated list of Fields that will be omitted when creating the metric name
    eg. "extra,hostname" will ignore fields called `extra` and `hostname` from the metric name
- start_with (string, optional, defaults to "")
    A `,` seperated list of Fields whose values will be used at the start of the metric name. 
    Other fields values will be appended after in alphabetical order.
    eg. "source,title" will start each metric name with the values %title%.%source%.


*Example Heka Configuration*

.. code-block:: ini

    [librato-encoder]
    type = "SandboxEncoder"
    filename = "lua_encoders/schema_librato.lua"
       [librato-encoder.config]
       ignore_fields = "level,hostname"

    [librato]
    type = "HttpOutput"
    message_matcher = "Type == 'heka.statmetric'"
    address = "https://metrics-api.librato.com/v1/metrics"
    encoder = "librato-encoder"
    username = "librato_username"
    password = "<PASSWORD>"

*Example Output*

.. code-block:: json

    {"gauges":{ "time_taken.video_encoding.mpeg" : {"value":441267,"measure_time":1417413408,"source":"0a5a7339e657"}}}

--]=]

require "cjson"
require "table"
require "string"


local stat_types = {
    gauge = "gauges",
    gauges = "gauges",
    counter = "counters",
    counters = "counters"
}
local ignore_fields_config = read_config("ignore_fields") or 'level'
local start_with_config = read_config("start_with") or ''

local ignore_fields = { value = true, type = true}
for f in string.gmatch(ignore_fields_config, '([^,]+)') do
    if f ~= "" and f ~= nil then
        ignore_fields[f] = true
    end
end
local start_with = {}
for f in string.gmatch(start_with_config, '([^,]+)') do
    if f ~= "" and f ~= nil then
        start_with[f] = true
    end
end


function metric_name()
    local field_names = {}

    while true do
        typ, name, value, representation, count = read_next_field()
        if not typ then break end
        -- skip ignore_fields and start_with fields
        if ignore_fields[name] == nil and start_with[name] == nil then
            table.insert(field_names, name)
        end
    end
    table.sort(field_names)

    local names = {}
    -- manually insert start_with fields in the start
    for key, _ in pairs(start_with) do
        local val = read_message("Fields[".. key .."]")
        if val then
            table.insert(names, val)
        end
    end

    for _, field in pairs(field_names) do
        local val = read_message("Fields[".. field .."]")
        if val then
            table.insert(names, val)
        end
    end

    return table.concat(names, '.')
end

function process_message()
    local ts = read_message("Timestamp") / 1e9
    local hostname = read_message("Hostname")

    local stat_type = stat_types[read_message("Fields[type]")]
    local value = read_message("Fields[value]")

    if not ts or not value then
        return -1
    end
    if not stat_type then return -1 end
    -- only process name if everything looks good
    local name = metric_name()
    if not name or name == "" then return -1 end
{'fraction_non_alphanumeric': 0.11311423101467773, 'fraction_numerical': 0.01148691767708998, 'mean_word_length': 3.008312020460358, 'pattern_counts': {'":': 8, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '904135', 'n_tokens_mistral': 2000, 'n_tokens_neox': 1938, 'n_words': 636}