<commit_before>local M_ = {}

-- enum of available question types
local QTYPE = {
    UNKNOWN = 0,        -- questions that we cannot answer
    PIPELINE = 1,       -- a question with a few consequent seaching actions
}

-- enum of available POI attributes
local POI_ATTR = {
    NAME = 0,

    COORDINATES = 1,
    OPEN_TIME = 2,
    PRICE = 3,

    ADDRESS = 20,
    CITY = 21,
    DISTRICT = 22,
    STREET = 23,

    DESC = 100
}

-- question types
-- Since each query is a single question and no more instance is required, 
-- we choose to use an object instance rather than an all-round class here.
local query_struct = {
    qtype = QTYPE.PIPELINE,

    input = {POI_ATTR.NAME, POI_ATTR.CITY},
    output = {{POI_ATTR.ADDRESS}},

    -- a simple mechanism to indicate whether a question as a whole contains
    -- sentiment, a value with larger integer means more sentiment
    sentiment = 0,
}

-- analyze a question, input the question string, and longitude and latitude in float numbers
--  @question a string of question
--  @lng longitude of the user's location when sending the query
--  @lat latitude of the user's location when sending the query
--
--  @return a question analysis report
--
local function analyze(question, lng, lat)
    return {
        errno = 0,
        errmsg = '',
        query_struct = query_struct,
        pos = {},
        parse = {},
        depparse = {},
    }
end

-- export symbols
M_.QTYPE = QTYPE
M_.POI_ATTR = POI_ATTR
M_.analyze = analyze
<commit_msg>tell apart from the input schema and input value<commit_after>local M_ = {}

-- enum of available question types
local QTYPE = {
    UNKNOWN = 0,        -- questions that we cannot answer
    PIPELINE = 1,       -- a question with a few consequent seaching actions
}

-- enum of available POI attributes
local POI_ATTR = {
    NAME = 0,

    COORDINATES = 1,
    OPEN_TIME = 2,
    PRICE = 3,

    ADDRESS = 20,
    CITY = 21,
    DISTRICT = 22,
    STREET = 23,

    DESC = 100
}

-- question types
-- Since each query is a single question and no more instance is required, 
-- we choose to use an object instance rather than an all-round class here.
local query_repr = {
    qtype = QTYPE.PIPELINE,

    input_schema = {POI_ATTR.NAME, POI_ATTR.CITY},
    input_value = {nil, nil},
    output = {{POI_ATTR.ADDRESS}},

    -- a simple mechanism to indicate whether a question as a whole contains
    -- sentiment, a value with larger integer means more sentiment
    sentiment = 0,
}

-- analyze a question, input the question string, and longitude and latitude in float numbers
--  @question a string of question
--  @lng longitude of the user's location when sending the query
--  @lat latitude of the user's location when sending the query
--
--  @return a question analysis report
--
local function analyze(question, lng, lat)
    return {
        errno = 0,
        errmsg = '',
        query_repr = query_repr,
        pos = {},
        parse = {},
        depparse = {},
    }
end

-- export symbols
M_.QTYPE = QTYPE
M_.POI_ATTR = POI_ATTR
M_.analyze = analyze

{'fraction_non_alphanumeric': 0.08827361563517916, 'fraction_numerical': 0.012377850162866449, 'mean_word_length': 2.4937428896473266, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3271890', 'n_tokens_mistral': 976, 'n_tokens_neox': 902, 'n_words': 369}