<commit_before>-- Use of this source code is governed by the Apache 2.0 license; see COPYING.
module(..., package.seeall)

local S = require("syscall")
local ffi = require("ffi")
local lib = require("core.lib")
local shm = require("core.shm")
local mem = require("lib.stream.mem")
local file = require("lib.stream.file")
local rpc = require("lib.yang.rpc")
local yang = require("lib.yang.yang")
local data = require("lib.yang.data")
local path_data = require("lib.yang.path_data")
local path_resolver = require("lib.yang.path_data").resolver

function show_usage(command, status, err_msg)
   if err_msg then print('error: '..err_msg) end
   print(require("program.config."..command:gsub('-','_')..".README_inc"))
   main.exit(status)
end

local parse_command_line_opts = {
   command = { required=true },
   with_config_file = { default=false },
   with_path = { default=false },
   with_value = { default=false },
   require_schema = { default=false },
   is_config = { default=true },
   usage = { default=show_usage },
   allow_extra_args = { default=false },
}

local function path_grammar(schema_name, path, is_config)
   local schema = yang.load_schema_by_name(schema_name)
   local grammar = data.data_grammar_from_schema(schema, is_config)
   local getter, subgrammar = path_resolver(grammar, path)
   return subgrammar
end

function data_parser(schema_name, path, is_config)
   local grammar = path_grammar(schema_name, path, is_config)
   local parser = data.data_parser_from_grammar(grammar)
   local validator = path_data.consistency_checker_from_grammar(grammar)
   return function (data)
      local config = parser(data)
      validator(config)
      print("validated")
      return config
   end
end

function config_parser(schema_name, path)
   return data_parser(schema_name, path, true)
end

function state_parser(schema_name, path)
   return data_parser(schema_name, path, false)<commit_msg>lib.yang: remove leftover debug print<commit_after>-- Use of this source code is governed by the Apache 2.0 license; see COPYING.
module(..., package.seeall)

local S = require("syscall")
local ffi = require("ffi")
local lib = require("core.lib")
local shm = require("core.shm")
local mem = require("lib.stream.mem")
local file = require("lib.stream.file")
local rpc = require("lib.yang.rpc")
local yang = require("lib.yang.yang")
local data = require("lib.yang.data")
local path_data = require("lib.yang.path_data")
local path_resolver = require("lib.yang.path_data").resolver

function show_usage(command, status, err_msg)
   if err_msg then print('error: '..err_msg) end
   print(require("program.config."..command:gsub('-','_')..".README_inc"))
   main.exit(status)
end

local parse_command_line_opts = {
   command = { required=true },
   with_config_file = { default=false },
   with_path = { default=false },
   with_value = { default=false },
   require_schema = { default=false },
   is_config = { default=true },
   usage = { default=show_usage },
   allow_extra_args = { default=false },
}

local function path_grammar(schema_name, path, is_config)
   local schema = yang.load_schema_by_name(schema_name)
   local grammar = data.data_grammar_from_schema(schema, is_config)
   local getter, subgrammar = path_resolver(grammar, path)
   return subgrammar
end

function data_parser(schema_name, path, is_config)
   local grammar = path_grammar(schema_name, path, is_config)
   local parser = data.data_parser_from_grammar(grammar)
   local validator = path_data.consistency_checker_from_grammar(grammar)
   return function (data)
      local config = parser(data)
      validator(config)
      return config
   end
end

function config_parser(schema_name, path)
   return data_parser(schema_name, path, true)
end

function state_parser(schema_name, path)
   return data_parser(schema_name, path, false)
{'fraction_non_alphanumeric': 0.1188170635959173, 'fraction_numerical': 0.0010468463752944255, 'mean_word_length': 4.90726429675425, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6486226', 'n_tokens_mistral': 1259, 'n_tokens_neox': 1226, 'n_words': 288}