<commit_before>local ExtendedCmdLine = require('onmt.utils.ExtendedCmdLine')

local Cuda = {
  fp16 = false,
  gpuIds = {},
  activated = false
}

local options = {
  {'-gpuid',     '0',   [[List of comma-separated GPU identifiers (1-indexed). CPU is used when set to 0.]],
                                 {valid=ExtendedCmdLine.listUInt}},
  {'-fallback_to_cpu', false, [[If GPU can't be use, rollback on the CPU.]]},
  {'-fp16', false, [[Use half-precision float on GPU.]]},
  {'-no_nccl', false, [[Disable usage of nccl in parallel mode.]]}
}

function Cuda.declareOpts(cmd)
  cmd:setCmdLineOptions(options)
end

function Cuda.init(opt, masterGPU)<commit_msg>Fix minor typo in option description<commit_after>local ExtendedCmdLine = require('onmt.utils.ExtendedCmdLine')

local Cuda = {
  fp16 = false,
  gpuIds = {},
  activated = false
}

local options = {
  {'-gpuid',     '0',   [[List of comma-separated GPU identifiers (1-indexed). CPU is used when set to 0.]],
                                 {valid=ExtendedCmdLine.listUInt}},
  {'-fallback_to_cpu', false, [[If GPU can't be used, rollback on the CPU.]]},
  {'-fp16', false, [[Use half-precision float on GPU.]]},
  {'-no_nccl', false, [[Disable usage of nccl in parallel mode.]]}
}

function Cuda.declareOpts(cmd)
  cmd:setCmdLineOptions(options)
end

function Cuda.init(opt, masterGPU)
{'fraction_non_alphanumeric': 0.14666666666666667, 'fraction_numerical': 0.01037037037037037, 'mean_word_length': 3.4735099337748343, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6772120', 'n_tokens_mistral': 467, 'n_tokens_neox': 441, 'n_words': 128}