<commit_before>import sets, cligen, cligen/[mfile, mslice] #mSlices MSlice Splitr

proc cols(input="/dev/stdin", delim="white", output="/dev/stdout", sepOut=" ",
          blanksOk=false, cut=false, origin=1, colNums: seq[int]) =
    ##Write just some columns of input to output; Memory map input if possible.
    var outFile = open(output, fmWrite)
    var colSet = initSet[int](rightSize(len(colNums)))
    if cut:
        for c in colNums: colSet.incl(c)
    let splitr = initSplitr(delim)
    var cols: seq[MSlice] = @[ ]
    for line in mSlices(input, eat='\0'):   #RO mmap | slices from stdio
        var wroteSomething = false
        splitr.split(line, cols)
        if cut:
            var ith = 0
            for j, f in cols:
                if (origin + j) in colSet or (origin + j - cols.len) in colSet:
                  continue
                if ith != 0: outFile.write sepOut
                outFile.write f
                wroteSomething = true
                inc(ith)
        else:
            for ith, i in colNums:
                let j = if i < 0: i + cols.len else: i - origin
                if j < 0 or j >= cols.len:
                    continue
                if ith != 0: outFile.write sepOut
                outFile.write cols[j]
                wroteSomething = true
        if wroteSomething or blanksOk:
            outFile.write "\n"

when isMainModule:
    dispatch(cols, help = {
             "input"   : "path to mmap|read as input",
             "delim"   : "inp delim chars; Any repeats => fold",
             "output"  : "path to write output file",
             "sepOut"  : "output field separator",
             "blanksOk": "allow blank output rows",
             "cut"     : "cut/censor specified columns, not keep",
             "origin"  : "origin for colNums; 0=>signed indexing" })
<commit_msg>Use ``urite`` to be more performant example code.<commit_after>import sets, cligen, cligen/[mfile, mslice, osUt] #mSlices MSlice Splitr

proc cols(input="/dev/stdin", delim="white", output="/dev/stdout", sepOut=" ",
          blanksOk=false, cut=false, origin=1, colNums: seq[int]) =
    ##Write just some columns of input to output; Memory map input if possible.
    var outFile = open(output, fmWrite)
    var colSet = initSet[int](rightSize(len(colNums)))
    if cut:
        for c in colNums: colSet.incl(c)
    let splitr = initSplitr(delim)
    var cols: seq[MSlice] = @[ ]
    for line in mSlices(input, eat='\0'):   #RO mmap | slices from stdio
        var wroteSomething = false
        splitr.split(line, cols)
        if cut:
            var ith = 0
            for j, f in cols:
                if (origin + j) in colSet or (origin + j - cols.len) in colSet:
                  continue
                if ith != 0: outFile.urite sepOut
                outFile.urite f
                wroteSomething = true
                inc(ith)
        else:
            for ith, i in colNums:
                let j = if i < 0: i + cols.len else: i - origin
                if j < 0 or j >= cols.len:
                    continue
                if ith != 0: outFile.urite sepOut
                outFile.urite cols[j]
                wroteSomething = true
        if wroteSomething or blanksOk:
            outFile.urite '\n'

when isMainModule:
    dispatch(cols, help = {
             "input"   : "path to mmap|read as input",
             "delim"   : "inp delim chars; Any repeats => fold",
             "output"  : "path to write output file",
             "sepOut"  : "output field separator",
             "blanksOk": "allow blank output rows",
             "cut"     : "cut/censor specified columns, not keep",
             "origin"  : "origin for colNums; 0=>signed indexing" })

{'fraction_non_alphanumeric': 0.10622317596566523, 'fraction_numerical': 0.004291845493562232, 'mean_word_length': 1.7338709677419355, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7202317', 'n_tokens_mistral': 1176, 'n_tokens_neox': 1119, 'n_words': 368}