<commit_before># Sqawk, an SQL Awk.
# Copyright (c) 2015, 2016, 2017, 2018, 2020 <NAME>
# License: MIT

namespace eval ::sqawk::parsers::tcl {
    variable formats {
        tcl
    }
    variable options {
        dicts 0
    }
}

::snit::type ::sqawk::parsers::tcl::parser {
    variable useDicts

    variable data
    variable i
    variable keys
    variable len

    constructor {channel options} {
        set useDicts [dict get $options dicts]
        set i [expr { $useDicts ? -1 : 0 }]
        set data [read $channel]
        set len [llength $data]
    }

    method next {} {
        if {$i == $len} {
            return -code break
        }

        if {!$useDicts} {
            set list [lindex $data $i]
            incr i
            return [list $list {*}$list]
        }

        if {$i == -1} {
            set allKeys {}
            foreach record $data {
                set allKeys [lsort -dictionary -unique \
                        [concat $allKeys [dict keys $record]]]
            }

            # Order the keys like they are ordered in the first row for
            # ergonomics.  Keys that aren't in the first row follow in
            # alphabetical order after those that are.
            set keys [dict keys [lindex $data 0]]
            foreach key $allKeys {
                if {$key ni $keys} {
                    lappend keys $key
                }
            }

            incr i
            return [list $keys {*}$keys]
        }

        set record [lindex $data $i]
        set row [list $record]
        foreach key $keys {
            if {[dict exists $record $key]} {
                lappend row [dict get $record $key]<commit_msg>fix(parsers/tcl): copy JSON parser $allKeys code<commit_after># Sqawk, an SQL Awk.
# Copyright (c) 2015, 2016, 2017, 2018, 2020 <NAME>
# License: MIT

namespace eval ::sqawk::parsers::tcl {
    variable formats {
        tcl
    }
    variable options {
        dicts 0
    }
}

::snit::type ::sqawk::parsers::tcl::parser {
    variable useDicts

    variable data
    variable i
    variable keys
    variable len

    constructor {channel options} {
        set useDicts [dict get $options dicts]
        set i [expr { $useDicts ? -1 : 0 }]
        set data [read $channel]
        set len [llength $data]
    }

    method next {} {
        if {$i == $len} {
            return -code break
        }

        if {!$useDicts} {
            set list [lindex $data $i]
            incr i
            return [list $list {*}$list]
        }

        if {$i == -1} {
            set allKeys [lsort -unique [concat {*}[lmap record $data {
                dict keys $record
            }]]]

            # Order the keys like they are ordered in the first row for
            # ergonomics.  Keys that aren't in the first row follow in
            # alphabetical order after those that are.
            set keys [dict keys [lindex $data 0]]
            foreach key $allKeys {
                if {$key ni $keys} {
                    lappend keys $key
                }
            }

            incr i
            return [list $keys {*}$keys]
        }

        set record [lindex $data $i]
        set row [list $record]
        foreach key $keys {
            if {[dict exists $record $key]} {
                lappend row [dict get $record $key]
{'fraction_non_alphanumeric': 0.09960878724044538, 'fraction_numerical': 0.01504664459825459, 'mean_word_length': 1.445916114790287, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '614858', 'n_tokens_mistral': 1023, 'n_tokens_neox': 940, 'n_words': 346}