<commit_before>
                         
   SERIALIZATION_FORMATS  
                         



COMPARISON ==>                    #Table-like:
                                  #  - based on CSV:
                                  #     - CSV (prefered): standard, widely used, simplistic
                                  #     - TSV: tab delimiter
                                  #     - DSV: any delimiter
                                  #  - Excel: featureful (e.g. formatting), proprietary
                                  #Tree:
                                  #  - simplistic:
                                  #     - INI: simplistic, but very few features and not standardized
                                  #     - TOML (prefered): simple and human-friendly, not fit for complex data
                                  #  - based on JSON:
                                  #     - JSON (prefered): standard, widely used, simple, fast
                                  #     - HJSON (prefered): adds comments, unquoted VAR|STR, no|trailing commas,
                                  #       multiline string, NaN|Inf, DATE
                                  #       Compared with JSON5:
                                  #        - more featureful, more maintained, better quality, different programming languages
                                  #        - less JavaScript-looking, slower
                                  #     - JSON5: adds ES5 syntax, i.e. comments, unquoted VAR, trailing commas,
                                  #       multiline string, NaN|Inf
                                  #     - CSON (avoid): adds CoffeeScript syntax, i.e. indentation, comments, unquoted VAR,
                                  #       no commas, multiline string
                                  #     - ESON (not maintained): interesting concept of pluggable special types for
                                  #       human-friendly types (time, dimensions, includes, ENVVAR|args)
                                  #  - featureful:
                                  #     - YAML (prefered): featureful, a bit slow, human-friendly, many types, references
                                  #     - XML: very featureful for query|transformation, strongly typed, very complex

PERFORMANCE ==>                   #This is based on the currently documented implementations, and in a JavaScript environment
                                  #Fast:
                                  #  - JSON:
                                  #     - fastest, used as unit here
                                  #     - 1ms to parse|stringify small file
                                  #  - XML: around same as JSON, although JSON much faster in JavaScript environment
                                  #Normal:
                                  #  - JSON5:
                                  #     - 4 time slower on parse|stringify
                                  #     - slower when parsing big files
                                  #  - YAML: 4-6 times slower on parse, 10-15 times on stringify, depending on schema used<commit_msg>no commit message<commit_after>
                         
   SERIALIZATION_FORMATS  
                         



COMPARISON ==>                    #Table-like:
                                  #  - based on CSV:
                                  #     - CSV (prefered): standard, widely used, simplistic
                                  #     - TSV: tab delimiter
                                  #     - DSV: any delimiter
                                  #  - Excel: featureful (e.g. formatting), proprietary
                                  #Tree:
                                  #  - simplistic:
                                  #     - INI: simplistic, but very few features and not standardized
                                  #     - TOML (prefered): simple and human-friendly, not fit for complex data
                                  #  - based on JSON:
                                  #     - JSON (prefered): standard, widely used, simple, fast
                                  #     - HJSON (prefered): adds comments, unquoted VAR|STR, no|trailing commas,
                                  #       multiline string, NaN|Inf, DATE
                                  #       Compared with JSON5:
                                  #        - more featureful, more maintained, better quality, different programming languages
                                  #        - less JavaScript-looking, slower
                                  #     - JSON5: adds ES5 syntax, i.e. comments, unquoted VAR, trailing commas,
                                  #       multiline string, NaN|Inf
                                  #     - CSON (avoid): adds CoffeeScript syntax, i.e. indentation, comments, unquoted VAR,
                                  #       no commas, multiline string
                                  #     - ESON (not maintained): interesting concept of pluggable special types for
                                  #       human-friendly types (time, dimensions, includes, ENVVAR|args)
                                  #  - featureful:
                                  #     - YAML (prefered): featureful, human-friendly, many types, references
                                  #     - XML: very featureful for query|transformation, strongly typed, very complex

PERFORMANCE ==>                   #This is based on the currently documented implementations, and in a JavaScript environment
                                  #Fast:
                                  #  - JSON:
                                  #     - fastest, used as unit here
                                  #     - 1ms to parse|stringify small file
                                  #  - XML: around same as JSON, although JSON much faster in JavaScript environment
                                  #Normal:
                                  #  - JSON5:
                                  #     - 4 time slower on parse|stringify
                                  #     - slower when parsing big files
                                  #  - YAML: 4-6 times slower on parse, 10-15 times on stringify, depending on schema used
{'fraction_non_alphanumeric': 0.1393013100436681, 'fraction_numerical': 0.0034934497816593887, 'mean_word_length': 1.0043757292882147, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 118, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1365617', 'n_tokens_mistral': 3163, 'n_tokens_neox': 3028, 'n_words': 428}