<commit_before>#!/usr/bin/env python
from __future__ import print_function

# a simple parser for Valve's KeyValue format
# https://developer.valvesoftware.com/wiki/KeyValues
#
# author: <NAME>, 2014
#
# use at your own risk

__version__ = "1.6"

import re
import sys

# Py2 & Py3 compability
if sys.version_info[0] >= 3:
    string_type = str
    next_method_name = '__next__'
    BOMS = '\ufffe\ufeff'

    def strip_bom(line):
        return line.lstrip(BOMS)
else:
    string_type = basestring
    next_method_name = 'next'
    BOMS = '\xef\xbb\xbf\xff\xfe\xfe\xff'
    BOMS_UNICODE = '\\ufffe\\ufeff'.decode('unicode-escape')

    def strip_bom(line):
        return line.lstrip(BOMS if isinstance(line, str) else BOMS_UNICODE)

###############################################
#
# Takes a file or str and returns dict
#
# Function assumes valid VDF as input.
# Invalid VDF will result in unexpected output
#
###############################################

<commit_msg>vdf: work around bom issue with files opened in binary mode<commit_after>#!/usr/bin/env python
from __future__ import print_function

# a simple parser for Valve's KeyValue format
# https://developer.valvesoftware.com/wiki/KeyValues
#
# author: <NAME>, 2014
#
# use at your own risk

__version__ = "1.6"

import codecs
import re
import sys

# Py2 & Py3 compability
if sys.version_info[0] >= 3:
    string_type = str
    next_method_name = '__next__'
    BOMS = '\ufffe\ufeff'
    BOMS = codecs.BOM_UTF16_LE

    def strip_bom(line):
        return line.lstrip(BOMS)
else:
    string_type = basestring
    next_method_name = 'next'
    BOMS = '\xef\xbb\xbf\xff\xfe\xfe\xff'
    BOMS_UNICODE = '\\ufffe\\ufeff'.decode('unicode-escape')

    def strip_bom(line):
        return line.lstrip(BOMS if isinstance(line, str) else BOMS_UNICODE)

###############################################
#
# Takes a file or str and returns dict
#
# Function assumes valid VDF as input.
# Invalid VDF will result in unexpected output
#
###############################################


{'fraction_non_alphanumeric': 0.18881804806277588, 'fraction_numerical': 0.010789602746444336, 'mean_word_length': 3.9156626506024095, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3255649', 'n_tokens_mistral': 707, 'n_tokens_neox': 695, 'n_words': 180}