<commit_before>#!/usr/bin/env python
#coding=utf-8
__author__ = '<NAME>'

import re
import json


def get_charset(html):
    """
    get charset in html page
    return : default if not exist `charset`
    """
    regex = r'charset=([a-zA-Z0-9-]+)?'
    pattern = re.compile(regex, re.IGNORECASE)
    if len(pattern.findall(html)) == 0:
        return 'UTF-8'
    return pattern.findall(html)[0]


def to_json_string(obj):
    return json.dumps(obj)


def trim(string):
    return string.strip()


def d(msg):
    print(msg)

<commit_msg>add dump file<commit_after># !/usr/bin/env python
# coding=utf-8
__author__ = '<NAME>'

import re
import json


def get_charset(html):
    """
    get charset in html page
    return : default if not exist `charset`
    """
    regex = r'charset=([a-zA-Z0-9-]+)?'
    pattern = re.compile(regex, re.IGNORECASE)
    if len(pattern.findall(html)) == 0:
        return 'UTF-8'
    return pattern.findall(html)[0]


def to_json_string(obj):
    return json.dumps(obj)


def trim(string):
    return string.strip()


def d(msg):
    print(msg)


def dump_file(content, filename="tmp.txt"):
    with open(filename, mode="w") as f:
        f.write(content)


{'fraction_non_alphanumeric': 0.15138772077375945, 'fraction_numerical': 0.010092514718250631, 'mean_word_length': 3.1319444444444446, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1943031', 'n_tokens_mistral': 452, 'n_tokens_neox': 428, 'n_words': 102}