<commit_before># -*- coding: utf-8 -*-
# dictionary generating script
#
# Copyright 2008 (C) <NAME>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#
#

import sys
import unicodedata

reload(sys)
sys.setdefaultencoding('UTF-8')

def nfd(u8str):
    return unicodedata.normalize('NFD', u8str.decode('UTF-8')).encode('UTF-8')

def out(u8str):
    return sys.stdout.write(u8str)

filenames = sys.argv[1:]

if len(filenames) < 1:
    print 'Usage: %s filenames...' % sys.argv[0]
    sys.exit(1)

## load

lines = []
for filename in filenames:
    lines += open(filename).readlines()

# comment out
lines = [line for line in lines if line[0] != '#']

# remove whitespaces
lines = [line.strip() for line in lines]

# empty line out
lines = [line for line in lines if line != '']

## 

def make_entry(str):
    d = str.split()
    name = d[0]
    info = {}
    for i in d[1:]:
        key, val = i.split(':')
        info[key] = val
    return (name, info)

data = map(make_entry, lines)
def compare_entry(a,b):<commit_msg>error message on wrong input file<commit_after># -*- coding: utf-8 -*-
# dictionary generating script
#
# Copyright 2008 (C) <NAME>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#
#

import sys
import unicodedata

reload(sys)
sys.setdefaultencoding('UTF-8')

def nfd(u8str):
    return unicodedata.normalize('NFD', u8str.decode('UTF-8')).encode('UTF-8')

def out(u8str):
    return sys.stdout.write(u8str)

filenames = sys.argv[1:]

if len(filenames) < 1:
    print 'Usage: %s filenames...' % sys.argv[0]
    sys.exit(1)

## load

lines = []
for filename in filenames:
    lines += open(filename).readlines()

# comment out
lines = [line for line in lines if line[0] != '#']

# remove whitespaces
lines = [line.strip() for line in lines]

# empty line out
lines = [line for line in lines if line != '']

## 

def make_entry(str):
    d = str.split()
    name = d[0]
    info = {}
    for i in d[1:]:
        try:
            key, val = i.split(':')
        except ValueError:
            sys.stderr.write('make-dic: wrong info "%s"\n' % i)
            raise ValueError
        info[key] = val
    return (name, info)

data = map(make_entry, lines)
def compare_entry(a,b):
{'fraction_non_alphanumeric': 0.1143099068585944, 'fraction_numerical': 0.019757267852102738, 'mean_word_length': 3.4804045512010116, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4640981', 'n_tokens_mistral': 1140, 'n_tokens_neox': 1107, 'n_words': 440}