<commit_before>#!/usr/bin/env python
# vim: set ts=4 sw=4 et: coding=UTF-8

# Copyright (c) 2013, SUSE LINUX Products GmbH, Nuernberg, Germany
# All rights reserved.
# See COPYING for details.

__version__ = '0.3'

import os
import sys

import cStringIO
import optparse
import re
import time
import tempfile
import subprocess
import shlex

#######################################################################

VERSION = '0.3'
LICENSES_CHANGES = 'licenses_changes.txt'
PKGCONFIG_CONVERSIONS = 'pkgconfig_conversions.txt'
BRACKETING_EXCLUDES = 'excludes-bracketing.txt'

re_comment = re.compile('^$|^\s*#')
re_define = re.compile('^\s*%define', re.IGNORECASE)

re_bindir = re.compile('%{_prefix}/bin([/\s$])')
re_sbindir = re.compile('%{_prefix}/sbin([/\s$])')
re_includedir = re.compile('%{_prefix}/include([/\s$])')
re_datadir = re.compile('%{_prefix}/share([/\s$])')
re_mandir = re.compile('%{_datadir}/man([/\s$])')
re_infodir = re.compile('%{_datadir}/info([/\s$])')
re_docdir = re.compile('%{_datadir}/doc([/\s$])')

re_macro = re.compile(r'(^|([^%]))%(\w+)(|(\W]))')
re_jobs = re.compile('%{(_smp_mflags|\?jobs:\s*-j\s*%(jobs|{jobs}))}')

global re_unbrace_keywords
global spec

def open_datafile(FILE):
    try:
        # the .. is appended as we are in spec_cleaner sub_folder
        f = open('{0}/{1}'.format(os.path.dirname(os.path.realpath(__file__ + '/../data/')), FILE), 'r')
    except IOError:
        f = open('/usr/share/spec-cleaner/{0}'.format(FILE), 'r')
    return f<commit_msg>Open the data files properly.<commit_after>#!/usr/bin/env python
# vim: set ts=4 sw=4 et: coding=UTF-8

# Copyright (c) 2013, SUSE LINUX Products GmbH, Nuernberg, Germany
# All rights reserved.
# See COPYING for details.

__version__ = '0.3'

import os
import sys

import cStringIO
import optparse
import re
import time
import tempfile
import subprocess
import shlex

#######################################################################

VERSION = '0.3'
LICENSES_CHANGES = 'licenses_changes.txt'
PKGCONFIG_CONVERSIONS = 'pkgconfig_conversions.txt'
BRACKETING_EXCLUDES = 'excludes-bracketing.txt'

re_comment = re.compile('^$|^\s*#')
re_define = re.compile('^\s*%define', re.IGNORECASE)

re_bindir = re.compile('%{_prefix}/bin([/\s$])')
re_sbindir = re.compile('%{_prefix}/sbin([/\s$])')
re_includedir = re.compile('%{_prefix}/include([/\s$])')
re_datadir = re.compile('%{_prefix}/share([/\s$])')
re_mandir = re.compile('%{_datadir}/man([/\s$])')
re_infodir = re.compile('%{_datadir}/info([/\s$])')
re_docdir = re.compile('%{_datadir}/doc([/\s$])')

re_macro = re.compile(r'(^|([^%]))%(\w+)(|(\W]))')
re_jobs = re.compile('%{(_smp_mflags|\?jobs:\s*-j\s*%(jobs|{jobs}))}')

global re_unbrace_keywords
global spec

def open_datafile(FILE):
    try:
        f = open('/usr/share/spec-cleaner/{0}'.format(FILE), 'r')
    except IOError:
        # the .. is appended as we are in spec_cleaner sub_folder
        f = open('{0}/{1}'.format(os.path.dirname(os.path.realpath(__file__)) + '/../data/', FILE), 'r')

    return f
{'fraction_non_alphanumeric': 0.24793114862628268, 'fraction_numerical': 0.009268454154253558, 'mean_word_length': 5.031936127744511, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3274374', 'n_tokens_mistral': 1194, 'n_tokens_neox': 1152, 'n_words': 209}