<commit_before># setup.py, config file for distutils

import __init__

from distutils.core import setup
from distutils.command.install_data import install_data
import os


class smart_install_data(install_data):
    def run(self):
        #need to change self.install_dir to the actual library dir
        install_cmd = self.get_finalized_command('install')
        self.install_dir = getattr(install_cmd, 'install_lib')
        return install_data.run(self)


data_files = []
for dirpath, dirnames, filenames in os.walk(r'.'):
    for dirname in ['.svn','build', 'dist', '_sgbak']:
        try:
            dirnames.remove(dirname)
        except ValueError:
            pass
    for filename in [fn for fn in filenames if os.path.splitext(fn)[-1].lower() in ('.pyc', '.pyo', '.scc')]:
        filenames.remove(filename)
    parts = ['UnRAR2']+dirpath.split(os.sep)[1:]
    
    data_files.append((os.path.join(*parts), [os.path.join(dirpath, fn) for fn in filenames]))<commit_msg>setup.py now correctly ignores Mercurial files<commit_after># setup.py, config file for distutils

import __init__

from distutils.core import setup
from distutils.command.install_data import install_data
import os


class smart_install_data(install_data):
    def run(self):
        #need to change self.install_dir to the actual library dir
        install_cmd = self.get_finalized_command('install')
        self.install_dir = getattr(install_cmd, 'install_lib')
        return install_data.run(self)


data_files = []
for dirpath, dirnames, filenames in os.walk(r'.'):
    for dirname in ['.svn','build', 'dist', '_sgbak', '.hg']:
        try:
            dirnames.remove(dirname)
        except ValueError:
            pass
    for filename in [fn for fn in filenames if os.path.splitext(fn)[-1].lower() in ('.pyc', '.pyo', '.scc')]:
        filenames.remove(filename)
    parts = ['UnRAR2']+dirpath.split(os.sep)[1:]
    
    data_files.append((os.path.join(*parts), [os.path.join(dirpath, fn) for fn in filenames]))
{'fraction_non_alphanumeric': 0.12724550898203593, 'fraction_numerical': 0.0029940119760479044, 'mean_word_length': 3.426048565121413, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2507012', 'n_tokens_mistral': 655, 'n_tokens_neox': 648, 'n_words': 165}