<commit_before>#!/usr/bin/python3
# -*- coding: utf-8 -*-
#-------------------------------------------------------------------------------
# This file is part of Pynspect package (https://pypi.python.org/pypi/pynspect).
# Originally part of Mentat system (https://mentat.cesnet.cz/).
#
# Copyright (C) since 2016 CESNET, z.s.p.o (http://www.ces.net/)
# Copyright (C) since 2016 <NAME> <<EMAIL>>
# Use of this source is governed by the MIT license, see LICENSE file.
#-------------------------------------------------------------------------------

# Resources:
#   https://packaging.python.org/en/latest/
#   https://python-packaging.readthedocs.io/en/latest/index.html

import sys
import os

# Always prefer setuptools over distutils
from setuptools import setup, find_packages
# To use a consistent encoding
from codecs import open

#
# Import local version of pynspect library, so that we can insert correct version
# number into documentation.
#
sys.path.insert(0, os.path.abspath('.'))
import pynspect

# Generate parsetab.py in advance so it can go into package.
from pynspect.jpath import *
from pynspect.rules import *
from pynspect.gparser import MentatFilterParser
from pynspect.filters import DataObjectFilter

parser = MentatFilterParser()
parser.build()

here = os.path.abspath(os.path.dirname(__file__))
<commit_msg>Fix: Fixed errors in setup.py file.<commit_after>#!/usr/bin/python3
# -*- coding: utf-8 -*-
#-------------------------------------------------------------------------------
# This file is part of Pynspect package (https://pypi.python.org/pypi/pynspect).
# Originally part of Mentat system (https://mentat.cesnet.cz/).
#
# Copyright (C) since 2016 CESNET, z.s.p.o (http://www.ces.net/)
# Copyright (C) since 2016 <NAME> <<EMAIL>>
# Use of this source is governed by the MIT license, see LICENSE file.
#-------------------------------------------------------------------------------

# Resources:
#   https://packaging.python.org/en/latest/
#   https://python-packaging.readthedocs.io/en/latest/index.html

import sys
import os

# Always prefer setuptools over distutils
from setuptools import setup, find_packages
# To use a consistent encoding
from codecs import open

#
# Import local version of pynspect library, so that we can insert correct version
# number into documentation.
#
sys.path.insert(0, os.path.abspath('.'))
import pynspect

# Generate parsetab.py in advance so it can go into package.
from pynspect.gparser import PynspectFilterParser
parser = PynspectFilterParser()
parser.build()

here = os.path.abspath(os.path.dirname(__file__))

{'fraction_non_alphanumeric': 0.22626888802789616, 'fraction_numerical': 0.008523827973653622, 'mean_word_length': 2.4797843665768196, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 8, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6103491', 'n_tokens_mistral': 767, 'n_tokens_neox': 735, 'n_words': 231}