<commit_before>from setuptools import setup, Extension
from setuptools.command.build_ext import build_ext
from setuptools import find_packages

import os
import sys
import subprocess

CLASSIFIERS = [
    "Development Status :: 4 - Beta",
    "Intended Audience :: Science/Research",
    "License :: OSI Approved :: MIT License",
    "Programming Language :: C++",
    "Programming Language :: Python",
    "Programming Language :: Python :: 3",
    "Topic :: Scientific/Engineering :: Artificial Intelligence",
    "Operating System :: Microsoft :: Windows",
    "Operating System :: POSIX",
    "Operating System :: Unix",
    "Operating System :: MacOS"
]

class CMakeExtension(Extension):
    def __init__(self, name, sourcedir='', extra_cmake_args='', extra_build_args=''):
        Extension.__init__(self, name, sources=[])
        self.sourcedir = os.path.abspath(sourcedir)
        self.extra_cmake_args = extra_cmake_args.split()
        self.extra_build_args = extra_build_args.split()
<commit_msg>FIX: missing platform import<commit_after>from setuptools import setup, Extension
from setuptools.command.build_ext import build_ext
from setuptools import find_packages

import os
import sys
import subprocess
import platform

CLASSIFIERS = [
    "Development Status :: 4 - Beta",
    "Intended Audience :: Science/Research",
    "License :: OSI Approved :: MIT License",
    "Programming Language :: C++",
    "Programming Language :: Python",
    "Programming Language :: Python :: 3",
    "Topic :: Scientific/Engineering :: Artificial Intelligence",
    "Operating System :: Microsoft :: Windows",
    "Operating System :: POSIX",
    "Operating System :: Unix",
    "Operating System :: MacOS"
]

# taken from github.com/pybind/cmake_example
class CMakeExtension(Extension):
    def __init__(self, name, sourcedir='', extra_cmake_args='', extra_build_args=''):
        Extension.__init__(self, name, sources=[])
        self.sourcedir = os.path.abspath(sourcedir)
        self.extra_cmake_args = extra_cmake_args.split()
        self.extra_build_args = extra_build_args.split()

{'fraction_non_alphanumeric': 0.11626794258373206, 'fraction_numerical': 0.0019138755980861245, 'mean_word_length': 4.214463840399002, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '430453', 'n_tokens_mistral': 628, 'n_tokens_neox': 602, 'n_words': 160}