<commit_before>#!/usr/bin/env python3

from setuptools import setup
from os import path
from setuptools.command.install import install

here = path.abspath(path.dirname(__file__))
readme = path.join(here, 'README.md')


def check_nltk(entries):
    """ checks whether required nltk parts are available """
    import nltk
    for entry in entries:
        try:
            nltk.data.find(entry[0])
        except LookupError:
            print("downloading %s", entry[1])
            nltk.download(entry[1])


class CustomInstall(install):

    def run(self):
        install.run(self)
        check_nltk([('corpora/wordnet', 'wordnet'),
                    ('tokenizers/punkt/english.pickle', 'punkt')])<commit_msg>check for heroku<commit_after>#!/usr/bin/env python3

from setuptools import setup
from os import path
from setuptools.command.install import install
import os

here = path.abspath(path.dirname(__file__))
readme = path.join(here, 'README.md')


def is_heroku():
    return int(os.environ.get("PORT", -1)) > 0


def check_nltk(entries):
    """ checks whether required nltk parts are available """
    import nltk
    for entry in entries:
        try:
            nltk.data.find(entry[0])
        except LookupError:
            print("downloading %s", entry[1])
            nltk.download(entry[1])


class CustomInstall(install):

    def run(self):
        install.run(self)
        if not is_heroku():
            check_nltk([('corpora/wordnet', 'wordnet'),
                        ('tokenizers/punkt/english.pickle', 'punkt')])
{'fraction_non_alphanumeric': 0.12475759534583064, 'fraction_numerical': 0.006464124111182934, 'mean_word_length': 2.8316831683168315, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '430196', 'n_tokens_mistral': 502, 'n_tokens_neox': 488, 'n_words': 114}