<commit_before>import nltk
import re
import shutil
import os
from urllib.parse import urlparse

from coalib.bears.GlobalBear import GlobalBear
from dependency_management.requirements.PipRequirement import PipRequirement
from coala_utils.ContextManagers import change_directory
from coalib.misc.Shell import run_shell_command
from coalib.results.Result import Result
from coalib.settings.FunctionMetadata import FunctionMetadata
from coalib.settings.Setting import typed_list


class GitCommitBear(GlobalBear):
    LANGUAGES = {'Git'}
    REQUIREMENTS = {PipRequirement('nltk', '3.2')}
    AUTHORS = {'The coala developers'}
    AUTHORS_EMAILS = {'<EMAIL>'}
    LICENSE = 'AGPL-3.0'
    ASCIINEMA_URL = 'https://asciinema.org/a/e146c9739ojhr8396wedsvf0d'
    CAN_DETECT = {'Formatting'}
    SUPPORTED_HOST_KEYWORD_REGEX = {
        'github': (r'[Cc]lose[sd]?'
                   r'|[Rr]esolve[sd]?'
                   r'|[Ff]ix(?:e[sd])?'),
        'gitlab': (r'[Cc]los(?:e[sd]?|ing)'
                   r'|[Rr]esolv(?:e[sd]?|ing)'
                   r'|[Ff]ix(?:e[sd]|ing)?')
    }
    CONCATENATION_KEYWORDS = [r',', r'\sand\s']

    _nltk_data_downloaded = False

    def setup_dependencies(self):
        if not self._nltk_data_downloaded and bool(
                self.section.get('shortlog_imperative_check', True)):
            nltk.download([
                'punkt',
                'averaged_perceptron_tagger',
            ])
            type(self)._nltk_data_downloaded = True
<commit_msg>GitCommitBear: Redirect excess output from nltk<commit_after>import nltk
import re
import shutil
import os
import logging
from urllib.parse import urlparse
from contextlib import redirect_stdout

from coalib.bears.GlobalBear import GlobalBear
from dependency_management.requirements.PipRequirement import PipRequirement
from coala_utils.ContextManagers import change_directory
from coalib.misc.Shell import run_shell_command
from coalib.results.Result import Result
from coalib.settings.FunctionMetadata import FunctionMetadata
from coalib.settings.Setting import typed_list


class GitCommitBear(GlobalBear):
    LANGUAGES = {'Git'}
    REQUIREMENTS = {PipRequirement('nltk', '3.2')}
    AUTHORS = {'The coala developers'}
    AUTHORS_EMAILS = {'<EMAIL>'}
    LICENSE = 'AGPL-3.0'
    ASCIINEMA_URL = 'https://asciinema.org/a/e146c9739ojhr8396wedsvf0d'
    CAN_DETECT = {'Formatting'}
    SUPPORTED_HOST_KEYWORD_REGEX = {
        'github': (r'[Cc]lose[sd]?'
                   r'|[Rr]esolve[sd]?'
                   r'|[Ff]ix(?:e[sd])?'),
        'gitlab': (r'[Cc]los(?:e[sd]?|ing)'
                   r'|[Rr]esolv(?:e[sd]?|ing)'
                   r'|[Ff]ix(?:e[sd]|ing)?')
    }
    CONCATENATION_KEYWORDS = [r',', r'\sand\s']

    _nltk_data_downloaded = False

    def setup_dependencies(self):
        if not self._nltk_data_downloaded and bool(
                self.section.get('shortlog_imperative_check', True)):
            logger = logging.getLogger()
            logger.write = lambda msg: logger.debug(
                msg) if msg != '\n' else None
            with redirect_stdout(logger):
                nltk.download([
                    'punkt',
                    'averaged_perceptron_tagger'
                ])
                type(self)._nltk_data_downloaded = True

{'fraction_non_alphanumeric': 0.11918516266342354, 'fraction_numerical': 0.00972940103374886, 'mean_word_length': 2.9213349225268175, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1338016', 'n_tokens_mistral': 1153, 'n_tokens_neox': 1097, 'n_words': 199}