<commit_before><?php
namespace Texts;

/**
 * Class Common
 * @package Texts
 * @author <NAME> <<EMAIL>>
 */
class Common
{
    /**
     * Local cache for stop-words.
     *
     * @var array
     */
    protected static $stopWords = array();

    /**
     * Create short announce for text with title.
     *
     * @param string $title
     * @param string $content
     * @param integer $returnSentances
     * @return string
     */
    public static function annotate($title = '', $content = '', $returnSentances = 3)
    {
        $replaces = array(
            '#'.preg_replace('#\s+#u', '\\s+', $title).'#iu' => '',
            '#\s+#iu'                                        => ' ',
            '#<table.*</table>#u'                            => '',
            '#<code.*</code>#u'                              => '',
        );
        $stopWords     = self::getStopWords('ru');
        $content       = preg_replace(array_keys($replaces), array_values($replaces), $content);
        $content       = strip_tags($content);
        $content_split = preg_replace('#[.!?]\s+([А-ЯA-Z0-9])#u', '<sent>\\1', $content);
        $sentences     = preg_split('#<sent>#iu', $content_split, -1, PREG_SPLIT_NO_EMPTY);
        $title         = ' ' . $title . ' ';

        $replaces = array(
            '#\s#'                                => '  ',
            '#[^a-zа-я][a-zа-я]{1,3}[^a-zа-я]#iu' => '',
            '#[^a-zа-я\s]#iu'                     => ' ',
            '#\s+#iu'                             => ' ',
        );

        $title   = preg_replace(array_keys($replaces), array_values($replaces), $title);
        $words   = preg_split('#\s+#iu', $title);
        $words   = array_map(array(get_called_class(), 'stemWord'), $words);
        $words   = array_filter($words, function ($item) use ($stopWords) {
            return !in_array($item, $stopWords);
        });
        $matches = array();

        foreach ($words as $word) {<commit_msg>Convert line breaks to sentence delimiters.<commit_after><?php
namespace Texts;

/**
 * Class Common
 * @package Texts
 * @author <NAME> <<EMAIL>>
 */
class Common
{
    /**
     * Local cache for stop-words.
     *
     * @var array
     */
    protected static $stopWords = array();

    /**
     * Create short announce for text with title.
     *
     * @param string $title
     * @param string $content
     * @param integer $returnSentances
     * @return string
     */
    public static function annotate($title = '', $content = '', $returnSentances = 3)
    {
        $replaces = array(
            '#'.preg_replace('#\s+#u', '\\s+', $title).'#iu' => '',
            '#\s+#iu'                                        => ' ',
            '#<table.*</table>#u'                            => '',
            '#<code.*</code>#u'                              => '',
        );
        $stopWords     = self::getStopWords('ru');
        $content       = preg_replace(array_keys($replaces), array_values($replaces), $content);
        $content       = strip_tags($content);
        $content_split = preg_replace('#[.!?\s]\s+([А-ЯA-Z0-9])#u', '<sent>\\1', $content);
        $sentences     = preg_split('#<sent>#iu', $content_split, -1, PREG_SPLIT_NO_EMPTY);
        $title         = ' ' . $title . ' ';

        $replaces = array(
            '#\s#'                                => '  ',
            '#[^a-zа-я][a-zа-я]{1,3}[^a-zа-я]#iu' => '',
            '#[^a-zа-я\s]#iu'                     => ' ',
            '#\s+#iu'                             => ' ',
        );

        $title   = preg_replace(array_keys($replaces), array_values($replaces), $title);
        $words   = preg_split('#\s+#iu', $title);
        $words   = array_map(array(get_called_class(), 'stemWord'), $words);
        $words   = array_filter($words, function ($item) use ($stopWords) {
            return !in_array($item, $stopWords);
        });
        $matches = array();

        foreach ($words as $word) {
{'fraction_non_alphanumeric': 0.19009100101112233, 'fraction_numerical': 0.003538928210313448, 'mean_word_length': 1.6067193675889329, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2601494', 'n_tokens_mistral': 1268, 'n_tokens_neox': 1243, 'n_words': 217}