<commit_before>'use strict';

/**
 * Dependencies.
 */

var emoji,
    nlcstToString;

emoji = require('./data/emoji.json');
nlcstToString = require('nlcst-to-string');

/**
 * Cached methods.
 */

var has;

has = Object.prototype.hasOwnProperty;

/**
 * Constants: node types.
 */

var EMOTICON_NODE;

EMOTICON_NODE = 'EmoticonNode';

/**
 * Constants: magic numbers.
 *
 * Gemoji's are treated by a parser as multiple nodes.
 * Because this modifier walks backwards, the first colon
 * never matches a gemoji it would normaly walk back to
 * the beginning (the first node). However, because the
 * longest gemoji is tokenized as `Punctuation` (`:`),
 * `Punctuation` (`+`), `Word` (`1`), and `Punctuation`
 * (`:`), we can safely break when the modifier walked
 * back more than 4 times.
 */

var MAX_GEMOJI_PART_COUNT;

MAX_GEMOJI_PART_COUNT = 4;

/**
 * Constants for emoji.
 */

var index,
    names,
    shortcodes,
    unicodes,
    unicodeKeys;

names = emoji.names;
unicodeKeys = emoji.unicode;

/**
 * Quick access to short-codes.
 */

unicodes = {};

index = -1;

while (unicodeKeys[++index]) {
    unicodes[unicodeKeys[index]] = true;
}

shortcodes = {};

index = -1;

while (names[++index]) {
    shortcodes[':' + names[index] + ':'] = true;
}

/**
 * Merge emoji and github-emoji (punctuation marks,
 * symbols, and words) into an `EmoticonNode`.
 *
 * @param {CSTNode} child
 * @param {number} index
 * @param {CSTNode} parent
 * @return {undefined|number} - Either void, or the
 *   next index to iterate over.
 */

function mergeEmoji(child, index, parent) {
    var siblings,
        siblingIndex,
        node,
        nodes,
        value;

    siblings = parent.children;

    if (child.type === 'WordNode') {
        value = nlcstToString(child);

        /**
         * Sometimes a unicode emoji is marked as a
         * word. Mark it as an `EmoticonNode`.
         */

        if (has.call(unicodes, value)) {
            siblings[index] = {
                'type': EMOTICON_NODE,
                'value': value
            };
        } else {
            /**
             * Sometimes a unicode emoji is split in two.
             * Remove the last and add its value to
             * the first.
             */

            node = siblings[index - 1];

            if (
                node &&
                has.call(unicodes, nlcstToString(node) + value)
            ) {
                node.type = EMOTICON_NODE;
                node.value = nlcstToString(node) + value;

                siblings.splice(index, 1);

                return index;
            }
        }
    } else if (has.call(unicodes, nlcstToString(child))) {
        child.type = EMOTICON_NODE;
    } else if (nlcstToString(child) === ':') {
        nodes = [];
        siblingIndex = index;

        while (siblingIndex--) {
            if ((index - siblingIndex) >= MAX_GEMOJI_PART_COUNT) {
                return;
            }
<commit_msg>Fix bug for emoticons embedded in words<commit_after>'use strict';

/**
 * Dependencies.
 */

var emoji,
    nlcstToString;

emoji = require('./data/emoji.json');
nlcstToString = require('nlcst-to-string');

/**
 * Cached methods.
 */

var has;

has = Object.prototype.hasOwnProperty;

/**
 * Constants: node types.
 */

var EMOTICON_NODE;

EMOTICON_NODE = 'EmoticonNode';

/**
 * Constants: magic numbers.
 *
 * Gemoji's are treated by a parser as multiple nodes.
 * Because this modifier walks backwards, the first colon
 * never matches a gemoji it would normaly walk back to
 * the beginning (the first node). However, because the
 * longest gemoji is tokenized as `Punctuation` (`:`),
 * `Punctuation` (`+`), `Word` (`1`), and `Punctuation`
 * (`:`), we can safely break when the modifier walked
 * back more than 4 times.
 */

var MAX_GEMOJI_PART_COUNT;

MAX_GEMOJI_PART_COUNT = 12;

/**
 * Constants for emoji.
 */

var index,
    names,
    shortcodes,
    unicodes,
    unicodeKeys;

names = emoji.names;
unicodeKeys = emoji.unicode;

/**
 * Quick access to short-codes.
 */

unicodes = {};

index = -1;

while (unicodeKeys[++index]) {
    unicodes[unicodeKeys[index]] = true;
}

shortcodes = {};

index = -1;

while (names[++index]) {
    shortcodes[':' + names[index] + ':'] = true;
}

/**
 * Merge emoji and github-emoji (punctuation marks,
 * symbols, and words) into an `EmoticonNode`.
 *
 * @param {CSTNode} child
 * @param {number} index
 * @param {CSTNode} parent
 * @return {undefined|number} - Either void, or the
 *   next index to iterate over.
 */

function mergeEmoji(child, index, parent) {
    var siblings,
        siblingIndex,
        node,
        nodes,
        value;

    siblings = parent.children;

    if (child.type === 'WordNode') {
        value = nlcstToString(child);

        /**
         * Sometimes a unicode emoji is marked as a
         * word. Mark it as an `EmoticonNode`.
         */

        if (has.call(unicodes, value)) {
            siblings[index] = {
                'type': EMOTICON_NODE,
                'value': value
            };
        } else {
            /**
             * Sometimes a unicode emoji is split in two.
             * Remove the last and add its value to
             * the first.
             */

            node = siblings[index - 1];

            if (
                node &&
                has.call(unicodes, nlcstToString(node) + value)
            ) {
                node.type = EMOTICON_NODE;
                node.value = nlcstToString(node) + value;

                siblings.splice(index, 1);

                return index;
            }
        }
    } else if (has.call(unicodes, nlcstToString(child))) {
        child.type = EMOTICON_NODE;
    } else if (nlcstToString(child) === ':') {
        nodes = [];
        siblingIndex = index;

        while (siblingIndex--) {
            if ((index - siblingIndex) > MAX_GEMOJI_PART_COUNT) {
                return;
            }

{'fraction_non_alphanumeric': 0.1280446549391069, 'fraction_numerical': 0.002537212449255751, 'mean_word_length': 2.0495100567302735, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2778687', 'n_tokens_mistral': 1915, 'n_tokens_neox': 1888, 'n_words': 500}