<commit_before>_ = require 'underscore'
_s = require 'underscore.string'

module.exports =

    # string manipulation
    # -------------------

    umlautReplacementMap:
        ae: /[äÄ]/g
        oe: /[öÖ]/g
        ue: /[üÜ]/g

    replaceUmlauts: (string) ->
        replaced = string
        for replacement, umlaut of module.exports.umlautReplacementMap
            do (replacement, umlaut) ->
                replaced = replaced.replace umlaut, replacement
        return replaced

    removeNonAlphanumeric: (string) ->
        string.replace /[^a-z0-9]/g, ''

    inits: (string, minPrefixLen = 1) ->
        throw new Error 'minPrefixLen must be > 0' if minPrefixLen < 1
        return [] if string is '' or minPrefixLen > string.length
        [minPrefixLen..string.length].map (i) -> string.substr 0, i

    tokenize: (string) ->

        # make lowercase
        # replace umlauts
        # split into words (split by whitespace)
        # remove non alphanumeric chars
        # remove empty words
        # remove duplicates

        lowerCase = string.toLowerCase().replace /[\-_]/g, ' '

        words = _s.words module.exports.replaceUmlauts lowerCase

        processed = words.map module.exports.removeNonAlphanumeric

        _.uniq processed.filter (word) -> word isnt ''<commit_msg>improve tokenize function<commit_after>_ = require 'underscore'
_s = require 'underscore.string'

module.exports =

    # string manipulation
    # -------------------

    umlautReplacementMap:
        ae: /[äÄ]/g
        oe: /[öÖ]/g
        ue: /[üÜ]/g

    replaceUmlauts: (string) ->
        replaced = string
        for replacement, umlaut of module.exports.umlautReplacementMap
            do (replacement, umlaut) ->
                replaced = replaced.replace umlaut, replacement
        return replaced

    removeNonAlphanumeric: (string) ->
        string.replace /[^a-z0-9]/g, ''

    inits: (string, minPrefixLen = 1) ->
        throw new Error 'minPrefixLen must be > 0' if minPrefixLen < 1
        return [] if string is '' or minPrefixLen > string.length
        [minPrefixLen..string.length].map (i) -> string.substr 0, i

    tokenize: (string) ->

        string = string.toLowerCase()

        # replace - and _ by whitespace to later break words on it
        string = string.replace /[\-_]/g, ' '

        string = module.exports.replaceUmlauts string

        # split into words
        words = _s.words string

        words = words.map module.exports.removeNonAlphanumeric

        # remove empty words
        words = words.filter (word) -> word isnt ''

        # remove duplicates
        _.uniq words
{'fraction_non_alphanumeric': 0.11684370257966616, 'fraction_numerical': 0.004552352048558422, 'mean_word_length': 2.0380184331797233, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4466582', 'n_tokens_mistral': 802, 'n_tokens_neox': 761, 'n_words': 207}