<commit_before>katasemeion.lexer = (function(tokens) {
    var self = this;

    var ifCharIs = function(character) {
        return {
            thenReturn: function(thenToken) {
                var _elseToken = function() { return null; };
                var tokenize = function(sourceStream) {
                    if (sourceStream.current === character) {
                        sourceStream.advanceCursor();
                        return thenToken();
                    }
                    return _elseToken();
                };
                tokenize.elseReturn = function(elseToken) {
                    _elseToken = elseToken;
                    return tokenize;
                };
                tokenize.unlessFollowedBy = function(alternateChar) {
                    return {
                        thenReturn: function(alternateToken) {
                            return function(sourceStream) {
                                return ifCharIs(character).
                                       thenReturn(function() {
                                           return ifCharIs(alternateChar).
                                                  thenReturn(alternateToken).
                                                  elseReturn(thenToken)(sourceStream);
                                       })(sourceStream);
                            };
                        }
                    };
                };
                tokenize.elifDoubledReturn = function(doubledToken) {
                    return tokenize.unlessFollowedBy(character)
                                   .thenReturn(doubledToken);
                };
                return tokenize;
            }
        };
    };

    self.tokenizeOpenBracket = ifCharIs('[').
                               thenReturn(tokens.OpenBracket).
                               elifDoubledReturn(tokens.DoubleOpenBracket);
    self.tokenizeCloseBracket = ifCharIs(']').
                                thenReturn(tokens.CloseBracket).
                                elifDoubledReturn(tokens.DoubleCloseBracket);
    self.tokenizeUnderscore = ifCharIs('_').
                              thenReturn(tokens.Underscore);
    self.tokenizeAsterisk = ifCharIs('*').
                            thenReturn(tokens.Asterisk).
                            elifDoubledReturn(tokens.DoubleAsterisk);
    self.tokenizeAt = ifCharIs('@').<commit_msg>Reduced indentation in tokenizer factory function<commit_after>katasemeion.lexer = (function(tokens) {
    var self = this;

    var ifCharIs = function(character) {
        var thenReturn = function(thenToken) {
            var _elseToken = function() { return null; };
            var tokenize = function(sourceStream) {
                if (sourceStream.current === character) {
                    sourceStream.advanceCursor();
                    return thenToken();
                }
                return _elseToken();
            };
            tokenize.elseReturn = function(elseToken) {
                _elseToken = elseToken;
                return tokenize;
            };
            tokenize.unlessFollowedBy = function(alternateChar) {
                return {
                    thenReturn: function(alternateToken) {
                        return function(sourceStream) {
                            return ifCharIs(character).
                                   thenReturn(function() {
                                       return ifCharIs(alternateChar).
                                              thenReturn(alternateToken).
                                              elseReturn(thenToken)(sourceStream);
                                   })(sourceStream);
                        };
                    }
                };
            };
            tokenize.elifDoubledReturn = function(doubledToken) {
                return tokenize.unlessFollowedBy(character)
                               .thenReturn(doubledToken);
            };
            return tokenize;
        };
        return {
            thenReturn: thenReturn
        };
    };

    self.tokenizeOpenBracket = ifCharIs('[').
                               thenReturn(tokens.OpenBracket).
                               elifDoubledReturn(tokens.DoubleOpenBracket);
    self.tokenizeCloseBracket = ifCharIs(']').
                                thenReturn(tokens.CloseBracket).
                                elifDoubledReturn(tokens.DoubleCloseBracket);
    self.tokenizeUnderscore = ifCharIs('_').
                              thenReturn(tokens.Underscore);
    self.tokenizeAsterisk = ifCharIs('*').
                            thenReturn(tokens.Asterisk).
                            elifDoubledReturn(tokens.DoubleAsterisk);
    self.tokenizeAt = ifCharIs('@').
{'fraction_non_alphanumeric': 0.08018375443725204, 'fraction_numerical': 0.0, 'mean_word_length': 1.1119929453262787, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1620237', 'n_tokens_mistral': 1074, 'n_tokens_neox': 1012, 'n_words': 151}