<commit_before>package lexer

import (
  "github.com/Southern/scanner"
  "regexp"
  "strings"
)

var regex = map[string]map[string]scanner.Definition{
  "comments": map[string]scanner.Definition{
    // Hash comments
    "hash": scanner.Definition{regexp.MustCompile("^#(.+)"), "COMMENT"},

    // Single line comments
    "oneline": scanner.Definition{regexp.MustCompile("^\\/{2,}\\s*(.+)"), "COMMENT"},

    // Multi-line comments
    "multiline": scanner.Definition{regexp.MustCompile("^/\\*([^*]|[\r\n]|(\\*+([^*/]|[\r\n])))*\\*+/"), "COMMENT"},
  },

  "strings": map[string]scanner.Definition{
    // Double quote strings
    "double": scanner.Definition{regexp.MustCompile("^\"([^\"\\n]|\\\")+\""), "STRING"},

    // Single quote strings
    "single": scanner.Definition{regexp.MustCompile("^'([^'\\n]|\\')+'"), "STRING"},
  },

  "operators": map[string]scanner.Definition{
    // Common operators found in almost all languages
    "common": scanner.Definition{regexp.MustCompile(strings.Join([]string{
      "^(",
      strings.Join([]string{
        // ++, --, &&, ||
        "([+\\-&|]{2})",
        // !, !=, <, <<, <=, <<=, >, >>, >>>, >=, >>=, >>>=, ^, ^=, +, +=, -,
        // -=, %, %=, *, *=
        "((<{2}|>{2,3}|[!|&<>^+\\-=%/*])=?)",
      }, "|"),
      ")",
    }, "")), "OPERATOR"},
  },
}<commit_msg>Strings shouldn't require text inside of them.<commit_after>package lexer

import (
  "github.com/Southern/scanner"
  "regexp"
  "strings"
)

var regex = map[string]map[string]scanner.Definition{
  "comments": map[string]scanner.Definition{
    // Hash comments
    "hash": scanner.Definition{regexp.MustCompile("^#(.+)"), "COMMENT"},

    // Single line comments
    "oneline": scanner.Definition{regexp.MustCompile("^\\/{2,}\\s*(.+)"), "COMMENT"},

    // Multi-line comments
    "multiline": scanner.Definition{regexp.MustCompile("^/\\*([^*]|[\r\n]|(\\*+([^*/]|[\r\n])))*\\*+/"), "COMMENT"},
  },

  "strings": map[string]scanner.Definition{
    // Double quote strings
    "double": scanner.Definition{regexp.MustCompile("^\"([^\"\\n]|\\\")*\""), "STRING"},

    // Single quote strings
    "single": scanner.Definition{regexp.MustCompile("^'([^'\\n]|\\')*'"), "STRING"},
  },

  "operators": map[string]scanner.Definition{
    // Common operators found in almost all languages
    "common": scanner.Definition{regexp.MustCompile(strings.Join([]string{
      "^(",
      strings.Join([]string{
        // ++, --, &&, ||
        "([+\\-&|]{2})",
        // !, !=, <, <<, <=, <<=, >, >>, >>>, >=, >>=, >>>=, ^, ^=, +, +=, -,
        // -=, %, %=, *, *=
        "((<{2}|>{2,3}|[!|&<>^+\\-=%/*])=?)",
      }, "|"),
      ")",
    }, "")), "OPERATOR"},
  },
}
{'fraction_non_alphanumeric': 0.3031657355679702, 'fraction_numerical': 0.0037243947858473, 'mean_word_length': 3.0574018126888216, 'pattern_counts': {'":': 18, '<': 19, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7333361', 'n_tokens_mistral': 1018, 'n_tokens_neox': 956, 'n_words': 118}