<commit_before>define(['lib/codemirror/lib/codemirror', 'underscore'], function(CodeMirror, _) {
(function () {
  "use strict";

  var spec = CodeMirror.resolveMode("text/css");
  var propertyKeywords =_.keys(spec.propertyKeywords);
  var valueKeywords = _.keys(spec.valueKeywords).concat(_.keys(spec.colorKeywords));

  function getHints(cm) {
    var cur = cm.getCursor(), token = cm.getTokenAt(cur);
    var inner = CodeMirror.innerMode(cm.getMode(), token.state);
    if (inner.mode.name != "css") return;

    if ((token.type === 'tag' || token.type === 'qualifier') && /^\./.test(token.string)) {
      token.type = 'class';
    } else if (token.type === 'builtin' && /^#/.test(token.string)) {
      token.type = 'id';
    }
    // If it's not a 'word-style' token, ignore the token.
    else if (!/^[\w$_-]*$/.test(token.string)) {
      token = {
        start: cur.ch, end: cur.ch, string: "", state: token.state,
        type: null
      };
      var stack = token.state.stack;
      var lastToken = stack && stack.length > 0 ? stack[stack.length - 1] : "";
      if (token.string == ":" || lastToken.indexOf("property") == 0)
        token.type = "variable";
      else if (token.string == "{" || lastToken.indexOf("rule") == 0)
        token.type = "property";
    }

    if (!token.type)
      return;

    var keywords = null;
    if (token.type.indexOf("property") == 0)
      keywords = propertyKeywords;
    else if (token.type.indexOf("variable") == 0)
      keywords = valueKeywords;
    else if (token.type === 'id' || token.type === 'class') {
      if (cm.hintprovider.css) {
        keywords = CodeMirror.hintprovider.css(cm, token);
      }
    }

    if (!keywords)
      return;

    function commonstring(str, sub) {
      function calculateCommon(strP, subP) {
        if (subP < sub.length) {
          var n = str.substring(strP).indexOf(sub.charAt(subP));
          if (n < 0) { return; }
          var p = 1;
          while (subP + p < sub.length && str.charAt(strP + n + p) === (sub.charAt(subP + p))) {
            p++;
          }
          if (subP + p >= sub.length) {
            return [{start: strP + n, len: p}];
          } else {
            var rest = calculateCommon(strP + n + p, subP + p);
            if (rest) {
              rest.push({start: strP + n, len: p});
              return rest;
            }
          }
        }
      }
      function calculateRelevance(common) {
        var mcp = _.max(common, function (c) { return c.len; });<commit_msg>minor fix - call hintprovider only when it's not null<commit_after>define(['lib/codemirror/lib/codemirror', 'underscore'], function(CodeMirror, _) {
(function () {
  "use strict";

  var spec = CodeMirror.resolveMode("text/css");
  var propertyKeywords =_.keys(spec.propertyKeywords);
  var valueKeywords = _.keys(spec.valueKeywords).concat(_.keys(spec.colorKeywords));

  function getHints(cm) {
    var cur = cm.getCursor(), token = cm.getTokenAt(cur);
    var inner = CodeMirror.innerMode(cm.getMode(), token.state);
    if (inner.mode.name != "css") return;

    if ((token.type === 'tag' || token.type === 'qualifier') && /^\./.test(token.string)) {
      token.type = 'class';
    } else if (token.type === 'builtin' && /^#/.test(token.string)) {
      token.type = 'id';
    }
    // If it's not a 'word-style' token, ignore the token.
    else if (!/^[\w$_-]*$/.test(token.string)) {
      token = {
        start: cur.ch, end: cur.ch, string: "", state: token.state,
        type: null
      };
      var stack = token.state.stack;
      var lastToken = stack && stack.length > 0 ? stack[stack.length - 1] : "";
      if (token.string == ":" || lastToken.indexOf("property") == 0)
        token.type = "variable";
      else if (token.string == "{" || lastToken.indexOf("rule") == 0)
        token.type = "property";
    }

    if (!token.type)
      return;

    var keywords = null;
    if (token.type.indexOf("property") == 0)
      keywords = propertyKeywords;
    else if (token.type.indexOf("variable") == 0)
      keywords = valueKeywords;
    else if (token.type === 'id' || token.type === 'class') {
      if (cm.hintprovider && cm.hintprovider.css) {
        keywords = CodeMirror.hintprovider.css(cm, token);
      }
    }

    if (!keywords)
      return;

    function commonstring(str, sub) {
      function calculateCommon(strP, subP) {
        if (subP < sub.length) {
          var n = str.substring(strP).indexOf(sub.charAt(subP));
          if (n < 0) { return; }
          var p = 1;
          while (subP + p < sub.length && str.charAt(strP + n + p) === (sub.charAt(subP + p))) {
            p++;
          }
          if (subP + p >= sub.length) {
            return [{start: strP + n, len: p}];
          } else {
            var rest = calculateCommon(strP + n + p, subP + p);
            if (rest) {
              rest.push({start: strP + n, len: p});
              return rest;
            }
          }
        }
      }
      function calculateRelevance(common) {
        var mcp = _.max(common, function (c) { return c.len; });
{'fraction_non_alphanumeric': 0.17294419246696904, 'fraction_numerical': 0.0031551962137645437, 'mean_word_length': 2.210126582278481, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6680754', 'n_tokens_mistral': 1673, 'n_tokens_neox': 1626, 'n_words': 381}