<commit_before>{
  "division": "Sogou",
  "sortIndex": 220,
  "userAgents": [
    {
      "userAgent": "Sogou",
      "properties": {
        "Parent": "DefaultProperties",
        "Comment": "Sogou",
        "Browser": "Sogou",
        "Browser_Type": "Bot/Crawler",
        "Frames": "true",
        "Tables": "true",
        "Crawler": "true"
      }
    },
    {
      "userAgent": "shaboyi spider",
      "properties": {
        "Parent": "Sogou",
        "Browser": "Sogou\/Shaboyi Spider"
      }
    },
    {
      "userAgent": "Sogou develop spider\/*",
      "properties": {
        "Parent": "Sogou",
        "Browser": "Sogou Develop Spider"
      }
    },
    {
      "userAgent": "Sogou head spider*",
      "properties": {
        "Parent": "Sogou",
        "Browser": "Sogou Head Spider"
      }
    },
    {
      "userAgent": "sogou js robot(*)",
      "properties": {
        "Parent": "Sogou"
      }
    },
    {
      "userAgent": "Sogou Orion spider\/*",
      "properties": {
        "Parent": "Sogou",
        "Browser": "Sogou Orion spider"
      }
    },
    {
      "userAgent": "Sogou Pic Agent",
      "properties": {
        "Parent": "Sogou",
        "Browser": "Sogou\/Image Crawler"
      }
    },
    {
      "userAgent": "Sogou Pic Spider\/*",
      "properties": {
        "Parent": "Sogou",
        "Browser": "Sogou Pic Spider"
      }
    },
    {
      "userAgent": "Sogou Push Spider\/*",
      "properties": {
        "Parent": "Sogou",
        "Browser": "Sogou Push Spider"
      }
    },
    {
      "userAgent": "sogou spider",
      "properties": {
        "Parent": "Sogou",
        "Browser": "Sogou\/Spider"
      }
    },
    {
      "userAgent": "sogou web spider*",
      "properties": {
        "Parent": "Sogou",
        "Browser": "sogou web spider"
      }
    },
    {
      "userAgent": "Sogou web spider*",
      "properties": {
        "Parent": "Sogou",
        "Browser": "sogou web spider"
      }
    },
    {
      "userAgent": "Sogou web spider/4.0*",
      "properties": {
        "Parent": "Sogou",
        "Browser": "sogou web spider",
        "Version": "4.0",
        "MajorVer": "4",
        "MinorVer": "0"
      }
    },
    {
      "userAgent": "Sogou-Test-Spider\/*",<commit_msg>remove duplicate rule definitions<commit_after>{
  "division": "Sogou",
  "sortIndex": 220,
  "userAgents": [
    {
      "userAgent": "Sogou",
      "properties": {
        "Parent": "DefaultProperties",
        "Comment": "Sogou",
        "Browser": "Sogou",
        "Browser_Type": "Bot/Crawler",
        "Frames": "true",
        "Tables": "true",
        "Crawler": "true"
      }
    },
    {
      "userAgent": "shaboyi spider",
      "properties": {
        "Parent": "Sogou",
        "Browser": "Sogou\/Shaboyi Spider"
      }
    },
    {
      "userAgent": "Sogou develop spider\/*",
      "properties": {
        "Parent": "Sogou",
        "Browser": "Sogou Develop Spider"
      }
    },
    {
      "userAgent": "Sogou head spider*",
      "properties": {
        "Parent": "Sogou",
        "Browser": "Sogou Head Spider"
      }
    },
    {
      "userAgent": "sogou js robot(*)",
      "properties": {
        "Parent": "Sogou"
      }
    },
    {
      "userAgent": "Sogou Orion spider\/*",
      "properties": {
        "Parent": "Sogou",
        "Browser": "Sogou Orion spider"
      }
    },
    {
      "userAgent": "Sogou Pic Agent",
      "properties": {
        "Parent": "Sogou",
        "Browser": "Sogou\/Image Crawler"
      }
    },
    {
      "userAgent": "Sogou Pic Spider\/*",
      "properties": {
        "Parent": "Sogou",
        "Browser": "Sogou Pic Spider"
      }
    },
    {
      "userAgent": "Sogou Push Spider\/*",
      "properties": {
        "Parent": "Sogou",
        "Browser": "Sogou Push Spider"
      }
    },
    {
      "userAgent": "sogou spider",
      "properties": {
        "Parent": "Sogou",
        "Browser": "Sogou\/Spider"
      }
    },
    {
      "userAgent": "Sogou-Test-Spider\/*",
{'fraction_non_alphanumeric': 0.18768693918245263, 'fraction_numerical': 0.0029910269192422734, 'mean_word_length': 1.6896782841823057, 'pattern_counts': {'":': 111, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5616847', 'n_tokens_mistral': 1451, 'n_tokens_neox': 1403, 'n_words': 269}