<commit_before>{
  "partialSitemapsPath": "/arena/solifugae_data/sitemaps/",
  "indexSitemapPath": "/arena/solifugae_data/",
  "indexSitemapName": "sitemap.xml",
  "s3": {
    "use": true,
    "bucket": "onefootball-solifugae",
    "partialSitemapsPath": "sitemaps/",
    "indexSitemapPath": "",
    "queue": "data/queue.json"
  },
  "crawler": {
    "entryDomain": "www.onefootball.com",
    "entryPath": "/",
    "queue": "/arena/solifugae_data/queue.json",
    "entryProtocol": "https",
    "prefixFolder": false,
    "fetchInterval": 1000,
    "concurrentConnections": 5,
    "defaultPriority": 0.5,
    "defaultFreq": "daily",
    "saveFrequency": 50,
    "allowedFolders": [
      "/ar",
      "/br",
      "/cl",
      "/co",
      "/de",
      "/en",
      "/es",
      "/fr",
      "/it",
      "/mx",
      "/nl"
    ],
    "userAgent": "Solifugae (compatible; Googlebot/2.1;)",
    "stripUrlParams": true,
    "folderDepth": 1,
    "timeToWait": 1000,
    "maxFileSize": 5
  }
}<commit_msg>update defs<commit_after>{
  "partialSitemapsPath": "/arena/solifugae_data/sitemaps/",
  "indexSitemapPath": "/arena/solifugae_data/",
  "indexSitemapName": "sitemap.xml",
  "s3": {
    "use": true,
    "bucket": "onefootball-solifugae",
    "partialSitemapsPath": "sitemaps/",
    "indexSitemapPath": "",
    "queue": "data/queue.json"
  },
  "crawler": {
    "entryDomain": "www.onefootball.com",
    "entryPath": "/",
    "queue": "/arena/solifugae_data/queue.json",
    "entryProtocol": "https",
    "prefixFolder": false,
    "fetchInterval": 1000,
    "concurrentConnections": 5,
    "defaultPriority": 0.5,
    "defaultFreq": "daily",
    "saveFrequency": 1000,
    "allowedFolders": [
      "/ar",
      "/br",
      "/cl",
      "/co",
      "/de",
      "/en",
      "/es",
      "/fr",
      "/it",
      "/mx",
      "/nl"
    ],
    "userAgent": "Solifugae (compatible; Googlebot/2.1;)",
    "stripUrlParams": true,
    "folderDepth": 1,
    "timeToWait": 10000,
    "maxFileSize": 5
  }
}
{'fraction_non_alphanumeric': 0.20668996505242138, 'fraction_numerical': 0.019470793809286072, 'mean_word_length': 2.697416974169742, 'pattern_counts': {'":': 52, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1502192', 'n_tokens_mistral': 800, 'n_tokens_neox': 757, 'n_words': 123}