<commit_before>{
  "_id": "_design/all",

  "validate_doc_update": "function(newDoc, oldDoc, userCtx) { if (newDoc['@type'] === 'WebPage') { if (!newDoc.url) { throw({forbidden: 'Document must have a url.'}); } } if (newDoc['@type'] === 'Identity') { if (!newDoc['@id'] || !newDoc['password']) { throw({forbidden: 'User must have a username and password.'}); } } }",

  "views": {
    "urls": {
      "map": "function(doc) { if (doc.url) { emit(doc.url, doc); } if (doc.hasPart && doc.hasPart.isBasedOnUrl) {emit(doc.hasPart.isBasedOnUrl, doc);} if (doc.hasPart && doc.hasPart.associatedMedia && doc.hasPart.associatedMedia.contentUrl) {emit(doc.hasPart.associatedMedia.contentUrl, doc);} }"
    },
    "by_url": {
      "map": "function(doc) { if (doc.url) { emit([doc.url, doc._id], doc); } }"
    },
    "domains": {
      "map": "function(doc) { if(doc['@type'] === 'WebPage') { if (doc.url) { var domain = doc.url.match(/^[a-z][a-z0-9+\\-.]*:\\/\\/([a-z0-9\\-._~%!$&'()*+,;=]+@)?([a-z0-9\\-._~%]+|\\[[a-z0-9\\-._~%!$&'()*+,;=:]+\\])/)[2]; if (domain) { emit(domain, 1); } } } }",
      "reduce": "_count"
    },
    "webpages": {
      "map": "function(doc) { if(doc['@type'] === 'WebPage') { emit(doc._id,doc); } }"
    },<commit_msg>Cut www from domains mapreduce<commit_after>{
  "_id": "_design/all",

  "validate_doc_update": "function(newDoc, oldDoc, userCtx) { if (newDoc['@type'] === 'WebPage') { if (!newDoc.url) { throw({forbidden: 'Document must have a url.'}); } } if (newDoc['@type'] === 'Identity') { if (!newDoc['@id'] || !newDoc['password']) { throw({forbidden: 'User must have a username and password.'}); } } }",

  "views": {
    "urls": {
      "map": "function(doc) { if (doc.url) { emit(doc.url, doc); } if (doc.hasPart && doc.hasPart.isBasedOnUrl) {emit(doc.hasPart.isBasedOnUrl, doc);} if (doc.hasPart && doc.hasPart.associatedMedia && doc.hasPart.associatedMedia.contentUrl) {emit(doc.hasPart.associatedMedia.contentUrl, doc);} }"
    },
    "by_url": {
      "map": "function(doc) { if (doc.url) { emit([doc.url, doc._id], doc); } }"
    },
    "domains": {
      "map": "function(doc) { if(doc['@type'] === 'WebPage') { if (doc.url) { var domain = doc.url.match(/^[a-z][a-z0-9+\\-.]*:\\/\\/([a-z0-9\\-._~%!$&'()*+,;=]+@)?([a-z0-9\\-._~%]+|\\[[a-z0-9\\-._~%!$&'()*+,;=:]+\\])/)[2]; if (domain) { emit(domain.replace(/^www./g, ''), 1); } } } }",
      "reduce": "_count"
    },
    "webpages": {
      "map": "function(doc) { if(doc['@type'] === 'WebPage') { emit(doc._id,doc); } }"
    },
{'fraction_non_alphanumeric': 0.29614926558158, 'fraction_numerical': 0.007939658594680429, 'mean_word_length': 3.7191011235955056, 'pattern_counts': {'":': 24, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '464275', 'n_tokens_mistral': 1025, 'n_tokens_neox': 1008, 'n_words': 166}