<commit_before>function(doc) {
  var output = new Document();
  var valid_fields = {
    mod_datetime: {
        field: "mod_datetime",
        store: "yes",
        index: "analyzed"
    },

    title: {
      field: "title",
      store: "yes",
      index: "analyzed"
    },

    url: {
        field: "url",
        store: "yes",
        index: "analyzed"
    },
    
    contents:       {
        field: "contents",
        store: "no",
        index: "analyzed"
    }
  };
  
  for (var field in valid_fields) {
    output.add(doc[field], valid_fields[field]);
  }
  
  output.add(
    doc["title"] + "\n" + doc["contents"], 
    {field: "default", store: "no", index: "analyzed"}
  );
  
  return output;
}<commit_msg>store a snippet of the contents to display with search results<commit_after>function(doc) {
  var output = new Document();
  var valid_fields = {
    mod_datetime: {
        field: "mod_datetime",
        store: "yes",
        index: "analyzed"
    },

    title: {
      field: "title",
      store: "yes",
      index: "analyzed"
    },

    url: {
        field: "url",
        store: "yes",
        index: "analyzed"
    },
    
    contents:       {
        field: "contents",
        store: "no",
        index: "analyzed"
    }
  };
  
  for (var field in valid_fields) {
    output.add(doc[field], valid_fields[field]);
  }
  
  output.add(
    doc["title"] + "\n" + doc["contents"], 
    {field: "default", store: "no", index: "analyzed"}
  );
  
  output.add(doc["contents"].substr(0, 140), {field: "snippet", store: "yes", index: "no"});
  
  return output;
}
{'fraction_non_alphanumeric': 0.16551724137931034, 'fraction_numerical': 0.0025078369905956114, 'mean_word_length': 2.057471264367816, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4415198', 'n_tokens_mistral': 549, 'n_tokens_neox': 536, 'n_words': 126}