<commit_before>function(doc) {
    var sanatize = function(str){
        str = str.trim()
        var loc = str.lastIndexOf(".");
        if(loc === (str.length -1 )){
            return str.substr(0, loc);
        }
        return str;
    }
    var publishers = ["national security agencysee resources", "consumer product safety commissionsee resources", "bureau of land management, department of the interiorsee resources", "national endowment for the humanitiessee resources", "department of the interiorsee resources", "national endowment for the artssee resources", "department of navysee resources", "u.s. courtssee resources", "federal judicial centersee resources", "securities and exchange commissionsee resources", "u.s. mint, treasurysee resources", "department of energysee resources", "national library of medicinesee resources", "department of commerce, international trade administrationsee resources", "the white housesee resources", "national constitution centersee resources", "the federal reservesee resources", "federal bureau of investigationsee resources", "national archives and records administrationsee resources", "national science foundationsee resources", "national academy of sciencessee resources", "department of housing and urban developmentsee resources", "multiple agenciessee resources", "abraham lincoln bicentennial commissionsee resources", "smithsonian institutionsee resources", "fish and wildlife service, department of interiorsee resources", "national institutes of healthsee resources", "national institute of standards and technologysee resources", "library of congresssee resources", "government printing officesee resources", "general services administrationsee resources", "national gallery of artsee resources", "department of justicesee resources", "department of agriculturesee resources", "institute of museum and library servicessee resources", "department of homeland securitysee resources", "federal trade commissionsee resources", "u.s. agency for international developmentsee resources", "u.s. geological surveysee resources", "national oceanic and atmospheric administrationsee resources", "department of health and human servicessee resources", "department of the treasurysee resources", "centers for disease control and preventionsee resources", "internal revenue servicesee resources", "office of naval researchsee resources", "department of commercesee resources", "u.s. institute of peacesee resources", "food and drug administrationsee resources", "national park servicesee resources", "u.s. global change research programsee resources", "holocaust memorial museumsee resources", "department of educationsee resources", "department of statesee resources", "department of veterans affairssee resources", "environmental protection agencysee resources", "small business administrationsee resources", "federal deposit insurance corporationsee resources", "federal emergency management agencysee resources", "house of representativessee resources", "u.s. census bureausee resources", "department of armysee resources", "national aeronautics and space administrationsee resources", "central intelligence agencysee resources", "national park service, teaching with historic placessee resources", "department of laborsee resources", "peace corpssee resources"];    
    if(doc.publisher){
        var lsPublisher =doc.publisher.toLowerCase();
        if (publishers.indexOf(lsPublisher) > -1){
            emit(lsPublisher, null);
        }
    } 
}<commit_msg>gov publishers<commit_after>function(doc) {
    var sanatize = function(str){
        str = str.trim()
        var loc = str.lastIndexOf(".");
        if(loc === (str.length -1 )){
            return str.substr(0, loc);
        }
        return str;
    }
    var publishers = ["national security agency", "consumer product safety commission", "bureau of land management, department of the interior", "national endowment for the humanities", "department of the interior", "national endowment for the arts", "department of navy", "u.s. courts", "federal judicial center", "securities and exchange commission", "u.s. mint, treasury", "department of energy", "national library of medicine", "department of commerce, international trade administration", "the white house", "national constitution center", "the federal reserve", "federal bureau of investigation", "national archives and records administration", "national science foundation", "national academy of sciences", "department of housing and urban development", "multiple agencies", "abraham lincoln bicentennial commission", "smithsonian institution", "fish and wildlife service, department of interior", "national institutes of health", "national institute of standards and technology", "library of congress", "government printing office", "general services administration", "national gallery of art", "department of justice", "department of agriculture", "institute of museum and library services", "department of homeland security", "federal trade commission", "u.s. agency for international development", "u.s. geological survey", "national oceanic and atmospheric administration", "department of health and human services", "department of the treasury", "centers for disease control and prevention", "internal revenue service", "office of naval research", "department of commerce", "u.s. institute of peace", "food and drug administration", "national park service", "u.s. global change research program", "holocaust memorial museum", "department of education", "department of state", "department of veterans affairs", "environmental protection agency", "small business administration", "federal deposit insurance corporation", "federal emergency management agency", "house of representatives", "u.s. census bureau", "department of army", "national aeronautics and space administration", "central intelligence agency", "national park service, teaching with historic places", "department of labor", "peace corps"];    
    if(doc.publisher){
        var lsPublisher =doc.publisher.toLowerCase();
        if (publishers.indexOf(lsPublisher) > -1){
            emit(lsPublisher, null);
        }
    } 
}
{'fraction_non_alphanumeric': 0.0922479365593138, 'fraction_numerical': 0.0009710309111506716, 'mean_word_length': 5.602564102564102, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5319228', 'n_tokens_mistral': 1455, 'n_tokens_neox': 1332, 'n_words': 635}