<commit_before>#include "file_scrubber.hpp"

#include <stdio.h>
#include <iostream>

#include "string_utils.hpp"
#include "error.hpp"

using std::ifstream;

namespace mocc {
    FileScrubber::FileScrubber(const char* fName, const char* commentFlag):
    	stream_(fName),
    	flag_(commentFlag){
       
            if( !stream_.good() ) {
                Error("Failed to open file.");
            }
        
        }
    	
    FileScrubber::~FileScrubber(){}
    
    std::string FileScrubber::getline(){
    	while(!stream_.eof()){
    		std::string line;
    		std::getline(stream_, line);
    		// Strip the comments
    		unsigned int commentPos = line.find(flag_, 0);
    		if(commentPos != std::string::npos){
    			line.erase(commentPos, std::string::npos);
    		}
    		// Remove whitespace
    		line = trim(line);
    		// If the result isnt empty, return the line.
    		if(!line.empty()){
    			return line;
    		}
    	}
    	return "";
    }
}<commit_msg>Fix overflow issue in file scrubber. npos is big<commit_after>#include "file_scrubber.hpp"

#include <stdio.h>
#include <iostream>

#include "string_utils.hpp"
#include "error.hpp"

using std::ifstream;

namespace mocc {
    FileScrubber::FileScrubber(const char* fName, const char* commentFlag):
    	stream_(fName),
    	flag_(commentFlag){
       
            if( !stream_.good() ) {
                Error("Failed to open file.");
            }
        
        }
    	
    FileScrubber::~FileScrubber(){}
    
    std::string FileScrubber::getline(){
    	while(!stream_.eof()){
    		std::string line;
    		std::getline(stream_, line);
    		// Strip the comments
    		size_t commentPos = line.find(flag_, 0);
    		if(commentPos != std::string::npos){
    			line.erase(commentPos, std::string::npos);
    		}
    		// Remove whitespace
    		line = trim(line);
    		// If the result isnt empty, return the line.
    		if(!line.empty()){
    			return line;
    		}
    	}
    	return "";
    }
}
{'fraction_non_alphanumeric': 0.13881877839475013, 'fraction_numerical': 0.0010095911155981827, 'mean_word_length': 2.2760330578512398, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1662868', 'n_tokens_mistral': 699, 'n_tokens_neox': 689, 'n_words': 135}