<commit_before>/**
 * An xhr wrapper that calls a callback whenever some more of the
 * response is available, without waiting for all of it.
 * 
 * This probably needs more development and testing more than most other parts 
 * of Oboe.
 *
 * TODO:
 *    error handling
 *    allow setting of request params and other such options
 *    x-browser testing, compatibility
 */
var streamingXhr = (function () {
   
   /* xhr2 already supports everything that we need so very little abstraction required.\
   *  listenToXhr2 is one of two possible values to use as listenToXhr  
   */
   function listenToXhr2(xhr, progressListener, completeListener) {      
      xhr.onprogress = progressListener;
      xhr.onload = completeListener;
   }

   /* xhr1 supports little so a bit more work is needed 
    * listenToXhr1 is one of two possible values to use as listenToXhr  
    */           
   function listenToXhr1(xhr, progressListener, completeListener){
   
      // We are recieving the content, check for progress as often as the browser allows. 
      // The progress listener makes sure there is something to report so just call it as often 
      // as possible regardless of if something happened to the xhr1 object.
      var interval = window.setInterval(progressListener, 0);      
   
      // handle the request being complete: 
      xhr.onreadystatechange = function() {

         if(this.readyState == 4 ) {
         
            // XHR is complete. Notify of completeness and stop notifying of progress:
            if( this.status == 200 ) {             
               completeListener();
            }
            
            window.clearInterval(interval);
         }               
      };
   }
      
   function supportsXhr2(){
      return ('onprogress' in new XMLHttpRequest());
   }      
   
   /* listenToXhr will be set to the appropriate function for XHR1 or XHR2 depending on what the browser
    * supports
    * 
    * @function
    * 
    * @param {XmlHttpRequest} xhr
    * @param {Function} progressListener
    * @param {Function} completeListener
    */
   var listenToXhr = supportsXhr2()? listenToXhr2 : listenToXhr1;   
   
   /**
    * Fetch something over ajax, calling back as often as new data is available.
    * 
    * @param {String} url
    * @param {Function(String nextResponseDrip)} progressCallback
    *    A callback to be called repeatedly as the input comes in.
    *    Will be passed the new string since the last call.
    * @param {Function(String wholeResponse)} doneCallback
    *    A callback to be called when the request is complete.
    *    Will be passed the total response
    */
   return function(url, progressCallback, doneCallback) {
      // TODO: in if in node, use require('http') instead of ajax
   
      doneCallback = doneCallback || always;
   
      var xhr = new XMLHttpRequest();
      var numberOfCharsGivenToCallback = 0;

      xhr.open("GET", url, true);
      xhr.send(null);

      function handleProgress() {
         
         try{
            var textSoFar = xhr.responseText;
         } catch(e) {
            // ie sometimes errors if you try to get the responseText too early but just
            // ignore it when this happens.
            return;
         }

         if( len(textSoFar) > numberOfCharsGivenToCallback ) {

            var latestText = textSoFar.substr(numberOfCharsGivenToCallback);

            progressCallback( latestText );

            numberOfCharsGivenToCallback = len(textSoFar);
         }<commit_msg>throttle callback frequency a bit for older browsers<commit_after>/**
 * An xhr wrapper that calls a callback whenever some more of the
 * response is available, without waiting for all of it.
 * 
 * This probably needs more development and testing more than most other parts 
 * of Oboe.
 *
 * TODO:
 *    error handling
 *    allow setting of request params and other such options
 *    x-browser testing, compatibility
 */
var streamingXhr = (function () {
   
   /* xhr2 already supports everything that we need so very little abstraction required.\
   *  listenToXhr2 is one of two possible values to use as listenToXhr  
   */
   function listenToXhr2(xhr, progressListener, completeListener) {      
      xhr.onprogress = progressListener;
      xhr.onload = completeListener;
   }

   /* xhr1 supports little so a bit more work is needed 
    * listenToXhr1 is one of two possible values to use as listenToXhr  
    */           
   function listenToXhr1(xhr, progressListener, completeListener){
   
      // We are recieving the content, check for progress as often as the browser allows. 
      // The progress listener makes sure there is something to report so just call it as often 
      // as possible regardless of if something happened to the xhr1 object.
      var interval = window.setInterval(progressListener, 50);      
   
      // handle the request being complete: 
      xhr.onreadystatechange = function() {

         if(this.readyState == 4 ) {
         
            // XHR is complete. Notify of completeness and stop notifying of progress:
            if( this.status == 200 ) {             
               completeListener();
            }
            
            window.clearInterval(interval);
         }               
      };
   }
      
   function browserSupportsXhr2(){
      return ('onprogress' in new XMLHttpRequest());
   }      
   
   /* listenToXhr will be set to the appropriate function for XHR1 or XHR2 depending on what the browser
    * supports
    * 
    * @function
    * 
    * @param {XmlHttpRequest} xhr
    * @param {Function} progressListener
    * @param {Function} completeListener
    */
   var listenToXhr = browserSupportsXhr2()? listenToXhr2 : listenToXhr1;
      
   /**
    * Fetch something over ajax, calling back as often as new data is available.
    * 
    * @param {String} url
    * @param {Function(String nextResponseDrip)} progressCallback
    *    A callback to be called repeatedly as the input comes in.
    *    Will be passed the new string since the last call.
    * @param {Function(String wholeResponse)} doneCallback
    *    A callback to be called when the request is complete.
    *    Will be passed the total response
    */
   return function(url, progressCallback, doneCallback) {
      // TODO: in if in node, use require('http') instead of ajax
   
      doneCallback = doneCallback || always;
   
      var xhr = new XMLHttpRequest();
      var numberOfCharsGivenToCallback = 0;

      xhr.open("GET", url, true);
      xhr.send(null);

      function handleProgress() {
         
         try{
            var textSoFar = xhr.responseText;
         } catch(e) {
            // ie sometimes errors if you try to get the responseText too early but just
            // ignore it when this happens.
            return;
         }
         
         if( len(textSoFar) > numberOfCharsGivenToCallback ) {

            var latestText = textSoFar.substr(numberOfCharsGivenToCallback);

            progressCallback( latestText );

            numberOfCharsGivenToCallback = len(textSoFar);
         }
{'fraction_non_alphanumeric': 0.0704957168936947, 'fraction_numerical': 0.005476758882179469, 'mean_word_length': 2.2240832956088727, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2929092', 'n_tokens_mistral': 1916, 'n_tokens_neox': 1829, 'n_words': 699}