<commit_before>var ObjectId = require('mongodb').ObjectId;
var global;
var tester;
var stats= 'nothing yet from explorer';

//generate a random 30 bits token that clearly identifies this process
var token = '';

function generateRandomBitToken(){
   if(token.length<20){
         token+=Math.ceil(Math.random()*1000)%2;
         generateRandomBitToken();
   }
   else{
    return;
   }   
}
generateRandomBitToken();
console.log(token);
  
//
module.exports = function(database){
    var Explorer = database.model('Explorer');
    var Urls = database.model('Urls');

    //
    var getUrl = function(cb){
         //
         function checkLock(){
               Explorer.find({locked: false}).toArray(function(err , results){
                  if(err){
                    stats = err;
                     throw new Error('DB connection error explorer check locked');
                  }
                  else if(results[0] == undefined){
                      //
                      stats = 'Another process is currently accessing database tying again in 29secs';
                      console.log('Another process is currently accessing database tying again in 29secs');
                      checkLock();
                  }
                  else {
                     console.log('Lock free on urls...');
                     lockAccessToUrls();  
                  }
             });

         };
         checkLock();

         //
         function lockAccessToUrls(){
             console.log('Locking access to urls...');
             Explorer.update(
                {},
                {
                   "$set": {
                       accessingDomain:token,
                       locked:true
                   }
                },
                function(err , result){
                    if(err){
                        stats=err;
                        throw new Error('DB connection error explorer locking error');
                    }
                    else {
                        console.log('process successfully locked database');
                        setTimeout(function(){
                           authenticateAccess();
                        } , 10000);
                    }
                }
             );
         }

         //
         function authenticateAccess(){
           console.log('Authenticating access');
           Explorer.find({locked: true , accessingDomain:token}).toArray(function(err , results){
                if(err){
                  stats= err;
                   throw new Error('DB connection error explorer authenticating');
                }
                else if(results[0] == undefined){
                    //
                    console.log('Unable to authenticate checking lock again in 29secs');
                    setTimeout(function(){
                       checkLock();
                    } , 30000); 
                }
                else {
                   console.log('Authentication completed...');
                   getAnyUrl(); 
                }
           });
         } 

        
         //
         function getAnyUrl(){ 
            console.log('getting url');
           
            Urls.find(  
                 {"lastVisited":{"$lte": (Date.now() - 60000*6)+''}}  
            ).toArray(function(err , results){//get url that have not been updated in the past 4mins
                  if(err){ 
                     throw new Error('DB connection error explorer getting any urls');
                  }
                  else if(results[0] == undefined){  
                      stats = 'Url available but last visited less than four mins ago ';
                      console.log('Url available but last visited less than four mins ago retrying in 10secs');
                      releaseLock(results[0]);
                  }
                  else {     
                      console.log(results);
                      releaseLock(results[0]); 
                  } 
             });
         }

         //update url object before updating explorer object....THATS THE SOLUTION      
         function releaseLock(urlObj){
              if(urlObj){
                   global=urlObj;
                   global.lastVisited = Date.now()+'';
                   Urls.update({_id : ObjectId(urlObj._id)},  {
                             "$set": {
                                   lastVisited: global.lastVisited
                              }
                          },
                          function(err , result){  
                              if(err){
                                  throw new Error('DB connection error release lock');
                              }
                              else { 
                                 stats="url gotten successfullly";
                                 setTimeout(function(){
                                     release();
                                 } , 15000); 
                              }
                           });

               }
               else{
                  stats = "No url ";
                  release();
               }

              function release(){
                 console.log('Releasing lock on database');
                     //release lock and return url back to main process
                 Explorer.update({}, {
                     "$set": {
                         accessingDomain: '',
                         locked:false
                     }
                  },
                  function(err , result){
                      if(err){
                          throw new Error('DB connection error explorer lock releasing error 1');
                      }
                      else {
                          stats = "Url object gotten";<commit_msg>changes updated<commit_after>var ObjectId = require('mongodb').ObjectId;
var global;
var tester;
var stats= 'nothing yet from explorer';

//generate a random 30 bits token that clearly identifies this process
var token = '';

function generateRandomBitToken(){
   if(token.length<20){
         token+=Math.ceil(Math.random()*1000)%2;
         generateRandomBitToken();
   }
   else{
    return;
   }   
}
generateRandomBitToken();
console.log(token);
  
//
module.exports = function(database){
    var Explorer = database.model('Explorer');
    var Urls = database.model('Urls');

    //
    var getUrl = function(cb){
         //
         (function checkLock(){
               Explorer.find({locked: false}).toArray(function(err , results){
                  if(err){
                    stats = err;
                     throw new Error('DB connection error explorer check locked');
                  }
                  else if(results[0] == undefined){
                      //
                      stats = 'Another process is currently accessing database tying again in 29secs';
                      console.log('Another process is currently accessing database tying again in 29secs');
                      checkLock();
                  }
                  else {
                     console.log('Lock free on urls...');
                     lockAccessToUrls();  
                  }
             });

         })();

         //
         function lockAccessToUrls(){
             console.log('Locking access to urls...');
             Explorer.update(
                {},
                {
                   "$set": {
                       accessingDomain:token,
                       locked:true
                   }
                },
                function(err , result){
                    if(err){
                        stats=err;
                        throw new Error('DB connection error explorer locking error');
                    }
                    else {
                        console.log('process successfully locked database');
                        setTimeout(function(){
                           authenticateAccess();
                        } , 2000);
                    }
                }
             );
         }

         //
         function authenticateAccess(){
           console.log('Authenticating access');
           Explorer.find({locked: true , accessingDomain:token}).toArray(function(err , results){
                if(err){
                  stats= err;
                   throw new Error('DB connection error explorer authenticating');
                }
                else if(results[0] == undefined){
                    //
                    console.log('Unable to authenticate checking lock again in 29secs');
                    checkLock();
                }
                else {
                   console.log('Authentication completed...');
                   getAnyUrl(); 
                }
           });
         } 

        
         //
         function getAnyUrl(){ 
            console.log('getting url');
           
            Urls.find(  
                 {"lastVisited":{"$lte": (Date.now() - 60000*6)+''}}  
            ).toArray(function(err , results){//get url that have not been updated in the past 4mins
                  if(err){ 
                     throw new Error('DB connection error explorer getting any urls');
                  }
                  else if(results[0] == undefined){  
                      stats = 'Url available but last visited less than four mins ago ';
                      console.log('Url available but last visited less than four mins ago retrying in 10secs');
                      releaseLock(results[0]);
                  }
                  else {     
                      console.log(results);
                      releaseLock(results[0]); 
                  } 
             });
         }

         //update url object before updating explorer object....THATS THE SOLUTION      
         function releaseLock(urlObj){
              if(urlObj){
                   global=urlObj;
                   global.lastVisited = Date.now()+'';
                   Urls.update({_id : ObjectId(urlObj._id)},  {
                             "$set": {
                                   lastVisited: global.lastVisited
                              }
                          },
                          function(err , result){  
                              if(err){
                                  throw new Error('DB connection error release lock');
                              }
                              else { 
                                 stats="url gotten successfullly";
                                 release(); 
                              }
                           });

               }
               else{
                  stats = "No url ";
                  release();
               }

              function release(){
                 console.log('Releasing lock on database');
                     //release lock and return url back to main process
                 Explorer.update({}, {
                     "$set": {
                         accessingDomain: '',
                         locked:false
                     }
                  },
                  function(err , result){
                      if(err){
                          throw new Error('DB connection error explorer lock releasing error 1');
                      }
                      else {
                          stats = "Url object gotten";
{'fraction_non_alphanumeric': 0.0917713181698428, 'fraction_numerical': 0.006937736014753666, 'mean_word_length': 0.9453365220362145, 'pattern_counts': {'":': 10, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3657594', 'n_tokens_mistral': 2584, 'n_tokens_neox': 2286, 'n_words': 595}