parameters: (self)
docstring: Runs Crawler objects using python threads.
  
      Number of concurrent threads is restricted to MAX_ALLOWED_THREADS.
  
      Returns:
        The number of registration pages found. -1 if no URLs are given.
  
      Raises:
        OSError: When creating the same directory that already exists.
docstring_summary: Runs Crawler objects using python threads.
docstring_tokens: Runs Crawler objects using python threads .
function: def Run(self):
      """Runs Crawler objects using python threads.
  
      Number of concurrent threads is restricted to MAX_ALLOWED_THREADS.
  
      Returns:
        The number of registration pages found. -1 if no URLs are given.
  
      Raises:
        OSError: When creating the same directory that already exists.
      """
      if self._urls_list:
        allThreads = []
        # originalNumThreads is the number of threads just before the
        # ThreadedCrawler starts creating new threads. As a standalone script it
        # will be 1.
        originalNumThreads = threading.active_count()
        for url in self._urls_list:
          self.logger.info('URL fed to a crawler thread: %s', url)
          t = WorkerThread(url)
          t.start()
          allThreads.append(t)
          while threading.active_count() >= (
              MAX_ALLOWED_THREADS + originalNumThreads):
            time.sleep(.4)
        while threading.active_count() > originalNumThreads:
          time.sleep(.4)
        self.logger.info('----------------')
        self.logger.info('--- FINISHED ---')
        self.logger.info('----------------')
        urls_no = 0
        urls_not_found_no = 0
        not_file_name = os.path.join(
            REGISTER_PAGE_DIR, NOT_FOUND_REG_PAGE_SITES_FILENAME)
        not_file_dir = os.path.dirname(not_file_name)
        try:
          os.makedirs(not_file_dir)
        except OSError as e:
          if e.errno != errno.EEXIST:
            raise
        fnot = open(not_file_name, 'wb')
        try:
          for t in sorted(allThreads, key=lambda t: t._url):
            urls_no += 1
            if not t.page_found:
              urls_not_found_no += 1
              fnot.write('%s' % t._url)
              fnot.write(os.linesep)
        except IOError as e:
          self.logger.error('Error: %s', e)
        finally:
          fnot.close()
        self.logger.info('Total number of URLs given: %d\n', urls_no)
        self.logger.info(
            'Registration pages found: %d\n', (urls_no - urls_not_found_no))
        self.logger.info(
            'URLs that did not return a registration page: %d\n',
            urls_not_found_no)
        return urls_no - urls_not_found_no
      else:
        self.logger.error('Error: no URLs were found.')
        return -1
function_tokens: def Run ( self ) : if self . _urls_list : allThreads = [ ] # originalNumThreads is the number of threads just before the # ThreadedCrawler starts creating new threads. As a standalone script it # will be 1. originalNumThreads = threading . active_count ( ) for url in self . _urls_list : self . logger . info ( 'URL fed to a crawler thread: %s' , url ) t = WorkerThread ( url ) t . start ( ) allThreads . append ( t ) while threading . active_count ( ) >= ( MAX_ALLOWED_THREADS + originalNumThreads ) : time . sleep ( .4 ) while threading . active_count ( ) > originalNumThreads : time . sleep ( .4 ) self . logger . info ( '----------------' ) self . logger . info ( '--- FINISHED ---' ) self . logger . info ( '----------------' ) urls_no = 0 urls_not_found_no = 0 not_file_name = os . path . join ( REGISTER_PAGE_DIR , NOT_FOUND_REG_PAGE_SITES_FILENAME ) not_file_dir = os . path . dirname ( not_file_name ) try : os . makedirs ( not_file_dir ) except OSError as e : if e . errno != errno . EEXIST : raise fnot = open ( not_file_name , 'wb' ) try : for t in sorted ( allThreads , key = lambda t : t . _url ) : urls_no += 1 if not t . page_found : urls_not_found_no += 1 fnot . write ( '%s' % t . _url ) fnot . write ( os . linesep ) except IOError as e : self . logger . error ( 'Error: %s' , e ) finally : fnot . close ( ) self . logger . info ( 'Total number of URLs given: %d\n' , urls_no ) self . logger . info ( 'Registration pages found: %d\n' , ( urls_no - urls_not_found_no ) ) self . logger . info ( 'URLs that did not return a registration page: %d\n' , urls_not_found_no ) return urls_no - urls_not_found_no else : self . logger . error ( 'Error: no URLs were found.' ) return - 1