<commit_before>// Filename: extractor.cxx
// Created by:  mike (09Jan97)
//
////////////////////////////////////////////////////////////////////
//
////////////////////////////////////////////////////////////////////
// Includes
////////////////////////////////////////////////////////////////////
#include "extractor.h"
#include "config_downloader.h"

#include <event.h>
#include <pt_Event.h>
#include <throw_event.h>
#include <eventParameter.h>
#include <filename.h>

////////////////////////////////////////////////////////////////////
// Defines
////////////////////////////////////////////////////////////////////

////////////////////////////////////////////////////////////////////
//       Class : ExtractorToken
// Description : Holds a request for the extractor.
////////////////////////////////////////////////////////////////////
class ExtractorToken : public ReferenceCount {
public:
  INLINE ExtractorToken(uint id, const Filename &source_file, 
	   	    	const string &event_name,
			const Filename &rel_path) {
    _id = id;
    _source_file = source_file;
    _event_name = event_name;
    _rel_path = rel_path;
  }
  int _id;
  Filename _source_file;
  string _event_name;
  Filename _rel_path;
};

////////////////////////////////////////////////////////////////////
//     Function: Extractor::Constructor
//       Access: Public
//  Description:
////////////////////////////////////////////////////////////////////
Extractor::
Extractor(void) : AsyncUtility() {
  PT(Buffer) buffer = new Buffer(extractor_buffer_size);
  init(buffer);
}

////////////////////////////////////////////////////////////////////
//     Function: Extractor::Constructor
//       Access: Public
//  Description:
////////////////////////////////////////////////////////////////////
Extractor::
Extractor(PT(Buffer) buffer) : AsyncUtility() {
  init(buffer);
}

////////////////////////////////////////////////////////////////////
//     Function: Extractor::Constructor
//       Access: Private 
//  Description:
////////////////////////////////////////////////////////////////////
void Extractor::
init(PT(Buffer) buffer) {
  nassertv(!buffer.is_null());
  _frequency = extractor_frequency;
  _token_board = new ExtractorTokenBoard;
  _buffer = buffer;
}

////////////////////////////////////////////////////////////////////
//     Function: Extractor::Destructor
//       Access: Public
//  Description:
////////////////////////////////////////////////////////////////////
Extractor::
~Extractor(void) {
  destroy_thread();

  delete _token_board;
}

////////////////////////////////////////////////////////////////////
//     Function: Extractor::request_extract
//       Access: Public
//  Description: 
////////////////////////////////////////////////////////////////////
int Extractor::
request_extract(const Filename &source_file, const string &event_name,
		const Filename &rel_path) {

  PT(ExtractorToken) tok;
  if (_threads_enabled) {

    // Make sure we actually are threaded
    if (!_threaded) {
      downloader_cat.info()
        << "Extractor::request_decompress() - create_thread() was "
        << "never called!  Calling it now..." << endl;
      create_thread();
    }

    // We need to grab the lock in order to signal the condition variable
#ifdef HAVE_IPC
    _lock.lock();
#endif<commit_msg>*** empty log message ***<commit_after>// Filename: extractor.cxx
// Created by:  mike (09Jan97)
//
////////////////////////////////////////////////////////////////////
//
////////////////////////////////////////////////////////////////////
// Includes
////////////////////////////////////////////////////////////////////
#include "extractor.h"
#include "config_downloader.h"

#include <event.h>
#include <pt_Event.h>
#include <throw_event.h>
#include <eventParameter.h>
#include <filename.h>

////////////////////////////////////////////////////////////////////
// Defines
////////////////////////////////////////////////////////////////////

////////////////////////////////////////////////////////////////////
//       Class : ExtractorToken
// Description : Holds a request for the extractor.
////////////////////////////////////////////////////////////////////
class ExtractorToken : public ReferenceCount {
public:
  INLINE ExtractorToken(uint id, const Filename &source_file, 
	   	    	const string &event_name,
			const Filename &rel_path) {
    _id = id;
    _source_file = source_file;
    _event_name = event_name;
    _rel_path = rel_path;
  }
  int _id;
  Filename _source_file;
  string _event_name;
  Filename _rel_path;
};

////////////////////////////////////////////////////////////////////
//     Function: Extractor::Constructor
//       Access: Public
//  Description:
////////////////////////////////////////////////////////////////////
Extractor::
Extractor(void) : AsyncUtility() {
  PT(Buffer) buffer = new Buffer(extractor_buffer_size);
  init(buffer);
}

////////////////////////////////////////////////////////////////////
//     Function: Extractor::Constructor
//       Access: Public
//  Description:
////////////////////////////////////////////////////////////////////
Extractor::
Extractor(PT(Buffer) buffer) : AsyncUtility() {
  init(buffer);
}

////////////////////////////////////////////////////////////////////
//     Function: Extractor::Constructor
//       Access: Private 
//  Description:
////////////////////////////////////////////////////////////////////
void Extractor::
init(PT(Buffer) buffer) {
  nassertv(!buffer.is_null());
  _frequency = extractor_frequency;
  _token_board = new ExtractorTokenBoard;
  _buffer = buffer;
}

////////////////////////////////////////////////////////////////////
//     Function: Extractor::Destructor
//       Access: Public
//  Description:
////////////////////////////////////////////////////////////////////
Extractor::
~Extractor(void) {
  destroy_thread();

  delete _token_board;
}

////////////////////////////////////////////////////////////////////
//     Function: Extractor::request_extract
//       Access: Public
//  Description: 
////////////////////////////////////////////////////////////////////
int Extractor::
request_extract(const Filename &source_file, const string &event_name,
		const Filename &rel_path) {

  PT(ExtractorToken) tok;
  if (_threads_enabled) {

    // Make sure we actually are threaded
    if (!_threaded) {
      downloader_cat.info()
        << "Extractor::request_extract() - create_thread() was "
        << "never called!  Calling it now..." << endl;
      create_thread();
    }

    // We need to grab the lock in order to signal the condition variable
#ifdef HAVE_IPC
    _lock.lock();
#endif
{'fraction_non_alphanumeric': 0.4274740171712607, 'fraction_numerical': 0.0012050007531254707, 'mean_word_length': 0.9993977717554954, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4506704', 'n_tokens_mistral': 1494, 'n_tokens_neox': 1445, 'n_words': 345}