<commit_before>use Logging, Memory, LibEv, IO, Indexer, Search, Partitions, Time;

// TODO: port to pure chapel
extern proc initialize_socket(port: c_int): c_int;

// trampolines
extern var c_accept_cb: opaque;

extern proc send(sockfd:c_int, buffer: c_string, len: size_t, flags: c_int);

config var port: c_int = 3033;

// TODO: we need to know which client context this is so that we can maintain parsing context
//       is the fd enough?
export proc handle_received_data(fd: c_int, buffer: c_string, read: size_t, buffer_size: size_t) {
  // writeln("from chpl: " + buffer);
  // accumulate string buffer
  var word = buffer;
  var trimmedWord: string = "";
  writeln("word: ", word);
  for i in 1..word.length {
  	var ch = word.substring(i);
  	if (ch != "\r" && ch != "\n") {
	  	writeln("ch: ", word.substring(i));
	  	trimmedWord += word.substring(i);
  	}
  }
//  writeln("trimmedWord: ", trimmedWord);
	if (trimmedWord == "dump") {
		dumpPartition(partitionForWord("dog"));
	} else {
		writeln("<adding>");
	  indexWord(trimmedWord, 1);
		dumpPostingTableForWord(trimmedWord);
		writeln("</adding>");
	}
  send(fd, buffer, read, 0);
}

proc initIndex() {
	writeln("This program is running on ", numLocales, " locales");
	writeln("It began running on locale #", here.id);
	writeln();


  initPartitions();
  initIndices();

  initIndexer();

 //  writeln("adding 3 entries");
	// indexWord("dog", 1);
	// indexWord("cat", 2);
	// indexWord("cat", 3);

 //  writeln("dumping partition holding word: dog");
	// dumpPartition(partitionForWord("dog"));

 //  writeln("dumping posting table for word: cat");
	// dumpPostingTableForWord("cat");
  var t: Timer;
  t.start();

  var infile = open("words.txt", iomode.r);
  var reader = infile.reader();
  var word: string;
  var docId: DocId = 1;
  while (reader.readln(word)) {
    enqueueIndexRequest(word, docId);
    docId = (docId + 1) % 1000 + 1; // fake different docs
  }

  waitForIndexer();
  t.stop();
  timing("indexing complete in ",t.elapsed(TimeUnits.microseconds), " microseconds");


  //dumpPostingTableForWord("the");

  // TODO: build execution Tree w/ conj / disj. (operator) nodes
  // test basic boolean operators
  writeln("conjunction");
  var conj = conjunction(["the", "dog"]);
  writeln(conj);

  writeln("disjunction");
  var disj = disjunction(["the", "dog"]);
  writeln(disj);
}

proc conjunction(words: [] string): domain(DocId) {
  writeln("finding conjunction of: ", words);
  var doms: [1..words.size] domain(DocId);
  
  for j in 1..words.size {
    var word = words[j];
    var entry = entryForWord(word);
    if (entry != nil) {
      on entry {
        for docId in documentIdsForEntry(entry) {
          doms[j] += docId;
        }
      }
    }
  }
<commit_msg>dont use indexer<commit_after>use Logging, Memory, LibEv, IO, Indexer, Search, Partitions, Time;

// TODO: port to pure chapel
extern proc initialize_socket(port: c_int): c_int;

// trampolines
extern var c_accept_cb: opaque;

extern proc send(sockfd:c_int, buffer: c_string, len: size_t, flags: c_int);

config var port: c_int = 3033;

// TODO: we need to know which client context this is so that we can maintain parsing context
//       is the fd enough?
export proc handle_received_data(fd: c_int, buffer: c_string, read: size_t, buffer_size: size_t) {
  // writeln("from chpl: " + buffer);
  // accumulate string buffer
  var word = buffer;
  var trimmedWord: string = "";
  writeln("word: ", word);
  for i in 1..word.length {
  	var ch = word.substring(i);
  	if (ch != "\r" && ch != "\n") {
	  	writeln("ch: ", word.substring(i));
	  	trimmedWord += word.substring(i);
  	}
  }
//  writeln("trimmedWord: ", trimmedWord);
	if (trimmedWord == "dump") {
		dumpPartition(partitionForWord("dog"));
	} else {
		writeln("<adding>");
	  indexWord(trimmedWord, 1);
		dumpPostingTableForWord(trimmedWord);
		writeln("</adding>");
	}
  send(fd, buffer, read, 0);
}

proc initIndex() {
	writeln("This program is running on ", numLocales, " locales");
	writeln("It began running on locale #", here.id);
	writeln();


  initPartitions();
  initIndices();

  // initIndexer();

 //  writeln("adding 3 entries");
	// indexWord("dog", 1);
	// indexWord("cat", 2);
	// indexWord("cat", 3);

 //  writeln("dumping partition holding word: dog");
	// dumpPartition(partitionForWord("dog"));

 //  writeln("dumping posting table for word: cat");
	// dumpPostingTableForWord("cat");
  var t: Timer;
  t.start();

  var infile = open("words.txt", iomode.r);
  var reader = infile.reader();
  var word: string;
  var docId: DocId = 1;
  while (reader.readln(word)) {
//    enqueueIndexRequest(word, docId);
    indexWord(word, docId);
    docId = (docId + 1) % 1000 + 1; // fake different docs
  }

//  waitForIndexer();
  t.stop();
  timing("indexing complete in ",t.elapsed(TimeUnits.microseconds), " microseconds");


  //dumpPostingTableForWord("the");

  // TODO: build execution Tree w/ conj / disj. (operator) nodes
  // test basic boolean operators
  // writeln("conjunction");
  // t.start();
  // var conj = conjunction(["the", "dog"]);
  // t.stop();
  // timing("conjunction complete in ",t.elapsed(TimeUnits.microseconds), " microseconds");
  // writeln(conj);

  // writeln("disjunction");
  // t.start();
  // var disj = disjunction(["the", "dog"]);
  // t.stop();
  // timing("disjunction complete in ",t.elapsed(TimeUnits.microseconds), " microseconds");
  // writeln(disj);
}

proc conjunction(words: [] string): domain(DocId) {
  writeln("finding conjunction of: ", words);
  var doms: [1..words.size] domain(DocId);
  
  for j in 1..words.size {
    var word = words[j];
    var entry = entryForWord(word);
    if (entry != nil) {
      on entry {
        for docId in documentIdsForEntry(entry) {
          doms[j] += docId;
        }
      }
    }
  }

{'fraction_non_alphanumeric': 0.16620784583620096, 'fraction_numerical': 0.006882312456985547, 'mean_word_length': 3.520217729393468, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4482603', 'n_tokens_mistral': 2092, 'n_tokens_neox': 1980, 'n_words': 522}