<commit_before>#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "tokenizer.h"
#include "sorted-list.h"
#include "index.h"
#include "search.h"

#define VERBOSE 1

int compareStrings(void* s1, void* s2) {
	return strcmp((char*)s1,(char*)s2);
}

int search(TokenizerT* tk, SortedListPtr sl, int op) {
	char* t = 0;
	SortedListPtr l = SLCreate(compareStrings);
	wordListNode * wl = malloc(sizeof(wordListNode));
	wordListNode * il = malloc(sizeof(wordListNode));
	memset(il,0,sizeof(wordListNode));
	memset(wl,0,sizeof(wordListNode));
	while(1) {
		SortedListIteratorPtr fl;
		t = TKGetNextToken(tk);
		if(t==0) { break; }
		wl->word = t;
		il = SLFind(sl,wl);
		if(il == 0) { continue; }
		fl = SLCreateIterator(il->fileList);
		while(1){
			fileListNode* f = SLNextItem(fl);
			wordListNode* item = SLFind(l,f->fileName);
			if(item == 0) {
				SLInsert(l,f->fileName);
			}
			/* oops... should have used fileNodes for keeping track of the found. need to fix */
		}
	}	
}

int loop(SortedListPtr sl) {
	char* input;
	char* command;
	int op;
	TokenizerT* tk;
	input = malloc(1000);
	memset(input,0,1000);

	while(1) {
		scanf("%[^\n]s",input);
		if(strcmp(input,"q")==0) {
			printf("Exiting.\n");
			return 0;
		}
		tk = TKCreate(input);
		if(tk==0) {
			printf("Error in tokenizing input.\n");
			return 1;
		}
		command = TKGetNextToken(tk);
		if(strcmp(command,"sa")==0) {
			op = 0;
		}else if(strcmp(command,"so")==0) {
			op = 1;
		}else{
			printf("Error: Invalid command.\n");
		}
		printf("Searching for %s, op = %d\n",input,op);
		if(search(tk,sl,op)) {
			return 1;
		}
	}
}

char *substring(const char* str, int beg, int n)
{
    char *ret = malloc(n+1);
    strncpy(ret, (str + beg), n);<commit_msg>I finally finished coding SO. SA is "in the works".<commit_after>#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "tokenizer.h"
#include "sorted-list.h"
#include "index.h"
#include "search.h"

#define VERBOSE 1

int compareStrings(void* s1, void* s2) {
	return strcmp((char*)s1,(char*)s2);
}

SortedListPtr search(TokenizerT* tk, SortedListPtr sl, int op) {
	char* t = 0;
	SortedListPtr l = SLCreate(compareFileNodes);
	wordListNode * wl = malloc(sizeof(wordListNode));
	wordListNode * il = malloc(sizeof(wordListNode));
	memset(il,0,sizeof(wordListNode));
	memset(wl,0,sizeof(wordListNode));
	while(1) {
		SortedListIteratorPtr fl;
		t = TKGetNextToken(tk);
		if(t==0) { break; }
		wl->word = t;
		il = SLFind(sl,wl);
		if(il == 0) { continue; }
		fl = SLCreateIterator(il->fileList);
		while(1){
			fileListNode* f = SLNextItem(fl);
			fileListNode* item = SLFind(l,f->fileName);
			if(item == 0) {
				SLInsert(l,f->fileName);
			}
			item->count++;
		}
	}	
	return l;
}

int loop(SortedListPtr sl) {
	char* input;
	char* command;
	int op;
	SortedListPtr sl = SLCreate(compareStrings);
	SortedListIteratorPtr si;
	TokenizerT* tk;
	input = malloc(1000);
	memset(input,0,1000);

	while(1) {
		SortedListPtr r;
		fileListNode fn;
		SortedListIteratorPtr i;
		scanf("%[^\n]s",input);
		if(strcmp(input,"q")==0) {
			printf("Exiting.\n");
			return 0;
		}
		tk = TKCreate(input);
		if(tk==0) {
			printf("Error in tokenizing input.\n");
			return 1;
		}
		command = TKGetNextToken(tk);
		if(strcmp(command,"sa")==0) {
			op = 0;
		}else if(strcmp(command,"so")==0) {
			op = 1;
		}else{
			printf("Error: Invalid command.\n");
		}
		printf("Searching for %s, op = %d\n",input,op);
		r = search(tk,sl,op);
		i = SLCreateIterator(r);
		while(1) {
			fn = SLNextItem(i);
			if(fn == 0) { break; }
			if(SLFind(sl,fn->filename) == 0) {
				SLInsert(sl,fn->filename);
			}
		}
	}
	si = SLCreateIterator(sl);
	while(1) {
		char* s = SLNextItem(si);
		if(s==0) {break;}
		printf("%s ",s);
	}
	return 0;
	
}

char *substring(const char* str, int beg, int n)
{
    char *ret = malloc(n+1);
    strncpy(ret, (str + beg), n);
{'fraction_non_alphanumeric': 0.18466718466718468, 'fraction_numerical': 0.01838901838901839, 'mean_word_length': 3.983225806451613, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3897245', 'n_tokens_mistral': 1746, 'n_tokens_neox': 1594, 'n_words': 330}