<commit_before>load('mongo-fulltext/_load.js');
var s = db.search_works;
s.drop();
var full_vector_norm = false;



var fixture = [
    { "_id" : 1, "title" : "fish", "content" : "groupers like <NAME>" },
    { "_id" : 2, "title" : "dogs", "content" : "whippets kick mongrels" },
    { "_id" : 3, "title" : "dogs & fish", "content" : "whippets kick groupers" }
];
var conf = db.fulltext_config;
conf.insert({'collection_name' : 'search_works', 'fields': {'title': 5, 'content': 1}, 'params': {'full_vector_norm': full_vector_norm}});


mft.get('util').load_records_from_list(fixture, 'search_works');
var search = mft.get('search');
search.mapReduceIndex('search_works');
search.mapReduceTermScore('search_works');

var ranks_result ;
var search_result ;

// TODO: add index on collection name (should we have an _id attribute too?)
ranks_result = search.mapReduceSearch('search_works', 'fish', true);

search_result = search.mapReduceNiceSearch("search_works", "fish");
// print(result, "Search result for 'fish'");

assert.eq(search_result.toArray(), [
    { "_id" : 1, "value" : { "_id" : 1, "title" : "fish", "content" : "groupers like <NAME>", "score" : 0.6757751801802742 } },
    { "_id" : 3, "value" : { "_id" : 3, "title" : "dogs & fish", "content" : "whippets kick groupers", "score" : 0.5622789375752065 } }],
    "search_works_1"
);
<commit_msg>made tests pass, although there are some worrying scores > 1 which should happen in the cosine model<commit_after>load('mongo-fulltext/_load.js');
var s = db.search_works;
s.drop();
var full_vector_norm = true;



var fixture = [
    { "_id" : 1, "title" : "fish", "content" : "groupers like <NAME>" },
    { "_id" : 2, "title" : "dogs", "content" : "whippets kick mongrels" },
    { "_id" : 3, "title" : "dogs & fish", "content" : "whippets kick groupers" }
];
var conf = db.fulltext_config;
conf.insert({'collection_name' : 'search_works', 'fields': {'title': 5, 'content': 1}, 'params': {'full_vector_norm': full_vector_norm}});


mft.get('util').load_records_from_list(fixture, 'search_works');
var search = mft.get('search');
search.mapReduceIndex('search_works');
search.mapReduceTermScore('search_works');

var ranks_result ;
var search_result ;

search_result = search.mapReduceNiceSearch("search_works", "fish");
// print(result, "Search result for 'fish'");

assert.eq(search_result.toArray(), [
    { "_id" : 1, "value" : { "_id" : 1, "title" : "fish", "content" : "groupers like <NAME>", "score" : 0.6757751801802742 } },
    { "_id" : 3, "value" : { "_id" : 3, "title" : "dogs & fish", "content" : "whippets kick groupers", "score" : 0.5622789375752065 } }],
    "search_works_1"
);

{'fraction_non_alphanumeric': 0.19411986430456088, 'fraction_numerical': 0.03354692800603091, 'mean_word_length': 5.073226544622425, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2744441', 'n_tokens_mistral': 1023, 'n_tokens_neox': 949, 'n_words': 225}