<commit_before>#!/usr/bin/env python

"""
Usage: ./search.py --create-index <files>
       ./search.py --create-index `find texts/ -name '??.txt'`
            - will index all given files (should be .txt!
              note that this script uses fetcher, which can really use
              a .sentences or .huninput file)
       ./search.py <query>
            - will perform a search
"""

from __future__ import unicode_literals

import sys
import os.path

#import pdb; pdb.set_trace()
PATH_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..', '..', '..'))
assert os.path.isdir(PATH_ROOT), PATH_ROOT

TEXTS_ROOT = PATH_ROOT + '/texts'
assert os.path.isdir(TEXTS_ROOT), TEXTS_ROOT

INDEX_DIR = PATH_ROOT + '/corthus/index'
assert os.path.isdir(INDEX_DIR), INDEX_DIR

EXTERNAL_DIR = PATH_ROOT + '/corthus/external' # for Whoosh
assert os.path.isdir(EXTERNAL_DIR), EXTERNAL_DIR<commit_msg>Correction.<commit_after>#!/usr/bin/env python

"""
Usage: ./search.py --create-index <files>
       ./search.py --create-index `find texts/ -name '??.txt'`
            - will index all given files (should be .txt!
              note that this script uses fetcher, which can really use
              a .sentences or .huninput file)
       ./search.py <query>
            - will perform a search
"""

from __future__ import unicode_literals

import sys
import os.path

# TODO get rid of this path madness (this is relative to site-packages)
PATH_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..', '..', '..'))
assert os.path.isdir(PATH_ROOT), PATH_ROOT

TEXTS_ROOT = PATH_ROOT + '/texts'
assert os.path.isdir(TEXTS_ROOT), TEXTS_ROOT

INDEX_DIR = PATH_ROOT + '/corthus/index'
assert os.path.isdir(INDEX_DIR), INDEX_DIR

EXTERNAL_DIR = PATH_ROOT + '/corthus/external' # for Whoosh
assert os.path.isdir(EXTERNAL_DIR), EXTERNAL_DIR
{'fraction_non_alphanumeric': 0.14323890967397113, 'fraction_numerical': 0.0, 'mean_word_length': 3.087336244541485, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2611312', 'n_tokens_mistral': 650, 'n_tokens_neox': 644, 'n_words': 162}