<commit_before>import argparse
import yaml
import os
from os.path import expanduser, isdir
import whoosh.index as ix
from whoosh.fields import *


def load_config_from(path):
  with open(expanduser(path)) as file:
    return yaml.load(file)

def parse_options():
  parser = argparse.ArgumentParser()
  parser.add_argument("command",
                      choices=["ls", "add", "rm", "edit", "view", "reindex"])
  parser.add_argument("--query")
  return parser.parse_args()

def create_or_load_index(index_path):
  index_full_path = expanduser(index_path)
  if isdir(index_full_path):
    return ix.open_dir(index_full_path)
  else:
    os.mkdir(index_full_path)
    schema = Schema(filename=TEXT(stored=True), content=TEXT)
    return ix.create_in(index_full_path, schema)

def main():
  config = load_config_from("~/.notes-cli/config.yaml")
  options = parse_options()
  index = create_or_load_index(config["indexdir"])
  print options

if __name__ == "__main__":
  main()<commit_msg>Reindex notes<commit_after>import argparse
import yaml
import shutil
import os
from os.path import expanduser, isdir
import whoosh.index as ix
from whoosh.fields import *


def command_reindex(index_full_path, notes_path):
  shutil.rmtree(index_full_path, True)
  os.mkdir(index_full_path)
  schema = Schema(filename=TEXT(stored=True), content=TEXT)
  index = ix.create_in(index_full_path, schema)
  writer = index.writer()
  for note in os.listdir(notes_path):
    note_full_path = os.path.join(notes_path, note)
    with open(note_full_path) as note_file:
      writer.add_document(filename=unicode(note_full_path, "utf-8"), content=unicode(note_file.read(), "utf-8"))
  return index

def load_config_from(path):
  with open(expanduser(path)) as file:
    return yaml.load(file)

def parse_options():
  parser = argparse.ArgumentParser()
  parser.add_argument("command",
                      choices=["ls", "add", "rm", "edit", "view", "reindex"])
  parser.add_argument("--query")
  return parser.parse_args()

def create_or_load_index(index_path, notes_path):
  index_full_path = expanduser(index_path)
  if isdir(index_full_path):
    return ix.open_dir(index_full_path)
  else:
    return command_reindex(index_full_path, notes_path)

def main():
  config = load_config_from("~/.notes-cli/config.yaml")
  options = parse_options()
  index = create_or_load_index(config["indexdir"], expanduser(config["notesdir"]))
  print options

if __name__ == "__main__":
  main()
{'fraction_non_alphanumeric': 0.12291412291412292, 'fraction_numerical': 0.000814000814000814, 'mean_word_length': 5.084158415841584, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '595908', 'n_tokens_mistral': 841, 'n_tokens_neox': 826, 'n_words': 171}