<commit_before>
import ftsdb

import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--find-text', help='search the YAML text field')
parser.add_argument('--exclude-text', help='search the YAML text field')
parser.add_argument('--find-title', help='search the YAML title field')
parser.add_argument('--exclude-title', help='search the YAML title field')
parser.add_argument('--show-filename',  action='store_const', const=True, default=False)
args = parser.parse_args()

print(args)

argsd = vars(args)
query = []
for field in ['text', 'title']:
   exclude_field = 'exclude_' + field
   include_field = 'find_' + field
   if exclude_field in argsd:
      entry = argsd.get(exclude_field)
      if entry: query.append(('-', field, entry))
   if include_field in argsd:
      entry = argsd.get(include_field)
      if entry: query.append(('', field, entry))

db = ftsdb.get()
gen = db.fts_search(query)

for row in gen:
   for field in row.keys():
      show_field = "show_" + field
      if argsd.get(show_field, None):
         if len(row[field]) > 0:
            print("%s = %s" % (field, row[field]))
   print()
<commit_msg>add initial support for title and text being shown<commit_after>
import ftsdb

import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--find-text', help='search the YAML text field')
parser.add_argument('--exclude-text', help='search the YAML text field')
parser.add_argument('--find-title', help='search the YAML title field')
parser.add_argument('--exclude-title', help='search the YAML title field')
parser.add_argument('--show-filename',  action='store_const', const=True, default=False)
parser.add_argument('--show-title',  action='store_const', const=True, default=False)
parser.add_argument('--show-text',  action='store_const', const=True, default=False)
args = parser.parse_args()

print(args)

argsd = vars(args)
query = []
for field in ['text', 'title']:
   exclude_field = 'exclude_' + field
   include_field = 'find_' + field
   if exclude_field in argsd:
      entry = argsd.get(exclude_field)
      if entry: query.append(('-', field, entry))
   if include_field in argsd:
      entry = argsd.get(include_field)
      if entry: query.append(('', field, entry))

db = ftsdb.get()
gen = db.fts_search(query)

for row in gen:
   for field in row.keys():
      show_field = "show_" + field
      if argsd.get(show_field, None):
         if len(row[field]) > 0:
            print("%s = %s" % (field, row[field]))
   print()

{'fraction_non_alphanumeric': 0.15156375300721733, 'fraction_numerical': 0.0008019246190858059, 'mean_word_length': 4.123203285420945, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5536474', 'n_tokens_mistral': 833, 'n_tokens_neox': 808, 'n_words': 216}