<commit_before># $Id$

import logging
import traceback

import pyparsing as pyp
import PyLucene

import canary.loader
from canary.concept import Concept
from canary.gazeteer import Feature
from canary.pubmed import Journal
from canary.study import Study
from canary.utils import render_capitalized

# All possible search fields, indexed by all valid forms.
SEARCH_FIELDS = {
    '1au':                  'first-author',
    'ab':                   'abstract',
    'abstract':             'abstract',
    'af':                   'affiliation', 
    'affiliation':          'affiliation', 
    'all':                  'all',
    'au':                   'author',
    'author':               'author',
    'authors':              'author',
    'date':                 'pubdate',
    'exp':                  'exposures',
    'exposure':             'exposures',
    'exposures':            'exposures',
    'gn':                   'grantnum',
    'grantnum':             'grantnum',
    'has-exposures':        'has-exposures', 
    'has-exposure-linkage': 'has-exposure-linkage',
    'has-genomic':          'has-genomic',
    'has-interspecies':     'has-interspecies',
    'has-outcomes':         'has-outcomes',
    'has-outcome-linkage':  'has-outcome-linkage',
    'has-relationships':    'has-relationships',
<<<<<<< .working
    'is':                   'issue', 
    'issn':                 'issn',
    'issue':                'issue',
    'jn':                   'journal',
    'journal':              'journal', 
    'keyword':              'keyword',
    'keywords':             'keyword',
    'kw':                   'keyword',
    'loc':                  'location',
    'location':             'location',
    'locations':            'location',
    'me':                   'methodology',
    'meth':                 'methodology',<commit_msg>Oops, left in a merge conflict delimiter. :(<commit_after># $Id$

import logging
import traceback

import pyparsing as pyp
import PyLucene

import canary.loader
from canary.concept import Concept
from canary.gazeteer import Feature
from canary.pubmed import Journal
from canary.study import Study
from canary.utils import render_capitalized

# All possible search fields, indexed by all valid forms.
SEARCH_FIELDS = {
    '1au':                  'first-author',
    'ab':                   'abstract',
    'abstract':             'abstract',
    'af':                   'affiliation', 
    'affiliation':          'affiliation', 
    'all':                  'all',
    'au':                   'author',
    'author':               'author',
    'authors':              'author',
    'date':                 'pubdate',
    'exp':                  'exposures',
    'exposure':             'exposures',
    'exposures':            'exposures',
    'gn':                   'grantnum',
    'grantnum':             'grantnum',
    'has-exposures':        'has-exposures', 
    'has-exposure-linkage': 'has-exposure-linkage',
    'has-genomic':          'has-genomic',
    'has-interspecies':     'has-interspecies',
    'has-outcomes':         'has-outcomes',
    'has-outcome-linkage':  'has-outcome-linkage',
    'has-relationships':    'has-relationships',
    'is':                   'issue', 
    'issn':                 'issn',
    'issue':                'issue',
    'jn':                   'journal',
    'journal':              'journal', 
    'keyword':              'keyword',
    'keywords':             'keyword',
    'kw':                   'keyword',
    'loc':                  'location',
    'location':             'location',
    'locations':            'location',
    'me':                   'methodology',
    'meth':                 'methodology',
{'fraction_non_alphanumeric': 0.13555675094136632, 'fraction_numerical': 0.0005379236148466917, 'mean_word_length': 1.48596256684492, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7082017', 'n_tokens_mistral': 1105, 'n_tokens_neox': 1026, 'n_words': 235}