<commit_before>"""Define search indexes for hs_core module."""

from haystack import indexes
from hs_core.models import BaseResource
from hs_geographic_feature_resource.models import GeographicFeatureMetaData
from hs_app_netCDF.models import NetcdfMetaData
from ref_ts.models import RefTSMetadata
from hs_app_timeseries.models import TimeSeriesMetaData
from django.db.models import Q
from datetime import datetime
from nameparser import HumanName
import probablepeople
from string import maketrans
import logging
# # SOLR extension needs to be installed for this to work
# from haystack.utils.geo import Point


def remove_whitespace(thing):
    intab = ""
    outtab = ""
    trantab = maketrans(intab, outtab)
    return str(thing).translate(trantab, " \t\r\n")


def normalize_name(name):
    """
    Normalize a name for sorting.

    This uses two powerful python libraries for differing reasons.

    `probablepeople` contains a discriminator between company and person names.
    This is used to determine whether to parse into last, first, middle or to
    leave the name alone.

    However, the actual name parser in `probablepeople` is unnecessarily complex,
    so that strings that it determines to be human names are parsed instead by
    `nameparser`.

    """
    sname = name.encode('utf-8').strip()  # remove spaces
    try:
        _, type = probablepeople.tag(sname)  # discard parser result
    except probablepeople.RepeatedLabelError:  # if it can't understand the name, punt
        return sname

    if type == 'Corporation':
        return sname  # do not parse and reorder company names

    # treat anything else as a human name
    nameparts = HumanName(sname)
    normalized = nameparts.last.capitalize()
    if nameparts.suffix:
        normalized = normalized + ' ' + nameparts.suffix
    normalized = normalized + ','
    if nameparts.title:
        normalized = normalized + ' ' + nameparts.title
    if nameparts.first:
        normalized = normalized + ' ' + nameparts.first.capitalize()
    if nameparts.middle:
        normalized = ' ' + normalized + ' ' + nameparts.middle.capitalize()
    return normalized.strip()
<commit_msg>Fix SOLR indexing bug involving unicode author names.<commit_after>"""Define search indexes for hs_core module."""

from haystack import indexes
from hs_core.models import BaseResource
from hs_geographic_feature_resource.models import GeographicFeatureMetaData
from hs_app_netCDF.models import NetcdfMetaData
from ref_ts.models import RefTSMetadata
from hs_app_timeseries.models import TimeSeriesMetaData
from django.db.models import Q
from datetime import datetime
from nameparser import HumanName
import probablepeople
from string import maketrans
import logging
# # SOLR extension needs to be installed for this to work
# from haystack.utils.geo import Point


def remove_whitespace(thing):
    intab = ""
    outtab = ""
    trantab = maketrans(intab, outtab)
    return str(thing).translate(trantab, " \t\r\n")


def normalize_name(name):
    """
    Normalize a name for sorting.

    This uses two powerful python libraries for differing reasons.

    `probablepeople` contains a discriminator between company and person names.
    This is used to determine whether to parse into last, first, middle or to
    leave the name alone.

    However, the actual name parser in `probablepeople` is unnecessarily complex,
    so that strings that it determines to be human names are parsed instead by
    `nameparser`.

    """
    sname = name.encode('utf-8').strip()  # remove spaces
    try:
        # probablepeople doesn't understand utf-8 encoding. Hand it pure unicode.
        _, type = probablepeople.tag(name)  # discard parser result
    except probablepeople.RepeatedLabelError:  # if it can't understand the name, punt
        return sname

    if type == 'Corporation':
        return sname  # do not parse and reorder company names

    # treat anything else as a human name
    nameparts = HumanName(sname)
    normalized = nameparts.last.capitalize()
    if nameparts.suffix:
        normalized = normalized + ' ' + nameparts.suffix
    normalized = normalized + ','
    if nameparts.title:
        normalized = normalized + ' ' + nameparts.title
    if nameparts.first:
        normalized = normalized + ' ' + nameparts.first.capitalize()
    if nameparts.middle:
        normalized = ' ' + normalized + ' ' + nameparts.middle.capitalize()
    return normalized.strip()

{'fraction_non_alphanumeric': 0.0718294051627385, 'fraction_numerical': 0.0006734006734006734, 'mean_word_length': 3.556237218813906, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1276346', 'n_tokens_mistral': 1229, 'n_tokens_neox': 1179, 'n_words': 436}