<commit_before>from collections import Counter, OrderedDict

import pandas
from django.core.management import BaseCommand
from django.db.models import Q

from juriscraper.lib.string_utils import titlecase
from pandas import to_pickle
from unidecode import unidecode

from cl.search.models import Court, Docket

titles = [
    'judge', 'magistrate', 'district', 'chief', 'senior',
    'bankruptcy', 'mag.', 'magistrate-judge', 'mag/judge', 'mag',
    'visiting', 'special', 'senior-judge', 'master', 'u.s.magistrate',
]
blacklist = [
    'a998', 'agb', 'am', 'ca', 'cet', 'ch.', 'cla', 'clerk', 'cp', 'cvb', 'db',
    'debt-magistrate', 'discovery', 'dj', 'docket', 'duty', 'duty', 'ec', 'eck',
    'general', 'grc', 'gs', 'hhl', 'hon', 'honorable', 'inactive', 'jne', 'jv',
    'kec', 'law', 'lc', 'llh', 'lq', 'maryland', 'mediator', 'merged', 'mj',
    'mmh', 'msh', 'mwd', 'no', 'none', 'prisoner', 'pslc', 'pro', 'pso', 'pt',
    'rmh', 'se', 'sf', 'successor', 'u.s.', 'tjc', 'unassigned', 'unassigned2',
    'unassigneddj', 'unknown', 'us', 'usdc', 'vjdistrict',
]

judge_normalizers = {
    # Generic Judge
    '': 'jud',
    'Judge': 'jud',
    'Judge Judge': 'jud',
    'District Judge': 'jud',
    'Visiting Judge': 'jud',
    'Bankruptcy Judge': 'jud',

    # Magistrate
    'Mag': 'mag',
    'Mag Judge': 'mag',
    'mag/judge': 'mag',
    'Magistrate': 'mag',
    'Magistrate Judge': 'mag',
    'Magistrate-Judge': 'mag',
    'Magistrate Judge Mag': 'mag',
    'Magistrate Judge Magistrate': 'mag',
    'Magistrate Judge Magistrate Judge': 'mag',

    # Chief
    'Chief': 'c-jud',
    'Chief Judge': 'c-jud',
    'Chief District Judge': 'c-jud',
    'Senior Judge': 'c-jud',
    'Senior-Judge': 'c-jud',

    # Chief Magistrate
    'Chief Magistrate': 'c-mag',
    'Chief Magistrate Judge': 'c-mag',

    # Special Master
    'Special Master': 'spec-m',
    'Chief Special Master': 'c-spec-m',
}


def normalize_judge_titles(title):
    """Normalize judge titles

    Take in a string like "Magistrate Judge" and return the normalized
    abbreviation from the POSITION_TYPES variable. Assumes that input is
    titlecased.

    Also normalizes things like:
     - District Judge --> Judge
     - Blank --> Judge
     - Bankruptcy Judge --> Judge
    """
    return judge_normalizers[title]


def split_name_title(judge):
    """Split a value from PACER and return the title and name"""
    judge = judge.replace(',', '')
    words = judge.lower().split()

    # Nuke bad junk (punct., j, blacklist, etc.)
    clean_words = []
    for i, w in enumerate(words):
        if any(['(' in w,
                ')' in w,
                w.startswith('-'),
                w.startswith('~'),
                (len(w) > 2 and '.' in w),
                (i == 0 and w == 'j.'),
                w in blacklist]):
            continue
        clean_words.append(w)

    title_words = []
    name_words = []
    for w in clean_words:
        if w in titles:
            title_words.append(w)
        else:
            name_words.append(w)
<commit_msg>Adds another variation and a catch for ones we don't know.<commit_after>from collections import Counter, OrderedDict

import pandas
from django.core.management import BaseCommand
from django.db.models import Q

from juriscraper.lib.string_utils import titlecase
from pandas import to_pickle
from unidecode import unidecode

from cl.search.models import Court, Docket

titles = [
    'judge', 'magistrate', 'district', 'chief', 'senior',
    'bankruptcy', 'mag.', 'magistrate-judge', 'mag/judge', 'mag',
    'visiting', 'special', 'senior-judge', 'master', 'u.s.magistrate',
]
blacklist = [
    'a998', 'agb', 'am', 'ca', 'cet', 'ch.', 'cla', 'clerk', 'cp', 'cvb', 'db',
    'debt-magistrate', 'discovery', 'dj', 'docket', 'duty', 'duty', 'ec', 'eck',
    'general', 'grc', 'gs', 'hhl', 'hon', 'honorable', 'inactive', 'jne', 'jv',
    'kec', 'law', 'lc', 'llh', 'lq', 'maryland', 'mediator', 'merged', 'mj',
    'mmh', 'msh', 'mwd', 'no', 'none', 'prisoner', 'pslc', 'pro', 'pso', 'pt',
    'rmh', 'se', 'sf', 'successor', 'u.s.', 'tjc', 'unassigned', 'unassigned2',
    'unassigneddj', 'unknown', 'us', 'usdc', 'vjdistrict',
]

judge_normalizers = {
    # Generic Judge
    '': 'jud',
    'Judge': 'jud',
    'Judge Judge': 'jud',
    'District Judge': 'jud',
    'Visiting Judge': 'jud',
    'Bankruptcy': 'jud',
    'Bankruptcy Judge': 'jud',

    # Magistrate
    'Mag': 'mag',
    'Mag Judge': 'mag',
    'mag/judge': 'mag',
    'Magistrate': 'mag',
    'Magistrate Judge': 'mag',
    'Magistrate-Judge': 'mag',
    'Magistrate Judge Mag': 'mag',
    'Magistrate Judge Magistrate': 'mag',
    'Magistrate Judge Magistrate Judge': 'mag',

    # Chief
    'Chief': 'c-jud',
    'Chief Judge': 'c-jud',
    'Chief District Judge': 'c-jud',
    'Senior Judge': 'c-jud',
    'Senior-Judge': 'c-jud',

    # Chief Magistrate
    'Chief Magistrate': 'c-mag',
    'Chief Magistrate Judge': 'c-mag',

    # Special Master
    'Special Master': 'spec-m',
    'Chief Special Master': 'c-spec-m',
}


def normalize_judge_titles(title):
    """Normalize judge titles

    Take in a string like "Magistrate Judge" and return the normalized
    abbreviation from the POSITION_TYPES variable. Assumes that input is
    titlecased.

    Also normalizes things like:
     - District Judge --> Judge
     - Blank --> Judge
     - Bankruptcy Judge --> Judge
    """
    return judge_normalizers.get(title, 'UNKNOWN: %s' % title)


def split_name_title(judge):
    """Split a value from PACER and return the title and name"""
    judge = judge.replace(',', '')
    words = judge.lower().split()

    # Nuke bad junk (punct., j, blacklist, etc.)
    clean_words = []
    for i, w in enumerate(words):
        if any(['(' in w,
                ')' in w,
                w.startswith('-'),
                w.startswith('~'),
                (len(w) > 2 and '.' in w),
                (i == 0 and w == 'j.'),
                w in blacklist]):
            continue
        clean_words.append(w)

    title_words = []
    name_words = []
    for w in clean_words:
        if w in titles:
            title_words.append(w)
        else:
            name_words.append(w)

{'fraction_non_alphanumeric': 0.18057124415039535, 'fraction_numerical': 0.0019364208487978054, 'mean_word_length': 2.809465273509527, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2274977', 'n_tokens_mistral': 2239, 'n_tokens_neox': 2059, 'n_words': 603}