<commit_before>import os
import string
from django.utils.text import slugify
from django.utils.timezone import now

os.environ['DJANGO_SETTINGS_MODULE'] = 'alert.settings'
import sys
execfile('/etc/courtlistener')
sys.path.append(INSTALL_ROOT)

from alert.lib.string_utils import anonymize, trunc
from alert.search.models import Document, save_doc_and_cite
from juriscraper.lib.string_utils import clean_string, harmonize, titlecase

import datetime
import re
import subprocess

BROWSER = 'firefox'


def merge_cases_simple(new, target_id):
    """Add `new` to the database, merging with target_id

     Merging is done by picking the best fields from each item.
    """
    target = Document.objects.get(pk=target_id)
    print "Merging %s with" % new.citation.case_name
    print "        %s" % target.citation.case_name

    if target.source == 'C':
        target.source = 'LC'
    elif target.source == 'R':
        target.source = 'LR'
    elif target.source == 'CR':
        target.source = 'LCR'

    # Recreate the slug from the new case name (this changes the URL, but the old will continue working)
    target.citation.slug = trunc(slugify(new.citation.case_name), 50)

    # Take the case name from the new item; they tend to be pretty good
    target.citation.case_name = new.citation.case_name

    # Add the docket number if the old doesn't exist, but keep the old if one does.
    if not target.citation.docket_number:
        target.citation.docket_number = new.citation.docket_number

    # Get the citations from the new item (ditch the old).
    target.citation.federal_cite_one = new.citation.federal_cite_one
    target.citation.federal_cite_two = new.citation.federal_cite_two
    target.citation.federal_cite_three = new.citation.federal_cite_three
    target.citation.state_cite_one = new.citation.state_cite_one
    target.citation.state_cite_two = new.citation.state_cite_two
    target.citation.state_cite_three = new.citation.state_cite_three
    target.citation.state_cite_regional = new.citation.state_cite_regional
    target.citation.specialty_cite_one = new.citation.specialty_cite_one
    target.citation.scotus_early_cite = new.citation.scotus_early_cite
    target.citation.lexis_cite = new.citation.lexis_cite
    target.citation.westlaw_cite = new.citation.westlaw_cite
    target.citation.neutral_cite = new.citation.neutral_cite

    # Add the URL if it's not a court one, replacing resource.org's info in some cases.
    if target.source == 'R':
        target.download_URL = new.download_URL

    # Add judge information if lacking. New is dirty, but better than none.
    if not target.judges:
        target.judges = new.judges

    # Add the text.
    target.html_lawbox, blocked = anonymize(new.html)
    if blocked:
        target.blocked = True
        target.date_blocked = now()

    target.extracted_by_ocr = False  # No longer true for any LB case.

    save_doc_and_cite(target, index=False)


def merge_cases_complex(case, target_ids):
    """Merge data from PRO with multiple cases that seem to be a match.

    The process here is a conservative one. We take *only* the information
    from PRO that is not already in CL in any form, and add only that.
    """<commit_msg>Tweaks.<commit_after>import os
import string
from django.utils.text import slugify
from django.utils.timezone import now

os.environ['DJANGO_SETTINGS_MODULE'] = 'alert.settings'
import sys
execfile('/etc/courtlistener')
sys.path.append(INSTALL_ROOT)

from alert.lib.string_utils import anonymize, trunc
from alert.search.models import Document, save_doc_and_cite
from juriscraper.lib.string_utils import clean_string, harmonize, titlecase

import datetime
import re
import subprocess

BROWSER = 'firefox'


def merge_cases_simple(new, target_id):
    """Add `new` to the database, merging with target_id

     Merging is done by picking the best fields from each item.
    """
    target = Document.objects.get(pk=target_id)
    print "Merging %s with" % new.citation.case_name
    print "        %s" % target.citation.case_name

    cached_source = target.source  # Original value is needed below.
    if target.source == 'C':
        target.source = 'LC'
    elif target.source == 'R':
        target.source = 'LR'
    elif target.source == 'CR':
        target.source = 'LCR'

    # Add the URL if it's not a court one, replacing resource.org's info in some cases.
    if cached_source == 'R':
        target.download_URL = new.download_URL

    # Recreate the slug from the new case name (this changes the URL, but the old will continue working)
    target.citation.slug = trunc(slugify(new.citation.case_name), 50)

    # Take the case name from the new item; they tend to be pretty good
    target.citation.case_name = new.citation.case_name

    # Add the docket number if the old doesn't exist, but keep the old if one does.
    if not target.citation.docket_number:
        target.citation.docket_number = new.citation.docket_number

    # Get the citations from the new item (ditch the old).
    target.citation.federal_cite_one = new.citation.federal_cite_one
    target.citation.federal_cite_two = new.citation.federal_cite_two
    target.citation.federal_cite_three = new.citation.federal_cite_three
    target.citation.state_cite_one = new.citation.state_cite_one
    target.citation.state_cite_two = new.citation.state_cite_two
    target.citation.state_cite_three = new.citation.state_cite_three
    target.citation.state_cite_regional = new.citation.state_cite_regional
    target.citation.specialty_cite_one = new.citation.specialty_cite_one
    target.citation.scotus_early_cite = new.citation.scotus_early_cite
    target.citation.lexis_cite = new.citation.lexis_cite
    target.citation.westlaw_cite = new.citation.westlaw_cite
    target.citation.neutral_cite = new.citation.neutral_cite

    # Add judge information if lacking. New is dirty, but better than none.
    if not target.judges:
        target.judges = new.judges

    # Add the text.
    target.html_lawbox, blocked = anonymize(new.html)
    if blocked:
        target.blocked = True
        target.date_blocked = now()

    target.extracted_by_ocr = False  # No longer true for any LB case.

    save_doc_and_cite(target, index=False)


def merge_cases_complex(case, target_ids):
    """Merge data from PRO with multiple cases that seem to be a match.

    The process here is a conservative one. We take *only* the information
    from PRO that is not already in CL in any form, and add only that.
    """
{'fraction_non_alphanumeric': 0.08066247508050912, 'fraction_numerical': 0.0006134028523232633, 'mean_word_length': 3.6222537207654146, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3935283', 'n_tokens_mistral': 2082, 'n_tokens_neox': 1945, 'n_words': 575}