<commit_before>"""
Process here will be to iterate over every item in the SCDB and to locate it in
CourtListener.

Location is done by:
 - Looking in the `scdb_id` field. During the first run of this
   program we expect this to fail for all items since they will not have this
   field populated yet. During subsequent runs, this field will have hits and
   will provide improved performance.
 - Looking for matching U.S. and docket number.

Once located, we update items:
 - Case name -- will this automatically fix the docket as well?
 - Citations (Lexis, US, L.Ed., etc.)
 - Docket number?
 - scdb_id
 - votes_majority & votes_minority
 - decision_direction
"""
import csv
import os

from django.core.management import BaseCommand

from cl.search.models import OpinionCluster
from datetime import date, datetime

SCDB_FILENAME = os.path.join(
    '/var/www/courtlistener/cl/corpus_importer/scdb/data',
    'SCDB_2015_01_caseCentered_Citation.csv'
)
SCDB_BEGINS = date(1946, 11, 18)
SCDB_ENDS = date(2014, 6, 19)

# Relevant numbers:
#  - 7907: After this point we don't seem to have any citations for items.


class Command(BaseCommand):
    help = 'Import data from the SCDB Case Centered CSV.'

    def add_arguments(self, parser):
        parser.add_argument(
            '--debug',
            action='store_true',
            default=False,
            help="Don't change the data. Only pretend."
        )

    def handle(self, *args, **options):
        self.debug = options['debug']
        self.iterate_scdb_and_take_actions(
            action_zero=lambda *args, **kwargs: None,
            action_one=self.enhance_item_with_scdb,
            action_many=self.get_human_review,
        )

    @staticmethod
    def set_if_falsy(obj, attribute, new_value):
        """Check if the value passed in is Falsy. If so, set it to the value of
        new_value.
        """
        current_value = getattr(obj, attribute)
        if current_value is not None:
            current_value = current_value.strip()
        if not current_value:
            setattr(obj, attribute, new_value)
        else:
            # Report if there's a difference -- that might spell trouble.
            if ''.join(current_value.split()) != ''.join(new_value.split()):
                print ("      WARNING: Didn't set '{attr}' attribute on obj "
                       "{obj_id} because it already had a value, but the new "
                       "value ('{new}') differs from current value "
                       "('{current}').".format(
                    attr=attribute,
                    obj_id=obj.pk,
                    new=new_value,
                    current=current_value,
                ))

    def enhance_item_with_scdb(self, cluster, scdb_info):
        """Good news: A single Cluster object was found for the SCDB record.

        Take that item and enhance it with the SCDB content.
        """
        print '    --> Enhancing cluster %s with data from SCDB.' % cluster.pk<commit_msg>Updates for ints.<commit_after>"""
Process here will be to iterate over every item in the SCDB and to locate it in
CourtListener.

Location is done by:
 - Looking in the `scdb_id` field. During the first run of this
   program we expect this to fail for all items since they will not have this
   field populated yet. During subsequent runs, this field will have hits and
   will provide improved performance.
 - Looking for matching U.S. and docket number.

Once located, we update items:
 - Case name -- will this automatically fix the docket as well?
 - Citations (Lexis, US, L.Ed., etc.)
 - Docket number?
 - scdb_id
 - votes_majority & votes_minority
 - decision_direction
"""
import csv
import os

from django.core.management import BaseCommand

from cl.search.models import OpinionCluster
from datetime import date, datetime

SCDB_FILENAME = os.path.join(
    '/var/www/courtlistener/cl/corpus_importer/scdb/data',
    'SCDB_2015_01_caseCentered_Citation.csv'
)
SCDB_BEGINS = date(1946, 11, 18)
SCDB_ENDS = date(2014, 6, 19)

# Relevant numbers:
#  - 7907: After this point we don't seem to have any citations for items.


class Command(BaseCommand):
    help = 'Import data from the SCDB Case Centered CSV.'

    def add_arguments(self, parser):
        parser.add_argument(
            '--debug',
            action='store_true',
            default=False,
            help="Don't change the data. Only pretend."
        )

    def handle(self, *args, **options):
        self.debug = options['debug']
        self.iterate_scdb_and_take_actions(
            action_zero=lambda *args, **kwargs: None,
            action_one=self.enhance_item_with_scdb,
            action_many=self.get_human_review,
        )

    @staticmethod
    def set_if_falsy(obj, attribute, new_value):
        """Check if the value passed in is Falsy. If so, set it to the value of
        new_value.
        """
        current_value = getattr(obj, attribute)
        if current_value is not None and isinstance(current_value, basestring):
            current_value = current_value.strip()
        if not current_value and current_value != 0:
            setattr(obj, attribute, new_value)
        else:
            # Report if there's a difference -- that might spell trouble.
            if ''.join(current_value.split()) != ''.join(new_value.split()):
                print ("      WARNING: Didn't set '{attr}' attribute on obj "
                       "{obj_id} because it already had a value, but the new "
                       "value ('{new}') differs from current value "
                       "('{current}').".format(
                    attr=attribute,
                    obj_id=obj.pk,
                    new=new_value,
                    current=current_value,
                ))

    def enhance_item_with_scdb(self, cluster, scdb_info):
        """Good news: A single Cluster object was found for the SCDB record.

        Take that item and enhance it with the SCDB content.
        """
        print '    --> Enhancing cluster %s with data from SCDB.' % cluster.pk
{'fraction_non_alphanumeric': 0.08828858507659364, 'fraction_numerical': 0.008400592983034097, 'mean_word_length': 2.327123287671233, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5102888', 'n_tokens_mistral': 1790, 'n_tokens_neox': 1692, 'n_words': 595}