<commit_before>import logging
import os
import pandas as pd
from django.core.management import BaseCommand
from django.conf import settings

from cl.corpus_importer.tasks import download_recap_item
from cl.recap.utils import (
    get_docketxml_url, get_pdf_url, get_document_filename, get_docket_filename
)

CSV_PATH = os.path.join(settings.BASE_DIR, 'cl', 'corpus_importer', 'recap',
                        "document_table.csv")
logger = logging.getLogger(__name__)


def load_csv():
    """Load the CSV data using pandas

    This CSV is generated with:

        mysql recap_prod -p -u recap < export.sql > out.csv

    export.sql contains:

        SELECT
            court, casenum, docnum, subdocnum, modified
        FROM
            uploads_document
        ORDER BY
            casenum
        WHERE
            available = 1;

    The resulting file is tab separated, and pandas will handle that just fine.
    """
    return pd.read_csv(CSV_PATH, delimiter='\t', dtype={
        'casenum': object,
        'docnum': object,
        'court': object,
    })


def make_download_tasks(data):
    """For every item in the CSV, send it to Celery for processing"""
    previous_casenum = None
    for index, item in data.iterrows():
        if item['casenum'] != previous_casenum:
            # New case, get the docket before getting the pdf
            logger.info("New docket found with casenum: %s" % item['casenum'])
            previous_casenum = item['casenum']
            filename = get_docket_filename(item['court'], item['casenum'])
            url = get_docketxml_url(item['court'], item['casenum'])
            download_recap_item.delay(url, filename)

        # Get the document
        filename = get_document_filename(item['court'], item['casenum'],
                                         item['docnum'], item['subdocnum'])
        url = get_pdf_url(item['court'], item['casenum'], filename)
        download_recap_item.delay(url, filename)


class Command(BaseCommand):
    help = ('Using a local CSV, download the XML data for RECAP content. '
            'Output is sent to the log.')

    def handle(self, *args, **kwargs):
        data = load_csv()
        make_download_tasks(data)<commit_msg>New version processes 50 at a time.<commit_after>import logging
import os
import pandas as pd
from celery.task import TaskSet
from django.core.management import BaseCommand
from django.conf import settings

from cl.corpus_importer.tasks import download_recap_item
from cl.recap.utils import (
    get_docketxml_url, get_pdf_url, get_document_filename, get_docket_filename
)

CSV_PATH = os.path.join(settings.BASE_DIR, 'cl', 'corpus_importer', 'recap',
                        "document_table.csv")
logger = logging.getLogger(__name__)


def load_csv():
    """Load the CSV data using pandas

    This CSV is generated with:

        mysql recap_prod -p -u recap < export.sql > out.csv

    export.sql contains:

        SELECT
            court, casenum, docnum, subdocnum, modified
        FROM
            uploads_document
        ORDER BY
            casenum
        WHERE
            available = 1;

    The resulting file is tab separated, and pandas will handle that just fine.
    """
    with open(CSV_PATH, 'r') as f:
        line_count = sum(1 for line in f) - 1
    data = pd.read_csv(CSV_PATH, delimiter='\t', dtype={
        'casenum': object,
        'docnum': object,
        'court': object,
    })
    return data, line_count


def make_download_tasks(data, line_count):
    """For every item in the CSV, send it to Celery for processing"""
    previous_casenum = None
    subtasks = []
    completed = 0
    for index, item in data.iterrows():
        last_item = (line_count == completed + 1)
        if item['casenum'] != previous_casenum:
            # New case, get the docket before getting the pdf
            logger.info("New docket found with casenum: %s" % item['casenum'])
            previous_casenum = item['casenum']
            filename = get_docket_filename(item['court'], item['casenum'])
            url = get_docketxml_url(item['court'], item['casenum'])
            subtasks.append(download_recap_item.subtask((url, filename)))

        # Get the document
        filename = get_document_filename(item['court'], item['casenum'],
                                         item['docnum'], item['subdocnum'])
        url = get_pdf_url(item['court'], item['casenum'], filename)
        subtasks.append(download_recap_item.subtask((url, filename)))

        # Every fifty items send the subtasks to Celery.
        if (len(subtasks) >= 50) or last_item:
            msg = ("Sent %s subtasks to celery. We have processed %s "
                   "rows so far." % (len(subtasks), completed))
            logger.info(msg)
            print msg
            job = TaskSet(tasks=subtasks)
            job.apply_async().join()
            subtasks = []

        completed += 1


class Command(BaseCommand):
    help = ('Using a local CSV, download the XML data for RECAP content. '
            'Output is sent to the log.')

    def handle(self, *args, **kwargs):
        data, line_count = load_csv()
        make_download_tasks(data, line_count)
{'fraction_non_alphanumeric': 0.10186789909493549, 'fraction_numerical': 0.0021182360870402464, 'mean_word_length': 2.50236008091706, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4330696', 'n_tokens_mistral': 1514, 'n_tokens_neox': 1447, 'n_words': 441}