<commit_before>from datetime import date
import os
import signal
import subprocess
from functools import wraps
import sys
from tqdm import tqdm
from fabric.decorators import task
from fabric.operations import local

import django


### helpers ###

os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings')
_django_setup = False
def setup_django(func):
    """
        For speed, avoid setting up django until we need it. Attach @setup_django to any tasks that rely on importing django packages.
    """
    @wraps(func)
    def wrapper(*args, **kwargs):
        global _django_setup
        if not _django_setup:
            sys.path.insert(0, '')
            django.setup()
            _django_setup = True
        return func(*args, **kwargs)
    return wrapper


@task(alias='run')
def run_django(port=None):
    if port is None:
        port = "0.0.0.0:8000" if os.environ.get('DOCKERIZED') else "127.0.0.1:8000"
    local('python manage.py runserver %s' % port)


@task
def run_frontend(port=None):
    node_proc = subprocess.Popen("npm run serve", shell=True, stdout=sys.stdout, stderr=sys.stderr)
    try:
        run_django()
    finally:
        os.kill(node_proc.pid, signal.SIGKILL)

### tasks ###

@task
@setup_django
def create_search_index():
    """ Create (or recreate) the search_view materialized view """
    from main.models import SearchIndex
    SearchIndex.create_search_index()


@task
@setup_django
def refresh_search_index():
    """ Update an existing search_view materialized view; will fail if create_search_index hasn't been run once """
    from main.models import SearchIndex
    SearchIndex.refresh_search_index()


@task
@setup_django
def report_tags():
    """ Report all HTML tags, attributes, and styles used in ContentNode.headnote and TextBlock.content. """
    from main.models import TextBlock, ContentNode
    from main.utils import parse_html_fragment
    from pprint import pprint
    import re

    tags = {}
    tag_styles = {}

    sanitized_fields = ((TextBlock, 'content'), (ContentNode, 'headnote'))

    for model, field in sanitized_fields:
        print("Getting tags from %s.%s" % (model.__name__, field))
        for obj in tqdm(model.objects.exclude(**{field: ''}).exclude(**{field: None}), total=float("inf")):
            tree = parse_html_fragment(getattr(obj, field))
            for el in tree.iter():
                tag = tags.setdefault(el.tag, set())
                for k, v in el.items():
                    tag.add(k)
                    if k == 'style':
                        tag_style = tag_styles.setdefault(el.tag, set())
                        v = re.compile(r'url\s*\(\s*[^\s)]+?\s*\)\s*').sub(' ', v)  # remove url()
                        for pair in v.split(';'):
                            tag_style.add(pair.split(':', 1)[0].strip().lower())

    print("Tags and attributes in use:")
    pprint(tags)

    print("Styles in use:")
    for tag in sorted(tag_styles.keys()):
        styles = tag_styles[tag]
        print("%s[%s]" % (tag, ",".join(s for s in styles if s)))

    print("Unique styles in use:")
    styles = set()
    for s in tag_styles.values():
        styles |= s
        print(" ".join(sorted(styles)))


@task
@setup_django
def compare_sanitized_html():
    """
        Report all changes that result from applying sanitize() to ContentNode.headnote and TextBlock.content.
    """
    from main.models import TextBlock, ContentNode
    from main.sanitize import sanitize
    from main.utils import parse_html_fragment, elements_equal

    sanitized_fields = (
        (TextBlock, 'content'),
        (ContentNode, 'headnote'),
    )
    for model, field in sanitized_fields:
        print("Getting tags from %s.%s" % (model.__name__, field))
        for obj in tqdm(model.objects.exclude(**{field: ''}).exclude(**{field: None}).iterator(), total=float("inf")):
            content = getattr(obj, field)
            sanitized = sanitize(content)
            if content != sanitized:
                content_tree = parse_html_fragment(content)
                sanitized_tree = parse_html_fragment(sanitized)
                elements_equal(content_tree, sanitized_tree, tidy_style_attrs=True)


@task
@setup_django
def load_uscode_index(index='uscode_index.jsonl', effective_date=date(2018,1,1)):
    """
    Import a jsonl file into the search index for US Code support
    """
    import json
    from main.models import USCodeIndex
    with open(index) as index_file:
        entries = []
        for line in index_file:
            ind_dict = json.loads(line)
            entries.append(USCodeIndex(citation=ind_dict['citation'], effective_date=effective_date, title=ind_dict['title'], gpo_id=ind_dict['gpo_id'], lii_url=ind_dict['lii_link']))
        USCodeIndex.objects.bulk_create(entries)
        for ind in USCodeIndex.objects.all():
            ind.save()

if __name__ == "__main__":
    # allow tasks to be run as "python fabfile.py task"
    # this is convenient for profiling, e.g. "kernprof -l fabfile.py refresh_search_index"
    from fabric.main import main
    main()

@task
@setup_django
def migrate_cases():
    from datetime import timedelta
    from itertools import groupby
    from pyquery import PyQuery
    from main.models import Case, LegalDocumentSource, LegalDocument, ContentNode, ContentAnnotation
    cap = LegalDocumentSource.objects.filter(name="CAP").get()
    legacy = LegalDocumentSource.objects.filter(name="Legacy").get()

    resource_map = {k:[x for x in v] for k,v in groupby(ContentNode.objects.filter(resource_type='Case').all(), lambda x:x.resource_id)}
    used_cases = set(resource_map.keys())
    resource_count = len([1 for x in resource_map.values() for y in x])
    pre_res_count = ContentNode.objects.filter(resource_type='Case').count()
    print(f"Migrating {len(used_cases)} cases and {resource_count}[{pre_res_count}] resources")
    cases = list(Case.objects.filter(id__in=used_cases).all())
    fully_imported = 0
    legacy_docs = 0
    odd_imported = 0
    prior_ld_count = LegalDocument.objects.count()
    res_count = 0
    failed_import = 0
    for case in tqdm(cases):
        ld = None
        if not case.capapi_id:
            cites = [x['cite'] for x in case.citations if 'cite' in x]
            ld = LegalDocument.objects.create(source=legacy,
                                              short_name=case.name_abbreviation,<commit_msg>Fixup citeless edgecase in migration<commit_after>from datetime import date
import os
import signal
import subprocess
from functools import wraps
import sys
from tqdm import tqdm
from fabric.decorators import task
from fabric.operations import local

import django


### helpers ###

os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings')
_django_setup = False
def setup_django(func):
    """
        For speed, avoid setting up django until we need it. Attach @setup_django to any tasks that rely on importing django packages.
    """
    @wraps(func)
    def wrapper(*args, **kwargs):
        global _django_setup
        if not _django_setup:
            sys.path.insert(0, '')
            django.setup()
            _django_setup = True
        return func(*args, **kwargs)
    return wrapper


@task(alias='run')
def run_django(port=None):
    if port is None:
        port = "0.0.0.0:8000" if os.environ.get('DOCKERIZED') else "127.0.0.1:8000"
    local('python manage.py runserver %s' % port)


@task
def run_frontend(port=None):
    node_proc = subprocess.Popen("npm run serve", shell=True, stdout=sys.stdout, stderr=sys.stderr)
    try:
        run_django()
    finally:
        os.kill(node_proc.pid, signal.SIGKILL)

### tasks ###

@task
@setup_django
def create_search_index():
    """ Create (or recreate) the search_view materialized view """
    from main.models import SearchIndex
    SearchIndex.create_search_index()


@task
@setup_django
def refresh_search_index():
    """ Update an existing search_view materialized view; will fail if create_search_index hasn't been run once """
    from main.models import SearchIndex
    SearchIndex.refresh_search_index()


@task
@setup_django
def report_tags():
    """ Report all HTML tags, attributes, and styles used in ContentNode.headnote and TextBlock.content. """
    from main.models import TextBlock, ContentNode
    from main.utils import parse_html_fragment
    from pprint import pprint
    import re

    tags = {}
    tag_styles = {}

    sanitized_fields = ((TextBlock, 'content'), (ContentNode, 'headnote'))

    for model, field in sanitized_fields:
        print("Getting tags from %s.%s" % (model.__name__, field))
        for obj in tqdm(model.objects.exclude(**{field: ''}).exclude(**{field: None}), total=float("inf")):
            tree = parse_html_fragment(getattr(obj, field))
            for el in tree.iter():
                tag = tags.setdefault(el.tag, set())
                for k, v in el.items():
                    tag.add(k)
                    if k == 'style':
                        tag_style = tag_styles.setdefault(el.tag, set())
                        v = re.compile(r'url\s*\(\s*[^\s)]+?\s*\)\s*').sub(' ', v)  # remove url()
                        for pair in v.split(';'):
                            tag_style.add(pair.split(':', 1)[0].strip().lower())

    print("Tags and attributes in use:")
    pprint(tags)

    print("Styles in use:")
    for tag in sorted(tag_styles.keys()):
        styles = tag_styles[tag]
        print("%s[%s]" % (tag, ",".join(s for s in styles if s)))

    print("Unique styles in use:")
    styles = set()
    for s in tag_styles.values():
        styles |= s
        print(" ".join(sorted(styles)))


@task
@setup_django
def compare_sanitized_html():
    """
        Report all changes that result from applying sanitize() to ContentNode.headnote and TextBlock.content.
    """
    from main.models import TextBlock, ContentNode
    from main.sanitize import sanitize
    from main.utils import parse_html_fragment, elements_equal

    sanitized_fields = (
        (TextBlock, 'content'),
        (ContentNode, 'headnote'),
    )
    for model, field in sanitized_fields:
        print("Getting tags from %s.%s" % (model.__name__, field))
        for obj in tqdm(model.objects.exclude(**{field: ''}).exclude(**{field: None}).iterator(), total=float("inf")):
            content = getattr(obj, field)
            sanitized = sanitize(content)
            if content != sanitized:
                content_tree = parse_html_fragment(content)
                sanitized_tree = parse_html_fragment(sanitized)
                elements_equal(content_tree, sanitized_tree, tidy_style_attrs=True)


@task
@setup_django
def load_uscode_index(index='uscode_index.jsonl', effective_date=date(2018,1,1)):
    """
    Import a jsonl file into the search index for US Code support
    """
    import json
    from main.models import USCodeIndex
    with open(index) as index_file:
        entries = []
        for line in index_file:
            ind_dict = json.loads(line)
            repealed = ind_dict['title'].startswith("Repealed")
            entries.append(USCodeIndex(citation=ind_dict['citation'], effective_date=effective_date, title=ind_dict['title'], gpo_id=ind_dict['gpo_id'], lii_url=ind_dict['lii_link'], repealed=repealed))
        USCodeIndex.objects.bulk_create(entries)
        for ind in USCodeIndex.objects.all():
            ind.save()

if __name__ == "__main__":
    # allow tasks to be run as "python fabfile.py task"
    # this is convenient for profiling, e.g. "kernprof -l fabfile.py refresh_search_index"
    from fabric.main import main
    main()

@task
@setup_django
def migrate_cases():
    from datetime import timedelta
    from itertools import groupby
    from pyquery import PyQuery
    from main.models import Case, LegalDocumentSource, LegalDocument, ContentNode, ContentAnnotation
    cap = LegalDocumentSource.objects.filter(name="CAP").get()
    legacy = LegalDocumentSource.objects.filter(name="Legacy").get()

    resource_map = {k:[x for x in v] for k,v in groupby(ContentNode.objects.filter(resource_type='Case').all(), lambda x:x.resource_id)}
    used_cases = set(resource_map.keys())
    resource_count = len([1 for x in resource_map.values() for y in x])
    pre_res_count = ContentNode.objects.filter(resource_type='Case').count()
    print(f"Migrating {len(used_cases)} cases and {resource_count}[{pre_res_count}] resources")
    cases = list(Case.objects.filter(id__in=used_cases).all())
    fully_imported = 0
    legacy_docs = 0
    odd_imported = 0
    prior_ld_count = LegalDocument.objects.count()
    res_count = 0
    failed_import = 0
    for case in tqdm(cases):
        ld = None
        if not case.capapi_id:
            cites = [x['cite'] for x in case.citations if 'cite' in x] if case.citations else []
            ld = LegalDocument.objects.create(source=legacy,
                                              short_name=case.name_abbreviation,
{'fraction_non_alphanumeric': 0.11624498611539648, 'fraction_numerical': 0.005091021289725393, 'mean_word_length': 2.975774302361239, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5166737', 'n_tokens_mistral': 3979, 'n_tokens_neox': 3854, 'n_words': 1033}