<commit_before>#!/usr/bin/env python

try:
    import ujson as json
except ImportError:
    try:
        import simplejson as json
    except ImportError:
        import json

import argparse
import codecs
import collections
import datetime
import itertools
import os
import operator
import re

import dateutil.parser
import jinja2


REPEAL_REGEXES = (
    re.compile(r'(repeal|deauthorize|defund).*(patient protection|affordable care act|obamacare)', re.I),
    re.compile(r'prohibit .* from enforcing the Patient Protection and Affordable Care Act', re.I),
    re.compile(r'no.*Internal Revenue Service.*carry out.* Patient Protection and Affordable Care Act', re.I),
    re.compile(r'To prevent implementation and enforcement of Obamacare.', re.I)
)

PPACA_BILL_ID = 'hr3590-111'

# most of these are attempts to repeal some Medicare portion of the ACA
NOT_REPEAL_BILL_IDS = frozenset([
    'hr1270-114',
    'hr1580-112',
    'hr2453-113',
    'hr452-112',
    'hr584-112',
    'hr6303-111',
    'hr1159-112',
    'hr763-113',
    'hr351-113',
    'hr1190-114',
    'hr1370-112',
    'hr6502-111',
    'hr928-114',
    'hr2835-113',
    'hr588-114',
    'hr2488-114',
    'hr1213-112',
    'hr3633-112',
    'hr5433-111',  # this one seems extra scummy
])

DEFINITELY_REPEAL_BILL_IDS = frozenset([
    'hr683-114',
    'hr5570-111',
    'hr2087-113',
])


def is_repeal(hr_summary_dict):
    if hr_summary_dict['bill_id'] in NOT_REPEAL_BILL_IDS:
        return False
    if hr_summary_dict['bill_id'] in DEFINITELY_REPEAL_BILL_IDS:
        return True
    if any(r.search(hr_summary_dict['official_title']) for r in REPEAL_REGEXES):
        return True
    titles = [hr_summary_dict.get(f) for f in ('official_title', 'short_title')]
    titles = map(lambda x: x.lower() if x else '', titles)
    if any('nobamacare' in title for title in titles):
        return True
    if any('defund obamacare' in title for title in titles):
        return True
    return False
<commit_msg>Remove a bill that should not be there<commit_after>#!/usr/bin/env python

try:
    import ujson as json
except ImportError:
    try:
        import simplejson as json
    except ImportError:
        import json

import argparse
import codecs
import collections
import datetime
import itertools
import os
import operator
import re

import dateutil.parser
import jinja2


REPEAL_REGEXES = (
    re.compile(r'(repeal|deauthorize|defund).*(patient protection|affordable care act|obamacare)', re.I),
    re.compile(r'prohibit .* from enforcing the Patient Protection and Affordable Care Act', re.I),
    re.compile(r'no.*Internal Revenue Service.*carry out.* Patient Protection and Affordable Care Act', re.I),
    re.compile(r'To prevent implementation and enforcement of Obamacare.', re.I)
)

PPACA_BILL_ID = 'hr3590-111'

# most of these are attempts to repeal some Medicare portion of the ACA
NOT_REPEAL_BILL_IDS = frozenset([
    'hr1270-114',
    'hr1580-112',
    'hr2453-113',
    'hr452-112',
    'hr584-112',
    'hr6303-111',
    'hr1159-112',
    'hr763-113',
    'hr351-113',
    'hr1190-114',
    'hr1370-112',
    'hr6502-111',
    'hr928-114',
    'hr2835-113',
    'hr588-114',
    'hr2488-114',
    'hr1213-112',
    'hr3633-112',
    'hr5433-111',  # this one seems extra scummy
    'hr5110-113',
])

DEFINITELY_REPEAL_BILL_IDS = frozenset([
    'hr683-114',
    'hr5570-111',
    'hr2087-113',
])


def is_repeal(hr_summary_dict):
    if hr_summary_dict['bill_id'] in NOT_REPEAL_BILL_IDS:
        return False
    if hr_summary_dict['bill_id'] in DEFINITELY_REPEAL_BILL_IDS:
        return True
    if any(r.search(hr_summary_dict['official_title']) for r in REPEAL_REGEXES):
        return True
    titles = [hr_summary_dict.get(f) for f in ('official_title', 'short_title')]
    titles = map(lambda x: x.lower() if x else '', titles)
    if any('nobamacare' in title for title in titles):
        return True
    if any('defund obamacare' in title for title in titles):
        return True
    return False

{'fraction_non_alphanumeric': 0.11380084851509857, 'fraction_numerical': 0.0791115547791365, 'mean_word_length': 3.356521739130435, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '441787', 'n_tokens_mistral': 1651, 'n_tokens_neox': 1406, 'n_words': 368}