<commit_before>import os
import csv
from functools import reduce
from opencivicdata.models import Bill


def get_all_subjects(billset=None):
    if not billset:
        billset = Bill.objects.all()

    subjects = reduce(lambda a,b: a+b, billset.values_list('subject', flat=True), [])
    # deduped list
    return sorted(set(subjects))


def get_all_locations():
    with open(os.path.join(os.path.dirname(__file__), '..', 'fl', 'places.csv')) as places:
        place_csv = csv.DictReader(places)
        places = sorted(row['name'] for row in place_csv)
    return places<commit_msg>Make list of locations a set to dedupe closes #104<commit_after>import os
import csv
from functools import reduce
from opencivicdata.models import Bill


def get_all_subjects(billset=None):
    if not billset:
        billset = Bill.objects.all()

    subjects = reduce(lambda a,b: a+b, billset.values_list('subject', flat=True), [])
    # deduped list
    return sorted(set(subjects))


def get_all_locations():
    with open(os.path.join(os.path.dirname(__file__), '..', 'fl', 'places.csv')) as places:
        place_csv = csv.DictReader(places)
        places = sorted(set(row['name'] for row in place_csv))
    return places
{'fraction_non_alphanumeric': 0.12273476112026359, 'fraction_numerical': 0.002471169686985173, 'mean_word_length': 3.9390243902439024, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2703386', 'n_tokens_mistral': 412, 'n_tokens_neox': 391, 'n_words': 102}