<commit_before>import requests
import hashlib
from datetime import datetime
from comport.department.models import Extractor
from comport.utils import random_string, random_date
import random


class JSONTestClient(object):

    def run(self, department, mutators=[]):
        baseurl = "http://localhost:5000/"
        password = "<PASSWORD>"
        extractor, envs = Extractor.from_department_and_password(
            department=department, password=password)
        comport_username = extractor.username
        comport_password = password

        data = []

        complaints = self.make_complaints()

        for mutator in mutators:
            complaints = mutator.mutate(complaints)

        data.extend(complaints)

        for i in range(0, len(data), 100):
            chunk = data[i:i + 100]
            payload = {'data': chunk}

            url = baseurl + "data/complaints"

            print(payload)

            p = requests.post(url, auth=(comport_username,
                                         comport_password), json=payload)
            if p.status_code != 200:
                print("error: %s" % p.text.encode("utf-8", "ignore"))

        data = []

        use_of_force_incidents = self.make_uof()

        for mutator in mutators:
            use_of_force_incidents = mutator.mutate(use_of_force_incidents)

        data.extend(use_of_force_incidents)

        for i in range(0, len(data), 100):
            chunk = data[i:i + 100]
            payload = {'data': chunk}

            url = baseurl + "data/UOF"

            print(payload)

            p = requests.post(url, auth=(comport_username,
                                         comport_password), json=payload)
            if p.status_code != 200:
                print("error: %s" % p.text.encode("utf-8", "ignore"))

        data = []

        ois_incidents = self.make_ois()

        for mutator in mutators:
            ois_incidents = mutator.mutate(ois_incidents)

        data.extend(ois_incidents)

        for i in range(0, len(data), 100):
            chunk = data[i:i + 100]
            payload = {'data': chunk}

            url = baseurl + "data/OIS"

            print(payload)

            p = requests.post(url, auth=(comport_username,
                                         comport_password), json=payload)
            if p.status_code != 200:
                print("error: %s" % p.text.encode("utf-8", "ignore"))

    def hash(self, text, key="bildad"):
        ''' Return an MD5 hash of the combined text and key.
        '''
        if not text or not key:
            return ""
        m = hashlib.md5()
        m.update((str(text) + key).encode('utf-8'))
        return m.hexdigest()

    def make_complaints(self, count=1000):
        complaints = []
        for x in range(0, count):
            assignment = self.generate_assignment()
            allegation = self.generate_allegation()
            complaints.append({
                "opaqueId": self.hash(random_string(10)),
                "serviceType": self.generate_service_type(),
                "source": self.generate_source(),
                "occuredDate": random_date(datetime(2014, 1, 1), datetime(2016, 1, 1)).strftime("%Y-%m-%d 0:0:00"),
                "division": assignment["division"],
                "precinct": assignment["precinct"],
                "shift": assignment["shift"],
                "beat": assignment["beat"],
                "allegationType": allegation["allegationType"],
                "allegation": allegation["allegation"],
                "disposition": self.generate_disposition(),
                "residentRace": self.generate_race(),
                "residentSex": self.generate_sex(),
                "residentAge": str(random.randint(15, 70)),
                "officerIdentifier": self.hash(random_string(10)),
                "officerRace": self.generate_race(),
                "officerSex": self.generate_sex(),
                "officerAge": str(random.randint(23, 50)),
                "officerYearsOfService": str(random.randint(0, 27))
            })

        return complaints

    def make_uof(self, count=1000):
        incidents = []
        for x in range(0, count):
            assignment = self.generate_assignment()
            incidents.append({
                "opaqueId": self.hash(random_string(10)),
                "occuredDate": random_date(datetime(2014, 1, 1), datetime(2016, 1, 1)).strftime("%Y-%m-%d 0:0:00"),
                "occuredTime": "",
                "division": assignment["division"],
                "precinct": assignment["precinct"],
                "shift": assignment["shift"],
                "beat": assignment["beat"],
                "disposition": self.generate_disposition(),
                "officerForceType": self.generate_officer_force_type(),
                "useOfForceReason": self.generate_use_of_force_reason(),
                "serviceType": self.generate_service_type(),
                "arrestMade": self.generate_bool(),
                "arrestCharges": self.generate_arrest_charges(),
                "residentWeaponUsed": "",
                "residentInjured": self.generate_bool(),
                "residentHospitalized": self.generate_bool(),
                "officerInjured": self.generate_bool(),
                "officerHospitalized": self.generate_bool(),
                "residentRace": self.generate_race(),
                "residentSex": self.generate_sex(),
                "residentAge": str(random.randint(15, 70)),
                "residentCondition": self.generate_condition(),
                "officerIdentifier": self.hash(random_string(10)),
                "officerRace": self.generate_race(),
                "officerSex": self.generate_sex(),
                "officerAge": str(random.randint(23, 50)),
                "officerYearsOfService": str(random.randint(0, 27)),
                "officerCondition": self.generate_condition()
            })

        return incidents

    def make_ois(self, count=1000):
        incidents = []
        for x in range(0, count):
            assignment = self.generate_assignment()
            incidents.append({
                "opaqueId": self.hash(random_string(10)),
                "serviceType": self.generate_service_type(),
                "occuredDate": random_date(datetime(2014, 1, 1), datetime(2016, 1, 1)).strftime("%Y-%m-%d 0:0:00"),
                "occuredTime": "",
                "division": assignment["division"],
                "precinct": assignment["precinct"],
                "shift": assignment["shift"],
                "beat": assignment["beat"],
                "disposition": self.generate_disposition(),
                "residentRace": self.generate_race(),
                "residentSex": self.generate_sex(),
                "residentAge": str(random.randint(15, 70)),
                "residentWeaponUsed": self.generate_ois_resident_force_type(),
                "residentCondition": self.generate_condition(),
                "officerIdentifier": self.hash(random_string(10)),
                "officerForceType": self.generate_ois_officer_force_type(),
                "officerRace": self.generate_race(),
                "officerSex": self.generate_sex(),
                "officerAge": str(random.randint(23, 50)),
                "officerYearsOfService": random.randint(0, 27),
                "officerCondition": self.generate_condition()
            })

        return incidents<commit_msg>improved fake data incidents are generated over the past 18 months officers can have more than one complaint output while loading fake data less overwhelming<commit_after>import requests
import hashlib
import datetime
from dateutil.relativedelta import relativedelta
from comport.department.models import Extractor
from comport.utils import random_string, random_date
import random


class JSONTestClient(object):

    def run(self, department, mutators=[]):
        baseurl = "http://localhost:5000/"
        password = "<PASSWORD>"
        extractor, envs = Extractor.from_department_and_password(
            department=department, password=password)
        comport_username = extractor.username
        comport_password = password
        end_date = datetime.datetime.now()
        start_date = end_date - relativedelta(months=18)

        data = []

        complaints = self.make_complaints(count=(900 + random.randint(0, 200)), start_date=start_date, end_date=end_date)

        for mutator in mutators:
            complaints = mutator.mutate(complaints)

        data.extend(complaints)

        print("* Adding {} fake complaints...".format(len(complaints)))

        for i in range(0, len(data), 100):
            chunk = data[i:i + 100]
            payload = {'data': chunk}

            url = baseurl + "data/complaints"

            print("- complaints {}-{}".format(i, i + len(chunk)))

            p = requests.post(url, auth=(comport_username,
                                         comport_password), json=payload)
            if p.status_code != 200:
                print("error: %s" % p.text.encode("utf-8", "ignore"))

        data = []

        use_of_force_incidents = self.make_uof(count=(900 + random.randint(0, 200)), start_date=start_date, end_date=end_date)

        for mutator in mutators:
            use_of_force_incidents = mutator.mutate(use_of_force_incidents)

        data.extend(use_of_force_incidents)

        print("* Adding {} fake uof...".format(len(use_of_force_incidents)))

        for i in range(0, len(data), 100):
            chunk = data[i:i + 100]
            payload = {'data': chunk}

            url = baseurl + "data/UOF"

            print("- uof {}-{}".format(i, i + len(chunk)))

            p = requests.post(url, auth=(comport_username,
                                         comport_password), json=payload)
            if p.status_code != 200:
                print("error: %s" % p.text.encode("utf-8", "ignore"))

        data = []

        ois_incidents = self.make_ois(count=(900 + random.randint(0, 200)), start_date=start_date, end_date=end_date)

        for mutator in mutators:
            ois_incidents = mutator.mutate(ois_incidents)

        data.extend(ois_incidents)

        print("* Adding {} fake ois...".format(len(ois_incidents)))

        for i in range(0, len(data), 100):
            chunk = data[i:i + 100]
            payload = {'data': chunk}

            url = baseurl + "data/OIS"

            print("- ois {}-{}".format(i, i + len(chunk)))

            p = requests.post(url, auth=(comport_username,
                                         comport_password), json=payload)
            if p.status_code != 200:
                print("error: %s" % p.text.encode("utf-8", "ignore"))

    def hash(self, text, key="bildad"):
        ''' Return an MD5 hash of the combined text and key.
        '''
        if not text or not key:
            return ""
        m = hashlib.md5()
        m.update((str(text) + key).encode('utf-8'))
        return m.hexdigest()

    def make_complaints(self, count=1000, start_date=datetime.datetime(2014, 1, 1), end_date=datetime.datetime(2016, 1, 1)):
        # make a smaller pool of officers so that it's possible to have more than one complaint per officer
        officers = []
        officer_count = round(count * .33)
        for x in range(0, count):
            officers.append({
                "officerIdentifier": self.hash(random_string(10)),
                "officerRace": self.generate_race(),
                "officerSex": self.generate_sex(),
                "officerAge": str(random.randint(23, 50)),
                "officerYearsOfService": str(random.randint(0, 27))
            })

        complaints = []
        for x in range(0, count):
            assignment = self.generate_assignment()
            allegation = self.generate_allegation()
            new_complaint = {
                "opaqueId": self.hash(random_string(10)),
                "serviceType": self.generate_service_type(),
                "source": self.generate_source(),
                "occuredDate": random_date(start_date, end_date).strftime("%Y-%m-%d 0:0:00"),
                "division": assignment["division"],
                "precinct": assignment["precinct"],
                "shift": assignment["shift"],
                "beat": assignment["beat"],
                "allegationType": allegation["allegationType"],
                "allegation": allegation["allegation"],
                "disposition": self.generate_disposition(),
                "residentRace": self.generate_race(),
                "residentSex": self.generate_sex(),
                "residentAge": str(random.randint(15, 70))
            }
            new_complaint.update(random.choice(officers))
            complaints.append(new_complaint)

        return complaints

    def make_uof(self, count=1000, start_date=datetime.datetime(2014, 1, 1), end_date=datetime.datetime(2016, 1, 1)):
        incidents = []
        for x in range(0, count):
            assignment = self.generate_assignment()
            incidents.append({
                "opaqueId": self.hash(random_string(10)),
                "occuredDate": random_date(start_date, end_date).strftime("%Y-%m-%d 0:0:00"),
                "occuredTime": "",
                "division": assignment["division"],
                "precinct": assignment["precinct"],
                "shift": assignment["shift"],
                "beat": assignment["beat"],
                "disposition": self.generate_disposition(),
                "officerForceType": self.generate_officer_force_type(),
                "useOfForceReason": self.generate_use_of_force_reason(),
                "serviceType": self.generate_service_type(),
                "arrestMade": self.generate_bool(),
                "arrestCharges": self.generate_arrest_charges(),
                "residentWeaponUsed": "",
                "residentInjured": self.generate_bool(),
                "residentHospitalized": self.generate_bool(),
                "officerInjured": self.generate_bool(),
                "officerHospitalized": self.generate_bool(),
                "residentRace": self.generate_race(),
                "residentSex": self.generate_sex(),
                "residentAge": str(random.randint(15, 70)),
                "residentCondition": self.generate_condition(),
                "officerIdentifier": self.hash(random_string(10)),
                "officerRace": self.generate_race(),
                "officerSex": self.generate_sex(),
                "officerAge": str(random.randint(23, 50)),
                "officerYearsOfService": str(random.randint(0, 27)),
                "officerCondition": self.generate_condition()
            })

        return incidents

    def make_ois(self, count=1000, start_date=datetime.datetime(2014, 1, 1), end_date=datetime.datetime(2016, 1, 1)):
        incidents = []
        for x in range(0, count):
            assignment = self.generate_assignment()
            incidents.append({
                "opaqueId": self.hash(random_string(10)),
                "serviceType": self.generate_service_type(),
                "occuredDate": random_date(start_date, end_date).strftime("%Y-%m-%d 0:0:00"),
                "occuredTime": "",
                "division": assignment["division"],
                "precinct": assignment["precinct"],
                "shift": assignment["shift"],
                "beat": assignment["beat"],
                "disposition": self.generate_disposition(),
                "residentRace": self.generate_race(),
                "residentSex": self.generate_sex(),
                "residentAge": str(random.randint(15, 70)),
                "residentWeaponUsed": self.generate_ois_resident_force_type(),
                "residentCondition": self.generate_condition(),
                "officerIdentifier": self.hash(random_string(10)),
                "officerForceType": self.generate_ois_officer_force_type(),
                "officerRace": self.generate_race(),
                "officerSex": self.generate_sex(),
                "officerAge": str(random.randint(23, 50)),
                "officerYearsOfService": random.randint(0, 27),
                "officerCondition": self.generate_condition()
            })

        return incidents
{'fraction_non_alphanumeric': 0.12688488414858404, 'fraction_numerical': 0.01986024273630011, 'mean_word_length': 2.042707944796718, 'pattern_counts': {'":': 136, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5192529', 'n_tokens_mistral': 4860, 'n_tokens_neox': 4308, 'n_words': 868}