<commit_before>#!/usr/bin/env python

import os
import random
import re
import uuid

import psycopg2
from psycopg2.extensions import adapt
import requests
import lxml.etree

from iatidataquality import app
from iatidq import models
from test_mapping import test_to_kind
import db


class NoIATIActivityFound(Exception):
    pass


def all_tests():
    sampling_tests = []
    all_tests = {test.description: test for test in models.Test.all()}
    for k in test_to_kind.keys():
        sampling_tests.append(all_tests[k])
    return sorted(sampling_tests, key=lambda x: x.description)


def all_orgs():
    all_orgs = models.Organisation.all()
    sample_org_ids = [x[0] for x in db.all_sample_orgs()]
    sample_orgs = filter(lambda x: x.id in sample_org_ids, all_orgs)
    return sorted(sample_orgs, key=lambda x: x.organisation_name)


def save_url(url, filename):
    resp = requests.get(url)
    with open(filename, 'w') as f:
        f.write(resp.content)


def query(*args, **kwargs):
    db_config = app.config['DATABASE_INFO']
    db = psycopg2.connect(**db_config)
    c = db.cursor()
    c.execute(*args)
    if kwargs.get("write"):
        db.commit()
    else:
        return c.fetchall()


class WorkItems(object):
    def __init__(self, org_ids, test_ids, create):
        self.org_ids = org_ids
        self.test_ids = test_ids
        if create:
            self.cleanup()

            query('''CREATE TABLE current_data_result AS
                     SELECT * FROM result
                     WHERE test_id = 46
                     AND result_data > 0;''', write=True)

            query('''CREATE TABLE sampling_current_result_tmp AS
                       SELECT * FROM result
                       WHERE test_id = ANY(%s)
                       AND result_data > 0;''', (test_ids,), write=True)

            query('''CREATE TABLE sampling_current_result AS
                       SELECT sampling_current_result_tmp.* FROM sampling_current_result_tmp, current_data_result
                       WHERE sampling_current_result_tmp.result_identifier = current_data_result.result_identifier;''', write=True)

    def test_desc_of_test_id(self, test_id):
        results = query('''select description from test where id = %s;''', (test_id,));
        assert len(results) == 1
        return results[0][0]

    def kind_of_test(self, test_id):
        test_desc = self.test_desc_of_test_id(test_id)
        return test_to_kind[test_desc]

    def cleanup(self):
        query('''DROP TABLE IF EXISTS current_data_result;''', write=True)
        query('''DROP TABLE IF EXISTS sampling_current_result_tmp;''', write=True)
        query('''DROP TABLE IF EXISTS sampling_current_result;''', write=True)

    def __iter__(self):
        for org_id in self.org_ids:
            print("Org: {}".format(org_id))
            for test_id in self.test_ids:
                print("Test: {}".format(test_id))
                sot = SampleOrgTest(org_id, test_id)
                sample_ids = sot.sample_activity_ids(20)

                test_kind = self.kind_of_test(test_id)

                for act_id in sample_ids:
                    try:
                        act = sot.xml_of_activity(act_id)
                        parent_act = sot.xml_of_parent_activity(act_id)<commit_msg>Create tmp sampling tables regardless of `create`<commit_after>#!/usr/bin/env python

import os
import random
import re
import uuid

import psycopg2
from psycopg2.extensions import adapt
import requests
import lxml.etree

from iatidataquality import app
from iatidq import models
from test_mapping import test_to_kind
import db


class NoIATIActivityFound(Exception):
    pass


def all_tests():
    sampling_tests = []
    all_tests = {test.description: test for test in models.Test.all()}
    for k in test_to_kind.keys():
        sampling_tests.append(all_tests[k])
    return sorted(sampling_tests, key=lambda x: x.description)


def all_orgs():
    all_orgs = models.Organisation.all()
    sample_org_ids = [x[0] for x in db.all_sample_orgs()]
    sample_orgs = filter(lambda x: x.id in sample_org_ids, all_orgs)
    return sorted(sample_orgs, key=lambda x: x.organisation_name)


def save_url(url, filename):
    resp = requests.get(url)
    with open(filename, 'w') as f:
        f.write(resp.content)


def query(*args, **kwargs):
    db_config = app.config['DATABASE_INFO']
    db = psycopg2.connect(**db_config)
    c = db.cursor()
    c.execute(*args)
    if kwargs.get("write"):
        db.commit()
    else:
        return c.fetchall()


class WorkItems(object):
    def __init__(self, org_ids, test_ids, create):
        self.org_ids = org_ids
        self.test_ids = test_ids

        query('''CREATE TABLE current_data_result AS
                 SELECT * FROM result
                 WHERE test_id = 46
                 AND result_data > 0;''', write=True)

        query('''CREATE TABLE sampling_current_result_tmp AS
                   SELECT * FROM result
                   WHERE test_id = ANY(%s)
                   AND result_data > 0;''', (test_ids,), write=True)

        query('''CREATE TABLE sampling_current_result AS
                   SELECT sampling_current_result_tmp.* FROM sampling_current_result_tmp, current_data_result
                   WHERE sampling_current_result_tmp.result_identifier = current_data_result.result_identifier;''', write=True)

    def test_desc_of_test_id(self, test_id):
        results = query('''select description from test where id = %s;''', (test_id,));
        assert len(results) == 1
        return results[0][0]

    def kind_of_test(self, test_id):
        test_desc = self.test_desc_of_test_id(test_id)
        return test_to_kind[test_desc]

    def cleanup(self):
        query('''DROP TABLE IF EXISTS current_data_result;''', write=True)
        query('''DROP TABLE IF EXISTS sampling_current_result_tmp;''', write=True)
        query('''DROP TABLE IF EXISTS sampling_current_result;''', write=True)

    def __iter__(self):
        for org_id in self.org_ids:
            print("Org: {}".format(org_id))
            for test_id in self.test_ids:
                print("Test: {}".format(test_id))
                sot = SampleOrgTest(org_id, test_id)
                sample_ids = sot.sample_activity_ids(20)

                test_kind = self.kind_of_test(test_id)

                for act_id in sample_ids:
                    try:
                        act = sot.xml_of_activity(act_id)
                        parent_act = sot.xml_of_parent_activity(act_id)
{'fraction_non_alphanumeric': 0.09849647131021785, 'fraction_numerical': 0.003988953666768947, 'mean_word_length': 2.575973669775096, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2955705', 'n_tokens_mistral': 2065, 'n_tokens_neox': 1982, 'n_words': 474}