<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import logging
from datapackage import pull_datapackage
logger = logging.getLogger(__name__)


# Module API

def process(conf, conn):

    # View/table
    query = """
        create table export_openaire_jp as
        select t.id, t.primary_register, t.primary_id, t.secondary_ids,
        t.scientific_title, t.public_title, t.description, t.brief_summary,
        t.registration_date, t.target_sample_size,
        array_agg(distinct r.primary_id) filter(where r.primary_id is not null) as identifiers,
        array_to_json(array_agg(json_build_object('url', r.source_url, 'sourceID', s.id, 'sourceName', s.name))) as jsonProv
        from trials as t
        left outer join trialrecords as r on t.id = r.trial_id
        left outer join sources as s on r.source_id = s.id
        group by t.id, t.primary_register, t.primary_id, t.secondary_ids,
        t.scientific_title, t.public_title, t.description, t.brief_summary,
        t.registration_date, t.target_sample_size;
    """
    conn['database'].engine.execute(query)

    # Import
    pull_datapackage(
        descriptor='target/openaire/datapackage.json', name='export',
        backend='sql', engine=conn['database'].engine, prefix='export_')

    # Log
    logger.info('Exported datapackage for OpenAIRE.')<commit_msg>fixed openaire table name<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import logging
from datapackage import pull_datapackage
logger = logging.getLogger(__name__)


# Module API

def process(conf, conn):

    # View/table
    query = """
        create table export_openaire as
        select t.id, t.primary_register, t.primary_id, t.secondary_ids,
        t.scientific_title, t.public_title, t.description, t.brief_summary,
        t.registration_date, t.target_sample_size,
        array_agg(distinct r.primary_id) filter(where r.primary_id is not null) as identifiers,
        array_to_json(array_agg(json_build_object('url', r.source_url, 'sourceID', s.id, 'sourceName', s.name))) as jsonProv
        from trials as t
        left outer join trialrecords as r on t.id = r.trial_id
        left outer join sources as s on r.source_id = s.id
        group by t.id, t.primary_register, t.primary_id, t.secondary_ids,
        t.scientific_title, t.public_title, t.description, t.brief_summary,
        t.registration_date, t.target_sample_size;
    """
    conn['database'].engine.execute(query)

    # Import
    pull_datapackage(
        descriptor='target/openaire/datapackage.json', name='export',
        backend='sql', engine=conn['database'].engine, prefix='export_')

    # Log
    logger.info('Exported datapackage for OpenAIRE.')
{'fraction_non_alphanumeric': 0.09874873182279337, 'fraction_numerical': 0.0006763611768684477, 'mean_word_length': 3.607476635514019, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4989572', 'n_tokens_mistral': 936, 'n_tokens_neox': 893, 'n_words': 228}