<commit_before><dataConfig>
    <dataSource type="JdbcDataSource" driver="org.postgresql.Driver" url="jdbc:postgresql://localhost:5432/defaultdb" user="zimmerman" password="" />
    <document>
        <entity 
            name="activity_transaction" 
            query="
                SELECT
                    iati_activity.iati_identifier as iati_identifier,
                    iati_transaction.id AS id,
                    activity_id,
                    ref,
                    CASE WHEN iati_transaction.humanitarian=TRUE THEN '1' ELSE '0' END AS humanitarian,
                    CASE WHEN iati_activity.humanitarian=TRUE THEN '1' ELSE '0' END AS activity_humanitarian,
                    transaction_type_id,
                    TO_CHAR(transaction_date::timestamp, 'YYYY-MM-DD') AS transaction_date,
                    currency_id,
                    TO_CHAR(value_date::timestamp, 'YYYY-MM-DD') AS value_date,
                    value,
                    disbursement_channel_id,
                    tied_status_id,
                    flow_type_id
                FROM iati_transaction, iati_activity
                WHERE activity_id=iati_activity.id
                ORDER BY iati_transaction.id
            "
            deltaQuery="
                SELECT
                    iati_transaction.id AS id
                FROM iati_transaction, iati_activity
                WHERE activity_id = iati_activity.id
                    AND iati_activity.last_updated_model > '${dataimporter.last_index_time}'
                ORDER BY iati_transaction.id
            "
            deltaImportQuery="
                SELECT
                    iati_activity.iati_identifier as iati_identifier,
                    iati_transaction.id AS id,
                    activity_id,
                    ref,
                    CASE WHEN iati_transaction.humanitarian=TRUE THEN '1' ELSE '0' END AS humanitarian,
                    CASE WHEN iati_activity.humanitarian=TRUE THEN '1' ELSE '0' END AS activity_humanitarian,
                    transaction_type_id,
                    TO_CHAR(transaction_date::timestamp, 'YYYY-MM-DD') AS transaction_date,
                    currency_id,
                    TO_CHAR(value_date::timestamp, 'YYYY-MM-DD') AS value_date,
                    value,
                    disbursement_channel_id,
                    tied_status_id,
                    flow_type_id
                FROM iati_transaction, iati_activity
                WHERE activity_id=iati_activity.id
                    AND iati_transaction.id='${dataimporter.delta.id}'
            "
        >
            <field column="iati_identifier" name="iati_identifier"/>
            <field column="ref" name="transaction_ref"/>
            <field column="humanitarian" name="transaction_humanitarian"/>
            <field column="activity_humanitarian" name="humanitarian"/>
            <field column="transaction_type_id" name="transaction_type"/>
            <field column="transaction_date" name="transaction_date_iso_date"/>
            <field column="currency_id" name="transaction_value_currency"/>
            <field column="value_date" name="transaction_value_date"/>
            <field column="value" name="transaction_value"/>
            <field column="disbursement_channel_id" name="transaction_disburstment_channel_code"/>
            <field column="flow_type_id" name="transaction_flow_type_code"/>
            <field column="tied_status_id" name="transaction_tied_status_code"/>
            <entity name="activity_title"
                query="
                     SELECT
                        id
                    FROM iati_title
                    WHERE activity_id=${activity_transaction.activity_id}
                "
            >
                <entity name="activity_title_narrative"
                    query="
                        SELECT
                            language_id AS lang,
                            iati_narrative.content AS content
                        FROM iati_narrative, django_content_type
                        WHERE iati_narrative.related_object_id=${activity_title.id}
                            AND iati_narrative.activity_id=${activity_transaction.activity_id}
                            AND django_content_type.model='title'
                            AND django_content_type.id=iati_narrative.related_content_type_id
                    "
                >
                    <field column="lang" name="title_lang"/>
                    <field column="content" name="title_narrative"/>
                </entity>
            </entity>
            <entity name="activity_description"
                query="
                   SELECT
                        id,
                        type_id AS type
                    FROM iati_description
                    WHERE activity_id=${activity_transaction.activity_id}
                "
            >
                <field column="type" name="description_type"/>
                <entity name="activity_description_narrative"
                    query="
                        SELECT
                            language_id AS lang,
                            content AS narrative
                        FROM iati_narrative, django_content_type
                        WHERE iati_narrative.related_object_id=${activity_description.id}
                            AND iati_narrative.activity_id=${activity_transaction.activity_id}
                            AND django_content_type.model='description'
                            AND django_content_type.id=iati_narrative.related_content_type_id
                    "
                >
                    <field column="lang" name="description_lang"/>
                    <field column="narrative" name="description_narrative"/>
                </entity>
            </entity>
            <entity name="activity_activity_date"
                query="
                    SELECT
                        type_id,
                        TO_CHAR(iso_date::timestamp, 'YYYY-MM-DD') AS iso_date
                    FROM iati_activitydate
                    WHERE activity_id=${activity_transaction.activity_id}
                "
            >
                <field column="type_id" name="activity_date_type"/>
                <field column="iso_date" name="activity_date_iso_date"/>
            </entity>
            <entity name="activity_reporting_org" 
                query="
                    SELECT 
                        organisation_id,
                        ref 
                    FROM iati_activityreportingorganisation 
                    WHERE activity_id=${activity_transaction.activity_id} LIMIT 1
                "
            >
                <field column="ref" name="reporting_org_ref"/>
                <field column="type_id" name="reporting_org_type"/>
                <field column="secondary_reporter" name="reporting_org_secondary_reporter"/><commit_msg>Added reporting_org_secondary_reporter to the transaction indexing<commit_after><dataConfig>
    <dataSource type="JdbcDataSource" driver="org.postgresql.Driver" url="jdbc:postgresql://localhost:5432/defaultdb" user="zimmerman" password="" />
    <document>
        <entity 
            name="activity_transaction" 
            query="
                SELECT
                    iati_activity.iati_identifier as iati_identifier,
                    iati_transaction.id AS id,
                    activity_id,
                    ref,
                    CASE WHEN iati_transaction.humanitarian=TRUE THEN '1' ELSE '0' END AS humanitarian,
                    CASE WHEN iati_activity.humanitarian=TRUE THEN '1' ELSE '0' END AS activity_humanitarian,
                    transaction_type_id,
                    TO_CHAR(transaction_date::timestamp, 'YYYY-MM-DD') AS transaction_date,
                    currency_id,
                    TO_CHAR(value_date::timestamp, 'YYYY-MM-DD') AS value_date,
                    value,
                    disbursement_channel_id,
                    tied_status_id,
                    flow_type_id
                FROM iati_transaction, iati_activity
                WHERE activity_id=iati_activity.id
                ORDER BY iati_transaction.id
            "
            deltaQuery="
                SELECT
                    iati_transaction.id AS id
                FROM iati_transaction, iati_activity
                WHERE activity_id = iati_activity.id
                    AND iati_activity.last_updated_model > '${dataimporter.last_index_time}'
                ORDER BY iati_transaction.id
            "
            deltaImportQuery="
                SELECT
                    iati_activity.iati_identifier as iati_identifier,
                    iati_transaction.id AS id,
                    activity_id,
                    ref,
                    CASE WHEN iati_transaction.humanitarian=TRUE THEN '1' ELSE '0' END AS humanitarian,
                    CASE WHEN iati_activity.humanitarian=TRUE THEN '1' ELSE '0' END AS activity_humanitarian,
                    transaction_type_id,
                    TO_CHAR(transaction_date::timestamp, 'YYYY-MM-DD') AS transaction_date,
                    currency_id,
                    TO_CHAR(value_date::timestamp, 'YYYY-MM-DD') AS value_date,
                    value,
                    disbursement_channel_id,
                    tied_status_id,
                    flow_type_id
                FROM iati_transaction, iati_activity
                WHERE activity_id=iati_activity.id
                    AND iati_transaction.id='${dataimporter.delta.id}'
            "
        >
            <field column="iati_identifier" name="iati_identifier"/>
            <field column="ref" name="transaction_ref"/>
            <field column="humanitarian" name="transaction_humanitarian"/>
            <field column="activity_humanitarian" name="humanitarian"/>
            <field column="transaction_type_id" name="transaction_type"/>
            <field column="transaction_date" name="transaction_date_iso_date"/>
            <field column="currency_id" name="transaction_value_currency"/>
            <field column="value_date" name="transaction_value_date"/>
            <field column="value" name="transaction_value"/>
            <field column="disbursement_channel_id" name="transaction_disburstment_channel_code"/>
            <field column="flow_type_id" name="transaction_flow_type_code"/>
            <field column="tied_status_id" name="transaction_tied_status_code"/>
            <entity name="activity_title"
                query="
                     SELECT
                        id
                    FROM iati_title
                    WHERE activity_id=${activity_transaction.activity_id}
                "
            >
                <entity name="activity_title_narrative"
                    query="
                        SELECT
                            language_id AS lang,
                            iati_narrative.content AS content
                        FROM iati_narrative, django_content_type
                        WHERE iati_narrative.related_object_id=${activity_title.id}
                            AND iati_narrative.activity_id=${activity_transaction.activity_id}
                            AND django_content_type.model='title'
                            AND django_content_type.id=iati_narrative.related_content_type_id
                    "
                >
                    <field column="lang" name="title_lang"/>
                    <field column="content" name="title_narrative"/>
                </entity>
            </entity>
            <entity name="activity_description"
                query="
                   SELECT
                        id,
                        type_id AS type
                    FROM iati_description
                    WHERE activity_id=${activity_transaction.activity_id}
                "
            >
                <field column="type" name="description_type"/>
                <entity name="activity_description_narrative"
                    query="
                        SELECT
                            language_id AS lang,
                            content AS narrative
                        FROM iati_narrative, django_content_type
                        WHERE iati_narrative.related_object_id=${activity_description.id}
                            AND iati_narrative.activity_id=${activity_transaction.activity_id}
                            AND django_content_type.model='description'
                            AND django_content_type.id=iati_narrative.related_content_type_id
                    "
                >
                    <field column="lang" name="description_lang"/>
                    <field column="narrative" name="description_narrative"/>
                </entity>
            </entity>
            <entity name="activity_activity_date"
                query="
                    SELECT
                        type_id,
                        TO_CHAR(iso_date::timestamp, 'YYYY-MM-DD') AS iso_date
                    FROM iati_activitydate
                    WHERE activity_id=${activity_transaction.activity_id}
                "
            >
                <field column="type_id" name="activity_date_type"/>
                <field column="iso_date" name="activity_date_iso_date"/>
            </entity>
            <entity name="activity_reporting_org" 
                query="
                    SELECT
                        ref,
                        iati_activityreportingorganisation.type_id AS type_id,
                        CASE WHEN secondary_reporter=TRUE THEN '1' ELSE CASE WHEN secondary_reporter=FALSE THEN '0' END END AS secondary_reporter,
                        organisation_id
                    FROM iati_activityreportingorganisation 
                    WHERE activity_id=${activity_transaction.activity_id} LIMIT 1
                "
            >
                <field column="ref" name="reporting_org_ref"/>
                <field column="type_id" name="reporting_org_type"/>
                <field column="secondary_reporter" name="reporting_org_secondary_reporter"/>
{'fraction_non_alphanumeric': 0.07126275061554696, 'fraction_numerical': 0.001969750263805839, 'mean_word_length': 1.4329967482457642, 'pattern_counts': {'":': 0, '<': 77, '<?xml version=': 0, '>': 79, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2481856', 'n_tokens_mistral': 3525, 'n_tokens_neox': 3241, 'n_words': 600}