<commit_before>*** Variables ***

${count} =   12       # use a multiple of 4
${database_url} =    
${field_mapping_method} =  
${time_to_pause_after_changing_mode} =  0

# tests won't work if there are records of these types in existence.
${core_objs_for_cleanup} =  DataImport__c,CustomObject3__c,CustomObject1__c
# you could also clean these up to have a cleaner test
${other_objs_for_cleanup} =   Opportunity,Account,Contact,CustomObject1__c,MaintenancePlan,General_Accounting_Unit__c,WorkOrder,npe01__OppPayment__c
${cleanup_first} =   core   # could also be "all" for maximum cleanliness or "none" for fresh scratch orgs
${data_generation_task} =     Set Variable If         "${field_mapping_method}"=="Help Text"  tasks.generate_bdi_data.generate_bdi_data         tasks.generate_bdi_CO_data.GenerateBDIData_CO

*** Settings ***

Resource  cumulusci/robotframework/CumulusCI.robot
Resource        robot/Cumulus/resources/NPSP.robot
Resource        robot/Cumulus/resources/Data.robot
Suite Setup      Run Keywords   Clear Generated Records
...                             AND  Setup BDI
...                             AND  Workaround Bug
...                             AND  Clear DataImport Records   
...                             AND  Generate Data  ${count}
Test Teardown      Run Keywords   Report BDI


*** Keywords ***
Clear DataImport Records
    ${all_objects} =   Catenate    ${core_objs_for_cleanup}  ,   ${other_objs_for_cleanup}
    # changed below to disable the "none" option temporarily.
    ${cleanup_objects} =   Set Variable If
    ...             "${cleanup_first}"=="core"  ${core_objs_for_cleanup}
    ...             "${cleanup_first}"=="all"  ${all_objects}
    ...             "${cleanup_first}"=="none"  ${core_objs_for_cleanup}     Error
    Should not be equal     ${cleanup_objects}      Error     
    ...                     Cleanup mode was not one of "core", "all" or "none"
    Run keyword if  "${cleanup_objects}"!="NONE"
    ...    Delete   ${cleanup_objects}

    Delete     Account     where=BillingCountry\='Tuvalu'    hardDelete=True
    Clear Generated Records

Clear Generated Records
    Python Display  Clearing
    Delete     Account_Soft_Credit__c     where=Account__r.BillingCountry\='Tuvalu'    hardDelete=True
    Delete     npe01__OppPayment__c     where=npe01__Opportunity__r.Account.BillingCountry\='Tuvalu'    hardDelete=True
    Delete     Opportunity     where=Account.BillingCountry\='Tuvalu'    hardDelete=True
    Delete     Account     where=BillingCountry\='Tuvalu'    hardDelete=True
    Delete     Contact     where=Title\='HRH'    hardDelete=True

Generate Data
    [Arguments]    ${count}
    ${count} =  Convert To Integer	${count}

    Run Task Class   tasks.generate_and_load_data.GenerateAndLoadData
    ...                 num_records=${count}
#    ...                 database_url=sqlite:////tmp/temp_db.db
    ...                 mapping=datasets/bdi_benchmark/mapping-CO.yml
    ...                 data_generation_task=${data_generation_task}

Setup BDI
    Configure BDI     ${field_mapping_method}

    Run Keyword If    ${time_to_pause_after_changing_mode}
    ...               Python Display    Pausing ${time_to_pause_after_changing_mode}    Seconds
    Run Keyword If    ${time_to_pause_after_changing_mode}
    ...               Sleep     ${time_to_pause_after_changing_mode}
    Run Keyword If    '${field_mapping_method}'=='Data Import Field Mapping'
    ...               Ensure Custom Metadata Was Deployed

Report BDI
    @{result} =   Salesforce Query  DataImport__c  
    ...           select=COUNT(Id)
    ...           Status__c=Imported

    ${imported_records} =   Set Variable    ${result}[0][expr0]

    Python Display  DataImport__c imported    ${result}[0][expr0]

    @{result} =   Salesforce Query  CustomObject3__c  
    ...           select=COUNT(Id)

    Python Display  CustomObject3__c imported    ${result}[0][expr0]

    # @{result} =   Salesforce Query  npe01__OppPayment__c  
    # ...           select=COUNT(Id)

    # Python Display  npe01__OppPayment__c imported    ${result}[0][expr0]<commit_msg>1.Support for persistent orgs 2. remove old stuff<commit_after>*** Variables ***

${count} =   12       # use a multiple of 4
${database_url} =    
${field_mapping_method} =  
${persistent_org} = 

# tests won't work if there are records of these types in existence.
${core_objs_for_cleanup} =  DataImport__c,CustomObject3__c,CustomObject1__c
# you could also clean these up to have a cleaner test
${other_objs_for_cleanup} =   Opportunity,Account,Contact,CustomObject1__c,MaintenancePlan,General_Accounting_Unit__c,WorkOrder,npe01__OppPayment__c
${cleanup_first} =   core   # could also be "all" for maximum cleanliness or "none" for fresh scratch orgs
${data_generation_task} =     Set Variable If         "${field_mapping_method}"=="Help Text"  tasks.generate_bdi_data.generate_bdi_data         tasks.generate_bdi_CO_data.GenerateBDIData_CO

*** Settings ***

Resource  cumulusci/robotframework/CumulusCI.robot
Resource        robot/Cumulus/resources/NPSP.robot
Resource        robot/Cumulus/resources/Data.robot
Suite Setup      Run Keywords   Clear Generated Records
...                             AND  Setup BDI
...                             AND  Workaround Bug
...                             AND  Clear DataImport Records   
...                             AND  Generate Data  ${count}
Test Teardown      Run Keywords   Report BDI


*** Keywords ***
Clear DataImport Records
    ${persistent_org} =     Convert To Boolean      ${persistent_org}
    Run keyword if  ${persistent_org}    
    ...    Python Display     Persistent org     ${persistent_org}
    ...    ELSE         Transient org      
    Run keyword if  ${persistent_org}     Clear Generated Records

Clear Generated Records
    Python Display  Clearing
    Delete     Account_Soft_Credit__c     where=Account__r.BillingCountry\='Tuvalu'    hardDelete=True
    Delete     npe01__OppPayment__c     where=npe01__Opportunity__r.Account.BillingCountry\='Tuvalu'    hardDelete=True
    Delete     Opportunity     where=Account.BillingCountry\='Tuvalu'    hardDelete=True
    Delete     Account     where=BillingCountry\='Tuvalu'    hardDelete=True
    Delete     Contact     where=Title\='HRH'    hardDelete=True

Generate Data
    [Arguments]    ${count}
    ${count} =  Convert To Integer	${count}

    Run Task Class   tasks.generate_and_load_data.GenerateAndLoadData
    ...                 num_records=${count}
#    ...                 database_url=sqlite:////tmp/temp_db.db
    ...                 mapping=datasets/bdi_benchmark/mapping-CO.yml
    ...                 data_generation_task=${data_generation_task}

Setup BDI
    Configure BDI     ${field_mapping_method}

    Run Keyword If    '${field_mapping_method}'=='Data Import Field Mapping'
    ...               Ensure Custom Metadata Was Deployed

Report BDI
    @{result} =   Salesforce Query  DataImport__c  
    ...           select=COUNT(Id)
    ...           Status__c=Imported

    ${imported_records} =   Set Variable    ${result}[0][expr0]

    Python Display  DataImport__c imported    ${result}[0][expr0]

    @{result} =   Salesforce Query  CustomObject3__c  
    ...           select=COUNT(Id)

    Python Display  CustomObject3__c imported    ${result}[0][expr0]

    # @{result} =   Salesforce Query  npe01__OppPayment__c  
    # ...           select=COUNT(Id)

    # Python Display  npe01__OppPayment__c imported    ${result}[0][expr0]
{'fraction_non_alphanumeric': 0.09253333333333333, 'fraction_numerical': 0.007333333333333333, 'mean_word_length': 2.539877300613497, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1069138', 'n_tokens_mistral': 2260, 'n_tokens_neox': 2181, 'n_words': 492}