<commit_before>projectId=cdf-athena
dataset=test_automation
serviceAccountAutoDetect=auto-detect
csvFormat=csv

## GCS-PLUGIN-PROPERTIES-START
gcsMandatoryProperties=referenceName, path, format
gcsInvalidBucketName=invalid?BucketName
gcsCsvFile=testdata/GCS_CSV_TEST.csv
gcsCsvFileSchema=[{"key":"EmployeeDepartment","value":"string"},{"key":"Employeename","value":"string"},\
  {"key":"Salary","value":"int"},{"key":"wotkhours","value":"int"}]
gcsCSVFileOutputSchema={ "type": "record", "name": "text", "fields": [ \
  { "name": "EmployeeDepartment", "type": "string" }, { "name": "Employeename", "type": "string" }, \
  { "name": "Salary", "type": "int" }, { "name": "wotkhours", "type": "int" } ] }
gcsTsvFile=testdata/GCS_TSV_TEST.tsv
gcsTsvFileSchema=[{"key":"testscenarioid","value":"string"},{"key":"testdescription","value":"string"},\
  {"key":"testconditionid","value":"string"},{"key":"testtype","value":"string"}]
gcsBlobFile=testdata/GCS_BLOB_TEST.zip
gcsBlobFileSchema=[{"key":"body","value":"bytes"}]
gcsDelimitedFile=testdata/GCS_DELIMITED_TEST.txt
gcsDelimitedFileSchema=[{"key":"id","value":"int"},{"key":"name","value":"string"},\
  {"key":"yearofbirth","value":"int"},{"key":"datatype","value":"long"},{"key":"tablename","value":"string"}]
gcsTextFile=testdata/GCS_TEXT_TEST.txt
gcsTextFileSchema=[{"key":"offset","value":"long"},{"key":"body","value":"string"}]
gcsOutputFieldTestFile=testdata/GCS_OUTPUTFIELD_TEST.csv
gcsOutputFieldTestFileSchema=[{"key":"id","value":"int"},{"key":"firstname","value":"string"},\
  {"key":"lastname","value":"string"},{"key":"photo","value":"string"},{"key":"outputfield","value":"string"}]
gcsOverrideSchema=[{"key":"id","value":"float"},{"key":"firstname","value":"string"},\
  {"key":"lastname","value":"string"},{"key":"photo","value":"string"},{"key":"outputfield","value":"string"}]
gcsCsvRangeFile=testdata/GCS_CSV_RANGE_TEST.csv
gcsCsvRangeFileSchema=[{"key":"Emp_ID","value":"int"},{"key":"First_Name","value":"string"},\
  {"key":"Last_Name","value":"string"},{"key":"Gender","value":"string"},{"key":"Date_of_Birth","value":"string"},\
  {"key":"Time_of_Birth","value":"string"},{"key":"Age_in_Yrs_","value":"double"},\
  {"key":"Weight_in_Kgs_","value":"int"},{"key":"Date_of_Joining","value":"string"},\
  {"key":"Quarter_of_Joining","value":"string"},{"key":"Year_of_Joining","value":"int"},\
  {"key":"Month_of_Joining","value":"int"},{"key":"Month_Name_of_Joining","value":"string"},\
  {"key":"Age_in_Company__Years_","value":"double"},{"key":"Salary","value":"int"},\
  {"key":"Last___Hike","value":"double"}]
gcsOverrideField=id
gcsInvalidOverrideField=invalid
gcsOverrideDataType=float
gcsMaxSplitSize=120
gcsMinSplitSize=100
gcsRegexPathFilter=.+TEST.csv
gcsIncorrectRegexPath=/tZY
gcsPathField=outputfield
gcsInvalidPathField=output
gcsDelimiter=@
gcsIncorrectDelimiter=#
gcsSkipHeaderTrue=True
gcsPathDateSuffix=yyyy-MM-dd-HH-mm
gcsDataTypeTest1File=testdata/GCS_DATATYPE_TEST_1.csv
gcsDataTypeTest2File=testdata/GCS_DATATYPE_TEST_2.csv
gcsReadRecursivePath=testdata/GCS_RECURSIVE_TEST
## GCS-PLUGIN-PROPERTIES-END

## BIGQUERY-PLUGIN-PROPERTIES-START
bqSourceSchema=[{"key":"Id","value":"long"},{"key":"Value","value":"long"},{"key":"UID","value":"string"}]
bqPartitionSourceSchema=[{"key":"transaction_id","value":"long"},{"key":"transaction_uid","value":"string"},\
  {"key":"transaction_date","value":"date"}]
bqMandatoryProperties=referenceName, dataset, table
bqIncorrectProjectId=incorrectprojectid
bqIncorrectDatasetProjectId=incorrectdatasetprojectid
bqIncorrectFormatProjectId=INCORRECTFORMAT
bqIncorrectFormatDatasetProjectId=INCORRECTFORMAT
bqIncorrectDataset=incorrectDataset
bqFilter=Id=20
bqInvalidFilter=workforce=attrition_techno
bqIncorrectTableName=incorrectTable
bqFuturePartitionStartDate=2022-05-01
bqFuturePartitionEndDate=2022-10-02
bqTruncateTableTrue=True
bqUpdateTableSchemaTrue=True
bqCreateTableQueryFile=testdata/BigQuery/BigQueryCreateTableQuery.txt
bqInsertDataQueryFile=testdata/BigQuery/BigQueryInsertDataQuery.txt
bqCreateViewQueryFile=testdata/BigQuery/BigQueryCreateViewQuery.txt
bqSourceSchemaDatatype=[{"key":"transaction_info","value":"boolean"},{"key":"transaction_num","value":"bytes"},\
  {"key":"transaction_uid","value":"string"},{"key":"transaction_date","value":"date"},{"key":"transaction_dt",\
  "value":"datetime"},{"key":"transaction_time","value":"time"},{"key":"latitude","value":"double"},\
  {"key":"unique_key","value":"long"},{"key":"business_ratio","value":"decimal"},\
  {"key":"updated_on","value":"timestamp"},{"key":"parent","value":"array"},{"key":"inputs","value":"record"},\
  {"key":"business_bigratio","value":"decimal"},{"key":"committer","value":"record"},{"key":"trailer","value":"array"},\
  {"key":"difference","value":"array"},{"key":"Userdata","value":"record"}]
bqPartitionField=Month_of_Joining
bqPartitionFieldTime=transaction_date
bqRangeStart=1
bqRangeEnd=10
bqRangeInterval=2
bqChunkSize=262144
bqInvalidChunkSize=26214
bqInvalidSinkDataset=$#%$
bqInvalidSinkTable=(*^*&*
bqInvalidTemporaryBucket=$#%$
## BIGQUERY-PLUGIN-PROPERTIES-END<commit_msg>CDAP-19170 - e2e BQ test data fix<commit_after>projectId=cdf-athena
dataset=test_automation
serviceAccountAutoDetect=auto-detect
csvFormat=csv

## GCS-PLUGIN-PROPERTIES-START
gcsMandatoryProperties=referenceName, path, format
gcsInvalidBucketName=invalid?BucketName
gcsCsvFile=testdata/GCS_CSV_TEST.csv
gcsCsvFileSchema=[{"key":"EmployeeDepartment","value":"string"},{"key":"Employeename","value":"string"},\
  {"key":"Salary","value":"int"},{"key":"wotkhours","value":"int"}]
gcsCSVFileOutputSchema={ "type": "record", "name": "text", "fields": [ \
  { "name": "EmployeeDepartment", "type": "string" }, { "name": "Employeename", "type": "string" }, \
  { "name": "Salary", "type": "int" }, { "name": "wotkhours", "type": "int" } ] }
gcsTsvFile=testdata/GCS_TSV_TEST.tsv
gcsTsvFileSchema=[{"key":"testscenarioid","value":"string"},{"key":"testdescription","value":"string"},\
  {"key":"testconditionid","value":"string"},{"key":"testtype","value":"string"}]
gcsBlobFile=testdata/GCS_BLOB_TEST.zip
gcsBlobFileSchema=[{"key":"body","value":"bytes"}]
gcsDelimitedFile=testdata/GCS_DELIMITED_TEST.txt
gcsDelimitedFileSchema=[{"key":"id","value":"int"},{"key":"name","value":"string"},\
  {"key":"yearofbirth","value":"int"},{"key":"datatype","value":"long"},{"key":"tablename","value":"string"}]
gcsTextFile=testdata/GCS_TEXT_TEST.txt
gcsTextFileSchema=[{"key":"offset","value":"long"},{"key":"body","value":"string"}]
gcsOutputFieldTestFile=testdata/GCS_OUTPUTFIELD_TEST.csv
gcsOutputFieldTestFileSchema=[{"key":"id","value":"int"},{"key":"firstname","value":"string"},\
  {"key":"lastname","value":"string"},{"key":"photo","value":"string"},{"key":"outputfield","value":"string"}]
gcsOverrideSchema=[{"key":"id","value":"float"},{"key":"firstname","value":"string"},\
  {"key":"lastname","value":"string"},{"key":"photo","value":"string"},{"key":"outputfield","value":"string"}]
gcsCsvRangeFile=testdata/GCS_CSV_RANGE_TEST.csv
gcsCsvRangeFileSchema=[{"key":"Emp_ID","value":"int"},{"key":"First_Name","value":"string"},\
  {"key":"Last_Name","value":"string"},{"key":"Gender","value":"string"},{"key":"Date_of_Birth","value":"string"},\
  {"key":"Time_of_Birth","value":"string"},{"key":"Age_in_Yrs_","value":"double"},\
  {"key":"Weight_in_Kgs_","value":"int"},{"key":"Date_of_Joining","value":"string"},\
  {"key":"Quarter_of_Joining","value":"string"},{"key":"Year_of_Joining","value":"int"},\
  {"key":"Month_of_Joining","value":"int"},{"key":"Month_Name_of_Joining","value":"string"},\
  {"key":"Age_in_Company__Years_","value":"double"},{"key":"Salary","value":"int"},\
  {"key":"Last___Hike","value":"double"}]
gcsOverrideField=id
gcsInvalidOverrideField=invalid
gcsOverrideDataType=float
gcsMaxSplitSize=120
gcsMinSplitSize=100
gcsRegexPathFilter=.+TEST.csv
gcsIncorrectRegexPath=/tZY
gcsPathField=outputfield
gcsInvalidPathField=output
gcsDelimiter=@
gcsIncorrectDelimiter=#
gcsSkipHeaderTrue=True
gcsPathDateSuffix=yyyy-MM-dd-HH-mm
gcsDataTypeTest1File=testdata/GCS_DATATYPE_TEST_1.csv
gcsDataTypeTest2File=testdata/GCS_DATATYPE_TEST_2.csv
gcsReadRecursivePath=testdata/GCS_RECURSIVE_TEST
## GCS-PLUGIN-PROPERTIES-END

## BIGQUERY-PLUGIN-PROPERTIES-START
bqSourceSchema=[{"key":"Id","value":"long"},{"key":"Value","value":"long"},{"key":"UID","value":"string"}]
bqPartitionSourceSchema=[{"key":"transaction_id","value":"long"},{"key":"transaction_uid","value":"string"},\
  {"key":"transaction_date","value":"date"}]
bqMandatoryProperties=referenceName, dataset, table
bqIncorrectProjectId=incorrectprojectid
bqIncorrectDatasetProjectId=incorrectdatasetprojectid
bqIncorrectFormatProjectId=INCORRECTFORMAT
bqIncorrectFormatDatasetProjectId=INCORRECTFORMAT
bqIncorrectDataset=incorrectDataset
bqFilter=Id=20
bqInvalidFilter=workforce=attrition_techno
bqIncorrectTableName=incorrectTable
bqFuturePartitionStartDate=2099-05-01
bqFuturePartitionEndDate=2099-10-02
bqTruncateTableTrue=True
bqUpdateTableSchemaTrue=True
bqCreateTableQueryFile=testdata/BigQuery/BigQueryCreateTableQuery.txt
bqInsertDataQueryFile=testdata/BigQuery/BigQueryInsertDataQuery.txt
bqCreateViewQueryFile=testdata/BigQuery/BigQueryCreateViewQuery.txt
bqSourceSchemaDatatype=[{"key":"transaction_info","value":"boolean"},{"key":"transaction_num","value":"bytes"},\
  {"key":"transaction_uid","value":"string"},{"key":"transaction_date","value":"date"},{"key":"transaction_dt",\
  "value":"datetime"},{"key":"transaction_time","value":"time"},{"key":"latitude","value":"double"},\
  {"key":"unique_key","value":"long"},{"key":"business_ratio","value":"decimal"},\
  {"key":"updated_on","value":"timestamp"},{"key":"parent","value":"array"},{"key":"inputs","value":"record"},\
  {"key":"business_bigratio","value":"decimal"},{"key":"committer","value":"record"},{"key":"trailer","value":"array"},\
  {"key":"difference","value":"array"},{"key":"Userdata","value":"record"}]
bqPartitionField=Month_of_Joining
bqPartitionFieldTime=transaction_date
bqRangeStart=1
bqRangeEnd=10
bqRangeInterval=2
bqChunkSize=262144
bqInvalidChunkSize=26214
bqInvalidSinkDataset=$#%$
bqInvalidSinkTable=(*^*&*
bqInvalidTemporaryBucket=$#%$
## BIGQUERY-PLUGIN-PROPERTIES-END
{'fraction_non_alphanumeric': 0.2290852228303362, 'fraction_numerical': 0.008991399530883503, 'mean_word_length': 19.183431952662723, 'pattern_counts': {'":': 282, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7243506', 'n_tokens_mistral': 3541, 'n_tokens_neox': 3477, 'n_words': 234}