<commit_before># Common Properties.
## GCP project id.
project.id=<project-id>
## GCS staging bucket
gcs.staging.bucket.path=<gcs-staging-path>

# HiveToGCS Template properties.
## Source Hive warehouse dir.
spark.sql.warehouse.dir=<warehouse-path>
## GCS output path.
hive.gcs.output.path=<gcs-output-path>
## Name of hive input table.
hive.input.table=<hive-input-table>
## Hive input db name.
hive.input.db=<hive-output-db>
## GCS output format. Optional, defaults to avro.
hive.gcs.output.format=<gcs-output-format>
## Optional, column to partition hive data.
hive.partition.col=<hive-partition-col>



# HiveToBQ Template properties.
# Required -Bigquery table name
hivetobq.bigquery.location=
#Required property
hivetobq.sql=
#Optional - Write mode to use while writing output to BQ. Supported values are - Append/Overwrite/ErrorIfExists/Ignore
hivetobq.write.mode=append
#Required Temporary directory to export bigquery data
hivetobq.temp.gcs.bucket=


# JDBCToBQ Template properties.
# Required -Bigquery table name
jdbctobq.bigquery.location=
# Required - JDBC URL and properties like username,password etc.,
jdbctobq.jdbc.url=
#Required property
jdbctobq.jdbc.driver.class.name=
#Required property
jdbctobq.sql=
#optional properties to partition sql, if one is specified then all needs to be specified
jdbctobq.sql.partitionColumn=
jdbctobq.sql.lowerBound=
jdbctobq.sql.upperBound=
jdbctobq.sql.numPartitions=
#Optional - Write mode to use while writing output to BQ. Supported values are - Append/Overwrite/ErrorIfExists/Ignore
jdbctobq.write.mode=
#Required Temporary directory to export bigquery data
jdbctobq.temp.gcs.bucket=


#JDBCTOGCS Template properties
# Required - GCS output location
jdbctogcs.output.location=
# Optional - Default output format is csv. Example values avro,csv,parquet
jdbctogcs.output.format=
# Optional - Default is overwrite. Supported values are - Append/Overwrite/ErrorIfExists/Ignore
jdbctogcs.write.mode=
# Required - JDBC URL and properties like username,password etc.,
jdbctogcs.jdbc.url=
#Required property
jdbctogcs.jdbc.driver.class.name=
#Required property
jdbctogcs.sql=
#optional properties to partition sql, if one is specified then all needs to be specified
jdbctogcs.sql.partitionColumn=
jdbctogcs.sql.lowerBound=
jdbctogcs.sql.upperBound=
jdbctogcs.sql.numPartitions=
# Optional property. column on which the output to be partitioned
jdbctogcs.output.partition.col=


# WordCount properties.
## Format of input file to be processed.
word.count.input.format=
## input path for word count.
word.count.input.path=
## input path for word count.
word.count.output.path=


# GCSToBigTable properties.
## Bigtable instance ud.
bigtable.instance.id=<bigtable instance id>

## BigTable output table name.
bigtable.output.table.name=<bigtable output table>
## Column from input source to be used as key column in Bigtable.
bigtable.key.col=<input col to be used as row key for BT>
## BigTable column family name.
bigtable.col.family.name=<bigtable col family>

# PubSubToBQ properties
## Project that contains the input Pub/Sub subscription to be read
pubsub.input.project.id=<pubsub project id>
## PubSub subscription name
pubsub.input.subscription=<pubsub subscription>
## Stream timeout, for how long the subscription will be read
pubsub.timeout.ms=60000
## Streaming duration, how often wil writes to BQ be triggered
pubsub.streaming.duration.seconds=15
## Number of streams that will read from Pub/Sub subscription in parallel
pubsub.total.receivers=5
## Project that contains the output table
pubsub.bq.output.project.id=<pubsub to bq output project id>
## BigQuery output dataset
pubsub.bq.output.dataset=<bq output dataset>
## BigQuery output table
pubsub.bq.output.table=<bq output table>
## Number of records to be written per message to BigQuery
pubsub.bq.batch.size=1000


#GCStoBigQuery properties
gcs.bigquery.input.format=<avro,parquet,csv>
gcs.bigquery.input.location=<input-file-location>
gcs.bigquery.output.dataset =<bq output dataset>
gcs.bigquery.output.table =<bq output table>
gcs.bigquery.temp.bucket.name =<temp bucket>

# Spanner To GCS properties
spanner.gcs.input.spanner.id=<spanner-id>
spanner.gcs.input.database.id=<database-id>
spanner.gcs.input.table.id=<table-id>
spanner.gcs.output.gcs.path=<gcs-path>
spanner.gcs.output.gcs.saveMode=<Append|Overwrite|ErrorIfExists|Ignore>
spanner.gcs.output.gcs.format=<avro|csv|parquet|json|orc>

# S3 to BQ
s3.bq.access.key=<s3-accesss-key>
s3.bq.secret.key=<s3-secret-key>
s3.bq.input.format=<avro,parquet,csv,json>
s3.bq.input.location=<s3-input-location>
s3.bq.output.dataset.name=<bq-dataset-name>
s3.bq.output.table.name=<bq-output-table>
s3.bq.ld.temp.bucket.name=<temp-bucket>

# Bigquery to GCS
bigquery.gcs.input.table=<project:dataset.table, eg: bigquery-public-data:samples.shakespeare>
bigquery.gcs.output.format=<avro,parquet,csv,json>
bigquery.gcs.output.location=<gcs-path>

# GCS to Spanner
gcs.spanner.input.format=<avro or parquet>
gcs.spanner.input.location=<gcs path>
gcs.spanner.output.instance=<spanner instance id>
gcs.spanner.output.database=<spanner database id>
gcs.spanner.output.table=<spanner table id>
gcs.spanner.output.saveMode=<Append|Overwrite|ErrorIfExists|Ignore>
gcs.spanner.output.primaryKey=<column[(,column)*] - primary key columns needed when creating the table>

# GCS to JDBC
gcs.jdbc.input.format=<avro or parquet>
gcs.jdbc.input.location=<gcs path>
gcs.jdbc.output.driver=<output jdbc driver><commit_msg>added orc section in template properties file<commit_after># Common Properties.
## GCP project id.
project.id=<project-id>
## GCS staging bucket
gcs.staging.bucket.path=<gcs-staging-path>

# HiveToGCS Template properties.
## Source Hive warehouse dir.
spark.sql.warehouse.dir=<warehouse-path>
## GCS output path.
hive.gcs.output.path=<gcs-output-path>
## Name of hive input table.
hive.input.table=<hive-input-table>
## Hive input db name.
hive.input.db=<hive-output-db>
## GCS output format. Optional, defaults to avro.
hive.gcs.output.format=<gcs-output-format>
## Optional, column to partition hive data.
hive.partition.col=<hive-partition-col>



# HiveToBQ Template properties.
# Required -Bigquery table name
hivetobq.bigquery.location=
#Required property
hivetobq.sql=
#Optional - Write mode to use while writing output to BQ. Supported values are - Append/Overwrite/ErrorIfExists/Ignore
hivetobq.write.mode=append
#Required Temporary directory to export bigquery data
hivetobq.temp.gcs.bucket=


# JDBCToBQ Template properties.
# Required -Bigquery table name
jdbctobq.bigquery.location=
# Required - JDBC URL and properties like username,password etc.,
jdbctobq.jdbc.url=
#Required property
jdbctobq.jdbc.driver.class.name=
#Required property
jdbctobq.sql=
#optional properties to partition sql, if one is specified then all needs to be specified
jdbctobq.sql.partitionColumn=
jdbctobq.sql.lowerBound=
jdbctobq.sql.upperBound=
jdbctobq.sql.numPartitions=
#Optional - Write mode to use while writing output to BQ. Supported values are - Append/Overwrite/ErrorIfExists/Ignore
jdbctobq.write.mode=
#Required Temporary directory to export bigquery data
jdbctobq.temp.gcs.bucket=


#JDBCTOGCS Template properties
# Required - GCS output location
jdbctogcs.output.location=
# Optional - Default output format is csv. Example values avro,csv,parquet
jdbctogcs.output.format=
# Optional - Default is overwrite. Supported values are - Append/Overwrite/ErrorIfExists/Ignore
jdbctogcs.write.mode=
# Required - JDBC URL and properties like username,password etc.,
jdbctogcs.jdbc.url=
#Required property
jdbctogcs.jdbc.driver.class.name=
#Required property
jdbctogcs.sql=
#optional properties to partition sql, if one is specified then all needs to be specified
jdbctogcs.sql.partitionColumn=
jdbctogcs.sql.lowerBound=
jdbctogcs.sql.upperBound=
jdbctogcs.sql.numPartitions=
# Optional property. column on which the output to be partitioned
jdbctogcs.output.partition.col=


# WordCount properties.
## Format of input file to be processed.
word.count.input.format=
## input path for word count.
word.count.input.path=
## input path for word count.
word.count.output.path=


# GCSToBigTable properties.
## Bigtable instance ud.
bigtable.instance.id=<bigtable instance id>

## BigTable output table name.
bigtable.output.table.name=<bigtable output table>
## Column from input source to be used as key column in Bigtable.
bigtable.key.col=<input col to be used as row key for BT>
## BigTable column family name.
bigtable.col.family.name=<bigtable col family>

# PubSubToBQ properties
## Project that contains the input Pub/Sub subscription to be read
pubsub.input.project.id=<pubsub project id>
## PubSub subscription name
pubsub.input.subscription=<pubsub subscription>
## Stream timeout, for how long the subscription will be read
pubsub.timeout.ms=60000
## Streaming duration, how often wil writes to BQ be triggered
pubsub.streaming.duration.seconds=15
## Number of streams that will read from Pub/Sub subscription in parallel
pubsub.total.receivers=5
## Project that contains the output table
pubsub.bq.output.project.id=<pubsub to bq output project id>
## BigQuery output dataset
pubsub.bq.output.dataset=<bq output dataset>
## BigQuery output table
pubsub.bq.output.table=<bq output table>
## Number of records to be written per message to BigQuery
pubsub.bq.batch.size=1000


#GCStoBigQuery properties
gcs.bigquery.input.format=<avro,parquet,csv>
gcs.bigquery.input.location=<input-file-location>
gcs.bigquery.output.dataset =<bq output dataset>
gcs.bigquery.output.table =<bq output table>
gcs.bigquery.temp.bucket.name =<temp bucket>

# Spanner To GCS properties
spanner.gcs.input.spanner.id=<spanner-id>
spanner.gcs.input.database.id=<database-id>
spanner.gcs.input.table.id=<table-id>
spanner.gcs.output.gcs.path=<gcs-path>
spanner.gcs.output.gcs.saveMode=<Append|Overwrite|ErrorIfExists|Ignore>
spanner.gcs.output.gcs.format=<avro|csv|parquet|json|orc>

# S3 to BQ
s3.bq.access.key=<s3-accesss-key>
s3.bq.secret.key=<s3-secret-key>
s3.bq.input.format=<avro,parquet,csv,json>
s3.bq.input.location=<s3-input-location>
s3.bq.output.dataset.name=<bq-dataset-name>
s3.bq.output.table.name=<bq-output-table>
s3.bq.ld.temp.bucket.name=<temp-bucket>

# Bigquery to GCS
bigquery.gcs.input.table=<project:dataset.table, eg: bigquery-public-data:samples.shakespeare>
bigquery.gcs.output.format=<avro,parquet,csv,json>
bigquery.gcs.output.location=<gcs-path>

# GCS to Spanner
gcs.spanner.input.format=<avro or parquet>
gcs.spanner.input.location=<gcs path>
gcs.spanner.output.instance=<spanner instance id>
gcs.spanner.output.database=<spanner database id>
gcs.spanner.output.table=<spanner table id>
gcs.spanner.output.saveMode=<Append|Overwrite|ErrorIfExists|Ignore>
gcs.spanner.output.primaryKey=<column[(,column)*] - primary key columns needed when creating the table>

# GCS to JDBC
gcs.jdbc.input.format=<avro or parquet or orc>
gcs.jdbc.input.location=<gcs path>
gcs.jdbc.output.driver=<output jdbc driver>
{'fraction_non_alphanumeric': 0.11095927849139109, 'fraction_numerical': 0.00419058030427257, 'mean_word_length': 5.5074096028452875, 'pattern_counts': {'":': 0, '<': 99, '<?xml version=': 0, '>': 99, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2262634', 'n_tokens_mistral': 3838, 'n_tokens_neox': 3785, 'n_words': 919}