<jupyter_start><jupyter_text>Example `credentials.env` file``` s3 credentialsS3_ENDPOINT=https://s3.us-east-1.amazonaws.comS3_BUCKET=ocp-odh-os-demo-s3S3_ACCESS_KEY=xxxS3_SECRET_KEY=xxx trino credentialsTRINO_USER=xxxTRINO_PASSWD=<PASSWORD>TRINO_HOST=trino-secure-odh-trino.apps.odh-cl1.apps.os-climate.orgTRINO_PORT=443```<jupyter_code>from dotenv import dotenv_values, load_dotenv
import os
import pathlib

dotenv_dir = os.environ.get('CREDENTIAL_DOTENV_DIR', os.environ.get('PWD', '/opt/app-root/src'))
dotenv_path = pathlib.Path(dotenv_dir) / 'credentials.env'
if os.path.exists(dotenv_path):
    load_dotenv(dotenv_path=dotenv_path,override=True)
import boto3
# Create an S3 client
s3 = boto3.client(
    service_name="s3",
    endpoint_url=os.environ['S3_ENDPOINT'],
    aws_access_key_id=os.environ['S3_ACCESS_KEY'],
    aws_secret_access_key=os.environ['S3_SECRET_KEY'],
)
import pandas as pd
obj = s3.get_object(
    Bucket=os.environ['S3_BUCKET'],
    Key='urgentem/UrgentemDataSampleEmissionsTargetsDec2020.csv')

# load the raw file from the bucket
dfEmissions = (pd.read_csv(obj['Body'])).convert_dtypes()

# convert columns to specific data types
dfEmissions = dfEmissions.convert_dtypes()

# map column names to a form that works for SQL
snakify_columns(dfEmissions, inplace=True)
#dfEmissions.head()
# a way to examine the structure of a pandas data frame
dfEmissions.info(verbose=True)
# parquet has multiple options for appending or updating data
# including adding new files, or appending, sharding directory trees, etc
dfEmissions.to_parquet('/tmp/emissions.parquet', index=False)
s3.upload_file(
    Bucket=os.environ['S3_BUCKET'],
    Key='urgentem/trino/itr_emissions_sample_test/emissions.parquet',
    Filename='/tmp/emissions.parquet'
)
import trino
conn = trino.dbapi.connect(
    auth=trino.auth.BasicAuthentication(os.environ['TRINO_USER'], os.environ['TRINO_PASSWD']),
    host=os.environ['TRINO_HOST'],
    port=int(os.environ['TRINO_PORT']),
    http_scheme='https',
    verify=True,
)
cur = conn.cursor()
# For this demonstration example, we just recreate table from scratch each time.
# in live data platform there will need to be policies and mechanisms for either
# appending new data, or overwriting old data, or saving off conditioned by a versioning number
# this is a data governance topic
cur.execute('drop table if exists hive.urgentem.itr_emissions_sample_test')
cur.fetchall()
# generate a sql schema that will correspond to the data types
# of columns in the pandas DF
# to-do: add some mechanisms for overriding types, either here
# or on the pandas data-frame itself before we write it out
schema = generate_table_schema_pairs(dfEmissions)

tabledef = """create table if not exists default.urgentem.itr_emissions_sample_test(
{schema}
) with (
    format = 'parquet',
    external_location = 's3a://ocp-odh-os-demo-s3/urgentem/trino/itr_emissions_sample_test/'
)""".format(schema=schema)
print(tabledef)

# tables created externally may not show up immediately in cloud-beaver
cur.execute(tabledef)
cur.fetchall()
# test creation of new table
#cur.execute('show tables in default.urgentem')
#cur.fetchall()
# test that we can get data
#cur.execute('select isin from default.urgentem.itr_emissions_sample_test')
#cur.fetchall()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10838631547078413, 'fraction_numerical': 0.009082652134423252, 'mean_word_length': 5.970464135021097, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '424768', 'license': 'FTL', 'n_tokens_mistral': 1144, 'n_tokens_neox': 1133, 'n_words': 264, 'path': 'notebooks/itr-ingest-raw-emissions.ipynb', 'repo_name': 'os-climate/data-platform-demo', 'size': '3296'}