<commit_before>import time, sys
from textwrap import dedent
import psycopg2

# TODO: set this up to use ssl certificates or an SSH tunnel, because
# otherwise postgres sends the password over the network as clear text.

# NOTE: instead of using the python csv writer, this directly writes tables to 
# file in the pyomo .tab format. This uses tabs between columns and the standard
# line break for the system it is run on. This does the following translations (only):
# - If a value contains double quotes, they get doubled.
# - If a value contains a single quote, tab or space character, the value gets enclosed in double quotes. 
#   (Note that pyomo doesn't allow quoting (and therefore spaces) in column headers.)
# - null values are converted to . (the pyomo/ampl standard for missing data)
# - any other values are simply passed to str().

# NOTE: this does not use the python csv writer because it doesn't support the quoting
# or null behaviors described above.

try:
    pghost='switch.eng.hawaii.edu'
    # note: the connection gets created when the module loads and never gets closed (until presumably python exits)
    con = psycopg2.connect(database='switch', host=pghost, user='switch_user')
    
except psycopg2.OperationalError:
    print dedent("""
        ############################################################################################
        Error while connecting to switch database on postgres server {server} as user 'switch_user'.
        Please ensure that there is a line like "*:*:*:switch_user:<password>" in 
        ~/.pgpass (which should be chmod 0600) or %APPDATA%\postgresql\pgpass.conf (Windows).    
        See http://www.postgresql.org/docs/9.1/static/libpq-pgpass.html for more details.
        ############################################################################################
        """.format(server=pghost))
    raise


# NOTE: ANSI SQL specifies single quotes for literal strings, and postgres conforms
# to this, so all the queries below should use single quotes around strings.

# NOTE: write_table() will automatically convert null values to '.', 
# so pyomo will recognize them as missing data

# NOTE: the code below could be made more generic, e.g., a list of
# table names and queries, which are then processed at the end.
# But that would be harder to debug, and wouldn't allow for ad hoc 
# calculations or writing .dat files (which are used for a few parameters)

def write_tables(**args):
    #########################
    # timescales
<commit_msg>using local postgres username for scenario_data.py<commit_after>import time, sys
from textwrap import dedent
import psycopg2

# TODO: set this up to use ssl certificates or an SSH tunnel, because
# otherwise postgres sends the password over the network as clear text.

# NOTE: instead of using the python csv writer, this directly writes tables to 
# file in the pyomo .tab format. This uses tabs between columns and the standard
# line break for the system it is run on. This does the following translations (only):
# - If a value contains double quotes, they get doubled.
# - If a value contains a single quote, tab or space character, the value gets enclosed in double quotes. 
#   (Note that pyomo doesn't allow quoting (and therefore spaces) in column headers.)
# - null values are converted to . (the pyomo/ampl standard for missing data)
# - any other values are simply passed to str().

# NOTE: this does not use the python csv writer because it doesn't support the quoting
# or null behaviors described above.

try:
    pghost='switch.eng.hawaii.edu'
    # note: the connection gets created when the module loads and never gets closed (until presumably python exits)
    con = psycopg2.connect(database='switch', host=pghost) #, user='switch_user')
    
except psycopg2.OperationalError:
    print dedent("""
        ############################################################################################
        Error while connecting to switch database on postgres server {server}.
        Please ensure that the PGUSER environment variable is set with your postgres username
        and there is a line like "*:*:*:<user>:<password>" in ~/.pgpass (which should be chmod 0600) 
        or in %APPDATA%\postgresql\pgpass.conf (Windows).    
        See http://www.postgresql.org/docs/9.1/static/libpq-pgpass.html for more details.
        ############################################################################################
        """.format(server=pghost))
    raise


# NOTE: ANSI SQL specifies single quotes for literal strings, and postgres conforms
# to this, so all the queries below should use single quotes around strings.

# NOTE: write_table() will automatically convert null values to '.', 
# so pyomo will recognize them as missing data

# NOTE: the code below could be made more generic, e.g., a list of
# table names and queries, which are then processed at the end.
# But that would be harder to debug, and wouldn't allow for ad hoc 
# calculations or writing .dat files (which are used for a few parameters)

def write_tables(**args):
    #########################
    # timescales

{'fraction_non_alphanumeric': 0.14740726354631967, 'fraction_numerical': 0.0034958244319285296, 'mean_word_length': 4.034213098729228, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7173315', 'n_tokens_mistral': 1331, 'n_tokens_neox': 1260, 'n_words': 638}