<commit_before>#!/bin/bash

# Setup #

# sintr -m 4000
# module load gcc mariadb
# mysql_install_db --basedir=${BRAZOS_MARIADB_ROOT} --defaults-file=${TMPDIR}/my.cnf
# nohup mysqld_safe --defaults-file=${TMPDIR}/my.cnf --innodb_fast_shutdown=0 --innodb_change_buffering=none &
# mysql --defaults-file=${TMPDIR}/my.cnf -e 'CREATE DATABASE google_1grams DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci'
# mysql --defaults-file=${TMPDIR}/my.cnf google_1grams < /dh/data/shared/db-backups/google_1grams.sql

# mysql --defaults-file=${TMPDIR}/my.cnf google_1grams < /dh/data/shared/db-backups/google_1grams/ngram3_lower.sql
# 
# 

# for file in /dh/data/shared/db-backups/google_1grams/chunks/ngram3_lower_* ; do  echo $file ; mysql --defaults-file=${TMPDIR}/my.cnf google_1grams -e "LOAD DATA INFILE '$file' INTO TABLE ngram3_lower"; done

# mysqladmin --defaults-file=${TMPDIR}/my.cnf --protocol=tcp shutdown

#########

MY_CNF="${TMPDIR}/my.cnf"
ROOT_DIR="${TMPDIR}/mariadb"
TMP_DIR="${ROOT_DIR}/tmp"
#DATA_DIR="${ROOT_DIR}/data"
DATA_DIR="${ROOT_DIR}/data"
SRC="/fdata/idhmc/serial/google_1grams/data"
SOCKET="${ROOT_DIR}/mysql.sock"
PORT=60${SLURM_JOB_ID: -3} #TODO This is a huge assumption about value of SLURM_JOB_ID > 1000

cat << EOF > ${TMPDIR}/emop.properties
db_host: 127.0.0.1:${PORT}
db_name: emop
db_user: root
db_pass:
db_ngrams: google_1grams
EOF

cat << EOF  > ${MY_CNF}

[mysqld]
socket = $SOCKET
port = $PORT
basedir = $BRAZOS_MARIADB_ROOT
tmpdir = $TMP_DIR
datadir = $DATA_DIR
user = $USER
pid-file = ${TMPDIR}/mariadb-${SLURM_JOB_ID}.pid<commit_msg>Update bootstrap emop.properties to use latest PageCorrector<commit_after>#!/bin/bash

# Setup #

# sintr -m 4000
# module load gcc mariadb
# mysql_install_db --basedir=${BRAZOS_MARIADB_ROOT} --defaults-file=${TMPDIR}/my.cnf
# nohup mysqld_safe --defaults-file=${TMPDIR}/my.cnf --innodb_fast_shutdown=0 --innodb_change_buffering=none &
# mysql --defaults-file=${TMPDIR}/my.cnf -e 'CREATE DATABASE google_1grams DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci'
# mysql --defaults-file=${TMPDIR}/my.cnf google_1grams < /dh/data/shared/db-backups/google_1grams.sql

# mysql --defaults-file=${TMPDIR}/my.cnf google_1grams < /dh/data/shared/db-backups/google_1grams/ngram3_lower.sql
# 
# 

# for file in /dh/data/shared/db-backups/google_1grams/chunks/ngram3_lower_* ; do  echo $file ; mysql --defaults-file=${TMPDIR}/my.cnf google_1grams -e "LOAD DATA INFILE '$file' INTO TABLE ngram3_lower"; done

# mysqladmin --defaults-file=${TMPDIR}/my.cnf --protocol=tcp shutdown

#########

MY_CNF="${TMPDIR}/my.cnf"
ROOT_DIR="${TMPDIR}/mariadb"
TMP_DIR="${ROOT_DIR}/tmp"
#DATA_DIR="${ROOT_DIR}/data"
DATA_DIR="${ROOT_DIR}/data"
SRC="/fdata/idhmc/serial/google_1grams/data"
SOCKET="${ROOT_DIR}/mysql.sock"
PORT=60${SLURM_JOB_ID: -3} #TODO This is a huge assumption about value of SLURM_JOB_ID > 1000

cat << EOF > ${TMPDIR}/emop.properties
ctx_db_driver: com.mysql.jdbc.Driver
ctx_db_url: jdbc:mysql://127.0.0.1:${PORT}/google_1grams
ctx_db_user: root
ctx_db_pass:
EOF

cat << EOF  > ${MY_CNF}

[mysqld]
socket = $SOCKET
port = $PORT
basedir = $BRAZOS_MARIADB_ROOT
tmpdir = $TMP_DIR
datadir = $DATA_DIR
user = $USER
pid-file = ${TMPDIR}/mariadb-${SLURM_JOB_ID}.pid
{'fraction_non_alphanumeric': 0.16301327570237728, 'fraction_numerical': 0.019759184933621488, 'mean_word_length': 5.328125, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2929957', 'n_tokens_mistral': 1357, 'n_tokens_neox': 1314, 'n_words': 227}