<commit_before>#!/bin/bash
SQOOP_METASTORE_HOST=localhost

sudo -u hdfs hadoop fs -mkdir -p  /etl/movielens/user_rating_fact
sudo -u hdfs hadoop fs -chown -R $USER: /etl/movielens/user_rating_fact

sqoop job --delete user_rating_import --meta-connect jdbc:hsqldb:hsql://${SQOOP_METASTORE_HOST}:16000/sqoop || :

sqoop job --create user_rating_import --meta-connect jdbc:hsqldb:hsql://${SQOOP_METASTORE_HOST}:16000/sqoop \
-- import --connect jdbc:mysql://mgrover-haa-2.vpc.cloudera.com:3306/oltp --username root \
--table user_rating -m 8 --incremental append --check-column timestamp \
--as-parquetfile --hive-import --warehouse-dir /etl/movielens/user_rating_fact --hive-table user_rating_fact

# The above command fails. We can either get rid of all hive things from the term or have to explicitly define
# HIVE_HOME and export it before we run this. Otherwise it fails with a misleading error message org.kitesdk.data.DatasetNotFoundException: Unknown dataset URI: hive?dataset=user_rating_fact

#Create user_rating_fact table in hive
hive -e "CREATE EXTERNAL TABLE IF NOT EXISTS user_rating_fact(id INT, timestamp TIMESTAMP,
 user_id INT, rating INT, movie_id INT)
ROW FORMAT SERDE 'parquet.hive.serde.ParquetHiveSerDe'
STORED AS
INPUTFORMAT 'parquet.hive.DeprecatedParquetInputFormat'
OUTPUTFORMAT 'parquet.hive.DeprecatedParquetOutputFormat'
LOCATION '/etl/movielens/user_rating_fact'"

sqoop job -exec user_rating_import --meta-connect jdbc:hsqldb:hsql://${SQOOP_METASTORE_HOST}:16000/sqoop

<commit_msg>Correct a bug in user_rating import:<commit_after>#!/bin/bash
SQOOP_METASTORE_HOST=localhost

sudo -u hdfs hadoop fs -mkdir -p  /etl/movielens/user_rating_fact
sudo -u hdfs hadoop fs -chown -R $USER: /etl/movielens/user_rating_fact

sqoop job --delete user_rating_import --meta-connect jdbc:hsqldb:hsql://${SQOOP_METASTORE_HOST}:16000/sqoop || :

sqoop job --create user_rating_import --meta-connect jdbc:hsqldb:hsql://${SQOOP_METASTORE_HOST}:16000/sqoop \
-- import --connect jdbc:mysql://mgrover-haa-2.vpc.cloudera.com:3306/oltp --username root \
--table user_rating -m 8 --incremental append --check-column timestamp \
--as-parquetfile --hive-import --warehouse-dir /etl/movielens/user_rating_fact --hive-table user_rating_fact

# The above command fails. We can either get rid of all hive things from the term or have to explicitly define
# HIVE_HOME and export it before we run this. Otherwise it fails with a misleading error message org.kitesdk.data.DatasetNotFoundException: Unknown dataset URI: hive?dataset=user_rating_fact

#Create user_rating_fact table in hive
hive -e "CREATE EXTERNAL TABLE IF NOT EXISTS user_rating_fact(id INT, timestamp TIMESTAMP,
 user_id INT, movie_id INT, rating INT)
ROW FORMAT SERDE 'parquet.hive.serde.ParquetHiveSerDe'
STORED AS
INPUTFORMAT 'parquet.hive.DeprecatedParquetInputFormat'
OUTPUTFORMAT 'parquet.hive.DeprecatedParquetOutputFormat'
LOCATION '/etl/movielens/user_rating_fact'"

sqoop job -exec user_rating_import --meta-connect jdbc:hsqldb:hsql://${SQOOP_METASTORE_HOST}:16000/sqoop


{'fraction_non_alphanumeric': 0.09944207417131605, 'fraction_numerical': 0.013784049885132918, 'mean_word_length': 5.195121951219512, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3109051', 'n_tokens_mistral': 1114, 'n_tokens_neox': 1055, 'n_words': 271}