<commit_before>#!/bin/bash
# Run on -4
# Need to copy Mysql connector jar to /var/lib/sqoop.
See documentation http://www.cloudera.com/content/cloudera/en/documentation/core/v5-2-x/topics/cdh_ig_jdbc_driver_install.html

sudo -u hdfs hadoop fs -mkdir -p  /data/movielens
sudo -u hdfs hadoop fs -chown -R $USER: /data/movielens

sudo -u hdfs hadoop fs -mkdir /user/$USER
sudo -u hdfs hadoop fs -chown -R $USER: /user/$USER

# Need to have MySQL instance permissions set properly so all nodes of cluster (not just the node running sqoop command) can access the database

# Cleanup if necessary
sudo -u hdfs hadoop fs -rmr /data/movielens/movie

#Sqoop command is
sqoop import --connect jdbc:mysql://mgrover-haa-2.vpc.cloudera.com:3306/oltp --username root --query 'SELECT movie.*, group_concat(genre.name) FROM movie JOIN movie_genre ON (movie.id = movie_genre.movie_id)
JOIN genre ON (movie_genre.genre_id = genre.id) GROUP BY movie.id WHERE $CONDITIONS' \
--split-by movie.id --as-avrodatafile --target-dir /data/movielens/movie
<commit_msg>Adding exit on error functionality in the sqoop import script Fixing a bug related to a missing #<commit_after>#!/bin/bash -e
# Run on -4
# Need to copy Mysql connector jar to /var/lib/sqoop.
# See documentation http://www.cloudera.com/content/cloudera/en/documentation/core/v5-2-x/topics/cdh_ig_jdbc_driver_install.html

sudo -u hdfs hadoop fs -mkdir -p  /data/movielens
sudo -u hdfs hadoop fs -chown -R $USER: /data/movielens

sudo -u hdfs hadoop fs -mkdir /user/$USER
sudo -u hdfs hadoop fs -chown -R $USER: /user/$USER

# Need to have MySQL instance permissions set properly so all nodes of cluster (not just the node running sqoop command) can access the database

# Cleanup if necessary
sudo -u hdfs hadoop fs -rmr /data/movielens/movie

#Sqoop command is
sqoop import --connect jdbc:mysql://mgrover-haa-2.vpc.cloudera.com:3306/oltp --username root --query 'SELECT movie.*, group_concat(genre.name) FROM movie JOIN movie_genre ON (movie.id = movie_genre.movie_id)
JOIN genre ON (movie_genre.genre_id = genre.id) GROUP BY movie.id WHERE $CONDITIONS' \
--split-by movie.id --as-avrodatafile --target-dir /data/movielens/movie

{'fraction_non_alphanumeric': 0.10824504836480885, 'fraction_numerical': 0.007369875633348687, 'mean_word_length': 3.9029345372460496, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5002710', 'n_tokens_mistral': 818, 'n_tokens_neox': 766, 'n_words': 252}