<commit_before>#!/bin/bash

help() {
  echo "usage: $0 localWorkdir srcFileInHdfs HDFS_USER"
  exit 1
}

if [ $# -ne 3 ] ; then
  help
fi

workdir=$1
hdfsFile="$2"
HDFS_USER=$3

#FSM states: STAGING, UNZIPPING, UPLOADING, CHOWNING, SUCCESS, FAILED
fsmFile="$(dirname $1)/fsm-$(basename $1).txt"

function finish {
    rm -rf $workdir
}
trap finish EXIT

function check_error {
    if [ $? -ne 0 ] ; then
      echo "FAILED" > $fsmFile
      exit 1	
    fi
}


prefix='hdfs://'
normalizedPath=${hdfsFile#${prefix}}
check_error

file=$(basename "$normalizedPath")
check_error
dir=$(dirname "$normalizedPath")
check_error

cd $workdir
check_error

filename=$(basename "$hdfsFile")

echo "STAGING" > $fsmFile
<%= node['hops']['base_dir'] %>/bin/hdfs dfs -copyToLocal "$hdfsFile" $workdir
check_error

echo "UNZIPPING" > $fsmFile

# State Machine State Change. File now staged to local FS.
# Java program checks if this file is written to see if this file exists:  $workdir/$(basename $hdfsfile)


# http://manpages.ubuntu.com/manpages/trusty/en/man1/dtrx.1.html
# , everything will be written to  a  dedicated  directory  that's named  after  the  archive.
# dtrx will also change the permissions to  ensure that the owner can read and write all those files.

cd $workdir
dtrx -o -n "$file"
check_error

# State Machine State Change. File now extracted to local FS.
# Java program checks if this file is written to see if this directory exists:  $workdir/$(basename $hdfsfile)


filedir="${file%%.*}"

case "$file" in 
  *.tar.gz)
     filedir=$(basename "$hdfsFile" .tar.gz)
    ;;
  *.bz2)
     filedir=$(basename "$hdfsFile" .bz2)
    ;;
  *.bzip2)
     filedir=$(basename "$hdfsFile" .bz2)
    ;;
  *.tgz)
     filedir=$(basename "$hdfsFile" .tgz)
    ;;
  *.gz)
     filedir=$(basename "$hdfsFile" .gz)
    ;;
  *.7z)
     filedir=$(basename "$hdfsFile" .7z)
    ;;
  *.rar)
     filedir=$(basename "$hdfsFile" .rar)
    ;;
  *.zip)
     filedir=$(basename "$hdfsFile" .zip)
    ;;
  *)
    help
    ;;
esac


#replace any spaces in the directory name with '_'
#Hadoop doesnt like spaces in paths.
escapedFiledir=${filedir// /_}
mv "$filedir" $escapedFiledir

echo "UPLOADING" > $fsmFile
<%= node['hops']['base_dir'] %>/bin/hdfs dfs -copyFromLocal $escapedFiledir "$dir"
check_error

echo "CHOWNING" > $fsmFile
<%= node['hops']['base_dir'] %>/bin/hdfs dfs -chown -R ${HDFS_USER} "$dir"
check_error

echo "SUCCESS" > $fsmFile

<commit_msg>change from HDFS_USER to new variable name<commit_after>#!/bin/bash

help() {
  echo "usage: $0 localWorkdir srcFileInHdfs IMPERSONATED_USERNAME"
  exit 1
}

if [ $# -ne 3 ] ; then
  help
fi

workdir=$1
hdfsFile="$2"
IMPERSONATED_USERNAME=$3

#FSM states: STAGING, UNZIPPING, UPLOADING, CHOWNING, SUCCESS, FAILED
fsmFile="$(dirname $1)/fsm-$(basename $1).txt"

function finish {
    rm -rf $workdir
}
trap finish EXIT

function check_error {
    if [ $? -ne 0 ] ; then
      echo "FAILED" > $fsmFile
      exit 1	
    fi
}


prefix='hdfs://'
normalizedPath=${hdfsFile#${prefix}}
check_error

file=$(basename "$normalizedPath")
check_error
dir=$(dirname "$normalizedPath")
check_error

cd $workdir
check_error

filename=$(basename "$hdfsFile")

echo "STAGING" > $fsmFile
<%= node['hops']['base_dir'] %>/bin/hdfs dfs -copyToLocal "$hdfsFile" $workdir
check_error

echo "UNZIPPING" > $fsmFile

# State Machine State Change. File now staged to local FS.
# Java program checks if this file is written to see if this file exists:  $workdir/$(basename $hdfsfile)


# http://manpages.ubuntu.com/manpages/trusty/en/man1/dtrx.1.html
# , everything will be written to  a  dedicated  directory  that's named  after  the  archive.
# dtrx will also change the permissions to  ensure that the owner can read and write all those files.

cd $workdir
dtrx -o -n "$file"
check_error

# State Machine State Change. File now extracted to local FS.
# Java program checks if this file is written to see if this directory exists:  $workdir/$(basename $hdfsfile)


filedir="${file%%.*}"

case "$file" in 
  *.tar.gz)
     filedir=$(basename "$hdfsFile" .tar.gz)
    ;;
  *.bz2)
     filedir=$(basename "$hdfsFile" .bz2)
    ;;
  *.bzip2)
     filedir=$(basename "$hdfsFile" .bz2)
    ;;
  *.tgz)
     filedir=$(basename "$hdfsFile" .tgz)
    ;;
  *.gz)
     filedir=$(basename "$hdfsFile" .gz)
    ;;
  *.7z)
     filedir=$(basename "$hdfsFile" .7z)
    ;;
  *.rar)
     filedir=$(basename "$hdfsFile" .rar)
    ;;
  *.zip)
     filedir=$(basename "$hdfsFile" .zip)
    ;;
  *)
    help
    ;;
esac


#replace any spaces in the directory name with '_'
#Hadoop doesnt like spaces in paths.
escapedFiledir=${filedir// /_}
mv "$filedir" $escapedFiledir

echo "UPLOADING" > $fsmFile
<%= node['hops']['base_dir'] %>/bin/hdfs dfs -copyFromLocal $escapedFiledir "$dir"
check_error

echo "CHOWNING" > $fsmFile
<%= node['hops']['base_dir'] %>/bin/hdfs dfs -chown -R ${IMPERSONATED_USERNAME} "$dir"
check_error

echo "SUCCESS" > $fsmFile


{'fraction_non_alphanumeric': 0.15011126846044912, 'fraction_numerical': 0.007283026502124216, 'mean_word_length': 3.7040913415794483, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4030344', 'n_tokens_mistral': 1943, 'n_tokens_neox': 1828, 'n_words': 502}