<commit_before>#!/usr/bin/env bash
# ------------------------------------------------------------------
shopt -s extglob

abspath_script="$(readlink -f -e "$0")"
script_absdir="$(dirname "$abspath_script")"
script_name="$(basename "$0" .sh)"

if [ $# -eq 0 ]
    then
        cat "$script_absdir/${script_name}_help.txt"
        exit 1
fi

TEMP=$(getopt -o ho:p:nt -l help,outdir:,prefix:,na,two -n "$script_name.sh" -- "$@")

if [ $? -ne 0 ]
then
  echo "Terminating..." >&2
  exit -1
fi

eval set -- "$TEMP"

# Defaults
outdir="$PWD"
prefix="merge_out"

while true
do
  case "$1" in
    -h|--help)			
      cat "$script_absdir/${script_name}_help.txt"
      exit
      ;;
    -o|--outdir)			
      outdir="$2"
      shift 2
      ;;
    -p|--prefix)			
      prefix="$2"
      shift 2
      ;;
    -n|--na)			
      na="x"
      shift
      ;;
    -t|--two)			
      two="x"
      shift
      ;;
    --)
      shift
      break
      ;;
    *)
      echo "$script_name.sh:Internal error!"
      exit 1
      ;;
  esac
done

# Temporary file and output
temp_file="/tmp/merge_temp.txt"
out_file="${outdir}/${prefix}.txt"

# Preprocess input files to handle two fields
if [ "$two" ]
then
  for file in "$@";
  do
    if [[ "$file" =~ \.gz$ ]];
    then
      zcat "$file" | sed 's/\t/_/' > "$temp_file"
      cat "$temp_file" > "$file"
    else
      cat "$file" | sed 's/\t/_/' > "$temp_file"
      cat "$temp_file" > "$file"
    fi
  done
fi

# Copy first file and shift argument
echo "$(date): Merging $1 ..."
first_file="$1"
if [[ "$first_file" =~ \.gz$ ]];
then
  zcat "$first_file" | sort -k 1b,1 > "$out_file" && shift
else
  cat "$first_file" | sort -k 1b,1 > "$out_file" && shift
fi

## Command options
cmd="join"
# Add null fields
if [ "$na" ]
then
  cmd+=" -e "NA" -a1 -a2 -o 0 1.2 2.2"
fi

# Loop through all the other files
for file in "$@";
do 
  i=3
  if [[ "$file" =~ \.gz$ ]];
  then
    echo "$(date): Merging ${file} ..."
    eval "$cmd" "$out_file" <(sort -k 1b,1 <(zcat "$file")) > "$temp_file"
    cat "$temp_file" > "$out_file" 
  else
    echo "$(date): Merging ${file} ..."
    eval "$cmd" "$out_file" <(sort -k 1b,1 "$file") > "$temp_file"
    cat "$temp_file" > "$out_file" 
  fi
  # Update command if na activated
  if [ "$na" ]
  then
    cmd+=" 1.${i}"
  fi
  i+=1
done<commit_msg>Bug fixed<commit_after>#!/usr/bin/env bash
# ------------------------------------------------------------------
shopt -s extglob

abspath_script="$(readlink -f -e "$0")"
script_absdir="$(dirname "$abspath_script")"
script_name="$(basename "$0" .sh)"

if [ $# -eq 0 ]
    then
        cat "$script_absdir/${script_name}_help.txt"
        exit 1
fi

TEMP=$(getopt -o ho:p:nt -l help,outdir:,prefix:,na,two -n "$script_name.sh" -- "$@")

if [ $? -ne 0 ]
then
  echo "Terminating..." >&2
  exit -1
fi

eval set -- "$TEMP"

# Defaults
outdir="$PWD"
prefix="merge_out"

while true
do
  case "$1" in
    -h|--help)			
      cat "$script_absdir/${script_name}_help.txt"
      exit
      ;;
    -o|--outdir)			
      outdir="$2"
      shift 2
      ;;
    -p|--prefix)			
      prefix="$2"
      shift 2
      ;;
    -n|--na)			
      na="x"
      shift
      ;;
    -t|--two)			
      two="x"
      shift
      ;;
    --)
      shift
      break
      ;;
    *)
      echo "$script_name.sh:Internal error!"
      exit 1
      ;;
  esac
done

# Temporary file and output
temp_file="/tmp/merge_temp.txt"
out_file="${outdir}/${prefix}.txt"

# Preprocess input files to handle two fields
if [ "$two" ]
then
  for file in "$@";
  do
    if [[ "$file" =~ \.gz$ ]];
    then
      zcat "$file" | sed 's/\t/_/' > "$temp_file"
      cat "$temp_file" > "$file"
    else
      cat "$file" | sed 's/\t/_/' > "$temp_file"
      cat "$temp_file" > "$file"
    fi
  done
fi

# Copy first file and shift argument
echo "$(date): Merging $1 ..."
first_file="$1"
if [[ "$first_file" =~ \.gz$ ]];
then
  zcat "$first_file" | sort -k 1b,1 > "$out_file" && shift
else
  cat "$first_file" | sort -k 1b,1 > "$out_file" && shift
fi

## Command options
cmd="join"
# Add null fields
if [ "$na" ]
then
  cmd+=" -e "NA" -a1 -a2 -o 0 1.2 2.2"
fi

# Loop through all the other files
i=3
for file in "$@";
do 
  if [[ "$file" =~ \.gz$ ]];
  then
    echo "$(date): Merging ${file} ..."
    eval "$cmd" "$out_file" <(sort -k 1b,1 <(zcat "$file")) > "$temp_file"
    cat "$temp_file" > "$out_file" 
  else
    echo "$(date): Merging ${file} ..."
    eval "$cmd" "$out_file" <(sort -k 1b,1 "$file") > "$temp_file"
    cat "$temp_file" > "$out_file" 
  fi
  # Update command if na activated
  if [ "$na" ]
  then
    cmd+=" 1.${i}"
  fi
  ((i++))
done
{'fraction_non_alphanumeric': 0.22482131254061077, 'fraction_numerical': 0.014078405891271389, 'mean_word_length': 2.050198150594452, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3261680', 'n_tokens_mistral': 1869, 'n_tokens_neox': 1789, 'n_words': 468}