<commit_before>#!/bin/bash

BIDMAT_ROOT="${BASH_SOURCE[0]}"
if [ ! `uname` = "Darwin" ]; then
  BIDMAT_ROOT=`readlink -f "${BIDMAT_ROOT}"`
else 
  while [ -L "${BIDMAT_ROOT}" ]; do
    BIDMAT_ROOT=`readlink "${BIDMAT_ROOT}"`
  done
fi
BIDMAT_ROOT=`dirname "$BIDMAT_ROOT"`
pushd "${BIDMAT_ROOT}"  > /dev/null
BIDMAT_ROOT=`pwd`
BIDMAT_ROOT="$( echo ${BIDMAT_ROOT} | sed s+/cygdrive/c+c:+ )" 

source="http://www.cs.berkeley.edu/~jfc/biddata"
cd ${BIDMAT_ROOT}/lib

if [ `uname` = "Darwin" ]; then
    subdir="osx"
    curl -o liblist.txt ${source}/lib/liblist_osx.txt 
elif [ "$OS" = "Windows_NT" ]; then
    subdir="win"
    curl -o liblist.txt ${source}/lib/liblist_win.txt
else
    subdir="linux"
    curl -o liblist.txt ${source}/lib/liblist_linux.txt
fi
curl -o exelist.txt ${source}/lib/exelist.txt

while read fname; do
    echo -e "\nDownloading ${fname}"
    curl --retry 2 -O ${source}/lib/${fname}
    chmod 755 ${fname}
done < liblist.txt

mv ${BIDMAT_ROOT}/lib/BIDMat.jar ${BIDMAT_ROOT}
rm ${BIDMAT_ROOT}/lib/BIDMach.jar 
<commit_msg>getdevlibs for arm<commit_after>#!/bin/bash

if [ "${ARCH}" = "" ];then
    ARCH=`arch`
fi

BIDMAT_ROOT="${BASH_SOURCE[0]}"
if [ ! `uname` = "Darwin" ]; then
  BIDMAT_ROOT=`readlink -f "${BIDMAT_ROOT}"`
else 
  while [ -L "${BIDMAT_ROOT}" ]; do
    BIDMAT_ROOT=`readlink "${BIDMAT_ROOT}"`
  done
fi
BIDMAT_ROOT=`dirname "$BIDMAT_ROOT"`
pushd "${BIDMAT_ROOT}"  > /dev/null
BIDMAT_ROOT=`pwd`
BIDMAT_ROOT="$( echo ${BIDMAT_ROOT} | sed s+/cygdrive/c+c:+ )" 

source="http://www.cs.berkeley.edu/~jfc/biddata"
cd ${BIDMAT_ROOT}/lib

if [ `uname` = "Darwin" ]; then
    subdir="osx"
    curl -o liblist.txt ${source}/lib/liblist_osx.txt 
elif [ "$OS" = "Windows_NT" ]; then
    subdir="win"
    curl -o liblist.txt ${source}/lib/liblist_win.txt
else
    if [[ "${ARCH}" == arm* ]]; then
        subdir="linux_arm"
        curl -o liblist.txt ${source}/lib/liblist_linux_arm.txt
    else
        subdir="linux"
        curl -o liblist.txt ${source}/lib/liblist_linux.txt
    fi
fi
curl -o exelist.txt ${source}/lib/exelist.txt

while read fname; do
    echo -e "\nDownloading ${fname}"
    curl --retry 2 -O ${source}/lib/${fname}
    chmod 755 ${fname}
done < liblist.txt

mv ${BIDMAT_ROOT}/lib/BIDMat.jar ${BIDMAT_ROOT}
rm ${BIDMAT_ROOT}/lib/BIDMach.jar 

{'fraction_non_alphanumeric': 0.1874455100261552, 'fraction_numerical': 0.004359197907585004, 'mean_word_length': 3.751552795031056, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '564723', 'n_tokens_mistral': 989, 'n_tokens_neox': 950, 'n_words': 186}