<commit_before>#!/bin/bash

set -ex

OPT=$(pwd)/out/compilers
rm -rf ${OPT}
mkdir -p ${OPT}

fetch() {
    curl -v ${http_proxy:+--proxy $http_proxy} -L "$*"
}

get_ghc() {
    local VER=$1
    local DIR=ghc-$VER

	pushd /tmp
	fetch https://downloads.haskell.org/~ghc/${VER}/ghc-${VER}-x86_64-deb8-linux.tar.xz | tar Jxf -
	cd /tmp/ghc-${VER}
	./configure --prefix=${OPT}/ghc
	make install
	popd
    rm -rf /tmp/ghc-${VER}
}

get_gdc() {
    vers=$1
    build=$2
    mkdir ${OPT}/gdc
    pushd ${OPT}/gdc
    fetch ftp://ftp.gdcproject.org/binaries/${vers}/x86_64-linux-gnu/gdc-${vers}+${build}.tar.xz | tar Jxf -
    popd
}

do_rust_install() {
    local DIR=$1
    pushd /tmp
    fetch http://static.rust-lang.org/dist/${DIR}.tar.gz | tar zxvf -
    cd ${DIR}
    ./install.sh --prefix=${OPT}/rust --verbose --without=rust-docs
    popd
    rm -rf /tmp/${DIR}
}

install_new_rust() {
    local NAME=$1
    
    do_rust_install rust-${NAME}-x86_64-unknown-linux-gnu
}
<commit_msg>Put temp files within build structure<commit_after>#!/bin/bash

set -ex

OPT=$(pwd)/out/compilers
rm -rf ${OPT}
mkdir -p ${OPT}
mkdir -p ${OPT}/tmp

fetch() {
    curl -v ${http_proxy:+--proxy $http_proxy} -L "$*"
}

get_ghc() {
    local VER=$1
    local DIR=ghc-$VER

	pushd ${OPT}/tmp
	fetch https://downloads.haskell.org/~ghc/${VER}/ghc-${VER}-x86_64-deb8-linux.tar.xz | tar Jxf -
	cd /tmp/ghc-${VER}
	./configure --prefix=${OPT}/ghc
	make install
	popd
    rm -rf ${OPT}/tmp/ghc-${VER}
}

get_gdc() {
    vers=$1
    build=$2
    mkdir ${OPT}/gdc
    pushd ${OPT}/gdc
    fetch ftp://ftp.gdcproject.org/binaries/${vers}/x86_64-linux-gnu/gdc-${vers}+${build}.tar.xz | tar Jxf -
    popd
}

do_rust_install() {
    local DIR=$1
    pushd ${OPT}/tmp
    fetch http://static.rust-lang.org/dist/${DIR}.tar.gz | tar zxvf -
    cd ${DIR}
    ./install.sh --prefix=${OPT}/rust --without=rust-docs
    popd
    rm -rf ${OPT}/tmp/${DIR}
}

install_new_rust() {
    local NAME=$1
    
    do_rust_install rust-${NAME}-x86_64-unknown-linux-gnu
}

{'fraction_non_alphanumeric': 0.20346534653465345, 'fraction_numerical': 0.01782178217821782, 'mean_word_length': 2.687956204379562, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6472240', 'n_tokens_mistral': 875, 'n_tokens_neox': 845, 'n_words': 153}