<commit_before>#!/usr/bin/env bash

# clean up
if [ ! -e repo-build ]; then
    mkdir repo-build
else
    rm -rf repo-build/*
fi

# sync git submodules
git submodule init
git submodule update
# loop through repos
for R in `find pkgs/* -maxdepth 0 -type d`
do

    REPO=$(echo $R | sed -e "s/pkgs\///")

    # create empty repos
    mkdir -p repo-build/dists/cldemo/$REPO/binary-amd64
    mkdir -p repo-build/dists/cldemo/$REPO/binary-i386
    mkdir -p repo-build/dists/cldemo/$REPO/binary-powerpc

    # loop through packages
    for P in `find pkgs/$REPO/* -maxdepth 0 -type d`
    do
        PKG=$(echo $P | sed -e "s/pkgs\/$REPO\///")
        dpkg-deb --build pkgs/$REPO/$PKG/debian repo-build/dists/cldemo/$REPO/binary-amd64/${PKG}_amd64.deb
    done

    # generate package lists
    dpkg-scanpackages -a amd64 repo-build/dists/cldemo/$REPO/binary-amd64 /dev/null | sed -e 's/Filename: repo-build\//Filename: /' > repo-build/dists/cldemo/$REPO/binary-amd64/Packages
    dpkg-scanpackages -a i386 repo-build/dists/cldemo/$REPO/binary-amd64 /dev/null | sed -e 's/Filename: repo-build\//Filename: /' > repo-build/dists/cldemo/$REPO/binary-i386/Packages
    dpkg-scanpackages -a powerpc repo-build/dists/cldemo/$REPO/binary-powerpc /dev/null | sed -e 's/Filename: repo-build\//Filename: /' > repo-build/dists/cldemo/$REPO/binary-powerpc/Packages

done

# copy static content
cp -R repo-static/* repo-build/<commit_msg>build script stops upon pkg failure<commit_after>#!/usr/bin/env bash

# clean up
if [ ! -e repo-build ]; then
    mkdir repo-build
else
    rm -rf repo-build/*
fi

# sync git submodules
git submodule init
git submodule update

architectures=(amd64 i386 powerpc)

# loop through repos
for R in `find pkgs/* -maxdepth 0 -type d`
do

    REPO=$(echo $R | sed -e "s/pkgs\///")

    echo ""
    echo "$REPO repo..."
    echo ""

    # create empty repos
    for ARCH in ${architectures[@]}; do
        mkdir -p repo-build/dists/cldemo/$REPO/binary-$ARCH
    done

    # loop through packages
    for P in `find pkgs/$REPO/* -maxdepth 0 -type d`
    do
        PKG=$(echo $P | sed -e "s/pkgs\/$REPO\///")
        if ! dpkg-deb --build pkgs/$REPO/$PKG/debian repo-build/dists/cldemo/$REPO/binary-amd64/${PKG}_amd64.deb
	then
	    echo "** ERROR ** while building $REPO/$PKG"
	    exit 1
	fi
    done

    # generate package lists
    for ARCH in ${architectures[@]}; do
        if ! dpkg-scanpackages -a $ARCH repo-build/dists/cldemo/$REPO/binary-$ARCH /dev/null | sed -e 's/Filename: repo-build\//Filename: /' > repo-build/dists/cldemo/$REPO/binary-$ARCH/Packages
        then
            echo "** ERROR ** while generating repo for  $REPO $ARCH"
            exit 1
        fi
    done

done

# copy static content
cp -R repo-static/* repo-build/

echo ""
echo "Done"
echo ""

exit 0
{'fraction_non_alphanumeric': 0.14663805436337626, 'fraction_numerical': 0.013948497854077254, 'mean_word_length': 2.581306017925736, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1125404', 'n_tokens_mistral': 1095, 'n_tokens_neox': 1019, 'n_words': 263}