<commit_before>#!/bin/sh
# This script downloads a entire pacman repo to a dir
# using the locally configured best mirror.
#
# Copyright (c) 2009 <NAME> <<EMAIL>>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

REPO="$1"
DEST="$2"

if [ -z "$REPO" -o -z "$DEST" ]; then
    echo "usage: $(basename $0) <reponame> <dest-dir>"
    exit 1
fi

if [ $EUID -ne 0 ]; then
    echo "This script must be run as root (for pacman -Sp)"
    exit 1
fi

[ -d "$DEST" ] || mkdir -p "$DEST"

#update repos
/usr/bin/pacman -Sy

#Ensure we have core/pkgname format, so we don't get crap from other repos
PKGS=$(/usr/bin/pacman -Sl $REPO | cut -d' ' -f1,2 | tr ' ' '/')

if [ -n "$PKGS" ]; then
    baseurl=""
    for url in $(/usr/bin/pacman -Sp $PKGS | grep '://'); do
        baseurl="$(dirname "$url")" #save for later
        pkgname="$(basename "$url")"
        wget -nv "$url" -O "$DEST/$pkgname"
    done
    wget -nv "$baseurl/$REPO.db.tar.gz" -O "$DEST/$REPO.db.tar.gz"
else
    echo "No packages to download... what'd you break?"
    exit 1
fi<commit_msg>Look into cache before fetching the pkg remotely.<commit_after>#!/bin/sh
# This script downloads a entire pacman repo to a dir
# using the locally configured best mirror.
#
# Copyright (c) 2009 <NAME> <<EMAIL>>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

REPO="$1"
DEST="$2"

if [ -z "$REPO" -o -z "$DEST" ]; then
    echo "usage: $(basename $0) <reponame> <dest-dir>"
    exit 1
fi

if [ $EUID -ne 0 ]; then
    echo "This script must be run as root (for pacman -Sp)"
    exit 1
fi

[ -d "$DEST" ] || mkdir -p "$DEST"

#update repos
/usr/bin/pacman -Sy

#Ensure we have core/pkgname format, so we don't get crap from other repos
PKGS=$(/usr/bin/pacman -Sl $REPO | cut -d' ' -f1,2 | tr ' ' '/')

if [ -n "$PKGS" ]; then
    baseurl=""
    cachedir="/var/cache/pacman/pkg"
    for url in $(/usr/bin/pacman -Sp $PKGS | grep '://'); do
        baseurl="$(dirname "$url")" #save for later
        pkgname="$(basename "$url")"
        cachedpkg="$cachedir/$pkgname"
        if [ -e "$cachedpkg" ]; then
                cp "$cachedpkg" "$DEST/$pkgname"
        else
                wget -nv "$url" -O "$DEST/$pkgname"
        fi
    done
    wget -nv "$baseurl/$REPO.db.tar.gz" -O "$DEST/$REPO.db.tar.gz"
else
    echo "No packages to download... what'd you break?"
    exit 1
fi
{'fraction_non_alphanumeric': 0.13335237007424328, 'fraction_numerical': 0.00799543118218161, 'mean_word_length': 2.994298745724059, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 2, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1867407', 'n_tokens_mistral': 1179, 'n_tokens_neox': 1129, 'n_words': 477}