<commit_before>#!/bin/bash

[[ $# -ne 1 ]] && echo "$0  ‹path to other.erldocs.com dir›" && exit 1
osite="$1"

# Extract URLs of Erlang projects to serve as seed.

root='https://erlangcentral.org/erlang-projects'
outf=seed.gh
touch $outf

source we.sh

function REST() {
    local p=$1 # 1..100
    local cat=$2 # stars forks updated
    local ordering=$3 # desc asc
    echo 'https://github.com/search?o='$ordering'&p='$p'&q=language%3Aerlang&ref=searchresults&s='$cat'&type=Repositories&utf8=%E2%9C%93'
}

function find() {
    for found in $(we -O - "$(REST $*)" | grep -Eoa '<a href="/([^"]+)">\1</a>' | cut -d '"' -f 2 | tr '[:upper:]' '[:lower:]'); do
        proj=github.com$found
        [[ -f $osite/$proj/meta.txt ]] && continue
        echo $proj
        echo $proj >>$outf
    done
}

for ordering in asc desc; do
    for category in stars forks updated; do
        for ((p=1; p<=100; p++)); do
            echo $p $category $ordering
            find $p $category $ordering
            sleep 5
        done
    done
done<commit_msg>gh.sh<commit_after>#!/bin/bash

[[ $# -ne 1 ]] && echo "$0  ‹path to other.erldocs.com dir›" && exit 1
osite="$1"
outf=seed.gh

# Extract URLs of Erlang projects to serve as seed.

source we.sh

function REST() {
    local p=$1 # 1..100
    local cat=$2 # stars forks updated
    local ordering=$3 # desc asc
    echo 'https://github.com/search?o='$ordering'&p='$p'&q=language%3Aerlang&ref=searchresults&s='$cat'&type=Repositories&utf8=%E2%9C%93'
}

function find() {
    for found in $(we -O - "$(REST $*)" | grep -Eoa '<a href="/([^"]+)">\1</a>' | cut -d '"' -f 2 | tr '[:upper:]' '[:lower:]'); do
        proj=github.com$found
        [[ -f $osite/$proj/meta.txt ]] && continue
        echo $proj
        echo $proj >>$outf
    done
}

function iter() {
    local category=$1
    local ordering=$2
    for ((p=1; p<=100; p++)); do
        echo $p $category $ordering
        find $p $category $ordering
        sleep 5
    done
}

# for ordering in asc desc; do
#     for category in stars forks updated; do
#        iter $category $ordering
#     done
# done

#iter stars asc
#iter forks asc
#iter updated asc
#iter stars desc
iter forks desc
iter updated desc
{'fraction_non_alphanumeric': 0.1765504753282028, 'fraction_numerical': 0.022634676324128564, 'mean_word_length': 2.7713310580204777, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4276859', 'n_tokens_mistral': 848, 'n_tokens_neox': 787, 'n_words': 231}