<commit_before>export DISTRO_NAME=debian
export DIB_RELEASE=${DIB_RELEASE:-stable}

if [ -n "${DIB_DEBIAN_DISTRIBUTION_MIRROR:-}" ]; then
    DIB_DISTRIBUTION_MIRROR=$DIB_DEBIAN_DISTRIBUTION_MIRROR
fi
export DIB_DISTRIBUTION_MIRROR=${DIB_DISTRIBUTION_MIRROR:-http://deb.debian.org/debian}

export DIB_DEBIAN_COMPONENTS=${DIB_DEBIAN_COMPONENTS:-main}
export DIB_DEBIAN_COMPONENTS_WS=${DIB_DEBIAN_COMPONENTS//,/ }

DIB_APT_SOURCES_CONF_DEFAULT=\
"default:deb ${DIB_DISTRIBUTION_MIRROR} ${DIB_RELEASE} ${DIB_DEBIAN_COMPONENTS_WS}
backports:deb ${DIB_DISTRIBUTION_MIRROR} ${DIB_RELEASE}-backports ${DIB_DEBIAN_COMPONENTS_WS}
updates:deb ${DIB_DISTRIBUTION_MIRROR} ${DIB_RELEASE}-updates ${DIB_DEBIAN_COMPONENTS_WS}
security:deb http://security.debian.org/ ${DIB_RELEASE}/updates ${DIB_DEBIAN_COMPONENTS_WS}
"

# NOTE(ianw): 2019-04 -- remove buster from here when released and it
# has security/backports.
if [ "${DIB_RELEASE}" = "testing" -o "${DIB_RELEASE}" = "unstable" -o "${DIB_RELEASE}" = "buster" ]; then
    DIB_APT_SOURCES_CONF_DEFAULT="default:deb ${DIB_DISTRIBUTION_MIRROR} ${DIB_RELEASE} ${DIB_DEBIAN_COMPONENTS_WS}"
fi

export DIB_APT_SOURCES_CONF=${DIB_APT_SOURCES_CONF:-${DIB_APT_SOURCES_CONF_DEFAULT}}<commit_msg>set default sources conf for buster as it now has a release<commit_after>export DISTRO_NAME=debian
export DIB_RELEASE=${DIB_RELEASE:-stable}

if [ -n "${DIB_DEBIAN_DISTRIBUTION_MIRROR:-}" ]; then
    DIB_DISTRIBUTION_MIRROR=$DIB_DEBIAN_DISTRIBUTION_MIRROR
fi
export DIB_DISTRIBUTION_MIRROR=${DIB_DISTRIBUTION_MIRROR:-http://deb.debian.org/debian}

export DIB_DEBIAN_COMPONENTS=${DIB_DEBIAN_COMPONENTS:-main}
export DIB_DEBIAN_COMPONENTS_WS=${DIB_DEBIAN_COMPONENTS//,/ }

DIB_APT_SOURCES_CONF_DEFAULT=\
"default:deb ${DIB_DISTRIBUTION_MIRROR} ${DIB_RELEASE} ${DIB_DEBIAN_COMPONENTS_WS}
backports:deb ${DIB_DISTRIBUTION_MIRROR} ${DIB_RELEASE}-backports ${DIB_DEBIAN_COMPONENTS_WS}
updates:deb ${DIB_DISTRIBUTION_MIRROR} ${DIB_RELEASE}-updates ${DIB_DEBIAN_COMPONENTS_WS}
security:deb http://security.debian.org/ ${DIB_RELEASE}/updates ${DIB_DEBIAN_COMPONENTS_WS}
"

if [ "${DIB_RELEASE}" = "testing" -o "${DIB_RELEASE}" = "unstable" ]; then
    DIB_APT_SOURCES_CONF_DEFAULT="default:deb ${DIB_DISTRIBUTION_MIRROR} ${DIB_RELEASE} ${DIB_DEBIAN_COMPONENTS_WS}"
fi

export DIB_APT_SOURCES_CONF=${DIB_APT_SOURCES_CONF:-${DIB_APT_SOURCES_CONF_DEFAULT}}
{'fraction_non_alphanumeric': 0.1270578303081469, 'fraction_numerical': 0.002532714225411566, 'mean_word_length': 9.821917808219178, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '421748', 'n_tokens_mistral': 1092, 'n_tokens_neox': 1005, 'n_words': 119}