<commit_before># ex:ts=4
#
# Insert some license text here at some point soon.
#

package provide portfetch 1.0
package require portutil 1.0

register com.apple.fetch target build fetch_main
register com.apple.fetch provides fetch
register com.apple.fetch requires main
register com.apple.fetch swdep build depends_fetch

# define options: distname master_sites
options master_sites patch_sites extract_sufx distfiles extract_only patchfiles use_zip use_bzip2

# Defaults
default extract_sufx tar.gz

set UI_PREFIX "---> "

proc suffix {distname} {
    global extract_sufx use_bzip2 use_zip
    if {[info exists extract_sufx]} {
	return ${distname}.${extract_sufx}
    } elseif {[tbool use_bzip2]} {
	return ${distname}.tar.bz2
    } elseif {[tbool use_zip]} {
	return ${distname}.zip
    } else {
	return ${distname}.tar.gz
    }
}

proc checkfiles {args} {
    global distdir distfiles patchfiles all_dist_files

    lappend filelist $distfiles
    if {[info exists patchfiles]} {
	set filelist [concat $filelist $patchfiles]
    }
    # Set all_dist_files to distfiles + patchfiles
    foreach file $filelist {
	if {![file exists files/$file]} {
	    lappend all_dist_files $file
	}
    }
}

proc fetchfiles {args} {
    global distpath all_dist_files master_sites UI_PREFIX ports_quiet

    if {![file isdirectory $distpath]} {
	file mkdir $distpath
    }

    foreach distfile $all_dist_files {
	if {![file isfile $distpath/$distfile]} {
	    ui_info "$UI_PREFIX $distfile doesn't seem to exist in $distpath"
	    foreach site $master_sites {
		ui_msg "$UI_PREFIX Attempting to fetch $distfile from $site"
		if [tbool ports_quiet] {
			set verboseflag -s
		} elseif [tbool ports_verbose] {
			set verboseflag -v
		} else {
			set verboseflag ""
		}
		if ![catch {system curl ${verboseflag} -o ${distpath}/${distfile} ${site}${distfile} 2>&1} result] {
		    set fetched 1
		    break
		}
	    }
	    if {![info exists fetched]} {
		return -1
	    } else {
		unset fetched<commit_msg>Better attempt at making curl DTRT<commit_after># ex:ts=4
#
# Insert some license text here at some point soon.
#

package provide portfetch 1.0
package require portutil 1.0

register com.apple.fetch target build fetch_main
register com.apple.fetch provides fetch
register com.apple.fetch requires main
register com.apple.fetch swdep build depends_fetch

# define options: distname master_sites
options master_sites patch_sites extract_sufx distfiles extract_only patchfiles use_zip use_bzip2

# Defaults
default extract_sufx tar.gz

set UI_PREFIX "---> "

proc suffix {distname} {
    global extract_sufx use_bzip2 use_zip
    if {[info exists extract_sufx]} {
	return ${distname}.${extract_sufx}
    } elseif {[tbool use_bzip2]} {
	return ${distname}.tar.bz2
    } elseif {[tbool use_zip]} {
	return ${distname}.zip
    } else {
	return ${distname}.tar.gz
    }
}

proc checkfiles {args} {
    global distdir distfiles patchfiles all_dist_files

    lappend filelist $distfiles
    if {[info exists patchfiles]} {
	set filelist [concat $filelist $patchfiles]
    }
    # Set all_dist_files to distfiles + patchfiles
    foreach file $filelist {
	if {![file exists files/$file]} {
	    lappend all_dist_files $file
	}
    }
}

proc fetchfiles {args} {
    global distpath all_dist_files master_sites UI_PREFIX ports_quiet ports_verbose

    if {![file isdirectory $distpath]} {
	file mkdir $distpath
    }

    foreach distfile $all_dist_files {
	if {![file isfile $distpath/$distfile]} {
	    ui_info "$UI_PREFIX $distfile doesn't seem to exist in $distpath"
	    foreach site $master_sites {
		ui_msg "$UI_PREFIX Attempting to fetch $distfile from $site"
		if [tbool ports_quiet] {
			set verboseflag -s
		} elseif [tbool ports_verbose] {
			set verboseflag -v
		} else {
			set verboseflag ""
		}
		if ![catch {system curl ${verboseflag} -o ${distpath}/${distfile} ${site}${distfile} 2>&1} result] {
		    set fetched 1
		    break
		}
	    }
	    if {![info exists fetched]} {
		return -1
	    } else {
		unset fetched
{'fraction_non_alphanumeric': 0.09523809523809523, 'fraction_numerical': 0.006482174021441037, 'mean_word_length': 3.648899188876014, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3188501', 'n_tokens_mistral': 1400, 'n_tokens_neox': 1312, 'n_words': 401}