<commit_before>#!/usr/bin/env tclsh
package require http
package require tls
package require autoproxy

set script_name resave
set version 0.0.1
set usage {Resave, a web scraping resource downloader.
Usage: resave [OPTION]... [URL]...

Mandatory arguments to long options are mandatory for short options too.

Options:
  -V,  --version           display the version and exit.
  -h,  --help              print this help.

  -r,  --resource          download resource if supported (default).
  -b,  --bookmark          save as bookmark.
  -o,  --output [OUT_DIR]  save files to OUT_DIR/...
  -q,  --quiet             quiet (no output).
}

# Optional args
set optargs {
    resource  1
    quiet     0
}

# Legitimize filename
proc legitimize {filename} {
    string map -nocase {
        "/" "-"
        "\n" " "
    } [string range $filename 0 128]
}

################
# Sub-commands #
################

# Print version
proc version {} {
    global script_name version
    puts stdout "$script_name $version"
}

# Print help
proc help {} {
    global usage
    puts stdout $usage
}

# Save URL as HTML bookmark
proc save_url {url} {
    global optargs
    if {[dict exists $optargs output]} {
        set output [dict get $optargs output]
    } else {
        set output {}
    }

    regexp {^([[:alpha:]]+)://} $url -> protocol
    if {[string equal $protocol https]} {
        ::http::register https 443 ::tls::socket
    }

    regexp {^[[:alpha:]]+://([^/]+)} $url -> domain
    set token [http::geturl $url]
    set data [http::data $token]
    regexp {(?i)<title>([^<]+)} $data -> title
    set filename "$title - $domain"
    http::cleanup $token

    set filename [legitimize $filename]
    set filename [file join $output $filename]

    if {[file exists "$filename.html"]} {
        for {set i 1} {[file exists "$filename ($i).html"]} {incr i} {}
        set filename "$filename ($i)"
    }
    set filename "$filename.html"

    set html "<meta http-equiv=\"refresh\" content=\"0; $url\">"

    set ofid [open $filename w]
    puts -nonewline $ofid $html
    close $ofid
}

# Save images
proc save_images {url} {
    global optargs
    set quiet [dict get $optargs quiet]
    if {[dict exists $optargs output]} {
        set output [dict get $optargs output]
    } else {
        set output {}
    }

    regexp {^[[:alpha:]]+://([^/]+)} $url -> domain
    set token [http::geturl $url]
    set data [http::data $token]
    http::cleanup $token

    regexp {<title>([^<]+)</title>} $data -> title
    set title [string trim $title]

    set matches [regexp -all -inline {http:[^\"\')]+(\.jpg|\.png|\.gif)} $data]
    set len [expr [llength $matches] / 2]
    set i 0
    foreach {imgUrl _} $matches {
        incr i
        set dirname "$title - $domain"

        set dirname [legitimize $dirname]
        set dirname [file join $output $dirname]

        if {![file isdirectory $dirname]} {
            file mkdir $dirname
        }

        regexp {/([^/]+)$} $imgUrl -> output_filename
        puts $output_filename

        set output_filename [legitimize $output_filename]<commit_msg>save_images: support HTTPS (TLS)<commit_after>#!/usr/bin/env tclsh
package require http
package require tls
package require autoproxy

set script_name resave
set version 0.0.1
set usage {Resave, a web scraping resource downloader.
Usage: resave [OPTION]... [URL]...

Mandatory arguments to long options are mandatory for short options too.

Options:
  -V,  --version           display the version and exit.
  -h,  --help              print this help.

  -r,  --resource          download resource if supported (default).
  -b,  --bookmark          save as bookmark.
  -o,  --output [OUT_DIR]  save files to OUT_DIR/...
  -q,  --quiet             quiet (no output).
}

# Optional args
set optargs {
    resource  1
    quiet     0
}

# Legitimize filename
proc legitimize {filename} {
    string map -nocase {
        "/" "-"
        "\n" " "
    } [string range $filename 0 128]
}

################
# Sub-commands #
################

# Print version
proc version {} {
    global script_name version
    puts stdout "$script_name $version"
}

# Print help
proc help {} {
    global usage
    puts stdout $usage
}

# Save URL as HTML bookmark
proc save_url {url} {
    global optargs
    if {[dict exists $optargs output]} {
        set output [dict get $optargs output]
    } else {
        set output {}
    }

    regexp {^([[:alpha:]]+)://} $url -> protocol
    if {[string equal $protocol https]} {
        ::http::register https 443 ::tls::socket
    }

    regexp {^[[:alpha:]]+://([^/]+)} $url -> domain
    set token [http::geturl $url]
    set data [http::data $token]
    regexp {(?i)<title>([^<]+)} $data -> title
    set filename "$title - $domain"
    http::cleanup $token

    set filename [legitimize $filename]
    set filename [file join $output $filename]

    if {[file exists "$filename.html"]} {
        for {set i 1} {[file exists "$filename ($i).html"]} {incr i} {}
        set filename "$filename ($i)"
    }
    set filename "$filename.html"

    set html "<meta http-equiv=\"refresh\" content=\"0; $url\">"

    set ofid [open $filename w]
    puts -nonewline $ofid $html
    close $ofid
}

# Save images
proc save_images {url} {
    global optargs
    set quiet [dict get $optargs quiet]
    if {[dict exists $optargs output]} {
        set output [dict get $optargs output]
    } else {
        set output {}
    }

    http::register https 443 [list ::tls::socket -ssl2 0 -ssl3 0 -tls1 1]

    regexp {^[[:alpha:]]+://([^/]+)} $url -> domain
    set token [http::geturl $url]
    set data [http::data $token]
    http::cleanup $token

    regexp {<title>([^<]+)</title>} $data -> title
    set title [string trim $title]

    set matches [regexp -all -inline {http:[^\"\')]+(\.jpg|\.png|\.gif)} $data]
    set len [expr [llength $matches] / 2]
    set i 0
    foreach {imgUrl _} $matches {
        incr i
        set dirname "$title - $domain"

        set dirname [legitimize $dirname]
        set dirname [file join $output $dirname]

        if {![file isdirectory $dirname]} {
            file mkdir $dirname
        }

        regexp {/([^/]+)$} $imgUrl -> output_filename
        puts $output_filename

        set output_filename [legitimize $output_filename]
{'fraction_non_alphanumeric': 0.15220845249443915, 'fraction_numerical': 0.0065141404512233875, 'mean_word_length': 2.426782798040283, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6555043', 'n_tokens_mistral': 2054, 'n_tokens_neox': 1977, 'n_words': 669}