<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"os/user"
	"path"
	"path/filepath"
	"regexp"
	"strings"

	ck101 "github.com/wuman/go-ck101"
)

var url string

func init() {
	flag.StringVar(&url, "ck101.url", "", "url to grab images from. should have pattern http://ck101.com/thread-2593278-1-1.html")
}

func main() {
	flag.Parse()

	if url == "" || !strings.HasPrefix(url, "http") {
		log.Fatalf("URL should be in the form of http://ck101.com/thread-2593278-1-1.html")
	}

	u, err := user.Current()
	if err != nil {
		log.Fatalf("Failed to get current user: %v", err)
	}
	basedir := filepath.Join(u.HomeDir, "Pictures/go-ck101/")
	threadId := regexp.MustCompile("thread-(\\d+)-.*").FindStringSubmatch(path.Base(url))[1]

	b, err := ck101.GrabPage(url)
	if err != nil {
		log.Fatalf("Failed to grab the page: %v", err)
	}

	targetDir := filepath.Join(basedir, fmt.Sprintf("%s - %s", threadId, b.Title))
	ck101.GrabImages(b, targetDir)
}
<commit_msg>catch thread id parsing error<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os/user"
	"path"
	"path/filepath"
	"regexp"
	"strings"

	ck101 "github.com/wuman/go-ck101"
)

var url string

func init() {
	flag.StringVar(&url, "ck101.url", "", "url to grab images from. should have pattern http://ck101.com/thread-2593278-1-1.html")
}

func main() {
	flag.Parse()

	if url == "" || !strings.HasPrefix(url, "http") {
		log.Fatalf("URL should be in the form of http://ck101.com/thread-2593278-1-1.html")
	}

	u, err := user.Current()
	if err != nil {
		log.Fatalf("Failed to get current user: %v", err)
	}
	basedir := filepath.Join(u.HomeDir, "Pictures/go-ck101/")
	matches := regexp.MustCompile("thread-(\\d+)-.*").FindStringSubmatch(path.Base(url))
	if matches == nil || len(matches) < 2 {
		log.Fatalf("URL should be in the form of http://ck101.com/thread-2593278-1-1.html")
	}
	threadId := matches[1]

	b, err := ck101.GrabPage(url)
	if err != nil {
		log.Fatalf("Failed to grab the page: %v", err)
	}

	targetDir := filepath.Join(basedir, fmt.Sprintf("%s - %s", threadId, b.Title))
	ck101.GrabImages(b, targetDir)
}

{'fraction_non_alphanumeric': 0.18995215311004784, 'fraction_numerical': 0.04736842105263158, 'mean_word_length': 3.8627906976744186, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1016061', 'n_tokens_mistral': 912, 'n_tokens_neox': 841, 'n_words': 185}