<commit_before>package main

import (
	"encoding/xml"
	"flag"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"strings"
	"time"

	"launchpad.net/xmlpath"
)

var path = xmlpath.MustCompile("//resumptionToken")

type options struct {
	from   string
	prefix string
	set    string
	until  string
}

type request struct {
	link  string
	opts  options
	token string
	verb  string
}

type response struct {
	Error struct {
		Code    string `xml:"code,attr"`
		Message string `xml:",chardata"`
	} `xml:"error"`
	Payload string `xml:",innerxml"`
}

func (r request) Link() string {
	v := url.Values{}
	if r.opts.from != "" {
		v.Add("from", r.opts.from)
	}
	if r.opts.set != "" {
		v.Add("set", r.opts.set)
	}
	if r.opts.until != "" {
		v.Add("until", r.opts.until)
	}
	if r.opts.prefix != "" {
		v.Add("metadataPrefix", r.opts.prefix)
	}
	if r.verb != "" {
		v.Add("verb", r.verb)
	}
	if r.token != "" {
		v.Add("resumptionToken", r.token)
	}
	encoded := v.Encode()
	if len(encoded) == 0 {
		return r.link
	}
	return fmt.Sprintf("%s?%s", r.link, encoded)
}

func ExtractToken(s string) string {
	root, err := xmlpath.Parse(strings.NewReader(s))
	if err != nil {
		log.Fatal(err)
	}
	if value, ok := path.String(root); ok {
		return value
	}
	return ""
}

func main() {
<commit_msg>shorten link assembly<commit_after>package main

import (
	"encoding/xml"
	"flag"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"strings"
	"time"

	"launchpad.net/xmlpath"
)

var path = xmlpath.MustCompile("//resumptionToken")

type options struct {
	from   string
	prefix string
	set    string
	until  string
}

type request struct {
	link  string
	opts  options
	token string
	verb  string
}

type response struct {
	Error struct {
		Code    string `xml:"code,attr"`
		Message string `xml:",chardata"`
	} `xml:"error"`
	Payload string `xml:",innerxml"`
}

func (r request) Link() string {
	vals := url.Values{}

	MayAdd := func(k, v string) {
		if v != "" {
			vals.Add(k, v)
		}
	}

	MayAdd("from", r.opts.from)
	MayAdd("set", r.opts.set)
	MayAdd("until", r.opts.until)
	MayAdd("metadataPrefix", r.opts.prefix)
	MayAdd("verb", r.verb)
	MayAdd("resumptionToken", r.token)

	encoded := vals.Encode()
	if len(encoded) == 0 {
		return r.link
	}
	return fmt.Sprintf("%s?%s", r.link, encoded)
}

func ExtractToken(s string) string {
	root, err := xmlpath.Parse(strings.NewReader(s))
	if err != nil {
		log.Fatal(err)
	}
	if value, ok := path.String(root); ok {
		return value
	}
	return ""
}

func main() {

{'fraction_non_alphanumeric': 0.170394469296462, 'fraction_numerical': 0.0008133387555917039, 'mean_word_length': 3.4324324324324325, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 18}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2185934', 'n_tokens_mistral': 1010, 'n_tokens_neox': 966, 'n_words': 228}