<commit_before>package main

import "fmt"
import "os"
import "net/http"
import "io/ioutil"
import "log"

import "github.com/moovweb/gokogiri"
import "github.com/moovweb/gokogiri/xml"

type Remote struct {
	VrouterUrl string
	Port int
	Table      string
}

type Webui struct {
	VrouterUrl string
	Port int
	Path      string
}

type File struct {
	Path string
}

type Sourcer interface {
	Load(descCol DescCollection) Collection
}

func load(url string, fromFile bool) *xml.XmlDocument {
	var data []byte
	if fromFile {
		file, _ := os.Open(url)
		data, _ = ioutil.ReadAll(file)
	} else {
		resp, err := http.Get(url)
		if err != nil {
			log.Fatal(err)
		}
		data, _ = ioutil.ReadAll(resp.Body)
	}
	doc, _ := gokogiri.ParseXml(data)
	return (doc)
}

// Parse data to XML
func fromDataToCollection(data []byte, descCol DescCollection, url string) Collection {
	doc, _ := gokogiri.ParseXml(data)
	ss, _ := doc.Search("/")
	if len(ss) < 1 {
		log.Fatal(fmt.Sprintf("%d Failed to search xpath '%s'", len(ss), descCol.BaseXpath))
	}
	col := Collection{node: ss[0], descCol: descCol, url: url}
	col.Init()
	return col
}

func (file File) Load(descCol DescCollection) Collection {
	f, _ := os.Open(file.Path)
	data, _ := ioutil.ReadAll(f)
	return fromDataToCollection(data, descCol, "file://"+file.Path)
}

func (page Remote) Load(descCol DescCollection) Collection {
	url := fmt.Sprintf("http://%s:%d/Snh_PageReq?x=begin:-1,end:-1,table:%s,", page.VrouterUrl, page.Port, page.Table)
	resp, err := http.Get(url)
	if err != nil {
		log.Fatal(err)
	}
	data, _ := ioutil.ReadAll(resp.Body)
	return fromDataToCollection(data, descCol, url)
}

func (page Webui) Load(descCol DescCollection) Collection {
	url := "http://" + page.VrouterUrl + ":8085/" + page.Path
	resp, err := http.Get(url)
	if err != nil {
		log.Fatal(err)
	}
	data, _ := ioutil.ReadAll(resp.Body)
	return fromDataToCollection(data, descCol, url)
}<commit_msg>load: bypass controller pagination<commit_after>package main

import "fmt"
import "os"
import "net/http"
import "io/ioutil"
import "log"

import "github.com/moovweb/gokogiri"
import "github.com/moovweb/gokogiri/xml"

type Remote struct {
	VrouterUrl string
	Port int
	Table      string
}

type Webui struct {
	VrouterUrl string
	Port int
	Path      string
}

type File struct {
	Path string
}

type Sourcer interface {
	Load(descCol DescCollection) Collection
}

func load(url string, fromFile bool) *xml.XmlDocument {
	var data []byte
	if fromFile {
		file, _ := os.Open(url)
		data, _ = ioutil.ReadAll(file)
	} else {
		resp, err := http.Get(url)
		if err != nil {
			log.Fatal(err)
		}
		data, _ = ioutil.ReadAll(resp.Body)
	}
	doc, _ := gokogiri.ParseXml(data)
	return (doc)
}

func dataToXml(data []byte) xml.Node {
	doc, _ := gokogiri.ParseXml(data)
	ss, _ := doc.Search("/")
	if len(ss) < 1 {
		log.Fatal("Failed to parse XML")
	}
	return ss[0]
}

func xmlToCollection(node xml.Node, descCol DescCollection, url string) Collection {
	col := Collection{node: node, descCol: descCol, url: url}
	col.Init()
	return col
}

// Parse data to XML
func fromDataToCollection(data []byte, descCol DescCollection, url string) Collection {
	return xmlToCollection(dataToXml(data), descCol, url)
}

func (file File) Load(descCol DescCollection) Collection {
	f, _ := os.Open(file.Path)
	data, _ := ioutil.ReadAll(f)
	return fromDataToCollection(data, descCol, "file://"+file.Path)
}

func (page Remote) Load(descCol DescCollection) Collection {
	url := fmt.Sprintf("http://%s:%d/Snh_PageReq?x=begin:-1,end:-1,table:%s,", page.VrouterUrl, page.Port, page.Table)
	resp, err := http.Get(url)
	if err != nil {
		log.Fatal(err)
	}
	data, _ := ioutil.ReadAll(resp.Body)
	return fromDataToCollection(data, descCol, url)
}

func (page Webui) Load(descCol DescCollection) Collection {
	url := fmt.Sprintf("http://%s:%d/%s", page.VrouterUrl, page.Port, page.Path)
	var data []byte
	var node xml.Node

	resp, err := http.Get(url)
	if err != nil {
		log.Fatal(err)
	}
	data, _ = ioutil.ReadAll(resp.Body)
	node = dataToXml(data)

	// Handle controller pagination
	//
	// Since it is not possible to query controller introspect
	// without pagination, we query it several time the controller
	// to rebuild a xml containing all nodes.
	currentNode := node
	for {
		if r, _ := currentNode.Search("/*/next_batch"); len(r) > 0 {
			currentUrl := fmt.Sprintf("http://%s:%d/Snh_%s?x=%s", page.VrouterUrl, page.Port, r[0].Attribute("link"), r[0].Content())
			resp, err := http.Get(currentUrl)
			if err != nil {
				log.Fatal(err)
			}
			data, _ = ioutil.ReadAll(resp.Body)
			currentNode = dataToXml(data)
			newLists, _ := currentNode.Search("/*/*/list")

			for _, l := range newLists {
				lists, _ := node.Search("/*/*/list")
				lists[len(lists) - 1].InsertAfter(l)
			}
		} else {
			break
		}
	}

	return xmlToCollection(node, descCol, url)
}
{'fraction_non_alphanumeric': 0.14890541098719537, 'fraction_numerical': 0.003304419661296985, 'mean_word_length': 4.0082730093071355, 'pattern_counts': {'":': 1, '<': 5, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 11}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7225145', 'n_tokens_mistral': 1876, 'n_tokens_neox': 1811, 'n_words': 436}