<commit_before>package writescript

import (
	"errors"
	"io/ioutil"
	"net/http"
	"net/url"
	"path/filepath"
	"strings"
)

const (
	KEYWORD_IMPORT = "#import "
)

type Plugin struct {
	ImportURLs      []string
	ImportCodeStack []string
	Js              []string // here we store the main plugin code
}

func (p *Plugin) ParseSource(src string) error {
	pluginLines := strings.Split(src, "\n")
	for _, v := range pluginLines {
		// fmt.Println("k", k, "v", v)

		if strings.Contains(v, KEYWORD_IMPORT) {
			// get the url...
			tmpUrl := strings.Split(v, KEYWORD_IMPORT)

			// check if import already exists, or is not at the list of known urls
			if len(p.ImportURLs) == 0 || !IsValueInList(tmpUrl[1], p.ImportURLs) {
				p.ImportURLs = append(p.ImportURLs, tmpUrl[1])
				data, err := p.request(tmpUrl[1])
				if err != nil {
					return err
				}
				p.ImportCodeStack = append(p.ImportCodeStack, string(data))
			}
		} else {
			p.Js = append(p.Js, v)
		}
	}
	return nil
}

func (p *Plugin) request(url string) ([]byte, error) {
	resp, errReq := http.Get(url)
	if errReq != nil {
		return []byte{}, errReq
	}
	defer resp.Body.Close()
	body, errBody := ioutil.ReadAll(resp.Body)
	if errBody != nil {
		return []byte{}, errBody
	}
	return body, nil
}

// LoadPlugin and return the source as a byte array
func LoadPlugin(src string) ([]byte, error) {
	var err error
	var dataReturn []byte

	switch PluginIsType(src) {

	case PluginTypeUnknown:
		dataReturn = []byte("")
		err = errors.New("No Plugin was set")
		break

	case PluginTypeFile:
		dataReturn, err = ioutil.ReadFile(src)
		break

	case PluginTypeURL:
		resp, errReq := http.Get(src)
		if errReq != nil {
			err = errReq
		}
		defer resp.Body.Close()
		body, errBody := ioutil.ReadAll(resp.Body)
		if errBody != nil {
			err = errBody
		}
		dataReturn = body
		break

	case PluginTypeString:
		dataReturn = []byte(src)
		break
	}

	// TODO: check if plugin keyword exists (in registry)
	// XXX: library of default plugins
	// fmt.Println("search if plugin is embedded...")
	// switch src {
	// case "golang-const":
	// 	// println("...golang-const")
	// 	dataReturn = PLUGIN_GOLANG_CONST
	// 	break
	// case "golang-cli":
	// 	// println("...golang-cli")
	// 	dataReturn = PLUGIN_GOLANG_CLI
	// 	break
	// default:
	// println("... default")
	// }

	// fmt.Println("==> pluginBytes:", string(pluginBytes))
	return dataReturn, err
}

const (<commit_msg>cleanup plugin.go<commit_after>package writescript

import (
	"errors"
	"io/ioutil"
	"net/http"
	"net/url"
	"path/filepath"
	"strings"
)

const (
	KEYWORD_IMPORT = "#import "
)

type Plugin struct {
	ImportURLs      []string
	ImportCodeStack []string
	Js              []string // here we store the main plugin code
}

func (p *Plugin) ParseSource(src string) error {
	pluginLines := strings.Split(src, "\n")
	for _, v := range pluginLines {
		// fmt.Println("k", k, "v", v)

		if strings.Contains(v, KEYWORD_IMPORT) {
			// get the url...
			tmpUrl := strings.Split(v, KEYWORD_IMPORT)

			// check if import already exists, or is not at the list of known urls
			if len(p.ImportURLs) == 0 || !IsValueInList(tmpUrl[1], p.ImportURLs) {
				p.ImportURLs = append(p.ImportURLs, tmpUrl[1])
				data, err := RequestPlugin(tmpUrl[1])
				if err != nil {
					return err
				}
				p.ImportCodeStack = append(p.ImportCodeStack, string(data))
			}
		} else {
			p.Js = append(p.Js, v)
		}
	}
	return nil
}

func RequestPlugin(url string) ([]byte, error) {
	resp, errReq := http.Get(url)
	if errReq != nil {
		return []byte{}, errReq
	}
	defer resp.Body.Close()
	body, errBody := ioutil.ReadAll(resp.Body)
	if errBody != nil {
		return []byte{}, errBody
	}
	return body, nil
}

// LoadPlugin and return the source as a byte array
func LoadPlugin(src string) ([]byte, error) {
	var err error
	var dataReturn []byte

	switch PluginIsType(src) {

	case PluginTypeUnknown:
		dataReturn = []byte("")
		err = errors.New("No Plugin was set")
		break

	case PluginTypeFile:
		dataReturn, err = ioutil.ReadFile(src)
		break

	case PluginTypeURL:
		dataReturn, err = RequestPlugin(src)
		break

	case PluginTypeString:
		dataReturn = []byte(src)
		break
	}

	// fmt.Println("==> pluginBytes:", string(pluginBytes))
	return dataReturn, err
}

const (
{'fraction_non_alphanumeric': 0.14410377358490567, 'fraction_numerical': 0.0018867924528301887, 'mean_word_length': 3.203171456888008, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3647577', 'n_tokens_mistral': 1668, 'n_tokens_neox': 1528, 'n_words': 409}