<commit_before>package main

import (
	"github.com/ardielle/ardielle-go/rdl"
	"fmt"
	"regexp"
	"encoding/json"
	"os"
	"flag"
	"io/ioutil"
	"net/url"
)

type uriInfo struct {
	Method string
	Path string
	PathRegex string
}

func main() {
	pOutdir := flag.String("o", ".", "Output directory")
	flag.String("s", "", "RDL source file")
	flag.Parse()
	var err error
	var data []byte
	if data, err = ioutil.ReadAll(os.Stdin); err == nil {
		err = genPathInfoFile(*pOutdir, data);
	}
	if err != nil {
		fmt.Fprintf(os.Stderr, "*** %v\n", err)
		os.Exit(1)
	}
	os.Exit(0)
}

func genPathInfoFile(outDir string, rdlJson []byte) error {
	var (
		schema rdl.Schema
		pathInfoJson []byte
		err error
	)
	if err = json.Unmarshal(rdlJson, &schema); err != nil {
		return err
	}
	pathInfos := extractPathInfo(&schema)
	if pathInfoJson, err = json.Marshal(pathInfos); err != nil {
		return err
	}
	if err = os.MkdirAll(outDir, 0755); err != nil {
		return err
	}
	fileName := string(schema.Name) + ".json"
	if err = ioutil.WriteFile(fileName, pathInfoJson, 0644); err != nil {
		return err
	}
	return nil
}

func extractPathInfo(schema *rdl.Schema) []uriInfo {
	pathInfos := []uriInfo{}
	for _, resource := range schema.Resources {
		pathInfo := uriInfo{}
		pathInfo.Method = resource.Method
		pathInfo.Path = resource.Path
		pathInfo.PathRegex = genUriRegex(pathInfo.Path, pathInfo.Method)
		pathInfos = append(pathInfos, pathInfo)
	}
	return pathInfos
}

var re = regexp.MustCompile("\\{[^}]+}")

func genUriRegex(path string, method string) string {
	u, _ := url.Parse(path)
	path = u.Path
	pathRegex := re.ReplaceAllString(path, `[^/]+`)
	if method == "GET" {
		pathRegex += `(/?\\?|/?\$)`
	} else {
		pathRegex += `/?\$`
	}
	return pathRegex
}<commit_msg>forget write file to -o specify output directory<commit_after>package main

import (
	"github.com/ardielle/ardielle-go/rdl"
	"fmt"
	"regexp"
	"encoding/json"
	"os"
	"flag"
	"io/ioutil"
	"net/url"
)

type uriInfo struct {
	Method string
	Path string
	PathRegex string
}

func main() {
	pOutdir := flag.String("o", ".", "Output directory")
	flag.String("s", "", "RDL source file")
	flag.Parse()
	var err error
	var data []byte
	if data, err = ioutil.ReadAll(os.Stdin); err == nil {
		err = genPathInfoFile(*pOutdir, data);
	}
	if err != nil {
		fmt.Fprintf(os.Stderr, "*** %v\n", err)
		os.Exit(1)
	}
	os.Exit(0)
}

func genPathInfoFile(outDir string, rdlJson []byte) error {
	var (
		schema rdl.Schema
		pathInfoJson []byte
		err error
	)
	if err = json.Unmarshal(rdlJson, &schema); err != nil {
		return err
	}
	pathInfos := extractPathInfo(&schema)
	if pathInfoJson, err = json.Marshal(pathInfos); err != nil {
		return err
	}
	if err = os.MkdirAll(outDir, 0755); err != nil {
		return err
	}
	fileName := outDir + "/" + string(schema.Name) + ".json"
	if err = ioutil.WriteFile(fileName, pathInfoJson, 0644); err != nil {
		return err
	}
	return nil
}

func extractPathInfo(schema *rdl.Schema) []uriInfo {
	pathInfos := []uriInfo{}
	for _, resource := range schema.Resources {
		pathInfo := uriInfo{}
		pathInfo.Method = resource.Method
		pathInfo.Path = resource.Path
		pathInfo.PathRegex = genUriRegex(pathInfo.Path, pathInfo.Method)
		pathInfos = append(pathInfos, pathInfo)
	}
	return pathInfos
}

var re = regexp.MustCompile("\\{[^}]+}")

func genUriRegex(path string, method string) string {
	u, _ := url.Parse(path)
	path = u.Path
	pathRegex := re.ReplaceAllString(path, `[^/]+`)
	if method == "GET" {
		pathRegex += `(/?\\?|/?\$)`
	} else {
		pathRegex += `/?\$`
	}
	return pathRegex
}
{'fraction_non_alphanumeric': 0.16393442622950818, 'fraction_numerical': 0.005652911249293386, 'mean_word_length': 3.847945205479452, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2476047', 'n_tokens_mistral': 1418, 'n_tokens_neox': 1360, 'n_words': 316}