<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"strings"
	"github.com/mozmark/OneCRL-Tools/oneCRL"	
)

type OneCRLSet struct {
	records oneCRL.Records
}

func (set *OneCRLSet) LoadRecord(record oneCRL.Record) {
	set.records.Data = append(set.records.Data, record)
}

func (set *OneCRLSet) FetchData(location string) error {
	if 0 == strings.Index(strings.ToLower(location),"http://") ||
	   0 == strings.Index(strings.ToLower(location),"https://") {
	   return oneCRL.LoadJSONFromURL(location, set)
   } else if location == "production" {
	   return oneCRL.LoadJSONFromURL(oneCRL.ProductionPrefix + oneCRL.RecordsPath, set)
   } else if location == "stage" {
	   return oneCRL.LoadJSONFromURL(oneCRL.StagePrefix + oneCRL.RecordsPath, set)
   } else {
	   return oneCRL.LoadRevocationsTxtFromFile(location, set)
   }
}

func main() {
	set1 := new(OneCRLSet)
	set2 := new(OneCRLSet)

	oneCRL.DefineFlags()
	flag.Parse()
	
	config := oneCRL.Config

	args := flag.Args()
	if len(args) < 1 {
		log.Fatal("We need at least one OneCRL file to operate on")
		return
	}
	if len(args) > 2 {
		log.Fatal("oneCRL diff only operates on two OneCRL data sets")
		return
	}
	if len(args) < 2 {
		// TODO: Read env. options and (optionally) fetch data from a different
		// environment than production
		set1.FetchData(config.GetRecordURL())
		set2.FetchData(args[0])
	} else {
		set1.FetchData(args[0])
		set2.FetchData(args[1])
	}

	fmt.Printf("set1 has %d entries\n",len((*set1).records.Data))<commit_msg>update oneCRLDiff to reflect config changes<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"strings"
	"github.com/mozmark/OneCRL-Tools/oneCRL"	
)

type OneCRLSet struct {
	records oneCRL.Records
}

func (set *OneCRLSet) LoadRecord(record oneCRL.Record) {
	set.records.Data = append(set.records.Data, record)
}

func (set *OneCRLSet) FetchData(location string) error {
	if 0 == strings.Index(strings.ToLower(location),"http://") ||
	   0 == strings.Index(strings.ToLower(location),"https://") {
	   return oneCRL.LoadJSONFromURL(location, set)
   } else if location == "production" {
	   return oneCRL.LoadJSONFromURL(oneCRL.ProductionPrefix + oneCRL.RecordsPath, set)
   } else if location == "stage" {
	   return oneCRL.LoadJSONFromURL(oneCRL.StagePrefix + oneCRL.RecordsPath, set)
   } else {
	   return oneCRL.LoadRevocationsTxtFromFile(location, set)
   }
}

func main() {
	set1 := new(OneCRLSet)
	set2 := new(OneCRLSet)

	oneCRL.DefineFlags()
	flag.Parse()
	
	config := oneCRL.GetConfig()

	args := flag.Args()
	if len(args) < 1 {
		log.Fatal("We need at least one OneCRL file to operate on")
		return
	}
	if len(args) > 2 {
		log.Fatal("oneCRL diff only operates on two OneCRL data sets")
		return
	}
	if len(args) < 2 {
		// TODO: Read env. options and (optionally) fetch data from a different
		// environment than production
		set1.FetchData(config.GetRecordURL())
		set2.FetchData(args[0])
	} else {
		set1.FetchData(args[0])
		set2.FetchData(args[1])
	}

	fmt.Printf("set1 has %d entries\n",len((*set1).records.Data))
{'fraction_non_alphanumeric': 0.1385878489326765, 'fraction_numerical': 0.010509031198686371, 'mean_word_length': 4.14527027027027, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6299631', 'n_tokens_mistral': 1175, 'n_tokens_neox': 1118, 'n_words': 244}