<commit_before>package app

import (
	"yap/alg/perceptron"
	"yap/alg/search"
	"yap/alg/transition"
	transitionmodel "yap/alg/transition/model"
	"yap/nlp/format/conll"
	"yap/nlp/format/conllu"
	"yap/nlp/format/lattice"
	"yap/nlp/format/mapping"
	"yap/nlp/format/segmentation"
	. "yap/nlp/parser/dependency/transition"
	"yap/nlp/parser/dependency/transition/morph"
	"yap/nlp/parser/disambig"
	"yap/nlp/parser/joint"
	nlp "yap/nlp/types"
	"yap/util"
	"yap/util/conf"

	"fmt"
	"log"
	"os"

	"github.com/gonuts/commander"
	"github.com/gonuts/flag"
)

var (
	JointStrategy, OracleStrategy string
	limitdev                      int
)

func SetupEnum(relations []string) {
	SetupRelationEnum(relations)
	SetupMorphTransEnum(relations)
	EWord, EPOS, EWPOS = util.NewEnumSet(APPROX_WORDS), util.NewEnumSet(APPROX_POS), util.NewEnumSet(APPROX_WORDS*5)
	EMHost, EMSuffix = util.NewEnumSet(APPROX_MHOSTS), util.NewEnumSet(APPROX_MSUFFIXES)
	EMorphProp = util.NewEnumSet(130) // random guess of number of possible values
	ETokens = util.NewEnumSet(10000)  // random guess of number of possible values
	// adding empty string as an element in the morph enum sets so that '0' default values
	// map to empty morphs
	EMHost.Add("")
	EMSuffix.Add("")
}

func CombineJointCorpus(graphs, goldLats, ambLats []interface{}) ([]interface{}, int) {
	if len(graphs) != len(goldLats) || len(graphs) != len(ambLats) {
		panic(fmt.Sprintf("Got mismatched training slice inputs (graphs, gold lattices, ambiguous lattices):", len(graphs), len(goldLats), len(ambLats)))
	}
	morphGraphs := make([]interface{}, len(graphs))
	var (
		numLatticeNoGold int
		noGold           int
	)
	prefix := log.Prefix()
	for i, goldGraph := range graphs {
		goldLat := goldLats[i].(nlp.LatticeSentence)
		ambLat := ambLats[i].(nlp.LatticeSentence)
		_, noGold = CombineToGoldMorph(goldLat, ambLat)
		log.SetPrefix(fmt.Sprintf("%v graph# %v ", prefix, i))
		morphGraphs[i], _ = morph.CombineToGoldMorph(goldGraph.(nlp.LabeledDependencyGraph), goldLat, ambLat)
		numLatticeNoGold += noGold
	}
	log.SetPrefix(prefix)
	return morphGraphs, numLatticeNoGold
}

func CombineToGoldMorphs(goldLats, ambLats []interface{}) ([]interface{}, int) {
	if len(goldLats) != len(ambLats) {
		panic(fmt.Sprintf("Got mismatched training slice inputs (gold lattices, ambiguous lattices):", len(goldLats), len(ambLats)))
	}
	morphGraphs := make([]interface{}, len(goldLats))
	var (
		numLatticeNoGold, numSentNoGold int
		numNoGold                       int
	)
	prefix := log.Prefix()
	for i, goldLat := range goldLats {
		ambLat := ambLats[i].(nlp.LatticeSentence)
		log.SetPrefix(fmt.Sprintf("%v lattice# %v ", prefix, i))
		morphGraphs[i], numNoGold = CombineToGoldMorph(goldLat.(nlp.LatticeSentence), ambLat)
		if numNoGold > 0 {
			numSentNoGold++
			numLatticeNoGold += numNoGold
		}
	}
	log.SetPrefix(prefix)
	return morphGraphs, numSentNoGold
}

func JointConfigOut(outModelFile string, b search.Interface, t transition.TransitionSystem) {
	log.Println("*** CONFIGURATION ***")
	log.Printf("Beam:             \t%s", b.Name())
	log.Printf("Transition System:\t%s", t.Name())
	log.Printf("Transition Oracle:\t%s", t.Oracle().Name())
	log.Printf("Iterations:\t\t%d", Iterations)
	log.Printf("Beam Size:\t\t%d", BeamSize)
	log.Printf("Beam Concurrent:\t%v", ConcurrentBeam)
	log.Printf("Parameter Func:\t%v", paramFuncName)
	log.Printf("Use Lemmas:\t\t%v", !lattice.IGNORE_LEMMA)
	log.Printf("Use POP:\t\t%v", UsePOP)
	log.Printf("Infuse Gold Dev:\t%v", combineGold)
	log.Printf("Limit (thousands):\t%v", limit)
	log.Printf("Use CoNLL-U:\t\t%v", useConllU)
	// log.Printf("Model file:\t\t%s", outModelFile)

	log.Println()
	log.Printf("Features File:\t%s", featuresFile)
	if !VerifyExists(featuresFile) {
		os.Exit(1)
	}
	log.Printf("Labels File:\t\t%s", labelsFile)
	if !VerifyExists(labelsFile) {
		os.Exit(1)
	}
	log.Println()
	log.Println("Data")
	log.Printf("Train file (conll):\t\t\t%s", tConll)
	if !VerifyExists(tConll) {
		return
	}
	log.Printf("Train file (disamb. lattice):\t%s", tLatDis)
	if !VerifyExists(tLatDis) {
		return
	}
	log.Printf("Train file (ambig.  lattice):\t%s", tLatAmb)
	if !VerifyExists(tLatAmb) {
		return
	}
	log.Printf("Test file  (ambig.  lattice):\t%s", input)
	if !VerifyExists(input) {
		return
	}
	if len(inputGold) > 0 {
		log.Printf("Test file  (disambig.  lattice):\t%s", inputGold)
		if !VerifyExists(inputGold) {
			return
		}
	}
	log.Printf("Out (disamb.) file:\t\t\t%s", outConll)
	log.Printf("Out (segmt.) file:\t\t\t%s", outSeg)
	log.Printf("Out (mapping.) file:\t\t\t%s", outMap)
	log.Printf("Out Train (segmt.) file:\t\t%s", tSeg)
}

func JointTrainAndParse(cmd *commander.Command, args []string) error {
	// *** SETUP ***
	paramFunc, exists := nlp.MDParams[paramFuncName]
	if !exists {
		log.Fatalln("Param Func", paramFuncName, "does not exist")
	}

	mdTrans := &disambig.MDTrans{
		ParamFunc: paramFunc,
		UsePOP:    UsePOP,
	}

	var (
		arcSystem     transition.TransitionSystem
		model         *transitionmodel.AvgMatrixSparse = &transitionmodel.AvgMatrixSparse{}
		terminalStack int
	)

	switch arcSystemStr {
	case "standard":
		arcSystem = &ArcStandard{}
		terminalStack = 1
	case "eager":
		arcSystem = &ArcEager{
			ArcStandard: ArcStandard{},
		}
		terminalStack = 0
	default:
		panic("Unknown arc system")
	}

	arcSystem.AddDefaultOracle()

	jointTrans := &joint.JointTrans{
		MDTrans:       mdTrans,
		ArcSys:        arcSystem,
		JointStrategy: JointStrategy,
	}
	jointTrans.AddDefaultOracle()
	jointTrans.Oracle().(*joint.JointOracle).OracleStrategy = OracleStrategy
	transitionSystem := transition.TransitionSystem(jointTrans)

	outModelFile := fmt.Sprintf("%s.b%d.i%d", modelFile, BeamSize, Iterations)
	modelExists := VerifyExists(outModelFile)
	REQUIRED_FLAGS := []string{"in", "oc", "om", "os", "f", "l", "jointstr", "oraclestr"}
	VerifyFlags(cmd, REQUIRED_FLAGS)

	if !modelExists {
		REQUIRED_FLAGS = []string{"it", "tc", "td", "tl", "in", "oc", "om", "os", "ots", "f", "l", "jointstr", "oraclestr"}
		VerifyFlags(cmd, REQUIRED_FLAGS)
	}
<commit_msg>Take model file as given<commit_after>package app

import (
	"yap/alg/perceptron"
	"yap/alg/search"
	"yap/alg/transition"
	transitionmodel "yap/alg/transition/model"
	"yap/nlp/format/conll"
	"yap/nlp/format/conllu"
	"yap/nlp/format/lattice"
	"yap/nlp/format/mapping"
	"yap/nlp/format/segmentation"
	. "yap/nlp/parser/dependency/transition"
	"yap/nlp/parser/dependency/transition/morph"
	"yap/nlp/parser/disambig"
	"yap/nlp/parser/joint"
	nlp "yap/nlp/types"
	"yap/util"
	"yap/util/conf"

	"fmt"
	"log"
	"os"

	"github.com/gonuts/commander"
	"github.com/gonuts/flag"
)

var (
	JointStrategy, OracleStrategy string
	limitdev                      int
)

func SetupEnum(relations []string) {
	SetupRelationEnum(relations)
	SetupMorphTransEnum(relations)
	EWord, EPOS, EWPOS = util.NewEnumSet(APPROX_WORDS), util.NewEnumSet(APPROX_POS), util.NewEnumSet(APPROX_WORDS*5)
	EMHost, EMSuffix = util.NewEnumSet(APPROX_MHOSTS), util.NewEnumSet(APPROX_MSUFFIXES)
	EMorphProp = util.NewEnumSet(130) // random guess of number of possible values
	ETokens = util.NewEnumSet(10000)  // random guess of number of possible values
	// adding empty string as an element in the morph enum sets so that '0' default values
	// map to empty morphs
	EMHost.Add("")
	EMSuffix.Add("")
}

func CombineJointCorpus(graphs, goldLats, ambLats []interface{}) ([]interface{}, int) {
	if len(graphs) != len(goldLats) || len(graphs) != len(ambLats) {
		panic(fmt.Sprintf("Got mismatched training slice inputs (graphs, gold lattices, ambiguous lattices):", len(graphs), len(goldLats), len(ambLats)))
	}
	morphGraphs := make([]interface{}, len(graphs))
	var (
		numLatticeNoGold int
		noGold           int
	)
	prefix := log.Prefix()
	for i, goldGraph := range graphs {
		goldLat := goldLats[i].(nlp.LatticeSentence)
		ambLat := ambLats[i].(nlp.LatticeSentence)
		_, noGold = CombineToGoldMorph(goldLat, ambLat)
		log.SetPrefix(fmt.Sprintf("%v graph# %v ", prefix, i))
		morphGraphs[i], _ = morph.CombineToGoldMorph(goldGraph.(nlp.LabeledDependencyGraph), goldLat, ambLat)
		numLatticeNoGold += noGold
	}
	log.SetPrefix(prefix)
	return morphGraphs, numLatticeNoGold
}

func CombineToGoldMorphs(goldLats, ambLats []interface{}) ([]interface{}, int) {
	if len(goldLats) != len(ambLats) {
		panic(fmt.Sprintf("Got mismatched training slice inputs (gold lattices, ambiguous lattices):", len(goldLats), len(ambLats)))
	}
	morphGraphs := make([]interface{}, len(goldLats))
	var (
		numLatticeNoGold, numSentNoGold int
		numNoGold                       int
	)
	prefix := log.Prefix()
	for i, goldLat := range goldLats {
		ambLat := ambLats[i].(nlp.LatticeSentence)
		log.SetPrefix(fmt.Sprintf("%v lattice# %v ", prefix, i))
		morphGraphs[i], numNoGold = CombineToGoldMorph(goldLat.(nlp.LatticeSentence), ambLat)
		if numNoGold > 0 {
			numSentNoGold++
			numLatticeNoGold += numNoGold
		}
	}
	log.SetPrefix(prefix)
	return morphGraphs, numSentNoGold
}

func JointConfigOut(outModelFile string, b search.Interface, t transition.TransitionSystem) {
	log.Println("*** CONFIGURATION ***")
	log.Printf("Beam:             \t%s", b.Name())
	log.Printf("Transition System:\t%s", t.Name())
	log.Printf("Transition Oracle:\t%s", t.Oracle().Name())
	log.Printf("Iterations:\t\t%d", Iterations)
	log.Printf("Beam Size:\t\t%d", BeamSize)
	log.Printf("Beam Concurrent:\t%v", ConcurrentBeam)
	log.Printf("Parameter Func:\t%v", paramFuncName)
	log.Printf("Use Lemmas:\t\t%v", !lattice.IGNORE_LEMMA)
	log.Printf("Use POP:\t\t%v", UsePOP)
	log.Printf("Infuse Gold Dev:\t%v", combineGold)
	log.Printf("Limit (thousands):\t%v", limit)
	log.Printf("Use CoNLL-U:\t\t%v", useConllU)
	// log.Printf("Model file:\t\t%s", outModelFile)

	log.Println()
	log.Printf("Features File:\t%s", featuresFile)
	if !VerifyExists(featuresFile) {
		os.Exit(1)
	}
	log.Printf("Labels File:\t\t%s", labelsFile)
	if !VerifyExists(labelsFile) {
		os.Exit(1)
	}
	log.Println()
	log.Println("Data")
	log.Printf("Train file (conll):\t\t\t%s", tConll)
	if !VerifyExists(tConll) {
		return
	}
	log.Printf("Train file (disamb. lattice):\t%s", tLatDis)
	if !VerifyExists(tLatDis) {
		return
	}
	log.Printf("Train file (ambig.  lattice):\t%s", tLatAmb)
	if !VerifyExists(tLatAmb) {
		return
	}
	log.Printf("Test file  (ambig.  lattice):\t%s", input)
	if !VerifyExists(input) {
		return
	}
	if len(inputGold) > 0 {
		log.Printf("Test file  (disambig.  lattice):\t%s", inputGold)
		if !VerifyExists(inputGold) {
			return
		}
	}
	log.Printf("Out (disamb.) file:\t\t\t%s", outConll)
	log.Printf("Out (segmt.) file:\t\t\t%s", outSeg)
	log.Printf("Out (mapping.) file:\t\t\t%s", outMap)
	log.Printf("Out Train (segmt.) file:\t\t%s", tSeg)
}

func JointTrainAndParse(cmd *commander.Command, args []string) error {
	// *** SETUP ***
	paramFunc, exists := nlp.MDParams[paramFuncName]
	if !exists {
		log.Fatalln("Param Func", paramFuncName, "does not exist")
	}

	mdTrans := &disambig.MDTrans{
		ParamFunc: paramFunc,
		UsePOP:    UsePOP,
	}

	var (
		arcSystem     transition.TransitionSystem
		model         *transitionmodel.AvgMatrixSparse = &transitionmodel.AvgMatrixSparse{}
		terminalStack int
	)

	switch arcSystemStr {
	case "standard":
		arcSystem = &ArcStandard{}
		terminalStack = 1
	case "eager":
		arcSystem = &ArcEager{
			ArcStandard: ArcStandard{},
		}
		terminalStack = 0
	default:
		panic("Unknown arc system")
	}

	arcSystem.AddDefaultOracle()

	jointTrans := &joint.JointTrans{
		MDTrans:       mdTrans,
		ArcSys:        arcSystem,
		JointStrategy: JointStrategy,
	}
	jointTrans.AddDefaultOracle()
	jointTrans.Oracle().(*joint.JointOracle).OracleStrategy = OracleStrategy
	transitionSystem := transition.TransitionSystem(jointTrans)

	outModelFile := modelFile
	modelExists := VerifyExists(outModelFile)
	REQUIRED_FLAGS := []string{"in", "oc", "om", "os", "f", "l", "jointstr", "oraclestr"}
	VerifyFlags(cmd, REQUIRED_FLAGS)

	if !modelExists {
		REQUIRED_FLAGS = []string{"it", "tc", "td", "tl", "in", "oc", "om", "os", "ots", "f", "l", "jointstr", "oraclestr"}
		VerifyFlags(cmd, REQUIRED_FLAGS)
	}

{'fraction_non_alphanumeric': 0.15937344604674292, 'fraction_numerical': 0.0026520802254268194, 'mean_word_length': 4.662599718442046, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7414838', 'n_tokens_mistral': 4797, 'n_tokens_neox': 4618, 'n_words': 861}