<commit_before>package com.yagadi.enguage.interpretant;

import java.io.File;
import java.util.Iterator;
import java.util.TreeSet;

import com.yagadi.enguage.util.Audit;
import com.yagadi.enguage.util.Strings;

public class Concepts {
	static private Audit audit = new Audit( "Concepts" );

    static private      TreeSet<String> names = new TreeSet<String>();
    static public final TreeSet<String> names() { return names; }
	static public                  void names( String location ) {
		audit.in( "names", location );
		for ( String fname : new File( location ).list() ) {
			String[] components = fname.split( "\\." );
			if (components.length > 1 && components[ 1 ].equals("txt")) {
				audit.debug( "adding concept: "+ components[ 0 ]);
				names.add( components[ 0 ]);
		}	}
		audit.out();
	}

	static private TreeSet<String> loaded = new TreeSet<String>();
	static public  TreeSet<String> loaded() { return loaded; }
	
	static private boolean matchesHyphenatedPattern( Strings s, String pattern ) {
		// where pattern may be "to_the_phrase-reply_with"
		Strings pcomp = new Strings( pattern, '-' );
		Iterator<String> ui  = s.iterator();
		Iterator<String> phi = pcomp.iterator();
		if (pcomp.size()>1) { // hyphens found...
			boolean first = true;
			while (ui.hasNext() && phi.hasNext()) { // read through the utterance & pattern
				Iterator<String> pwi = new Strings( phi.next(), '_' ).iterator();
				if (pwi.hasNext()) {
					String pw = pwi.next(),
					       ut = ui.next();
					// find the first matching token
					if (!first)
						while (!ut.equals(pw) && ui.hasNext())
							ut = ui.next();
					//now go thru' matching tokens
					while (ut.equals(pw) && ui.hasNext() && pwi.hasNext()) {
						ut = ui.next();
						pw = pwi.next();
					}
					// return if we have a mismatch
					if (!pw.equals(ut)) return false;
					// hyphen represents at least 1 utterance string, just read over it
					if (phi.hasNext())
						if (ui.hasNext())
							ui.next();
						else
							return false; // hyphen but no text...
					else
						return !ui.hasNext();
					first = false;
			}	}
		} else if(pcomp.size() == 1) // no hyphens found - simple match
			return s.contains( new Strings( phi.next(), '_' ));
		
		return true;
	}

	public static Strings matches( Strings utterance ) {
		audit.in( "matches", utterance.toString() );
		// matches: utt=[martin is a wally], candiates=[ "is_a+has_a" ] => add( is_a+has_a )
		Strings matches = new Strings();
		for (String candidate : names() ) { // e.g. "is_a+has_a" OR "to_the_phrase-reply_with"
			Strings candid = new Strings( candidate, '+' );
			// matching: "to the phrase my name is martin reply hello martin" with "to_the_phrase-reply"
			for (String c : candid) { // e.g. c="to_the_phrase-reply-"
				if (matchesHyphenatedPattern(utterance, c))
					matches.add( candidate );
		}	}
		return audit.out( matches );
	}

	// backwards compatibility... STATICally load a repertoire file
	static public void load( String name ) {
		audit.in( "load", "name="+ name );

		// as with autoloading, make sure it is singular..
		//if (Plural.isPlural( name ))
		//	name = Plural.singular( name );

		// add in name as to what is loaded.
		if (!loaded.contains( name )) {
			// loading repertoires won't use undo - disable
			Allopoiesis.undoEnabledIs( false );
			if ( Concept.load( name )) {
				loaded.add( name );
				audit.debug( "LOADED>>>>>>>>>>>>:"+ name );
			} else
				audit.debug( "LOAD skipping already loaded:"+ name );
			Allopoiesis.undoEnabledIs( true );
		}<commit_msg>Tidy: privatise names<commit_after>package com.yagadi.enguage.interpretant;

import java.io.File;
import java.util.Iterator;
import java.util.TreeSet;

import com.yagadi.enguage.util.Audit;
import com.yagadi.enguage.util.Strings;

public class Concepts {
	static private Audit audit = new Audit( "Concepts" );

	static private       TreeSet<String> names = new TreeSet<String>();
	static private final TreeSet<String> names() { return names; }
	static public                   void names( String location ) {
		audit.in( "names", location );
		for ( String fname : new File( location ).list() ) {
			String[] components = fname.split( "\\." );
			if (components.length > 1 && components[ 1 ].equals("txt")) {
				audit.debug( "adding concept: "+ components[ 0 ]);
				names.add( components[ 0 ]);
		}	}
		audit.out();
	}

	static private TreeSet<String> loaded = new TreeSet<String>();
	static public  TreeSet<String> loaded() { return loaded; }
	
	static private boolean matchesHyphenatedPattern( Strings s, String pattern ) {
		// where pattern may be "to_the_phrase-reply_with"
		Strings pcomp = new Strings( pattern, '-' );
		Iterator<String> ui  = s.iterator();
		Iterator<String> phi = pcomp.iterator();
		if (pcomp.size()>1) { // hyphens found...
			boolean first = true;
			while (ui.hasNext() && phi.hasNext()) { // read through the utterance & pattern
				Iterator<String> pwi = new Strings( phi.next(), '_' ).iterator();
				if (pwi.hasNext()) {
					String pw = pwi.next(),
					       ut = ui.next();
					// find the first matching token
					if (!first)
						while (!ut.equals(pw) && ui.hasNext())
							ut = ui.next();
					//now go thru' matching tokens
					while (ut.equals(pw) && ui.hasNext() && pwi.hasNext()) {
						ut = ui.next();
						pw = pwi.next();
					}
					// return if we have a mismatch
					if (!pw.equals(ut)) return false;
					// hyphen represents at least 1 utterance string, just read over it
					if (phi.hasNext())
						if (ui.hasNext())
							ui.next();
						else
							return false; // hyphen but no text...
					else
						return !ui.hasNext();
					first = false;
			}	}
		} else if(pcomp.size() == 1) // no hyphens found - simple match
			return s.contains( new Strings( phi.next(), '_' ));
		
		return true;
	}

	public static Strings matches( Strings utterance ) {
		audit.in( "matches", utterance.toString() );
		// matches: utt=[martin is a wally], candiates=[ "is_a+has_a" ] => add( is_a+has_a )
		Strings matches = new Strings();
		for (String candidate : names() ) { // e.g. "is_a+has_a" OR "to_the_phrase-reply_with"
			Strings candid = new Strings( candidate, '+' );
			// matching: "to my name is martin reply hello martin" with "to-reply-"
			for (String c : candid) { // e.g. c="to_the_phrase-reply-"
				if (matchesHyphenatedPattern(utterance, c))
					matches.add( candidate );
		}	}
		return audit.out( matches );
	}

	// backwards compatibility... STATICally load a repertoire file
	static public void load( String name ) {
		audit.in( "load", "name="+ name );

		// as with autoloading, make sure it is singular..
		//if (Plural.isPlural( name ))
		//	name = Plural.singular( name );

		// add in name as to what is loaded.
		if (!loaded.contains( name )) {
			// loading repertoires won't use undo - disable
			Allopoiesis.undoEnabledIs( false );
			if ( Concept.load( name )) {
				loaded.add( name );
				audit.debug( "LOADED>>>>>>>>>>>>:"+ name );
			} else
				audit.debug( "LOAD skipping already loaded:"+ name );
			Allopoiesis.undoEnabledIs( true );
		}
{'fraction_non_alphanumeric': 0.15112994350282485, 'fraction_numerical': 0.0019774011299435027, 'mean_word_length': 2.9382647385984426, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2757509', 'n_tokens_mistral': 2728, 'n_tokens_neox': 2286, 'n_words': 680}