<commit_before>package ee.stacc.productivity.edsl.parser;

import static org.junit.Assert.*;

import java.util.HashMap;
import java.util.Map;
import java.util.Set;

import org.junit.Test;

import ee.stacc.productivity.edsl.lexer.automata.State;
import ee.stacc.productivity.edsl.sqlparser.IAbstractStack;
import ee.stacc.productivity.edsl.sqlparser.LRParser;


public class FixpointParsingTest {

	@Test
	public void test() throws Exception {
		
	}
	
	public interface IAbstractStackSet {
		boolean add(IAbstractStack stack);
	}
	
	private IAbstractStackSet getSet(Map<State, IAbstractStackSet> abstractStackSets, State state) {
		IAbstractStackSet set = abstractStackSets.get(state);
		if (set == null) {
			set = null;
			abstractStackSets.put(state, set);
		}
		return set;
	}
	
	private boolean parse (LRParser parser, State initial) {
		Map<State, IAbstractStackSet> abstractStackSets = new HashMap<State, IAbstractStackSet>();
		IAbstractStack initialStack = new SimpleStack(parser.getInitialState());
		getSet(abstractStackSets, initial).add(initialStack);
		
		return true;
	}
}<commit_msg>First tests of a fixpoint parser pass<commit_after>package ee.stacc.productivity.edsl.parser;

import static org.junit.Assert.*;

import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;

import org.junit.Test;

import ee.stacc.productivity.edsl.lexer.automata.AutomataParser;
import ee.stacc.productivity.edsl.lexer.automata.AutomataUtils;
import ee.stacc.productivity.edsl.lexer.automata.IAlphabetConverter;
import ee.stacc.productivity.edsl.lexer.automata.State;
import ee.stacc.productivity.edsl.lexer.automata.Transition;
import ee.stacc.productivity.edsl.sqlparser.IAbstractStack;
import ee.stacc.productivity.edsl.sqlparser.IParserState;
import ee.stacc.productivity.edsl.sqlparser.LRParser;


public class FixpointParsingTest {

	@Test
	public void test() throws Exception {
		final LRParser parser = LRParser.build("../ee.stacc.productivity.edsl.sqlparser/generated/sql.xml");
		State initial;
		
		initial = AutomataParser.parse("A - B:S; B - C:I; C - D:F; D - E:I; E - X:X; X - !X1:X");
		AutomataUtils.generate(initial, "");
		assertTrue(doParse(parser, initial));
		
		initial = AutomataParser.parse("A - B:S; B - C:S; C - D:F; D - E:I; E - X:X; X - !X1:X");
		assertFalse(doParse(parser, initial));
		
	}

	private boolean doParse(final LRParser parser, State initial) {
		boolean parse = new FixpointParser(parser, new IAlphabetConverter() {
			
			@Override
			public int convert(int c) {
				Map<String, Integer> namesToTokenNumbers = parser.getNamesToTokenNumbers();
				switch (c) {
				case 'S':
					return namesToTokenNumbers.get("SELECT");
				case 'I':
					return namesToTokenNumbers.get("ID");
				case 'F':
					return namesToTokenNumbers.get("FROM");
				case 'X':
					return namesToTokenNumbers.get("$end");
				case ',':
					return namesToTokenNumbers.get("','");
				}
				throw new IllegalStateException("Unknown token: " + c);
			}
		}, new IAbstractStackSetFactory() {
			
			@Override
			public IAbstractStackSet newAbstractStackSet() {
				return new StackSet();
			}
		}).parse(initial);
		return parse;
	}
	
	public interface IAbstractStackSet {
		
		/**
		 * @return true if there was an actual change 
		 */
		boolean add(IAbstractStack stack);
		
		boolean allAccepting();
		boolean hasError();
		/**
		 * @return a java.util.Set of all IAbstractStack objects, which is 
		 *         tolerant for concurrent modification (e.g., a copy)
		 */
		Set<IAbstractStack> asJavaSet();
	}

	private static class StackSet implements IAbstractStackSet {

		private final Set<IAbstractStack> stacks = new HashSet<IAbstractStack>();
		
		@Override
		public boolean allAccepting() {
			for (IAbstractStack stack : stacks) {
				if (stack.top() != IParserState.ACCEPT) {
					return false;
				}
			}
			return true;
		}

		@Override
		public boolean hasError() {
			for (IAbstractStack stack : stacks) {
				if (stack.top() == IParserState.ERROR) {
					return true;
				}
			}
			return false;
		}

		@Override
		public Set<IAbstractStack> asJavaSet() {
			return new HashSet<IAbstractStack>(stacks);
		}

		@Override
		public boolean add(IAbstractStack stack) {
			return stacks.add(stack);
		}
		
	}
	
	public interface IAbstractStackSetFactory {
		IAbstractStackSet newAbstractStackSet();
	}
	
	private static final class FixpointParser {
	
		private final Map<State, IAbstractStackSet> abstractStackSets = new HashMap<State, IAbstractStackSet>();
		private final IAlphabetConverter alphabetConverter;
		private final LRParser parser;
		private final IAbstractStackSetFactory factory;
		
		public FixpointParser(LRParser parser,
				IAlphabetConverter alphabetConverter,
				IAbstractStackSetFactory factory) {
			this.parser = parser;
			this.alphabetConverter = alphabetConverter;
			this.factory = factory;
		}
		
		private IAbstractStackSet getSet(State state) {
			IAbstractStackSet set = abstractStackSets.get(state);
			if (set == null) {
				set = factory.newAbstractStackSet();
				abstractStackSets.put(state, set);
			}
			return set;
		}

		public boolean parse (State initial) {
			IAbstractStack initialStack = new SimpleStack(parser.getInitialState());
			getSet(initial).add(initialStack);
			return dfs(initial);
		}
	
		private boolean dfs(State current) {
			IAbstractStackSet setForCurrent = getSet(current);
			
			if (current.isAccepting()) {
				if (!setForCurrent.allAccepting()) {
					return false;
				}
			}
			
			for (Transition transition : current.getOutgoingTransitions()) {
				int tokenIndex = alphabetConverter.convert(transition.getInChar());
				IAbstractStackSet setForTo = getSet(transition.getTo());
				Collection<IAbstractStack> hashSet = setForCurrent.asJavaSet();
				boolean changes = false;
				for (IAbstractStack stack : hashSet) {
					Set<IAbstractStack> newStacks = parser.processToken(tokenIndex, stack);
					for (IAbstractStack newStack : newStacks) {
						if (setForTo.add(newStack)) {
							changes = true;
						}
					}
				}
				if (changes) {
					if (setForCurrent.hasError()) {
						return false;
					}
					if (!dfs(transition.getTo())) {
						return false;
					}
				}
			}
			
			return true;
		}
	}
}
{'fraction_non_alphanumeric': 0.11128826530612244, 'fraction_numerical': 0.00031887755102040814, 'mean_word_length': 3.8552631578947367, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5533230', 'n_tokens_mistral': 2334, 'n_tokens_neox': 1947, 'n_words': 439}