<commit_before>/*
 *  Copyright (C) 2011-2012 The Animo Project
 *  http://animotron.org
 *  	
 *  This file is part of Animotron.
 *  
 *  Animotron is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as 
 *  published by the Free Software Foundation, either version 3 of 
 *  the License, or (at your option) any later version.
 *  
 *  Animotron is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *  
 *  You should have received a copy of 
 *  the GNU Affero General Public License along with Animotron.  
 *  If not, see <http://www.gnu.org/licenses/>.
 */
package org.animotron.statement.query;

import java.util.Arrays;
import java.util.Set;

import org.animotron.graph.index.Order;
import org.animotron.io.Pipe;
import org.animotron.manipulator.OnQuestion;
import org.animotron.manipulator.PFlow;
import org.animotron.manipulator.QCAVector;
import org.animotron.statement.operator.AN;
import org.animotron.statement.operator.Reference;
import org.animotron.statement.operator.Utils;
import org.animotron.statement.value.VALUE;
import org.neo4j.graphdb.Node;
import org.neo4j.graphdb.Path;
import org.neo4j.graphdb.Relationship;
import org.neo4j.graphdb.index.IndexHits;

import javolution.util.FastSet;

/**
 * Query operator 'ALL'.
 * 
 * Return perfect 'all' USE.
 * 
 * @author <a href="mailto:<EMAIL>"><NAME></a>
 * @author <a href="mailto:<EMAIL>"><NAME></a>
 */
public class ALL extends AbstractQuery implements Reference {
	
	public static final ALL _ = new ALL();
	
	private ALL() { super("all", "~~"); }

    public OnQuestion onCalcQuestion() {
        return new Calc();
    }
    
    class Calc extends OnQuestion {
        @Override
        public void act(final PFlow pf) {
			//System.out.println("ALL **************************");
            
        	Pipe pipe = Utils.getByREF(pf, pf.getVector());
        	QCAVector theVector;
        	
            while ((theVector = pipe.take()) != null) {
            	
            	Relationship the = theVector.getAnswer();

    			//check, maybe, result was already calculated
    			if (!Utils.results(the, pf)) {
    				
    				Node node = the.getEndNode();

    				FastSet<Node> uses = FastSet.newInstance();
    				FastSet<Node> weaks = FastSet.newInstance();
    				FastSet<Path> directed = FastSet.newInstance();

					try {
	    				getUSEs(pf, node, uses, weaks, directed);
						
						//System.out.println(uses);
						
						Set<Relationship> list = getExpected(pf);
						if (list !=null && !list.isEmpty()) {
							System.out.println("after predicate "+Arrays.toString(list.toArray()));
							for (Relationship r : list) {
								if (setFiltering(r.getEndNode(), uses, weaks))
									//System.out.print("answered ");
									//Utils.debug(r);
			        				pf.sendAnswer( r );
							}
							return;
						}

	    				boolean underUSE = false;
						Node n = getClosestIntersection(directed);
	    				if (n != null) {
	    					node = n;
	    					underUSE = true;
	    				}
						
		        		Relationship res;
						if (underUSE 
								&& isLeaf(node) 
								&& (res = getThe(node)) != null  
								&& filtering(pf, res, uses, weaks))
							
			            	try {
			            		pf.sendAnswer( res );
			            	} catch (Exception e) {}
						
				        for (Path path : td_IS_leaf.traverse(node)) {
				        	
				        	//System.out.println(path);
	
				        	Relationship r = path.lastRelationship();
				        	if (!Utils.haveContext(r.getEndNode())) {
				        		
				        		//XXX: need better check, it can be reference from other then AN
				        		if (!isLeaf(r.getStartNode()))
				        			continue;
	
			        			try {
					        		res = getThe(r.getStartNode());
				        			if (filtering(pf, res, uses, weaks)) {
				        				pf.sendAnswer( res );
				        			}
			        			} catch (Exception e) {
			        				for (Path p : Utils.td_THE.traverse(r.getStartNode())) {
			        					res = p.lastRelationship();
					        			if (filtering(pf, res, uses, weaks)) {
					        				pf.sendAnswer( res );
					        			}
			        				}
								}
				        	} else {
				    			IndexHits<Relationship> hits = Order.context(r.getEndNode());
				    			try {
				    				for (Relationship rr : hits) {
				    					
				    					if (rr.isType(AN._) || rr.isType(VALUE._)) {
				    						if (filtering(pf, rr, r.getEndNode(), uses, weaks)) {
						        				pf.sendAnswer( rr );
				    						}
				    					}
				    				}
				    			} finally {
				    				hits.close();
				    			}
				        	}
				        }
					} finally {
						FastSet.recycle(uses);
						FastSet.recycle(weaks);
						FastSet.recycle(directed);
					}
    			}
            }
        }
    }
}<commit_msg>optimize ALL<commit_after>/*
 *  Copyright (C) 2011-2012 The Animo Project
 *  http://animotron.org
 *  	
 *  This file is part of Animotron.
 *  
 *  Animotron is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as 
 *  published by the Free Software Foundation, either version 3 of 
 *  the License, or (at your option) any later version.
 *  
 *  Animotron is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *  
 *  You should have received a copy of 
 *  the GNU Affero General Public License along with Animotron.  
 *  If not, see <http://www.gnu.org/licenses/>.
 */
package org.animotron.statement.query;

import java.util.Arrays;
import java.util.Set;

import org.animotron.graph.index.Order;
import org.animotron.manipulator.OnQuestion;
import org.animotron.manipulator.PFlow;
import org.animotron.statement.operator.AN;
import org.animotron.statement.operator.Reference;
import org.animotron.statement.operator.Utils;
import org.animotron.statement.value.VALUE;
import org.neo4j.graphdb.Node;
import org.neo4j.graphdb.Path;
import org.neo4j.graphdb.Relationship;
import org.neo4j.graphdb.index.IndexHits;

import javolution.util.FastSet;

/**
 * Query operator 'ALL'.
 * 
 * Return perfect 'all' USE.
 * 
 * @author <a href="mailto:<EMAIL>"><NAME></a>
 * @author <a href="mailto:<EMAIL>"><NAME></a>
 */
public class ALL extends AbstractQuery implements Reference {
	
	public static final ALL _ = new ALL();
	
	private ALL() { super("all", "~~"); }

    public OnQuestion onCalcQuestion() {
        return new Calc();
    }
    
    class Calc extends OnQuestion {
        @Override
        public void act(final PFlow pf) {
			//System.out.println("ALL **************************");
            
			FastSet<Relationship> thes = FastSet.newInstance();
			try {
				Utils.getTHELikeBag(pf, pf.getVector(), thes);
				
				for (FastSet.Record rc = thes.head(), end = thes.tail(); (rc = rc.getNext()) != end;) {
					Relationship the = thes.valueOf(rc);

	    			//check, maybe, result was already calculated
	    			if (!Utils.results(the, pf)) {
	    				
	    				Node node = the.getEndNode();
	
	    				FastSet<Node> uses = FastSet.newInstance();
	    				FastSet<Node> weaks = FastSet.newInstance();
	    				FastSet<Path> directed = FastSet.newInstance();
	
						try {
		    				getUSEs(pf, node, uses, weaks, directed);
							
							//System.out.println(uses);
							
							Set<Relationship> list = getExpected(pf);
							if (list !=null && !list.isEmpty()) {
								System.out.println("after predicate "+Arrays.toString(list.toArray()));
								for (Relationship r : list) {
									if (setFiltering(r.getEndNode(), uses, weaks))
										//System.out.print("answered ");
										//Utils.debug(r);
				        				pf.sendAnswer( r );
								}
								continue;
							}
	
		    				boolean underUSE = false;
							Node n = getClosestIntersection(directed);
		    				if (n != null) {
		    					node = n;
		    					underUSE = true;
		    				}
							
			        		Relationship res;
							if (underUSE 
									&& isLeaf(node) 
									&& (res = getThe(node)) != null  
									&& filtering(pf, res, uses, weaks))
								
				            	try {
				            		pf.sendAnswer( res );
				            	} catch (Exception e) {}
							
					        for (Path path : td_IS_leaf.traverse(node)) {
					        	
					        	//System.out.println(path);
		
					        	Relationship r = path.lastRelationship();
					        	if (!Utils.haveContext(r.getEndNode())) {
					        		
					        		//XXX: need better check, it can be reference from other then AN
					        		if (!isLeaf(r.getStartNode()))
					        			continue;
		
				        			try {
						        		res = getThe(r.getStartNode());
					        			if (filtering(pf, res, uses, weaks)) {
					        				pf.sendAnswer( res );
					        			}
				        			} catch (Exception e) {
				        				for (Path p : Utils.td_THE.traverse(r.getStartNode())) {
				        					res = p.lastRelationship();
						        			if (filtering(pf, res, uses, weaks)) {
						        				pf.sendAnswer( res );
						        			}
				        				}
									}
					        	} else {
					    			IndexHits<Relationship> hits = Order.context(r.getEndNode());
					    			try {
					    				for (Relationship rr : hits) {
					    					
					    					if (rr.isType(AN._) || rr.isType(VALUE._)) {
					    						if (filtering(pf, rr, r.getEndNode(), uses, weaks)) {
							        				pf.sendAnswer( rr );
					    						}
					    					}
					    				}
					    			} finally {
					    				hits.close();
					    			}
					        	}
					        }
						} finally {
							FastSet.recycle(uses);
							FastSet.recycle(weaks);
							FastSet.recycle(directed);
						}
	    			}
	            }
			} finally {
				FastSet.recycle(thes);
			}
        }
    }
}
{'fraction_non_alphanumeric': 0.12193675889328064, 'fraction_numerical': 0.002569169960474308, 'mean_word_length': 1.836603139013453, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3829094', 'n_tokens_mistral': 4130, 'n_tokens_neox': 3121, 'n_words': 735}