<commit_before>/*
  Copyright 2014-2016 <NAME>
  Laboratoire d'informatique formelle
  Université du Québec à Chicoutimi, Canada

  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
 */
package ca.uqac.lif.bullwinkle;

import java.io.File;
import java.io.IOException;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;

import ca.uqac.lif.bullwinkle.BnfRule.InvalidRuleException;
import ca.uqac.lif.util.EmptyException;
import ca.uqac.lif.util.FileReadWrite;
import ca.uqac.lif.util.MutableString;

public class BnfParser
{
  private LinkedList<BnfRule> m_rules;

  private BnfRule m_startRule;

  private static transient final String s_CRLF = System.getProperty("line.separator");

  private transient boolean m_debugMode = false;

  /**
   * Maximum number of recursion steps when parsing a string.
   * Parsing will stop immediately when reaching this depth.
   * This is an upper bound that prevents the parser from entering
   * an infinite recursion.
   */
  public static transient int s_maxRecursionSteps = 50;
  
  /**
   * Sets whether partial parsing is enabled
   */
  private boolean m_partialParsing = false;

  public BnfParser()
  {
    super();
    m_startRule = null;
    m_rules = new LinkedList<BnfRule>();
  }

  /**
   * Creates a new parser by copying the rules from another parser
   * @param parser The parser to copy the rules from
   */
  public BnfParser(BnfParser parser)
  {
    super();
    m_rules = new LinkedList<BnfRule>();
    m_rules.addAll(parser.m_rules);
    m_startRule = parser.m_startRule;
  }

  public BnfParser(File f) throws IOException, InvalidGrammarException
  {
    this();
    String contents = FileReadWrite.readFile(f);
    setGrammar(contents);
  }
  
  /**
   * Instructs the parser to perform partial parsing. In partial parsing,
   * a string can contain instances of non-terminal tokens. For example,
   * given the rules
   * <pre>
   * &lt;S&gt; := &lt;A&gt; b
   * &lt;A&gt; := foo | bar
   * </pre>
   * With partial parsing, the string <tt>&lt;A&gt; b</tt> will parse.
   * In this case, note that the resulting parse tree can have non-terminal
   * tokens as leaves.
   * @param b Set to true to enable partial parsing
   */
  public void setPartialParsing(boolean b)
  {
    m_partialParsing = b;
  }

  /**
   * Whether the matching is sensitive to case. This is a program-wide
   * value
   * @param b True if parsing is case-sensitive, false otherwise
   */
  public static void setCaseSensitive(boolean b)
  {
    Token.setCaseSensitive(b);
  }

  public static void setMaxRecursionSteps(int steps)
  {
    if (steps > 0)
    {
      s_maxRecursionSteps = steps;
    }
  }

  public void setDebugMode(boolean b)
  {
    m_debugMode = b;
  }

  @Override
  public String toString()
  {
    StringBuilder out = new StringBuilder();
    for (BnfRule rule : m_rules)
    {
      out.append(rule).append(";\n");
    }
    return out.toString();
  }

  public void addCaseToRule(int index, String rule_name, String case_string)
  {
    BnfRule rule = getRule(rule_name);
    if (rule == null)
    {
      return;
    }
    NonTerminalToken ntok = new NonTerminalToken(case_string);
    TokenString ts = new TokenString();
    ts.add(ntok);
    rule.addAlternative(index, ts);
  }

  public void addCaseToRule(String rule_name, String case_string)
  {
    addCaseToRule(0, rule_name, case_string);
  }

  protected BnfRule getRule(String rule_name)
  {
    for (BnfRule rule : m_rules)
    {
      String lhs = rule.getLeftHandSide().getName();
      if (rule_name.compareTo(lhs) == 0)
      {
        return rule;
      }
    }
    return null;
  }

  public void setGrammar(String grammar) throws InvalidGrammarException
  {
    List<BnfRule> rules = getRules(grammar);
    addRules(rules);
  }

  public static List<BnfRule> getRules(String grammar) throws InvalidGrammarException
  {
    List<BnfRule> rules = new LinkedList<BnfRule>();
    if (grammar == null)
    {
      throw new InvalidGrammarException("Null argument given");
    }
    String[] lines = grammar.split(s_CRLF);
    String current_rule = "";
    for (String line : lines)<commit_msg>Added function to get the alternatives given a left hand rule. Had to add this because getRule is protected.<commit_after>/*
  Copyright 2014-2016 <NAME>
  Laboratoire d'informatique formelle
  Université du Québec à Chicoutimi, Canada

  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
 */
package ca.uqac.lif.bullwinkle;

import java.io.File;
import java.io.IOException;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;

import ca.uqac.lif.bullwinkle.BnfRule.InvalidRuleException;
import ca.uqac.lif.util.EmptyException;
import ca.uqac.lif.util.FileReadWrite;
import ca.uqac.lif.util.MutableString;

public class BnfParser
{
  private LinkedList<BnfRule> m_rules;

  private BnfRule m_startRule;

  private static transient final String s_CRLF = System.getProperty("line.separator");

  private transient boolean m_debugMode = false;

  /**
   * Maximum number of recursion steps when parsing a string.
   * Parsing will stop immediately when reaching this depth.
   * This is an upper bound that prevents the parser from entering
   * an infinite recursion.
   */
  public static transient int s_maxRecursionSteps = 50;
  
  /**
   * Sets whether partial parsing is enabled
   */
  private boolean m_partialParsing = false;

  public BnfParser()
  {
    super();
    m_startRule = null;
    m_rules = new LinkedList<BnfRule>();
  }

  /**
   * Creates a new parser by copying the rules from another parser
   * @param parser The parser to copy the rules from
   */
  public BnfParser(BnfParser parser)
  {
    super();
    m_rules = new LinkedList<BnfRule>();
    m_rules.addAll(parser.m_rules);
    m_startRule = parser.m_startRule;
  }

  public BnfParser(File f) throws IOException, InvalidGrammarException
  {
    this();
    String contents = FileReadWrite.readFile(f);
    setGrammar(contents);
  }
  
  /**
   * Instructs the parser to perform partial parsing. In partial parsing,
   * a string can contain instances of non-terminal tokens. For example,
   * given the rules
   * <pre>
   * &lt;S&gt; := &lt;A&gt; b
   * &lt;A&gt; := foo | bar
   * </pre>
   * With partial parsing, the string <tt>&lt;A&gt; b</tt> will parse.
   * In this case, note that the resulting parse tree can have non-terminal
   * tokens as leaves.
   * @param b Set to true to enable partial parsing
   */
  public void setPartialParsing(boolean b)
  {
    m_partialParsing = b;
  }

  /**
   * Whether the matching is sensitive to case. This is a program-wide
   * value
   * @param b True if parsing is case-sensitive, false otherwise
   */
  public static void setCaseSensitive(boolean b)
  {
    Token.setCaseSensitive(b);
  }

  public static void setMaxRecursionSteps(int steps)
  {
    if (steps > 0)
    {
      s_maxRecursionSteps = steps;
    }
  }

  public void setDebugMode(boolean b)
  {
    m_debugMode = b;
  }

  @Override
  public String toString()
  {
    StringBuilder out = new StringBuilder();
    for (BnfRule rule : m_rules)
    {
      out.append(rule).append(";\n");
    }
    return out.toString();
  }

  public void addCaseToRule(int index, String rule_name, String case_string)
  {
    BnfRule rule = getRule(rule_name);
    if (rule == null)
    {
      return;
    }
    NonTerminalToken ntok = new NonTerminalToken(case_string);
    TokenString ts = new TokenString();
    ts.add(ntok);
    rule.addAlternative(index, ts);
  }

  public void addCaseToRule(String rule_name, String case_string)
  {
    addCaseToRule(0, rule_name, case_string);
  }

  protected BnfRule getRule(String rule_name)
  {
    for (BnfRule rule : m_rules)
    {
      String lhs = rule.getLeftHandSide().getName();
      if (rule_name.compareTo(lhs) == 0)
      {
        return rule;
      }
    }
    return null;
  }
  
  public List<TokenString> getAlternatives(String rule_name)
  {
    for (BnfRule rule : m_rules)
    {
      String lhs = rule.getLeftHandSide().getName();
      if (rule_name.compareTo(lhs) == 0)
      {
        return rule.getAlternatives();
      }
    }
    return null;
  }

  public void setGrammar(String grammar) throws InvalidGrammarException
  {
    List<BnfRule> rules = getRules(grammar);
    addRules(rules);
  }

  public static List<BnfRule> getRules(String grammar) throws InvalidGrammarException
  {
    List<BnfRule> rules = new LinkedList<BnfRule>();
    if (grammar == null)
    {
      throw new InvalidGrammarException("Null argument given");
    }
    String[] lines = grammar.split(s_CRLF);
    String current_rule = "";
    for (String line : lines)
{'fraction_non_alphanumeric': 0.09245667686034659, 'fraction_numerical': 0.003567787971457696, 'mean_word_length': 3.216158143532445, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4040253', 'n_tokens_mistral': 3156, 'n_tokens_neox': 3045, 'n_words': 941}