<commit_before>package com.jayantkrish.jklol.cvsm.ccg;

import java.util.Arrays;
import java.util.List;

import joptsimple.OptionParser;
import joptsimple.OptionSet;
import joptsimple.OptionSpec;

import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.jayantkrish.jklol.ccg.lambda.ApplicationExpression;
import com.jayantkrish.jklol.ccg.lambda.ConstantExpression;
import com.jayantkrish.jklol.ccg.lambda.Expression;
import com.jayantkrish.jklol.ccg.lambda.ExpressionParser;
import com.jayantkrish.jklol.cli.AbstractCli;
import com.jayantkrish.jklol.cvsm.ccg.CcgLfReader.LogicalFormConversionError;
import com.jayantkrish.jklol.util.IndexedList;
import com.jayantkrish.jklol.util.IoUtils;

public class ConvertCncToCvsm extends AbstractCli {
  
  private OptionSpec<String> cncParses;
  private OptionSpec<String> mentions;
  private OptionSpec<String> relationDictionary;
  private OptionSpec<String> lfTemplates;
  
  private CcgLfReader reader;
  
  public ConvertCncToCvsm() {
    super();
  }

  @Override
  public void initializeOptions(OptionParser parser) {
    cncParses = parser.accepts("cncParses").withRequiredArg().ofType(String.class).required();
    mentions = parser.accepts("mentions").withRequiredArg().ofType(String.class).required();
    relationDictionary = parser.accepts("relationDictionary").withRequiredArg().ofType(String.class).required();
    lfTemplates = parser.accepts("lfTemplates").withRequiredArg().ofType(String.class).required();
  }

  @Override
  public void run(OptionSet options) {
    reader = CcgLfReader.parseFrom(IoUtils.readLines(options.valueOf(lfTemplates)));
    ExpressionParser exp = new ExpressionParser();

    List<RelationExtractionExample> examples = readExamples(IoUtils.readLines(options.valueOf(mentions)));
    IndexedList<String> relDict = IndexedList.create(IoUtils.readLines(options.valueOf(relationDictionary)));
    
    List<String> lines = IoUtils.readLines(options.valueOf(cncParses));
    Expression ccgExpression = null;
    List<Expression> wordExpressions = null;
    List<Expression> expressions = Lists.newArrayList();
    for (String line : lines) {
      if (!line.startsWith("(")) {
        continue;
      }
      
      if (line.startsWith("(ccg")) {
        if (ccgExpression != null) {
          int parseNum = Integer.parseInt(((ConstantExpression) ((ApplicationExpression) ccgExpression).getArguments().get(0)).getName());
          RelationExtractionExample sentence = examples.get(parseNum - 1);
          expressions.add(convertExpression(sentence, ccgExpression, wordExpressions, relDict));
        }
        ccgExpression = exp.parseSingleExpression(line);
        wordExpressions = Lists.newArrayList();
      } else if (line.startsWith("(w")) {
        wordExpressions.add(exp.parseSingleExpression(line));
      }
    }

    if (ccgExpression != null) {
      int parseNum = Integer.parseInt(((ConstantExpression) ((ApplicationExpression) ccgExpression).getArguments().get(0)).getName());
      RelationExtractionExample sentence = examples.get(parseNum - 1);
      expressions.add(convertExpression(sentence, ccgExpression, wordExpressions, relDict));
    }
  }
  
  private List<RelationExtractionExample> readExamples(List<String> lines) {
    List<RelationExtractionExample> examples = Lists.newArrayList();
    for (int i = 0; i < lines.size(); i++) {
      String line = lines.get(i);
      if (line.matches("^[0-9].*")) {
        String label = lines.get(i + 1);
        
        String sentence = line.split("\t")[1];
        sentence = sentence.replaceFirst("^\\s*\"(.*)\"\\s*$", "$1");        
        examples.add(new RelationExtractionExample(sentence, label));
      }
    }
    return examples;
  }
  
  private Expression convertExpression(RelationExtractionExample example, Expression ccgExpression, 
      List<Expression> initialWordExpressions, IndexedList<String> relDict) {
    ccgExpression = ((ApplicationExpression) ccgExpression).getArguments().get(1);

    // Find the marked mentions to identify the sentence span to retain.
    Span e1Span = mapSpanToTokenizedSpan(example.getE1Span(), wordExpressions);
    Span e2Span = mapSpanToTokenizedSpan(example.getE2Span(), wordExpressions);
    if (e1Span == null || e2Span == null) {
      System.err.println("No conversion: no span for " + example.getE1Span() + " and " + example.getE2Span());
      return null;
    }
    Expression spanningExpression = reader.findSpanningExpression(ccgExpression, e1Span.getStart(), e2Span.getEnd());
    if (spanningExpression == null) {
      System.err.println("No conversion: no atomic type: " + ccgExpression);
      return null;
    }

    try {
      Expression parsedExpression = reader.parse(spanningExpression, wordExpressions);
      parsedExpression = new ApplicationExpression(new ConstantExpression("op:softmax"), Arrays.asList(new ApplicationExpression(new ConstantExpression("op:matvecmul"), Arrays.asList(new ConstantExpression("weights:softmax"), parsedExpression))));
<commit_msg>fixing handling of unparsable sentences<commit_after>package com.jayantkrish.jklol.cvsm.ccg;

import java.util.Arrays;
import java.util.List;

import joptsimple.OptionParser;
import joptsimple.OptionSet;
import joptsimple.OptionSpec;

import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.jayantkrish.jklol.ccg.lambda.ApplicationExpression;
import com.jayantkrish.jklol.ccg.lambda.ConstantExpression;
import com.jayantkrish.jklol.ccg.lambda.Expression;
import com.jayantkrish.jklol.ccg.lambda.ExpressionParser;
import com.jayantkrish.jklol.cli.AbstractCli;
import com.jayantkrish.jklol.cvsm.ccg.CcgLfReader.LogicalFormConversionError;
import com.jayantkrish.jklol.util.IndexedList;
import com.jayantkrish.jklol.util.IoUtils;

public class ConvertCncToCvsm extends AbstractCli {
  
  private OptionSpec<String> cncParses;
  private OptionSpec<String> mentions;
  private OptionSpec<String> relationDictionary;
  private OptionSpec<String> lfTemplates;
  
  private CcgLfReader reader;
  
  public ConvertCncToCvsm() {
    super();
  }

  @Override
  public void initializeOptions(OptionParser parser) {
    cncParses = parser.accepts("cncParses").withRequiredArg().ofType(String.class).required();
    mentions = parser.accepts("mentions").withRequiredArg().ofType(String.class).required();
    relationDictionary = parser.accepts("relationDictionary").withRequiredArg().ofType(String.class).required();
    lfTemplates = parser.accepts("lfTemplates").withRequiredArg().ofType(String.class).required();
  }

  @Override
  public void run(OptionSet options) {
    reader = CcgLfReader.parseFrom(IoUtils.readLines(options.valueOf(lfTemplates)));
    ExpressionParser exp = new ExpressionParser();

    List<RelationExtractionExample> examples = readExamples(IoUtils.readLines(options.valueOf(mentions)));
    IndexedList<String> relDict = IndexedList.create(IoUtils.readLines(options.valueOf(relationDictionary)));
    
    List<String> lines = IoUtils.readLines(options.valueOf(cncParses));
    Expression ccgExpression = null;
    List<Expression> wordExpressions = null;
    List<Expression> expressions = Lists.newArrayList();
    for (String line : lines) {
      if (!line.startsWith("(")) {
        continue;
      }
      
      if (line.startsWith("(ccg")) {
        if (ccgExpression != null) {
          int parseNum = Integer.parseInt(((ConstantExpression) ((ApplicationExpression) ccgExpression).getArguments().get(0)).getName());
          RelationExtractionExample sentence = examples.get(parseNum - 1);
          expressions.add(convertExpression(sentence, ccgExpression, wordExpressions, relDict));
        }
        ccgExpression = exp.parseSingleExpression(line);
        wordExpressions = Lists.newArrayList();
      } else if (line.startsWith("(w")) {
        wordExpressions.add(exp.parseSingleExpression(line));
      }
    }

    if (ccgExpression != null) {
      int parseNum = Integer.parseInt(((ConstantExpression) ((ApplicationExpression) ccgExpression).getArguments().get(0)).getName());
      RelationExtractionExample sentence = examples.get(parseNum - 1);
      expressions.add(convertExpression(sentence, ccgExpression, wordExpressions, relDict));
    }
  }
  
  private List<RelationExtractionExample> readExamples(List<String> lines) {
    List<RelationExtractionExample> examples = Lists.newArrayList();
    for (int i = 0; i < lines.size(); i++) {
      String line = lines.get(i);
      if (line.matches("^[0-9].*")) {
        String label = lines.get(i + 1);
        
        String sentence = line.split("\t")[1];
        sentence = sentence.replaceFirst("^\\s*\"(.*)\"\\s*$", "$1");        
        examples.add(new RelationExtractionExample(sentence, label));
      }
    }
    return examples;
  }
  
  private Expression convertExpression(RelationExtractionExample example, Expression ccgExpression, 
      List<Expression> initialWordExpressions, IndexedList<String> relDict) {
    int parseNum = Integer.parseInt(((ConstantExpression) ((ApplicationExpression) ccgExpression).getArguments().get(0)).getName());
    ccgExpression = ((ApplicationExpression) ccgExpression).getArguments().get(1);

    List<Expression> wordExpressions = Lists.newArrayList(initialWordExpressions);
    for (Expression wordExpression : initialWordExpressions) {
      int wordParseNum = Integer.parseInt(((ConstantExpression) 
          ((ApplicationExpression) wordExpressions).getArguments().get(0)).getName());
      
      if (wordParseNum != parseNum) {
        wordExpressions.remove(wordExpression);
      }
    }

    // Find the marked mentions to identify the sentence span to retain.
    Span e1Span = mapSpanToTokenizedSpan(example.getE1Span(), wordExpressions);
    Span e2Span = mapSpanToTokenizedSpan(example.getE2Span(), wordExpressions);
    if (e1Span == null || e2Span == null) {
      System.err.println("No conversion: no span for " + example.getE1Span() + " and " + example.getE2Span());
      return null;
    }
    Expression spanningExpression = reader.findSpanningExpression(ccgExpression, e1Span.getStart(), e2Span.getEnd());
    if (spanningExpression == null) {
      System.err.println("No conversion: no atomic type: " + ccgExpression);
      return null;
    }

    try {
      Expression parsedExpression = reader.parse(spanningExpression, wordExpressions);
      parsedExpression = new ApplicationExpression(new ConstantExpression("op:softmax"), Arrays.asList(new ApplicationExpression(new ConstantExpression("op:matvecmul"), Arrays.asList(new ConstantExpression("weights:softmax"), parsedExpression))));

{'fraction_non_alphanumeric': 0.12179306456160136, 'fraction_numerical': 0.004134949722770416, 'mean_word_length': 4.50258531540848, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7056245', 'n_tokens_mistral': 3137, 'n_tokens_neox': 3066, 'n_words': 541}