<commit_before>package som.langserv;

import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;

import com.oracle.truffle.api.source.Source;
import com.oracle.truffle.api.source.SourceSection;
import com.oracle.truffle.api.vm.PolyglotEngine;
import com.oracle.truffle.api.vm.PolyglotEngine.Builder;

import io.typefox.lsapi.CompletionItemImpl;
import io.typefox.lsapi.CompletionList;
import io.typefox.lsapi.CompletionListImpl;
import io.typefox.lsapi.Diagnostic;
import io.typefox.lsapi.DiagnosticImpl;
import io.typefox.lsapi.DocumentHighlight;
import io.typefox.lsapi.Location;
import io.typefox.lsapi.LocationImpl;
import io.typefox.lsapi.MessageParams;
import io.typefox.lsapi.MessageParamsImpl;
import io.typefox.lsapi.PositionImpl;
import io.typefox.lsapi.RangeImpl;
import io.typefox.lsapi.SymbolInformation;
import io.typefox.lsapi.SymbolInformationImpl;
import som.VM;
import som.VMOptions;
import som.compiler.MixinBuilder.MixinDefinitionError;
import som.compiler.MixinDefinition;
import som.compiler.MixinDefinition.SlotDefinition;
import som.compiler.Parser.ParseError;
import som.compiler.SourcecodeCompiler;
import som.interpreter.SomLanguage;
import som.interpreter.nodes.ExpressionNode;
import som.interpreter.nodes.MessageSendNode.AbstractUninitializedMessageSendNode;
import som.interpreter.nodes.dispatch.Dispatchable;
import som.vmobjects.SInvokable;
import som.vmobjects.SSymbol;
import tools.SourceCoordinate;
import tools.language.StructuralProbe;

public class SomAdapter {

  private final SomWindow window;

  private final Map<String, SomStructures> structuralProbes = new HashMap<>();
  private final SomCompiler compiler = new SomCompiler();

  public SomAdapter(final SomWindow window) {
    this.window = window;
    initializePolyglot();
  }

  private void initializePolyglot() {
    String coreLib = System.getProperty("som.langserv.core-lib");
    if (coreLib == null) {
      throw new IllegalArgumentException("The som.langserv.core-lib system property needs to be set. For instance: -Dsom.langserv.core-lib=/SOMns/core-lib");
    }

    String[] args = new String[] {"--kernel", coreLib + "/Kernel.som",
                                  "--platform", coreLib + "/Platform.som"};
    Builder builder = PolyglotEngine.newBuilder();
    builder.config(SomLanguage.MIME_TYPE, SomLanguage.CMD_ARGS, args);
    builder.config(SomLanguage.MIME_TYPE, SomLanguage.AVOID_EXIT, true);
    VMOptions vmOptions = new VMOptions(args);
    PolyglotEngine engine = builder.build();
    VM.setEngine(engine);
    engine.getInstruments().values().forEach(i -> i.setEnabled(false));

    // Trigger object system initialization
    engine.getLanguages().get(SomLanguage.MIME_TYPE).getGlobalObject();
  }

  private SomStructures getProbe(final String documentUri) {
    synchronized (structuralProbes) {
      return structuralProbes.get(documentUri);
    }
  }

  public ArrayList<DiagnosticImpl> parse(final String text, final String sourceUri)
      throws URISyntaxException {
    URI uri = new URI(sourceUri);
    Source source = Source.newBuilder(text).
        name(uri.getPath()).
        mimeType(SomLanguage.MIME_TYPE).
        uri(uri).build();

    try {
      // clean out old structural data
      SomStructures newProbe = new SomStructures(source);
      synchronized (structuralProbes) {
        structuralProbes.put(sourceUri, newProbe);
      }
      synchronized (newProbe) {
        compiler.compileModule(source, newProbe);
      }
    } catch (ParseError e) {
      return toDiagnostics(e);
    } catch (MixinDefinitionError e) {
      // TODO
    }
    return new ArrayList<>();
  }

  private ArrayList<DiagnosticImpl> toDiagnostics(final ParseError e) {
    ArrayList<DiagnosticImpl> diagnostics = new ArrayList<>();

    DiagnosticImpl d = new DiagnosticImpl();
    d.setSeverity(Diagnostic.SEVERITY_ERROR);

    SourceCoordinate coord = e.getSourceCoordinate();

    RangeImpl r = new RangeImpl();
    r.setStart(pos(coord.startLine, coord.startColumn));
    r.setEnd(pos(coord.startLine, coord.startColumn + 2)); // TODO: better upper limit /IntMax??
    d.setRange(r);
    d.setMessage(e.getMessage());
    d.setSource("Parser");

    diagnostics.add(d);
    return diagnostics;
  }

  private static PositionImpl pos(final int startLine, final int startChar) {
    PositionImpl pos = new PositionImpl();
    pos.setLine(startLine - 1);
    pos.setCharacter(startChar - 1);
    return pos;
  }
<commit_msg>Indicate parser errors for full remaining line<commit_after>package som.langserv;

import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;

import com.oracle.truffle.api.source.Source;
import com.oracle.truffle.api.source.SourceSection;
import com.oracle.truffle.api.vm.PolyglotEngine;
import com.oracle.truffle.api.vm.PolyglotEngine.Builder;

import io.typefox.lsapi.CompletionItemImpl;
import io.typefox.lsapi.CompletionList;
import io.typefox.lsapi.CompletionListImpl;
import io.typefox.lsapi.Diagnostic;
import io.typefox.lsapi.DiagnosticImpl;
import io.typefox.lsapi.DocumentHighlight;
import io.typefox.lsapi.Location;
import io.typefox.lsapi.LocationImpl;
import io.typefox.lsapi.MessageParams;
import io.typefox.lsapi.MessageParamsImpl;
import io.typefox.lsapi.PositionImpl;
import io.typefox.lsapi.RangeImpl;
import io.typefox.lsapi.SymbolInformation;
import io.typefox.lsapi.SymbolInformationImpl;
import som.VM;
import som.VMOptions;
import som.compiler.MixinBuilder.MixinDefinitionError;
import som.compiler.MixinDefinition;
import som.compiler.MixinDefinition.SlotDefinition;
import som.compiler.Parser.ParseError;
import som.compiler.SourcecodeCompiler;
import som.interpreter.SomLanguage;
import som.interpreter.nodes.ExpressionNode;
import som.interpreter.nodes.MessageSendNode.AbstractUninitializedMessageSendNode;
import som.interpreter.nodes.dispatch.Dispatchable;
import som.vmobjects.SInvokable;
import som.vmobjects.SSymbol;
import tools.SourceCoordinate;
import tools.language.StructuralProbe;

public class SomAdapter {

  private final SomWindow window;

  private final Map<String, SomStructures> structuralProbes = new HashMap<>();
  private final SomCompiler compiler = new SomCompiler();

  public SomAdapter(final SomWindow window) {
    this.window = window;
    initializePolyglot();
  }

  private void initializePolyglot() {
    String coreLib = System.getProperty("som.langserv.core-lib");
    if (coreLib == null) {
      throw new IllegalArgumentException("The som.langserv.core-lib system property needs to be set. For instance: -Dsom.langserv.core-lib=/SOMns/core-lib");
    }

    String[] args = new String[] {"--kernel", coreLib + "/Kernel.som",
                                  "--platform", coreLib + "/Platform.som"};
    Builder builder = PolyglotEngine.newBuilder();
    builder.config(SomLanguage.MIME_TYPE, SomLanguage.CMD_ARGS, args);
    builder.config(SomLanguage.MIME_TYPE, SomLanguage.AVOID_EXIT, true);
    VMOptions vmOptions = new VMOptions(args);
    PolyglotEngine engine = builder.build();
    VM.setEngine(engine);
    engine.getInstruments().values().forEach(i -> i.setEnabled(false));

    // Trigger object system initialization
    engine.getLanguages().get(SomLanguage.MIME_TYPE).getGlobalObject();
  }

  private SomStructures getProbe(final String documentUri) {
    synchronized (structuralProbes) {
      return structuralProbes.get(documentUri);
    }
  }

  public ArrayList<DiagnosticImpl> parse(final String text, final String sourceUri)
      throws URISyntaxException {
    URI uri = new URI(sourceUri);
    Source source = Source.newBuilder(text).
        name(uri.getPath()).
        mimeType(SomLanguage.MIME_TYPE).
        uri(uri).build();

    try {
      // clean out old structural data
      SomStructures newProbe = new SomStructures(source);
      synchronized (structuralProbes) {
        structuralProbes.put(sourceUri, newProbe);
      }
      synchronized (newProbe) {
        compiler.compileModule(source, newProbe);
      }
    } catch (ParseError e) {
      return toDiagnostics(e);
    } catch (MixinDefinitionError e) {
      // TODO
    }
    return new ArrayList<>();
  }

  private ArrayList<DiagnosticImpl> toDiagnostics(final ParseError e) {
    ArrayList<DiagnosticImpl> diagnostics = new ArrayList<>();

    DiagnosticImpl d = new DiagnosticImpl();
    d.setSeverity(Diagnostic.SEVERITY_ERROR);

    SourceCoordinate coord = e.getSourceCoordinate();

    RangeImpl r = new RangeImpl();
    r.setStart(pos(coord.startLine, coord.startColumn));
    r.setEnd(pos(coord.startLine, Integer.MAX_VALUE));
    d.setRange(r);
    d.setMessage(e.getMessage());
    d.setSource("Parser");

    diagnostics.add(d);
    return diagnostics;
  }

  private static PositionImpl pos(final int startLine, final int startChar) {
    PositionImpl pos = new PositionImpl();
    pos.setLine(startLine - 1);
    pos.setCharacter(startChar - 1);
    return pos;
  }

{'fraction_non_alphanumeric': 0.10812288181917569, 'fraction_numerical': 0.0005466273095003826, 'mean_word_length': 4.239404352806415, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7235329', 'n_tokens_mistral': 2746, 'n_tokens_neox': 2736, 'n_words': 560}