<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package org.almibe.ligature.parsers

import javafx.animation.Animation
import javafx.animation.KeyFrame
import javafx.animation.Timeline
import javafx.application.Application
import javafx.application.Platform
import javafx.embed.swing.SwingNode
import javafx.event.ActionEvent
import javafx.geometry.Orientation
import javafx.scene.Scene
import javafx.scene.control.*
import javafx.scene.layout.BorderPane
import javafx.scene.layout.HBox
import javafx.stage.Stage
import javafx.util.Duration
import org.almibe.ligature.parser.NTriplesLexer
import org.almibe.ligature.parser.NTriplesParser
import org.almibe.ligature.parser.TurtleLexer
import org.almibe.ligature.parser.TurtleParser
import org.antlr.v4.gui.TreeViewer
import org.antlr.v4.runtime.CharStreams
import org.antlr.v4.runtime.CommonTokenStream

import javax.swing.*

class ParseTreeDebugView extends Application {
    final String defaultDocument = "<http://one.example/subject1> <http://one.example/predicate1> <http://one.example/object1> ."
    Stage stage
    Scene scene
    final BorderPane topControls = new BorderPane()
    final TextArea textArea = new TextArea(defaultDocument)
    final SwingNode swingNode = new SwingNode()
    final ScrollPane bottomScrollPane = new ScrollPane(swingNode)
    final SplitPane splitPane = new SplitPane(topControls, bottomScrollPane)
    final ToggleGroup radioGroup = new ToggleGroup()
    final RadioButton ntriplesButton = new RadioButton("N-Triples")
    final RadioButton turtleButton = new RadioButton("Turtle")
    final HBox buttonBox = new HBox(ntriplesButton, turtleButton)

    @Override
    void start(Stage primaryStage) throws Exception {
        this.stage = stage
        topControls.center = textArea
        topControls.bottom = buttonBox
        ntriplesButton.toggleGroup = radioGroup
        turtleButton.toggleGroup = radioGroup
        ntriplesButton.selected = true
        splitPane.orientation = Orientation.VERTICAL
        splitPane.setDividerPositions(0.2, 0.8)
        scene = new Scene(splitPane, 1200, 800)
        primaryStage.title = "Ligature Parser Debugger"
        primaryStage.scene = scene
        primaryStage.show()
        swingNode.content = createNTriplesTreeView(defaultDocument)

        Timeline timeline = new Timeline(new KeyFrame(
                Duration.seconds(1.0),
                { ActionEvent event ->
                    checkDisplay()
                }
        ))
        timeline.cycleCount = Animation.INDEFINITE
        timeline.play()
    }

    void checkDisplay() {
        Platform.runLater {
            if (ntriplesButton.selected) {
                updateSwingNode(createNTriplesTreeView(textArea.text))
            } else {
                updateSwingNode(createTurtleTreeView(textArea.text))
            }
        }
    }

    void updateSwingNode(JComponent component) {
        SwingUtilities.invokeLater {
            swingNode.content = component
        }
    }

    JComponent createTurtleTreeView(String text) {
        def stream = CharStreams.fromString(text)
        def lexer = new TurtleLexer(stream)
        def tokenStream = new CommonTokenStream(lexer)
        def parser = new TurtleParser(tokenStream)
        def tree = parser.turtleDoc()
        return new TreeViewer(parser.ruleNames.toList(), tree)
    }

    JComponent createNTriplesTreeView(String text) {
        def stream = CharStreams.fromString(text)
        def lexer = new NTriplesLexer(stream)
        def tokenStream = new CommonTokenStream(lexer)
        def parser = new NTriplesParser(tokenStream)
        def tree = parser.ntriplesDoc()
        return new TreeViewer(parser.ruleNames.toList(), tree)
    }

    static void main(String[] args) {
        launch(ParseTreeDebugView.class, args)
    }
}<commit_msg>closing #20<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package org.almibe.ligature.parsers

import javafx.animation.Animation
import javafx.animation.KeyFrame
import javafx.animation.Timeline
import javafx.application.Application
import javafx.application.Platform
import javafx.embed.swing.SwingNode
import javafx.event.ActionEvent
import javafx.geometry.Orientation
import javafx.scene.Scene
import javafx.scene.control.*
import javafx.scene.layout.BorderPane
import javafx.scene.layout.HBox
import javafx.stage.Stage
import javafx.util.Duration
import org.almibe.ligature.parser.NTriplesLexer
import org.almibe.ligature.parser.NTriplesParser
import org.almibe.ligature.parser.TurtleLexer
import org.almibe.ligature.parser.TurtleParser
import org.antlr.v4.gui.TreeViewer
import org.antlr.v4.runtime.CharStreams
import org.antlr.v4.runtime.CommonTokenStream

import javax.swing.*

class ParseTreeDebugView extends Application {
    final String defaultDocument = "<http://one.example/subject1> <http://one.example/predicate1> <http://one.example/object1> ."
    Stage stage
    Scene scene
    final BorderPane topControls = new BorderPane()
    final TextArea textArea = new TextArea(defaultDocument)
    final SwingNode swingNode = new SwingNode()
    final ScrollPane bottomScrollPane = new ScrollPane(swingNode)
    final SplitPane splitPane = new SplitPane(topControls, bottomScrollPane)
    final ToggleGroup radioGroup = new ToggleGroup()
    final RadioButton ntriplesButton = new RadioButton("N-Triples")
    final RadioButton turtleButton = new RadioButton("Turtle")
    final HBox buttonBox = new HBox(ntriplesButton, turtleButton)

    @Override
    void start(Stage primaryStage) throws Exception {
        this.stage = stage
        topControls.center = textArea
        topControls.bottom = buttonBox
        ntriplesButton.toggleGroup = radioGroup
        turtleButton.toggleGroup = radioGroup
        ntriplesButton.selected = true
        splitPane.orientation = Orientation.VERTICAL
        splitPane.setDividerPositions(0.2, 0.8)
        scene = new Scene(splitPane, 1200, 800)
        primaryStage.title = "Ligature Parser Debugger"
        primaryStage.scene = scene
        primaryStage.show()
        checkDisplay()

        Timeline timeline = new Timeline(new KeyFrame(
                Duration.seconds(1.0),
                { ActionEvent event ->
                    checkDisplay()
                }
        ))
        timeline.cycleCount = Animation.INDEFINITE
        timeline.play()
    }

    void checkDisplay() {
        Platform.runLater {
            TreeViewer component = ntriplesButton.selected ? createNTriplesTreeView(textArea.text) :
                createTurtleTreeView(textArea.text)
            updateSwingNode(component)
        }
    }

    void updateSwingNode(TreeViewer component) {
        SwingUtilities.invokeLater {
            component.scale = 1.5
            swingNode.content = component
        }
    }

    TreeViewer createTurtleTreeView(String text) {
        def stream = CharStreams.fromString(text)
        def lexer = new TurtleLexer(stream)
        def tokenStream = new CommonTokenStream(lexer)
        def parser = new TurtleParser(tokenStream)
        def tree = parser.turtleDoc()
        return new TreeViewer(parser.ruleNames.toList(), tree)
    }

    TreeViewer createNTriplesTreeView(String text) {
        def stream = CharStreams.fromString(text)
        def lexer = new NTriplesLexer(stream)
        def tokenStream = new CommonTokenStream(lexer)
        def parser = new NTriplesParser(tokenStream)
        def tree = parser.ntriplesDoc()
        return new TreeViewer(parser.ruleNames.toList(), tree)
    }

    static void main(String[] args) {
        launch(ParseTreeDebugView.class, args)
    }
}
{'fraction_non_alphanumeric': 0.07912945024531387, 'fraction_numerical': 0.00629009938357026, 'mean_word_length': 3.0727459016393444, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5397049', 'n_tokens_mistral': 2335, 'n_tokens_neox': 2201, 'n_words': 523}