<commit_before>/**
 * *****************************************************************************
 * Copyright (c) 2016
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *     http://www.apache.org/licenses/LICENSE-2.0
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 ******************************************************************************
 */
package com.exalttech.trex.ui.controllers;

import com.exalttech.trex.ui.models.datastore.Connection;
import com.exalttech.trex.ui.util.TrexAlertBuilder;
import javafx.beans.property.BooleanProperty;
import javafx.beans.property.SimpleBooleanProperty;
import javafx.beans.value.ObservableValue;
import javafx.event.ActionEvent;
import javafx.fxml.FXML;
import javafx.fxml.Initializable;
import javafx.scene.Node;
import javafx.scene.control.*;
import javafx.scene.image.Image;
import javafx.scene.image.ImageView;
import javafx.scene.input.MouseEvent;
import javafx.scene.layout.AnchorPane;
import javafx.stage.Stage;

import org.apache.commons.io.FileUtils;
import org.apache.log4j.Logger;

import com.google.common.base.Strings;
import com.google.common.eventbus.EventBus;
import com.google.inject.Inject;

import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.util.*;

import com.fasterxml.jackson.databind.ObjectMapper;

import com.xored.javafx.packeteditor.controllers.FieldEditorController;
import com.xored.javafx.packeteditor.events.ScapyClientNeedConnectEvent;

import com.exalttech.trex.core.ConnectionManager;
import com.exalttech.trex.remote.models.profiles.Packet;
import com.exalttech.trex.remote.models.profiles.Profile;
import com.exalttech.trex.remote.models.profiles.Stream;
import com.exalttech.trex.ui.StreamBuilderType;
import com.exalttech.trex.ui.dialog.DialogView;
import com.exalttech.trex.ui.models.PacketInfo;
import com.exalttech.trex.ui.views.streams.binders.BuilderDataBinding;
import com.exalttech.trex.ui.views.streams.builder.PacketBuilderHelper;
import com.exalttech.trex.ui.views.streams.viewer.PacketHex;
import com.exalttech.trex.ui.views.streams.viewer.PacketParser;
import com.exalttech.trex.util.TrafficProfile;
import com.exalttech.trex.util.Util;


public class PacketBuilderHomeController extends DialogView implements Initializable {
    private static final Logger LOG = Logger.getLogger(PacketBuilderHomeController.class.getName());

    private enum SelectedEditMode {
        Cancel,
        Advanced,
        Simple
    }

    @FXML
    private AnchorPane hexPane;
    @FXML
    private Button nextStreamBtn;
    @FXML
    private Button streamEditorModeBtn;
    @FXML
    private Button prevStreamBtn;
    @FXML
    private Button saveButton;
    @FXML
    private StreamPropertiesViewController streamPropertiesController;
    @FXML
    private PacketViewerController packetViewerController;
    @FXML
    private ProtocolSelectionController protocolSelectionController;
    @FXML
    private ProtocolDataController protocolDataController;
    @FXML
    private Tab packetViewerTab;
    @FXML
    private Tab packetEditorTab;
    @FXML
    private Tab fieldEngineTab;
    @FXML
    private Tab packetViewerWithTreeTab;
    @FXML
    private Tab protocolSelectionTab;
    @FXML
    private Tab protocolDataTab;
    @FXML
    private Tab advanceSettingsTab;
    @FXML
    private AdvancedSettingsController advancedSettingsController;
    @FXML
    private Tab streamPropertiesTab;
    @FXML
    private TabPane streamTabPane;

    @Inject
    private FieldEditorController packetBuilderController;
    @Inject
    private EventBus eventBus;

    private PacketInfo packetInfo = null;
    private PacketParser parser;
    private PacketHex packetHex;
    private Profile selectedProfile;
    private boolean isBuildPacket = false;
    private List<Profile> profileList;
    private String yamlFileName;
    private int currentSelectedProfileIndex;
    private BuilderDataBinding builderDataBinder;
    private TrafficProfile trafficProfile;
    private BooleanProperty isImportedStreamProperty = new SimpleBooleanProperty(false);

    @Override
    public void initialize(URL url, ResourceBundle rb) {
        trafficProfile = new TrafficProfile();
        packetHex = new PacketHex(hexPane);
        nextStreamBtn.setGraphic(new ImageView(new Image("/icons/next_stream.png")));
        prevStreamBtn.setGraphic(new ImageView(new Image("/icons/prev_stream.png")));
        packetInfo = new PacketInfo();
        parser = new PacketParser();
    }

    public boolean initStreamBuilder(String pcapFileBinary, List<Profile> profileList, int selectedProfileIndex, String yamlFileName, StreamBuilderType type) throws Exception {
        selectedProfile = profileList.get(selectedProfileIndex);
        this.profileList = profileList;
        this.yamlFileName = yamlFileName;
        currentSelectedProfileIndex = selectedProfileIndex;

        if (selectedProfile.getStream().getAdvancedMode() && !ConnectionManager.getInstance().isScapyConnected()) {
            SelectedEditMode action = prepareToAdvancedMode();
            if (action == SelectedEditMode.Cancel) {
                return false;
            } else {
                selectedProfile.getStream().setAdvancedMode(action == SelectedEditMode.Advanced);
            }
        }

        packetBuilderController.reset();
        streamPropertiesController.init(profileList, selectedProfileIndex);
        updateNextPrevButtonState();

        switch (type) {
            case BUILD_STREAM:
                initStreamBuilder(new BuilderDataBinding());
                break;
            case EDIT_STREAM:
                initEditStream(pcapFileBinary);
                break;
            default:
                break;
        }

        if (selectedProfile.getStream().getAdvancedMode()) {
            showAdvancedModeTabs();
        } else {
            showSimpleModeTabs();
        }

        return true;
    }

    private SelectedEditMode prepareToAdvancedMode() {
        String warningText = "There is no connection to Scapy server\n" +
                "Please, refer to documentation about\n" +
                "Scapy server and advanced mode.";

        ButtonType tryConnect = new ButtonType("Connect", ButtonBar.ButtonData.YES);
        ButtonType continueSimple = new ButtonType("Simple Mode");

        while (true) {
            Optional<ButtonType> userSelection = TrexAlertBuilder.build()
                    .setType(Alert.AlertType.WARNING)
                    .setHeader("Scapy server connection required")
                    .setContent(warningText)
                    .setButtons(tryConnect, continueSimple, ButtonType.CANCEL)
                    .getAlert()
                    .showAndWait();

            if (!userSelection.isPresent() || userSelection.get().equals(ButtonType.CANCEL)) {
                return SelectedEditMode.Cancel;
            } else if (userSelection.get().equals(continueSimple)) {
                return SelectedEditMode.Simple;
            }

            eventBus.post(new ScapyClientNeedConnectEvent());

            if (ConnectionManager.getInstance().isScapyConnected()) {
                return SelectedEditMode.Advanced;
            }

            TrexAlertBuilder.build()
                    .setType(Alert.AlertType.ERROR)
                    .setContent("Connection to Scapy server failed.")
                    .getAlert()
                    .showAndWait();
        }
    }

    private void initEditStream(String pcapFileBinary) throws IOException {
        streamTabPane.setDisable(false);
        saveButton.setDisable(false);
        streamEditorModeBtn.setDisable(false);
        Stream currentStream = selectedProfile.getStream();
        streamEditorModeBtn.setText(currentStream.getAdvancedMode() ? "Simple mode" : "Advanced mode");
        if (!Util.isNullOrEmpty(currentStream.getPacket().getMeta())) {
            BuilderDataBinding dataBinding = getDataBinding();
            if (dataBinding != null) {
                initStreamBuilder(dataBinding);
                return;
            } else {
                streamTabPane.setDisable(true);
                saveButton.setDisable(true);
                streamEditorModeBtn.setDisable(true);
            }
        } else {
            isImportedStreamProperty.set(true);
        }

        if (isImportedStreamProperty.get()) {
            streamTabPane.getTabs().remove(protocolDataTab);
            streamTabPane.getTabs().remove(protocolSelectionTab);
            streamTabPane.getTabs().remove(advanceSettingsTab);
            streamTabPane.getTabs().remove(packetViewerWithTreeTab);
            streamTabPane.getTabs().remove(packetEditorTab);
            streamTabPane.getTabs().remove(fieldEngineTab);
        }
        
        if (pcapFileBinary != null) {
            try {
                isBuildPacket = false;
                File pcapFile = trafficProfile.decodePcapBinary(pcapFileBinary);
                parser.parseFile(pcapFile.getAbsolutePath(), packetInfo);
                packetHex.setData(packetInfo);
            } catch (IOException ex) {
                LOG.error("Failed to load PCAP value", ex);
            }
        }
        String base64UserModel = currentStream.getPacket().getModel();
        if (!Strings.isNullOrEmpty(base64UserModel)) {
            packetBuilderController.loadUserModel(base64UserModel);
        } else {
            byte[] base64Packet = currentStream.getPacket().getBinary().getBytes();
            byte[] packet = Base64.getDecoder().decode(base64Packet);
            packetBuilderController.loadPcapBinary(packet);
        }
    }
    
    private BuilderDataBinding getDataBinding() {
        String meta = selectedProfile.getStream().getPacket().getMeta();
        boolean emptyMeta = meta == null; 
        isImportedStreamProperty.setValue(emptyMeta);
        if (emptyMeta) {
            return null;
        }
        final String metaJSON = new String(Base64.getDecoder().decode(meta));
        if (metaJSON.charAt(0) != '{') {
            TrexAlertBuilder.build()
                    .setType(Alert.AlertType.ERROR)
                    .setTitle("Warning")
                    .setHeader("Stream initialization failed")
                    .setContent("This stream couldn't be edited due to outdated data format.")
                    .getAlert()
                    .showAndWait();
            return null;
        }
        try {
            return new ObjectMapper().readValue(metaJSON, BuilderDataBinding.class);
        } catch (Exception exc) {
            LOG.error("Can't read packet meta", exc);
            TrexAlertBuilder.build()
                    .setType(Alert.AlertType.ERROR)
                    .setTitle("Warning")
                    .setHeader("Stream initialization failed")
                    .setContent(exc.getMessage())
                    .getAlert()
                    .showAndWait();
            return null;
        }
    }

    private void initStreamBuilder(BuilderDataBinding builderDataBinder) {
        isImportedStreamProperty.setValue(false);
        isBuildPacket = true;
        String packetEditorModel = selectedProfile.getStream().getPacket().getModel();
        if (!Strings.isNullOrEmpty(packetEditorModel)) {
            packetBuilderController.loadUserModel(packetEditorModel);
        }
        this.builderDataBinder = builderDataBinder;
        // initialize builder tabs
        protocolSelectionController.bindSelections(builderDataBinder.getProtocolSelection());
        protocolDataController.bindSelection(builderDataBinder);
        advancedSettingsController.bindSelections(builderDataBinder.getAdvancedPropertiesDB());

        packetViewerWithTreeTab.selectedProperty().addListener((ObservableValue<? extends Boolean> observable, Boolean oldValue, Boolean newValue) -> {
            if (newValue) {
                try {
                    // handle it and send the result to packet viewer controller
                    packetViewerController.setPacketView(protocolDataController.getProtocolData());
                } catch (Exception ex) {
                    LOG.error("Error creating packet", ex);
                }
            }
        });
    }

    private void showSimpleModeTabs() {
        streamTabPane.getTabs().clear();
        if (isImportedStreamProperty.get()) {
            streamTabPane.getTabs().addAll(
                    streamPropertiesTab,
                    packetViewerTab
            );
        } else {
            streamTabPane.getTabs().addAll(
                    streamPropertiesTab,
                    protocolSelectionTab,
                    protocolDataTab,
                    advanceSettingsTab,
                    packetViewerWithTreeTab<commit_msg>Added defaul simpleModel on creating new stream<commit_after>/**
 * *****************************************************************************
 * Copyright (c) 2016
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *     http://www.apache.org/licenses/LICENSE-2.0
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 ******************************************************************************
 */
package com.exalttech.trex.ui.controllers;

import com.exalttech.trex.ui.models.datastore.Connection;
import com.exalttech.trex.ui.util.TrexAlertBuilder;
import javafx.beans.property.BooleanProperty;
import javafx.beans.property.SimpleBooleanProperty;
import javafx.beans.value.ObservableValue;
import javafx.event.ActionEvent;
import javafx.fxml.FXML;
import javafx.fxml.Initializable;
import javafx.scene.Node;
import javafx.scene.control.*;
import javafx.scene.image.Image;
import javafx.scene.image.ImageView;
import javafx.scene.input.MouseEvent;
import javafx.scene.layout.AnchorPane;
import javafx.stage.Stage;

import org.apache.commons.io.FileUtils;
import org.apache.log4j.Logger;

import com.google.common.base.Strings;
import com.google.common.eventbus.EventBus;
import com.google.inject.Inject;

import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.util.*;

import com.fasterxml.jackson.databind.ObjectMapper;

import com.xored.javafx.packeteditor.controllers.FieldEditorController;
import com.xored.javafx.packeteditor.events.ScapyClientNeedConnectEvent;

import com.exalttech.trex.core.ConnectionManager;
import com.exalttech.trex.remote.models.profiles.Packet;
import com.exalttech.trex.remote.models.profiles.Profile;
import com.exalttech.trex.remote.models.profiles.Stream;
import com.exalttech.trex.ui.StreamBuilderType;
import com.exalttech.trex.ui.dialog.DialogView;
import com.exalttech.trex.ui.models.PacketInfo;
import com.exalttech.trex.ui.views.streams.binders.BuilderDataBinding;
import com.exalttech.trex.ui.views.streams.builder.PacketBuilderHelper;
import com.exalttech.trex.ui.views.streams.viewer.PacketHex;
import com.exalttech.trex.ui.views.streams.viewer.PacketParser;
import com.exalttech.trex.util.TrafficProfile;
import com.exalttech.trex.util.Util;


public class PacketBuilderHomeController extends DialogView implements Initializable {
    private static final Logger LOG = Logger.getLogger(PacketBuilderHomeController.class.getName());

    private enum SelectedEditMode {
        Cancel,
        Advanced,
        Simple
    }

    @FXML
    private AnchorPane hexPane;
    @FXML
    private Button nextStreamBtn;
    @FXML
    private Button streamEditorModeBtn;
    @FXML
    private Button prevStreamBtn;
    @FXML
    private Button saveButton;
    @FXML
    private StreamPropertiesViewController streamPropertiesController;
    @FXML
    private PacketViewerController packetViewerController;
    @FXML
    private ProtocolSelectionController protocolSelectionController;
    @FXML
    private ProtocolDataController protocolDataController;
    @FXML
    private Tab packetViewerTab;
    @FXML
    private Tab packetEditorTab;
    @FXML
    private Tab fieldEngineTab;
    @FXML
    private Tab packetViewerWithTreeTab;
    @FXML
    private Tab protocolSelectionTab;
    @FXML
    private Tab protocolDataTab;
    @FXML
    private Tab advanceSettingsTab;
    @FXML
    private AdvancedSettingsController advancedSettingsController;
    @FXML
    private Tab streamPropertiesTab;
    @FXML
    private TabPane streamTabPane;

    @Inject
    private FieldEditorController packetBuilderController;
    @Inject
    private EventBus eventBus;

    private PacketInfo packetInfo = null;
    private PacketParser parser;
    private PacketHex packetHex;
    private Profile selectedProfile;
    private boolean isBuildPacket = false;
    private List<Profile> profileList;
    private String yamlFileName;
    private int currentSelectedProfileIndex;
    private BuilderDataBinding builderDataBinder;
    private TrafficProfile trafficProfile;
    private BooleanProperty isImportedStreamProperty = new SimpleBooleanProperty(false);

    @Override
    public void initialize(URL url, ResourceBundle rb) {
        trafficProfile = new TrafficProfile();
        packetHex = new PacketHex(hexPane);
        nextStreamBtn.setGraphic(new ImageView(new Image("/icons/next_stream.png")));
        prevStreamBtn.setGraphic(new ImageView(new Image("/icons/prev_stream.png")));
        packetInfo = new PacketInfo();
        parser = new PacketParser();
    }

    public boolean initStreamBuilder(String pcapFileBinary, List<Profile> profileList, int selectedProfileIndex, String yamlFileName, StreamBuilderType type) throws Exception {
        selectedProfile = profileList.get(selectedProfileIndex);
        this.profileList = profileList;
        this.yamlFileName = yamlFileName;
        currentSelectedProfileIndex = selectedProfileIndex;

        if (selectedProfile.getStream().getAdvancedMode() && !ConnectionManager.getInstance().isScapyConnected()) {
            SelectedEditMode action = prepareToAdvancedMode();
            if (action == SelectedEditMode.Cancel) {
                return false;
            } else {
                selectedProfile.getStream().setAdvancedMode(action == SelectedEditMode.Advanced);
            }
        }

        packetBuilderController.reset();
        streamPropertiesController.init(profileList, selectedProfileIndex);
        updateNextPrevButtonState();

        switch (type) {
            case BUILD_STREAM:
                initStreamBuilder(new BuilderDataBinding());
                break;
            case EDIT_STREAM:
                initEditStream(pcapFileBinary);
                break;
            default:
                break;
        }

        if (selectedProfile.getStream().getAdvancedMode()) {
            showAdvancedModeTabs();
        } else {
            showSimpleModeTabs();
        }

        return true;
    }

    private SelectedEditMode prepareToAdvancedMode() {
        String warningText = "There is no connection to Scapy server\n" +
                "Please, refer to documentation about\n" +
                "Scapy server and advanced mode.";

        ButtonType tryConnect = new ButtonType("Connect", ButtonBar.ButtonData.YES);
        ButtonType continueSimple = new ButtonType("Simple Mode");

        while (true) {
            Optional<ButtonType> userSelection = TrexAlertBuilder.build()
                    .setType(Alert.AlertType.WARNING)
                    .setHeader("Scapy server connection required")
                    .setContent(warningText)
                    .setButtons(tryConnect, continueSimple, ButtonType.CANCEL)
                    .getAlert()
                    .showAndWait();

            if (!userSelection.isPresent() || userSelection.get().equals(ButtonType.CANCEL)) {
                return SelectedEditMode.Cancel;
            } else if (userSelection.get().equals(continueSimple)) {
                return SelectedEditMode.Simple;
            }

            eventBus.post(new ScapyClientNeedConnectEvent());

            if (ConnectionManager.getInstance().isScapyConnected()) {
                return SelectedEditMode.Advanced;
            }

            TrexAlertBuilder.build()
                    .setType(Alert.AlertType.ERROR)
                    .setContent("Connection to Scapy server failed.")
                    .getAlert()
                    .showAndWait();
        }
    }

    private void initEditStream(String pcapFileBinary) throws IOException {
        streamTabPane.setDisable(false);
        saveButton.setDisable(false);
        streamEditorModeBtn.setDisable(false);
        Stream currentStream = selectedProfile.getStream();
        streamEditorModeBtn.setText(currentStream.getAdvancedMode() ? "Simple mode" : "Advanced mode");
        if (!Util.isNullOrEmpty(currentStream.getPacket().getMeta())) {
            BuilderDataBinding dataBinding = getDataBinding();
            if (dataBinding != null) {
                initStreamBuilder(dataBinding);
                return;
            } else {
                streamTabPane.setDisable(true);
                saveButton.setDisable(true);
                streamEditorModeBtn.setDisable(true);
            }
        } else {
            isImportedStreamProperty.set(true);
        }

        if (isImportedStreamProperty.get()) {
            streamTabPane.getTabs().remove(protocolDataTab);
            streamTabPane.getTabs().remove(protocolSelectionTab);
            streamTabPane.getTabs().remove(advanceSettingsTab);
            streamTabPane.getTabs().remove(packetViewerWithTreeTab);
            streamTabPane.getTabs().remove(packetEditorTab);
            streamTabPane.getTabs().remove(fieldEngineTab);
        }
        
        if (pcapFileBinary != null) {
            try {
                isBuildPacket = false;
                File pcapFile = trafficProfile.decodePcapBinary(pcapFileBinary);
                parser.parseFile(pcapFile.getAbsolutePath(), packetInfo);
                packetHex.setData(packetInfo);
            } catch (IOException ex) {
                LOG.error("Failed to load PCAP value", ex);
            }
        }
        String base64UserModel = currentStream.getPacket().getModel();
        if (!Strings.isNullOrEmpty(base64UserModel)) {
            packetBuilderController.loadUserModel(base64UserModel);
        } else {
            byte[] base64Packet = currentStream.getPacket().getBinary().getBytes();
            byte[] packet = Base64.getDecoder().decode(base64Packet);
            packetBuilderController.loadPcapBinary(packet);
        }
    }
    
    private BuilderDataBinding getDataBinding() {
        String meta = selectedProfile.getStream().getPacket().getMeta();
        boolean emptyMeta = meta == null; 
        isImportedStreamProperty.setValue(emptyMeta);
        if (emptyMeta) {
            return null;
        }
        final String metaJSON = new String(Base64.getDecoder().decode(meta));
        if (metaJSON.charAt(0) != '{') {
            TrexAlertBuilder.build()
                    .setType(Alert.AlertType.ERROR)
                    .setTitle("Warning")
                    .setHeader("Stream initialization failed")
                    .setContent("This stream couldn't be edited due to outdated data format.")
                    .getAlert()
                    .showAndWait();
            return null;
        }
        try {
            return new ObjectMapper().readValue(metaJSON, BuilderDataBinding.class);
        } catch (Exception exc) {
            LOG.error("Can't read packet meta", exc);
            TrexAlertBuilder.build()
                    .setType(Alert.AlertType.ERROR)
                    .setTitle("Warning")
                    .setHeader("Stream initialization failed")
                    .setContent(exc.getMessage())
                    .getAlert()
                    .showAndWait();
            return null;
        }
    }

    private void initStreamBuilder(BuilderDataBinding builderDataBinder) {
        isImportedStreamProperty.setValue(false);
        isBuildPacket = true;
        String packetEditorModel = selectedProfile.getStream().getPacket().getModel();
        this.builderDataBinder = builderDataBinder;
        if (!Strings.isNullOrEmpty(packetEditorModel)) {
            packetBuilderController.loadUserModel(packetEditorModel);
        } else {
            packetBuilderController.loadSimpleUserModel(this.builderDataBinder.serializeAsPacketModel());
        }
        // initialize builder tabs
        protocolSelectionController.bindSelections(builderDataBinder.getProtocolSelection());
        protocolDataController.bindSelection(builderDataBinder);
        advancedSettingsController.bindSelections(builderDataBinder.getAdvancedPropertiesDB());

        packetViewerWithTreeTab.selectedProperty().addListener((ObservableValue<? extends Boolean> observable, Boolean oldValue, Boolean newValue) -> {
            if (newValue) {
                try {
                    // handle it and send the result to packet viewer controller
                    packetViewerController.setPacketView(protocolDataController.getProtocolData());
                } catch (Exception ex) {
                    LOG.error("Error creating packet", ex);
                }
            }
        });
    }

    private void showSimpleModeTabs() {
        streamTabPane.getTabs().clear();
        if (isImportedStreamProperty.get()) {
            streamTabPane.getTabs().addAll(
                    streamPropertiesTab,
                    packetViewerTab
            );
        } else {
            streamTabPane.getTabs().addAll(
                    streamPropertiesTab,
                    protocolSelectionTab,
                    protocolDataTab,
                    advanceSettingsTab,
                    packetViewerWithTreeTab
{'fraction_non_alphanumeric': 0.09653137647993343, 'fraction_numerical': 0.0018156371751711616, 'mean_word_length': 2.5823848238482383, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1568995', 'n_tokens_mistral': 6759, 'n_tokens_neox': 6494, 'n_words': 1347}