<commit_before># global
import logging
import copy
import numpy as np

from PyQt5 import QtCore
from PyQt5 import QtGui, QtWidgets

from twisted.internet import threads
from twisted.internet import reactor

# sas-global
from sas.sascalc.invariant import invariant
from sas.qtgui.Plotting.PlotterData import Data1D
import sas.qtgui.Utilities.GuiUtils as GuiUtils

# local
from .UI.TabbedInvariantUI import Ui_tabbedInvariantUI
from .InvariantDetails import DetailsDialog
from .InvariantUtils import WIDGETS

# The minimum q-value to be used when extrapolating
Q_MINIMUM = 1e-5
# The maximum q-value to be used when extrapolating
Q_MAXIMUM = 10
# the ratio of maximum q value/(qmax of data) to plot the theory data
Q_MAXIMUM_PLOT = 3
# Default number of points of interpolation: high and low range
NPOINTS_Q_INTERP = 10
# Default power law for interpolation
DEFAULT_POWER_LOW = 4

# Background of line edits if settings OK or wrong
BG_WHITE = "background-color: rgb(255, 255, 255);"
BG_RED = "background-color: rgb(244, 170, 164);"


class InvariantWindow(QtWidgets.QDialog, Ui_tabbedInvariantUI):
    # The controller which is responsible for managing signal slots connections
    # for the gui and providing an interface to the data model.
    name = "Invariant"  # For displaying in the combo box in DataExplorer
    ext = 'inv'  # File extension used for saving analyses

    def __init__(self, parent=None):
        super(InvariantWindow, self).__init__()
        self.setupUi(self)

        self.setWindowTitle(self.title())

        # initial input params
        self._background = 0.0
        self._scale = 1.0
        self._contrast = 8.0e-6
        self._porod = None

        self.parent = parent

        self._manager = parent
        self._reactor = reactor
        self._model_item = QtGui.QStandardItem()

        self.detailsDialog = DetailsDialog(self)
        self.detailsDialog.cmdOK.clicked.connect(self.enabling)

        self._low_extrapolate = False
        self._low_guinier = True
        self._low_fit = False
        self._low_points = NPOINTS_Q_INTERP
        self._low_power_value = DEFAULT_POWER_LOW

        self._high_extrapolate = False
        self._high_fit = False
        self._high_points = NPOINTS_Q_INTERP
        self._high_power_value = DEFAULT_POWER_LOW

        # Define plots
        self.high_extrapolation_plot = None
        self.low_extrapolation_plot = None

        # no reason to have this widget resizable
        self.resize(self.minimumSizeHint())

        self.communicate = self._manager.communicator()
        self.communicate.dataDeletedSignal.connect(self.removeData)

        self._data = None
        self._path = ""
        self._calculator = None

        self._allow_close = False

        # Modify font in order to display Angstrom symbol correctly
        new_font = 'font-family: -apple-system, "Helvetica Neue", "Ubuntu";'
        self.lblTotalQUnits.setStyleSheet(new_font)
        self.lblSpecificSurfaceUnits.setStyleSheet(new_font)
        self.lblInvariantTotalQUnits.setStyleSheet(new_font)
        self.lblContrastUnits.setStyleSheet(new_font)
        self.lblPorodCstUnits.setStyleSheet(new_font)
        self.lblExtrapolQUnits.setStyleSheet(new_font)

        # To remove blue square around line edits
        self.txtBackgd.setAttribute(QtCore.Qt.WA_MacShowFocusRect, False)
        self.txtContrast.setAttribute(QtCore.Qt.WA_MacShowFocusRect, False)
        self.txtScale.setAttribute(QtCore.Qt.WA_MacShowFocusRect, False)
        self.txtPorodCst.setAttribute(QtCore.Qt.WA_MacShowFocusRect, False)
        self.txtNptsHighQ.setAttribute(QtCore.Qt.WA_MacShowFocusRect, False)
        self.txtNptsLowQ.setAttribute(QtCore.Qt.WA_MacShowFocusRect, False)
        self.txtPowerLowQ.setAttribute(QtCore.Qt.WA_MacShowFocusRect, False)
        self.txtPowerHighQ.setAttribute(QtCore.Qt.WA_MacShowFocusRect, False)

        # Let's choose the Standard Item Model.
        self.model = QtGui.QStandardItemModel(self)

        # Connect buttons to slots.
        # Needs to be done early so default values propagate properly.
        self.setupSlots()

        # Set up the model.
        self.setupModel()

        # Set up the mapper
        self.setupMapper()

        # Default enablement
        self.cmdCalculate.setEnabled(False)

        # validator: double
        self.txtExtrapolQMin.setValidator(GuiUtils.DoubleValidator())
        self.txtExtrapolQMax.setValidator(GuiUtils.DoubleValidator())
        self.txtBackgd.setValidator(GuiUtils.DoubleValidator())
        self.txtContrast.setValidator(GuiUtils.DoubleValidator())
        self.txtScale.setValidator(GuiUtils.DoubleValidator())
        self.txtPorodCst.setValidator(GuiUtils.DoubleValidator())

        # validator: integer number
        self.txtNptsLowQ.setValidator(QtGui.QIntValidator())
        self.txtNptsHighQ.setValidator(QtGui.QIntValidator())
        self.txtPowerLowQ.setValidator(GuiUtils.DoubleValidator())
        self.txtPowerHighQ.setValidator(GuiUtils.DoubleValidator())

        self.mapper.toFirst()

    def get_low_q_extrapolation_upper_limit(self):
        q_value = self._data.x[int(self.txtNptsLowQ.text()) - 1]
        return q_value

    def set_low_q_extrapolation_upper_limit(self, value):
        n_pts = (np.abs(self._data.x - value)).argmin() + 1
        item = QtGui.QStandardItem(str(n_pts))
        self.model.setItem(WIDGETS.W_NPTS_LOWQ, item)

    def get_high_q_extrapolation_lower_limit(self):
        q_value = self._data.x[len(self._data.x) - int(self.txtNptsHighQ.text()) - 1]
        return q_value

    def set_high_q_extrapolation_lower_limit(self, value):
        n_pts = len(self._data.x) - (np.abs(self._data.x - value)).argmin() + 1
        item = QtGui.QStandardItem(str(int(n_pts)))
        self.model.setItem(WIDGETS.W_NPTS_HIGHQ, item)

    def enabling(self):
        """ """
        self.cmdStatus.setEnabled(True)

    def setClosable(self, value=True):
        """ Allow outsiders close this widget """
        assert isinstance(value, bool)

        self._allow_close = value

    def isSerializable(self):
        """
        Tell the caller that this perspective writes its state
        """
        return True

    def closeEvent(self, event):
        """
        Overwrite QDialog close method to allow for custom widget close
        """
        if self._allow_close:
            # reset the closability flag
            self.setClosable(value=False)
            # Tell the MdiArea to close the container if it is visible
            if self.parentWidget():
                self.parentWidget().close()
            event.accept()
        else:
            event.ignore()
            # Maybe we should just minimize
            self.setWindowState(QtCore.Qt.WindowMinimized)

    def updateFromModel(self):
        """ Update the globals based on the data in the model """
        self._background = float(self.model.item(WIDGETS.W_BACKGROUND).text())
        self._contrast = float(self.model.item(WIDGETS.W_CONTRAST).text())
        self._scale = float(self.model.item(WIDGETS.W_SCALE).text())
        if self.model.item(WIDGETS.W_POROD_CST).text() != 'None' and self.model.item(WIDGETS.W_POROD_CST).text() != '':
            self._porod = float(self.model.item(WIDGETS.W_POROD_CST).text())

        # Low extrapolating
        self._low_extrapolate = str(self.model.item(WIDGETS.W_ENABLE_LOWQ).text()) == 'true'
        self._low_points = float(self.model.item(WIDGETS.W_NPTS_LOWQ).text())
        self._low_guinier = str(self.model.item(WIDGETS.W_LOWQ_GUINIER).text()) == 'true'
        self._low_fit = str(self.model.item(WIDGETS.W_LOWQ_FIT).text()) == 'true'
        self._low_power_value = float(self.model.item(WIDGETS.W_LOWQ_POWER_VALUE).text())

        # High extrapolating
        self._high_extrapolate = str(self.model.item(WIDGETS.W_ENABLE_HIGHQ).text()) == 'true'
        self._high_points = float(self.model.item(WIDGETS.W_NPTS_HIGHQ).text())
        self._high_fit = str(self.model.item(WIDGETS.W_HIGHQ_FIT).text()) == 'true'
        self._high_power_value = float(self.model.item(WIDGETS.W_HIGHQ_POWER_VALUE).text())

    def calculateInvariant(self):
        """ Use twisted to thread the calculations away """
        # Find out if extrapolation needs to be used.
        extrapolation = None
        if self._low_extrapolate and self._high_extrapolate:
            extrapolation = "both"
        elif self._high_extrapolate:
            extrapolation = "high"
        elif self._low_extrapolate:
            extrapolation = "low"

        # modify the Calculate button to indicate background process
        self.cmdCalculate.setText("Calculating...")
        self.cmdCalculate.setEnabled(False)

        # Send the calculations to separate thread.
        d = threads.deferToThread(self.calculateThread, extrapolation)

        # Add deferred callback for call return
        d.addCallback(self.deferredPlot)
        d.addErrback(self.calculationFailed)

    def calculationFailed(self, reason):
        print("calculation failed: ", reason)
        self.allow_calculation()

    def deferredPlot(self, model):
        """
        Run the GUI/model update in the main thread
        """
        reactor.callFromThread(lambda: self.plotResult(model))
        self.allow_calculation()

    def allow_calculation(self):
        # Set the calculate button to available
        self.cmdCalculate.setEnabled(True)
        self.cmdCalculate.setText("Calculate")

    def plotResult(self, model):
        """ Plot result of calculation """

        self.model = model
        self._data = GuiUtils.dataFromItem(self._model_item)
        # Send the modified model item to DE for keeping in the model
        plots = [self._model_item]
        if self.high_extrapolation_plot:
            self.high_extrapolation_plot.plot_role = Data1D.ROLE_DEFAULT
            self.high_extrapolation_plot.symbol = "Line"
            self.high_extrapolation_plot.show_errors = False
            self.high_extrapolation_plot.show_q_range_sliders = True
            self.high_extrapolation_plot.slider_update_on_move = False
            self.high_extrapolation_plot.slider_low_q_input = self.txtNptsHighQ
            self.high_extrapolation_plot.slider_low_q_setter = self.set_high_q_extrapolation_lower_limit
            self.high_extrapolation_plot.slider_low_q_getter = self.get_high_q_extrapolation_lower_limit
            self.high_extrapolation_plot.slider_high_q_input = self.txtExtrapolQMax
            GuiUtils.updateModelItemWithPlot(self._model_item, self.high_extrapolation_plot,
                                             self.high_extrapolation_plot.title)
            plots.append(self.high_extrapolation_plot)
        if self.low_extrapolation_plot:
            self.low_extrapolation_plot.plot_role = Data1D.ROLE_DEFAULT
            self.low_extrapolation_plot.symbol = "Line"
            self.low_extrapolation_plot.show_errors = False
            self.low_extrapolation_plot.show_q_range_sliders = True
            self.low_extrapolation_plot.slider_update_on_move = False
            self.low_extrapolation_plot.slider_high_q_input = self.txtNptsLowQ
            self.low_extrapolation_plot.slider_high_q_setter = self.set_low_q_extrapolation_upper_limit
            self.low_extrapolation_plot.slider_high_q_getter = self.get_low_q_extrapolation_upper_limit
            self.low_extrapolation_plot.slider_low_q_input = self.txtExtrapolQMin
            GuiUtils.updateModelItemWithPlot(self._model_item, self.low_extrapolation_plot,
                                             self.low_extrapolation_plot.title)
            plots.append(self.low_extrapolation_plot)
        if len(plots) > 1:
            self.communicate.plotRequestedSignal.emit(plots, None)

        # Update the details dialog in case it is open
        self.updateDetailsWidget(model)

    def updateDetailsWidget(self, model):
        """
        On demand update of the details widget
        """
        if self.detailsDialog.isVisible():
            self.onStatus()

    def calculateThread(self, extrapolation):
        """
        Perform Invariant calculations.<commit_msg>Apply QRangeSlider strings to invariant perspective perspective.<commit_after># global
import logging
import copy
import numpy as np

from PyQt5 import QtCore
from PyQt5 import QtGui, QtWidgets

from twisted.internet import threads
from twisted.internet import reactor

# sas-global
from sas.sascalc.invariant import invariant
from sas.qtgui.Plotting.PlotterData import Data1D
import sas.qtgui.Utilities.GuiUtils as GuiUtils

# local
from .UI.TabbedInvariantUI import Ui_tabbedInvariantUI
from .InvariantDetails import DetailsDialog
from .InvariantUtils import WIDGETS

# The minimum q-value to be used when extrapolating
Q_MINIMUM = 1e-5
# The maximum q-value to be used when extrapolating
Q_MAXIMUM = 10
# the ratio of maximum q value/(qmax of data) to plot the theory data
Q_MAXIMUM_PLOT = 3
# Default number of points of interpolation: high and low range
NPOINTS_Q_INTERP = 10
# Default power law for interpolation
DEFAULT_POWER_LOW = 4

# Background of line edits if settings OK or wrong
BG_WHITE = "background-color: rgb(255, 255, 255);"
BG_RED = "background-color: rgb(244, 170, 164);"


class InvariantWindow(QtWidgets.QDialog, Ui_tabbedInvariantUI):
    # The controller which is responsible for managing signal slots connections
    # for the gui and providing an interface to the data model.
    name = "Invariant"  # For displaying in the combo box in DataExplorer
    ext = 'inv'  # File extension used for saving analyses

    def __init__(self, parent=None):
        super(InvariantWindow, self).__init__()
        self.setupUi(self)

        self.setWindowTitle(self.title())

        # initial input params
        self._background = 0.0
        self._scale = 1.0
        self._contrast = 8.0e-6
        self._porod = None

        self.parent = parent

        self._manager = parent
        self._reactor = reactor
        self._model_item = QtGui.QStandardItem()

        self.detailsDialog = DetailsDialog(self)
        self.detailsDialog.cmdOK.clicked.connect(self.enabling)

        self._low_extrapolate = False
        self._low_guinier = True
        self._low_fit = False
        self._low_points = NPOINTS_Q_INTERP
        self._low_power_value = DEFAULT_POWER_LOW

        self._high_extrapolate = False
        self._high_fit = False
        self._high_points = NPOINTS_Q_INTERP
        self._high_power_value = DEFAULT_POWER_LOW

        # Define plots
        self.high_extrapolation_plot = None
        self.low_extrapolation_plot = None

        # no reason to have this widget resizable
        self.resize(self.minimumSizeHint())

        self.communicate = self._manager.communicator()
        self.communicate.dataDeletedSignal.connect(self.removeData)

        self._data = None
        self._path = ""
        self._calculator = None

        self._allow_close = False

        # Modify font in order to display Angstrom symbol correctly
        new_font = 'font-family: -apple-system, "Helvetica Neue", "Ubuntu";'
        self.lblTotalQUnits.setStyleSheet(new_font)
        self.lblSpecificSurfaceUnits.setStyleSheet(new_font)
        self.lblInvariantTotalQUnits.setStyleSheet(new_font)
        self.lblContrastUnits.setStyleSheet(new_font)
        self.lblPorodCstUnits.setStyleSheet(new_font)
        self.lblExtrapolQUnits.setStyleSheet(new_font)

        # To remove blue square around line edits
        self.txtBackgd.setAttribute(QtCore.Qt.WA_MacShowFocusRect, False)
        self.txtContrast.setAttribute(QtCore.Qt.WA_MacShowFocusRect, False)
        self.txtScale.setAttribute(QtCore.Qt.WA_MacShowFocusRect, False)
        self.txtPorodCst.setAttribute(QtCore.Qt.WA_MacShowFocusRect, False)
        self.txtNptsHighQ.setAttribute(QtCore.Qt.WA_MacShowFocusRect, False)
        self.txtNptsLowQ.setAttribute(QtCore.Qt.WA_MacShowFocusRect, False)
        self.txtPowerLowQ.setAttribute(QtCore.Qt.WA_MacShowFocusRect, False)
        self.txtPowerHighQ.setAttribute(QtCore.Qt.WA_MacShowFocusRect, False)

        # Let's choose the Standard Item Model.
        self.model = QtGui.QStandardItemModel(self)

        # Connect buttons to slots.
        # Needs to be done early so default values propagate properly.
        self.setupSlots()

        # Set up the model.
        self.setupModel()

        # Set up the mapper
        self.setupMapper()

        # Default enablement
        self.cmdCalculate.setEnabled(False)

        # validator: double
        self.txtExtrapolQMin.setValidator(GuiUtils.DoubleValidator())
        self.txtExtrapolQMax.setValidator(GuiUtils.DoubleValidator())
        self.txtBackgd.setValidator(GuiUtils.DoubleValidator())
        self.txtContrast.setValidator(GuiUtils.DoubleValidator())
        self.txtScale.setValidator(GuiUtils.DoubleValidator())
        self.txtPorodCst.setValidator(GuiUtils.DoubleValidator())

        # validator: integer number
        self.txtNptsLowQ.setValidator(QtGui.QIntValidator())
        self.txtNptsHighQ.setValidator(QtGui.QIntValidator())
        self.txtPowerLowQ.setValidator(GuiUtils.DoubleValidator())
        self.txtPowerHighQ.setValidator(GuiUtils.DoubleValidator())

        self.mapper.toFirst()

    def get_low_q_extrapolation_upper_limit(self):
        q_value = self._data.x[int(self.txtNptsLowQ.text()) - 1]
        return q_value

    def set_low_q_extrapolation_upper_limit(self, value):
        n_pts = (np.abs(self._data.x - value)).argmin() + 1
        item = QtGui.QStandardItem(str(n_pts))
        self.model.setItem(WIDGETS.W_NPTS_LOWQ, item)

    def get_high_q_extrapolation_lower_limit(self):
        q_value = self._data.x[len(self._data.x) - int(self.txtNptsHighQ.text()) - 1]
        return q_value

    def set_high_q_extrapolation_lower_limit(self, value):
        n_pts = len(self._data.x) - (np.abs(self._data.x - value)).argmin() + 1
        item = QtGui.QStandardItem(str(int(n_pts)))
        self.model.setItem(WIDGETS.W_NPTS_HIGHQ, item)

    def enabling(self):
        """ """
        self.cmdStatus.setEnabled(True)

    def setClosable(self, value=True):
        """ Allow outsiders close this widget """
        assert isinstance(value, bool)

        self._allow_close = value

    def isSerializable(self):
        """
        Tell the caller that this perspective writes its state
        """
        return True

    def closeEvent(self, event):
        """
        Overwrite QDialog close method to allow for custom widget close
        """
        if self._allow_close:
            # reset the closability flag
            self.setClosable(value=False)
            # Tell the MdiArea to close the container if it is visible
            if self.parentWidget():
                self.parentWidget().close()
            event.accept()
        else:
            event.ignore()
            # Maybe we should just minimize
            self.setWindowState(QtCore.Qt.WindowMinimized)

    def updateFromModel(self):
        """ Update the globals based on the data in the model """
        self._background = float(self.model.item(WIDGETS.W_BACKGROUND).text())
        self._contrast = float(self.model.item(WIDGETS.W_CONTRAST).text())
        self._scale = float(self.model.item(WIDGETS.W_SCALE).text())
        if self.model.item(WIDGETS.W_POROD_CST).text() != 'None' and self.model.item(WIDGETS.W_POROD_CST).text() != '':
            self._porod = float(self.model.item(WIDGETS.W_POROD_CST).text())

        # Low extrapolating
        self._low_extrapolate = str(self.model.item(WIDGETS.W_ENABLE_LOWQ).text()) == 'true'
        self._low_points = float(self.model.item(WIDGETS.W_NPTS_LOWQ).text())
        self._low_guinier = str(self.model.item(WIDGETS.W_LOWQ_GUINIER).text()) == 'true'
        self._low_fit = str(self.model.item(WIDGETS.W_LOWQ_FIT).text()) == 'true'
        self._low_power_value = float(self.model.item(WIDGETS.W_LOWQ_POWER_VALUE).text())

        # High extrapolating
        self._high_extrapolate = str(self.model.item(WIDGETS.W_ENABLE_HIGHQ).text()) == 'true'
        self._high_points = float(self.model.item(WIDGETS.W_NPTS_HIGHQ).text())
        self._high_fit = str(self.model.item(WIDGETS.W_HIGHQ_FIT).text()) == 'true'
        self._high_power_value = float(self.model.item(WIDGETS.W_HIGHQ_POWER_VALUE).text())

    def calculateInvariant(self):
        """ Use twisted to thread the calculations away """
        # Find out if extrapolation needs to be used.
        extrapolation = None
        if self._low_extrapolate and self._high_extrapolate:
            extrapolation = "both"
        elif self._high_extrapolate:
            extrapolation = "high"
        elif self._low_extrapolate:
            extrapolation = "low"

        # modify the Calculate button to indicate background process
        self.cmdCalculate.setText("Calculating...")
        self.cmdCalculate.setEnabled(False)

        # Send the calculations to separate thread.
        d = threads.deferToThread(self.calculateThread, extrapolation)

        # Add deferred callback for call return
        d.addCallback(self.deferredPlot)
        d.addErrback(self.calculationFailed)

    def calculationFailed(self, reason):
        print("calculation failed: ", reason)
        self.allow_calculation()

    def deferredPlot(self, model):
        """
        Run the GUI/model update in the main thread
        """
        reactor.callFromThread(lambda: self.plotResult(model))
        self.allow_calculation()

    def allow_calculation(self):
        # Set the calculate button to available
        self.cmdCalculate.setEnabled(True)
        self.cmdCalculate.setText("Calculate")

    def plotResult(self, model):
        """ Plot result of calculation """

        self.model = model
        self._data = GuiUtils.dataFromItem(self._model_item)
        # Send the modified model item to DE for keeping in the model
        plots = [self._model_item]
        if self.high_extrapolation_plot:
            self.high_extrapolation_plot.plot_role = Data1D.ROLE_DEFAULT
            self.high_extrapolation_plot.symbol = "Line"
            self.high_extrapolation_plot.show_errors = False
            self.high_extrapolation_plot.show_q_range_sliders = True
            self.high_extrapolation_plot.slider_update_on_move = False
            self.high_extrapolation_plot.slider_perspective_name = self.name
            self.high_extrapolation_plot.slider_low_q_input = ['txtNptsHighQ']
            self.high_extrapolation_plot.slider_low_q_setter = ['set_high_q_extrapolation_lower_limit']
            self.high_extrapolation_plot.slider_low_q_getter = ['get_high_q_extrapolation_lower_limit']
            self.high_extrapolation_plot.slider_high_q_input = ['txtExtrapolQMax']
            GuiUtils.updateModelItemWithPlot(self._model_item, self.high_extrapolation_plot,
                                             self.high_extrapolation_plot.title)
            plots.append(self.high_extrapolation_plot)
        if self.low_extrapolation_plot:
            self.low_extrapolation_plot.plot_role = Data1D.ROLE_DEFAULT
            self.low_extrapolation_plot.symbol = "Line"
            self.low_extrapolation_plot.show_errors = False
            self.low_extrapolation_plot.show_q_range_sliders = True
            self.low_extrapolation_plot.slider_update_on_move = False
            self.low_extrapolation_plot.slider_perspective_name = self.name
            self.low_extrapolation_plot.slider_high_q_input = ['txtNptsLowQ']
            self.low_extrapolation_plot.slider_high_q_setter = ['set_low_q_extrapolation_upper_limit']
            self.low_extrapolation_plot.slider_high_q_getter = ['get_low_q_extrapolation_upper_limit']
            self.low_extrapolation_plot.slider_low_q_input = ['txtExtrapolQMin']
            GuiUtils.updateModelItemWithPlot(self._model_item, self.low_extrapolation_plot,
                                             self.low_extrapolation_plot.title)
            plots.append(self.low_extrapolation_plot)
        if len(plots) > 1:
            self.communicate.plotRequestedSignal.emit(plots, None)

        # Update the details dialog in case it is open
        self.updateDetailsWidget(model)

    def updateDetailsWidget(self, model):
        """
        On demand update of the details widget
        """
        if self.detailsDialog.isVisible():
            self.onStatus()

    def calculateThread(self, extrapolation):
        """
        Perform Invariant calculations.
{'fraction_non_alphanumeric': 0.0865047673376253, 'fraction_numerical': 0.0035041968869692773, 'mean_word_length': 2.869914853358562, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4822048', 'n_tokens_mistral': 7578, 'n_tokens_neox': 7386, 'n_words': 1399}