<commit_before>/*
 * Copyright (c) 2011 Stiftung Deutsches Elektronen-Synchrotron,
 * Member of the Helmholtz Association, (DESY), HAMBURG, GERMANY.
 *
 * THIS SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN "../AS IS" BASIS.
 * WITHOUT WARRANTY OF ANY KIND, EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED
 * TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR PARTICULAR PURPOSE AND
 * NON-INFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
 * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
 * THE USE OR OTHER DEALINGS IN THE SOFTWARE. SHOULD THE SOFTWARE PROVE DEFECTIVE
 * IN ANY RESPECT, THE USER ASSUMES THE COST OF ANY NECESSARY SERVICING, REPAIR OR
 * CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE.
 * NO USE OF ANY SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER.
 * DESY HAS NO OBLIGATION TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS,
 * OR MODIFICATIONS.
 * THE FULL LICENSE SPECIFYING FOR THE SOFTWARE THE REDISTRIBUTION, MODIFICATION,
 * USAGE AND OTHER RIGHTS AND OBLIGATIONS IS INCLUDED WITH THE DISTRIBUTION OF THIS
 * PROJECT IN THE FILE LICENSE.HTML. IF THE LICENSE IS NOT INCLUDED YOU MAY FIND A COPY
 * AT HTTP://WWW.DESY.DE/LEGAL/LICENSE.HTM
 */
package org.csstudio.sds;

import static org.junit.Assert.assertEquals;

import java.io.File;
import java.net.URL;
import java.util.ArrayList;

import org.csstudio.dal.CssApplicationContext;
import org.csstudio.domain.desy.junit.ConditionalClassRunner;
import org.csstudio.domain.desy.junit.OsCondition;
import org.csstudio.domain.desy.junit.RunIf;
import org.csstudio.domain.desy.softioc.BasicSoftIocConfigurator;
import org.csstudio.domain.desy.softioc.Caget;
import org.csstudio.domain.desy.softioc.DBR;
import org.csstudio.domain.desy.softioc.ISoftIocConfigurator;
import org.csstudio.domain.desy.softioc.SoftIoc;
import org.csstudio.platform.logging.CentralLogger;
import org.csstudio.platform.model.pvs.IProcessVariableAddress;
import org.csstudio.sds.eventhandling.AbstractBehavior;
import org.csstudio.sds.internal.connection.BehaviorConnector;
import org.csstudio.sds.model.AbstractTextTypeWidgetModel;
import org.csstudio.sds.model.AbstractWidgetModel;
import org.csstudio.sds.model.DisplayModel;
import org.csstudio.sds.model.LabelModel;
import org.csstudio.sds.model.TextTypeEnum;
import org.eclipse.core.runtime.FileLocator;
import org.epics.css.dal.simple.ChannelListener;
import org.epics.css.dal.simple.ConnectionParameters;
import org.epics.css.dal.simple.RemoteInfo;
import org.epics.css.dal.simple.SimpleDALBroker;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;

import com.cosylab.util.CommonException;

/**
 * TODO (hrickens) :
 *
 * @author hrickens
 * @since 25.11.2011
 */
@RunWith(ConditionalClassRunner.class)
@RunIf(conditionClass = OsCondition.class, arguments = { OsCondition.WIN })
public class CAJPrecisionHeadlessTest {

    private static SoftIoc _softIoc;


    @BeforeClass
    public static void startUpSoftIoc() throws Exception {
        final URL dbBundleResourceUrl = CAJPrecisionHeadlessTest.class.getClassLoader()
                .getResource("resources/db/CAJPrecisionTest.db");
        final URL dbFileUrl = FileLocator.toFileURL(dbBundleResourceUrl);

        final ISoftIocConfigurator cfg = new BasicSoftIocConfigurator().with(new File(dbFileUrl
                                                                                      .getFile()));
        _softIoc = new SoftIoc(cfg);
        _softIoc.start();
        while (!_softIoc.isStartUpDone()) {
            // wait IOC startup finished
        }
        setSystemPropertys();
        Thread.sleep(400);
    }


    /**
     * @throws java.lang.Exception
     */
    @Before
    public void setUp() throws Exception {
    }

    @Test(timeout = 5000)
    public void testCAGetPrecision() throws Exception {
//        while (!_softIoc.isStartUpDone()) {
//            // wait IOC startup finished
//        }
        final Caget caget = new Caget();
        caget.setDbr(DBR.DBR_TIME_STRING);
        caget.setWaitTime(3);
        final String recodName = "DALPrecisionTest1";
        final ArrayList<String> result = caget.caget(recodName);
        assertEquals(8, result.size());
        final String[] split = result.get(4).trim().split(":");
        assertEquals(2, split.length);
        final String value = split[1].trim();
        assertEquals("The display Value of " + recodName + " was wrong: ", "12345.6789", value);
    }

    @Test(timeout = 5000)
    public void testSDSLabelDoublePrecision() throws Exception {
        final String recordName = "DALPrecisionTest1";
        final String behaviorId = "behavior.desy.label.connectiondefault";

        final LabelModel labelModel = buildLabelModel(recordName, behaviorId);
        labelModel.setPropertyValue(AbstractTextTypeWidgetModel.PROP_TEXT_TYPE, TextTypeEnum.DOUBLE.getIndex());

        buildConnection(behaviorId, labelModel);

        Thread.sleep(200);
        final String stringValue = labelModel.getStringProperty(LabelModel.PROP_TEXTVALUE);
        assertEquals("The display Value of " + recordName + " was wrong: ",
                     "12345.678901234",
                     stringValue);
    }

    @Test(timeout = 5000)
    public void testSDSLabelTextPrecision() throws Exception {
        final String recordName = "DALPrecisionTest1";
        final String behaviorId = "behavior.desy.label.connectiondefault";

        final LabelModel labelModel = buildLabelModel(recordName, behaviorId);
        labelModel.setPropertyValue(AbstractTextTypeWidgetModel.PROP_TEXT_TYPE, TextTypeEnum.TEXT.getIndex());

        buildConnection(behaviorId, labelModel);

        Thread.sleep(200);
        final String stringValue = labelModel.getStringProperty(LabelModel.PROP_TEXTVALUE);
        assertEquals("The display Value of " + recordName + " was wrong: ",
                     "12345.6789",
                     stringValue);
    }

    @Test(timeout = 5000)
    public void testSDSLabelExpPrecision() throws Exception {
        final String recordName = "DALPrecisionTest1";
        final String behaviorId = "behavior.desy.label.connectiondefault";

        final LabelModel labelModel = buildLabelModel(recordName, behaviorId);
        labelModel.setPropertyValue(AbstractTextTypeWidgetModel.PROP_TEXT_TYPE, TextTypeEnum.EXP.getIndex());

        buildConnection(behaviorId, labelModel);

        Thread.sleep(200);
        final String stringValue = labelModel.getStringProperty(LabelModel.PROP_TEXTVALUE);
        // TODO (hrickens): Not sure that is the correct way/layer
        assertEquals("The display Value of " + recordName + " was wrong: ",
                     "12345.678901234",
                     stringValue);
    }

    @Test(timeout = 5000)
    public void testSDSLabelHexPrecision() throws Exception {
        final String recordName = "DALPrecisionTest1";
        final String behaviorId = "behavior.desy.label.connectiondefault";

        final LabelModel labelModel = buildLabelModel(recordName, behaviorId);
        labelModel.setPropertyValue(AbstractTextTypeWidgetModel.PROP_TEXT_TYPE, TextTypeEnum.HEX.getIndex());

        buildConnection(behaviorId, labelModel);

        Thread.sleep(200);
        final String stringValue = labelModel.getStringProperty(LabelModel.PROP_TEXTVALUE);
        // TODO (hrickens): Not sure that is the correct way/layer
        assertEquals("The display Value of " + recordName + " was wrong: ",
                     "12345.678901234",
                     stringValue);
    }

    @Test(timeout = 5000)
    public void testSDSLabelAliasPrecision() throws Exception {
        final String recordName = "DALPrecisionTest1";
        final String behaviorId = "behavior.desy.label.connectiondefault";

        final LabelModel labelModel = buildLabelModel(recordName, behaviorId);
        labelModel.setPropertyValue(AbstractTextTypeWidgetModel.PROP_TEXT_TYPE, TextTypeEnum.ALIAS.getIndex());

        buildConnection(behaviorId, labelModel);

        Thread.sleep(200);
        final String stringValue = labelModel.getStringProperty(LabelModel.PROP_TEXTVALUE);
        assertEquals("The display Value of " + recordName + " was wrong: ",
                     "DALPrecisionTest1",
                     stringValue);
    }

    /**
     * @param behaviorId
     * @param labelModel
     * @throws InterruptedException
     */
    private void buildConnection(final String behaviorId, final LabelModel labelModel) throws InterruptedException {
        final AbstractBehavior behavior = SdsPlugin.getDefault().getBehaviourService()
                .getBehavior(behaviorId, labelModel.getTypeID());

        final IProcessVariableAddress mainPv = labelModel.getMainPvAdress();
        final Class<?> javaType = labelModel.getJavaType();
        final RemoteInfo remoteInfo = new RemoteInfo(RemoteInfo.DAL_TYPE_PREFIX + "EPICS",
                                                     mainPv.getProperty(),
                                                     null,
                                                     null);
        final ConnectionParameters connectionParameters = new ConnectionParameters(remoteInfo,
                                                                                   javaType);
        if (behavior != null) {
            final BehaviorConnector behaviorConnector = new BehaviorConnector(labelModel,
                                                                              connectionParameters,
                                                                              behavior);

            // .. connect to the control system to receive dynamic values and meta data
            register(connectionParameters, behaviorConnector);

            // .. connect to the widget to receive manual changes and forward them to the
            // control system
            final String[] settablePropertyIds = behavior.getSettablePropertyIds();

            for (final String id : settablePropertyIds) {
                labelModel.addPropertyChangeListener(id, behaviorConnector);
            }
        }
    }

    /**
     * @param recordName
     * @param behaviorId
     * @return
     */
    private LabelModel buildLabelModel(final String recordName, final String behaviorId) {
        final DisplayModel displayModel = new DisplayModel();
        final LabelModel labelModel = new LabelModel();
        labelModel.setColor(AbstractWidgetModel.PROP_COLOR_BACKGROUND, "${Invalid}");
        labelModel.setPrimarPv(recordName);
        labelModel.setPropertyValue(AbstractWidgetModel.PROP_BEHAVIOR, behaviorId);
        displayModel.addWidget(labelModel);
        return labelModel;
    }

    /**<commit_msg>o.c.sds: extends the Precision test<commit_after>/*
 * Copyright (c) 2011 Stiftung Deutsches Elektronen-Synchrotron,
 * Member of the Helmholtz Association, (DESY), HAMBURG, GERMANY.
 *
 * THIS SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN "../AS IS" BASIS.
 * WITHOUT WARRANTY OF ANY KIND, EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED
 * TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR PARTICULAR PURPOSE AND
 * NON-INFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
 * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
 * THE USE OR OTHER DEALINGS IN THE SOFTWARE. SHOULD THE SOFTWARE PROVE DEFECTIVE
 * IN ANY RESPECT, THE USER ASSUMES THE COST OF ANY NECESSARY SERVICING, REPAIR OR
 * CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE.
 * NO USE OF ANY SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER.
 * DESY HAS NO OBLIGATION TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS,
 * OR MODIFICATIONS.
 * THE FULL LICENSE SPECIFYING FOR THE SOFTWARE THE REDISTRIBUTION, MODIFICATION,
 * USAGE AND OTHER RIGHTS AND OBLIGATIONS IS INCLUDED WITH THE DISTRIBUTION OF THIS
 * PROJECT IN THE FILE LICENSE.HTML. IF THE LICENSE IS NOT INCLUDED YOU MAY FIND A COPY
 * AT HTTP://WWW.DESY.DE/LEGAL/LICENSE.HTM
 */
package org.csstudio.sds;

import static org.junit.Assert.assertEquals;

import java.io.File;
import java.net.URL;
import java.util.ArrayList;

import org.csstudio.dal.CssApplicationContext;
import org.csstudio.domain.desy.junit.ConditionalClassRunner;
import org.csstudio.domain.desy.junit.OsCondition;
import org.csstudio.domain.desy.junit.RunIf;
import org.csstudio.domain.desy.softioc.BasicSoftIocConfigurator;
import org.csstudio.domain.desy.softioc.Caget;
import org.csstudio.domain.desy.softioc.DBR;
import org.csstudio.domain.desy.softioc.ISoftIocConfigurator;
import org.csstudio.domain.desy.softioc.SoftIoc;
import org.csstudio.platform.logging.CentralLogger;
import org.csstudio.platform.model.pvs.IProcessVariableAddress;
import org.csstudio.sds.eventhandling.AbstractBehavior;
import org.csstudio.sds.internal.connection.BehaviorConnector;
import org.csstudio.sds.model.AbstractTextTypeWidgetModel;
import org.csstudio.sds.model.AbstractWidgetModel;
import org.csstudio.sds.model.DisplayModel;
import org.csstudio.sds.model.LabelModel;
import org.csstudio.sds.model.TextTypeEnum;
import org.eclipse.core.runtime.FileLocator;
import org.epics.css.dal.simple.ChannelListener;
import org.epics.css.dal.simple.ConnectionParameters;
import org.epics.css.dal.simple.RemoteInfo;
import org.epics.css.dal.simple.SimpleDALBroker;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;

import com.cosylab.util.CommonException;

/**
 * TODO (hrickens) :
 *
 * @author hrickens
 * @since 25.11.2011
 */
@RunWith(ConditionalClassRunner.class)
@RunIf(conditionClass = OsCondition.class, arguments = { OsCondition.WIN })
public class CAJPrecisionHeadlessTest {

    private static SoftIoc _softIoc;


    @BeforeClass
    public static void startUpSoftIoc() throws Exception {
        final URL dbBundleResourceUrl = CAJPrecisionHeadlessTest.class.getClassLoader()
                .getResource("resources/db/CAJPrecisionTest.db");
        final URL dbFileUrl = FileLocator.toFileURL(dbBundleResourceUrl);

        final ISoftIocConfigurator cfg = new BasicSoftIocConfigurator().with(new File(dbFileUrl
                                                                                      .getFile()));
        _softIoc = new SoftIoc(cfg);
        _softIoc.start();
        while (!_softIoc.isStartUpDone()) {
            // wait IOC startup finished
        }
        setSystemPropertys();
        Thread.sleep(400);
    }


    /**
     * @throws java.lang.Exception
     */
    @Before
    public void setUp() throws Exception {
    }

    @Test(timeout = 5000)
    public void testCAGetPrecision() throws Exception {
//        while (!_softIoc.isStartUpDone()) {
//            // wait IOC startup finished
//        }
        final Caget caget = new Caget();
        caget.setDbr(DBR.DBR_TIME_STRING);
        caget.setWaitTime(3);
        final String recodName = "DALPrecisionTest1";
        final ArrayList<String> result = caget.caget(recodName);
        assertEquals(8, result.size());
        final String[] split = result.get(4).trim().split(":");
        assertEquals(2, split.length);
        final String value = split[1].trim();
        assertEquals("The display Value of " + recodName + " was wrong: ", "12345.6789", value);
    }

    @Test(timeout = 5000)
    public void testSDSLabelBehaviorDoublePrecision() throws Exception {
        final String recordName = "DALPrecisionTest1";
        final String behaviorId = "behavior.desy.label.connectiondefault";

        final LabelModel labelModel = buildLabelModel(recordName, behaviorId);
        labelModel.setPropertyValue(AbstractTextTypeWidgetModel.PROP_TEXT_TYPE, TextTypeEnum.DOUBLE.getIndex());

        buildConnection(behaviorId, labelModel);

        Thread.sleep(200);
        final String stringValue = labelModel.getStringProperty(LabelModel.PROP_TEXTVALUE);
        assertEquals("The display Value of " + recordName + " was wrong: ",
                     "12345.678901234",
                     stringValue);
    }

    @Test(timeout = 5000)
    public void testSDSLabelBehaviorTextPrecision() throws Exception {
        final String recordName = "DALPrecisionTest1";
        final String behaviorId = "behavior.desy.label.connectiondefault";

        final LabelModel labelModel = buildLabelModel(recordName, behaviorId);
        labelModel.setPropertyValue(AbstractTextTypeWidgetModel.PROP_TEXT_TYPE, TextTypeEnum.TEXT.getIndex());

        buildConnection(behaviorId, labelModel);

        Thread.sleep(200);
        final String stringValue = labelModel.getStringProperty(LabelModel.PROP_TEXTVALUE);
        assertEquals("The display Value of " + recordName + " was wrong: ",
                     "12345.6789",
                     stringValue);
    }

    @Test(timeout = 5000)
    public void testSDSLabelBehaviorExpPrecision() throws Exception {
        final String recordName = "DALPrecisionTest1";
        final String behaviorId = "behavior.desy.label.connectiondefault";

        final LabelModel labelModel = buildLabelModel(recordName, behaviorId);
        labelModel.setPropertyValue(AbstractTextTypeWidgetModel.PROP_TEXT_TYPE, TextTypeEnum.EXP.getIndex());

        buildConnection(behaviorId, labelModel);

        Thread.sleep(200);
        final String stringValue = labelModel.getStringProperty(LabelModel.PROP_TEXTVALUE);
        // TODO (hrickens): Not sure that is the correct way/layer
        assertEquals("The display Value of " + recordName + " was wrong: ",
                     "12345.678901234",
                     stringValue);
    }

    @Test(timeout = 5000)
    public void testSDSLabelBehaviorHexPrecision() throws Exception {
        final String recordName = "DALPrecisionTest1";
        final String behaviorId = "behavior.desy.label.connectiondefault";

        final LabelModel labelModel = buildLabelModel(recordName, behaviorId);
        labelModel.setPropertyValue(AbstractTextTypeWidgetModel.PROP_TEXT_TYPE, TextTypeEnum.HEX.getIndex());

        buildConnection(behaviorId, labelModel);

        Thread.sleep(200);
        final String stringValue = labelModel.getStringProperty(LabelModel.PROP_TEXTVALUE);
        // TODO (hrickens): Not sure that is the correct way/layer
        assertEquals("The display Value of " + recordName + " was wrong: ",
                     "12345.678901234",
                     stringValue);
    }

    @Test(timeout = 5000)
    public void testSDSLabelBehaviorAliasPrecision() throws Exception {
        final String recordName = "DALPrecisionTest1";
        final String behaviorId = "behavior.desy.label.connectiondefault";

        final LabelModel labelModel = buildLabelModel(recordName, behaviorId);
        labelModel.setPropertyValue(AbstractTextTypeWidgetModel.PROP_TEXT_TYPE, TextTypeEnum.ALIAS.getIndex());

        buildConnection(behaviorId, labelModel);

        Thread.sleep(200);
        final String stringValue = labelModel.getStringProperty(LabelModel.PROP_TEXTVALUE);
        assertEquals("The display Value of " + recordName + " was wrong: ",
                     "DALPrecisionTest1",
                     stringValue);
    }

    @Test(timeout = 5000)
    public void testSDSLabelAliasPrecision() throws Exception {
        final String recordName = "DALPrecisionTest1";
        final LabelModel labelModel = buildLabelModel(recordName, null);
        labelModel.setPropertyValue(AbstractTextTypeWidgetModel.PROP_TEXT_TYPE, TextTypeEnum.ALIAS.getIndex());
        labelModel.setJavaType(String.class);
        buildConnection(null, labelModel);

    }


    /**
     * @param behaviorId
     * @param labelModel
     * @throws InterruptedException
     */
    private void buildConnection(final LabelModel labelModel) throws InterruptedException {


//        ConnectionUtilNew.connectDynamizedProperties(labelModel.getPropertyInternal(name), labelModel.getMainPvAdress(), false, labelModel.get, labelModel.getRoot().getRuntimeContext());
//
//        final IProcessVariableAddress mainPv = labelModel.getMainPvAdress();
//        final Class<?> javaType = labelModel.getJavaType();
//        System.err.println("javaType: "+javaType.getSimpleName());
//        final RemoteInfo remoteInfo = new RemoteInfo(RemoteInfo.DAL_TYPE_PREFIX + "EPICS",
//                                                     mainPv.getProperty(),
//                                                     null,
//                                                     null);
//        final ConnectionParameters connectionParameters = new ConnectionParameters(remoteInfo,
//                                                                                   javaType);
//
//        new SinglePropertyReadConnector(connectionParameters, javaType, "");

    }
    private void buildConnection(final String behaviorId, final LabelModel labelModel) throws InterruptedException {
        final AbstractBehavior behavior = SdsPlugin.getDefault().getBehaviourService()
                .getBehavior(behaviorId, labelModel.getTypeID());

        final IProcessVariableAddress mainPv = labelModel.getMainPvAdress();
        final Class<?> javaType = labelModel.getJavaType();
        final RemoteInfo remoteInfo = new RemoteInfo(RemoteInfo.DAL_TYPE_PREFIX + "EPICS",
                                                     mainPv.getProperty(),
                                                     null,
                                                     null);
        System.err.println("javaType: "+javaType.getSimpleName());
        final ConnectionParameters connectionParameters = new ConnectionParameters(remoteInfo,
                                                                                   javaType);
        if (behavior != null) {
            final BehaviorConnector behaviorConnector = new BehaviorConnector(labelModel,
                                                                              connectionParameters,
                                                                              behavior);

            // .. connect to the control system to receive dynamic values and meta data
            register(connectionParameters, behaviorConnector);

            // .. connect to the widget to receive manual changes and forward them to the
            // control system
            final String[] settablePropertyIds = behavior.getSettablePropertyIds();

            for (final String id : settablePropertyIds) {
                labelModel.addPropertyChangeListener(id, behaviorConnector);
            }
        }
    }

    /**
     * @param recordName
     * @param behaviorId
     * @return
     */
    private LabelModel buildLabelModel(final String recordName, final String behaviorId) {
        final DisplayModel displayModel = new DisplayModel();
        final LabelModel labelModel = new LabelModel();
        labelModel.setColor(AbstractWidgetModel.PROP_COLOR_BACKGROUND, "${Invalid}");
        labelModel.setPrimarPv(recordName);
        if(behaviorId!=null&&!behaviorId.isEmpty()) {
            labelModel.setPropertyValue(AbstractWidgetModel.PROP_BEHAVIOR, behaviorId);
        }
        displayModel.addWidget(labelModel);
        return labelModel;
    }

    /**
{'fraction_non_alphanumeric': 0.08582944420873992, 'fraction_numerical': 0.010903691132795927, 'mean_word_length': 2.7503579952267305, 'pattern_counts': {'":': 2, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 47, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2502111', 'n_tokens_mistral': 6496, 'n_tokens_neox': 6097, 'n_words': 1478}