<commit_before><?php

namespace DTA\MetadataBundle\Controller;

use Symfony\Component\HttpFoundation\Request;
use Symfony\Component\HttpFoundation\Response;
use Exception;

/**
 * TODO: Nice to have: if the labels on the horizontal forms (column layout) would slide down the screen (position: fixed)
 * to support orientation in vertically long forms (where only the left border is visible)
 */

/**
 * Base class for all domain controllers. Contains generic actions (list all records, new record)
 * that are derived from the database schema.
 */
class ORMController extends DTADomainController {

    public $package = null;

    // TODO: pressing enter in a modal form leads to a form submit and all data is lost, 
    // because the server redirects to the ajax response page

    /**
     * Deletes a record from the database after having asked for a confirmation.
     * @param type $package   like in genericViewOneAction
     * @param type $className   like in genericViewOneAction
     * @param type $recordId    like in genericViewOneAction
     */
    public function genericDeleteOneAction(Request $request, $package, $className, $recordId) {

        // get confirmation
        if ( ! $request->isMethod("POST")) {

            return $this->renderWithDomainData("DTAMetadataBundle:ORM:confirmDelete.html.twig", array(
                'className' => $className,
                'recordId' => $recordId,
            ));

        // try to really delete data on confirm POST
        } else {

            // confirmed
            if( $request->get("reallyDelete") ){                                    // && $request->get('recordId')

                $classNames = $this->relatedClassNames($package, $className);
                $query = new $classNames['query'];
                $record = $query->findOneById($recordId);

                if($record === null){
                    $this->addErrorFlash("The record ($className #$recordId) to be deleted doesn't exist.");
                }
                else{
                    $record->delete();
                    $this->addSuccessFlash("Datensatz gelöscht.");
                }

            // abort
            } else {

                if( ! $request->get("reallyDelete") ){
                    $this->addWarningFlash("Datensatz nicht gelöscht.");
                } else if(! $request->get('recordId')){
                    $this->addErrorFlash("Datensatz kann nicht gelöscht werden: Er existiert nicht.");
                }

            }

            return $this->genericViewAllAction($package, $className);

        }
    }

    /**
     * Renders a single entity for ajax display (i.e. without any additional markup like menus or footers).
     * @param string $package      domain/object model package (Data, Classification, Workflow, Master)
     * @param string $className    model class
     * @param int    $recordId
     */
    public function genericViewAction(Request $request, $package, $className, $recordId) {

        $classNames = $this->relatedClassNames($package, $className);

        // for retrieving the entities
        $query    = new $classNames['query'];
//        $formType = new $classNames['formType'];

        $record = $query->findOneById($recordId);

        $form = $this->genericCreateOrEdit($request, $record);

//        return $this->renderWithDomainData("DTAMetadataBundle:ORM:createOrEdit.html.twig", array(
//                    'form' => $form['form']->createView(),
//                    'transaction' => $form['transaction'],    // whether the form is for edit or create
//                    'className' => $className,
//                    'recordId' => $recordId,
//                ));

        return $this->renderWithDomainData("DTAMetadataBundle:Form:genericBaseForm.html.twig", array(
                    'className' => $className,
                    'form' => $form['form']->createView(),
                ));
    }

    /**
     *
     */
    private static function extractPureAccessor($accessor){
        if(strncmp($accessor, "accessor:", strlen("accessor:"))) {
            return $accessor;
        }elseif(!strncmp($accessor, "accessor:get", strlen("accessor:get"))) {
            $modifiedAccessor = substr($accessor,strlen("accessor:get"));
            return $modifiedAccessor;
        }else{
            throw new InvalidArgumentException(sprintf(
                'Could not extract pure accessor of \'%s\'.', $accessor));
        }
    }

    /**
     * Adds a sorting or filtering criteria to the given query.
     * If no fitting filter/order method is available the original query will be returned and critical logging entry is added.
     * @param $query The source query where the criteria will be added.
     * @param $type 'filter' or 'order'
     * @param $columnCaption The caption which determines the column of the criteria.
     * @param $modelClass To get the correct accessors the model class is needed.
     * @param $data depends on the type value: 'filter' -> filterstring, 'order' -> sorting direction (asc/desc)
     * @return mixed the modifeid query
     * @throws InvalidArgumentException
     */
    private function addCriteria($query, $type, $columnCaption, $modelClass, $data){
        if(!in_array($type, array('filter','order')))
            throw new \InvalidArgumentException(sprintf(
                '\'%s\' is not allowed as value for $type. Use \'filter\' or \'order\'.',$type));
        $criteriaAccessor = $type.'By'.ORMController::extractPureAccessor($modelClass::$tableRowViewAccessors[$columnCaption]);

        // is the column embedded?
        $embeddedAccessor = explode('Of',$criteriaAccessor);
        if(count($embeddedAccessor)>1){
            $useQuery = "use$embeddedAccessor[1]Query";
            $criteriaAccessor = $embeddedAccessor[0];
            $query = $query->$useQuery();
            // is it a representative?
        }

        // adding "String" to the function name prevents collision with propel filter method
        if(method_exists($query,$criteriaAccessor."String")){
            $criteriaAccessor = $criteriaAccessor."String";
            //$this->get('logger')->critical($type." via: ".$criteriaAccessor." ".$data);
            $query = $query->$criteriaAccessor($data);
        }
        elseif(is_callable(array($query,$criteriaAccessor))){
            //$this->get('logger')->critical($type." via: ".$criteriaAccessor." ".$data);
            $query = $query->$criteriaAccessor($data);
        }else{
            $this->get('logger')->critical("not callable: ".$criteriaAccessor);
        }

        // is the column embedded? Then Close it.
        if(count($embeddedAccessor)>1){
            $query = $query->endUse();
        }
        return $query;
    }

    /**
     * @param $request
     * @param $package
     * @param $className
     * @param bool $hiddenIdColumn
     * @return \ModelCriteria
     */
    protected function getSortedFilteredQuery($request, $package, $className, $hiddenIdColumn = false){
        $classNames = $this->relatedClassNames($package, $className);
        $modelClass = new $classNames["model"];
        $query = $modelClass::getRowViewQueryObject();
        $columns = $modelClass::getTableViewColumnNames();

        //$this->get('logger')->critical(urldecode($request));
        //$this->get('logger')->critical("hiddenIdColumn: ".$hiddenIdColumn);

        // SORTING
        if($request->get('order')) {
            $direction = $request->get('order')[0]['dir'];
            $orderColumnCaption = $columns[$request->get('order')[0]['column'] - ($hiddenIdColumn?1:0)];
            $query = $this->addCriteria($query,'order',$orderColumnCaption,$modelClass,$direction);

            // use the class specific default sorting
        }elseif(method_exists($classNames['query'], 'sqlSort')){
            $query = $classNames['query']::sqlSort($query, \ModelCriteria::ASC);
        }
        // SORTING END

        $this->get('logger')->critical("sorted query: " . $query->toString());

        // FILTERING
        if($request->get('search')['value']) {
            $this->get('logger')->critical("filterColumns: " . implode(", ",$modelClass::getTableViewFilterColumns()));
            $this->get('logger')->critical("FILTER");
            $filterString = $request->get('search')['value'];

            $filterColumnCaptions = $modelClass::getTableViewFilterColumns();
            if(!empty($filterColumnCaptions)){
                $query = $this->addCriteria($query,'filter',array_shift($filterColumnCaptions),$modelClass,$filterString);
            }
            foreach($filterColumnCaptions as $columnCaption){
                $query = $this->addCriteria($query->_or(),'filter',$columnCaption,$modelClass,$filterString);
            }
        }
        return $query;

        // filtering is more difficult than initially thought!
        // - using ILIKE, case insensitive search is performed
        // - using % and _ several and a single character can be wildcarded, respectively
        // - a union mechanism seems to be missing (search on each column and union the matches), could be simulated by appending the search results manually, but that destroys sorting
        //   another option might be to use criteria (the $query->where(...) methods) because there is an _or() function to use with them but it is not obvious how to formulate
        //   expressions like the ones below in this syntax (especially if it comes to date specifications which are related by different columns)

        /* @var $query \DTA\MetadataBundle\Model\Data\BookQuery */
//        $result2 = $modelClass::getRowViewQueryObject()->usePublicationQuery()->useTitleQuery()->useTitlefragmentQuery()->filterByName('%lage%', \ModelCriteria::ILIKE)->endUse()->endUse()->endUse()->find();
//        $result1 = $modelClass::getRowViewQueryObject()->usePublicationQuery()->useTitleQuery()->useTitlefragmentQuery()->filterByName('%sinn%', \ModelCriteria::ILIKE)->endUse()->endUse()->endUse()->find();
//        /* @var $result1 \PropelObjectCollection */
//        foreach($result2 as $result)
//        $result1->append($result);
//        $query = $query->usePublicationQuery()->useDatespecificationRelatedByPublicationdateIdQuery()->filterByYear('1811')->endUse()->endUse();
//        $query = $query->wh // where('year = ?', 1811);



    }

    protected function getPaginatedEntities($request, $query){
        // pagination offset
        $offset = $request->get('start');
        $numRecords = $request->get('length');

        /* @var $query \DTA\MetadataBundle\Model\Data\PublicationQuery */
        $entities = $query->setFormatter(\ModelCriteria::FORMAT_ON_DEMAND)
                          ->offset($offset)
                          ->limit($numRecords)
                          ->find();

        return $entities;

//        } catch (\PropelException $exc) {
////            $query = $modelClass::getRowViewQueryObject();
//            $query->offset(NULL)->limit(NULL);
//            $allEntities = $query->find();
//            $entities = array();//array_slice($entities->toArray(), $offset, $numRecords);
//            for ($i = 0; $i < $numRecords; $i++) {
//                $entities[] = $allEntities->get($offset+$i);
//            }
//        }
    }

    /**
     *
     * @param type $entities Propel entities
     * @param type $columns  The attributes to display, needs to be a subset of the table row view defined columns
     * @param type $package  Data, Workflow, Classification or Master
     * @param type $className The unqualified class name of the entity
     * @param bool $addIdColumn If true, an id column will be added
     * @return type array containing one array per record, listing the values of the given attributes (columns)
     */
    protected function formatAsArray($entities, $columns, $package, $className, $addIdColumn = false){
        $result = array();
        foreach($entities as $entity) {
            $row = array();
            if($addIdColumn){
                $row = array($entity->getId());
            }
            for ($i = 0; $i < count($columns); $i++) {
                $column = $columns[$i];
                $attribute = $entity->getAttributeByTableViewColumName($column);

                if(is_object($attribute)){
                    $value = $attribute->__toString();
                } else {
                    $value = $attribute;
                }
                // add an edit button to the first column entry
                if($i === 0){
                    $editLink = "";
                    if($entity->getSpecialization()!== null){
                        $classNameParts = explode('\\',get_class($entity->getSpecialization()));
                        $linkClassName = array_pop($classNameParts);
                        $linkPackage = array_pop($classNameParts);
                        $editLink = $this->generateUrl(
                            $linkPackage . '_genericCreateOrEdit',
                            array(
                                'package' => $linkPackage,
                                'className' => $linkClassName,
                                'recordId' => $entity->getSpecialization()->getId()
                            )
                        );
                    }else{
                        $editLink = $this->generateUrl(
                            $package . '_genericCreateOrEdit',
                            array(
                                'package' => $package,
                                'className' => $className,
                                'recordId' => $entity->getId()
                            )
                        );
                    }
//                    $deleteLink = $this->generateUrl($package . '_deleteRecord', array(
//                        'package'=>$package, 'className'=>$className, 'recordId'=>$entity->getId() 
//                    ));
                    $row[] = "<a href='$editLink'><span class='glyphicon glyphicon-edit'></span></a>"
//                            ."<a href='$deleteLink'><span class='glyphicon glyphicon-trash'></span></a> "
                            ."$value";
                } elseif(is_object($attribute)) {
                        // reflection
                        $classNameParts = explode('\\',get_class($attribute));
                        $linkClassName = array_pop($classNameParts);
                        $linkPackage = array_pop($classNameParts);
                        $editLink = $this->generateUrl(
                            $linkPackage . '_genericCreateOrEdit',
                            array(
                                'package' => $linkPackage,
                                'className' => $linkClassName,
                                'recordId' => $attribute->getId()
                            )
                        );
                        $row[] = "<a href='$editLink'>$value</a>";
                }else{
                    $row[] = $value;
                }
            }
            $result[] = $row;
	    }

        return $result;

    }
    /**
     * Responds to XHR requests of the data tables module.
     * @param \Symfony\Component\HttpFoundation\Request $request
     * @param String $package
     * @param String $className
     * @param bool $addIdColumn
     * @return Response<commit_msg>cleared ORMController code (added function getEditingLinkForObject)<commit_after><?php

namespace DTA\MetadataBundle\Controller;

use Symfony\Component\HttpFoundation\Request;
use Symfony\Component\HttpFoundation\Response;
use Exception;

/**
 * TODO: Nice to have: if the labels on the horizontal forms (column layout) would slide down the screen (position: fixed)
 * to support orientation in vertically long forms (where only the left border is visible)
 */

/**
 * Base class for all domain controllers. Contains generic actions (list all records, new record)
 * that are derived from the database schema.
 */
class ORMController extends DTADomainController {

    public $package = null;

    // TODO: pressing enter in a modal form leads to a form submit and all data is lost, 
    // because the server redirects to the ajax response page

    /**
     * Deletes a record from the database after having asked for a confirmation.
     * @param type $package   like in genericViewOneAction
     * @param type $className   like in genericViewOneAction
     * @param type $recordId    like in genericViewOneAction
     */
    public function genericDeleteOneAction(Request $request, $package, $className, $recordId) {

        // get confirmation
        if ( ! $request->isMethod("POST")) {

            return $this->renderWithDomainData("DTAMetadataBundle:ORM:confirmDelete.html.twig", array(
                'className' => $className,
                'recordId' => $recordId,
            ));

        // try to really delete data on confirm POST
        } else {

            // confirmed
            if( $request->get("reallyDelete") ){                                    // && $request->get('recordId')

                $classNames = $this->relatedClassNames($package, $className);
                $query = new $classNames['query'];
                $record = $query->findOneById($recordId);

                if($record === null){
                    $this->addErrorFlash("The record ($className #$recordId) to be deleted doesn't exist.");
                }
                else{
                    $record->delete();
                    $this->addSuccessFlash("Datensatz gelöscht.");
                }

            // abort
            } else {

                if( ! $request->get("reallyDelete") ){
                    $this->addWarningFlash("Datensatz nicht gelöscht.");
                } else if(! $request->get('recordId')){
                    $this->addErrorFlash("Datensatz kann nicht gelöscht werden: Er existiert nicht.");
                }

            }

            return $this->genericViewAllAction($package, $className);

        }
    }

    /**
     * Renders a single entity for ajax display (i.e. without any additional markup like menus or footers).
     * @param string $package      domain/object model package (Data, Classification, Workflow, Master)
     * @param string $className    model class
     * @param int    $recordId
     */
    public function genericViewAction(Request $request, $package, $className, $recordId) {

        $classNames = $this->relatedClassNames($package, $className);

        // for retrieving the entities
        $query    = new $classNames['query'];
//        $formType = new $classNames['formType'];

        $record = $query->findOneById($recordId);

        $form = $this->genericCreateOrEdit($request, $record);

//        return $this->renderWithDomainData("DTAMetadataBundle:ORM:createOrEdit.html.twig", array(
//                    'form' => $form['form']->createView(),
//                    'transaction' => $form['transaction'],    // whether the form is for edit or create
//                    'className' => $className,
//                    'recordId' => $recordId,
//                ));

        return $this->renderWithDomainData("DTAMetadataBundle:Form:genericBaseForm.html.twig", array(
                    'className' => $className,
                    'form' => $form['form']->createView(),
                ));
    }

    /**
     *
     */
    private static function extractPureAccessor($accessor){
        if(strncmp($accessor, "accessor:", strlen("accessor:"))) {
            return $accessor;
        }elseif(!strncmp($accessor, "accessor:get", strlen("accessor:get"))) {
            $modifiedAccessor = substr($accessor,strlen("accessor:get"));
            return $modifiedAccessor;
        }else{
            throw new InvalidArgumentException(sprintf(
                'Could not extract pure accessor of \'%s\'.', $accessor));
        }
    }

    /**
     * Adds a sorting or filtering criteria to the given query.
     * If no fitting filter/order method is available the original query will be returned and critical logging entry is added.
     * @param $query The source query where the criteria will be added.
     * @param $type 'filter' or 'order'
     * @param $columnCaption The caption which determines the column of the criteria.
     * @param $modelClass To get the correct accessors the model class is needed.
     * @param $data depends on the type value: 'filter' -> filterstring, 'order' -> sorting direction (asc/desc)
     * @return mixed the modifeid query
     * @throws InvalidArgumentException
     */
    private function addCriteria($query, $type, $columnCaption, $modelClass, $data){
        if(!in_array($type, array('filter','order')))
            throw new \InvalidArgumentException(sprintf(
                '\'%s\' is not allowed as value for $type. Use \'filter\' or \'order\'.',$type));
        $criteriaAccessor = $type.'By'.ORMController::extractPureAccessor($modelClass::$tableRowViewAccessors[$columnCaption]);

        // is the column embedded?
        $embeddedAccessor = explode('Of',$criteriaAccessor);
        if(count($embeddedAccessor)>1){
            $useQuery = "use$embeddedAccessor[1]Query";
            $criteriaAccessor = $embeddedAccessor[0];
            $query = $query->$useQuery();
        }

        // adding "String" to the function name prevents collision with propel filter method
        if(method_exists($query,$criteriaAccessor."String")){
            $criteriaAccessor = $criteriaAccessor."String";
            //$this->get('logger')->critical($type." via: ".$criteriaAccessor." ".$data);
            $query = $query->$criteriaAccessor($data);
        }
        elseif(is_callable(array($query,$criteriaAccessor))){
            //$this->get('logger')->critical($type." via: ".$criteriaAccessor." ".$data);
            $query = $query->$criteriaAccessor($data);
        }else{
            $this->get('logger')->critical("not callable: ".$criteriaAccessor);
        }

        // is the column embedded? Then Close it.
        if(count($embeddedAccessor)>1){
            $query = $query->endUse();
        }
        return $query;
    }

    /**
     * @param $request
     * @param $package
     * @param $className
     * @param bool $hiddenIdColumn
     * @return \ModelCriteria
     */
    protected function getSortedFilteredQuery($request, $package, $className, $hiddenIdColumn = false){
        $classNames = $this->relatedClassNames($package, $className);
        $modelClass = new $classNames["model"];
        $query = $modelClass::getRowViewQueryObject();
        $columns = $modelClass::getTableViewColumnNames();

        //$this->get('logger')->critical(urldecode($request));
        //$this->get('logger')->critical("hiddenIdColumn: ".$hiddenIdColumn);

        // SORTING
        if($request->get('order')) {
            $direction = $request->get('order')[0]['dir'];
            $orderColumnCaption = $columns[$request->get('order')[0]['column'] - ($hiddenIdColumn?1:0)];
            $query = $this->addCriteria($query,'order',$orderColumnCaption,$modelClass,$direction);

            // use the class specific default sorting
        }elseif(method_exists($classNames['query'], 'sqlSort')){
            $query = $classNames['query']::sqlSort($query, \ModelCriteria::ASC);
        }
        // SORTING END

        $this->get('logger')->critical("sorted query: " . $query->toString());

        // FILTERING
        if($request->get('search')['value']) {
            $this->get('logger')->critical("filterColumns: " . implode(", ",$modelClass::getTableViewFilterColumns()));
            $this->get('logger')->critical("FILTER");
            $filterString = $request->get('search')['value'];

            $filterColumnCaptions = $modelClass::getTableViewFilterColumns();
            if(!empty($filterColumnCaptions)){
                $query = $this->addCriteria($query,'filter',array_shift($filterColumnCaptions),$modelClass,$filterString);
            }
            foreach($filterColumnCaptions as $columnCaption){
                $query = $this->addCriteria($query->_or(),'filter',$columnCaption,$modelClass,$filterString);
            }
        }
        return $query;

        // filtering is more difficult than initially thought!
        // - using ILIKE, case insensitive search is performed
        // - using % and _ several and a single character can be wildcarded, respectively
        // - a union mechanism seems to be missing (search on each column and union the matches), could be simulated by appending the search results manually, but that destroys sorting
        //   another option might be to use criteria (the $query->where(...) methods) because there is an _or() function to use with them but it is not obvious how to formulate
        //   expressions like the ones below in this syntax (especially if it comes to date specifications which are related by different columns)

        /* @var $query \DTA\MetadataBundle\Model\Data\BookQuery */
//        $result2 = $modelClass::getRowViewQueryObject()->usePublicationQuery()->useTitleQuery()->useTitlefragmentQuery()->filterByName('%lage%', \ModelCriteria::ILIKE)->endUse()->endUse()->endUse()->find();
//        $result1 = $modelClass::getRowViewQueryObject()->usePublicationQuery()->useTitleQuery()->useTitlefragmentQuery()->filterByName('%sinn%', \ModelCriteria::ILIKE)->endUse()->endUse()->endUse()->find();
//        /* @var $result1 \PropelObjectCollection */
//        foreach($result2 as $result)
//        $result1->append($result);
//        $query = $query->usePublicationQuery()->useDatespecificationRelatedByPublicationdateIdQuery()->filterByYear('1811')->endUse()->endUse();
//        $query = $query->wh // where('year = ?', 1811);



    }

    protected function getPaginatedEntities($request, $query){
        // pagination offset
        $offset = $request->get('start');
        $numRecords = $request->get('length');

        /* @var $query \DTA\MetadataBundle\Model\Data\PublicationQuery */
        $entities = $query->setFormatter(\ModelCriteria::FORMAT_ON_DEMAND)
                          ->offset($offset)
                          ->limit($numRecords)
                          ->find();

        return $entities;

//        } catch (\PropelException $exc) {
////            $query = $modelClass::getRowViewQueryObject();
//            $query->offset(NULL)->limit(NULL);
//            $allEntities = $query->find();
//            $entities = array();//array_slice($entities->toArray(), $offset, $numRecords);
//            for ($i = 0; $i < $numRecords; $i++) {
//                $entities[] = $allEntities->get($offset+$i);
//            }
//        }
    }

    private function getEditingLinkForObject($obj){
        $classNameParts = explode('\\',get_class($obj));
        $linkClassName = array_pop($classNameParts);
        $linkPackage = array_pop($classNameParts);
        return $this->generateUrl(
            $linkPackage . '_genericCreateOrEdit',
            array(
                'package' => $linkPackage,
                'className' => $linkClassName,
                'recordId' => $obj->getId()
            )
        );
    }

    /**
     *
     * @param type $entities Propel entities
     * @param type $columns  The attributes to display, needs to be a subset of the table row view defined columns
     * @param type $package  Data, Workflow, Classification or Master
     * @param type $className The unqualified class name of the entity
     * @param bool $addIdColumn If true, an id column will be added
     * @return type array containing one array per record, listing the values of the given attributes (columns)
     */
    protected function formatAsArray($entities, $columns, $package, $className, $addIdColumn = false){
        $result = array();
        foreach($entities as $entity) {
            $row = array();
            if($addIdColumn){
                $row = array($entity->getId());
            }
            for ($i = 0; $i < count($columns); $i++) {
                $column = $columns[$i];
                $attribute = $entity->getAttributeByTableViewColumName($column);

                if(is_object($attribute)){
                    $value = $attribute->__toString();
                } else {
                    $value = $attribute;
                }
                // add an edit button to the first column entry
                if($i === 0){
                    $specialized = $entity->getSpecialization();
                    $editLink = $specialized!==null?$this->getEditingLinkForObject($specialized):$this->getEditingLinkForObject($entity);
//                    $deleteLink = $this->generateUrl($package . '_deleteRecord', array(
//                        'package'=>$package, 'className'=>$className, 'recordId'=>$entity->getId() 
//                    ));
                    $row[] = "<a href='$editLink'><span class='glyphicon glyphicon-edit'></span></a>"
//                            ."<a href='$deleteLink'><span class='glyphicon glyphicon-trash'></span></a> "
                            ."$value";
                } elseif(is_object($attribute)) {
                        $editLink = $this->getEditingLinkForObject($attribute);
                        $row[] = "<a href='$editLink'>$value</a>";
                }else{
                    $row[] = $value;
                }
            }
            $result[] = $row;
	    }

        return $result;

    }
    /**
     * Responds to XHR requests of the data tables module.
     * @param \Symfony\Component\HttpFoundation\Request $request
     * @param String $package
     * @param String $className
     * @param bool $addIdColumn
     * @return Response
{'fraction_non_alphanumeric': 0.13761807119567043, 'fraction_numerical': 0.0016134996134323843, 'mean_word_length': 2.1571686299479995, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 264, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5038974', 'n_tokens_mistral': 7745, 'n_tokens_neox': 7450, 'n_words': 1866}