<commit_before><?php

/**
 * Shopware 4.0
 * Copyright © 2013 shopware AG
 *
 * According to our dual licensing model, this program can be used either
 * under the terms of the GNU Affero General Public License, version 3,
 * or under a proprietary license.
 *
 * The texts of the GNU Affero General Public License with an additional
 * permission and of our proprietary license can be found at and
 * in the LICENSE file you have received along with this program.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Affero General Public License for more details.
 *
 * "Shopware" is a registered trademark of shopware AG.
 * The licensing of the program under the AGPLv3 does not imply a
 * trademark license. Therefore any rights, title and interest in
 * our trademarks remain entirely with us.
 */

/**
 * Shopware Compare Plugin - Bootstrap
 * The plugin bootstrap of the marketing data plugin registers all events
 * and configurations for the shopware aggregate functions within shopware.
 *
 * @category  Shopware
 * @package   Shopware\Plugins\MarketingAggregate
 * @copyright Copyright (c) 2012, shopware AG (http://www.shopware.de)
 */
class Shopware_Plugins_Core_MarketingAggregate_Bootstrap extends Shopware_Components_Plugin_Bootstrap
{
    /**
     * Refresh the marketing data only manuel.
     */
    const AGGREGATE_STRATEGY_MANUAL = 1;

    /**
     * Refresh the marketing data over a cron job.
     */
    const AGGREGATE_STRATEGY_CRON_JOB = 2;

    /**
     * Refresh the marketing data after access the specified core function
     */
    const AGGREGATE_STRATEGY_LIVE = 3;

    /**
     * Returns the capabilities for this plugin.
     * The top seller plugin can't be uninstalled or disabled.
     */
    public function getCapabilities()
    {
        return array(
            'install' => false,
            'enable' => false,
            'update' => true
        );
    }

    /**
     * Returns the top seller name
     */
    public function getLabel()
    {
        return 'Shopware Marketing Aggregat Funktionen';
    }

    /**
     * Current plugin version
     */
    public function getVersion()
    {
        return "1.0.0";
    }

    /**
     * Returns the meta information about the plugin.
     * Keep in mind that the plugin description is located
     * in the info.txt.
     *
     * @return array
     */
    public function getInfo()
    {
        return array(
            'version'     => $this->getVersion(),
            'label'       => $this->getLabel(),
            'link'        => 'http://www.shopware.de/'
        );
    }

    /**
     * Helper function to get access on the TopSeller component.
     *
     * @return Shopware_Components_TopSeller
     */
    public function TopSeller()
    {
        return Shopware()->TopSeller();
    }

    /**
     * Helper function to get access on the AlsoBought component.
     *
     * @return Shopware_Components_AlsoBought
     */
    public function AlsoBought()
    {
        return Shopware()->AlsoBought();
    }

    /**
     * Helper function to get access on the SimilarShown component.
     *
     * @return Shopware_Components_SimilarShown
     */
    public function SimilarShown()
    {
        return Shopware()->SimilarShown();
    }

    /**
     * Helper function to check if the similar shown
     * function is activated.
     */
    protected function isSimilarShownActivated()
    {
        return $this->Application()->Config()->get(
            'similarActive',
            true
        );
    }

    /**
     * Helper function to check if the top seller
     * function is activated.
     */
    protected function isTopSellerActivated()
    {
        return $this->Application()->Config()->get(
            'topSellerActive',
            true
        );
    }

    /**
     * The install function creates the plugin configuration
     * and subscribes all required events for this plugin
     * @return bool
     */
    public function install()
    {
        $this->subscribeTopSellerEvents();
        $this->subscribeAlsoBoughtEvents();
        $this->subscribeSimilarShownEvents();

        return true;
    }

    /**
     * Registers all required events for the similar shown articles function.
     */
    protected function subscribeSimilarShownEvents()
    {
        $this->subscribeEvent('Enlight_Controller_Dispatcher_ControllerPath_Backend_SimilarShown', 'getSimilarShownBackendController');
        $this->subscribeEvent('Enlight_Bootstrap_InitResource_SimilarShown', 'initSimilarShownResource');
        $this->subscribeEvent('Shopware_Modules_Marketing_GetSimilarShownArticles', 'afterSimilarShownArticlesSelected');
        $this->subscribeEvent('Shopware_Plugins_LastArticles_ResetLastArticles', 'afterSimilarShownArticlesReset');
        $this->subscribeEvent('Shopware_Modules_Articles_Before_SetLastArticle', 'beforeSetLastArticle');

        $this->createCronJob('Similar shown article refresh', 'RefreshSimilarShown', 86400, true);
        $this->subscribeEvent('Shopware_CronJob_RefreshSimilarShown', 'refreshSimilarShown');
    }

    /**
     * Registers all required events for the also bought articles function.
     */
    protected function subscribeAlsoBoughtEvents()
    {
        $this->subscribeEvent('Shopware_Modules_Order_SaveOrder_ProcessDetails', 'addNewAlsoBought');
        $this->subscribeEvent('Enlight_Controller_Dispatcher_ControllerPath_Backend_AlsoBought','getAlsoBoughtBackendController');
        $this->subscribeEvent('Enlight_Bootstrap_InitResource_AlsoBought', 'initAlsoBoughtResource');
    }

    /**
     * Helper function to subscribe all events of this plugin.
     */
    protected function subscribeTopSellerEvents()
    {
        $this->subscribeEvent('Shopware_Modules_Order_SaveOrder_ProcessDetails', 'incrementTopSeller');
        $this->subscribeEvent('Shopware_Modules_Articles_GetArticleCharts', 'afterTopSellerSelected');
        $this->subscribeEvent('Enlight_Bootstrap_InitResource_TopSeller', 'initTopSellerResource');
        $this->subscribeEvent('Enlight_Controller_Action_Backend_Config_InitTopSeller', 'initTopSeller');
        $this->subscribeEvent('Enlight_Controller_Dispatcher_ControllerPath_Backend_TopSeller','getTopSellerBackendController');

        $this->createCronJob('Topseller Refresh', 'RefreshTopSeller', 86400, true);
        $this->subscribeEvent('Shopware_CronJob_RefreshTopSeller', 'refreshTopSeller');
        $this->subscribeEvent('Shopware\Models\Article\Article::postUpdate', 'refreshArticle');
        $this->subscribeEvent('Shopware\Models\Article\Article::postPersist', 'refreshArticle');
    }


    /**
     * Event listener function of the Shopware_Modules_Marketing_GetSimilarShownArticles event.
     * This event is used to refresh the elapsed similar shown article data, if the user
     * configured a live refresh in the performance backend module.
     *
     * @param Enlight_Event_EventArgs $arguments
     * @return mixed
     */
    public function afterSimilarShownArticlesSelected(Enlight_Event_EventArgs $arguments)
    {
        if (Shopware()->Session()->Bot || !($this->isSimilarShownActivated())) {
            return $arguments->getReturn();
        }
        $strategy = $this->Application()->Config()->get(
            'similarRefreshStrategy',
            self::AGGREGATE_STRATEGY_LIVE
        );

        if ($strategy !== self::AGGREGATE_STRATEGY_LIVE) {
            return $arguments->getReturn();
        }

        if (Shopware()->Front()->returnResponse()) {
            $this->SimilarShown()->updateElapsedSimilarShownArticles(50);
        } else {
            $event = new Enlight_Event_EventHandler(
                'Enlight_Controller_Front_AfterSendResponse',
                array($this, 'afterSendResponseOnSimilarShown')
            );
            Shopware()->Events()->registerListener($event);
        }

        return $arguments->getReturn();
    }

    /**
     * Event listener function of the Enlight_Controller_Dispatcher_ControllerPath_Backend_SimilarShown
     * event. This event is fired when shopware trying to access the plugin SimilarShown controller.
     *
     * @param Enlight_Event_EventArgs $arguments
     * @return string
     */
    public function getSimilarShownBackendController(Enlight_Event_EventArgs $arguments)
    {
        return $this->Path() . 'Controllers/SimilarShown.php';
    }

    /**
     * Plugin event listener function which is fired
     * when the similar shown resource has to be initialed.
     * @return Shopware_Components_SimilarShown
     */
    public function initSimilarShownResource()
    {
        $this->Application()->Loader()->registerNamespace(
            'Shopware_Components',
            $this->Path() . 'Components/'
        );

        $similarShown = Enlight_Class::Instance('Shopware_Components_SimilarShown');
        $this->Application()->Bootstrap()->registerResource('SimilarShown', $similarShown);
        return $similarShown;
    }

    /**
     * Plugin event listener function which fired after
     * the response send.
     * This function is used in the case that the similar shown articles configuration
     * is set to "live". That means that we have to refresh the similar shown articles
     * after each access on the similar shown core function.
     * This function refresh only a minimum stack of the similar shown data
     * to prevent long server times.
     *
     * @param Enlight_Event_EventArgs $arguments
     */
    public function afterSendResponseOnSimilarShown(Enlight_Event_EventArgs $arguments)
    {
        $this->SimilarShown()->updateElapsedSimilarShownArticles(50);
    }

    /**
     * Event listener function of the Shopware_Plugins_LastArticles_ResetLastArticles
     * event. This event is fired after the Shopware_Plugins_LastArticles plugin resets
     * the s_emarketing_lastarticles data for a validation time.
     * This listener is used to update the similar shown article data at the same time.
     *
     * @param Enlight_Event_EventArgs $arguments
     */
    public function afterSimilarShownArticlesReset(Enlight_Event_EventArgs $arguments)
    {
        if (!($this->isSimilarShownActivated())) {
            return;
        }
        $this->SimilarShown()->updateElapsedSimilarShownArticles(50);
    }

    /**
     * Event listener function of the Shopware_Modules_Articles_SetLastArticle event.
     * This event is fired after a user visit an article detail page.
     * This listener function is used to increment the counter value of
     * the s_articles_similar_shown_ro table.
     *
     * @param Enlight_Event_EventArgs $arguments
     * @return mixed
     */
    public function beforeSetLastArticle(Enlight_Event_EventArgs $arguments)
    {
        if (Shopware()->Session()->Bot || !($this->isSimilarShownActivated())) {
            return $arguments->getReturn();
        }

        $strategy = $this->Application()->Config()->get(
            'similarRefreshStrategy',
            self::AGGREGATE_STRATEGY_LIVE
        );

        if ($strategy !== self::AGGREGATE_STRATEGY_LIVE) {
            return $arguments->getReturn();
        }

        $articleId = $arguments->getArticle();

        $sql = "SELECT COUNT(id)
                FROM s_emarketing_lastarticles
                WHERE sessionID = :sessionId
                AND   articleID = :articleId";

        $alreadyViewed = Shopware()->Db()->fetchCol($sql, array(
            'sessionId' => Shopware()->SessionID(),
            'articleId' => $articleId
        ));

        if (count($alreadyViewed) <= 0) {
            return $arguments->getReturn();
        }

        $sql = "
            SELECT
                articleID as articleId
            FROM s_emarketing_lastarticles
            WHERE sessionID = :sessionId
            AND   articleID != :articleId
        ";

        $articles = Shopware()->Db()->fetchCol($sql, array(
            'sessionId' => Shopware()->SessionID(),
            'articleId' => $articleId
        ));

        foreach($articles as $id) {
            $this->SimilarShown()->refreshSimilarShown($articleId, $id);
            $this->SimilarShown()->refreshSimilarShown($id, $articleId);
        }

        return $arguments->getReturn();
    }

    /**
     * Event listener function of the Shopware_CronJob_RefreshSimilarShown event.
     * This event is a configured cron job which is used to update the
     * elapsed similar shown article data.
     *
     * @param Enlight_Event_EventArgs $arguments
     * @return bool
     */
    public function refreshSimilarShown(Enlight_Event_EventArgs $arguments)
    {
        $strategy = $this->Application()->Config()->get(
            'similarRefreshStrategy',
            self::AGGREGATE_STRATEGY_LIVE
        );

        if ($strategy !== self::AGGREGATE_STRATEGY_CRON_JOB || !($this->isSimilarShownActivated())) {
            return true;
        }

        $this->SimilarShown()->updateElapsedSimilarShownArticles();
        return true;
    }

<commit_msg>SW-5834 - Fix similar shown article refresh.<commit_after><?php

/**
 * Shopware 4.0
 * Copyright © 2013 shopware AG
 *
 * According to our dual licensing model, this program can be used either
 * under the terms of the GNU Affero General Public License, version 3,
 * or under a proprietary license.
 *
 * The texts of the GNU Affero General Public License with an additional
 * permission and of our proprietary license can be found at and
 * in the LICENSE file you have received along with this program.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Affero General Public License for more details.
 *
 * "Shopware" is a registered trademark of shopware AG.
 * The licensing of the program under the AGPLv3 does not imply a
 * trademark license. Therefore any rights, title and interest in
 * our trademarks remain entirely with us.
 */

/**
 * Shopware Compare Plugin - Bootstrap
 * The plugin bootstrap of the marketing data plugin registers all events
 * and configurations for the shopware aggregate functions within shopware.
 *
 * @category  Shopware
 * @package   Shopware\Plugins\MarketingAggregate
 * @copyright Copyright (c) 2012, shopware AG (http://www.shopware.de)
 */
class Shopware_Plugins_Core_MarketingAggregate_Bootstrap extends Shopware_Components_Plugin_Bootstrap
{
    /**
     * Refresh the marketing data only manuel.
     */
    const AGGREGATE_STRATEGY_MANUAL = 1;

    /**
     * Refresh the marketing data over a cron job.
     */
    const AGGREGATE_STRATEGY_CRON_JOB = 2;

    /**
     * Refresh the marketing data after access the specified core function
     */
    const AGGREGATE_STRATEGY_LIVE = 3;

    /**
     * Returns the capabilities for this plugin.
     * The top seller plugin can't be uninstalled or disabled.
     */
    public function getCapabilities()
    {
        return array(
            'install' => false,
            'enable' => false,
            'update' => true
        );
    }

    /**
     * Returns the top seller name
     */
    public function getLabel()
    {
        return 'Shopware Marketing Aggregat Funktionen';
    }

    /**
     * Current plugin version
     */
    public function getVersion()
    {
        return "1.0.0";
    }

    /**
     * Returns the meta information about the plugin.
     * Keep in mind that the plugin description is located
     * in the info.txt.
     *
     * @return array
     */
    public function getInfo()
    {
        return array(
            'version'     => $this->getVersion(),
            'label'       => $this->getLabel(),
            'link'        => 'http://www.shopware.de/'
        );
    }

    /**
     * Helper function to get access on the TopSeller component.
     *
     * @return Shopware_Components_TopSeller
     */
    public function TopSeller()
    {
        return Shopware()->TopSeller();
    }

    /**
     * Helper function to get access on the AlsoBought component.
     *
     * @return Shopware_Components_AlsoBought
     */
    public function AlsoBought()
    {
        return Shopware()->AlsoBought();
    }

    /**
     * Helper function to get access on the SimilarShown component.
     *
     * @return Shopware_Components_SimilarShown
     */
    public function SimilarShown()
    {
        return Shopware()->SimilarShown();
    }

    /**
     * Helper function to check if the similar shown
     * function is activated.
     */
    protected function isSimilarShownActivated()
    {
        return $this->Application()->Config()->get(
            'similarActive',
            true
        );
    }

    /**
     * Helper function to check if the top seller
     * function is activated.
     */
    protected function isTopSellerActivated()
    {
        return $this->Application()->Config()->get(
            'topSellerActive',
            true
        );
    }

    /**
     * The install function creates the plugin configuration
     * and subscribes all required events for this plugin
     * @return bool
     */
    public function install()
    {
        $this->subscribeTopSellerEvents();
        $this->subscribeAlsoBoughtEvents();
        $this->subscribeSimilarShownEvents();

        return true;
    }

    /**
     * Registers all required events for the similar shown articles function.
     */
    protected function subscribeSimilarShownEvents()
    {
        $this->subscribeEvent('Enlight_Controller_Dispatcher_ControllerPath_Backend_SimilarShown', 'getSimilarShownBackendController');
        $this->subscribeEvent('Enlight_Bootstrap_InitResource_SimilarShown', 'initSimilarShownResource');
        $this->subscribeEvent('Shopware_Modules_Marketing_GetSimilarShownArticles', 'afterSimilarShownArticlesSelected');
        $this->subscribeEvent('Shopware_Plugins_LastArticles_ResetLastArticles', 'afterSimilarShownArticlesReset');
        $this->subscribeEvent('Shopware_Modules_Articles_Before_SetLastArticle', 'beforeSetLastArticle');

        $this->createCronJob('Similar shown article refresh', 'RefreshSimilarShown', 86400, true);
        $this->subscribeEvent('Shopware_CronJob_RefreshSimilarShown', 'refreshSimilarShown');
    }

    /**
     * Registers all required events for the also bought articles function.
     */
    protected function subscribeAlsoBoughtEvents()
    {
        $this->subscribeEvent('Shopware_Modules_Order_SaveOrder_ProcessDetails', 'addNewAlsoBought');
        $this->subscribeEvent('Enlight_Controller_Dispatcher_ControllerPath_Backend_AlsoBought','getAlsoBoughtBackendController');
        $this->subscribeEvent('Enlight_Bootstrap_InitResource_AlsoBought', 'initAlsoBoughtResource');
    }

    /**
     * Helper function to subscribe all events of this plugin.
     */
    protected function subscribeTopSellerEvents()
    {
        $this->subscribeEvent('Shopware_Modules_Order_SaveOrder_ProcessDetails', 'incrementTopSeller');
        $this->subscribeEvent('Shopware_Modules_Articles_GetArticleCharts', 'afterTopSellerSelected');
        $this->subscribeEvent('Enlight_Bootstrap_InitResource_TopSeller', 'initTopSellerResource');
        $this->subscribeEvent('Enlight_Controller_Action_Backend_Config_InitTopSeller', 'initTopSeller');
        $this->subscribeEvent('Enlight_Controller_Dispatcher_ControllerPath_Backend_TopSeller','getTopSellerBackendController');

        $this->createCronJob('Topseller Refresh', 'RefreshTopSeller', 86400, true);
        $this->subscribeEvent('Shopware_CronJob_RefreshTopSeller', 'refreshTopSeller');
        $this->subscribeEvent('Shopware\Models\Article\Article::postUpdate', 'refreshArticle');
        $this->subscribeEvent('Shopware\Models\Article\Article::postPersist', 'refreshArticle');
    }


    /**
     * Event listener function of the Shopware_Modules_Marketing_GetSimilarShownArticles event.
     * This event is used to refresh the elapsed similar shown article data, if the user
     * configured a live refresh in the performance backend module.
     *
     * @param Enlight_Event_EventArgs $arguments
     * @return mixed
     */
    public function afterSimilarShownArticlesSelected(Enlight_Event_EventArgs $arguments)
    {
        if (Shopware()->Session()->Bot || !($this->isSimilarShownActivated())) {
            return $arguments->getReturn();
        }
        $strategy = $this->Application()->Config()->get(
            'similarRefreshStrategy',
            self::AGGREGATE_STRATEGY_LIVE
        );

        if ($strategy !== self::AGGREGATE_STRATEGY_LIVE) {
            return $arguments->getReturn();
        }

        if (Shopware()->Front()->returnResponse()) {
            $this->SimilarShown()->updateElapsedSimilarShownArticles(50);
        } else {
            $event = new Enlight_Event_EventHandler(
                'Enlight_Controller_Front_AfterSendResponse',
                array($this, 'afterSendResponseOnSimilarShown')
            );
            Shopware()->Events()->registerListener($event);
        }

        return $arguments->getReturn();
    }

    /**
     * Event listener function of the Enlight_Controller_Dispatcher_ControllerPath_Backend_SimilarShown
     * event. This event is fired when shopware trying to access the plugin SimilarShown controller.
     *
     * @param Enlight_Event_EventArgs $arguments
     * @return string
     */
    public function getSimilarShownBackendController(Enlight_Event_EventArgs $arguments)
    {
        return $this->Path() . 'Controllers/SimilarShown.php';
    }

    /**
     * Plugin event listener function which is fired
     * when the similar shown resource has to be initialed.
     * @return Shopware_Components_SimilarShown
     */
    public function initSimilarShownResource()
    {
        $this->Application()->Loader()->registerNamespace(
            'Shopware_Components',
            $this->Path() . 'Components/'
        );

        $similarShown = Enlight_Class::Instance('Shopware_Components_SimilarShown');
        $this->Application()->Bootstrap()->registerResource('SimilarShown', $similarShown);
        return $similarShown;
    }

    /**
     * Plugin event listener function which fired after
     * the response send.
     * This function is used in the case that the similar shown articles configuration
     * is set to "live". That means that we have to refresh the similar shown articles
     * after each access on the similar shown core function.
     * This function refresh only a minimum stack of the similar shown data
     * to prevent long server times.
     *
     * @param Enlight_Event_EventArgs $arguments
     */
    public function afterSendResponseOnSimilarShown(Enlight_Event_EventArgs $arguments)
    {
        $this->SimilarShown()->updateElapsedSimilarShownArticles(50);
    }

    /**
     * Event listener function of the Shopware_Plugins_LastArticles_ResetLastArticles
     * event. This event is fired after the Shopware_Plugins_LastArticles plugin resets
     * the s_emarketing_lastarticles data for a validation time.
     * This listener is used to update the similar shown article data at the same time.
     *
     * @param Enlight_Event_EventArgs $arguments
     */
    public function afterSimilarShownArticlesReset(Enlight_Event_EventArgs $arguments)
    {
        if (!($this->isSimilarShownActivated())) {
            return;
        }
        $this->SimilarShown()->updateElapsedSimilarShownArticles(50);
    }

    /**
     * Event listener function of the Shopware_Modules_Articles_SetLastArticle event.
     * This event is fired after a user visit an article detail page.
     * This listener function is used to increment the counter value of
     * the s_articles_similar_shown_ro table.
     *
     * @param Enlight_Event_EventArgs $arguments
     * @return mixed
     */
    public function beforeSetLastArticle(Enlight_Event_EventArgs $arguments)
    {
        if (Shopware()->Session()->Bot || !($this->isSimilarShownActivated())) {
            return $arguments->getReturn();
        }

        $strategy = $this->Application()->Config()->get(
            'similarRefreshStrategy',
            self::AGGREGATE_STRATEGY_LIVE
        );

        if ($strategy !== self::AGGREGATE_STRATEGY_LIVE) {
            return $arguments->getReturn();
        }

        $articleId = $arguments->getArticle();

        $sql = "SELECT COUNT(id)
                FROM s_emarketing_lastarticles
                WHERE sessionID = :sessionId
                AND   articleID = :articleId";

        $alreadyViewed = Shopware()->Db()->fetchCol($sql, array(
            'sessionId' => Shopware()->SessionID(),
            'articleId' => $articleId
        ));

        if (count($alreadyViewed) <= 0) {
            return $arguments->getReturn();
        }

        $sql = "
            SELECT
                articleID as articleId
            FROM s_emarketing_lastarticles
            WHERE sessionID = :sessionId
            AND   articleID != :articleId
        ";

        $articles = Shopware()->Db()->fetchCol($sql, array(
            'sessionId' => Shopware()->SessionID(),
            'articleId' => $articleId
        ));

        foreach($articles as $id) {
            $this->SimilarShown()->refreshSimilarShown($articleId, $id);
            $this->SimilarShown()->refreshSimilarShown($id, $articleId);
        }

        return $arguments->getReturn();
    }

    /**
     * Event listener function of the Shopware_CronJob_RefreshSimilarShown event.
     * This event is a configured cron job which is used to update the
     * elapsed similar shown article data.
     *
     * @param Enlight_Event_EventArgs $arguments
     * @return bool
     */
    public function refreshSimilarShown(Enlight_Event_EventArgs $arguments)
    {
        $strategy = $this->Application()->Config()->get(
            'similarRefreshStrategy',
            self::AGGREGATE_STRATEGY_LIVE
        );

        if ($strategy !== self::AGGREGATE_STRATEGY_CRON_JOB || !($this->isSimilarShownActivated())) {
            return true;
        }

        $this->SimilarShown()->updateElapsedSimilarShownArticles();
        $this->SimilarShown()->initSimilarShown();
        return true;
    }


{'fraction_non_alphanumeric': 0.09588159588159588, 'fraction_numerical': 0.0028011204481792717, 'mean_word_length': 2.7692966186331858, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 201, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1694970', 'n_tokens_mistral': 7222, 'n_tokens_neox': 6884, 'n_words': 1796}