<commit_before><?php

namespace console\controllers;

use backend\services\AccessLogService;
use backend\services\DistributeLogService;
use backend\services\ToolService;
use Yii;
use yii\console\Controller;

/**
 * 日志处理后台程序
 */
class LogdealController extends Controller {

    /**
     * 分析nginx配置文件
     * $message string 指定是那个代理的文件
     * $fitdata string 指定是那一天的数据
     * @return type
     */
    public function actionNginxaccessfile($message = '17', $fitdata = '') {
        //
        set_time_limit(0);
        ini_set('memory_limit', '4024M');
        $save_rs = false;
        //步增判断，当天的数据，最后一部分的数据不会入库
        $step = TRUE;
        //如果有传入时间参数那么以时间参数为准 
        if (empty($fitdata)) {
            $fitdata = date("Ymd", time());
        } else {
            $step = FALSE;
        }
        if ($message == '17') {
            $dir = \Yii::$app->params['proxy17'];
            $source = "17";
        } else {
            $dir = \Yii::$app->params['proxy21'];
            $source = "21";
        }
        $dir.='/' . $fitdata;
        //test data
        $dir = \Yii::$app->params['proxytest'];
        $handle = dir($dir);

        $fitFileArray = \Yii::$app->params['nginxfitfile'];

        while ($entry = $handle->read()) {
            if (!in_array($entry, array('.', '..'))) {
                $file_url = $dir . "/" . $entry;
                //获取文件名22
                $short_name = ToolService::parseFileName($entry);<commit_msg>增加一个用户访问路线图的功能。<commit_after><?php

namespace console\controllers;

use backend\services\AccessLogService;
use backend\services\DistributeLogService;
use backend\services\ToolService;
use Yii;
use yii\console\Controller;

/**
 * 日志处理后台程序
 */
class LogdealController extends Controller {

    /**
     * 分析nginx配置文件
     * $message string 指定是那个代理的文件
     * $fitdata string 指定是那一天的数据
     * @return type
     */
    public function actionNginxaccessfile($message = '17', $fitdata = '') {
        //
        set_time_limit(0);
        ini_set('memory_limit', '4024M');
        $save_rs = false;
        //步增判断，当天的数据，最后一部分的数据不会入库
        $step = TRUE;
        //如果有传入时间参数那么以时间参数为准 
        if (empty($fitdata)) {
            $fitdata = date("Ymd", time());
        } else {
            $step = FALSE;
        }
        if ($message == '17') {
            $dir = \Yii::$app->params['proxy17'];
            $source = "17";
        } else {
            $dir = \Yii::$app->params['proxy21'];
            $source = "21";
        }
        $dir.='/' . $fitdata;
        //test data
        //$dir = \Yii::$app->params['proxytest'];
        $handle = dir($dir);

        $fitFileArray = \Yii::$app->params['nginxfitfile'];

        while ($entry = $handle->read()) {
            if (!in_array($entry, array('.', '..'))) {
                $file_url = $dir . "/" . $entry;
                //获取文件名22
                $short_name = ToolService::parseFileName($entry);
{'fraction_non_alphanumeric': 0.16390614216701174, 'fraction_numerical': 0.013112491373360938, 'mean_word_length': 1.9253279515640767, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3507602', 'n_tokens_mistral': 1031, 'n_tokens_neox': 1026, 'n_words': 165}