<commit_before><?php
/**
 * @copyright Copyright (C) 2015 <NAME>
 * @license https://github.com/fetus-hina/stat.ink/blob/master/LICENSE MIT
 * @author <NAME> <<EMAIL>>
 */

namespace app\components\web;

use Yii;
use yii\base\Component;
use yii\web\ResponseFormatterInterface;
use app\components\helpers\Resource;

class CsvResponseFormatter extends Component implements ResponseFormatterInterface
{
    public $inputCharset;
    public $outputCharset;

    public function format($response)
    {
        $this->inputCharset = @$response->data['inputCharset'] ?: Yii::$app->charset;
        $this->outputCharset = @$response->data['outputCharset'] ?: Yii::$app->charset;
        
        // 代替文字
        $substitute = new Resource(
            mb_substitute_character(),
            function ($old) {
                mb_substitute_character($old);
            }
        );
        mb_substitute_character(0x3013);

        $tmpfile = tmpfile();
        foreach ($response->data['rows'] as $row) {
            fwrite($tmpfile, $this->formatRow($row) . "\x0d\x0a");
        }
        fseek($tmpfile, 0, SEEK_SET);
        $response->content = null;
        $response->stream = $tmpfile;
    }

    protected function formatRow(array $row)
    {
        $ret = array_map(
            function ($cell) {
                $cell = mb_convert_encoding((string)$cell, $this->outputCharset, $this->inputCharset);
                if (!preg_match('/[",\x0d\x0a]/', $cell)) {
                    return $cell;
                }
                return '"' . mb_str_replace('"', '""', $cell, $this->outputCharset)  . '"';
            },
            $row
        );
        return implode(',', $ret);
    }
}
<commit_msg>:+1: CSV出力時にBOM出力有無の設定と代替文字を変更可能にした<commit_after><?php
/**
 * @copyright Copyright (C) 2015 <NAME>
 * @license https://github.com/fetus-hina/stat.ink/blob/master/LICENSE MIT
 * @author <NAME> <<EMAIL>>
 */

namespace app\components\web;

use Yii;
use yii\base\Component;
use yii\web\ResponseFormatterInterface;
use app\components\helpers\Resource;

class CsvResponseFormatter extends Component implements ResponseFormatterInterface
{
    public $inputCharset;
    public $outputCharset;
    public $substituteCharacter;
    public $appendBOM;

    public function format($response)
    {
        $this->inputCharset = $response->data['inputCharset'] ?? Yii::$app->charset;
        $this->outputCharset = $response->data['outputCharset'] ?? Yii::$app->charset;
        $this->substituteCharacter = $response->data['substituteCharacter'] ?? 0x3013;
        $this->appendBOM = $response->data['appendBOM'] ?? false;
        
        // 代替文字
        $substitute = new Resource(
            mb_substitute_character(),
            function ($old) {
                mb_substitute_character($old);
            }
        );
        mb_substitute_character($this->substituteCharacter);

        $tmpfile = tmpfile();
        if ($this->appendBOM && in_array($this->outputCharset, ['UTF-8', 'UTF-16', 'UTF-32'])) {
            fwrite($tmpfile, mb_convert_encoding("\xfe\xff", $this->outputCharset, 'UTF-16BE'));
        }
        foreach ($response->data['rows'] as $row) {
            fwrite($tmpfile, $this->formatRow($row) . "\x0d\x0a");
        }
        fseek($tmpfile, 0, SEEK_SET);
        $response->content = null;
        $response->stream = $tmpfile;
    }

    protected function formatRow(array $row)
    {
        $ret = array_map(
            function ($cell) {
                $cell = mb_convert_encoding((string)$cell, $this->outputCharset, $this->inputCharset);
                if (!preg_match('/[",\x0d\x0a]/', $cell)) {
                    return $cell;
                }
                return '"' . mb_str_replace('"', '""', $cell, $this->outputCharset)  . '"';
            },
            $row
        );
        return implode(',', $ret);
    }
}

{'fraction_non_alphanumeric': 0.15653298835705046, 'fraction_numerical': 0.009314359637774904, 'mean_word_length': 2.347186147186147, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 45, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1674873', 'n_tokens_mistral': 1241, 'n_tokens_neox': 1194, 'n_words': 203}