<commit_before><?php

/**
 * Class FileFabricate
 *
 * テスト用にファイルを簡単に出力するためのツール。
 * テスト後に(register_shutdown_functionで)自動的にファイルは削除される。
 */
class FileFabricate {
    private static $base_str = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";

    public static $dir = null;

    public static function from2DimensionalArray($data) {
        return new FileFabricateDataCells($data);
    }

    public static function fromString($string) {
        return new FileFabricateFile($string);
    }

    public static function fromCsv($path, $delimiter = ',', $enclosure = '"') {
        $data = [];
        $fh = fopen($path, "r");
        $maxsize = filesize($path);
        while (($line = fgetcsv($fh, $maxsize, $delimiter, $enclosure)) !== FALSE) {
            $data[] = $line;
        }
        return new FileFabricateDataCells($data);
    }

    public static function defineTemplate($definition) {
        return new FileFabricateTemplate($definition);
    }

    public static function value_integer($min = 1, $max = null) {
        return self::value_callback(function ($i) use ($min, $max) {
            if ($max === null) {
                $row = $i + $min;
            } else {
                $row = $i % ($max - $min + 1) + $min; //maxが指定されたらmax内をループする
            }
            return $row;
        });
    }

    public static function value_string($size) {
        $base = self::$base_str;
        return self::value_callback(function ($i) use ($size, $base) {
            $pos = $i % strlen($base);
            return str_repeat($base[$pos], $size);
        });
    }

    public static function value_date($format = 'Y-m-d H:i:s', $basedate = '2000-01-01 00:00:00') {
        return self::value_callback(function ($i) use ($format, $basedate) {
            return date($format, strtotime($basedate) + $i * 3600 * 24);
        });
    }

    public static function value_rotation($array) {
        return self::value_callback(function ($i) use ($array) {
            $pos = $i % count($array);
            return $array[$pos];
        });
    }

    public static function value_callback($callback) {
        return new FileFabricateFabricatorByLabel($callback);
    }
}

/**
 * Class FileFabricateDataCells
 *
 * csv/tsv を作成できる２次元データ
 */
class FileFabricateDataCells {
    private $cells; //２次元配列

    private $withoutBrAtEof = false;
    private $changes = [];
    private $delimiter;
    private $enclosure;

    public function __construct($cells) {
        $this->cells = $cells;
    }

    public function toCsv($delimiter = ',', $enclosure = '"') {
        $this->delimiter = $delimiter;
        $this->enclosure = $enclosure;
        return new FileFabricateFile($this);
    }

    public function __toString() {
        if (!empty($this->changes)) {
            foreach ($this->changes as $change) {
                list($i, $col, $value) = $change;
                $this->cells[$i][$col] = $value;
            }
        }
        $fh_memory = fopen("php://memory", "rw");
        foreach ($this->cells as $row) {
            fputcsv($fh_memory, $row, $this->delimiter, $this->enclosure);
        }
        $size = ftell($fh_memory);
        fseek($fh_memory, 0);
        $str = fread($fh_memory, $size);
        fclose($fh_memory);
        if ($this->withoutBrAtEof) {
            $str = preg_replace('/\n$/', "", $str);
        }
        return $str;
    }

    public function toTsv($delimiter = "\t", $enclosure = '"') {
        return $this->toCsv($delimiter, $enclosure);
    }

    /**
     * ファイル末尾には末尾の改行コードをつけない。
     *
     * @return $this
     */
    public function withoutBrAtEof() {
        $this->withoutBrAtEof = true;
        return $this;
    }

    public function changeValue($i, $label, $value) {
        $col = array_search($label, $this->cells[0]);
        if ($col === FALSE) {
            throw new LogicException("Unkown Label: " . $label);
        }
        if (count($this->cells) < $i) {
            throw new LogicException("More than the number of rows (" . count($this->cells) . ") : " . $i);
        }

        $this->changes[] = [$i, $col, $value];

        return $this;
    }
}

class FileFabricateFileSettings {
    public $encodeTo = 'UTF-8';
    public $bom = null;
    public $directory = null;
    public $filename = null;
    public $alreadyMade = false;
}

/**
 * Class FileFabricateFile
 *
 * ファイル出力可能な文字列データ
 */
class FileFabricateFile {

    /** @var string|FileFabricateDataCells */
    private $data;

    /** @var FileFabricateFileSettings */
    protected $settings;

    private $path = null; //作成済みのファイルパス

    /**
     * @param string|FileFabricateDataCells $data
     */
    public function __construct($data) {
        $this->settings = new FileFabricateFileSettings();
        $this->data = $data;
    }

    public function encodeTo($encoding) {
        $this->settings->encodeTo = $encoding;
        $this->__resetFile();
        return $this;
    }<commit_msg>UTF-16LE で出力した際にBOMがつくように<commit_after><?php

/**
 * Class FileFabricate
 *
 * テスト用にファイルを簡単に出力するためのツール。
 * テスト後に(register_shutdown_functionで)自動的にファイルは削除される。
 */
class FileFabricate {
    private static $base_str = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";

    public static $dir = null;

    public static function from2DimensionalArray($data) {
        return new FileFabricateDataCells($data);
    }

    public static function fromString($string) {
        return new FileFabricateFile($string);
    }

    public static function fromCsv($path, $delimiter = ',', $enclosure = '"') {
        $data = [];
        $fh = fopen($path, "r");
        $maxsize = filesize($path);
        while (($line = fgetcsv($fh, $maxsize, $delimiter, $enclosure)) !== FALSE) {
            $data[] = $line;
        }
        return new FileFabricateDataCells($data);
    }

    public static function defineTemplate($definition) {
        return new FileFabricateTemplate($definition);
    }

    public static function value_integer($min = 1, $max = null) {
        return self::value_callback(function ($i) use ($min, $max) {
            if ($max === null) {
                $row = $i + $min;
            } else {
                $row = $i % ($max - $min + 1) + $min; //maxが指定されたらmax内をループする
            }
            return $row;
        });
    }

    public static function value_string($size) {
        $base = self::$base_str;
        return self::value_callback(function ($i) use ($size, $base) {
            $pos = $i % strlen($base);
            return str_repeat($base[$pos], $size);
        });
    }

    public static function value_date($format = 'Y-m-d H:i:s', $basedate = '2000-01-01 00:00:00') {
        return self::value_callback(function ($i) use ($format, $basedate) {
            return date($format, strtotime($basedate) + $i * 3600 * 24);
        });
    }

    public static function value_rotation($array) {
        return self::value_callback(function ($i) use ($array) {
            $pos = $i % count($array);
            return $array[$pos];
        });
    }

    public static function value_callback($callback) {
        return new FileFabricateFabricatorByLabel($callback);
    }
}

/**
 * Class FileFabricateDataCells
 *
 * csv/tsv を作成できる２次元データ
 */
class FileFabricateDataCells {
    private $cells; //２次元配列

    private $withoutBrAtEof = false;
    private $changes = [];
    private $delimiter;
    private $enclosure;

    public function __construct($cells) {
        $this->cells = $cells;
    }

    public function toCsv($delimiter = ',', $enclosure = '"') {
        $this->delimiter = $delimiter;
        $this->enclosure = $enclosure;
        return new FileFabricateFile($this);
    }

    public function __toString() {
        if (!empty($this->changes)) {
            foreach ($this->changes as $change) {
                list($i, $col, $value) = $change;
                $this->cells[$i][$col] = $value;
            }
        }
        $fh_memory = fopen("php://memory", "rw");
        foreach ($this->cells as $row) {
            fputcsv($fh_memory, $row, $this->delimiter, $this->enclosure);
        }
        $size = ftell($fh_memory);
        fseek($fh_memory, 0);
        $str = fread($fh_memory, $size);
        fclose($fh_memory);
        if ($this->withoutBrAtEof) {
            $str = preg_replace('/\n$/', "", $str);
        }
        return $str;
    }

    public function toTsv($delimiter = "\t", $enclosure = '"') {
        return $this->toCsv($delimiter, $enclosure);
    }

    /**
     * ファイル末尾には末尾の改行コードをつけない。
     *
     * @return $this
     */
    public function withoutBrAtEof() {
        $this->withoutBrAtEof = true;
        return $this;
    }

    public function changeValue($i, $label, $value) {
        $col = array_search($label, $this->cells[0]);
        if ($col === FALSE) {
            throw new LogicException("Unkown Label: " . $label);
        }
        if (count($this->cells) < $i) {
            throw new LogicException("More than the number of rows (" . count($this->cells) . ") : " . $i);
        }

        $this->changes[] = [$i, $col, $value];

        return $this;
    }
}

class FileFabricateFileSettings {
    public $encodeTo = 'UTF-8';
    public $bom = null;
    public $directory = null;
    public $filename = null;
    public $alreadyMade = false;
}

/**
 * Class FileFabricateFile
 *
 * ファイル出力可能な文字列データ
 */
class FileFabricateFile {

    /** @var string|FileFabricateDataCells */
    private $data;

    /** @var FileFabricateFileSettings */
    protected $settings;

    private $path = null; //作成済みのファイルパス

    /**
     * @param string|FileFabricateDataCells $data
     */
    public function __construct($data) {
        $this->settings = new FileFabricateFileSettings();
        $this->data = $data;
    }

    public function encodeTo($encoding) {
        $this->settings->encodeTo = $encoding;
        if ($encoding === 'UTF-16LE') {
            $this->settings->bom = "\xff\xfe";
        }
        $this->__resetFile();
        return $this;
    }
{'fraction_non_alphanumeric': 0.14908509149085092, 'fraction_numerical': 0.0059994000599940004, 'mean_word_length': 2.2912142152023693, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7210098', 'n_tokens_mistral': 3461, 'n_tokens_neox': 3245, 'n_words': 666}