<commit_before><?php

/**
 * Description of marc21
 *
 * @author Heinz
 */
//
//  Returns a structure like the one below
//
//        $tagInd[0]->tag = '100';
//        $tagInd[0]->ind = '1_';
//        $tagInd[0]->subs = Array();
//        $tagInd[0]->subs[0]->code = 'a';
//        $tagInd[0]->subs[0]->data = 'mein Buch';
//        $tagInd[0]->subs[1]->code = 'b';
//        $tagInd[0]->subs[1]->data = 'dein Buch';
//        
//
class m21File {

    private $fh, $filter, $leader, $dict, $data = array(), $nRecords, $dataLen;

    function __construct($m21File) {
        $this->fh = fopen($m21File, 'rb');
        $this->filter = '';
        $this->nRecords = 0;
    }

    function decodeRecord() {

        $tagInd = Array();
        $m21 = $this->readM21Record($this->fh);

        while ($m21) {
            $i = 0;
            $nTags = (strlen($this->dict) - 1) / 12;
            /*
             * ***********************************************
             * iterate over directory entries, each 12 charactes
             * ***********************************************
             */
            for ($j = 0, $jj = -1, $i = 0; $j < $nTags; $j++, $i+=12) {
                $tag = mb_substr($this->dict, $i, 3);
                if ($this->filter && !strpos($this->filter, $tag . '|')) {
                    continue; //tag not in filter; skip it
                }

                $len = mb_substr($this->dict, $i + 3, 4) * 1;
                $offset = mb_substr($this->dict, $i + 3 + 4, 5) * 1;
                $jj++;
                $tagInd[$jj] = (new stdClass());
                $tagInd[$jj]->tag = $tag;
                /*
                 * ***********************************************
                 * indicators ?
                 * ***********************************************
                 */
                $tagInd[$jj]->ind = '  ';
                if ($tag >= '010') {
                    $tagInd[$jj]->ind = '__';
                    if ($this->data[$offset] > ' ') {
                        $tagInd[$jj]->ind{0} = $this->data[$offset];
                    }
                    $offset++;
                    if ($this->data[$offset] > ' ') {
                        $tagInd[$jj]->ind{1} = $this->data[$offset];
                    }
                    if ($tagInd[$jj]->ind === '__') {
                        $tagInd[$jj]->ind = '';
                    }
                    $offset++;
                }
                /*
                 * ***********************************************
                 * iterate over subfields separated by 'x1f' 
                 * ***********************************************
                 */
                $s = 0;
                $len = $offset + $len;
                while ($offset < $len && $this->data[$offset] !== "\x1E") {
                    if ($this->data[$offset] === "\x1F") {
                        /*
                         * ***********************************************
                         *  save subfield code and data
                         * ***********************************************
                         */
                        $tagInd[$jj]->subs[$s] = new stdClass();
                        $tagInd[$jj]->subs[$s]->code = $this->data[++$offset];
                        $offset++;
                        $nc = 0;
                        while ($this->data[$offset + $nc] >= ' ') {
                            //  $tagInd[$jj]->subs[$s]->data.=$this->data[$offset++];<commit_msg>initila commit<commit_after><?php

/**
 * Description of marc21
 *
 * @author Heinz
 */
//
//  Returns a structure like the one below
//
//        $tagInd[0]->tag = '100';
//        $tagInd[0]->ind = '1_';
//        $tagInd[0]->subs = Array();
//        $tagInd[0]->subs[0]->code = 'a';
//        $tagInd[0]->subs[0]->data = 'mein Buch';
//        $tagInd[0]->subs[1]->code = 'b';
//        $tagInd[0]->subs[1]->data = 'dein Buch';
//        
//
class m21File {

    private $fh, $filter, $leader, $dict, $data = array(), $nRecords, $dataLen;

    function __construct($m21File) {
        $this->fh = fopen($m21File, 'rb');
        $this->filter = '';
        $this->nRecords = 0;
    }

    function decodeRecord() {

        $tagInd = Array();
        $m21 = $this->readM21Record($this->fh);

        while ($m21) {
            $i = 0;
            $nTags = (strlen($this->dict) - 1) / 12;
            /*
             * ***********************************************
             * iterate over directory entries, each 12 charactes
             * ***********************************************
             */
            for ($j = 0, $jj = -1, $i = 0; $j < $nTags; $j++, $i+=12) {
                $tag = mb_substr($this->dict, $i, 3);
                if ($this->filter && !strpos($this->filter, $tag . '|')) {
                    continue; //tag not in filter; skip it
                }

                $len = mb_substr($this->dict, $i + 3, 4) * 1;
                $offset = mb_substr($this->dict, $i + 3 + 4, 5) * 1;
                $jj++;
                $tagInd[$jj] = (new stdClass());
                $tagInd[$jj]->tag = $tag;
                /*
                 * ***********************************************
                 * indicators ?
                 * ***********************************************
                 */
                $tagInd[$jj]->ind = '  ';
                if ($tag >= '010') {
                    $tagInd[$jj]->ind = '__';
                    if ($this->data[$offset] > ' ') {
                        $tagInd[$jj]->ind{0} = $this->data[$offset];
                    }
                    $offset++;
                    if ($this->data[$offset] > ' ') {
                        $tagInd[$jj]->ind{1} = $this->data[$offset];
                    }
                    if ($tagInd[$jj]->ind === '__') {
                        $tagInd[$jj]->ind = '';
                    }
                    $offset++;
                }
                /*
                 * ***********************************************
                 * iterate over subfields separated by 'x1f' 
                 * ***********************************************
                 */
                $s = 0;
                $len = $offset + $len;
                while ($offset < $len && $this->data[$offset] !== "\x1E") {
                    if ($this->data[$offset] === "\x1F") {
                        /*
                         * ***********************************************
                         *  save subfield code and data
                         * ***********************************************
                         */
                        $tagInd[$jj]->subs[$s] = new stdClass();
                        $tagInd[$jj]->subs[$s]->code = $this->data[++$offset];
                        $offset++;
                        $nc = 0;
                        while ($this->data[$offset + $nc] >= ' ') {                      
{'fraction_non_alphanumeric': 0.27268817204301077, 'fraction_numerical': 0.01691756272401434, 'mean_word_length': 1.2139003490955251, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 94, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2033875', 'n_tokens_mistral': 2011, 'n_tokens_neox': 1839, 'n_words': 337}