<commit_before><?php

namespace Colligator;

use Colligator\Document;
use Colligator\Events\Marc21RecordImported;
use Colligator\Subject;
use Danmichaelo\QuiteSimpleXMLElement\QuiteSimpleXMLElement;
use Event;
use Scriptotek\SimpleMarcParser\BibliographicRecord;
use Scriptotek\SimpleMarcParser\HoldingsRecord;
use Scriptotek\SimpleMarcParser\Parser as MarcParser;
use Scriptotek\SimpleMarcParser\ParserException;

class Marc21Importer
{
    public $record;
    public $parser;

    /**
     * Create a new job instance.
     *
     * @param $record
     */
    public function __construct(MarcParser $parser = null, DescriptionScraper $scraper)
    {
        $this->parser = $parser ?: new MarcParser();
        $this->scraper = $scraper ?: new DescriptionScraper();
    }

    /**
     * Parse using SimpleMarcParser and separate bibliographic and holdings.
     *
     * @param QuiteSimpleXMLElement $data
     *
     * @return array
     */
    public function parseRecord(QuiteSimpleXMLElement $data)
    {
        $biblio = null;
        $holdings = array();
        foreach ($data->xpath('.//marc:record') as $rec) {
            $parsed = $this->parser->parse($rec);
            if ($parsed instanceof BibliographicRecord) {
                $biblio = $parsed->toArray();
            } elseif ($parsed instanceof HoldingsRecord) {
                $holdings[] = $parsed->toArray();
            }
        }

        return array($biblio, $holdings);
    }

    /**
     * @param array $biblio
     * @param array $holdings
     *
     * @return null|Document
     */
    public function importParsedRecord(array $biblio, array $holdings = [])
    {
        // Convert Carbon date objects to ISO8601 strings
        if (isset($biblio['created'])) {
            $biblio['created'] = $biblio['created']->toIso8601String();
        }
        if (isset($biblio['modified'])) {
            $biblio['modified'] = $biblio['modified']->toIso8601String();
        }
        foreach ($holdings as &$holding) {
            $holding['created'] = $holding['created']->toIso8601String();
            if (isset($holding['acquired'])) {
                $holding['acquired'] = $holding['acquired']->toIso8601String();
            }
        }

        // Find existing Document or create a new one
        $doc = Document::firstOrNew(['bibsys_id' => $biblio['id']]);

        // Update Document
        $doc->bibliographic = $biblio;
        $doc->holdings = $holdings;

        // Extract description from bibliographic record if no description exists
        if (isset($biblio['description']) && is_null($doc->description)) {
            $this->scraper->updateDocument($doc, $biblio['description']);
        }

        if (!$doc->save()) {
            $this->error("Document $biblio->id could not be saved!");

            return;
        }

        // Check other form
        $other_id = array_get($biblio, 'other_form.id');
        if (!empty($other_id)) {
            // TODO: Add a separate jobb that updates e-books weekly or so..
            $doc2 = Document::where('bibsys_id', '=', $other_id)->first();
            if (is_null($doc2)) {
                $record = \SruClient::first('bs.objektid=' . $other_id);
                \Log::debug('Importing related record ' . $other_id);
                if (is_null($record)) {
                    die("uh oh");
                } else {
                    $this->import($record->data);
                }
            }
        }

        // Sync subjects
        $subject_ids = [];
        foreach ($biblio['subjects'] as $value) {
            $subject = Subject::lookup($value['vocabulary'], $value['term']);
            if (is_null($subject)) {
                $subject = Subject::create($value);
            }
            $subject_ids[] = $subject->id;
        }
        $doc->subjects()->sync($subject_ids);

        // Sync genres
        $genre_ids = [];
        foreach ($biblio['genres'] as $value) {
            $genre = Genre::lookup($value['vocabulary'], $value['term']);
            if (is_null($genre)) {
                $genre = Genre::create($value);
            }
            $genre_ids[] = $genre->id;
        }
        $doc->genres()->sync($genre_ids);

        // Extract cover from bibliographic record if no local cover exists
        if (isset($biblio['cover_image']) && is_null($doc->cover)) {
            try {<commit_msg>Fix wrong vocabulary codes from Bibsys.<commit_after><?php

namespace Colligator;

use Colligator\Document;
use Colligator\Events\Marc21RecordImported;
use Colligator\Subject;
use Danmichaelo\QuiteSimpleXMLElement\QuiteSimpleXMLElement;
use Event;
use Scriptotek\SimpleMarcParser\BibliographicRecord;
use Scriptotek\SimpleMarcParser\HoldingsRecord;
use Scriptotek\SimpleMarcParser\Parser as MarcParser;
use Scriptotek\SimpleMarcParser\ParserException;

class Marc21Importer
{
    public $record;
    public $parser;

    /**
     * Create a new job instance.
     *
     * @param $record
     */
    public function __construct(MarcParser $parser = null, DescriptionScraper $scraper)
    {
        $this->parser = $parser ?: new MarcParser();
        $this->scraper = $scraper ?: new DescriptionScraper();
    }

    /**
     * Parse using SimpleMarcParser and separate bibliographic and holdings.
     *
     * @param QuiteSimpleXMLElement $data
     *
     * @return array
     */
    public function parseRecord(QuiteSimpleXMLElement $data)
    {
        $biblio = null;
        $holdings = array();
        foreach ($data->xpath('.//marc:record') as $rec) {
            $parsed = $this->parser->parse($rec);
            if ($parsed instanceof BibliographicRecord) {
                $biblio = $parsed->toArray();
            } elseif ($parsed instanceof HoldingsRecord) {
                $holdings[] = $parsed->toArray();
            }
        }

        return array($biblio, $holdings);
    }

    /**
     * Fixes wrong vocabulary codes returned by the Bibsys SRU service.
     * Bibsys strips dashes in 648, 650, 655, but not in 651, so we have
     * to do that ourselves. This affects 'no-ubo-mn' and 'no-ubo-mr'.
     */
    public function fixVocabularyCode($code)
    {
        return str_replace('-', '', $code);
    }

    /**
     * @param array $biblio
     * @param array $holdings
     *
     * @return null|Document
     */
    public function importParsedRecord(array $biblio, array $holdings = [])
    {
        // Convert Carbon date objects to ISO8601 strings
        if (isset($biblio['created'])) {
            $biblio['created'] = $biblio['created']->toIso8601String();
        }
        if (isset($biblio['modified'])) {
            $biblio['modified'] = $biblio['modified']->toIso8601String();
        }
        foreach ($holdings as &$holding) {
            $holding['created'] = $holding['created']->toIso8601String();
            if (isset($holding['acquired'])) {
                $holding['acquired'] = $holding['acquired']->toIso8601String();
            }
        }

        // Find existing Document or create a new one
        $doc = Document::firstOrNew(['bibsys_id' => $biblio['id']]);

        // Update Document
        $doc->bibliographic = $biblio;
        $doc->holdings = $holdings;

        // Extract description from bibliographic record if no description exists
        if (isset($biblio['description']) && is_null($doc->description)) {
            $this->scraper->updateDocument($doc, $biblio['description']);
        }

        if (!$doc->save()) {
            $this->error("Document $biblio->id could not be saved!");

            return;
        }

        // Check other form
        $other_id = array_get($biblio, 'other_form.id');
        if (!empty($other_id)) {
            // TODO: Add a separate jobb that updates e-books weekly or so..
            $doc2 = Document::where('bibsys_id', '=', $other_id)->first();
            if (is_null($doc2)) {
                $record = \SruClient::first('bs.objektid=' . $other_id);
                \Log::debug('Importing related record ' . $other_id);
                if (is_null($record)) {
                    die("uh oh");
                } else {
                    $this->import($record->data);
                }
            }
        }

        // Sync subjects
        $subject_ids = [];
        foreach ($biblio['subjects'] as $value) {
            $value['vocabulary'] = $this->fixVocabularyCode($value['vocabulary']);
            $subject = Subject::lookup($value['vocabulary'], $value['term']);
            if (is_null($subject)) {
                $subject = Subject::create($value);
            }
            $subject_ids[] = $subject->id;
        }
        $doc->subjects()->sync($subject_ids);

        // Sync genres
        $genre_ids = [];
        foreach ($biblio['genres'] as $value) {
            $genre = Genre::lookup($value['vocabulary'], $value['term']);
            if (is_null($genre)) {
                $genre = Genre::create($value);
            }
            $genre_ids[] = $genre->id;
        }
        $doc->genres()->sync($genre_ids);

        // Extract cover from bibliographic record if no local cover exists
        if (isset($biblio['cover_image']) && is_null($doc->cover)) {
            try {
{'fraction_non_alphanumeric': 0.14515253501248507, 'fraction_numerical': 0.006948214091846705, 'mean_word_length': 2.1322679360761647, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 64, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '316775', 'n_tokens_mistral': 2785, 'n_tokens_neox': 2619, 'n_words': 560}