<commit_before><?php

namespace Simgroep\ConcurrentSpiderBundle\DocumentResolver\Type;

use Simgroep\ConcurrentSpiderBundle\DocumentResolver\Type\TypeAbstract;
use VDB\Spider\Resource;
use Simgroep\ConcurrentSpiderBundle\DocumentResolver\DocumentDataExtractor;
use Smalot\PdfParser\Parser;
use Simgroep\ConcurrentSpiderBundle\InvalidContentException;

/**
 * Pdf Resolver Document Type
 */
class Pdf extends TypeAbstract implements DocumentTypeInterface
{
    /**
     * @var \Smalot\PdfParser\Parser
     */
    private $pdfParser;

    /**
     * @param \Smalot\PdfParser\Parser $pdfParser
     * @param \VDB\Spider\Resource $resource
     */
    public function __construct(Parser $pdfParser)
    {
        $this->pdfParser = $pdfParser;
    }

    /**
     * Returns the filename of the resource. If not found it returns null.
     *
     * @param \VDB\Spider\Resource $resource
     *
     * @return string
     */
    protected function getFileNameFromResource(Resource $resource)
    {
        $regex = '/.*filename=[\'\"]([^\'\"]+)/';

        if (preg_match($regex, $resource->getResponse()->getContentDisposition(), $matches)) {
            return $matches['1'];
        }
    }

    /**
     * Extracts content from a PDF File and returns document data.
     *
     * @param \VDB\Spider\Resource $resource
     *
     * @return array
     *
     * @throws \Simgroep\ConcurrentSpiderBundle\InvalidContentException
     */
    public function getData(Resource $resource)
    {
        $content = $this->extractContentFromResource($resource);

        if (strlen($content) < self::MINIMAL_CONTENT_LENGTH) {
            throw new InvalidContentException(
                sprintf("PDF didn't contain enough content (minimal chars is %s)", self::MINIMAL_CONTENT_LENGTH)
            );
        }

        $dataExtractor = new DocumentDataExtractor($resource);

        $url = $dataExtractor->getUrl();
        $title = $this->getFileNameFromResource($resource);

        if (null === $title) {
            $title = $this->getTitleByUrl($url) ? : '';
        }

        $data = [
            'document' => [
                'id' => $dataExtractor->getId(),
                'url' => $url,
                'content' => $content,
                'strippedContent' => $content,
                'title' => $title,
                'tstamp' => date('Y-m-d\TH:i:s\Z'),
                'contentLength' => strlen($content),
                'lastModified' => $dataExtractor->getLastModified(),
                'date' => date('Y-m-d\TH:i:s\Z'),
                'publishedDate' => date('Y-m-d\TH:i:s\Z'),
                'SIM_archief' => $dataExtractor->getSimArchief(),
                'SIM.simfaq' => $dataExtractor->getSimfaq(),
                'type' => ['appliation/pdf']
            ],
        ];

        return $data;
    }

    /**
     *
     * Extract content from resource
     *
     * @param \VDB\Spider\Resource $resource
     *<commit_msg>Fixed comments.<commit_after><?php

namespace Simgroep\ConcurrentSpiderBundle\DocumentResolver\Type;

use Simgroep\ConcurrentSpiderBundle\DocumentResolver\Type\TypeAbstract;
use VDB\Spider\Resource;
use Simgroep\ConcurrentSpiderBundle\DocumentResolver\DocumentDataExtractor;
use Smalot\PdfParser\Parser;
use Simgroep\ConcurrentSpiderBundle\InvalidContentException;

/**
 * Pdf Resolver Document Type
 */
class Pdf extends TypeAbstract implements DocumentTypeInterface
{
    /**
     * @var \Smalot\PdfParser\Parser
     */
    private $pdfParser;

    /**
     * @param \Smalot\PdfParser\Parser $pdfParser
     * @param \VDB\Spider\Resource $resource
     */
    public function __construct(Parser $pdfParser)
    {
        $this->pdfParser = $pdfParser;
    }

    /**
     * Returns the filename of the resource. If not found it returns null.
     *
     * @param \VDB\Spider\Resource $resource
     *
     * @return string|null
     */
    protected function getFileNameFromResource(Resource $resource)
    {
        $regex = '/.*filename=[\'\"]([^\'\"]+)/';

        if (preg_match($regex, $resource->getResponse()->getContentDisposition(), $matches)) {
            return $matches['1'];
        }
    }

    /**
     * Extracts content from a PDF File and returns document data.
     *
     * @param \VDB\Spider\Resource $resource
     *
     * @return array
     *
     * @throws \Simgroep\ConcurrentSpiderBundle\InvalidContentException
     */
    public function getData(Resource $resource)
    {
        $content = $this->extractContentFromResource($resource);

        if (strlen($content) < self::MINIMAL_CONTENT_LENGTH) {
            throw new InvalidContentException(
                sprintf("PDF didn't contain enough content (minimal chars is %s)", self::MINIMAL_CONTENT_LENGTH)
            );
        }

        $dataExtractor = new DocumentDataExtractor($resource);

        $url = $dataExtractor->getUrl();
        $title = $this->getFileNameFromResource($resource);

        if (null === $title) {
            $title = $this->getTitleByUrl($url) ? : '';
        }

        $data = [
            'document' => [
                'id' => $dataExtractor->getId(),
                'url' => $url,
                'content' => $content,
                'strippedContent' => $content,
                'title' => $title,
                'tstamp' => date('Y-m-d\TH:i:s\Z'),
                'contentLength' => strlen($content),
                'lastModified' => $dataExtractor->getLastModified(),
                'date' => date('Y-m-d\TH:i:s\Z'),
                'publishedDate' => date('Y-m-d\TH:i:s\Z'),
                'SIM_archief' => $dataExtractor->getSimArchief(),
                'SIM.simfaq' => $dataExtractor->getSimfaq(),
                'type' => ['application/pdf']
            ],
        ];

        return $data;
    }

    /**
     *
     * Extract content from resource
     *
     * @param \VDB\Spider\Resource $resource
     *
{'fraction_non_alphanumeric': 0.13835198372329605, 'fraction_numerical': 0.00033909799932180403, 'mean_word_length': 2.176628971459343, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4095155', 'n_tokens_mistral': 1770, 'n_tokens_neox': 1709, 'n_words': 313}