<commit_before><?php

namespace AppBundle\Model\Logic\Collector;

use AppBundle\Exception\ParseException;
use AppBundle\Model\Logic\Parser\DateTime\DateTimeParserFactory;
use AppBundle\Request\AvitoRequest;
use AppBundle\Storage\FileStorage;
use Monolog\Logger;
use Schema\Parse\Record\Source;
use PHPHtmlParser\Dom;

class AvitoCollector implements CollectorInterface
{
    private $request;
    private $logger;
    private $storage;
    private $parser_datetime;
    private $last_hours;
    private $unique_ids;

    /**
     * AvitoCollector constructor.
     * @param AvitoRequest          $request
     * @param DateTimeParserFactory $parser_datetime_factory
     * @param Logger                $logger
     * @param string                $file_dir
     * @param int                   $last_hours
     */
    public function __construct(
        AvitoRequest $request,
        DateTimeParserFactory $parser_datetime_factory,
        Logger $logger,
        string $file_dir,
        int $last_hours
    )
    {
        $this->request         = $request;
        $this->logger          = $logger;
        $this->storage         = new FileStorage($file_dir);
        $this->parser_datetime = $parser_datetime_factory->init(Source::TYPE_AVITO);

        $this->last_hours = $last_hours;
        $this->unique_ids = [];
    }

    /**
     * @param Source $source
     * @return string
     */
    private function getConfigName(Source $source)
    {
        return 'config_' . $source->getId();
    }

    /**
     * @param Source $source
     * @return AvitoConfig
     */
    private function getConfigFromFile(Source $source)
    {
        $config_name = $this->getConfigName($source);

        $date       = (new \DateTime())->modify(sprintf('- %s hours', $this->last_hours));
        $new_config =
            (new AvitoConfig())
                ->setPage(1)
                ->setTimestamp($date->getTimestamp())
                ->setFinish(false);

        if (!$this->storage->exists($config_name)) {

            return $new_config;
        }

        $instance = $this->storage->get($config_name);

        $config = unserialize($instance);

        if (!($config instanceof AvitoConfig)) {

            return $new_config;
        }

        return $config;
    }

    /**
     * @param Source       $source
     * @param VkWallConfig $config
     * @return bool
     */
    private function setConfigToFile(Source $source, AvitoConfig $config)
    {
        $config_name = $this->getConfigName($source);

        return $this->storage->put($config_name, serialize($config));
    }

    /**
     * @param Source $source
     * @return array
     * @throws ParseException
     */
    public function collect(Source $source)
    {
        $this->logger->debug('Processing collect...', [
            'source_id'   => $source->getId(),
            'source_type' => $source->getType()
        ]);

        try {

            $config = $this->getConfigFromFile($source);

            if ($config->isFinish()) {
                $this->setConfigToFile($source, $config->setPage(1)->setFinish(false));

                $this->logger->debug('There are no more notes', [
                    'source_id'   => $source->getId(),
                    'source_type' => $source->getType()
                ]);

                return [];
            }

            usleep(1000000);

            $link_list = $source->getLink();

            $this->logger->debug('Collect list requesting...', [
                'source_id'   => $source->getId(),
                'source_type' => $source->getType(),
                'link'        => $link_list,
                'page'        => $config->getPage()
            ]);

            try {

                $response = $this->request->getList($link_list, $config->getPage());

            } catch (\Exception $e) {

                $this->logger->error('Collect list requesting... done', [
                    'source_id'   => $source->getId(),
                    'source_type' => $source->getType(),
                    'link'        => $link_list,
                    'page'        => $config->getPage()
                ]);

                $this->setConfigToFile($source,
                    $config
                        ->setPage(1)
                        ->setFinish(false)
                );

                return [];
            }

            $this->logger->debug('Collect list requesting... done', [
                'source_id'   => $source->getId(),
                'source_type' => $source->getType(),
                'link'        => $link_list,
                'page'        => $config->getPage()
            ]);

            $contents = $response->getBody()->getContents();

            $notes = $this->getLinks($source, $contents);

            $raws    = [];
            $finish  = false;

            $index = 0;
            foreach ($notes as $raw) {
                $index++;

                if ($index > 5) {
                    continue;
                }

                if ($raw->getTimestamp() < $config->getTimestamp()) {

                    $finish = true;

                    $this->logger->debug('Done by timestamp', [
                        'source_id'   => $source->getId(),
                        'source_type' => $source->getType(),
                    ]);

                    break;
                }
<commit_msg>avito user agent<commit_after><?php

namespace AppBundle\Model\Logic\Collector;

use AppBundle\Exception\ParseException;
use AppBundle\Model\Logic\Parser\DateTime\DateTimeParserFactory;
use AppBundle\Request\AvitoRequest;
use AppBundle\Storage\FileStorage;
use Monolog\Logger;
use Schema\Parse\Record\Source;
use PHPHtmlParser\Dom;

class AvitoCollector implements CollectorInterface
{
    private $request;
    private $logger;
    private $storage;
    private $parser_datetime;
    private $last_hours;
    private $unique_ids;

    /**
     * AvitoCollector constructor.
     * @param AvitoRequest          $request
     * @param DateTimeParserFactory $parser_datetime_factory
     * @param Logger                $logger
     * @param string                $file_dir
     * @param int                   $last_hours
     */
    public function __construct(
        AvitoRequest $request,
        DateTimeParserFactory $parser_datetime_factory,
        Logger $logger,
        string $file_dir,
        int $last_hours
    )
    {
        $this->request         = $request;
        $this->logger          = $logger;
        $this->storage         = new FileStorage($file_dir);
        $this->parser_datetime = $parser_datetime_factory->init(Source::TYPE_AVITO);

        $this->last_hours = $last_hours;
        $this->unique_ids = [];
    }

    /**
     * @param Source $source
     * @return string
     */
    private function getConfigName(Source $source)
    {
        return 'config_' . $source->getId();
    }

    /**
     * @param Source $source
     * @return AvitoConfig
     */
    private function getConfigFromFile(Source $source)
    {
        $config_name = $this->getConfigName($source);

        $date       = (new \DateTime())->modify(sprintf('- %s hours', $this->last_hours));
        $new_config =
            (new AvitoConfig())
                ->setPage(1)
                ->setTimestamp($date->getTimestamp())
                ->setFinish(false);

        if (!$this->storage->exists($config_name)) {

            return $new_config;
        }

        $instance = $this->storage->get($config_name);

        $config = unserialize($instance);

        if (!($config instanceof AvitoConfig)) {

            return $new_config;
        }

        return $config;
    }

    /**
     * @param Source       $source
     * @param VkWallConfig $config
     * @return bool
     */
    private function setConfigToFile(Source $source, AvitoConfig $config)
    {
        $config_name = $this->getConfigName($source);

        return $this->storage->put($config_name, serialize($config));
    }

    /**
     * @param Source $source
     * @return array
     * @throws ParseException
     */
    public function collect(Source $source)
    {
        $this->logger->debug('Processing collect...', [
            'source_id'   => $source->getId(),
            'source_type' => $source->getType()
        ]);

        try {

            $config = $this->getConfigFromFile($source);

            if ($config->isFinish()) {
                $this->setConfigToFile($source, $config->setPage(1)->setFinish(false));

                $this->logger->debug('There are no more notes', [
                    'source_id'   => $source->getId(),
                    'source_type' => $source->getType()
                ]);

                return [];
            }

            usleep(1000000);

            $link_list = $source->getLink();

            $this->logger->debug('Collect list requesting...', [
                'source_id'   => $source->getId(),
                'source_type' => $source->getType(),
                'link'        => $link_list,
                'page'        => $config->getPage()
            ]);

            try {

                $response = $this->request->getList($link_list, $config->getPage());

            } catch (\Exception $e) {

                $this->logger->error('Collect list requesting... done', [
                    'source_id'   => $source->getId(),
                    'source_type' => $source->getType(),
                    'link'        => $link_list,
                    'page'        => $config->getPage()
                ]);

                $this->setConfigToFile($source,
                    $config
                        ->setPage(1)
                        ->setFinish(false)
                );

                return [];
            }

            $this->logger->debug('Collect list requesting... done', [
                'source_id'   => $source->getId(),
                'source_type' => $source->getType(),
                'link'        => $link_list,
                'page'        => $config->getPage()
            ]);

            $contents = $response->getBody()->getContents();

            $notes = $this->getLinks($source, $contents);

            $raws    = [];
            $finish  = false;
            foreach ($notes as $raw) {

                if ($raw->getTimestamp() < $config->getTimestamp()) {

                    $finish = true;

                    $this->logger->debug('Done by timestamp', [
                        'source_id'   => $source->getId(),
                        'source_type' => $source->getType(),
                    ]);

                    break;
                }

{'fraction_non_alphanumeric': 0.1371610845295056, 'fraction_numerical': 0.0020639834881320948, 'mean_word_length': 1.5539051269765214, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 172, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7387609', 'n_tokens_mistral': 2942, 'n_tokens_neox': 2777, 'n_words': 517}