<commit_before><?php


namespace CultuurNet\UDB3\Event;

use Broadway\CommandHandling\CommandHandler;
use Broadway\Repository\RepositoryInterface;
use CultuurNet\UDB3\Keyword;
use CultuurNet\UDB3\Search\SearchServiceInterface;
use Psr\Log\LoggerAwareInterface;
use Psr\Log\LoggerAwareTrait;

class EventCommandHandler extends CommandHandler implements LoggerAwareInterface
{
    use LoggerAwareTrait;

    /**
     * @var RepositoryInterface
     */
    protected $eventRepository;

    /**
     * @var SearchServiceInterface
     */
    protected $searchService;


    public function __construct(
      RepositoryInterface $eventRepository,
      SearchServiceInterface $searchService)
    {
        $this->eventRepository = $eventRepository;
        $this->searchService = $searchService;
    }

    public function handleTagEvents(TagEvents $tagEvents)
    {
        foreach ($tagEvents->getEventIds() as $eventId) {
            $this->tagEvent($tagEvents->getKeyword(), $eventId);
        }
    }

    public function handleTagQuery(TagQuery $tagQuery)
    {
        $query = $tagQuery->getQuery();
        $totalItemCount = 0;

        // do a pre query to test if the query is valid and check the item count
        try {
            $preQueryResult = $this->searchService->search($query, 1, 0);
            $totalItemCount = $preQueryResult['totalItems'];
        }
        catch (\Guzzle\Http\Exception\ClientErrorResponseException $e) {
            if ($this->logger) {
                $this->logger->error(
                  'query_was_not_tagged',
                  array(
                    'query' => $query,
                    'error' => $e->getMessage(),
                    'exception_class' => get_class($e),
                  )
                );
            }
        }

        if($totalItemCount < 1) {
            if ($this->logger) {
                $this->logger->error(
                  'query_was_not_tagged',
                  array(
                    'query' => $query,
                    'error' => "query did not return any results"
                  )
                );
            }
        } else {
            // change this pageSize value to increase or decrease the page size;
            $pageSize = 10;
            $allResults = [];
            $pageCount =  ceil($totalItemCount / $pageSize);
            $pageCounter = 0;

            //Page querying the search service;
            while($pageCounter < $pageCount) {
                $results = $this->searchService->search($query, $pageSize, ($pageCounter * $pageSize));
                $allResults = array_merge_recursive($allResults, $results);
                ++$pageCounter;
            };

            // Iterate all the results and get their IDs
            // by stripping them from the json-LD representation
            $eventIds = [];
            foreach($allResults['member'] as $event) {
                $expoId = explode('/',$event['@id']);
                $eventIds[] = array_pop($expoId);
            }

            // Use the full list of IDs to tag all the events at once
            $this->tagEventsById($eventIds, $tagQuery->getKeyword());
        };
    }

    /**
     * Tags a single event with a keyword.
     *
     * @param Keyword $keyword
     * @param $eventId
     */
    private function tagEvent(Keyword $keyword, $eventId)
    {
        /** @var Event $event */
        $event = $this->eventRepository->load($eventId);
        $event->tag($keyword);
        try {
            $this->eventRepository->add($event);
<commit_msg>iii-61: Tag events immediately while retrieving paged result sets, order by creation date, log duplicates just in case<commit_after><?php


namespace CultuurNet\UDB3\Event;

use Broadway\CommandHandling\CommandHandler;
use Broadway\Repository\RepositoryInterface;
use CultuurNet\UDB3\Keyword;
use CultuurNet\UDB3\Search\SearchServiceInterface;
use Psr\Log\LoggerAwareInterface;
use Psr\Log\LoggerAwareTrait;

class EventCommandHandler extends CommandHandler implements LoggerAwareInterface
{
    use LoggerAwareTrait;

    /**
     * @var RepositoryInterface
     */
    protected $eventRepository;

    /**
     * @var SearchServiceInterface
     */
    protected $searchService;


    public function __construct(
      RepositoryInterface $eventRepository,
      SearchServiceInterface $searchService)
    {
        $this->eventRepository = $eventRepository;
        $this->searchService = $searchService;
    }

    public function handleTagEvents(TagEvents $tagEvents)
    {
        foreach ($tagEvents->getEventIds() as $eventId) {
            $this->tagEvent($tagEvents->getKeyword(), $eventId);
        }
    }

    public function handleTagQuery(TagQuery $tagQuery)
    {
        $query = $tagQuery->getQuery();
        $totalItemCount = 0;

        // do a pre query to test if the query is valid and check the item count
        try {
            $preQueryResult = $this->searchService->search($query, 1, 0);
            $totalItemCount = $preQueryResult['totalItems'];
        }
        catch (\Guzzle\Http\Exception\ClientErrorResponseException $e) {
            if ($this->logger) {
                $this->logger->error(
                  'query_was_not_tagged',
                  array(
                    'query' => $query,
                    'error' => $e->getMessage(),
                    'exception_class' => get_class($e),
                  )
                );
            }
        }

        if($totalItemCount < 1) {
            if ($this->logger) {
                $this->logger->error(
                  'query_was_not_tagged',
                  array(
                    'query' => $query,
                    'error' => "query did not return any results"
                  )
                );
            }
        } else {
            // change this pageSize value to increase or decrease the page size;
            $pageSize = 10;
            $pageCount =  ceil($totalItemCount / $pageSize);
            $pageCounter = 0;
            $taggedEventIds = [];

            //Page querying the search service;
            while ($pageCounter < $pageCount) {
                $start = $pageCounter * $pageSize;
                // Sort ascending by creation date to make sure we get a quite consistent paging.
                $sort = 'creationdate asc';
                $results = $this->searchService->search($query, $pageSize, $start, $sort);

                // Iterate the results of the current page and get their IDs
                // by stripping them from the json-LD representation
                foreach ($results['member'] as $event) {
                    $expoId = explode('/',$event['@id']);
                    $eventId = array_pop($expoId);

                    if (!array_key_exists($eventId, $taggedEventIds)) {
                        $taggedEventIds[$eventId] = $pageCounter;

                        $this->tagEvent($tagQuery->getKeyword(), $eventId);
                    }
                    else {
                        if ($this->logger) {
                            $this->logger->error(
                                'query_duplicate_event',
                                array(
                                    'query' => $query,
                                    'error' => "found duplicate event {$eventId} on page {$pageCounter}, occurred first time on page {$taggedEventIds[$eventId]}"
                                )
                            );
                        }
                    }
                }
                ++$pageCounter;
            };
        };
    }

    /**
     * Tags a single event with a keyword.
     *
     * @param Keyword $keyword
     * @param $eventId
     */
    private function tagEvent(Keyword $keyword, $eventId)
    {
        /** @var Event $event */
        $event = $this->eventRepository->load($eventId);
        $event->tag($keyword);
        try {
            $this->eventRepository->add($event);

{'fraction_non_alphanumeric': 0.10967661067936826, 'fraction_numerical': 0.0027575833542241163, 'mean_word_length': 1.6342027071640806, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 66, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '649050', 'n_tokens_mistral': 2120, 'n_tokens_neox': 2005, 'n_words': 476}