<commit_before><?php

namespace Ayamel\SearchBundle\Controller;

use Ayamel\ApiBundle\Controller\ApiController;
use Ayamel\SearchBundle\Model\Result;
use Ayamel\SearchBundle\Model\Query;
use Ayamel\SearchBundle\Model\Hit;
use Ayamel\SearchBundle\Model\Facet;
use Ayamel\SearchBundle\Model\FacetValue;
use Nelmio\ApiDocBundle\Annotation\ApiDoc;
use Symfony\Component\HttpFoundation\Request;
use JMS\Serializer\DeserializationContext;
use Elastica\Query\QueryString as ESQueryString;
use Elastica\Query as ESQuery;
use Elastica\Filter\Terms as TermsFilter;
use Elastica\Filter\Nested as NestedFilter;
use Elastica\Filter\Missing as MissingFilter;
use Elastica\Filter\BoolOr as BoolOrFilter;
use Elastica\Filter\BoolAnd as BoolAndFilter;
use Elastica\Facet\Terms as TermsFacet;

/**
 * Search controller for discovering Resources, which provides an simplified
 * GET API on top of ElasticSearch.
 *
 * @package AyamelSearchBundle
 */
class SearchV1 extends ApiController
{
    /**
     * Search for Resource objects based on a text query, filtering by criteria.  Searches include all publicly
     * available resources, as well as resources visible to the requesting client.  Searches may contain a variety
     * of filters and facets, described below.
     *
     * ### Filters ###
     *
     * When searching, you may filter results on certain fields.  All filters are specified in the format
     * `filter:<fieldName>=<values>`. Values can be passed as comma-delimited, to specify multiple allowed
     * values.  For example:
     *
     * * **?q=colorless%20green%20dreams&filter:type=video** - will return matches only where
     *     the resource is of type `video`
     * * **?q=colorless%20green%20dreams&filter:type=video,audio** - will return matches where
     *     the resource is either video, or audio
     *
     * For fields that can contain multiple values, such as `subjectDomains`, there are additional ways to specify a filter.
     *
     * * **?q=colorless%20green%20dreams&filter:subjectDomains=language,science** - will return matches where the resource contains either "language" or "science", or possibly both, as one of the values.
     * * **?q=colorless%20green%20dreams&filter:subjectDomains[]=language&filter:subjectDomains[]=science** - will contain matches where `subjectDomains` contains *both* "language" and "science".
     *
     * Any field that contains multiple values can be passed as an array shown above to specify an "AND" requirement.  Otherwise, a
     * comma-delimited list of values is interpreted as an "OR" requirement.
     *
     * ### Facets ###
     *
     * Facets allow you see the count of potential hits containing values for certain fields.  Facets are available for the same
     * fields that you are allowed to filter on, and are specified in the format `facet:<fieldName>`, or `facet:<fieldName>=<size>`.
     * By default, facets will contain a max of 10 values.  You may increase the size by specifying a larger facet size.  Also, note
     * that facet values are always ordered by the highest number of hits first.  So, if a particular facet potentially contains
     * many values, you will need to increase the size of the facet to see values with lower counts.  A few examples:
     *
     * * **?q=colorless%20green%20dreams&facet:type** - show the type facet
     * * **?q=colorless%20green%20dreams&facet:subjectDomains=20** - show the subject domains, including value counts for the top 20 most used values
     *
     * @ApiDoc(
     *      resource=true,
     *      description="Search for resources",
     *      output="Ayamel\SearchBundle\Model\Result",
     *      filters={
     *          {"name"="q", "required"="yes", "description"="The text to search on."},
     *          {"name"="limit", "default"="10", "description"="How many results to return.  Max 100."},
     *          {"name"="skip", "default"="0", "description"="Which result to start at. This in combination with `limit` can be used for paginating results.  Max 1000."},
     *          {"name"="filter:type", "description"="comma-delimited list of Resource types."},
     *          {"name"="filter:client", "description"="**Not yet implemented** Comma-delimited list of API Client ids."},
     *          {"name"="filter:clientUser", "description"="**Not yet implemented** Comma-delimited list of API Client User ids."},
     *          {"name"="filter:language", "description"="**Not yet implemented** Comma-delimited list of langauge codes.  Can be specified as an array."},
     *          {"name"="filter:subjectDomains", "description"="Comma-delimited list of subject domains.  Can be specified as an array."},
     *          {"name"="filter:functionalDomains", "description"="Comma-delimited list of functional domains.  Can be specified as an array."},
     *          {"name"="filter:registers", "description"="Comma-delimited list of registers.  Can be specified as an array."},
     *          {"name"="facet:type", "description"="Include facet for type."},
     *          {"name"="facet:client", "description"="**Not yet implemented** Include facet for client."},
     *          {"name"="facet:clientUser", "description"="**Not yet implemented** Include facet for client."},
     *          {"name"="facet:language", "description"="**Not yet implemented** Include facet for language."},
     *          {"name"="facet:subjectDomains", "description"="Include facet for subject domains."},
     *          {"name"="facet:functionalDomains", "description"="Include facet for functional domains."},
     *          {"name"="facet:registers", "description"="Include facet for registers."}
     *      }
     * )
     */
    public function searchAction(Request $request)
    {
        $q = $request->query;
        if (!$queryText = $q->get('q', false)) {
            throw $this->createHttpException(400, "Searches must include a string query via the [q] parameter.");
        }

        //create an Elastica query, and set the text to query
        $query = new ESQuery();
        $queryString = new ESQueryString();
        $queryString->setQuery($queryText);
        $queryString->setDefaultOperator('AND');
        $query->setQuery($queryString);

        //limit and skip, with some internally enforced ranges
        $limit = ($l = $q->get('limit', 20)) > 100 ? 100 : (int) $l;
        $skip = ($s = $q->get('skip', 0)) > 1000 ? 1000 : (int) $s;
        $query->setFrom($skip);
        $query->setLimit($limit);

        //TODO: nested field filters
        //  * resource.client.id
        //  * resource.clientUser.id
        //  * resource.language
        //    * languages.iso639_3 OR langauges.bcp47

        //create query filters, always enforcing a visibility filter
        $queryFilters = [$this->createVisibilityFilter()];

        //TODO: enforce filters derived from future AuthorizationPolicy

        if ($filterValue = $q->get('filter:type', false)) {
            $queryFilters[] = new TermsFilter('type', explode(',', strtolower($filterValue)));
        }
        if ($filterValue = $q->get('filter:subjectDomains', false)) {
            foreach ((array) $filterValue as $val) {
                $queryFilters[] = new TermsFilter('subjectDomains', explode(',', strtolower($val)));
            }
        }
        if ($filterValue = $q->get('filter:functionalDomains', false)) {
            foreach ((array) $filterValue as $val) {
                $queryFilters[] = new TermsFilter('functionalDomains', explode(',', strtolower($val)));
            }
        }
        if ($filterValue = $q->get('filter:registers', false)) {
            foreach ((array) $filterValue as $val) {
                $queryFilters[] = new TermsFilter('registers', explode(',', strtolower($val)));
            }
        }
        if ($filterValue = $q->get('filter:client', false)) {
            //TODO
            // $f = new NestedFilter();
            // $f->setPath('client');
            // $f->setFilter(new TermsFilter('client.id', explode(',', strtolower($filterValue))));
            // $queryFilters[] = $f;
        }
        if ($filterValue = $q->get('clientUser', false)) {
            //TODO
        }
        if ($filterValue = $q->get('filter:language', false)) {
            //TODO
            // $filterValue = explode(',', strtolower($filterValue));
            // $queryFilters[] = (new BoolOrFilter())
            //     ->addFilter(new TermsFilter('languages.iso639_3', $langs))
            //     ->addFilter(new TermsFilter('languages.bcp47', $langs))
            // ;
        }

        //add all the filters to the query
        $queryFilter = (new BoolAndFilter())->setFilters($queryFilters);
        $query->setFilter($queryFilter);

        //TODO: nested field facets
        //  * resource.client.id
        //  * resource.clientUser.id
        //  * resource.language ... how to do this, really?
        $queryFacets = [];
        if ($q->has('facet:type')) {
            $queryFacets[] = $this->createFacet('type', $q->get('facet:type', false));
        }
        if ($q->has('facet:subjectDomains')) {
            $queryFacets[] = $this->createFacet('subjectDomains', $q->get('facet:subjectDomains', false));
        }
        if ($q->has('facet:functionalDomains')) {
            $queryFacets[] = $this->createFacet('functionalDomains', $q->get('facet:functionalDomains', false));
        }
        if ($q->has('facet:registers')) {
            $queryFacets[] = $this->createFacet('registers', $q->get('facet:registers', false));
        }

        //add all the facets to the query
        foreach ($queryFacets as $facet) {
            //the main query filter also needs to apply to any facets used
            $facet->setFilter($queryFilter);
            $query->addFacet($facet);
        }

        //perform query
        $type = $this->container->get('fos_elastica.index.ayamel')->getType('resource');
        $resultSet = $type->search($query);

        //transform raw result into API result
        $result = Result::createFromArray([
            'query' => Query::createFromArray([
                'limit' => $limit,
                'skip' => $skip,
                'total' => $resultSet->getTotalHits(),
                'time' => $resultSet->getTotalTime()
            ]),
            'hits' => $this->filterResults($resultSet->getResults()),
            'facets' => $this->filterFacets($resultSet->getFacets())
        ]);

        return $this->createServiceResponse(['result' => $result], 200);<commit_msg>update Search Controller to include new facets and filters<commit_after><?php

namespace Ayamel\SearchBundle\Controller;

use Ayamel\ApiBundle\Controller\ApiController;
use Ayamel\SearchBundle\Model\Result;
use Ayamel\SearchBundle\Model\Query;
use Ayamel\SearchBundle\Model\Hit;
use Ayamel\SearchBundle\Model\Facet;
use Ayamel\SearchBundle\Model\FacetValue;
use Nelmio\ApiDocBundle\Annotation\ApiDoc;
use Symfony\Component\HttpFoundation\Request;
use JMS\Serializer\DeserializationContext;
use Elastica\Query\QueryString as ESQueryString;
use Elastica\Query as ESQuery;
use Elastica\Filter\Terms as TermsFilter;
use Elastica\Filter\Nested as NestedFilter;
use Elastica\Filter\Missing as MissingFilter;
use Elastica\Filter\BoolOr as BoolOrFilter;
use Elastica\Filter\BoolAnd as BoolAndFilter;
use Elastica\Facet\Terms as TermsFacet;

/**
 * Search controller for discovering Resources, which provides an simplified
 * GET API on top of ElasticSearch.
 *
 * @package AyamelSearchBundle
 */
class SearchV1 extends ApiController
{
    /**
     * Search for Resource objects based on a text query, filtering by criteria.  Searches include all publicly
     * available resources, as well as resources visible to the requesting client.  Searches may contain a variety
     * of filters and facets, described below.
     *
     * ### Filters ###
     *
     * When searching, you may filter results on certain fields.  All filters are specified in the format
     * `filter:<fieldName>=<values>`. Values can be passed as comma-delimited, to specify multiple allowed
     * values.  For example:
     *
     * * **?q=colorless%20green%20dreams&filter:type=video** - will return matches only where
     *     the resource is of type `video`
     * * **?q=colorless%20green%20dreams&filter:type=video,audio** - will return matches where
     *     the resource is either video, or audio
     *
     * For fields that can contain multiple values, such as `topics`, there are additional ways to specify a filter.
     *
     * * **?q=colorless%20green%20dreams&filter:topics=language,science** - will return matches where the resource contains either "language" or "science", or possibly both, as one of the values.
     * * **?q=colorless%20green%20dreams&filter:topics[]=language&filter:topics[]=science** - will contain matches where `topics` contains *both* "language" and "science".
     *
     * Any field that contains multiple values can be passed as an array shown above to specify an "AND" requirement.  Otherwise, a
     * comma-delimited list of values is interpreted as an "OR" requirement.
     *
     * ### Facets ###
     *
     * Facets allow you see the count of potential hits containing values for certain fields.  Facets are available for the same
     * fields that you are allowed to filter on, and are specified in the format `facet:<fieldName>`, or `facet:<fieldName>=<size>`.
     * By default, facets will contain a max of 10 values.  You may increase the size by specifying a larger facet size.  Also, note
     * that facet values are always ordered by the highest number of hits first.  So, if a particular facet potentially contains
     * many values, you will need to increase the size of the facet to see values with lower counts.  A few examples:
     *
     * * **?q=colorless%20green%20dreams&facet:type** - show the type facet
     * * **?q=colorless%20green%20dreams&facet:topics=20** - show the topics, including value counts for the top 20 most used values
     *
     * @ApiDoc(
     *      resource=true,
     *      description="Search for resources",
     *      output="Ayamel\SearchBundle\Model\Result",
     *      filters={
     *          {"name"="q", "required"="yes", "description"="The text to search on."},
     *          {"name"="limit", "default"="10", "description"="How many results to return.  Max 100."},
     *          {"name"="skip", "default"="0", "description"="Which result to start at. This in combination with `limit` can be used for paginating results.  Max 1000."},
     *          {"name"="filter:type", "description"="comma-delimited list of Resource types."},
     *          {"name"="filter:client", "description"="**Not yet implemented** Comma-delimited list of API Client ids."},
     *          {"name"="filter:clientUser", "description"="**Not yet implemented** Comma-delimited list of API Client User ids."},
     *          {"name"="filter:language", "description"="**Not yet implemented** Comma-delimited list of langauge codes.  Can be specified as an array."},
     *          {"name"="filter:topics", "description"="Comma-delimited list of topics.  Can be specified as an array."},
     *          {"name"="filter:functions", "description"="Comma-delimited list of functions.  Can be specified as an array."},
     *          {"name"="filter:registers", "description"="Comma-delimited list of registers.  Can be specified as an array."},
     *          {"name"="facet:type", "description"="Include facet for type."},
     *          {"name"="facet:client", "description"="**Not yet implemented** Include facet for client."},
     *          {"name"="facet:clientUser", "description"="**Not yet implemented** Include facet for client."},
     *          {"name"="facet:language", "description"="**Not yet implemented** Include facet for language."},
     *          {"name"="facet:topics", "description"="Include facet for topics."},
     *          {"name"="facet:functions", "description"="Include facet for functions."},
     *          {"name"="facet:registers", "description"="Include facet for registers."}
     *      }
     * )
     */
    public function searchAction(Request $request)
    {
        $q = $request->query;
        if (!$queryText = $q->get('q', false)) {
            throw $this->createHttpException(400, "Searches must include a string query via the [q] parameter.");
        }

        //create an Elastica query, and set the text to query
        $query = new ESQuery();
        $queryString = new ESQueryString();
        $queryString->setQuery($queryText);
        $queryString->setDefaultOperator('AND');
        $query->setQuery($queryString);

        //limit and skip, with some internally enforced ranges
        $limit = ($l = $q->get('limit', 20)) > 100 ? 100 : (int) $l;
        $skip = ($s = $q->get('skip', 0)) > 1000 ? 1000 : (int) $s;
        $query->setFrom($skip);
        $query->setLimit($limit);

        //TODO: nested field filters
        //  * resource.client.id
        //  * resource.clientUser.id
        //  * resource.language
        //    * languages.iso639_3 OR langauges.bcp47

        //create query filters, always enforcing a visibility filter
        $queryFilters = [$this->createVisibilityFilter()];

        //TODO: enforce filters derived from future AuthorizationPolicy

        if ($filterValue = $q->get('filter:type', false)) {
            $queryFilters[] = new TermsFilter('type', explode(',', strtolower($filterValue)));
        }
        if ($filterValue = $q->get('filter:topics', false)) {
            foreach ((array) $filterValue as $val) {
                $queryFilters[] = new TermsFilter('topics', explode(',', strtolower($val)));
            }
        }
        if ($filterValue = $q->get('filter:functions', false)) {
            foreach ((array) $filterValue as $val) {
                $queryFilters[] = new TermsFilter('functions', explode(',', strtolower($val)));
            }
        }
        if ($filterValue = $q->get('filter:registers', false)) {
            foreach ((array) $filterValue as $val) {
                $queryFilters[] = new TermsFilter('registers', explode(',', strtolower($val)));
            }
        }
        if ($filterValue = $q->get('filter:authenticity', false)) {
            foreach ((array) $filterValue as $val) {
                $queryFilters[] = new TermsFilter('authenticity', explode(',', strtolower($val)));
            }
        }
        if ($filterValue = $q->get('filter:format', false)) {
            foreach ((array) $filterValue as $val) {
                $queryFilters[] = new TermsFilter('format', explode(',', strtolower($val)));
            }
        }
        if ($filterValue = $q->get('filter:genres', false)) {
            foreach ((array) $filterValue as $val) {
                $queryFilters[] = new TermsFilter('genres', explode(',', strtolower($val)));
            }
        }
        if ($filterValue = $q->get('filter:client', false)) {
            //TODO
            // $f = new NestedFilter();
            // $f->setPath('client');
            // $f->setFilter(new TermsFilter('client.id', explode(',', strtolower($filterValue))));
            // $queryFilters[] = $f;
        }
        if ($filterValue = $q->get('clientUser', false)) {
            //TODO
        }
        if ($filterValue = $q->get('filter:language', false)) {
            //TODO
            // $filterValue = explode(',', strtolower($filterValue));
            // $queryFilters[] = (new BoolOrFilter())
            //     ->addFilter(new TermsFilter('languages.iso639_3', $langs))
            //     ->addFilter(new TermsFilter('languages.bcp47', $langs))
            // ;
        }

        //add all the filters to the query
        $queryFilter = (new BoolAndFilter())->setFilters($queryFilters);
        $query->setFilter($queryFilter);

        //TODO: nested field facets
        //  * resource.client.id
        //  * resource.clientUser.id
        //  * resource.language ... how to do this, really?
        $queryFacets = [];
        if ($q->has('facet:type')) {
            $queryFacets[] = $this->createFacet('type', $q->get('facet:type', false));
        }
        if ($q->has('facet:topics')) {
            $queryFacets[] = $this->createFacet('topics', $q->get('facet:topics', false));
        }
        if ($q->has('facet:functions')) {
            $queryFacets[] = $this->createFacet('functions', $q->get('facet:functions', false));
        }
        if ($q->has('facet:registers')) {
            $queryFacets[] = $this->createFacet('registers', $q->get('facet:registers', false));
        }
        if ($q->has('facet:formats')) {
            $queryFacets[] = $this->createFacet('formats', $q->get('facet:formats', false));
        }
        if ($q->has('facet:genres')) {
            $queryFacets[] = $this->createFacet('genres', $q->get('facet:genres', false));
        }
        if ($q->has('facet:authenticity')) {
            $queryFacets[] = $this->createFacet('authenticity', $q->get('facet:authenticity', false));
        }
        //add all the facets to the query
        foreach ($queryFacets as $facet) {
            //the main query filter also needs to apply to any facets used
            $facet->setFilter($queryFilter);
            $query->addFacet($facet);
        }

        //perform query
        $type = $this->container->get('fos_elastica.index.ayamel')->getType('resource');
        $resultSet = $type->search($query);

        //transform raw result into API result
        $result = Result::createFromArray([
            'query' => Query::createFromArray([
                'limit' => $limit,
                'skip' => $skip,
                'total' => $resultSet->getTotalHits(),
                'time' => $resultSet->getTotalTime()
            ]),
            'hits' => $this->filterResults($resultSet->getResults()),
            'facets' => $this->filterFacets($resultSet->getFacets())
        ]);

        return $this->createServiceResponse(['result' => $result], 200);
{'fraction_non_alphanumeric': 0.14473803686552256, 'fraction_numerical': 0.0069009352583310635, 'mean_word_length': 2.728334461746784, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 143, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2551438', 'n_tokens_mistral': 6387, 'n_tokens_neox': 6132, 'n_words': 1706}