<commit_before>module ElasticSearch exposing (..)

import Http
import Decoder exposing (buildRecordDecoder)
import Json.Decode as Decode
import Json.Encode as Encode
import Types exposing (..)


type ClauseKind
    = Match
    | Prefix
    | Term


clauseName : ClauseKind -> String
clauseName kind =
    case kind of
        Prefix ->
            "prefix"

        Match ->
            "match"

        _ ->
            "term"


encodeClause : ClauseKind -> String -> String -> Encode.Value
encodeClause kind name value =
    Encode.object
        [ ( clauseName kind
          , Encode.object [ ( name, Encode.string value ) ]
          )
        ]


extractClauses : ClauseKind -> String -> List String -> Maybe Encode.Value
extractClauses kind field values =
    case values of
        [] ->
            Nothing

        [ "" ] ->
            Nothing

        [ value ] ->
            -- Only this value
            Just <|
                Encode.object
                    [ ( "bool"
                      , Encode.object
                            [ ( "must", encodeClause kind field value ) ]
                      )
                    ]

        values ->
            -- Any of those values
            Just <|
                Encode.object
                    [ ( "bool"
                      , Encode.object
                            [ ( "should"
                              , values
                                    |> List.map (encodeClause kind field)
                                    |> Encode.list
                              )
                            ]
                      )
                    ]


encodeAggregate : String -> List (Maybe Encode.Value) -> ( String, Encode.Value )
encodeAggregate field clauses =
    ( field
    , Encode.object
        [ ( "filter", encodeFilter clauses )
        , ( "aggs"
          , Encode.object
                [ ( field ++ "_agg"
                  , Encode.object
                        [ ( "terms"
                          , Encode.object
                                [ ( "field", Encode.string field )
                                , ( "size", Encode.int 1000 )
                                ]
                          )
                        ]
                  )
                ]
          )
        ]
    )


encodeFilter : List (Maybe Encode.Value) -> Encode.Value
encodeFilter clauses =
    Encode.object
        [ ( "bool"
          , Encode.object
                [ ( "must"
                  , clauses
                        |> List.filterMap identity
                        |> Encode.list
                  )
                ]
          )
        ]


encodeQuery : Filters -> Int -> Encode.Value
encodeQuery { page, product, channel, locale, version, platform, buildId } pageSize =
    let
        productClauses =
            extractClauses Term "source.product" product

        channelClauses =
            extractClauses Term "target.channel" channel

        versionClauses =
            extractClauses Term "target.version" version

        localeClauses =
            extractClauses Term "target.locale" locale

        platformClauses =
            extractClauses Term "target.platform" platform

        buildIdClauses =
            extractClauses Prefix "build.id" [ buildId ]
    in
        Encode.object
            [ ( "size", Encode.int pageSize )
            , ( "from", Encode.int <| (page - 1) * pageSize )
            , ( "sort", Encode.list [ Encode.object [ ( "download.date", Encode.string "desc" ) ] ] )<commit_msg>Polish code.<commit_after>module ElasticSearch exposing (..)

import Http
import Decoder exposing (buildRecordDecoder)
import Json.Decode as Decode
import Json.Encode as Encode
import Types exposing (..)


type ClauseKind
    = Match
    | Prefix
    | Term


type alias EncodedAggregate =
    Encode.Value


type alias EncodedClause =
    Encode.Value


type alias EncodedFilter =
    Encode.Value


type alias EncodedQuery =
    Encode.Value


clauseName : ClauseKind -> String
clauseName kind =
    case kind of
        Prefix ->
            "prefix"

        Match ->
            "match"

        _ ->
            "term"


encodeClause : ClauseKind -> String -> String -> EncodedClause
encodeClause kind name value =
    Encode.object
        [ ( clauseName kind
          , Encode.object [ ( name, Encode.string value ) ]
          )
        ]


extractClauses : ClauseKind -> String -> List String -> Maybe EncodedClause
extractClauses kind field values =
    case (List.filter (not << String.isEmpty) values) of
        [] ->
            Nothing

        [ value ] ->
            -- Only this value
            Just <|
                Encode.object
                    [ ( "bool"
                      , Encode.object
                            [ ( "must", encodeClause kind field value ) ]
                      )
                    ]

        values ->
            -- Any of those values
            Just <|
                Encode.object
                    [ ( "bool"
                      , Encode.object
                            [ ( "should"
                              , values
                                    |> List.map (encodeClause kind field)
                                    |> Encode.list
                              )
                            ]
                      )
                    ]


encodeAggregate : String -> List (Maybe EncodedClause) -> ( String, EncodedAggregate )
encodeAggregate field clauses =
    ( field
    , Encode.object
        [ ( "filter", encodeFilter clauses )
        , ( "aggs"
          , Encode.object
                [ ( field ++ "_agg"
                  , Encode.object
                        [ ( "terms"
                          , Encode.object
                                [ ( "field", Encode.string field )
                                , ( "size", Encode.int 1000 )
                                ]
                          )
                        ]
                  )
                ]
          )
        ]
    )


encodeFilter : List (Maybe EncodedClause) -> EncodedFilter
encodeFilter clauses =
    Encode.object
        [ ( "bool"
          , Encode.object
                [ ( "must"
                  , clauses
                        |> List.filterMap identity
                        |> Encode.list
                  )
                ]
          )
        ]


encodeQuery : Filters -> Int -> EncodedQuery
encodeQuery { page, product, channel, locale, version, platform, buildId } pageSize =
    let
        productClauses =
            extractClauses Term "source.product" product

        channelClauses =
            extractClauses Term "target.channel" channel

        versionClauses =
            extractClauses Term "target.version" version

        localeClauses =
            extractClauses Term "target.locale" locale

        platformClauses =
            extractClauses Term "target.platform" platform

        buildIdClauses =
            extractClauses Prefix "build.id" [ buildId ]
    in
        Encode.object
            [ ( "size", Encode.int pageSize )
            , ( "from", Encode.int <| (page - 1) * pageSize )
            , ( "sort", Encode.list [ Encode.object [ ( "download.date", Encode.string "desc" ) ] ] )
{'fraction_non_alphanumeric': 0.0846968238691049, 'fraction_numerical': 0.0013749484394335212, 'mean_word_length': 1.0872309899569583, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2454320', 'n_tokens_mistral': 1862, 'n_tokens_neox': 1731, 'n_words': 423}