<commit_before>module Score where

import Data.Char
import Data.List

-- TODO: test this properly
matchLength :: String -> String -> Int
matchLength [] _  =  1
matchLength _ []  =  0
matchLength (firstChar:rest) str = 
    case elemIndex firstChar str of
        Just startPos -> let endPos = endMatch rest (drop (startPos + 1) str) 
                                                    (startPos + 1)
                         in if endPos == 0 
                            then 0
                            else endPos - startPos
        Nothing -> 0
  where 
    endMatch [] _ lastIndex = lastIndex 
    endMatch (q:qs) s lastIndex = case elemIndex q s of
        Just i -> endMatch qs (drop (i + 1) s) (i + 1 + lastIndex)
        Nothing -> 0


minMatchLength :: String -> String -> Int
minMatchLength [] _ = 1
minMatchLength _ [] = 0
minMatchLength query@(q:_) choice = 
  let lengths = filter (>0) 
                $ map (matchLength query)
                $ filter (isPrefixOf [q])
                $ tails choice
  in if null lengths 
     then 0
     else minimum lengths

score :: String -> String -> Int
score q choice
    | null q      = 1
    | null choice = 0
    | otherwise = minMatchLength (map toLower q) (map toLower choice)<commit_msg>Finish implementing selecta's scoring algorithm.<commit_after>module Score where

import Data.Char
import Data.List

-- TODO: test this properly
matchLength :: String -> String -> Int
matchLength [] _  =  1
matchLength _ []  =  0
matchLength (firstChar:rest) str = 
    case elemIndex firstChar str of
        Just startPos -> let endPos = endMatch rest (drop (startPos + 1) str) 
                                                    (startPos + 1)
                         in if endPos == 0 
                            then 0
                            else endPos - startPos
        Nothing -> 0
  where 
    endMatch [] _ lastIndex = lastIndex 
    endMatch (q:qs) s lastIndex = case elemIndex q s of
        Just i -> endMatch qs (drop (i + 1) s) (i + 1 + lastIndex)
        Nothing -> 0


minMatchLength :: String -> String -> Int
minMatchLength [] _ = 1
minMatchLength _ [] = 0
minMatchLength query@(q:_) choice = 
  let lengths = filter (>0) 
                $ map (matchLength query)
                $ filter (isPrefixOf [q])
                $ tails choice
  in if null lengths 
     then 0
     else minimum lengths

score :: Fractional n => String -> String -> n
score q choice
    | null q      = 1
    | null choice = 0
    | otherwise = let minLength = minMatchLength (map toLower q) (map toLower choice)
                  in if minLength > 0 
                     then ((fromIntegral $ length q) / fromIntegral minLength) -- penalize bigger match lengths
                          / fromIntegral (length choice) -- penalize longer choice strings
                     else 0 
{'fraction_non_alphanumeric': 0.0812807881773399, 'fraction_numerical': 0.011963406052076003, 'mean_word_length': 1.6545284780578897, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6655281', 'n_tokens_mistral': 802, 'n_tokens_neox': 736, 'n_words': 288}