<commit_before>{-# LANGUAGE DataKinds, TypeFamilies, UndecidableInstances, TypeOperators #-}

--------------------------------------------------------------------------------
-- |
-- Module      :  Data.Comp.SubsumeCommon
-- Copyright   :  (c) 2014 <NAME>
-- License     :  BSD3
-- Maintainer  :  <NAME> <<EMAIL>>
-- Stability   :  experimental
-- Portability :  non-portable (GHC Extensions)
--
-- Shared parts of the implementation of signature subsumption for
-- both the base and the multi library.
--
--------------------------------------------------------------------------------

module Data.Comp.SubsumeCommon
    ( ComprEmb
    , Pos (..)
    , Emb (..)
    ) where

-- | This type is used in its promoted form only. It represents
-- pointers from the left-hand side of a subsumption to the right-hand
-- side.
data Pos = Here | Le Pos | Ri Pos | Sum Pos Pos

-- | This type is used in its promoted form only. It represents
-- possible results for checking for subsumptions. 'Found' indicates a
-- subsumption was found; 'NotFound' indicates no such subsumption was
-- found. 'Ambiguous' indicates that there are duplicates on the left-
-- or the right-hand side.
data Emb = Found Pos | NotFound | Ambiguous

-- | This type family takes a position type and compresses it. That
-- means it replaces each nested occurrence of 
-- 
-- @
--   Sum (prefix (Le Here)) (prefix (Ri Here))@
-- @
---
-- with 
-- 
-- @
--   prefix Here@
-- @
-- 
-- where @prefix@ is some composition of @Le@ and @Ri@. The rational
-- behind this type family is that it provides a more compact proof
-- term of a subsumption, and thus yields more efficient
-- implementations of 'inj' and 'prj'.

type family ComprPos (p :: Pos) :: Pos where
    ComprPos Here = Here
    ComprPos (Le p) = Le (ComprPos p)
    ComprPos (Ri p) = Ri (ComprPos p)
    ComprPos (Sum l r) = CombineMaybe (Sum l r) (Combine (ComprPos l) (ComprPos r))
<commit_msg>fix bug in post-processing of embeddings<commit_after>{-# LANGUAGE DataKinds, TypeFamilies, UndecidableInstances, TypeOperators #-}

--------------------------------------------------------------------------------
-- |
-- Module      :  Data.Comp.SubsumeCommon
-- Copyright   :  (c) 2014 <NAME>
-- License     :  BSD3
-- Maintainer  :  <NAME> <<EMAIL>>
-- Stability   :  experimental
-- Portability :  non-portable (GHC Extensions)
--
-- Shared parts of the implementation of signature subsumption for
-- both the base and the multi library.
--
--------------------------------------------------------------------------------

module Data.Comp.SubsumeCommon
    ( ComprEmb
    , Pos (..)
    , Emb (..)
    ) where

-- | This type is used in its promoted form only. It represents
-- pointers from the left-hand side of a subsumption to the right-hand
-- side.
data Pos = Here | Le Pos | Ri Pos | Sum Pos Pos

-- | This type is used in its promoted form only. It represents
-- possible results for checking for subsumptions. 'Found' indicates a
-- subsumption was found; 'NotFound' indicates no such subsumption was
-- found. 'Ambiguous' indicates that there are duplicates on the left-
-- or the right-hand side.
data Emb = Found Pos | NotFound | Ambiguous

-- | This type family takes a position type and compresses it. That
-- means it replaces each nested occurrence of 
-- 
-- @
--   Sum (prefix (Le Here)) (prefix (Ri Here))@
-- @
---
-- with 
-- 
-- @
--   prefix Here@
-- @
-- 
-- where @prefix@ is some composition of @Le@ and @Ri@. The rational
-- behind this type family is that it provides a more compact proof
-- term of a subsumption, and thus yields more efficient
-- implementations of 'inj' and 'prj'.

type family ComprPos (p :: Pos) :: Pos where
    ComprPos Here = Here
    ComprPos (Le p) = Le (ComprPos p)
    ComprPos (Ri p) = Ri (ComprPos p)
    ComprPos (Sum l r) = CombineRec (ComprPos l) (ComprPos r)


-- | Helper type family for 'ComprPos'. Note that we could have
-- defined this as a type synonym. But if we do that, performance
-- becomes abysmal. I presume that the reason for this huge impact on
-- performance lies in the fact that right-hand side of the defining
-- equation duplicates the two arguments @l@ and @r@.
type family CombineRec l r where
    CombineRec l r = CombineMaybe (Sum l r) (Combine l r)

{'fraction_non_alphanumeric': 0.17702227432590856, 'fraction_numerical': 0.0023446658851113715, 'mean_word_length': 2.164688427299703, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3805587', 'n_tokens_mistral': 1261, 'n_tokens_neox': 1217, 'n_words': 524}