<commit_before>-- |
-- Module      : Text.Search.Whistlepig
-- Copyright   : (c) <NAME> 2012
-- License     : BSD3
-- 
-- Maintainer  : <EMAIL>
-- Stability   : experimental
-- Portability : portable
-- 
-- This module provides a high level interface to the Whistlepig C library.
-- 
module Text.Search.Whistlepig
       ( -- * Introduction
         -- $intro

         -- * Tutorial
         -- $tutorial

         -- * Interface
         -- ** Index management
         WP.Index       -- :: *
       , WP.DocID       -- :: *
       , WP.Results     -- :: *
       , WP.indexExists -- :: FilePath -> IO Bool
       , WP.deleteIndex -- :: FilePath -> IO ()
       , create         -- :: MonadResource m => FilePath -> m WP.Index
       , load           -- :: MonadResource m => FilePath -> m WP.Index
       , open           -- :: MonadResource m => FilePath -> m WP.Index
       , withIndex      -- :: FilePath -> (WP.Index -> ResourceT IO a) -> IO a
       , indexSize      -- :: MonadResource m => WP.Index -> m Word64
       , addEntry       -- :: MonadResource m => WP.Index -> WP.Entry -> m WP.DocID
       , addLabel       -- :: MonadResource m => WP.Index -> String -> WP.DocID -> m ()
       , removeLabel    -- :: MonadResource m => WP.Index -> String -> WP.DocID -> m ()

         -- ** Entry management
       , WP.Entry       -- :: *

         -- ** Queries
       , WP.Query       -- :: *
       , newQuery       -- :: Monadresource m => String -> String -> m WP.Query
       , andQuery       -- :: Monadresource m => Query -> Query -> m WP.Query
       , orQuery        -- :: Monadresource m => Query -> Query -> m WP.Query

       , runQuery       -- :: MonadResource m => WP.Index -> WP.Query -> m WP.Results
       , countResults   -- :: MonadResource m => WP.Index -> WP.Query -> m Word32

         -- *** Quasi quoting
       ) where

import Data.Word
import Control.Applicative

import Control.Monad.IO.Class (liftIO)
import Control.Monad.Trans.Resource

import qualified Text.Search.Whistlepig.IO as WP

-------------------------------------------------------------------------------
-- Introduction

-- $intro
-- Whistlepig is a minimalist realtime full-text search index. Its goal is to
-- be as small and maintainable as possible, while still remaining useful,
-- performant and scalable to large corpora. If you want realtime full-text
-- search without the frills, Whistlepig may be for you.
--


-------------------------------------------------------------------------------
-- Tutorial

-- $tutorial
-- Lorem ipsum...
--


-------------------------------------------------------------------------------
-- Index management

-- | Creates an 'Index'. If it already exists, this function throws an error.
create :: MonadResource m => FilePath -> m WP.Index
create path = liftIO (WP.indexExists path) >>= bool err create'
  where err = throwErr "Whistlepig.create: index already exists"
        create' = snd <$> allocate (open' path) close'
        open' p =    WP.createIndex p
                 >>= onoesIfErr ("whistlepig: could not open index "++p)
        close' idx =   WP.closeIndex idx
                   >>= onoesIfErr2 ("whistlepig: could not close index "++path)

-- | Load an 'Index'. If it does not exist, this function throws an error.
load :: MonadResource m => FilePath -> m WP.Index
load path = liftIO (WP.indexExists path) >>= bool load' err
  where err = throwErr "Whistlepig.load: index doesn't exist"
        load'   = snd <$> allocate (open' path) close'
        open' p =   WP.loadIndex p
                >>= onoesIfErr ("whistlepig: could not load index "++p)
        close' idx =   WP.closeIndex idx
                   >>= onoesIfErr2 ("whistlepig: could not close index "++path)<commit_msg>Whitespace and grammar police are at your door.<commit_after>-- |
-- Module      : Text.Search.Whistlepig
-- Copyright   : (c) <NAME> 2012
-- License     : BSD3
--
-- Maintainer  : <EMAIL>
-- Stability   : experimental
-- Portability : portable
--
-- This module provides a high level interface to the Whistlepig C library.
--
module Text.Search.Whistlepig
       ( -- * Introduction
         -- $intro

         -- * Tutorial
         -- $tutorial

         -- * Interface
         -- ** Index management
         WP.Index       -- :: *
       , WP.DocID       -- :: *
       , WP.Results     -- :: *
       , WP.indexExists -- :: FilePath -> IO Bool
       , WP.deleteIndex -- :: FilePath -> IO ()
       , create         -- :: MonadResource m => FilePath -> m WP.Index
       , load           -- :: MonadResource m => FilePath -> m WP.Index
       , open           -- :: MonadResource m => FilePath -> m WP.Index
       , withIndex      -- :: FilePath -> (WP.Index -> ResourceT IO a) -> IO a
       , indexSize      -- :: MonadResource m => WP.Index -> m Word64
       , addEntry       -- :: MonadResource m => WP.Index -> WP.Entry -> m WP.DocID
       , addLabel       -- :: MonadResource m => WP.Index -> String -> WP.DocID -> m ()
       , removeLabel    -- :: MonadResource m => WP.Index -> String -> WP.DocID -> m ()

         -- ** Entry management
       , WP.Entry       -- :: *

         -- ** Queries
       , WP.Query       -- :: *
       , newQuery       -- :: Monadresource m => String -> String -> m WP.Query
       , andQuery       -- :: Monadresource m => Query -> Query -> m WP.Query
       , orQuery        -- :: Monadresource m => Query -> Query -> m WP.Query

       , runQuery       -- :: MonadResource m => WP.Index -> WP.Query -> m WP.Results
       , countResults   -- :: MonadResource m => WP.Index -> WP.Query -> m Word32

         -- *** Quasi quoting
       ) where

import Data.Word
import Control.Applicative

import Control.Monad.IO.Class (liftIO)
import Control.Monad.Trans.Resource

import qualified Text.Search.Whistlepig.IO as WP

-------------------------------------------------------------------------------
-- Introduction

-- $intro
-- Whistlepig is a minimalist realtime full-text search index. Its goal is to
-- be as small and maintainable as possible, while still remaining useful,
-- performant and scalable to large corpora. If you want realtime full-text
-- search without the frills, Whistlepig may be for you.
--


-------------------------------------------------------------------------------
-- Tutorial

-- $tutorial
-- Lorem ipsum...
--


-------------------------------------------------------------------------------
-- Index management

-- | Creates an 'Index'. If it already exists, this function throws an error.
create :: MonadResource m => FilePath -> m WP.Index
create path = liftIO (WP.indexExists path) >>= bool err create'
  where err = throwErr "Whistlepig.create: index already exists"
        create' = snd <$> allocate (open' path) close'
        open' p =    WP.createIndex p
                 >>= onoesIfErr ("Whistlepig: could not open index "++p)
        close' idx =   WP.closeIndex idx
                   >>= onoesIfErr2 ("Whistlepig: could not close index "++path)

-- | Load an 'Index'. If it does not exist, this function throws an error.
load :: MonadResource m => FilePath -> m WP.Index
load path = liftIO (WP.indexExists path) >>= bool load' err
  where err = throwErr "Whistlepig.load: index doesn't exist"
        load'   = snd <$> allocate (open' path) close'
        open' p =   WP.loadIndex p
                >>= onoesIfErr ("Whistlepig: could not load index "++p)
        close' idx =   WP.closeIndex idx
                   >>= onoesIfErr2 ("Whistlepig: could not close index "++path)
{'fraction_non_alphanumeric': 0.18855622089155022, 'fraction_numerical': 0.002927478376580173, 'mean_word_length': 1.573972602739726, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 121, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2336434', 'n_tokens_mistral': 2126, 'n_tokens_neox': 2045, 'n_words': 666}