<commit_before>{-# LANGUAGE OverloadedStrings, NoMonomorphismRestriction, ScopedTypeVariables #-}

module HEP.Parser.LHEParser.Parser.Conduit where

import Control.Monad.IO.Class

import Data.XML.Types
import Data.Conduit as C
import Data.Conduit.List as CL
import Data.Conduit.Util.Control as CU
import qualified Data.Text as T

import HEP.Parser.LHEParser.Type
import HEP.Parser.LHEParser.Parser.Text

import Text.XML.Stream.Render

import Prelude hiding (dropWhile, takeWhile)


-- | check event starting
 
isEventStart :: Event -> Bool 
isEventStart (EventBeginElement name _) = nameLocalName name == "event" 
isEventStart _ = False

-- | check event ending 

isEventEnd :: Event -> Bool 
isEventEnd (EventEndElement name) = nameLocalName name == "event"
isEventEnd _ = False


-- | 
  
parseSingleEvent :: [Event] -> Maybe LHEvent
parseSingleEvent ((EventContent content):_)  =
  case content of 
    ContentText txt -> Just (getEvent txt)
    _ -> Nothing
parseSingleEvent _ = Nothing

-- | 

chunkLHEventConduit :: (Monad m) => Conduit Event m [Event] 
chunkLHEventConduit = C.sequence $ do 
                        CU.dropWhile (not.isEventStart)
                        CL.drop 1 
                        ev <- CU.takeWhileR (not.isEventEnd)
                        CL.drop 1 
                        return ev

-- | 

parseLHEventConduit :: (Monad m) => Conduit [Event] m (Maybe LHEvent) 
parseLHEventConduit = CL.map parseSingleEvent 

-- | 

parseEventConduit :: (Monad m) => Sink (Maybe LHEvent) m a -> Sink Event m a 
parseEventConduit x = chunkLHEventConduit =$ CL.filter (not.null) =$ parseLHEventConduit =$ x

-- | 

parseLHEHeader :: (Monad m) => Conduit Event m Event 
parseLHEHeader = CU.takeWhile (not.isEventStart)

-- | 

textLHEHeader :: (MonadIO m, MonadThrow m, MonadUnsafeIO m) => Sink Event m [T.Text]
textLHEHeader = parseLHEHeader =$ renderText def =$ CL.consume 
  
  
<commit_msg>rename to chunkLHEvent, parseLHEvent, parseEvent<commit_after>{-# LANGUAGE OverloadedStrings, NoMonomorphismRestriction, ScopedTypeVariables #-}

module HEP.Parser.LHEParser.Parser.Conduit where

import Control.Monad.IO.Class

import Data.XML.Types
import Data.Conduit as C
import Data.Conduit.List as CL
import Data.Conduit.Util.Control as CU
import qualified Data.Text as T

import HEP.Parser.LHEParser.Type
import HEP.Parser.LHEParser.Parser.Text

import Text.XML.Stream.Render

import Prelude hiding (dropWhile, takeWhile)


-- | check event starting
 
isEventStart :: Event -> Bool 
isEventStart (EventBeginElement name _) = nameLocalName name == "event" 
isEventStart _ = False

-- | check event ending 

isEventEnd :: Event -> Bool 
isEventEnd (EventEndElement name) = nameLocalName name == "event"
isEventEnd _ = False


-- | 
  
parseSingleEvent :: [Event] -> Maybe LHEvent
parseSingleEvent ((EventContent content):_)  =
  case content of 
    ContentText txt -> Just (getEvent txt)
    _ -> Nothing
parseSingleEvent _ = Nothing

-- | 

chunkLHEvent :: (Monad m) => Conduit Event m [Event] 
chunkLHEvent = C.sequence $ do 
                 CU.dropWhile (not.isEventStart)
                 CL.drop 1 
                 ev <- CU.takeWhileR (not.isEventEnd)
                 CL.drop 1 
                 return ev

-- | 

parseLHEvent :: (Monad m) => Conduit [Event] m (Maybe LHEvent) 
parseLHEvent = CL.map parseSingleEvent 

-- | 

parseEvent :: (Monad m) => Sink (Maybe LHEvent) m a -> Sink Event m a 
parseEvent x = chunkLHEvent =$ CL.filter (not.null) =$ parseLHEvent =$ x

-- | 

parseLHEHeader :: (Monad m) => Conduit Event m Event 
parseLHEHeader = CU.takeWhile (not.isEventStart)

-- | 

textLHEHeader :: (MonadIO m, MonadThrow m, MonadUnsafeIO m) => Sink Event m [T.Text]
textLHEHeader = parseLHEHeader =$ renderText def =$ CL.consume 
  
  

{'fraction_non_alphanumeric': 0.10191082802547771, 'fraction_numerical': 0.0010615711252653928, 'mean_word_length': 3.1146288209606987, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6529388', 'n_tokens_mistral': 1297, 'n_tokens_neox': 1236, 'n_words': 330}