<commit_before>{-# LANGUAGE ScopedTypeVariables, BangPatterns, GADTs, FlexibleContexts, 
             FlexibleInstances, TypeFamilies #-}

module HEP.Parser.LHCOAnalysis.Parse where

import HEP.Parser.LHCOAnalysis.PhysObj

import qualified Data.List.Split as SP

import Data.ByteString.Lex.Lazy.Double
import Data.ByteString.Internal  
import qualified Data.ByteString.Lazy.Char8 as B

import Debug.Trace


data Parsed = Comment B.ByteString | Zero Int | Nonzero (Int,EachObj) | Strange
              deriving (Show)


type Merged = Parsed

parsestr :: B.ByteString -> [PhyEventClassified]
parsestr str1 = 
    let strlines = B.lines str1 
        parsed = map classify_line strlines
        commentfiltered = filter (not.isComment) parsed
        strangefiltered = filter (not.isStrange) commentfiltered
        splitted  = tail $ SP.split (SP.whenElt isZero) strangefiltered
       
        splitted' = mergeZeroParsed $ splitted 
                
        unnonzeroed = map unnonzeroaction splitted'
                      
        unnonzeroaction (x,xs) = (x, map unNonzero xs)
                      
        classified  = map constructPhysObjClass unnonzeroed 
    in  classified 





mergeZeroParsed :: [[Parsed]] -> [(Int,[Parsed])]
mergeZeroParsed [] = []
mergeZeroParsed (x0:x1:xs) = mergeZero x0 x1 : mergeZeroParsed xs

mergeZero [Zero evnum] x1 = (evnum,x1) 

--mergeZero x0 x1 = x1

isComment :: Parsed -> Bool 
isComment (Comment _) = True
isComment _ = False

isStrange :: Parsed -> Bool
isStrange Strange = True
isStrange _ = False

isZero :: Parsed -> Bool
isZero (Zero _) = True
isZero _ = False

unNonzero :: Parsed -> (Int, EachObj)
unNonzero (Nonzero x) = x
unNonzero _ = undefined

constructPhysObjClass :: (Int,[(Int, EachObj)]) -> PhyEventClassified
constructPhysObjClass (xid,lst) = 
    let classified = foldl addEachObjtoClassified (zeroevent {eventid = xid}) lst
        sortedclassified = sortPhyEventC classified 
    in  sortedclassified



classify_line :: B.ByteString -> Parsed 
classify_line bstr = let trimmed = B.dropWhile (isSpaceChar8) bstr
                     in if B.null trimmed 
                        then Strange 
                        else let h = B.head trimmed 
                             in case h of 
                                  '#' -> Comment trimmed 
                                  '0' -> parse_zero trimmed
                                  _   -> parse_nonzero trimmed


parse_zero :: B.ByteString -> Parsed
parse_zero bstr = let bstrlst = B.split ' ' bstr
                      _:a2:[] = take 2 $ filter (not . B.null) bstrlst
                      b2 = trace ("a2 = " ++ show a2) $ maybe 0 fst $ B.readInt a2
                      
                  in  trace ("b2 = " ++ show b2 ) $ Zero b2


parse_nonzero :: B.ByteString -> Parsed
parse_nonzero bstr 
    = let bstrlst = B.split ' ' bstr
          a1:a2:a3:a4:a5:a6:a7:a8:a9:a10:a11:[]
              = take 11 $ filter (not . B.null) bstrlst
          objid = maybe 0 fst $ B.readInt a1 
          typ = maybe 0 fst $ B.readInt a2
          eta = maybe 0 fst $ readDouble a3
          phi = maybe 0 fst $ readDouble a4
          pt  = maybe 0 fst $ readDouble a5
          jmas= maybe 0 fst $ readDouble a6
          ntrk= maybe 0 fst $ readDouble a7
          btag= maybe 0 fst $ readDouble a8
          hadem= maybe 0 fst $ readDouble a9
          dum1= maybe 0 fst $ readDouble a10
          dum2= maybe 0 fst $ readDouble a11
      in  Nonzero (objid,  
                   mkEachObj typ eta phi pt jmas ntrk  
                             btag hadem dum1 dum2 )
                           
mkEachObj :: Int -> Double -> Double -> Double -> Double -> Double 
                 -> Double -> Double -> Double -> Double -> EachObj
mkEachObj typ eta phi pt jmas ntrk btag hadem dum1 dum2 
  = case typ of
           0 -> EO $ ObjPhoton   { etaphiptphoton = (eta,phi,pt) }
           1 -> EO $ ObjElectron { etaphiptelectron = (eta,phi,pt)
                                 , chargeelectron = ntrktoecharge ntrk }
           2 -> EO $ ObjMuon     { etaphiptmuon  = (eta,phi,pt)
                                 , chargemuon    = ntrktoecharge ntrk }
           3 -> EO $ ObjTau      { etaphipttau   = (eta,phi,pt)
                                 , chargetau     = ntrktoecharge ntrk 
                                 , prongtau      = ntrktotauprong ntrk }
           4 -> if btag == 0.0 
                then EO $ ObjJet  { etaphiptjet   = (eta,phi,pt)
                                  , mjet          = jmas
                                  , numtrkjet     = round ntrk }
                else EO $ ObjBJet { etaphiptbjet  = (eta,phi,pt)
                                  , mbjet         = jmas
                                  , numtrkbjet    = round ntrk }
           6 -> EO $ ObjMET { phiptmet = (phi,pt) }
           _ -> error "not a predefined type"
     
addEachObjtoClassified :: PhyEventClassified -> (Int,EachObj) 
                          -> PhyEventClassified
addEachObjtoClassified phyclass (idx,(EO photon@(ObjPhoton _))) 
  = phyclass { photonlst  = (idx,photon) : (photonlst phyclass) }
addEachObjtoClassified phyclass (idx,(EO electron@(ObjElectron _ _))) 
  = phyclass { electronlst = (idx,electron) : (electronlst phyclass) }
addEachObjtoClassified phyclass (idx,(EO muon@(ObjMuon _ _)))  
  = phyclass { muonlst     = (idx,muon) : (muonlst phyclass) }
addEachObjtoClassified phyclass (idx,(EO tau@(ObjTau _ _ _)))
  = phyclass { taulst      = (idx,tau) : (taulst phyclass) }
addEachObjtoClassified phyclass (idx,(EO jet@(ObjJet _ _ _))) 
  = phyclass { jetlst      = (idx,jet) : (jetlst phyclass) }
addEachObjtoClassified phyclass (idx,(EO bjet@(ObjBJet _ _ _)))
  = phyclass { bjetlst     = (idx,bjet) : (bjetlst phyclass) }
addEachObjtoClassified phyclass (idx,(EO met'@(ObjMET _)))  
  = phyclass { met         = met' }

<commit_msg>erase redundant trace message in Parse.hs<commit_after>{-# LANGUAGE ScopedTypeVariables, BangPatterns, GADTs, FlexibleContexts, 
             FlexibleInstances, TypeFamilies #-}

module HEP.Parser.LHCOAnalysis.Parse where

import HEP.Parser.LHCOAnalysis.PhysObj

import qualified Data.List.Split as SP

import Data.ByteString.Lex.Lazy.Double
import Data.ByteString.Internal  
import qualified Data.ByteString.Lazy.Char8 as B


data Parsed = Comment B.ByteString | Zero Int | Nonzero (Int,EachObj) | Strange
              deriving (Show)


type Merged = Parsed

parsestr :: B.ByteString -> [PhyEventClassified]
parsestr str1 = 
    let strlines = B.lines str1 
        parsed = map classify_line strlines
        commentfiltered = filter (not.isComment) parsed
        strangefiltered = filter (not.isStrange) commentfiltered
        splitted  = tail $ SP.split (SP.whenElt isZero) strangefiltered
       
        splitted' = mergeZeroParsed $ splitted 
                
        unnonzeroed = map unnonzeroaction splitted'
                      
        unnonzeroaction (x,xs) = (x, map unNonzero xs)
                      
        classified  = map constructPhysObjClass unnonzeroed 
    in  classified 





mergeZeroParsed :: [[Parsed]] -> [(Int,[Parsed])]
mergeZeroParsed [] = []
mergeZeroParsed (x0:x1:xs) = mergeZero x0 x1 : mergeZeroParsed xs

mergeZero [Zero evnum] x1 = (evnum,x1) 

--mergeZero x0 x1 = x1

isComment :: Parsed -> Bool 
isComment (Comment _) = True
isComment _ = False

isStrange :: Parsed -> Bool
isStrange Strange = True
isStrange _ = False

isZero :: Parsed -> Bool
isZero (Zero _) = True
isZero _ = False

unNonzero :: Parsed -> (Int, EachObj)
unNonzero (Nonzero x) = x
unNonzero _ = undefined

constructPhysObjClass :: (Int,[(Int, EachObj)]) -> PhyEventClassified
constructPhysObjClass (xid,lst) = 
    let classified = foldl addEachObjtoClassified (zeroevent {eventid = xid}) lst
        sortedclassified = sortPhyEventC classified 
    in  sortedclassified



classify_line :: B.ByteString -> Parsed 
classify_line bstr = let trimmed = B.dropWhile (isSpaceChar8) bstr
                     in if B.null trimmed 
                        then Strange 
                        else let h = B.head trimmed 
                             in case h of 
                                  '#' -> Comment trimmed 
                                  '0' -> parse_zero trimmed
                                  _   -> parse_nonzero trimmed


parse_zero :: B.ByteString -> Parsed
parse_zero bstr = let bstrlst = B.split ' ' bstr
                      _:a2:[] = take 2 $ filter (not . B.null) bstrlst
                      b2 = maybe 0 fst $ B.readInt a2
                      
                  in Zero b2


parse_nonzero :: B.ByteString -> Parsed
parse_nonzero bstr 
    = let bstrlst = B.split ' ' bstr
          a1:a2:a3:a4:a5:a6:a7:a8:a9:a10:a11:[]
              = take 11 $ filter (not . B.null) bstrlst
          objid = maybe 0 fst $ B.readInt a1 
          typ = maybe 0 fst $ B.readInt a2
          eta = maybe 0 fst $ readDouble a3
          phi = maybe 0 fst $ readDouble a4
          pt  = maybe 0 fst $ readDouble a5
          jmas= maybe 0 fst $ readDouble a6
          ntrk= maybe 0 fst $ readDouble a7
          btag= maybe 0 fst $ readDouble a8
          hadem= maybe 0 fst $ readDouble a9
          dum1= maybe 0 fst $ readDouble a10
          dum2= maybe 0 fst $ readDouble a11
      in  Nonzero (objid,  
                   mkEachObj typ eta phi pt jmas ntrk  
                             btag hadem dum1 dum2 )
                           
mkEachObj :: Int -> Double -> Double -> Double -> Double -> Double 
                 -> Double -> Double -> Double -> Double -> EachObj
mkEachObj typ eta phi pt jmas ntrk btag hadem dum1 dum2 
  = case typ of
           0 -> EO $ ObjPhoton   { etaphiptphoton = (eta,phi,pt) }
           1 -> EO $ ObjElectron { etaphiptelectron = (eta,phi,pt)
                                 , chargeelectron = ntrktoecharge ntrk }
           2 -> EO $ ObjMuon     { etaphiptmuon  = (eta,phi,pt)
                                 , chargemuon    = ntrktoecharge ntrk }
           3 -> EO $ ObjTau      { etaphipttau   = (eta,phi,pt)
                                 , chargetau     = ntrktoecharge ntrk 
                                 , prongtau      = ntrktotauprong ntrk }
           4 -> if btag == 0.0 
                then EO $ ObjJet  { etaphiptjet   = (eta,phi,pt)
                                  , mjet          = jmas
                                  , numtrkjet     = round ntrk }
                else EO $ ObjBJet { etaphiptbjet  = (eta,phi,pt)
                                  , mbjet         = jmas
                                  , numtrkbjet    = round ntrk }
           6 -> EO $ ObjMET { phiptmet = (phi,pt) }
           _ -> error "not a predefined type"
     
addEachObjtoClassified :: PhyEventClassified -> (Int,EachObj) 
                          -> PhyEventClassified
addEachObjtoClassified phyclass (idx,(EO photon@(ObjPhoton _))) 
  = phyclass { photonlst  = (idx,photon) : (photonlst phyclass) }
addEachObjtoClassified phyclass (idx,(EO electron@(ObjElectron _ _))) 
  = phyclass { electronlst = (idx,electron) : (electronlst phyclass) }
addEachObjtoClassified phyclass (idx,(EO muon@(ObjMuon _ _)))  
  = phyclass { muonlst     = (idx,muon) : (muonlst phyclass) }
addEachObjtoClassified phyclass (idx,(EO tau@(ObjTau _ _ _)))
  = phyclass { taulst      = (idx,tau) : (taulst phyclass) }
addEachObjtoClassified phyclass (idx,(EO jet@(ObjJet _ _ _))) 
  = phyclass { jetlst      = (idx,jet) : (jetlst phyclass) }
addEachObjtoClassified phyclass (idx,(EO bjet@(ObjBJet _ _ _)))
  = phyclass { bjetlst     = (idx,bjet) : (bjetlst phyclass) }
addEachObjtoClassified phyclass (idx,(EO met'@(ObjMET _)))  
  = phyclass { met         = met' }


{'fraction_non_alphanumeric': 0.0899931693989071, 'fraction_numerical': 0.012807377049180328, 'mean_word_length': 1.831278704375151, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 67, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5199769', 'n_tokens_mistral': 3921, 'n_tokens_neox': 3658, 'n_words': 955}