<commit_before>--import Debug.Trace
import Text.ParserCombinators.Parsec as PS
import System.IO
import System.Environment
import Data.List
import qualified Data.ByteString.Lazy as BL
import Data.Binary.Put
--import Control.Applicative as A
    
valouevFile :: GenParser Char st [(String, String, String, [Float])]
valouevFile =
            do  result <- PS.many record
                eof
                return result

record :: GenParser Char st (String, String, String, [Float])
record = do map_name <- fieldContent ; _ <- eol
            enz_name <- parseEnzyme ; _ <- char '\t'; enz_acr_name <- fieldContent; _ <- char '\t'; frags <- sepBy fieldContent (char '\t'); _ <- eol
            _ <- eol
            return (map_name, enz_name, enz_acr_name, fmap read frags)

parseEnzyme :: GenParser Char st String
parseEnzyme = do (char '\t' >> fieldContent)
                 PS.<|>
                 fieldContent
                 


fieldContent :: GenParser Char st String
fieldContent = PS.many (noneOf "\t\n")


-- The end of line character is \n
eol :: GenParser Char st Char
eol = char '\n'

parseOM :: [Char] -> Either ParseError [(String, String, String, [Float])]  
parseOM input = parse valouevFile "(unknown)" input



extract_frags :: (String, String, String, [Float]) -> [Float]
extract_frags (_, _, _, frags) = frags

frag_delim :: [Float]
frag_delim = [100000.0]

dumpnode :: (Float, Int) -> Put
dumpnode (lab, value) = do
  putWord32host $ fromIntegral $ quantize $ round lab * 1000
  putWord32host $ fromIntegral value

enumerate :: [Float] -> [(Float, Int)]
enumerate order_list = (zip order_list [1..])
                       
dumpOrderList :: [Float] -> Put
dumpOrderList order_list = do
  let enumerated = enumerate order_list
  let actions = fmap dumpnode enumerated   -- broken, need to do full list, not just first element
  sequence_ actions
  

         
dumpNodes :: [[Float]] -> Put
dumpNodes skip_nodes = do
  let num_nodes = fromIntegral $ sum $ fmap length skip_nodes
  putWord32host $ fromIntegral $ num_nodes + 2 -- for initial and final
  dumpnode initnode
  let actions = fmap dumpOrderList skip_nodes
  sequence_ actions
  dumpnode finalnode
  --return ()

 -- this attaches numbers in the same order as they are written to disk, FIXME: figure out a way to assign these numbers at the time they are written 
enumerateOrderList :: (Int, [a]) -> [(a, Int)]
enumerateOrderList (start, list) = zip list [start..]

enumerateNodes :: [[a]] -> [[(a, Int)]]
enumerateNodes skip_nodes = let lengths = fmap length skip_nodes
                                prefixes = scanl (+) 1 lengths
                                prefix_list_pairs = zip prefixes skip_nodes in
                            fmap enumerateOrderList prefix_list_pairs

bin_size :: Int
bin_size = 100

quantize :: Int -> Int
quantize val = if (val `mod` bin_size) < (bin_size `quot` 2)
               then val - (val `mod` bin_size)
               else val - (val `mod` bin_size) + bin_size

max_skipnode :: Int                                
max_skipnode = 4

initnode :: (Float, Int)
initnode = (0.0, 2100000000) -- fixme, should be 2**32 - 1 but not sure if unsigned clean

finalnode :: (Float, Int)
finalnode = (0.0, 0)

--FIXME Edge and Node should probably be types
dumpEdgePiece :: (Float, Int) -> Put
dumpEdgePiece (_, pos) = do putWord32host $ fromIntegral pos
<commit_msg>refactored some to mark backbone nodes<commit_after>--import Debug.Trace
import Text.ParserCombinators.Parsec as PS
import System.IO
import System.Environment
import Data.List
import qualified Data.ByteString.Lazy as BL
import Data.Binary.Put
--import Control.Applicative as A
    
valouevFile :: GenParser Char st [(String, String, String, [Float])]
valouevFile =
            do  result <- PS.many record
                eof
                return result

record :: GenParser Char st (String, String, String, [Float])
record = do map_name <- fieldContent ; _ <- eol
            enz_name <- parseEnzyme ; _ <- char '\t'; enz_acr_name <- fieldContent; _ <- char '\t'; frags <- sepBy fieldContent (char '\t'); _ <- eol
            _ <- eol
            return (map_name, enz_name, enz_acr_name, fmap read frags)

parseEnzyme :: GenParser Char st String
parseEnzyme = do (char '\t' >> fieldContent)
                 PS.<|>
                 fieldContent
                 


fieldContent :: GenParser Char st String
fieldContent = PS.many (noneOf "\t\n")


-- The end of line character is \n
eol :: GenParser Char st Char
eol = char '\n'

parseOM :: [Char] -> Either ParseError [(String, String, String, [Float])]  
parseOM input = parse valouevFile "(unknown)" input



extract_frags :: (String, String, String, [Float]) -> [Float]
extract_frags (_, _, _, frags) = frags

frag_delim :: [Float]
frag_delim = [100000.0]

dumpnode :: (Int, Int) -> Put
dumpnode (lab, value) = do putWord32host $ fromIntegral lab
                           putWord32host $ fromIntegral value

enumerate :: [a] -> [(a, Int)]
enumerate list = (zip list [1..])
                       
dumpOrderList :: [Int] -> Put
dumpOrderList order_list = do
  let enumerated = enumerate order_list
  let actions = fmap dumpnode enumerated   -- broken, need to do full list, not just first element
  sequence_ actions

make_skipnode n = n
make_backbone n = n

process_backbone_node :: Float -> Int                  
process_backbone_node = (\lab -> fromIntegral $ make_backbone $ quantize $ round lab * 1000)

process_backbone_nodes :: [Float] -> [Int]
process_backbone_nodes labs = fmap process_backbone_node labs

process_actual_skipnode :: Float -> Int
process_actual_skipnode = (\lab -> fromIntegral $ make_skipnode $ quantize $ round lab * 1000)                              
process_actual_skipnodes :: [Float] -> [Int]                             
process_actual_skipnodes labs = fmap process_actual_skipnode labs
         
dumpNodes :: [[Float]] -> Put
dumpNodes skip_nodes = do
  let num_nodes = fromIntegral $ sum $ fmap length skip_nodes
  putWord32host $ fromIntegral $ num_nodes + 2 -- for initial and final
  dumpnode initnode
  let actual_skipnodes = take ((length skip_nodes) - 1) skip_nodes
  let backbone = [last skip_nodes]
  let actions = (fmap dumpOrderList (fmap process_actual_skipnodes actual_skipnodes)) ++ (fmap dumpOrderList (fmap process_backbone_nodes backbone))
  sequence_ actions
  dumpnode finalnode
  --return ()

 -- this attaches numbers in the same order as they are written to disk, FIXME: figure out a way to assign these numbers at the time they are written 
enumerateOrderList :: (Int, [a]) -> [(a, Int)]
enumerateOrderList (start, list) = zip list [start..]

enumerateNodes :: [[a]] -> [[(a, Int)]]
enumerateNodes skip_nodes = let lengths = fmap length skip_nodes
                                prefixes = scanl (+) 1 lengths
                                prefix_list_pairs = zip prefixes skip_nodes in
                            fmap enumerateOrderList prefix_list_pairs

bin_size :: Int
bin_size = 100

quantize :: Int -> Int
quantize val = if (val `mod` bin_size) < (bin_size `quot` 2)
               then val - (val `mod` bin_size)
               else val - (val `mod` bin_size) + bin_size

-- the order of the automaton.  This counts the backbone as a degenerate case of skip nodes, you might call it the 0th order skip nodes.  So an automaton with a backbone and skipnodes that sum two consecutive nodes in the backbone for each skipnode would have a value of 2.                    
max_skipnode :: Int                                
max_skipnode = 3

initnode :: (Int, Int)
initnode = (process_backbone_node 0.0, 2100000000) -- fixme, should be 2**32 - 1 but not sure if unsigned clean

finalnode :: (Int, Int)
finalnode = (process_backbone_node 0.0, 0)

--FIXME Edge and Node should probably be types
dumpEdgePiece :: (Float, Int) -> Put
dumpEdgePiece (_, pos) = do putWord32host $ fromIntegral pos

{'fraction_non_alphanumeric': 0.09789500380421, 'fraction_numerical': 0.012553892974892214, 'mean_word_length': 2.7503566333808847, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7170546', 'n_tokens_mistral': 2446, 'n_tokens_neox': 2296, 'n_words': 830}