<commit_before>﻿#light "off"

module ExpandAlter

open IL
open Rule
open Production
open Source
open System

let extract_one_rule (rule:Rule.t<'a,'b>) = 
    let (name,args,body,_public,metaArgs) = (rule.name,rule.args,rule.body,rule._public,rule.metaArgs)
    in
    let rec f b =
    match b
    with
    |PAlt     (a,b) -> List.concat [f a;f b] 
    |PSeq     (a,b) -> let wrap = List.unzip(List.map (fun (x:Production.elem<'c,'d>) ->
                                                (x.rule,fun rule -> 
                                                            {omit=x.omit;
                                                             rule = rule;
                                                             binding=x.binding;
                                                             checker=x.checker}))
                                             a  )              
                       in
                       let new_body = (List.map f (fst wrap))  
                       in 
                       let rec gen lst = 
                           match lst 
                           with
                           | hd::tl -> [for x in hd -> x::(List.concat(gen tl))]
                           | []     -> []
                       in
                       List.map (fun x -> PSeq ((List.map2 (fun c f -> f c) x (snd wrap)),b))(gen new_body)   
    |PRef   (a,b) as t   -> [t]
    |PLiteral (a)
    |PToken   (a) as t   -> [t]
    | _             -> (Console.WriteLine("incorrect tree for alternative expanding!");
                       failwith "incorrect tree for alternative expanding!")
    in 
    let new_body = f body
    in
    List.map (fun x -> {name = name;
                        args =args;
                        body =x;
                        _public =_public;
                        metaArgs=metaArgs}) new_body                    <commit_msg>Now it is about 20 lines (40%) shorter. Once again, I extremely apologize for possible bugs. Please review carefully to study f#/Caml programming<commit_after>﻿#light "off"

module ExpandAlter

open IL
open Rule
open Production
open Source
open System

let extract_one_rule (rule:Rule.t<'a,'b>) = 
    let rec expand = function
    |PAlt     (a,b) -> expand a  @ expand b
    |PSeq     (a,b) -> let wrap = List.map (fun x -> (x.rule, fun r -> {x with rule = r})) a
                                  |> List.unzip
                       in
                       let rec gen = function
                           | hd::tl -> [for x in hd -> x :: ( gen tl |> List.concat)]
                           | []     -> []
                       in 
                       fst wrap |> List.map expand |> gen 
                       |> List.map (fun x -> PSeq ((List.map2 ( |> ) x (snd wrap)),b))
    |PRef   _ 
    |PLiteral _
    |PToken   _ as t   -> [t]
    | _             -> (print_endline "incorrect tree for alternative expanding!"
                        ; failwith "incorrect tree for alternative expanding!")
    in 
    expand rule.body |> List.map (fun x -> {rule with body = x})
{'fraction_non_alphanumeric': 0.10485183979159883, 'fraction_numerical': 0.0019537609899055682, 'mean_word_length': 1.023715415019763, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 17, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7023768', 'n_tokens_mistral': 847, 'n_tokens_neox': 811, 'n_words': 249}