<commit_before>﻿namespace TextOn.Atom

open System
open System.Text.RegularExpressions

type ParsedAttributeName = string
type ParsedVariableName = string
type ParsedFunctionName = string

type ParsedSentenceNode =
    | ParsedStringValue of string
    | ParsedVariable of ParsedVariableName
    | ParsedSimpleChoice of ParsedSentenceNode[]
    | ParsedSimpleSeq of ParsedSentenceNode[]

type ParsedCondition =
    | ParsedUnconditional
    | ParsedOr of ParsedCondition * ParsedCondition
    | ParsedAnd of ParsedCondition * ParsedCondition
    | ParsedAreEqual of ParsedAttributeName * string
    | ParsedAreNotEqual of ParsedAttributeName * string

type ParsedNode =
    | ParsedSentence of ParsedSentenceNode * ParsedCondition
    | ParsedFunctionInvocation of ParsedFunctionName * ParsedCondition
    | ParsedSeq of ParsedNode[] * ParsedCondition
    | ParsedChoice of ParsedNode[] * ParsedCondition
    | ParsedParagraphBreak of ParsedCondition

[<RequireQualifiedAccess>]
module FunctionParser =
    /// Parse the CategorizedAttributedTokenSet for a function definition into a tree.
    let parseFunction (tokenSet:CategorizedAttributedTokenSet) =

        ()<commit_msg>Not sure where I'm going with this<commit_after>﻿namespace TextOn.Atom

open System
open System.Text.RegularExpressions

type ParsedAttributeName = string
type ParsedVariableName = string
type ParsedFunctionName = string

type ParsedSentenceNode =
    | ParsedStringValue of string
    | ParsedVariable of ParsedVariableName
    | ParsedSimpleChoice of ParsedSentenceNode[]
    | ParsedSimpleSeq of ParsedSentenceNode[]

type ParsedCondition =
    | ParsedUnconditional
    | ParsedOr of ParsedCondition * ParsedCondition
    | ParsedAnd of ParsedCondition * ParsedCondition
    | ParsedAreEqual of ParsedAttributeName * string
    | ParsedAreNotEqual of ParsedAttributeName * string

type ParseError = {
    LineNumber : int
    StartLocation : int
    EndLocation : int
    ErrorText : string }

type ParsedNode =
    | ParsedSentence of int * ParsedSentenceNode * ParsedCondition
    | ParsedFunctionInvocation of int * ParsedFunctionName * ParsedCondition
    | ParsedSeq of int * ParsedNode[] * ParsedCondition
    | ParsedChoice of int * ParsedNode[] * ParsedCondition
    | ParsedParagraphBreak of int * ParsedCondition
    | ParseErrors of ParseError[]

type ParsedFunctionDefinition = {
    File : string
    StartLine : int
    EndLine : int
    Index : int
    HasErrors : bool
    Name : ParsedFunctionName
    Tree : ParsedNode }

[<RequireQualifiedAccess>]
module FunctionParser =
    let rec private parseFunctionInner (tokens:AttributedTokenizedLine[]) =
        ()

    /// Parse the CategorizedAttributedTokenSet for a function definition into a tree.
    let parseFunction (tokenSet:CategorizedAttributedTokenSet) : ParsedFunctionDefinition =

        failwith ""
{'fraction_non_alphanumeric': 0.0527972027972028, 'fraction_numerical': 0.0, 'mean_word_length': 4.288354898336414, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2866858', 'n_tokens_mistral': 791, 'n_tokens_neox': 781, 'n_words': 217}