<commit_before>﻿// --------------------------------------------------------------------------------------
// F# Markdown (MarkdownParser.fs)
// (c) <NAME>, 2012, Available under Apache 2.0 license.
// --------------------------------------------------------------------------------------
// Modifications (c) by <NAME>, 2015, 2016
// --------------------------------------------------------------------------------------

module internal FountainSharp.Parse.Parser

open System
open System.IO
open System.Collections.Generic

open FSharp.Collections
open FountainSharp.Parse.Collections
open FountainSharp.Parse.Patterns
open FountainSharp.Parse.Patterns.List
open FountainSharp.Parse.Patterns.String

//====== Parser
// Part 1: Inline Formatting

/// Succeeds when the specified character list starts with an escaped 
/// character - in that case, returns the character and the tail of the list
let inline (|EscapedChar|_|) input = 
  match input with
  | '\\'::( ( '*' | '\\' | '`' | '_' | '{' | '}' | '[' | ']' 
            | '(' | ')' | '>' | '#' | '.' | '!' | '+' | '-' | '$') as c) ::rest -> Some(c, rest)
  | _ -> None

/// matches a delimited list of characters that starts with some sub list, called the 
/// delimiter bracket, (like ['*'; '*'] for bold) that occurs at the beginning and the end of the text.
/// returns a wrapped list and then the rest of the characters after the delimited text.
let (|DelimitedText|_|) delimiterBracket input =
  let startl, endl = delimiterBracket, delimiterBracket
  // Like List.partitionUntilEquals, but skip over escaped characters
  let rec loop acc = function
    | EscapedChar(x, xs) -> loop (x::'\\'::acc) xs
    | input when List.startsWith endl input -> Some(List.rev acc, input)
    | x::xs -> loop (x::acc) xs
    | [] -> None
  // If it starts with 'startl', let's search for 'endl'
  if List.startsWith delimiterBracket input then
    match loop [] (List.skip delimiterBracket.Length input) with
    | Some(pre, post) -> Some(pre, List.skip delimiterBracket.Length post)
    | None -> None
  else None

/// recognizes emphasized text of Italic, Bold, etc.
/// take somethign like "*some text* some more text" and return a sequence of TextSpans: italic<"some text">::rest
let (|Emphasized|_|) = function
  // if it starts with either `_` or `*`
  //   1) the code `(('_' | '*')` :: tail)` decomposes the input into a sequence of either `'_'::tail` or `'*'::tail`
  //   2) `as input` binds that sequence to a variable
  | (('_' | '*') :: tail) as input ->
    match input with
    // the *** case in which it is both italic and strong
    | DelimitedText ['*'; '*'; '*'] (body, rest) -> 
        Some(body, Italic >> List.singleton >> Strong, rest)
    | DelimitedText['*'; '*'] (body, rest) -> 
        Some(body, Strong, rest)
    | DelimitedText['_'] (body, rest) ->
        Some(body, Underline, rest)
    | DelimitedText['*'] (body, rest) -> 
        Some(body, Italic, rest)
    | _ -> None
  | _ -> None

/// recognizes notes which start with "[[" and end with "]]"
let (|Note|_|) = function
  // the *** case in which it is both italic and strong
  | DelimitedWith ['['; '['] [']';']'] (body, rest) -> 
      Some (body, Note, rest)
  | _ -> None


/// Parses a body of a block and recognizes all inline tags.
/// returns a sequence of FountainSpan
let rec parseChars acc input = seq {

  // Zero or one literals, depending whether there is some accumulated input
  let accLiterals = Lazy.Create(fun () ->
    if List.isEmpty acc then [] 
    else [Literal(String(List.rev acc |> Array.ofList))] )

  match input with 

  // Recognizes explicit line-break at the end of line
  | ' '::' '::('\n' | '\r')::rest
  | ' '::' '::'\r'::'\n'::rest ->
      yield! accLiterals.Value
      yield HardLineBreak
      yield! parseChars [] rest

  // Encode & as an HTML entity
  | '&'::'a'::'m'::'p'::';'::rest 
  | '&'::rest ->
      yield! parseChars (';'::'p'::'m'::'a'::'&'::acc) rest      

  // Ignore escaped characters that might mean something else
  | EscapedChar(c, rest) ->
      yield! parseChars (c::acc) rest

  // Handle emphasised text
  | Emphasized (body, f, rest) ->
      yield! accLiterals.Value
      let body = parseChars [] body |> List.ofSeq
      yield f(body)
      yield! parseChars [] rest

  // Notes
  | Note (body, f, rest) ->
      yield! accLiterals.Value
      let body = parseChars [] body |> List.ofSeq
      //TODO: why won't it accept this?
      // until i figure out what's happening here, Note won't work, because it just pulls the text out entirely.
      //yield f(body)
      yield! parseChars [] rest

  // This calls itself recursively on the rest of the list
  | x::xs -> 
      yield! parseChars (x::acc) xs 
  | [] ->
      yield! accLiterals.Value }

/// Parse body of a block into a list of Markdown inline spans      
let parseSpans (String.TrimBoth s) = 
  parseChars [] (s.ToCharArray() |> List.ofArray) |> List.ofSeq

//======================================================================================
// Part 2: Block Formatting

/// Recognizes a Section (# Some section, ## another section), prefixed with '#'s
let (|Section|_|) = function
  | String.StartsWithRepeated "#" (n, header) :: rest ->
      let header = 
        // Drop "##" at the end, but only when it is preceded by some whitespace
        // (For example "## Hello F#" should be "Hello F#")
        if header.EndsWith "#" then
          let noHash = header.TrimEnd [| '#' |]
          if noHash.Length > 0 && Char.IsWhiteSpace(noHash.Chars(noHash.Length - 1)) 
          then noHash else header
        else header        
      Some(n, header.Trim(), rest)
  | rest ->
      None

/// Recognizes a SceneHeading (prefixed with INT/EXT, etc. or a single period)
let (|SceneHeading|_|) = function
  // TODO: Make this StartsWithAnyCaseInsensitive
  | String.StartsWithAny [ "INT"; "EXT"; "EST"; "INT./EXT."; "INT/EXT"; "I/E" ] heading:string :: rest ->
     Some(heading.Trim(), rest)
  | String.StartsWith "." heading:string :: rest ->
     Some(heading.Trim(), rest) 
  | rest ->
     None

// CHARACTER TODO: "BOB (OS)"
let (|Character|_|) (list:string list) =
  match list with
  | [] -> None
  | head :: rest ->
    if (head.Length = 0) then 
      None
    // matches "@McAVOY"
    else if (head.StartsWith "@") then
      Some(head.Trim(), rest)
    // matches "BOB" or "BOB JOHNSON" or "R2D2" but not "25D2"
    else if (System.Char.IsUpper (head.[0]) && head |> Seq.forall (fun c -> (System.Char.IsUpper c|| System.Char.IsWhiteSpace c || System.Char.IsNumber c))) then
      Some(head.Trim(), rest)
    // matches "BOB (*)"
    //else if (
    else
      None


// DIALOG

/// Recognizes a PageBreak (3 or more consecutive equals and nothign more)
let (|PageBreak|_|) = function
  | String.StartsWithRepeated "=" text :: rest ->
    if (fst text) >= 3 then
      match (snd text).Trim() with
      | "" -> //after the trim, there should be nothing left.<commit_msg>added some more writlines.<commit_after>﻿// --------------------------------------------------------------------------------------
// F# Markdown (MarkdownParser.fs)
// (c) <NAME>, 2012, Available under Apache 2.0 license.
// --------------------------------------------------------------------------------------
// Modifications (c) by <NAME>, 2015, 2016
// --------------------------------------------------------------------------------------

module internal FountainSharp.Parse.Parser

open System
open System.IO
open System.Collections.Generic

open FSharp.Collections
open FountainSharp.Parse.Collections
open FountainSharp.Parse.Patterns
open FountainSharp.Parse.Patterns.List
open FountainSharp.Parse.Patterns.String

//====== Parser
// Part 1: Inline Formatting

/// Succeeds when the specified character list starts with an escaped 
/// character - in that case, returns the character and the tail of the list
let inline (|EscapedChar|_|) input = 
  match input with
  | '\\'::( ( '*' | '\\' | '`' | '_' | '{' | '}' | '[' | ']' 
            | '(' | ')' | '>' | '#' | '.' | '!' | '+' | '-' | '$') as c) ::rest -> Some(c, rest)
  | _ -> None

/// matches a delimited list of characters that starts with some sub list, called the 
/// delimiter bracket, (like ['*'; '*'] for bold) that occurs at the beginning and the end of the text.
/// returns a wrapped list and then the rest of the characters after the delimited text.
let (|DelimitedText|_|) delimiterBracket input =
  let startl, endl = delimiterBracket, delimiterBracket
  // Like List.partitionUntilEquals, but skip over escaped characters
  let rec loop acc = function
    | EscapedChar(x, xs) -> loop (x::'\\'::acc) xs
    | input when List.startsWith endl input -> Some(List.rev acc, input)
    | x::xs -> loop (x::acc) xs
    | [] -> None
  // If it starts with 'startl', let's search for 'endl'
  if List.startsWith delimiterBracket input then
    match loop [] (List.skip delimiterBracket.Length input) with
    | Some(pre, post) -> Some(pre, List.skip delimiterBracket.Length post)
    | None -> None
  else None

/// recognizes emphasized text of Italic, Bold, etc.
/// take somethign like "*some text* some more text" and return a sequence of TextSpans: italic<"some text">::rest
let (|Emphasized|_|) = function
  // if it starts with either `_` or `*`
  //   1) the code `(('_' | '*')` :: tail)` decomposes the input into a sequence of either `'_'::tail` or `'*'::tail`
  //   2) `as input` binds that sequence to a variable
  | (('_' | '*') :: tail) as input ->
    match input with
    // the *** case in which it is both italic and strong
    | DelimitedText ['*'; '*'; '*'] (body, rest) -> 
        Some(body, Italic >> List.singleton >> Strong, rest)
    | DelimitedText['*'; '*'] (body, rest) -> 
        Some(body, Strong, rest)
    | DelimitedText['_'] (body, rest) ->
        Some(body, Underline, rest)
    | DelimitedText['*'] (body, rest) -> 
        Some(body, Italic, rest)
    | _ -> None
  | _ -> None

/// recognizes notes which start with "[[" and end with "]]"
let (|Note|_|) = function
  // the *** case in which it is both italic and strong
  | DelimitedWith ['['; '['] [']';']'] (body, rest) -> 
      Some (body, Note, rest)
  | _ -> None


/// Parses a body of a block and recognizes all inline tags.
/// returns a sequence of FountainSpan
let rec parseChars acc input = seq {

  // Zero or one literals, depending whether there is some accumulated input
  let accLiterals = Lazy.Create(fun () ->
    if List.isEmpty acc then [] 
    else [Literal(String(List.rev acc |> Array.ofList))] )

  match input with 

  // Recognizes explicit line-break at the end of line
  | ' '::' '::('\n' | '\r')::rest
  | ' '::' '::'\r'::'\n'::rest ->
      yield! accLiterals.Value
      yield HardLineBreak
      yield! parseChars [] rest

  // Encode & as an HTML entity
  | '&'::'a'::'m'::'p'::';'::rest 
  | '&'::rest ->
      yield! parseChars (';'::'p'::'m'::'a'::'&'::acc) rest      

  // Ignore escaped characters that might mean something else
  | EscapedChar(c, rest) ->
      yield! parseChars (c::acc) rest

  // Handle emphasised text
  | Emphasized (body, f, rest) ->
      yield! accLiterals.Value
      let body = parseChars [] body |> List.ofSeq
      yield f(body)
      yield! parseChars [] rest

  // Notes
  | Note (body, f, rest) ->
      yield! accLiterals.Value
      let body = parseChars [] body |> List.ofSeq
      //TODO: why won't it accept this?
      // until i figure out what's happening here, Note won't work, because it just pulls the text out entirely.
      //yield f(body)
      yield! parseChars [] rest

  // This calls itself recursively on the rest of the list
  | x::xs -> 
      yield! parseChars (x::acc) xs 
  | [] ->
      yield! accLiterals.Value }

/// Parse body of a block into a list of Markdown inline spans      
let parseSpans (String.TrimBoth s) = 
  parseChars [] (s.ToCharArray() |> List.ofArray) |> List.ofSeq

//======================================================================================
// Part 2: Block Formatting

/// Recognizes a Section (# Some section, ## another section), prefixed with '#'s
let (|Section|_|) = function
  | String.StartsWithRepeated "#" (n, header) :: rest ->
      let header = 
        // Drop "##" at the end, but only when it is preceded by some whitespace
        // (For example "## Hello F#" should be "Hello F#")
        if header.EndsWith "#" then
          let noHash = header.TrimEnd [| '#' |]
          if noHash.Length > 0 && Char.IsWhiteSpace(noHash.Chars(noHash.Length - 1)) 
          then noHash else header
        else header        
      Some(n, header.Trim(), rest)
  | rest ->
      None

/// Recognizes a SceneHeading (prefixed with INT/EXT, etc. or a single period)
let (|SceneHeading|_|) = function
  // TODO: Make this StartsWithAnyCaseInsensitive
  | String.StartsWithAny [ "INT"; "EXT"; "EST"; "INT./EXT."; "INT/EXT"; "I/E" ] heading:string :: rest ->
     Some(heading.Trim(), rest)
  | String.StartsWith "." heading:string :: rest ->
     Some(heading.Trim(), rest) 
  | rest ->
     None

// CHARACTER TODO: "BOB (OS)"
let (|Character|_|) (list:string list) =
  printfn "testing for character"
  match list with
  | [] -> None
  | head :: rest ->
    if (head.Length = 0) then 
      None
    // matches "@McAVOY"
    else if (head.StartsWith "@") then
      Some(head.Trim(), rest)
    // matches "BOB" or "BOB JOHNSON" or "R2D2" but not "25D2"
    else if (System.Char.IsUpper (head.[0]) && head |> Seq.forall (fun c -> (System.Char.IsUpper c|| System.Char.IsWhiteSpace c || System.Char.IsNumber c))) then
      Some(head.Trim(), rest)
    // matches "BOB (*)"
    //else if (
    else
      None


/// Recognizes a PageBreak (3 or more consecutive equals and nothign more)
let (|PageBreak|_|) = function
  | String.StartsWithRepeated "=" text :: rest ->
    if (fst text) >= 3 then
      match (snd text).Trim() with
      | "" -> //after the trim, there should be nothing left.
{'fraction_non_alphanumeric': 0.20004300458715596, 'fraction_numerical': 0.0041571100917431195, 'mean_word_length': 2.462282878411911, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 105, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1084806', 'n_tokens_mistral': 4489, 'n_tokens_neox': 4250, 'n_words': 1415}