<commit_before>{-# LANGUAGE DeriveFunctor       #-}
{-# LANGUAGE DeriveFoldable      #-}
{-# LANGUAGE DeriveTraversable   #-}
{-# LANGUAGE FlexibleContexts    #-}
{-# LANGUAGE OverloadedStrings   #-}
{-# LANGUAGE ViewPatterns        #-}

module Unison.Util.Pretty (
   Pretty,
   ColorText,
   align,
   backticked,
   bulleted,
   bracket,
   -- breakable
   callout,
   excerptSep,
   excerptSep',
   excerptColumn2,
   excerptColumn2Headed,
   warnCallout, blockedCallout, fatalCallout, okCallout,
   column2,
   column3,
   column3sep,
   commas,
   commented,
   oxfordCommas,
   plural,
   dashed,
   flatMap,
   group,
   hang',
   hang,
   hangUngrouped',
   hangUngrouped,
   indent,
   indentAfterNewline,
   indentN,
   indentNAfterNewline,
   leftPad,
   lines,
   linesNonEmpty,
   linesSpaced,
   lit,
   map,
   nest,
   num,
   newline,
   lineSkip,
   nonEmpty,
   numbered,
   numberedColumn2,
   orElse,
   orElses,
   paragraphyText,
   parenthesize,
   parenthesizeCommas,
   parenthesizeIf,
   preferredWidth,
   preferredHeight,
   render,
   renderUnbroken,
   rightPad,
   sep,
   sepNonEmpty,
   sepSpaced,
   shown,
   softbreak,
   spaceIfBreak,
   spaced,
   spacedMap,
   spacesIfBreak,
   string,
   surroundCommas,
   syntaxToColor,
   text,
   toANSI,
   toAnsiUnbroken,
   toHTML,
   toPlain,
   toPlainUnbroken,
   withSyntax,
   wrap,
   wrapColumn2,
   wrapString,
   black, red, green, yellow, blue, purple, cyan, white, hiBlack, hiRed, hiGreen, hiYellow, hiBlue, hiPurple, hiCyan, hiWhite, bold,
   border,
   Width
  ) where

import Unison.Prelude

import           Data.Char                      ( isSpace )
import           Data.List                      ( foldr1, intersperse )
import           Prelude                 hiding ( lines , map )
import           Unison.Util.AnnotatedText      ( annotateMaybe )
import qualified Unison.Util.ColorText         as CT
import qualified Unison.Util.SyntaxText        as ST
import           Unison.Util.Monoid             ( intercalateMap )
import qualified Data.ListLike                 as LL
import qualified Data.Sequence                 as Seq
import qualified Data.Text                     as Text
import qualified Data.Char as Char

type Width = Int
type ColorText = CT.ColorText

data Pretty s = Pretty { delta :: Delta, out :: F s (Pretty s) } deriving Eq

instance Functor Pretty where
  fmap f (Pretty d o) = Pretty d (mapLit f $ fmap (fmap f) o)

data F s r
  = Empty | Group r | Lit s | Wrap (Seq r) | OrElse r r | Append (Seq r)
  deriving (Eq, Show, Foldable, Traversable, Functor)

mapLit :: (s -> t) -> F s r -> F t r
mapLit f (Lit s) = Lit (f s)
mapLit _ Empty = Empty
mapLit _ (Group r) = Group r
mapLit _ (Wrap s) = Wrap s
mapLit _ (OrElse r s) = OrElse r s
mapLit _ (Append s) = Append s

lit :: (IsString s, LL.ListLike s Char) => s -> Pretty s
lit s = lit' (foldMap chDelta $ LL.toList s) s

lit' :: Delta -> s -> Pretty s
lit' d s = Pretty d (Lit s)

orElse :: Pretty s -> Pretty s -> Pretty s
orElse p1 p2 = Pretty (delta p1) (OrElse p1 p2)

orElses :: [Pretty s] -> Pretty s
orElses [] = mempty
orElses ps = foldr1 orElse ps

wrapImpl :: IsString s => [Pretty s] -> Pretty s
wrapImpl [] = mempty
wrapImpl (p:ps) = wrap_ . Seq.fromList $
  p : fmap (\p -> (" " <> p) `orElse` (newline <> p)) ps

wrapString :: (LL.ListLike s Char, IsString s) => String -> Pretty s
wrapString s = wrap (lit $ fromString s)

-- 1. Convert multiple adjacent blank lines to a single blank line.
-- 2. Combine adjacent non-blank lines into one line.
-- 3. Wrap each remaining line.
paragraphyText :: (LL.ListLike s Char, IsString s) => Text -> Pretty s
paragraphyText t = lines . fmap (wrap . text) .  step2 . step1 $ Text.lines t where
  step1 (line1:line2:rest) | isBlank line1 && isBlank line2 = step1 (line1:rest)
  step1 (line:rest) = line : step1 rest
  step1 [] = []
  step2 (line:line2:rest) | not (isBlank line) || not (isBlank line2) = step2 $ (line <> " " <> line2) : rest 
  step2 (line:rest) = line : step2 rest
  step2 [] = []
  isBlank line = Text.all Char.isSpace line

wrap :: (LL.ListLike s Char, IsString s) => Pretty s -> Pretty s
wrap p = wrapImpl (toLeaves [p]) where
  toLeaves [] = []
  toLeaves (hd:tl) = case out hd of
    Empty -> toLeaves tl
    Lit s -> wordify s ++ toLeaves tl
    Group _ -> hd : toLeaves tl
    OrElse a _ -> toLeaves (a:tl)
    Wrap _ -> hd : toLeaves tl
    Append hds -> toLeaves (toList hds ++ tl)
  wordify s0 = let s = LL.dropWhile isSpace s0 in
    if LL.null s then []
    else case LL.break isSpace s of (word1, s) -> lit word1 : wordify s

wrap_ :: Seq (Pretty s) -> Pretty s
wrap_ ps = Pretty (foldMap delta ps) (Wrap ps)

group :: Pretty s -> Pretty s
group p = Pretty (delta p) (Group p)<commit_msg>fix bug in paragraphyText<commit_after>{-# LANGUAGE DeriveFunctor       #-}
{-# LANGUAGE DeriveFoldable      #-}
{-# LANGUAGE DeriveTraversable   #-}
{-# LANGUAGE FlexibleContexts    #-}
{-# LANGUAGE OverloadedStrings   #-}
{-# LANGUAGE ViewPatterns        #-}

module Unison.Util.Pretty (
   Pretty,
   ColorText,
   align,
   backticked,
   bulleted,
   bracket,
   -- breakable
   callout,
   excerptSep,
   excerptSep',
   excerptColumn2,
   excerptColumn2Headed,
   warnCallout, blockedCallout, fatalCallout, okCallout,
   column2,
   column3,
   column3sep,
   commas,
   commented,
   oxfordCommas,
   plural,
   dashed,
   flatMap,
   group,
   hang',
   hang,
   hangUngrouped',
   hangUngrouped,
   indent,
   indentAfterNewline,
   indentN,
   indentNAfterNewline,
   leftPad,
   lines,
   linesNonEmpty,
   linesSpaced,
   lit,
   map,
   nest,
   num,
   newline,
   lineSkip,
   nonEmpty,
   numbered,
   numberedColumn2,
   orElse,
   orElses,
   paragraphyText,
   parenthesize,
   parenthesizeCommas,
   parenthesizeIf,
   preferredWidth,
   preferredHeight,
   render,
   renderUnbroken,
   rightPad,
   sep,
   sepNonEmpty,
   sepSpaced,
   shown,
   softbreak,
   spaceIfBreak,
   spaced,
   spacedMap,
   spacesIfBreak,
   string,
   surroundCommas,
   syntaxToColor,
   text,
   toANSI,
   toAnsiUnbroken,
   toHTML,
   toPlain,
   toPlainUnbroken,
   withSyntax,
   wrap,
   wrapColumn2,
   wrapString,
   black, red, green, yellow, blue, purple, cyan, white, hiBlack, hiRed, hiGreen, hiYellow, hiBlue, hiPurple, hiCyan, hiWhite, bold,
   border,
   Width
  ) where

import Unison.Prelude

import           Data.Char                      ( isSpace )
import           Data.List                      ( foldr1, intersperse )
import           Prelude                 hiding ( lines , map )
import           Unison.Util.AnnotatedText      ( annotateMaybe )
import qualified Unison.Util.ColorText         as CT
import qualified Unison.Util.SyntaxText        as ST
import           Unison.Util.Monoid             ( intercalateMap )
import qualified Data.ListLike                 as LL
import qualified Data.Sequence                 as Seq
import qualified Data.Text                     as Text
import qualified Data.Char as Char

type Width = Int
type ColorText = CT.ColorText

data Pretty s = Pretty { delta :: Delta, out :: F s (Pretty s) } deriving Eq

instance Functor Pretty where
  fmap f (Pretty d o) = Pretty d (mapLit f $ fmap (fmap f) o)

data F s r
  = Empty | Group r | Lit s | Wrap (Seq r) | OrElse r r | Append (Seq r)
  deriving (Eq, Show, Foldable, Traversable, Functor)

mapLit :: (s -> t) -> F s r -> F t r
mapLit f (Lit s) = Lit (f s)
mapLit _ Empty = Empty
mapLit _ (Group r) = Group r
mapLit _ (Wrap s) = Wrap s
mapLit _ (OrElse r s) = OrElse r s
mapLit _ (Append s) = Append s

lit :: (IsString s, LL.ListLike s Char) => s -> Pretty s
lit s = lit' (foldMap chDelta $ LL.toList s) s

lit' :: Delta -> s -> Pretty s
lit' d s = Pretty d (Lit s)

orElse :: Pretty s -> Pretty s -> Pretty s
orElse p1 p2 = Pretty (delta p1) (OrElse p1 p2)

orElses :: [Pretty s] -> Pretty s
orElses [] = mempty
orElses ps = foldr1 orElse ps

wrapImpl :: IsString s => [Pretty s] -> Pretty s
wrapImpl [] = mempty
wrapImpl (p:ps) = wrap_ . Seq.fromList $
  p : fmap (\p -> (" " <> p) `orElse` (newline <> p)) ps

wrapString :: (LL.ListLike s Char, IsString s) => String -> Pretty s
wrapString s = wrap (lit $ fromString s)

-- 1. Convert multiple adjacent blank lines to a single blank line.
-- 2. Combine adjacent non-blank lines into one line.
-- 3. Wrap each remaining line.
paragraphyText :: (LL.ListLike s Char, IsString s) => Text -> Pretty s
paragraphyText t = lines . fmap (wrap . text) .  step2 . step1 $ Text.lines t where
  step1 (line1:line2:rest) | isBlank line1 && isBlank line2 = step1 (line1:rest)
  step1 (line:rest) = line : step1 rest
  step1 [] = []
  step2 (line:line2:rest) | not (isBlank line) && not (isBlank line2) = step2 $ (line <> " " <> line2) : rest 
  step2 (line:rest) = line : step2 rest
  step2 [] = []
  isBlank line = Text.all Char.isSpace line

wrap :: (LL.ListLike s Char, IsString s) => Pretty s -> Pretty s
wrap p = wrapImpl (toLeaves [p]) where
  toLeaves [] = []
  toLeaves (hd:tl) = case out hd of
    Empty -> toLeaves tl
    Lit s -> wordify s ++ toLeaves tl
    Group _ -> hd : toLeaves tl
    OrElse a _ -> toLeaves (a:tl)
    Wrap _ -> hd : toLeaves tl
    Append hds -> toLeaves (toList hds ++ tl)
  wordify s0 = let s = LL.dropWhile isSpace s0 in
    if LL.null s then []
    else case LL.break isSpace s of (word1, s) -> lit word1 : wordify s

wrap_ :: Seq (Pretty s) -> Pretty s
wrap_ ps = Pretty (foldMap delta ps) (Wrap ps)

group :: Pretty s -> Pretty s
group p = Pretty (delta p) (Group p)
{'fraction_non_alphanumeric': 0.1058278981695771, 'fraction_numerical': 0.008626130864717021, 'mean_word_length': 2.587547169811321, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 67, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7579618', 'n_tokens_mistral': 3456, 'n_tokens_neox': 3344, 'n_words': 1082}