<commit_before>{-# LANGUAGE PatternGuards, CPP #-}
{-# LANGUAGE TemplateHaskell #-}
{-# OPTIONS_GHC -fno-warn-orphans #-}
-----------------------------------------------------------------------------
-- |
-- Module      :  Text.CSL.Parser
-- Copyright   :  (c) <NAME>
-- License     :  BSD-style (see LICENSE)
--
-- Maintainer  :  <NAME> <<EMAIL>>
-- Stability   :  unstable
-- Portability :  unportable
--
-- The CS Language parsers
--
-----------------------------------------------------------------------------

module Text.CSL.Parser where

import Text.CSL.Reference ( readNum )
import Text.CSL.Style
import Text.CSL.Pickle
#ifdef EMBED_DATA_FILES
import Data.FileEmbed
import qualified Data.ByteString as S
#else
import Paths_pandoc_citeproc (getDataFileName)
import System.Directory  ( doesFileExist   )
#endif

import Data.Char        ( isUpper, toUpper, toLower )
import Data.Maybe       ( catMaybes                 )
import qualified Data.ByteString.Lazy as L
import Data.ByteString.Lazy.UTF8 ( fromString )

#ifdef USE_NETWORK
import Network.HTTP ( getResponseBody, mkRequest, RequestMethod(..) )
import Network.Browser ( browse, setAllowRedirects, setUserAgent, request )
import Network.URI ( parseURI, URI(..) )
#endif

-- | Read and parse a CSL style file into the internal style
-- representation, the 'Style'.
readCSLFile :: FilePath -> IO Style
readCSLFile src = do
#ifdef USE_NETWORK
  let readURI u = do rsp <- browse $ do
                              setAllowRedirects True
                              setUserAgent "citeproc-hs"
                              request $ mkRequest GET u
                     getResponseBody (Right $ snd rsp)
  f <- case parseURI src of
         Just u | uriScheme u `elem` ["http:","https:"] -> readURI u
         _                                              -> readFile' src
#else
  f <- readFile' src
#endif
  parseCSL' f

-- | Parse a 'String' into a fully localized 'Style'
parseCSL :: String -> IO Style
parseCSL = parseCSL' . fromString

parseCSL' :: L.ByteString -> IO Style
parseCSL' f = do
  let s = readXmlString xpStyle f
  l <- readLocaleFile (styleDefaultLocale s)
  return s { styleLocale = mergeLocales (styleDefaultLocale s) l (styleLocale s)}

#ifdef EMBED_DATA_FILES
localeFiles :: [(FilePath, S.ByteString)]
localeFiles = $(embedDir "locales/")
#endif

readLocaleFile :: String -> IO Locale
readLocaleFile s = do
#ifdef EMBED_DATA_FILES
  f <- case s of
         x | length x == 2 -> let fn = ("locales-" ++
                                        maybe "en-US"
                                        id (lookup x langBase) ++ ".xml")
                              in case lookup fn localeFiles of
                                   Just x' -> return x'
                                   _       -> error "could not load the locale file"
            | otherwise     -> case lookup ("locales-" ++ take 5 x ++ ".xml") localeFiles of
                                 Just x' -> return x'
                                 _       -> error "could not load the locale file"
  return $ readXmlString xpLocale $ L.fromChunks [f]
#else
  f <- case s of
         x | length x == 2 -> getDataFileName ("locales/locales-" ++
                                               maybe "en-US"
                                               id (lookup x langBase) ++ ".xml")
           | otherwise     -> getDataFileName ("locales/locales-" ++ take 5 x ++ ".xml")
  b <- doesFileExist f
  if b
     then readXmlFile xpLocale f
     else readLocaleFile $ take 2 s
#endif

instance XmlPickler Layout where
    xpickle = xpWrap (uncurry3 Layout, \(Layout f d e) -> (f,d,e)) $
              xpIElem "layout" $
              xpTriple xpickle xpDelimiter xpickle

instance XmlPickler Element where
    xpickle = xpAlt tag ps
        where
          tag (Choose       {}) =  0
          tag (Macro        {}) =  1
          tag (Const        {}) =  2
          tag (Variable     {}) =  4<commit_msg>Avoid infinite loop if locale files not found!<commit_after>{-# LANGUAGE PatternGuards, CPP #-}
{-# LANGUAGE TemplateHaskell #-}
{-# OPTIONS_GHC -fno-warn-orphans #-}
-----------------------------------------------------------------------------
-- |
-- Module      :  Text.CSL.Parser
-- Copyright   :  (c) <NAME>
-- License     :  BSD-style (see LICENSE)
--
-- Maintainer  :  <NAME> <<EMAIL>>
-- Stability   :  unstable
-- Portability :  unportable
--
-- The CS Language parsers
--
-----------------------------------------------------------------------------

module Text.CSL.Parser where

import Text.CSL.Reference ( readNum )
import Text.CSL.Style
import Text.CSL.Pickle
#ifdef EMBED_DATA_FILES
import Data.FileEmbed
import qualified Data.ByteString as S
#else
import Paths_pandoc_citeproc (getDataFileName)
import System.Directory  ( doesFileExist   )
#endif

import Data.Char        ( isUpper, toUpper, toLower )
import Data.Maybe       ( catMaybes                 )
import qualified Data.ByteString.Lazy as L
import Data.ByteString.Lazy.UTF8 ( fromString )

#ifdef USE_NETWORK
import Network.HTTP ( getResponseBody, mkRequest, RequestMethod(..) )
import Network.Browser ( browse, setAllowRedirects, setUserAgent, request )
import Network.URI ( parseURI, URI(..) )
#endif

-- | Read and parse a CSL style file into the internal style
-- representation, the 'Style'.
readCSLFile :: FilePath -> IO Style
readCSLFile src = do
#ifdef USE_NETWORK
  let readURI u = do rsp <- browse $ do
                              setAllowRedirects True
                              setUserAgent "citeproc-hs"
                              request $ mkRequest GET u
                     getResponseBody (Right $ snd rsp)
  f <- case parseURI src of
         Just u | uriScheme u `elem` ["http:","https:"] -> readURI u
         _                                              -> readFile' src
#else
  f <- readFile' src
#endif
  parseCSL' f

-- | Parse a 'String' into a fully localized 'Style'
parseCSL :: String -> IO Style
parseCSL = parseCSL' . fromString

parseCSL' :: L.ByteString -> IO Style
parseCSL' f = do
  let s = readXmlString xpStyle f
  l <- readLocaleFile (styleDefaultLocale s)
  return s { styleLocale = mergeLocales (styleDefaultLocale s) l (styleLocale s)}

#ifdef EMBED_DATA_FILES
localeFiles :: [(FilePath, S.ByteString)]
localeFiles = $(embedDir "locales/")
#endif

readLocaleFile :: String -> IO Locale
readLocaleFile s = do
#ifdef EMBED_DATA_FILES
  f <- case s of
         x | length x == 2 -> let fn = ("locales-" ++
                                        maybe "en-US"
                                        id (lookup x langBase) ++ ".xml")
                              in case lookup fn localeFiles of
                                   Just x' -> return x'
                                   _       -> error "could not load the locale file"
            | otherwise     -> case lookup ("locales-" ++ take 5 x ++ ".xml") localeFiles of
                                 Just x' -> return x'
                                 _       -> error "could not load the locale file"
  return $ readXmlString xpLocale $ L.fromChunks [f]
#else
  f <- case s of
         x | length x == 2 -> getDataFileName ("locales/locales-" ++
                                               maybe "en-US"
                                               id (lookup x langBase) ++ ".xml")
           | otherwise     -> getDataFileName ("locales/locales-" ++ take 5 x ++ ".xml")
  exists <- doesFileExist f
  if not exists && length s > 2
     then readLocaleFile $ take 2 s  -- try again with base locale
     else readXmlFile xpLocale f
#endif

instance XmlPickler Layout where
    xpickle = xpWrap (uncurry3 Layout, \(Layout f d e) -> (f,d,e)) $
              xpIElem "layout" $
              xpTriple xpickle xpDelimiter xpickle

instance XmlPickler Element where
    xpickle = xpAlt tag ps
        where
          tag (Choose       {}) =  0
          tag (Macro        {}) =  1
          tag (Const        {}) =  2
          tag (Variable     {}) =  4
{'fraction_non_alphanumeric': 0.13485113835376533, 'fraction_numerical': 0.002877157868401301, 'mean_word_length': 1.6170212765957446, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 32, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1742352', 'n_tokens_mistral': 2264, 'n_tokens_neox': 2192, 'n_words': 719}