<commit_before>module Main where

import Text.RDF.Core
import qualified Text.RDF.TriplesGraph   as NG
import qualified Text.RDF.NTriplesParser as NP
import qualified Text.RDF.MGraph         as TG
import qualified Text.RDF.TurtleParser   as TP

import Data.ByteString.Char8(ByteString)
import qualified Data.ByteString.Char8 as B

import System.Environment
import System.IO
import System.Exit
import System.Console.GetOpt

import Control.Monad

import Data.List
import Data.Char(isLetter)
import Text.Printf(hPrintf)

-- TODO: doesn't obey the output format yet, because there is no serializer for turtle yet.

main :: IO ()
main = 
  do (opts, args) <- (getArgs >>= compilerOpts)
     when (elem Help opts)
       (putStrLn (usageInfo header options) >> exitWith ExitSuccess)
     when (elem Version opts)
       (putStrLn version >> exitWith ExitSuccess)
     when (null args)
       (ioError (userError ("\n\n" ++ "INPUT-URI required\n\n" ++ usageInfo header options)))
     let quiet         = elem Quiet opts
         inputUri      = head args
         inputFormat   = getWithDefault (InputFormat "turtle") opts
         outputFormat  = getWithDefault (OutputFormat "ntriples") opts
         inputBaseUri  = getInputBaseUri inputUri args opts
         outputBaseUri = getWithDefault (OutputBaseUri inputBaseUri) opts
     when (not quiet)
       (hPrintf stderr "      INPUT-URI:  %s\n\n" inputUri     >>
        hPrintf stderr "   INPUT-FORMAT:  %s\n"   inputFormat  >>
        hPrintf stderr " INPUT-BASE-URI:  %s\n\n" inputBaseUri >>
        hPrintf stderr "  OUTPUT-FORMAT:  %s\n"   outputFormat >>
        hPrintf stderr "OUTPUT-BASE-URI:  %s\n\n" outputBaseUri)
     let mInputUri  = if inputBaseUri == "-" then Nothing else Just (BaseUrl $ s2b inputBaseUri)
         docUri     = inputUri
     case (inputFormat, isUri $ s2b inputUri) of
       ("turtle",    True) -> TP.parseURL mInputUri docUri inputUri
                                >>= \(res :: Either ParseFailure TG.MGraph) -> write res
       ("turtle",   False) -> TP.parseFile mInputUri docUri inputUri
                                >>= \(res :: Either ParseFailure TG.MGraph) -> write res
       ("ntriples",  True) -> NP.parseURL inputUri
                                >>= \(res :: Either ParseFailure NG.TriplesGraph) -> write res
       ("ntriples", False) -> NP.parseFile inputUri
                                >>= \(res :: Either ParseFailure NG.TriplesGraph) -> write res
       (str     ,   _    ) -> putStrLn ("Invalid format: " ++ str) >> exitFailure
     --putStrLn $ show (opts, args)
<commit_msg>Rdf4hMain.hs: added ability to read from stdin if INPUT-URI is '-'.<commit_after>module Main where

import Text.RDF.Core
import qualified Text.RDF.TriplesGraph   as NG
import qualified Text.RDF.NTriplesParser as NP
import qualified Text.RDF.MGraph         as TG
import qualified Text.RDF.TurtleParser   as TP

import Data.ByteString.Char8(ByteString)
import qualified Data.ByteString.Char8 as B

import System.Environment
import System.IO
import System.Exit
import System.Console.GetOpt

import Control.Monad

import Data.List
import Data.Char(isLetter)
import Text.Printf(hPrintf)

-- TODO: doesn't obey the output format yet, because there is no serializer for turtle yet.

main :: IO ()
main = 
  do (opts, args) <- (getArgs >>= compilerOpts)
     when (elem Help opts)
       (putStrLn (usageInfo header options) >> exitWith ExitSuccess)
     when (elem Version opts)
       (putStrLn version >> exitWith ExitSuccess)
     when (null args)
       (ioError (userError ("\n\n" ++ "INPUT-URI required\n\n" ++ usageInfo header options)))
     let quiet         = elem Quiet opts
         inputUri      = head args
         inputFormat   = getWithDefault (InputFormat "turtle") opts
         outputFormat  = getWithDefault (OutputFormat "ntriples") opts
         inputBaseUri  = getInputBaseUri inputUri args opts
         outputBaseUri = getWithDefault (OutputBaseUri inputBaseUri) opts
     when (not quiet)
       (hPrintf stderr "      INPUT-URI:  %s\n\n" inputUri     >>
        hPrintf stderr "   INPUT-FORMAT:  %s\n"   inputFormat  >>
        hPrintf stderr " INPUT-BASE-URI:  %s\n\n" inputBaseUri >>
        hPrintf stderr "  OUTPUT-FORMAT:  %s\n"   outputFormat >>
        hPrintf stderr "OUTPUT-BASE-URI:  %s\n\n" outputBaseUri)
     let mInputUri  = if inputBaseUri == "-" then Nothing else Just (BaseUrl $ s2b inputBaseUri)
         docUri     = inputUri
     case (inputFormat, isUri $ s2b inputUri) of
       ("turtle",    True) -> TP.parseURL mInputUri docUri inputUri
                                >>= \(res :: Either ParseFailure TG.MGraph) -> write res
       ("turtle",   False) -> (if inputUri /= "-" 
                                  then TP.parseFile mInputUri docUri inputUri 
                                  else getContents >>= return . TP.parseString mInputUri docUri)
                                >>= \(res :: Either ParseFailure TG.MGraph) -> write res
       ("ntriples",  True) -> NP.parseURL inputUri
                                >>= \(res :: Either ParseFailure NG.TriplesGraph) -> write res
       ("ntriples", False) -> (if inputUri /= "-"
                                  then NP.parseFile inputUri
                                  else getContents >>= return . NP.parseString)
                                >>= \(res :: Either ParseFailure NG.TriplesGraph) -> write res
       (str     ,   _    ) -> putStrLn ("Invalid format: " ++ str) >> exitFailure
     --putStrLn $ show (opts, args)

{'fraction_non_alphanumeric': 0.10009049773755656, 'fraction_numerical': 0.0016289592760180996, 'mean_word_length': 2.2717584369449377, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 73, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3652471', 'n_tokens_mistral': 1662, 'n_tokens_neox': 1607, 'n_words': 484}