<commit_before>import System.IO
import Text.XML.HaXml
import Text.XML.HaXml.Posn
import Text.XML.HaXml.Util
import Codec.Binary.UTF8.String (encodeString, decodeString)
import Network.Curl hiding (Content)
{- test stub
curlGetString :: String -> [a] -> IO (a, String)
curlGetString path _ = do
    str <- readFile "test/new.rss"
    return (200, str)
---}

import Config

data Entry = Entry { title :: String }
  deriving (Show)

entries :: String -> [Entry]
entries = map entry . (deep $ tag "title") . rootContent
  where
    rootContent :: String -> Content Posn
    rootContent str = cont
      where
        Document _ _ rootElement _ = xmlParse "" str
        cont = CElem rootElement noPos

    entry :: Content Posn -> Entry
    entry t = Entry {title = tagTextContent t}

main :: IO ()
main = do
    config <- loadConfig "nicodicbot.config"
    (code, rss) <- curlGetString (rssUri config) []
    mapM_ (putStrLn . title) $ entries $ decodeString rss
<commit_msg>get rss title and link<commit_after>import System.IO
import Text.XML.HaXml
import Text.XML.HaXml.Posn
import Text.XML.HaXml.Util
import Codec.Binary.UTF8.String (encodeString, decodeString)
import Network.Curl hiding (Content)

import Config

data Entry = Entry {
    title :: String,
    link :: String}

instance Show Entry where
    show e = "{title = \"" ++ title e ++ "\", link = " ++ link e ++ "}"

entries :: String -> [Entry]
entries = map entry . (deep $ tag "item") . rootContent
  where
    rootContent :: String -> Content Posn
    rootContent str = cont
      where
        Document _ _ rootElement _ = xmlParse "" str
        cont = CElem rootElement noPos

    entry :: Content Posn -> Entry
    entry item = Entry {
        title = tagText "title" item,
        link = tagText "link" item}

    tagText :: String -> Content Posn -> String
    tagText t = tagTextContent . head . (deep $ tag t)

main :: IO ()
main = do
    config <- loadConfig "nicodicbot.config"
    (code, rss) <- curlGetString (rssUri config) []
    mapM_ (putStrLn . show) $ entries $ decodeString rss

{'fraction_non_alphanumeric': 0.11450752062105773, 'fraction_numerical': 0.0024260067928190197, 'mean_word_length': 2.935114503816794, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2826648', 'n_tokens_mistral': 640, 'n_tokens_neox': 629, 'n_words': 210}