<commit_before>{-# LANGUAGE OverloadedStrings #-}
module Scrapers.Einstein (
     scrapEinstein
   , EinsteinScrapResult
   ) where

import Prelude
import Helpers.Scraping (openUrl)
import Text.HTML.TagSoup
import Data.List.Split (splitEvery)
import Control.Monad (guard, liftM2)
import Data.Char (isNumber)
import Data.Maybe (listToMaybe)
import qualified Data.Text as T

-- I use tag soup as it apperently fixes unicode characters
type EinsteinScrapResult = Maybe (Int, [[String]])

scrapEinstein :: IO EinsteinScrapResult
scrapEinstein = do
    body <- openUrl "http://www.butlercatering.se/Lunchmeny"
    return (parse body)

printMenuToStdout :: IO()
printMenuToStdout = do
    Just (week, sss) <- scrapEinstein
    putStrLn $ "\nVecka " ++ show week ++ ":"
    mapM_ (putStrLn . unlines) sss

parse :: String -> Maybe (Int, [[String]])
parse body = liftM2 (,) mweek mmenu<commit_msg>Fix so parse from multiple ES urls [close #32]<commit_after>{-# LANGUAGE OverloadedStrings #-}
module Scrapers.Einstein (
     scrapEinstein
   , EinsteinScrapResult
   ) where

import Prelude
import Helpers.Scraping (openUrl)
import Text.HTML.TagSoup
import Data.List.Split (splitEvery)
import Control.Monad (guard, liftM2)
import Data.Char (isNumber)
import Data.Maybe (listToMaybe)
import Control.Monad (msum)
import qualified Data.Text as T

-- I use tag soup as it apperently fixes unicode characters
type EinsteinScrapResult = Maybe (Int, [[String]])

scrapEinstein :: IO EinsteinScrapResult
scrapEinstein = do
    let urls = ["http://www.butlercatering.se/" ++ x | x <- ["", "Lunchmeny"]]
    fmap (msum . map parse) $ mapM openUrl urls

printMenuToStdout :: IO ()
printMenuToStdout = do
    Just (week, sss) <- scrapEinstein
    putStrLn $ "\nVecka " ++ show week ++ ":"
    mapM_ (putStrLn . unlines) sss

parse :: String -> Maybe (Int, [[String]])
parse body = liftM2 (,) mweek mmenu
{'fraction_non_alphanumeric': 0.11424017003188097, 'fraction_numerical': 0.003188097768331562, 'mean_word_length': 4.230555555555555, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5245423', 'n_tokens_mistral': 653, 'n_tokens_neox': 625, 'n_words': 187}