<commit_before>-- | Download and import feeds from various sources.

module HN.Model.Feeds where

import HN.Data
import HN.Monads
import HN.Model.Items
import HN.Types
import HN.Curl

import qualified HN.Model.Mailman (downloadFeed)

import Control.Applicative
import Network.URI
import Snap.App

import Text.Feed.Import
import Text.Feed.Query
import Text.Feed.Types

--------------------------------------------------------------------------------
-- Various service feeds

importHaskellCafeNative :: Model c s (Either String ())
importHaskellCafeNative =
  importMailman 50
                HaskellCafeNative
                "https://mail.haskell.org/pipermail/haskell-cafe/"
                (\item -> return (item { niTitle = strip (niTitle item) }))

  where strip x | isPrefixOf "re: " (map toLower x) = strip (drop 4 x)
                | isPrefixOf label x = drop (length label) x
                | otherwise = x
        label = "[Haskell-cafe]"

importPlanetHaskell :: Model c s (Either String ())
importPlanetHaskell =
  importGeneric PlanetHaskell "http://planet.haskell.org/rss20.xml"

importJobs :: Model c s (Either String ())
importJobs =
  importGeneric Jobs "http://www.haskellers.com/feed/jobs"

importStackOverflow :: Model c s (Either String ())
importStackOverflow = do
  importGeneric StackOverflow "http://stackoverflow.com/feeds/tag/haskell"
  importGeneric StackOverflow "http://programmers.stackexchange.com/feeds/tag/haskell"
  importGeneric StackOverflow "http://codereview.stackexchange.com/feeds/tag/haskell"

importHaskellWiki :: Model c s (Either String ())
importHaskellWiki =
  importGeneric HaskellWiki "http://wiki.haskell.org/index.php?title=Special:RecentChanges&feed=atom"

importHackage :: Model c s (Either String ())
importHackage =
  importGeneric Hackage "http://hackage.haskell.org/packages/recent.rss"

-- | Import all vimeo content.
importVimeo :: Model c s (Either String ())
importVimeo = do
  importGeneric Vimeo "https://vimeo.com/channels/haskell/videos/rss"
  importGeneric Vimeo "https://vimeo.com/channels/galois/videos/rss"
  importGenerically Vimeo "https://vimeo.com/rickasaurus/videos/rss"
    (\ni -> if isInfixOf "haskell" (map toLower (niTitle ni)) then return ni else Nothing)

-- | Import @remember'd IRC quotes from ircbrowse.
importIrcQuotes :: Model c s (Either String ())
importIrcQuotes = do
  importGeneric IrcQuotes
                "http://ircbrowse.net/quotes.rss"
<commit_msg>Turn StackOverflow URL to HTTPS<commit_after>-- | Download and import feeds from various sources.

module HN.Model.Feeds where

import HN.Data
import HN.Monads
import HN.Model.Items
import HN.Types
import HN.Curl

import qualified HN.Model.Mailman (downloadFeed)

import Control.Applicative
import Network.URI
import Snap.App

import Text.Feed.Import
import Text.Feed.Query
import Text.Feed.Types

--------------------------------------------------------------------------------
-- Various service feeds

importHaskellCafeNative :: Model c s (Either String ())
importHaskellCafeNative =
  importMailman 50
                HaskellCafeNative
                "https://mail.haskell.org/pipermail/haskell-cafe/"
                (\item -> return (item { niTitle = strip (niTitle item) }))

  where strip x | isPrefixOf "re: " (map toLower x) = strip (drop 4 x)
                | isPrefixOf label x = drop (length label) x
                | otherwise = x
        label = "[Haskell-cafe]"

importPlanetHaskell :: Model c s (Either String ())
importPlanetHaskell =
  importGeneric PlanetHaskell "http://planet.haskell.org/rss20.xml"

importJobs :: Model c s (Either String ())
importJobs =
  importGeneric Jobs "http://www.haskellers.com/feed/jobs"

importStackOverflow :: Model c s (Either String ())
importStackOverflow = do
  importGeneric StackOverflow "https://stackoverflow.com/feeds/tag/haskell"
  importGeneric StackOverflow "https://programmers.stackexchange.com/feeds/tag/haskell"
  importGeneric StackOverflow "https://codereview.stackexchange.com/feeds/tag/haskell"

importHaskellWiki :: Model c s (Either String ())
importHaskellWiki =
  importGeneric HaskellWiki "http://wiki.haskell.org/index.php?title=Special:RecentChanges&feed=atom"

importHackage :: Model c s (Either String ())
importHackage =
  importGeneric Hackage "http://hackage.haskell.org/packages/recent.rss"

-- | Import all vimeo content.
importVimeo :: Model c s (Either String ())
importVimeo = do
  importGeneric Vimeo "https://vimeo.com/channels/haskell/videos/rss"
  importGeneric Vimeo "https://vimeo.com/channels/galois/videos/rss"
  importGenerically Vimeo "https://vimeo.com/rickasaurus/videos/rss"
    (\ni -> if isInfixOf "haskell" (map toLower (niTitle ni)) then return ni else Nothing)

-- | Import @remember'd IRC quotes from ircbrowse.
importIrcQuotes :: Model c s (Either String ())
importIrcQuotes = do
  importGeneric IrcQuotes
                "http://ircbrowse.net/quotes.rss"

{'fraction_non_alphanumeric': 0.13800122124974556, 'fraction_numerical': 0.0020354162426216163, 'mean_word_length': 3.2545454545454544, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 11, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7352771', 'n_tokens_mistral': 1633, 'n_tokens_neox': 1486, 'n_words': 382}