<commit_before>-- Copyright 2008, 2009, 2010, 2011, 2012 <NAME>

-- This file is part of Vocabulink.

-- Vocabulink is free software: you can redistribute it and/or modify it under
-- the terms of the GNU Affero General Public License as published by the Free
-- Software Foundation, either version 3 of the License, or (at your option)
-- any later version.

-- Vocabulink is distributed in the hope that it will be useful, but WITHOUT
-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-- FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License
-- for more details.

-- You should have received a copy of the GNU Affero General Public License
-- along with Vocabulink. If not, see <http://www.gnu.org/licenses/>.

-- Creating links is great, but viewing a link doesn't mean that you've learned
-- it. We need a way to present links to members for regular (scheduled)
-- reviews.

module Vocabulink.Review ( newReview, scheduleNextReview
                         , reviewStats, dailyReviewStats, detailedReviewStats
                         , learnPage, upcomingLinks
                         ) where

-- For now, we have only 1 review algorithm (SuperMemo 2).

import qualified Vocabulink.Review.SM2 as SM2

import Vocabulink.App
import Vocabulink.CGI
import Vocabulink.Config
import Vocabulink.Html
import Vocabulink.Link
import Vocabulink.Link.Html
import Vocabulink.Member
import Vocabulink.Page
import Vocabulink.Utils

import qualified Data.Aeson.Encode
import qualified Data.Aeson.Generic
import qualified Data.Aeson.Types
import qualified Data.Text
import qualified Data.Text.Lazy
import Data.Text.Lazy.Builder (toLazyText)
import Data.Time.Calendar (toGregorian)
import qualified Data.Vector as V

import Prelude hiding (div, span, id)

-- Review Scheduling

-- When a member indicates that they want to review a link, we just add it to
-- the @link_to_review@ relation. This may change if we ever support multiple
-- review sets. The link review time is set to the current time by default so
-- that it immediately shows up for review (something we want no matter which
-- algorithm we're using).

-- We need to give the user feedback that they've successfully added the link
-- to their review set. For now, we redirect them back to the referrer because
-- we assume it will be the link page (which will then indicate in some way
-- that they're reviewing the link now). However, this is a good candidate for
-- an asynchronous JavaScript call.

newReview :: Member -> Integer -> App ()
newReview member linkNo = do
  $(execute' "INSERT INTO link_to_review (member_no, link_no) \
                                 \VALUES ({memberNumber member}, {linkNo})")

-- We need to schedule the next review based on the review algorithm in use.
-- The algorithm needs to know how well the item was remembered. Also, we log
-- the amount of time it took to recall the item. The SM2 algorithm does not
-- use this information (nor any SuperMemo algorithm that I know of), but we
-- may find it useful when analyzing data later.

-- @recallGrade@ is passed as a real number between 0 and 1 to allow for future
-- variations in recall rating (such as fewer choices than 1 through 5 or less
-- discrete options like a slider). 0 indicates complete failure while 1
-- indicates perfect recall.

-- @recallTime@ is the time in milliseconds.

-- All database updates during this process are wrapped in a transaction.

scheduleNextReview :: Member -> Integer -> Float -> Integer -> UTCTime -> App ()
scheduleNextReview member linkNo recallGrade recallTime reviewedAt = do
  previous <- fromJust <$> previousInterval member linkNo
  diff <- SM2.reviewInterval (memberNumber member) linkNo previous recallGrade
  h <- asks appDB
  liftIO $ withTransaction h $ do
    $(execute
      "INSERT INTO link_review (member_no, link_no, recall_grade, recall_time, actual_time, \
                               \target_time) \
                       \VALUES ({memberNumber member}, {linkNo}, {recallGrade}, {recallTime}, {reviewedAt}, \
                               \(SELECT target_time FROM link_to_review \
                                \WHERE member_no = {memberNumber member} AND link_no = {linkNo}))") h
    $(execute
      "UPDATE link_to_review \
      \SET target_time = {reviewedAt}::timestamp with time zone + {diff}::interval \
      \WHERE member_no = {memberNumber member} AND link_no = {linkNo}") h

dueForReview :: Maybe Member -> String -> String -> Int -> App [Link]
dueForReview member learn' known' n =
  case member of
    Nothing -> return []
    Just m  -> map linkFromTuple <$> $(queryTuples'
      "SELECT l.link_no, learn, known, \
             \learn_lang, known_lang, ll.name, kl.name, \
             \s.link_no IS NOT NULL, COALESCE(linkword) \
      \FROM link_to_review l \
      \INNER JOIN link USING (link_no) \
      \INNER JOIN language ll ON (ll.abbr = learn_lang) \
      \INNER JOIN language kl ON (kl.abbr = known_lang) \
      \LEFT JOIN link_soundalike s USING (link_no) \
      \LEFT JOIN link_linkword w USING (link_no) \
      \WHERE member_no = {memberNumber m} AND current_timestamp >= target_time \
        \AND learn_lang = {learn'} AND known_lang = {known'} \
      \ORDER BY target_time ASC \
      \LIMIT {n}")

-- First, use words with existing stories. Second, use linkwords or
-- soundalikes. Finally, use whatever.
newForReview :: Maybe Member -> String -> String -> Int -> App [Link]
newForReview member learn' known' n =
  case member of
    Nothing -> do
      storied <- map linkFromTuple <$> $(queryTuples'
        "SELECT l.link_no, learn, known, \
               \learn_lang, known_lang, ll.name, kl.name, \
               \s.link_no IS NOT NULL, COALESCE(linkword) \
        \FROM link l \
        \INNER JOIN language ll ON (ll.abbr = learn_lang) \
        \INNER JOIN language kl ON (kl.abbr = known_lang) \
        \LEFT JOIN link_soundalike s ON (s.link_no = l.link_no) \
        \LEFT JOIN link_linkword w ON (w.link_no = l.link_no) \
        \INNER JOIN linkword_story ss ON (ss.link_no = l.link_no) \
        \WHERE learn_lang = {learn'} AND known_lang = {known'} \
          \AND NOT deleted \
        \ORDER BY random() LIMIT {n}")
      if length storied >= n
        then return storied
        else do
          special <- map linkFromTuple <$> $(queryTuples'
            "SELECT l.link_no, learn, known, \<commit_msg>Changing due for review logic to help when someone gets backed up with links to review.<commit_after>-- Copyright 2008, 2009, 2010, 2011, 2012 <NAME>

-- This file is part of Vocabulink.

-- Vocabulink is free software: you can redistribute it and/or modify it under
-- the terms of the GNU Affero General Public License as published by the Free
-- Software Foundation, either version 3 of the License, or (at your option)
-- any later version.

-- Vocabulink is distributed in the hope that it will be useful, but WITHOUT
-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-- FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License
-- for more details.

-- You should have received a copy of the GNU Affero General Public License
-- along with Vocabulink. If not, see <http://www.gnu.org/licenses/>.

-- Creating links is great, but viewing a link doesn't mean that you've learned
-- it. We need a way to present links to members for regular (scheduled)
-- reviews.

module Vocabulink.Review ( newReview, scheduleNextReview
                         , reviewStats, dailyReviewStats, detailedReviewStats
                         , learnPage, upcomingLinks
                         ) where

-- For now, we have only 1 review algorithm (SuperMemo 2).

import qualified Vocabulink.Review.SM2 as SM2

import Vocabulink.App
import Vocabulink.CGI
import Vocabulink.Config
import Vocabulink.Html
import Vocabulink.Link
import Vocabulink.Link.Html
import Vocabulink.Member
import Vocabulink.Page
import Vocabulink.Utils

import qualified Data.Aeson.Encode
import qualified Data.Aeson.Generic
import qualified Data.Aeson.Types
import qualified Data.Text
import qualified Data.Text.Lazy
import Data.Text.Lazy.Builder (toLazyText)
import Data.Time.Calendar (toGregorian)
import qualified Data.Vector as V

import Prelude hiding (div, span, id)

-- Review Scheduling

-- When a member indicates that they want to review a link, we just add it to
-- the @link_to_review@ relation. This may change if we ever support multiple
-- review sets. The link review time is set to the current time by default so
-- that it immediately shows up for review (something we want no matter which
-- algorithm we're using).

-- We need to give the user feedback that they've successfully added the link
-- to their review set. For now, we redirect them back to the referrer because
-- we assume it will be the link page (which will then indicate in some way
-- that they're reviewing the link now). However, this is a good candidate for
-- an asynchronous JavaScript call.

newReview :: Member -> Integer -> App ()
newReview member linkNo = do
  $(execute' "INSERT INTO link_to_review (member_no, link_no) \
                                 \VALUES ({memberNumber member}, {linkNo})")

-- We need to schedule the next review based on the review algorithm in use.
-- The algorithm needs to know how well the item was remembered. Also, we log
-- the amount of time it took to recall the item. The SM2 algorithm does not
-- use this information (nor any SuperMemo algorithm that I know of), but we
-- may find it useful when analyzing data later.

-- @recallGrade@ is passed as a real number between 0 and 1 to allow for future
-- variations in recall rating (such as fewer choices than 1 through 5 or less
-- discrete options like a slider). 0 indicates complete failure while 1
-- indicates perfect recall.

-- @recallTime@ is the time in milliseconds.

-- All database updates during this process are wrapped in a transaction.

scheduleNextReview :: Member -> Integer -> Float -> Integer -> UTCTime -> App ()
scheduleNextReview member linkNo recallGrade recallTime reviewedAt = do
  previous <- fromJust <$> previousInterval member linkNo
  diff <- SM2.reviewInterval (memberNumber member) linkNo previous recallGrade
  h <- asks appDB
  liftIO $ withTransaction h $ do
    $(execute
      "INSERT INTO link_review (member_no, link_no, recall_grade, recall_time, actual_time, \
                               \target_time) \
                       \VALUES ({memberNumber member}, {linkNo}, {recallGrade}, {recallTime}, {reviewedAt}, \
                               \(SELECT target_time FROM link_to_review \
                                \WHERE member_no = {memberNumber member} AND link_no = {linkNo}))") h
    $(execute
      "UPDATE link_to_review \
      \SET target_time = {reviewedAt}::timestamp with time zone + {diff}::interval \
      \WHERE member_no = {memberNumber member} AND link_no = {linkNo}") h

-- There are at least 2 ways to decide which links should be brought up for
-- review first:
--
-- The method that Vocabulink used originally was taking the ones with the
-- oldest target time. They're the ones that have been waiting longest for
-- review, so we should review them first, right? Well, the problem is that you
-- get into a case when too many links are due for review where you spend
-- longer going through and reviewing all the due links before returning to the
-- beginning of the list and starting over again. If the list of links due for
-- review is long enough, it could take weeks to get through the list and a
-- link that was only waiting a few days for review will have been forgotten by
-- the time we get around to it. This breaks the whole principle of spaced
-- repetition as soon as you fall behind enough.

-- The method that Vocabulink now uses is to prioritize links based on when you
-- started learning them. The links that you started learning longest ago come
-- up for review with high priority as soon as they're due. Links you began
-- learning later have to wait until there's time. This way, if the learner
-- gets backed up, at least they're maintaining the links that they've been
-- spending the longest on (and are presumably the most familiar with). This
-- should also have the side effect of reducing the size of the queue faster
-- because links that the learner has been studying longer are likely to have
-- bigger payoff (time until the next review date) if they nail them.
dueForReview :: Maybe Member -> String -> String -> Int -> App [Link]
dueForReview member learn' known' n =
  case member of
    Nothing -> return []
    Just m  -> map linkFromTuple <$> $(queryTuples'
      "SELECT l.link_no, learn, known, \
             \learn_lang, known_lang, ll.name, kl.name, \
             \s.link_no IS NOT NULL, COALESCE(linkword) \
      \FROM link_to_review l \
      \INNER JOIN link USING (link_no) \
      \INNER JOIN language ll ON (ll.abbr = learn_lang) \
      \INNER JOIN language kl ON (kl.abbr = known_lang) \
      \LEFT JOIN link_soundalike s USING (link_no) \
      \LEFT JOIN link_linkword w USING (link_no) \
      \WHERE member_no = {memberNumber m} AND current_timestamp >= target_time \
        \AND learn_lang = {learn'} AND known_lang = {known'} \
      \ORDER BY added_time ASC \
      \LIMIT {n}")

-- First, use words with existing stories. Second, use linkwords or
-- soundalikes. Finally, use whatever.
newForReview :: Maybe Member -> String -> String -> Int -> App [Link]
newForReview member learn' known' n =
  case member of
    Nothing -> do
      storied <- map linkFromTuple <$> $(queryTuples'
        "SELECT l.link_no, learn, known, \
               \learn_lang, known_lang, ll.name, kl.name, \
               \s.link_no IS NOT NULL, COALESCE(linkword) \
        \FROM link l \
        \INNER JOIN language ll ON (ll.abbr = learn_lang) \
        \INNER JOIN language kl ON (kl.abbr = known_lang) \
        \LEFT JOIN link_soundalike s ON (s.link_no = l.link_no) \
        \LEFT JOIN link_linkword w ON (w.link_no = l.link_no) \
        \INNER JOIN linkword_story ss ON (ss.link_no = l.link_no) \
        \WHERE learn_lang = {learn'} AND known_lang = {known'} \
          \AND NOT deleted \
        \ORDER BY random() LIMIT {n}")
      if length storied >= n
        then return storied
        else do
          special <- map linkFromTuple <$> $(queryTuples'
            "SELECT l.link_no, learn, known, \
{'fraction_non_alphanumeric': 0.08079479368595957, 'fraction_numerical': 0.004638604264746608, 'mean_word_length': 2.8366533864541834, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2725213', 'n_tokens_mistral': 4067, 'n_tokens_neox': 3897, 'n_words': 1807}