<commit_before>﻿namespace SourceWrappers

open Mastodon.Model
open FSharp.Control

type MastodonPostWrapper(status: Status) =
    member __.SpoilerText = status.SpoilerText
    interface IPostBase with
        member __.Title = ""
        member __.HTMLDescription = status.Content
        member __.Mature = status.Sensitive.GetValueOrDefault()
        member __.Adult = status.Sensitive.GetValueOrDefault()
        member __.Tags = status.Tags |> Seq.map (fun t -> t.Name)
        member __.Timestamp = status.CreatedAt
        member __.ViewURL = status.Url

type MastodonPhotoPostWrapper(status: Status, attachment: Attachment) =
    inherit MastodonPostWrapper(status)
    interface IRemotePhotoPost with
        member __.ImageURL = attachment.Url
        member __.ThumbnailURL = attachment.PreviewUrl

type MastodonVideoPostWrapper(status: Status, attachment: Attachment) =
    inherit MastodonPostWrapper(status)
    interface IRemoteVideoPost with
        member __.VideoURL = attachment.Url
        member __.ThumbnailURL = attachment.PreviewUrl

type MastodonSourceWrapper(domain: string, token: string, photosOnly: bool) =
    inherit AsyncSeqWrapper()

    let user_task = lazy(Mastodon.Api.Accounts.VerifyCredentials(domain, token))

    let getUser = user_task.Force() |> Async.AwaitTask

    let isPhoto (a: Attachment) = a.Type = "image"
    let isAnimatedGif (a: Attachment) = a.Type = "gifv"

    override __.Name =
        if photosOnly then
            sprintf "%s (images)" domain
        else
            sprintf "%s (text + images)" domain

    override __.FetchSubmissionsInternal() = asyncSeq {
        let! user = getUser

        let mutable maxId = 0L
        let mutable more = true

        while more do
            let! toots = Mastodon.Api.Accounts.Statuses(domain, token, user.Id, maxId) |> Async.AwaitTask

            for t in toots do
                if isNull t.Reblog then
                    let photos = t.MediaAttachments |> Seq.filter isPhoto
                    let gifs =
                        if photosOnly
                        then Seq.empty
                        else t.MediaAttachments |> Seq.filter isAnimatedGif
                    if Seq.isEmpty photos && Seq.isEmpty gifs then
                        if not photosOnly then
                            yield new MastodonPostWrapper(t) :> IPostBase
                    else
                        for a in photos do
                            yield new MastodonPhotoPostWrapper(t, a) :> IPostBase
                        for a in gifs do
                            yield new MastodonVideoPostWrapper(t, a) :> IPostBase
                
            more <- not (Seq.isEmpty toots)
            if more then
                maxId <- (toots |> Seq.map (fun t -> t.Id) |> Seq.min) - 1L
    }

    override __.FetchUserInternal() = async {
        let! u = getUser
        return {
            username = u.UserName
            icon_url = u.AvatarUrl |> Option.ofObj
        }
    }<commit_msg>Mastodon: use maxId from result<commit_after>﻿namespace SourceWrappers

open Mastodon.Model
open FSharp.Control

type MastodonPostWrapper(status: Status) =
    member __.SpoilerText = status.SpoilerText
    interface IPostBase with
        member __.Title = ""
        member __.HTMLDescription = status.Content
        member __.Mature = status.Sensitive.GetValueOrDefault()
        member __.Adult = status.Sensitive.GetValueOrDefault()
        member __.Tags = status.Tags |> Seq.map (fun t -> t.Name)
        member __.Timestamp = status.CreatedAt
        member __.ViewURL = status.Url

type MastodonPhotoPostWrapper(status: Status, attachment: Attachment) =
    inherit MastodonPostWrapper(status)
    interface IRemotePhotoPost with
        member __.ImageURL = attachment.Url
        member __.ThumbnailURL = attachment.PreviewUrl

type MastodonVideoPostWrapper(status: Status, attachment: Attachment) =
    inherit MastodonPostWrapper(status)
    interface IRemoteVideoPost with
        member __.VideoURL = attachment.Url
        member __.ThumbnailURL = attachment.PreviewUrl

type MastodonSourceWrapper(domain: string, token: string, photosOnly: bool) =
    inherit AsyncSeqWrapper()

    let user_task = lazy(Mastodon.Api.Accounts.VerifyCredentials(domain, token))

    let getUser = user_task.Force() |> Async.AwaitTask

    let isPhoto (a: Attachment) = a.Type = "image"
    let isAnimatedGif (a: Attachment) = a.Type = "gifv"

    override __.Name =
        if photosOnly then
            sprintf "%s (images)" domain
        else
            sprintf "%s (text + images)" domain

    override __.FetchSubmissionsInternal() = asyncSeq {
        let! user = getUser

        let mutable maxId = 0L
        let mutable more = true

        while more do
            let! toots = Mastodon.Api.Accounts.Statuses(domain, token, user.Id, maxId) |> Async.AwaitTask

            for t in toots do
                if isNull t.Reblog then
                    let photos = t.MediaAttachments |> Seq.filter isPhoto
                    let gifs =
                        if photosOnly
                        then Seq.empty
                        else t.MediaAttachments |> Seq.filter isAnimatedGif
                    if Seq.isEmpty photos && Seq.isEmpty gifs then
                        if not photosOnly then
                            yield new MastodonPostWrapper(t) :> IPostBase
                    else
                        for a in photos do
                            yield new MastodonPhotoPostWrapper(t, a) :> IPostBase
                        for a in gifs do
                            yield new MastodonVideoPostWrapper(t, a) :> IPostBase
                
            if toots.MaxId.HasValue then
                maxId <- toots.MaxId.Value
                more <- true
            else
                more <- false
    }

    override __.FetchUserInternal() = async {
        let! u = getUser
        return {
            username = u.UserName
            icon_url = u.AvatarUrl |> Option.ofObj
        }
    }
{'fraction_non_alphanumeric': 0.07255259234719232, 'fraction_numerical': 0.0004969355640218651, 'mean_word_length': 1.8697718631178708, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1651415', 'n_tokens_mistral': 1702, 'n_tokens_neox': 1629, 'n_words': 449}