<commit_before>import {
    StorageModule,
    StorageModuleConfig,
    StorageModuleConstructorArgs,
} from '@worldbrain/storex-pattern-modules'
import { normalizeUrl } from '@worldbrain/memex-url-utils'
import { ReadableData } from 'src/reader/types'
import { now } from 'moment'
import Readability from 'readability/Readability'
import { fetchDOMFromUrl } from 'src/page-analysis/background/fetch-page-data'
import DOMPurify from 'dompurify'

export default class ReaderStorage extends StorageModule {
    constructor(private options: StorageModuleConstructorArgs) {
        super(options)
    }

    getConfig = (): StorageModuleConfig => ({
        collections: {
            readable: {
                version: new Date('2020-05-12'),
                fields: {
                    url: { type: 'string' },
                    fullUrl: { type: 'text' },
                    title: { type: 'string' },
                    length: { type: 'string' },
                    content: { type: 'text', optional: true },
                    textContent: { type: 'text', optional: true },
                    strategy: { type: 'string' },
                },
                indices: [
                    { field: 'url', pk: true },
                    { field: 'fullUrl' },
                    { field: 'title' },
                ],
            },
        },
        operations: {
            createReadable: {
                operation: 'createObject',
                collection: 'readable',
            },
            findReadableByUrl: {
                operation: 'findObject',
                collection: 'readable',
                args: {
                    url: '$url:string',
                },
            },
        },
    })

    readableExists = async (url: string): Promise<boolean> => {
        const normalizedUrl = normalizeUrl(url, {})
        const existingPage = await this.operation('findReadableByUrl', {
            url: normalizedUrl,
        })
        return !!existingPage
    }

    createReadableIfNotExists = async (
        readableData: ReadableData,
    ): Promise<void> => {
        const normalizedUrl = normalizeUrl(readableData.url, {})
        const exists = await this.readableExists(normalizedUrl)
        if (!exists) {
            await this.operation('createReadable', {
                ...readableData,
                url: normalizedUrl,
            })
        }
    }

    parseAndSaveReadable = async ({
        fullUrl,
        doc,
    }: {
        fullUrl: string
        doc?: Document
    }): Promise<ReadableData> => {
        const normalizedUrl = normalizeUrl(fullUrl, {})

        let document = doc
        if (!document) {
            console.log(`Reader::Parser Document fetching from URL ${fullUrl}`)

            document = await fetchDOMFromUrl(fullUrl, 5000).run()
        } else {
            console.log(`Reader::Parser Document given`)
        }

        console.log(
            `Reader::Parser Original Doc Size - ${
                document.body.outerHTML.length / 1000
            }k`,
        )
        console.time('Reader::Parser::ParseTime')

        const article = new Readability(document).parse()
        article.content = DOMPurify.sanitize(article.contnet)
        console.timeEnd('Reader::Parser::ParseTime')
        console.log(
            `Reader::Parser Readable Doc Size - ${
                document.body.outerHTML.length / 1000
            }k`,
        )

        const readableData = {
            title: article.title,
            content: article.content,
            textContent: article.textContent,
            url: normalizedUrl,
            length: article.length,
            fullUrl,
            strategy: 'mozilla/readability',
            created: now(),
        }

        await this.operation('createReadable', readableData)

        return readableData as ReadableData<commit_msg>fix typo leading to content not being displayed<commit_after>import {
    StorageModule,
    StorageModuleConfig,
    StorageModuleConstructorArgs,
} from '@worldbrain/storex-pattern-modules'
import { normalizeUrl } from '@worldbrain/memex-url-utils'
import { ReadableData } from 'src/reader/types'
import { now } from 'moment'
import Readability from 'readability/Readability'
import { fetchDOMFromUrl } from 'src/page-analysis/background/fetch-page-data'
import DOMPurify from 'dompurify'

export default class ReaderStorage extends StorageModule {
    constructor(private options: StorageModuleConstructorArgs) {
        super(options)
    }

    getConfig = (): StorageModuleConfig => ({
        collections: {
            readable: {
                version: new Date('2020-05-12'),
                fields: {
                    url: { type: 'string' },
                    fullUrl: { type: 'text' },
                    title: { type: 'string' },
                    length: { type: 'string' },
                    content: { type: 'text', optional: true },
                    textContent: { type: 'text', optional: true },
                    strategy: { type: 'string' },
                },
                indices: [
                    { field: 'url', pk: true },
                    { field: 'fullUrl' },
                    { field: 'title' },
                ],
            },
        },
        operations: {
            createReadable: {
                operation: 'createObject',
                collection: 'readable',
            },
            findReadableByUrl: {
                operation: 'findObject',
                collection: 'readable',
                args: {
                    url: '$url:string',
                },
            },
        },
    })

    readableExists = async (url: string): Promise<boolean> => {
        const normalizedUrl = normalizeUrl(url, {})
        const existingPage = await this.operation('findReadableByUrl', {
            url: normalizedUrl,
        })
        return !!existingPage
    }

    createReadableIfNotExists = async (
        readableData: ReadableData,
    ): Promise<void> => {
        const normalizedUrl = normalizeUrl(readableData.url, {})
        const exists = await this.readableExists(normalizedUrl)
        if (!exists) {
            await this.operation('createReadable', {
                ...readableData,
                url: normalizedUrl,
            })
        }
    }

    parseAndSaveReadable = async ({
        fullUrl,
        doc,
    }: {
        fullUrl: string
        doc?: Document
    }): Promise<ReadableData> => {
        const normalizedUrl = normalizeUrl(fullUrl, {})

        let document = doc
        if (!document) {
            console.log(`Reader::Parser Document fetching from URL ${fullUrl}`)

            document = await fetchDOMFromUrl(fullUrl, 5000).run()
        } else {
            console.log(`Reader::Parser Document given`)
        }

        console.log(
            `Reader::Parser Original Doc Size - ${
                document.body.outerHTML.length / 1000
            }k`,
        )
        console.time('Reader::Parser::ParseTime')

        const article = new Readability(document).parse()
        article.content = DOMPurify.sanitize(article.content)
        console.timeEnd('Reader::Parser::ParseTime')
        console.log(
            `Reader::Parser Readable Doc Size - ${
                document.body.outerHTML.length / 1000
            }k`,
        )

        const readableData = {
            title: article.title,
            content: article.content,
            textContent: article.textContent,
            url: normalizedUrl,
            length: article.length,
            fullUrl,
            strategy: 'mozilla/readability',
            created: now(),
        }

        await this.operation('createReadable', readableData)

        return readableData as ReadableData
{'fraction_non_alphanumeric': 0.10874583226468325, 'fraction_numerical': 0.005129520389843549, 'mean_word_length': 1.6563351498637602, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6253363', 'n_tokens_mistral': 2040, 'n_tokens_neox': 1905, 'n_words': 426}