<commit_before>/* eslint promise/param-names: 0 */
import initSearchIndex from 'search-index'
import stream from 'stream'
import stopword from 'stopword'

import pipeline from './search-index-pipeline'
import { convertMetaDocId } from 'src/activity-logger'

const indexOpts = {
    batchSize: 500,
    appendOnly: true,
    indexPath: 'worldbrain-index',
    logLevel: 'info',
    preserveCase: false,
    compositeField: false,
    nGramLength: 1,
    separator: /[|' .,\-|(\n)]+/,
    stopwords: stopword.en,
    fieldOptions: {
        visits: {
            fieldedSearch: true,
        },
        bookmarks: {
            fieldedSearch: true,
        },
        latest: {
            sortable: true,
        },
        content: {
            fieldedSearch: true,
        },
        url: {
            weight: 10,
            fieldedSearch: true,
            separator: '/',
        },
    },
}

const standardResponse = (resolve, reject) =>
    (err, data = true) => err ? reject(err) : resolve(data)

// Simply extracts the timestamp component out the ID of a visit or bookmark doc,
//  which is the only data we want at the moment.
const transformMetaDoc = doc => convertMetaDocId(doc._id).timestamp

/**
 * NOTE: Assumes sorted arrays, last index containing latest.
 * @param {Array<string>} visits Sorted array of UNIX timestamp strings.
 * @param {Array<string>} bookmarks Sorted array of UNIX timestamp strings.
 * @returns {string} Latest UNIX timestamp of all params.
 */
const getLatestMeta = (visits, bookmarks) => {
    const numVisits = visits.length
    const numBookmarks = bookmarks.length

    if (numVisits && numBookmarks) { // Both arrays have timestamps
        return visits[numVisits - 1] > bookmarks[numBookmarks - 1]
            ? visits[numVisits - 1]
            : bookmarks[numBookmarks - 1]
    } else if (numBookmarks) { // Only bookmark array has timestamps
        return bookmarks[numBookmarks - 1]
    } else { // Only visit array has timestamps
        return visits[numVisits - 1]
    }
}

// Groups input docs into standard index doc structure
const transformPageAndMetaDocs = ({ pageDoc, visitDocs = [], bookmarkDocs = [] }) => {
    const visits = visitDocs.map(transformMetaDoc)
    const bookmarks = bookmarkDocs.map(transformMetaDoc)

    return {
        ...pipeline(pageDoc),
        latest: getLatestMeta(visits, bookmarks),
        visits,
        bookmarks,
    }
}

// Wraps instance creation in a Promise for use in async interface methods
const indexP = new Promise((...args) => initSearchIndex(indexOpts, standardResponse(...args)))

/**
 * @param doc The doc to queue up for adding into the index.
 * @returns Boolean denoting that the add was successful (else error thrown).
 */
async function addConcurrent(doc) {
    const index = await indexP
    const input = [doc]

    return new Promise((...args) => index.concurrentAdd(indexOpts, input, standardResponse(...args)))
}

/**
 * Adds a new page doc + any associated visit/bookmark docs to the index. This method
 * uses the concurrency-safe index add method.
 * @param {any} Object containing a `pageDoc` (required) and optionally any associated `visitDocs` and `bookmarkDocs`.
 * @returns {Promise} Promise resolving when indexing is complete, or rejecting for any index errors.
 */
export async function addPageConcurrent({ pageDoc, visitDocs = [], bookmarkDocs = [] }) {
    const indexDoc = transformPageAndMetaDocs({ pageDoc, visitDocs, bookmarkDocs })

    return await addConcurrent(indexDoc)
}

/**
 * Adds a new page doc + any associated visit/bookmark docs to the index. This method
 * is *NOT* concurrency safe.
 * @param {any} Object containing a `pageDoc` (required) and optionally any associated `visitDocs` and `bookmarkDocs`.
 * @returns {Promise} Promise resolving when indexing is complete, or rejecting for any index errors.
 */
export async function addPage({ pageDoc, visitDocs = [], bookmarkDocs = [] }) {
    const index = await indexP

    const indexDoc = transformPageAndMetaDocs({ pageDoc, visitDocs, bookmarkDocs })

    return new Promise((resolve, reject) => {
        // Set up add pipeline
        const inputStream = new stream.Readable({ objectMode: true })
        inputStream
            .pipe(index.defaultPipeline())
            .pipe(index.add())
            .on('finish', resolve)
            .on('error', reject)

        // Push index doc onto stream + null to signify end-of-stream
        inputStream.push(indexDoc)
        inputStream.push(null)
    })
}

/**
 * Returns a function that affords adding either a visit or bookmark to an index.
 * The function should perform the following update, using the associated page doc ID:
 *  - grab the existing index doc matching the page doc ID
 *  - perform update to appropriate timestamp field, by APPENDING new timestamp to field
 *  - delete the original doc form the index
 *  - add the updated doc into the index
 * NOTE: this appends the timestamp extracted from the given metaDoc, hence to maintain an
 * order, this should generally only get called on new visits/bookmark events.
 */
const addTimestamp = field => async metaDoc => {
    const indexDocId = metaDoc.page._id // Index docs share ID with corresponding pouch page doc
    const existingDoc = await get(indexDocId) // Get existing indexed doc
    if (!existingDoc) {
        throw new Error('Page associated with timestamp is not recorded in the index')
    }

    (existingDoc[field] || []).push(transformMetaDoc(metaDoc)) // Perform in-memory update
    await del(indexDocId) // Delete existing doc
    return addConcurrent(existingDoc) // Add new updated doc
}

export const addVisit = addTimestamp('visits')
export const addBookmark = addTimestamp('bookmarks')

/**
 * Returns a function that affords removing either a visit or bookmark from the index.
 * The function should perform the following update, using the associated page doc ID:
 *  - grab the existing index doc matching the page doc ID
 *  - perform update to appropriate timestamp field, removing the existing ID
 *  - delete the original doc form the index
 *  - add the updated doc into the index
 */
const removeTimestamp = field => async metaDoc => {
    const indexDocId = metaDoc.page._id // Index docs share ID with corresponding pouch page doc
    const existingDoc = await get(indexDocId) // Get existing doc
    if (!existingDoc) {
        throw new Error('Page associated with timestamp is not recorded in the index')<commit_msg>Ensure new visit/bookmark updates index latest field<commit_after>/* eslint promise/param-names: 0 */
import initSearchIndex from 'search-index'
import stream from 'stream'
import stopword from 'stopword'

import pipeline from './search-index-pipeline'
import { convertMetaDocId } from 'src/activity-logger'

const indexOpts = {
    batchSize: 500,
    appendOnly: true,
    indexPath: 'worldbrain-index',
    logLevel: 'info',
    preserveCase: false,
    compositeField: false,
    nGramLength: 1,
    separator: /[|' .,\-|(\n)]+/,
    stopwords: stopword.en,
    fieldOptions: {
        visits: {
            fieldedSearch: true,
        },
        bookmarks: {
            fieldedSearch: true,
        },
        latest: {
            sortable: true,
        },
        content: {
            fieldedSearch: true,
        },
        url: {
            weight: 10,
            fieldedSearch: true,
            separator: '/',
        },
    },
}

const standardResponse = (resolve, reject) =>
    (err, data = true) => err ? reject(err) : resolve(data)

// Simply extracts the timestamp component out the ID of a visit or bookmark doc,
//  which is the only data we want at the moment.
const transformMetaDoc = doc => convertMetaDocId(doc._id).timestamp

/**
 * NOTE: Assumes sorted arrays, last index containing latest.
 * @param {Array<string>} visits Sorted array of UNIX timestamp strings.
 * @param {Array<string>} bookmarks Sorted array of UNIX timestamp strings.
 * @returns {string} Latest UNIX timestamp of all params.
 */
const getLatestMeta = (visits, bookmarks) => {
    const numVisits = visits.length
    const numBookmarks = bookmarks.length

    if (numVisits && numBookmarks) { // Both arrays have timestamps
        return visits[numVisits - 1] > bookmarks[numBookmarks - 1]
            ? visits[numVisits - 1]
            : bookmarks[numBookmarks - 1]
    } else if (numBookmarks) { // Only bookmark array has timestamps
        return bookmarks[numBookmarks - 1]
    } else { // Only visit array has timestamps
        return visits[numVisits - 1]
    }
}

// Groups input docs into standard index doc structure
const transformPageAndMetaDocs = ({ pageDoc, visitDocs = [], bookmarkDocs = [] }) => {
    const visits = visitDocs.map(transformMetaDoc)
    const bookmarks = bookmarkDocs.map(transformMetaDoc)

    return {
        ...pipeline(pageDoc),
        latest: getLatestMeta(visits, bookmarks),
        visits,
        bookmarks,
    }
}

// Wraps instance creation in a Promise for use in async interface methods
const indexP = new Promise((...args) => initSearchIndex(indexOpts, standardResponse(...args)))

/**
 * @param doc The doc to queue up for adding into the index.
 * @returns Boolean denoting that the add was successful (else error thrown).
 */
async function addConcurrent(doc) {
    const index = await indexP
    const input = [doc]

    return new Promise((...args) => index.concurrentAdd(indexOpts, input, standardResponse(...args)))
}

/**
 * Adds a new page doc + any associated visit/bookmark docs to the index. This method
 * uses the concurrency-safe index add method.
 * @param {any} Object containing a `pageDoc` (required) and optionally any associated `visitDocs` and `bookmarkDocs`.
 * @returns {Promise} Promise resolving when indexing is complete, or rejecting for any index errors.
 */
export async function addPageConcurrent({ pageDoc, visitDocs = [], bookmarkDocs = [] }) {
    const indexDoc = transformPageAndMetaDocs({ pageDoc, visitDocs, bookmarkDocs })

    return await addConcurrent(indexDoc)
}

/**
 * Adds a new page doc + any associated visit/bookmark docs to the index. This method
 * is *NOT* concurrency safe.
 * @param {any} Object containing a `pageDoc` (required) and optionally any associated `visitDocs` and `bookmarkDocs`.
 * @returns {Promise} Promise resolving when indexing is complete, or rejecting for any index errors.
 */
export async function addPage({ pageDoc, visitDocs = [], bookmarkDocs = [] }) {
    const index = await indexP

    const indexDoc = transformPageAndMetaDocs({ pageDoc, visitDocs, bookmarkDocs })

    return new Promise((resolve, reject) => {
        // Set up add pipeline
        const inputStream = new stream.Readable({ objectMode: true })
        inputStream
            .pipe(index.defaultPipeline())
            .pipe(index.add())
            .on('finish', resolve)
            .on('error', reject)

        // Push index doc onto stream + null to signify end-of-stream
        inputStream.push(indexDoc)
        inputStream.push(null)
    })
}

/**
 * Returns a function that affords adding either a visit or bookmark to an index.
 * The function should perform the following update, using the associated page doc ID:
 *  - grab the existing index doc matching the page doc ID
 *  - perform update to appropriate timestamp field, by APPENDING new timestamp to field + `latest` (sort) field
 *  - delete the original doc form the index
 *  - add the updated doc into the index
 * NOTE: this appends the timestamp extracted from the given metaDoc, hence to maintain an
 * order, this should generally only get called on new visits/bookmark events.
 */
const addTimestamp = field => async metaDoc => {
    const indexDocId = metaDoc.page._id // Index docs share ID with corresponding pouch page doc
    const existingDoc = await get(indexDocId) // Get existing indexed doc
    if (!existingDoc) {
        throw new Error('Page associated with timestamp is not recorded in the index')
    }

    const newTimestamp = transformMetaDoc(metaDoc)

    // Perform in-memory updates of timestamp array + "latest"/sort field
    existingDoc.latest = newTimestamp
    existingDoc[field] = [
        ...(existingDoc[field] || []),
        newTimestamp,
    ]

    await del(indexDocId) // Delete existing doc
    return addConcurrent(existingDoc) // Add new updated doc
}

export const addVisit = addTimestamp('visits')
export const addBookmark = addTimestamp('bookmarks')

/**
 * Returns a function that affords removing either a visit or bookmark from the index.
 * The function should perform the following update, using the associated page doc ID:
 *  - grab the existing index doc matching the page doc ID
 *  - perform update to appropriate timestamp field, removing the existing ID
 *  - delete the original doc form the index
 *  - add the updated doc into the index
 */
const removeTimestamp = field => async metaDoc => {
    const indexDocId = metaDoc.page._id // Index docs share ID with corresponding pouch page doc
    const existingDoc = await get(indexDocId) // Get existing doc
    if (!existingDoc) {
        throw new Error('Page associated with timestamp is not recorded in the index')
{'fraction_non_alphanumeric': 0.09465301478953356, 'fraction_numerical': 0.0019719378081152826, 'mean_word_length': 3.095031055900621, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1849853', 'n_tokens_mistral': 3600, 'n_tokens_neox': 3525, 'n_words': 1365}