<commit_before>import { decodeHTML } from "entities"
import { DatabaseConnection } from "db/DatabaseConnection"
import {
    WORDPRESS_DB_NAME,
    WORDPRESS_DB_HOST,
    WORDPRESS_DB_PORT,
    WORDPRESS_DB_USER,
    WORDPRESS_DB_PASS,
    WORDPRESS_API_PASS,
    WORDPRESS_API_USER
} from "serverSettings"
import { WORDPRESS_URL, BAKED_BASE_URL } from "settings"
import * as Knex from "knex"
import fetch from "node-fetch"
const urlSlug = require("url-slug")

import { defaultTo } from "charts/Util"
import { Base64 } from "js-base64"
import { registerExitHandler } from "./cleanup"

class WPDB {
    conn?: DatabaseConnection

    knex(tableName?: string | Knex.Raw | Knex.QueryBuilder | undefined) {
        if (!knexInstance) {
            knexInstance = Knex({
                client: "mysql",
                connection: {
                    host: WORDPRESS_DB_HOST,
                    port: WORDPRESS_DB_PORT,
                    user: WORDPRESS_DB_USER,
                    password: <PASSWORD>,
                    database: WORDPRESS_DB_NAME
                }
            })

            registerExitHandler(async () => {
                if (knexInstance) await knexInstance.destroy()
            })
        }

        return knexInstance(tableName)
    }

    async connect() {
        this.conn = new DatabaseConnection({
            host: WORDPRESS_DB_HOST,
            port: WORDPRESS_DB_PORT,
            user: WORDPRESS_DB_USER,
            password: <PASSWORD>,
            database: WORDPRESS_DB_NAME
        })
        await this.conn.connect()

        registerExitHandler(async () => {
            if (this.conn) this.conn.end()
        })
    }

    async query(queryStr: string, params?: any[]): Promise<any[]> {
        if (!this.conn) await this.connect()

        return this.conn!.query(queryStr, params)
    }

    async get(queryStr: string, params?: any[]): Promise<any> {
        if (!this.conn) await this.connect()

        return this.conn!.get(queryStr, params)
    }
}

const wpdb = new WPDB()

const WP_API_ENDPOINT = `${WORDPRESS_URL}/wp-json/wp/v2`
const OWID_API_ENDPOINT = `${WORDPRESS_URL}/wp-json/owid/v1`
const WP_GRAPHQL_ENDPOINT = `${WORDPRESS_URL}/wp/graphql`

async function apiQuery(
    endpoint: string,
    params?: {
        isAuthenticated?: boolean
        searchParams?: Array<[string, string | number]>
    }
): Promise<any> {
    const url = new URL(endpoint)

    if (params && params.searchParams) {
        params.searchParams.forEach(param => {
            url.searchParams.append(param[0], String(param[1]))
        })
    }

    if (params && params.isAuthenticated) {
        return fetch(url.toString(), {
            headers: [
                [
                    "Authorization",
                    "Basic " +
                        Base64.encode(
                            `${WORDPRESS_API_USER}:${WORDPRESS_API_PASS}`
                        )
                ]
            ]
        })
    } else {
        return fetch(url.toString())
    }
}

export async function query(queryStr: string, params?: any[]): Promise<any[]> {
    return wpdb.query(queryStr, params)
}

export async function get(queryStr: string, params?: any[]): Promise<any> {
    return wpdb.get(queryStr, params)
}

export async function connect() {
    await wpdb.connect()
}

export async function end() {
    if (wpdb.conn) wpdb.conn.end()
    if (knexInstance) await knexInstance.destroy()
}

// Retrieve a map of post ids to authors
let cachedAuthorship: Map<number, string[]> | undefined
export async function getAuthorship(): Promise<Map<number, string[]>> {
    if (cachedAuthorship) return cachedAuthorship

    const authorRows = await wpdb.query(`
        SELECT object_id, terms.description FROM wp_term_relationships AS rels
        LEFT JOIN wp_term_taxonomy AS terms ON terms.term_taxonomy_id=rels.term_taxonomy_id
        WHERE terms.taxonomy='author'
        ORDER BY rels.term_order ASC
    `)

    const authorship = new Map<number, string[]>()
    for (const row of authorRows) {
        let authors = authorship.get(row.object_id)
        if (!authors) {
            authors = []
            authorship.set(row.object_id, authors)
        }
        authors.push(
            row.description
                .split(" ")
                .slice(0, 2)
                .join(" ")
        )
    }

    cachedAuthorship = authorship
    return authorship
}

export interface EntryMeta {
    slug: string
    title: string
    excerpt: string
    kpi: string
}

export interface CategoryWithEntries {
    name: string
    slug: string
    entries: EntryMeta[]
    subcategories: CategoryWithEntries[]
}

export async function getCategoriesByPostId(): Promise<Map<number, string[]>> {
    const categoriesByPostId = new Map<number, string[]>()
    const rows = await wpdb.query(`
        SELECT object_id, terms.name FROM wp_term_relationships AS rels
        LEFT JOIN wp_terms AS terms ON terms.term_id=rels.term_taxonomy_id
    `)

    for (const row of rows) {
        let cats = categoriesByPostId.get(row.object_id)
        if (!cats) {
            cats = []
            categoriesByPostId.set(row.object_id, cats)
        }
        cats.push(row.name)
    }

    return categoriesByPostId
}

export async function getTagsByPostId(): Promise<Map<number, string[]>> {
    const tagsByPostId = new Map<number, string[]>()
    const rows = await wpdb.query(`
        SELECT p.id, t.name
        FROM wp_posts p
        JOIN wp_term_relationships tr
            on (p.id=tr.object_id)
        JOIN wp_term_taxonomy tt
            on (tt.term_taxonomy_id=tr.term_taxonomy_id
            and tt.taxonomy='post_tag')
        JOIN wp_terms t
            on (tt.term_id=t.term_id)
    `)

    for (const row of rows) {
        let cats = tagsByPostId.get(row.id)
        if (!cats) {
            cats = []
            tagsByPostId.set(row.id, cats)
        }
        cats.push(row.name)
    }

    return tagsByPostId
}

export interface EntryNode {
    slug: string
    title: string
    excerpt: string
    kpi: string
}

// Retrieve a list of categories and their associated entries
let cachedEntries: CategoryWithEntries[] = []
export async function getEntriesByCategory(): Promise<CategoryWithEntries[]> {
    if (cachedEntries.length) return cachedEntries

    const first = 100
    // The filtering of cached entries below makes the $first argument
    // less accurate, as it does not represent the exact number of entries
    // returned per subcategories but rather their maximum number of entries.
    const where = {
        orderby: "TERM_ORDER",
        termTaxonomId: 44 // Entries category ID
    }
    const whereNested = {
        orderby: "TERM_ORDER"
    }
    const query = `
    query getEntriesByCategory($first: Int, $where: RootQueryToCategoryConnectionWhereArgs!, $whereNested: CategoryToCategoryConnectionWhereArgs!) {
        categories(first: $first, where: $where) {
          nodes {
            name
            children(first: $first, where: $whereNested) {
              nodes {
                ...categoryWithEntries
                children(first: $first, where: $whereNested) {
                  nodes {
                    ...categoryWithEntries
                  }
                }
              }
            }
          }
        }
      }

      fragment categoryWithEntries on Category {
        name
        slug
        pages(first: $first) {
          nodes {
            slug
            title
            excerpt
            kpi
          }
        }
      }
      `

    const response = await fetch(WP_GRAPHQL_ENDPOINT, {
        method: "POST",
        headers: {
            "Content-Type": "application/json",
            Accept: "application/json"
        },
        body: JSON.stringify({
            query,
            variables: { first, where, whereNested }
        })
    })
    const json = await response.json()

    interface CategoryNode {
        name: string
        slug: string
        pages: any
        children: any
    }

    const getEntryNode = ({ slug, title, excerpt, kpi }: EntryNode) => ({
        slug,
        title: decodeHTML(title),
        excerpt: decodeHTML(excerpt),
        kpi
    })

    const isEntryInSubcategories = (entry: EntryNode, subcategories: any) => {
        return subcategories.some((subcategory: any) => {
            return subcategory.pages.nodes.some(
                (node: EntryNode) => entry.slug === node.slug
            )
        })
    }

    cachedEntries = json.data.categories.nodes[0].children.nodes.map(
        ({ name, slug, pages, children }: CategoryNode) => ({
            name: decodeHTML(name),
            slug,
            entries: pages.nodes
                .filter(
                    (node: EntryNode) =>
                        // Entries are being listed in root categories even when they
                        // belong to subcategories. It is then necessary to filter them
                        // out to avoid duplicates.
                        // https://github.com/wp-graphql/wp-graphql/issues/1100
                        !isEntryInSubcategories(node, children.nodes)
                )
                .map((node: EntryNode) => getEntryNode(node)),
            subcategories: children.nodes
                .filter(
                    (subcategory: CategoryNode) =>
                        subcategory.pages.nodes.length !== 0<commit_msg>fix: handle "null" excerpt in subcategories<commit_after>import { decodeHTML } from "entities"
import { DatabaseConnection } from "db/DatabaseConnection"
import {
    WORDPRESS_DB_NAME,
    WORDPRESS_DB_HOST,
    WORDPRESS_DB_PORT,
    WORDPRESS_DB_USER,
    WORDPRESS_DB_PASS,
    WORDPRESS_API_PASS,
    WORDPRESS_API_USER
} from "serverSettings"
import { WORDPRESS_URL, BAKED_BASE_URL } from "settings"
import * as Knex from "knex"
import fetch from "node-fetch"
const urlSlug = require("url-slug")

import { defaultTo } from "charts/Util"
import { Base64 } from "js-base64"
import { registerExitHandler } from "./cleanup"

class WPDB {
    conn?: DatabaseConnection

    knex(tableName?: string | Knex.Raw | Knex.QueryBuilder | undefined) {
        if (!knexInstance) {
            knexInstance = Knex({
                client: "mysql",
                connection: {
                    host: WORDPRESS_DB_HOST,
                    port: WORDPRESS_DB_PORT,
                    user: WORDPRESS_DB_USER,
                    password: <PASSWORD>,
                    database: WORDPRESS_DB_NAME
                }
            })

            registerExitHandler(async () => {
                if (knexInstance) await knexInstance.destroy()
            })
        }

        return knexInstance(tableName)
    }

    async connect() {
        this.conn = new DatabaseConnection({
            host: WORDPRESS_DB_HOST,
            port: WORDPRESS_DB_PORT,
            user: WORDPRESS_DB_USER,
            password: <PASSWORD>,
            database: WORDPRESS_DB_NAME
        })
        await this.conn.connect()

        registerExitHandler(async () => {
            if (this.conn) this.conn.end()
        })
    }

    async query(queryStr: string, params?: any[]): Promise<any[]> {
        if (!this.conn) await this.connect()

        return this.conn!.query(queryStr, params)
    }

    async get(queryStr: string, params?: any[]): Promise<any> {
        if (!this.conn) await this.connect()

        return this.conn!.get(queryStr, params)
    }
}

const wpdb = new WPDB()

const WP_API_ENDPOINT = `${WORDPRESS_URL}/wp-json/wp/v2`
const OWID_API_ENDPOINT = `${WORDPRESS_URL}/wp-json/owid/v1`
const WP_GRAPHQL_ENDPOINT = `${WORDPRESS_URL}/wp/graphql`

async function apiQuery(
    endpoint: string,
    params?: {
        isAuthenticated?: boolean
        searchParams?: Array<[string, string | number]>
    }
): Promise<any> {
    const url = new URL(endpoint)

    if (params && params.searchParams) {
        params.searchParams.forEach(param => {
            url.searchParams.append(param[0], String(param[1]))
        })
    }

    if (params && params.isAuthenticated) {
        return fetch(url.toString(), {
            headers: [
                [
                    "Authorization",
                    "Basic " +
                        Base64.encode(
                            `${WORDPRESS_API_USER}:${WORDPRESS_API_PASS}`
                        )
                ]
            ]
        })
    } else {
        return fetch(url.toString())
    }
}

export async function query(queryStr: string, params?: any[]): Promise<any[]> {
    return wpdb.query(queryStr, params)
}

export async function get(queryStr: string, params?: any[]): Promise<any> {
    return wpdb.get(queryStr, params)
}

export async function connect() {
    await wpdb.connect()
}

export async function end() {
    if (wpdb.conn) wpdb.conn.end()
    if (knexInstance) await knexInstance.destroy()
}

// Retrieve a map of post ids to authors
let cachedAuthorship: Map<number, string[]> | undefined
export async function getAuthorship(): Promise<Map<number, string[]>> {
    if (cachedAuthorship) return cachedAuthorship

    const authorRows = await wpdb.query(`
        SELECT object_id, terms.description FROM wp_term_relationships AS rels
        LEFT JOIN wp_term_taxonomy AS terms ON terms.term_taxonomy_id=rels.term_taxonomy_id
        WHERE terms.taxonomy='author'
        ORDER BY rels.term_order ASC
    `)

    const authorship = new Map<number, string[]>()
    for (const row of authorRows) {
        let authors = authorship.get(row.object_id)
        if (!authors) {
            authors = []
            authorship.set(row.object_id, authors)
        }
        authors.push(
            row.description
                .split(" ")
                .slice(0, 2)
                .join(" ")
        )
    }

    cachedAuthorship = authorship
    return authorship
}

export interface EntryMeta {
    slug: string
    title: string
    excerpt: string
    kpi: string
}

export interface CategoryWithEntries {
    name: string
    slug: string
    entries: EntryMeta[]
    subcategories: CategoryWithEntries[]
}

export async function getCategoriesByPostId(): Promise<Map<number, string[]>> {
    const categoriesByPostId = new Map<number, string[]>()
    const rows = await wpdb.query(`
        SELECT object_id, terms.name FROM wp_term_relationships AS rels
        LEFT JOIN wp_terms AS terms ON terms.term_id=rels.term_taxonomy_id
    `)

    for (const row of rows) {
        let cats = categoriesByPostId.get(row.object_id)
        if (!cats) {
            cats = []
            categoriesByPostId.set(row.object_id, cats)
        }
        cats.push(row.name)
    }

    return categoriesByPostId
}

export async function getTagsByPostId(): Promise<Map<number, string[]>> {
    const tagsByPostId = new Map<number, string[]>()
    const rows = await wpdb.query(`
        SELECT p.id, t.name
        FROM wp_posts p
        JOIN wp_term_relationships tr
            on (p.id=tr.object_id)
        JOIN wp_term_taxonomy tt
            on (tt.term_taxonomy_id=tr.term_taxonomy_id
            and tt.taxonomy='post_tag')
        JOIN wp_terms t
            on (tt.term_id=t.term_id)
    `)

    for (const row of rows) {
        let cats = tagsByPostId.get(row.id)
        if (!cats) {
            cats = []
            tagsByPostId.set(row.id, cats)
        }
        cats.push(row.name)
    }

    return tagsByPostId
}

export interface EntryNode {
    slug: string
    title: string
    // in some edge cases (entry alone in a subcategory), WPGraphQL returns
    // null instead of an empty string)
    excerpt: string | null
    kpi: string
}

// Retrieve a list of categories and their associated entries
let cachedEntries: CategoryWithEntries[] = []
export async function getEntriesByCategory(): Promise<CategoryWithEntries[]> {
    if (cachedEntries.length) return cachedEntries

    const first = 100
    // The filtering of cached entries below makes the $first argument
    // less accurate, as it does not represent the exact number of entries
    // returned per subcategories but rather their maximum number of entries.
    const where = {
        orderby: "TERM_ORDER",
        termTaxonomId: 44 // Entries category ID
    }
    const whereNested = {
        orderby: "TERM_ORDER"
    }
    const query = `
    query getEntriesByCategory($first: Int, $where: RootQueryToCategoryConnectionWhereArgs!, $whereNested: CategoryToCategoryConnectionWhereArgs!) {
        categories(first: $first, where: $where) {
          nodes {
            name
            children(first: $first, where: $whereNested) {
              nodes {
                ...categoryWithEntries
                children(first: $first, where: $whereNested) {
                  nodes {
                    ...categoryWithEntries
                  }
                }
              }
            }
          }
        }
      }

      fragment categoryWithEntries on Category {
        name
        slug
        pages(first: $first) {
          nodes {
            slug
            title
            excerpt
            kpi
          }
        }
      }
      `

    const response = await fetch(WP_GRAPHQL_ENDPOINT, {
        method: "POST",
        headers: {
            "Content-Type": "application/json",
            Accept: "application/json"
        },
        body: JSON.stringify({
            query,
            variables: { first, where, whereNested }
        })
    })
    const json = await response.json()

    interface CategoryNode {
        name: string
        slug: string
        pages: any
        children: any
    }

    const getEntryNode = ({ slug, title, excerpt, kpi }: EntryNode) => ({
        slug,
        title: decodeHTML(title),
        excerpt: excerpt === null ? "" : decodeHTML(excerpt),
        kpi
    })

    const isEntryInSubcategories = (entry: EntryNode, subcategories: any) => {
        return subcategories.some((subcategory: any) => {
            return subcategory.pages.nodes.some(
                (node: EntryNode) => entry.slug === node.slug
            )
        })
    }

    cachedEntries = json.data.categories.nodes[0].children.nodes.map(
        ({ name, slug, pages, children }: CategoryNode) => ({
            name: decodeHTML(name),
            slug,
            entries: pages.nodes
                .filter(
                    (node: EntryNode) =>
                        // Entries are being listed in root categories even when they
                        // belong to subcategories. It is then necessary to filter them
                        // out to avoid duplicates.
                        // https://github.com/wp-graphql/wp-graphql/issues/1100
                        !isEntryInSubcategories(node, children.nodes)
                )
                .map((node: EntryNode) => getEntryNode(node)),
            subcategories: children.nodes
                .filter(
                    (subcategory: CategoryNode) =>
                        subcategory.pages.nodes.length !== 0
{'fraction_non_alphanumeric': 0.09972197450558674, 'fraction_numerical': 0.0024130514609452865, 'mean_word_length': 1.923477994172673, 'pattern_counts': {'":': 2, '<': 41, '<?xml version=': 0, '>': 63, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7602767', 'n_tokens_mistral': 5499, 'n_tokens_neox': 5275, 'n_words': 1310}