<commit_before>import parse = require("s-expression")
import pointer from "json8-pointer"
import { isArray, tail, without } from "lodash"
// This type models what we get from the s-expression library. This library
// transforms lists in S-Expressions into array, strings inside double quotes
// as String (note the uppercase! It really uses the rarely used String object) and
// all other tokes like numbers or symbols as normal string.
export type SExprAtom = string | String | SExprAtom[] // eslint-disable-line @typescript-eslint/ban-types

export enum Arity {
    nullary = "nullary",
    unary = "unary",
    binary = "binary",
    nary = "nary",
}

const arityToNumberMap = new Map([
    [Arity.nullary, 0],
    [Arity.unary, 1],
    [Arity.binary, 2],
    [Arity.nary, undefined],
])

export interface Operation {
    toSql(): string
    toSExpr(): string
    expressionType: ExpressionType
}

export abstract class NumericOperation implements Operation {
    abstract toSql(): string
    abstract toSExpr(): string
    expressionType: ExpressionType = ExpressionType.numeric
}

export abstract class BooleanOperation implements Operation {
    abstract toSql(): string
    abstract toSExpr(): string
    expressionType: ExpressionType = ExpressionType.boolean
}

export abstract class StringOperation implements Operation {
    abstract toSql(): string
    abstract toSExpr(): string
    expressionType: ExpressionType = ExpressionType.string
}

export enum ExpressionType {
    numeric = "numeric",
    boolean = "boolean",
    string = "string",
    any = "any", // This is used for values we can't type check e.g. jsonpath queries that might return anything
}

export class BooleanAtom extends BooleanOperation {
    constructor(public value: boolean) {
        super()
    }
    toSql(): string {
        return this.value.toString()
    }

    toSExpr(): string {
        return this.value.toString()
    }
}

export class NumberAtom extends NumericOperation {
    constructor(public value: number) {
        super()
    }
    toSql(): string {
        return this.value.toString()
    }

    toSExpr(): string {
        return this.value.toString()
    }
}

export class StringAtom extends StringOperation {
    constructor(public value: string) {
        super()
    }
    expressionType = ExpressionType.string
    escapedValue(): string {
        return this.value.toString().replace("'", "''").replace("\\", "\\\\")
    }
    toSql(): string {
        return `'${this.escapedValue()}'` // escape single quotes to avoid SQL injection attacks :)
    }

    toSExpr(): string {
        return `"${this.value.toString()}"`
    }
}

function jsonPointerToJsonPath(jsonpointer: string): string {
    const pointerFragments = pointer.decode(jsonpointer)
    let path = "$"
    for (const fragment of pointerFragments) {
        const number = Number.parseInt(fragment)
        if (!Number.isNaN(number)) path += `[${number}]`
        else path += `.${fragment}`<commit_msg>fix(bulk FASTT editor): Fix escaping bugs<commit_after>import parse = require("s-expression")
import pointer from "json8-pointer"
import { isArray, tail, without } from "lodash"
// This type models what we get from the s-expression library. This library
// transforms lists in S-Expressions into array, strings inside double quotes
// as String (note the uppercase! It really uses the rarely used String object) and
// all other tokes like numbers or symbols as normal string.
export type SExprAtom = string | String | SExprAtom[] // eslint-disable-line @typescript-eslint/ban-types

export enum Arity {
    nullary = "nullary",
    unary = "unary",
    binary = "binary",
    nary = "nary",
}

const arityToNumberMap = new Map([
    [Arity.nullary, 0],
    [Arity.unary, 1],
    [Arity.binary, 2],
    [Arity.nary, undefined],
])

export interface Operation {
    toSql(): string
    toSExpr(): string
    expressionType: ExpressionType
}

export abstract class NumericOperation implements Operation {
    abstract toSql(): string
    abstract toSExpr(): string
    expressionType: ExpressionType = ExpressionType.numeric
}

export abstract class BooleanOperation implements Operation {
    abstract toSql(): string
    abstract toSExpr(): string
    expressionType: ExpressionType = ExpressionType.boolean
}

export abstract class StringOperation implements Operation {
    abstract toSql(): string
    abstract toSExpr(): string
    expressionType: ExpressionType = ExpressionType.string
}

export enum ExpressionType {
    numeric = "numeric",
    boolean = "boolean",
    string = "string",
    any = "any", // This is used for values we can't type check e.g. jsonpath queries that might return anything
}

export class BooleanAtom extends BooleanOperation {
    constructor(public value: boolean) {
        super()
    }
    toSql(): string {
        return this.value.toString()
    }

    toSExpr(): string {
        return this.value.toString()
    }
}

export class NumberAtom extends NumericOperation {
    constructor(public value: number) {
        super()
    }
    toSql(): string {
        return this.value.toString()
    }

    toSExpr(): string {
        return this.value.toString()
    }
}
const quoteReplaceRegex = /'/g
const backslashReplaceRegex = /\\/g
const doubleQuoteReplaceRegex = /"/g
export class StringAtom extends StringOperation {
    constructor(public value: string) {
        super()
    }
    expressionType = ExpressionType.string
    escapedValue(): string {
        return this.value
            .toString()
            .replace(quoteReplaceRegex, "''")
            .replace(backslashReplaceRegex, "\\\\")
    }
    toSql(): string {
        return `'${this.escapedValue()}'` // escape single quotes to avoid SQL injection attacks :)
    }

    toSExpr(): string {
        return `"${this.value
            .toString()
            .replace(doubleQuoteReplaceRegex, '\\"')}"`
    }
}

function jsonPointerToJsonPath(jsonpointer: string): string {
    const pointerFragments = pointer.decode(jsonpointer)
    let path = "$"
    for (const fragment of pointerFragments) {
        const number = Number.parseInt(fragment)
        if (!Number.isNaN(number)) path += `[${number}]`
        else path += `.${fragment}`
{'fraction_non_alphanumeric': 0.10938251804330393, 'fraction_numerical': 0.0012830793905372895, 'mean_word_length': 3.151797603195739, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5431554', 'n_tokens_mistral': 1716, 'n_tokens_neox': 1691, 'n_words': 539}