<commit_before>function urlSampleToFull(url: string): string {
    return url
        .replace("img-master", "img-original")
        .replace(/_master\d+\./, ".");
}
function urlSampleToThumbnail(url: string): string {
    return url.replace("/img-master/", "/c/150x150/img-master/");
}

export const source: ISource = {
    name: "Pixiv",
    modifiers: [],
    forcedTokens: [],
    searchFormat: {
        and: " ",
    },
    auth: {
        oauth2: {
            type: "oauth2",
            authType: "password",
            tokenUrl: "https://oauth.secure.pixiv.net/auth/token",
        },
    },
    apis: {
        json: {
            name: "JSON",
            auth: ["oauth2"],
            maxLimit: 1000, // Actual max limit is higher but unnecessary, slow, and unreliable
            search: {
                url: (query: ISearchQuery, opts: IUrlOptions): string => {
                    const params: string[] = [
                        "q=" + query.search,
                        "page=" + query.page,
                        "per_page=" + opts.limit,
                        "period=all",
                        "order=desc",
                        "mode=caption",
                        "sort=date",<commit_msg>Fix Tumblr source name<commit_after>function urlSampleToFull(url: string): string {
    return url
        .replace("img-master", "img-original")
        .replace(/_master\d+\./, ".");
}
function urlSampleToThumbnail(url: string): string {
    return url.replace("/img-master/", "/c/150x150/img-master/");
}

export const source: ISource = {
    name: "Tumblr",
    modifiers: [],
    forcedTokens: [],
    searchFormat: {
        and: " ",
    },
    auth: {
        oauth2: {
            type: "oauth2",
            authType: "password",
            tokenUrl: "https://oauth.secure.pixiv.net/auth/token",
        },
    },
    apis: {
        json: {
            name: "JSON",
            auth: ["oauth2"],
            maxLimit: 1000, // Actual max limit is higher but unnecessary, slow, and unreliable
            search: {
                url: (query: ISearchQuery, opts: IUrlOptions): string => {
                    const params: string[] = [
                        "q=" + query.search,
                        "page=" + query.page,
                        "per_page=" + opts.limit,
                        "period=all",
                        "order=desc",
                        "mode=caption",
                        "sort=date",
{'fraction_non_alphanumeric': 0.1438965238480194, 'fraction_numerical': 0.010509296685529508, 'mean_word_length': 1.431237721021611, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5293108', 'n_tokens_mistral': 726, 'n_tokens_neox': 683, 'n_words': 146}