<commit_before>function completeImage(img: IImage): IImage {
    if (img["ext"] && img["ext"][0] === ".") {
        img["ext"] = img["ext"].mid(1);
    }

    const hasMd5 = img["md5"] && img["md5"].length > 0;

    if (!img["file_url"] || img["file_url"].length < 5) {
        img["file_url"] = hasMd5
            ? `/_images/${img["md5"]}.${img["ext"]}`
            : `/_images/${img["id"]}.${img["ext"]}`;
    }

    if (!img["preview_url"] || img["preview_url"].length < 5) {
        img["preview_url"] = hasMd5
            ? `/_thumbs/${img["md5"]}.jpg`
            : `/_thumbs/${img["id"]}.jpg`;
    }

    return img;
}

export const source: ISource = {
    name: "Shimmie",
    modifiers: [],
    forcedTokens: ["filename", "date"],
    tagFormat: {
        case: "lower",
        wordSeparator: "_",
    },
    auth: {},
    apis: {
        rss: {
            name: "RSS",
            auth: [],
            search: {
                url: (query: any, opts: any, previous: any): IError | string => {
                    if (query.search.length > 0) {
                        return { error: "Tag search is impossible with Shimmie RSS API." };
                    }
                    return "/rss/images/" + query.page;
                },
                parse: (src: string): IParsedSearch => {
                    const data = Grabber.parseXML(src).rss.channel.item;

                    const images: IImage[] = [];
                    for (const image of data) {
                        const img: IImage = {
                            page_url: image["link"]["#text"],
                            preview_url: image["media:thumbnail"]["#text"] || image["media:thumbnail"]["@attributes"]["url"],
                            file_url: image["media:content"]["#text"] || image["media:content"]["@attributes"]["url"],
                            created_at: image["pubDate"]["#text"],
                        };

                        const info = image["title"]["#text"].split(" - ");
                        if (info.length === 2) {
                            img["id"] = parseInt(info[0], 10);
                            img["tags"] = info[1].toLowerCase().split(" ");
                        } else {
                            img["id"] = Grabber.regexToConst("id", "/(?<id>\\d+)", img["page_url"]);
                        }

                        // Some additional fields can be found parsing the HTML description
                        const desc = image["description"]["#text"];
                        const matches = Grabber.regexMatches(" // (?<width>\\d+)x(?<height>\\d+) // (?<filesize>[^'\" ]*)(?: // (?<ext>[^'\"&]*))?['\"&]", desc);
                        if (matches && matches.length > 0) {
                            const match = matches[0];
                            img["width"] = match["width"];
                            img["height"] = match["height"];
                            img["file_size"] = match["filesize"];
                            if (match["ext"]) {
                                img["ext"] = match["ext"];
                            }
                        }

                        images.push(completeImage(img));
                    }

                    return { images };
                },
            },
        },
        html: {
            name: "Regex",
            auth: [],
            search: {
                url: (query: any, opts: any, previous: any): string => {
                    if (query.search.length > 0) {
                        return "/post/list/" + query.search + "/" + query.page;
                    }
                    return "/post/list/" + query.page;
                },
                parse: (src: string): IParsedSearch => {
                    return {
                        tags: Grabber.regexToTags('<li class="tag-type-(?<type>[^"]+)">[^<]*<a href="[^"]+">[^<]*</a>[^<]*<a href="[^"]+">(?<name>[^<]+)</a>[^<]*</li>|<a class=[\'"]tag_name[\'"] href=[\'"]([^\'"]+)(?:/1)?[\'"]>(?<name_2>[^<]+)</a>', src),
                        images: Grabber.regexToImages("<a(?: class=['\"][^'\"]*['\"])? href=['\"][^'\">]*/post/view/(?<id>[^'\"]+)['\"][^>]*>[^<]*(?<image><img(?: id=['\"](?:[^'\"]*)['\"])? title=['\"](?<tags>[^'\"/]*)(?: // (?<width>[^'\"]+)x(?<height>[^'\"]+) // (?<filesize>[^'\"]*)(?: // (?<ext>[^'\"]*))?)?['\"] alt=['\"][^'\"]*['\"](?: height=['\"][^'\"]*['\"])? width=['\"][^'\"]*['\"](?: height=['\"][^'\"]*['\"])?[^>]*(?:src|data-original)=['\"][^'\"]*(?<preview_url>/_thumbs/(?<md5>[^/]*)/[^'\"]*\\.jpg|/thumb/(?<md5_2>[^'\"]*)\\.jpg|questionable\\.jpg)['\"][^>]*>).*?</a>|<a href=['\"][^'\">]*/i(?<id_2>[^'\"]+)['\"](?: class=['\"][^'\"]*['\"])?[^>]*>[^<]*(?<image_2><img(?: id=['\"](?:[^'\"]*)['\"])? src=['\"][^'\"]*(?<preview_url_2>/_thumbs/(?<md5_3>[^'\"]*)(?:_th)?\\.jpg|/thumb/(?<md5_4>[^'\"]*)\\.jpg|questionable\\.jpg)['\"] title=['\"](?<tags_2>[^'\"/]+) // (?<width_2>[^'\"]+)x(?<height_2>[^'\"]+) // (?<filesize_2>[^'\"]*)(?: // (?<ext_2>[^'\"]*))?['\"] alt=['\"][^'\"]*['\"] ?/?>)[^<]*</a>", src).map(completeImage),
                        pageCount: Grabber.regexToConst("page", "<a href=['\"]/post/list(?:/[^/]+)?/(?<page>\\d*)['\"]>Last</a>", src),
                    };
                },
            },
            details: {
                url: (id: number, md5: string): string => {
                    return "/post/view/" + id;
                },
                parse: (src: string): IParsedDetails => {
                    return {
                        tags: Grabber.regexToTags('<li class="tag-type-(?<type>[^"]+)">[^<]*<a href="[^"]+">[^<]*</a>[^<]*<a href="[^"]+">(?<name>[^<]+)</a>[^<]*</li>|<a class=[\'"]tag_name[\'"] href=[\'"]([^\'"]+)(?:/1)?[\'"]>(?<name_2>[^<]+)</a>', src),
                        imageUrl: Grabber.regexToConst("url", "<img.+?id=['\"]main_image['\"] src=['\"](?<url>[^']+)['\"][^>]*>", src),
                        createdAt: Grabber.regexToConst("date", "<time datetime=['\"](?<date>[^'\"]+)['\"]>", src),
                    };
                },
            },
            check: {
                url: (): string => {
                    return "/";
                },
                parse: (src: string): boolean => {
                    return src.indexOf("Running Shimmie") !== -1<commit_msg>Add better support for tag count on some Shimmie sources<commit_after>function completeImage(img: IImage): IImage {
    if (img["ext"] && img["ext"][0] === ".") {
        img["ext"] = img["ext"].mid(1);
    }

    const hasMd5 = img["md5"] && img["md5"].length > 0;

    if (!img["file_url"] || img["file_url"].length < 5) {
        img["file_url"] = hasMd5
            ? `/_images/${img["md5"]}.${img["ext"]}`
            : `/_images/${img["id"]}.${img["ext"]}`;
    }

    if (!img["preview_url"] || img["preview_url"].length < 5) {
        img["preview_url"] = hasMd5
            ? `/_thumbs/${img["md5"]}.jpg`
            : `/_thumbs/${img["id"]}.jpg`;
    }

    return img;
}

export const source: ISource = {
    name: "Shimmie",
    modifiers: [],
    forcedTokens: ["filename", "date"],
    tagFormat: {
        case: "lower",
        wordSeparator: "_",
    },
    auth: {},
    apis: {
        rss: {
            name: "RSS",
            auth: [],
            search: {
                url: (query: any, opts: any, previous: any): IError | string => {
                    if (query.search.length > 0) {
                        return { error: "Tag search is impossible with Shimmie RSS API." };
                    }
                    return "/rss/images/" + query.page;
                },
                parse: (src: string): IParsedSearch => {
                    const data = Grabber.parseXML(src).rss.channel.item;

                    const images: IImage[] = [];
                    for (const image of data) {
                        const img: IImage = {
                            page_url: image["link"]["#text"],
                            preview_url: image["media:thumbnail"]["#text"] || image["media:thumbnail"]["@attributes"]["url"],
                            file_url: image["media:content"]["#text"] || image["media:content"]["@attributes"]["url"],
                            created_at: image["pubDate"]["#text"],
                        };

                        const info = image["title"]["#text"].split(" - ");
                        if (info.length === 2) {
                            img["id"] = parseInt(info[0], 10);
                            img["tags"] = info[1].toLowerCase().split(" ");
                        } else {
                            img["id"] = Grabber.regexToConst("id", "/(?<id>\\d+)", img["page_url"]);
                        }

                        // Some additional fields can be found parsing the HTML description
                        const desc = image["description"]["#text"];
                        const matches = Grabber.regexMatches(" // (?<width>\\d+)x(?<height>\\d+) // (?<filesize>[^'\" ]*)(?: // (?<ext>[^'\"&]*))?['\"&]", desc);
                        if (matches && matches.length > 0) {
                            const match = matches[0];
                            img["width"] = match["width"];
                            img["height"] = match["height"];
                            img["file_size"] = match["filesize"];
                            if (match["ext"]) {
                                img["ext"] = match["ext"];
                            }
                        }

                        images.push(completeImage(img));
                    }

                    return { images };
                },
            },
        },
        html: {
            name: "Regex",
            auth: [],
            search: {
                url: (query: any, opts: any, previous: any): string => {
                    if (query.search.length > 0) {
                        return "/post/list/" + query.search + "/" + query.page;
                    }
                    return "/post/list/" + query.page;
                },
                parse: (src: string): IParsedSearch => {
                    return {
                        tags: Grabber.regexToTags('<li class="tag-type-(?<type>[^"]+)">[^<]*<a href="[^"]+">[^<]*</a>[^<]*<a href="[^"]+">(?<name>[^<]+)</a>[^<]*</li>|<a class=[\'"]tag_name[\'"] href=[\'"]([^\'"]+)(?:/1)?[\'"]>(?<name_2>[^<]+)</a>(?:</td><td class=[\'"]tag_count_cell[\'"]>[^<]*<span class=[\'"]tag_count[\'"]>(?<count>\\d+)</span>)?', src),
                        images: Grabber.regexToImages("<a(?: class=['\"][^'\"]*['\"])? href=['\"][^'\">]*/post/view/(?<id>[^'\"]+)['\"][^>]*>[^<]*(?<image><img(?: id=['\"](?:[^'\"]*)['\"])? title=['\"](?<tags>[^'\"/]*)(?: // (?<width>[^'\"]+)x(?<height>[^'\"]+) // (?<filesize>[^'\"]*)(?: // (?<ext>[^'\"]*))?)?['\"] alt=['\"][^'\"]*['\"](?: height=['\"][^'\"]*['\"])? width=['\"][^'\"]*['\"](?: height=['\"][^'\"]*['\"])?[^>]*(?:src|data-original)=['\"][^'\"]*(?<preview_url>/_thumbs/(?<md5>[^/]*)/[^'\"]*\\.jpg|/thumb/(?<md5_2>[^'\"]*)\\.jpg|questionable\\.jpg)['\"][^>]*>).*?</a>|<a href=['\"][^'\">]*/i(?<id_2>[^'\"]+)['\"](?: class=['\"][^'\"]*['\"])?[^>]*>[^<]*(?<image_2><img(?: id=['\"](?:[^'\"]*)['\"])? src=['\"][^'\"]*(?<preview_url_2>/_thumbs/(?<md5_3>[^'\"]*)(?:_th)?\\.jpg|/thumb/(?<md5_4>[^'\"]*)\\.jpg|questionable\\.jpg)['\"] title=['\"](?<tags_2>[^'\"/]+) // (?<width_2>[^'\"]+)x(?<height_2>[^'\"]+) // (?<filesize_2>[^'\"]*)(?: // (?<ext_2>[^'\"]*))?['\"] alt=['\"][^'\"]*['\"] ?/?>)[^<]*</a>", src).map(completeImage),
                        pageCount: Grabber.regexToConst("page", "<a href=['\"]/post/list(?:/[^/]+)?/(?<page>\\d*)['\"]>Last</a>", src),
                    };
                },
            },
            details: {
                url: (id: number, md5: string): string => {
                    return "/post/view/" + id;
                },
                parse: (src: string): IParsedDetails => {
                    return {
                        tags: Grabber.regexToTags('<li class="tag-type-(?<type>[^"]+)">[^<]*<a href="[^"]+">[^<]*</a>[^<]*<a href="[^"]+">(?<name>[^<]+)</a>[^<]*</li>|<a class=[\'"]tag_name[\'"] href=[\'"]([^\'"]+)(?:/1)?[\'"]>(?<name_2>[^<]+)</a>(?:</td><td class=[\'"]tag_count_cell[\'"]>[^<]*<span class=[\'"]tag_count[\'"]>(?<count>\\d+)</span>)?', src),
                        imageUrl: Grabber.regexToConst("url", "<img.+?id=['\"]main_image['\"] src=['\"](?<url>[^']+)['\"][^>]*>", src),
                        createdAt: Grabber.regexToConst("date", "<time datetime=['\"](?<date>[^'\"]+)['\"]>", src),
                    };
                },
            },
            check: {
                url: (): string => {
                    return "/";
                },
                parse: (src: string): boolean => {
                    return src.indexOf("Running Shimmie") !== -1
{'fraction_non_alphanumeric': 0.2894962236237639, 'fraction_numerical': 0.006540527914038776, 'mean_word_length': 1.6244380874540254, 'pattern_counts': {'":': 0, '<': 169, '<?xml version=': 0, '>': 171, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4529320', 'n_tokens_mistral': 4386, 'n_tokens_neox': 4117, 'n_words': 556}