<commit_before>package com.rarchives.ripme.ripper.rippers;

import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.rarchives.ripme.ripper.AbstractHTMLRipper;
import com.rarchives.ripme.utils.Http;

public class HentaiCafeRipper extends AbstractHTMLRipper {

    public HentaiCafeRipper(URL url) throws IOException {
    super(url);
    }

        @Override
        public String getHost() {
            return "hentai";
        }

        @Override
        public String getDomain() {
            return "hentai.cafe";
        }

        @Override
        public String getGID(URL url) throws MalformedURLException {
            Pattern p = Pattern.compile("https?://hentai\\.cafe/([a-zA-Z1-9_-]*)/?$");
            Matcher m = p.matcher(url.toExternalForm());
            if (m.matches()) {
                return m.group(1);
            }
            throw new MalformedURLException("Expected hentai.cafe URL format: " +
                            "hentai.cafe/COMIC - got " + url + " instead");
        }

        @Override
        public Document getFirstPage() throws IOException {
            // "url" is an instance field of the superclass
            Document tempDoc = Http.url(url).get();
            return Http.url(tempDoc.select("div.last > p > a.x-btn").attr("href")).get();
        }

        @Override
        public Document getNextPage(Document doc) throws IOException {
            String nextPageURL = doc.select("div[id=page] > div.inner > a").attr("href");
            int totalPages = Integer.parseInt(doc.select("div.panel > div.topbar > div > div.topbar_right > div.tbtitle > div.text").text().replace(" ⤵", ""));
            String[] nextPageURLSplite = nextPageURL.split("/");
            // This checks if the next page number is greater than the total number of pages
            if (totalPages >= Integer.parseInt(nextPageURLSplite[nextPageURLSplite.length -1])) {
                return Http.url(nextPageURL).get();
            }
            throw new IOException("No more pages");
        }

        @Override
        public List<String> getURLsFromPage(Document doc) {
            List<String> result = new ArrayList<String>();
                result.add(doc.select("div[id=page] > div.inner > a > img.open").attr("src"));
                return result;
        }

        @Override
        public void downloadURL(URL url, int index) {
            addURLToDownload(url, getPrefix(index));
        }
    }
<commit_msg>hentai.cafe now supports urls with unicode<commit_after>package com.rarchives.ripme.ripper.rippers;

import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.rarchives.ripme.ripper.AbstractHTMLRipper;
import com.rarchives.ripme.utils.Http;

public class HentaiCafeRipper extends AbstractHTMLRipper {

    public HentaiCafeRipper(URL url) throws IOException {
    super(url);
    }

        @Override
        public String getHost() {
            return "hentai";
        }

        @Override
        public String getDomain() {
            return "hentai.cafe";
        }

        @Override
        public String getGID(URL url) throws MalformedURLException {
            Pattern p = Pattern.compile("https?://hentai\\.cafe/([a-zA-Z1-9_\\-%]*)/?$");
            Matcher m = p.matcher(url.toExternalForm());
            if (m.matches()) {
                return m.group(1);
            }
            throw new MalformedURLException("Expected hentai.cafe URL format: " +
                            "hentai.cafe/COMIC - got " + url + " instead");
        }

        @Override
        public Document getFirstPage() throws IOException {
            // "url" is an instance field of the superclass
            Document tempDoc = Http.url(url).get();
            return Http.url(tempDoc.select("div.last > p > a.x-btn").attr("href")).get();
        }

        @Override
        public Document getNextPage(Document doc) throws IOException {
            String nextPageURL = doc.select("div[id=page] > div.inner > a").attr("href");
            int totalPages = Integer.parseInt(doc.select("div.panel > div.topbar > div > div.topbar_right > div.tbtitle > div.text").text().replace(" ⤵", ""));
            String[] nextPageURLSplite = nextPageURL.split("/");
            // This checks if the next page number is greater than the total number of pages
            if (totalPages >= Integer.parseInt(nextPageURLSplite[nextPageURLSplite.length -1])) {
                return Http.url(nextPageURL).get();
            }
            throw new IOException("No more pages");
        }

        @Override
        public List<String> getURLsFromPage(Document doc) {
            List<String> result = new ArrayList<String>();
                result.add(doc.select("div[id=page] > div.inner > a > img.open").attr("src"));
                return result;
        }

        @Override
        public void downloadURL(URL url, int index) {
            addURLToDownload(url, getPrefix(index));
        }
    }

{'fraction_non_alphanumeric': 0.12095895386850708, 'fraction_numerical': 0.001452960406828914, 'mean_word_length': 2.2682492581602376, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '376678', 'n_tokens_mistral': 1600, 'n_tokens_neox': 1575, 'n_words': 351}