<commit_before>package com.cherokeelessons.cll2ev1;

import java.nio.charset.StandardCharsets;
import java.util.List;

import com.badlogic.gdx.Gdx;
import com.badlogic.gdx.files.FileHandle;
import com.cherokeelessons.cll2ev1.models.CardData;
import com.cherokeelessons.cll2ev1.models.GameCard;

public class LoadImageFilenames implements Runnable {

	private boolean debug = true;
	private static final String UTF_8 = StandardCharsets.UTF_8.name();
	private FileHandle cardImageDir = Gdx.files.internal("card-data/images/");

	private void log(String message) {
		Gdx.app.log(this.getClass().getSimpleName(), message);
	}

	private List<GameCard> cards;

	public LoadImageFilenames(CLL2EV1 game) {
		cards = game.cards;
	}

	@Override
	public void run() {
		// each top level directory is a two-digit number that is the chapter
		// number in the csv file
		String[] dirs = cardImageDir.child("0_dirs.txt").readString(UTF_8).split("\n");
		for (String dir : dirs) {
			FileHandle subDir = cardImageDir.child(dir);
			String[] imageFiles = subDir.child("0_files.txt").readString(UTF_8).split("\n");
			for (GameCard card : cards) {
				CardData cd = card.getData();
				if (cd == null || cd.images == null) {
					continue;
				}
				String c = cd.chapter < 10 ? "0" + cd.chapter : "" + cd.chapter;
				if (!c.equals(dir)) {
					continue;
				}
				String[] imagePrefixes = cd.images.split(";\\s*");
				for (String imagePrefix : imagePrefixes) {
					for (String imageFile : imageFiles) {
						if (!imageFile.endsWith(".png") && !imageFile.endsWith(".jpg")
								&& !imageFile.endsWith(".jpeg")) {
							continue;
						}
						if (imageFile.startsWith(imagePrefix + ".")) {
							cd.addImageFile(subDir.child(imageFile).path());
							continue;
						}
						if (imageFile.startsWith(imagePrefix + "_")) {
							cd.addImageFile(subDir.child(imageFile).path());
							continue;
						}
					}
				}
			}
		}
		if (debug) {
			log("=== DEBUG - CARD DATA IMAGE FILE ASSIGNMENTS:");
			for (GameCard card : cards) {
				CardData data = card.getData();
				if (data.hasImageFiles()) {
					log("CARD: " + data.chapter + " - " + data.images + " - " + data.nextRandomImageFile());
				}
			}<commit_msg>do a FULL blacklist check for each filename!<commit_after>package com.cherokeelessons.cll2ev1;

import java.nio.charset.StandardCharsets;
import java.util.List;

import com.badlogic.gdx.Gdx;
import com.badlogic.gdx.files.FileHandle;
import com.cherokeelessons.cll2ev1.models.CardData;
import com.cherokeelessons.cll2ev1.models.GameCard;

public class LoadImageFilenames implements Runnable {

	private boolean debug = false;
	private static final String UTF_8 = StandardCharsets.UTF_8.name();
	private FileHandle cardImageDir = Gdx.files.internal("card-data/images/");

	private void log(String message) {
		Gdx.app.log(this.getClass().getSimpleName(), message);
	}

	private List<GameCard> cards;

	public LoadImageFilenames(CLL2EV1 game) {
		cards = game.cards;
	}

	@Override
	public void run() {
		// each top level directory is a two-digit number that is the chapter
		// number in the csv file
		String[] dirs = cardImageDir.child("0_dirs.txt").readString(UTF_8).split("\n");
		for (String dir : dirs) {
			FileHandle subDir = cardImageDir.child(dir);
			String[] imageFiles = subDir.child("0_files.txt").readString(UTF_8).split("\n");
			for (GameCard card : cards) {
				CardData cd = card.getData();
				if (cd == null || cd.images == null) {
					continue;
				}
				String c = cd.chapter < 10 ? "0" + cd.chapter : "" + cd.chapter;
				if (!c.equals(dir)) {
					continue;
				}
				String[] imagePrefixes = cd.images.split(";\\s*");
				String[] imageBlacklistPrefixes = cd.blacklistPic.split(";\\s*");
				for (String imagePrefix : imagePrefixes) {
					nextImage: for (String imageFile : imageFiles) {
						//skip not recogized image files
						if (!imageFile.endsWith(".png") && !imageFile.endsWith(".jpg")
								&& !imageFile.endsWith(".jpeg")) {
							continue nextImage;
						}
						//matches as correct, add it to the correct side of things
						if (imageFile.startsWith(imagePrefix + ".")) {
							cd.addImageFile(subDir.child(imageFile).path());
							continue nextImage;
						}
						//matches as correct, add it to the correct side of things
						if (imageFile.startsWith(imagePrefix + "_")) {
							cd.addImageFile(subDir.child(imageFile).path());
							continue nextImage;
						}
						//see if it is ok to add to the wrong pics list
						for (String prefix: imageBlacklistPrefixes) {
							if (imageFile.startsWith(prefix+".")){
								continue nextImage;
							}
							if (imageFile.startsWith(prefix+"_")){
								continue nextImage;
							}
						}
						cd.addWrongImageFile(subDir.child(imageFile).path());
					}
				}
			}
		}
		if (debug) {
			log("=== DEBUG - CARD DATA IMAGE FILE ASSIGNMENTS:");
			for (GameCard card : cards) {
				CardData data = card.getData();
				if (data.hasImageFiles() && data.hasWrongImageFiles()) {
					log("");
					log("CARD [c]: " + data.chapter + " - " + data.nextRandomImageFile() + " - " + data.getImageFiles().size());
					log("CARD [w]: " + data.chapter + " - " + data.nextRandomWrongImageFile() + " - " + data.getWrongImageFiles().size());
				}
			}
{'fraction_non_alphanumeric': 0.1462578556465435, 'fraction_numerical': 0.006474957151018854, 'mean_word_length': 3.2117080994386527, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7489196', 'n_tokens_mistral': 2128, 'n_tokens_neox': 1714, 'n_words': 374}