<commit_before>package de.altenerding.biber.pinkie.business.nuLiga.control;

import de.altenerding.biber.pinkie.business.nuLiga.StandingEntry;
import de.altenerding.biber.pinkie.business.nuLiga.TeamScheduleEntry;
import de.altenerding.biber.pinkie.business.team.control.TeamProvider;
import de.altenerding.biber.pinkie.business.team.entity.Team;
import org.apache.logging.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import javax.inject.Inject;
import javax.persistence.EntityManager;
import javax.persistence.PersistenceContext;
import javax.transaction.Transactional;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

public class NuLigaProcessor {

	private Logger logger;
	private TeamProvider teamProvider;
	@PersistenceContext
	private EntityManager em;

	@Transactional
	public void loadNuLigaData() throws Exception {
		emptyData();
		List<Team> teams = teamProvider.getTeams();

		for (Team team : teams) {
			Document document = Jsoup.connect(team.getUrlStanding()).get();
			loadTeamStandings(team, document);
			document = Jsoup.connect(team.getUrlTeamSchedule()).get();
			loadTeamSchedule(team, document);
		}
	}

	private void emptyData() {
		em.createNamedQuery("TeamScheduleEntry.deleteAll").executeUpdate();
	}

	private void loadTeamSchedule(Team team, Document document) {
		logger.info("Loading season schedule from nuLiga for team={} with id={}", team.getName(), team.getId());
		List<Element> tables = document.select("table");
		Element standingTable = tables.get(1);
		Elements rows = standingTable.select("tr");

		List<TeamScheduleEntry> seasonScheduleEntries = new ArrayList<>();

		for (int i = 1; i < rows.size(); i++) { //first row is the col names so skip it.
			Element row = rows.get(i);
			Elements cols = row.select("td");

			TeamScheduleEntry entry = new TeamScheduleEntry();
			entry.setTeam(team);

			int columnCounter = 0;
			if (cols.size() == 11) {
				String day = cols.get(columnCounter++).text(); //Column 1
				if (day.replace("\u00A0", "").isEmpty()) {
					entry.setDay(seasonScheduleEntries.get(i - 2).getDay());
				} else {
					entry.setDay(day);
				}

				String date = cols.get(columnCounter++).text(); //Column 2
				if (date.replace("\u00A0", "").isEmpty()) {
					entry.setDate(seasonScheduleEntries.get(i - 2).getDate());
				} else {
					entry.setDate(date);
				}

				String time = cols.get(columnCounter++).text().replace("\u00A0", "").trim(); //Column 3
				entry.setTime(time);
			} else {
				entry.setInactive(true);
				String inactiveReason = cols.get(columnCounter++).text(); //Column 1
				entry.setInactiveReason(inactiveReason);
				columnCounter++; //Skip column 2
			}
			@SuppressWarnings("unused")
			/*
				ignore Hallennr
				column index is different depending on case above
			 */
			String col4 = cols.get(columnCounter++).text();
			String matchId = cols.get(columnCounter++).text();
			entry.setMatchId(Long.parseLong(matchId));
			String homeTeam = cols.get(columnCounter++).text().replace("\u00A0", "").trim();
			entry.setHomeTeam(homeTeam);
			String guestTeam = cols.get(columnCounter++).text().replace("\u00A0", "").trim();
			entry.setGuestTeam(guestTeam);
			String result = cols.get(columnCounter++).text().replace("\u00A0", "").trim();
			entry.setResult(result);
			@SuppressWarnings({"unused", "UnusedAssignment"})
			String col9 = cols.get(columnCounter++).text(); //ignore emtpy cell

			seasonScheduleEntries.add(entry);

			em.persist(entry);
			em.flush();
		}
	}

	private void loadTeamStandings(Team team, Document document) throws IOException {
		logger.info("Updating standings data for team={} with id={}", team.getName(), team.getId());
		List<Element> tables = document.select("table");
		Element standingTable = tables.get(0);
		Elements rows = standingTable.select("tr");

		for (int i = 1; i < rows.size(); i++) { //first row is the col names so skip it.
			Element row = rows.get(i);
			Elements cols = row.select("td");<commit_msg>Delete old nuLiga data and reset table sequence to prevent overflow<commit_after>package de.altenerding.biber.pinkie.business.nuLiga.control;

import de.altenerding.biber.pinkie.business.nuLiga.entity.StandingEntry;
import de.altenerding.biber.pinkie.business.nuLiga.entity.TeamScheduleEntry;
import de.altenerding.biber.pinkie.business.team.control.TeamProvider;
import de.altenerding.biber.pinkie.business.team.entity.Team;
import org.apache.logging.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import javax.inject.Inject;
import javax.persistence.EntityManager;
import javax.persistence.PersistenceContext;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

public class NuLigaProcessor {

	private Logger logger;
	private TeamProvider teamProvider;
	@PersistenceContext
	private EntityManager em;

	public void loadNuLigaTeamData() throws Exception {
		emptyTeamData();
		List<Team> teams = teamProvider.getTeams();

		for (Team team : teams) {
			Document document = Jsoup.connect(team.getUrlStanding()).get();
			loadTeamStandings(team, document);
			document = Jsoup.connect(team.getUrlTeamSchedule()).get();
			loadTeamSchedule(team, document);
		}
	}

	private void emptyTeamData() {
		//Delete old data
		em.createNamedQuery("TeamScheduleEntry.deleteAll").executeUpdate();
		em.createNamedQuery("StandingEntry.deleteAll").executeUpdate();
		//reset sequence to prevent an overflow
		em.createNativeQuery("ALTER SEQUENCE standing_id_seq RESTART WITH 1").executeUpdate();
		em.createNativeQuery("ALTER SEQUENCE teamschedule_id_seq RESTART WITH 1").executeUpdate();
	}

	private void loadTeamSchedule(Team team, Document document) {
		logger.info("Loading season schedule from nuLiga for team={} with id={}", team.getName(), team.getId());
		List<Element> tables = document.select("table");
		Element standingTable = tables.get(1);
		Elements rows = standingTable.select("tr");

		List<TeamScheduleEntry> seasonScheduleEntries = new ArrayList<>();

		for (int i = 1; i < rows.size(); i++) { //first row is the col names so skip it.
			Element row = rows.get(i);
			Elements cols = row.select("td");

			TeamScheduleEntry entry = new TeamScheduleEntry();
			entry.setTeam(team);

			int columnCounter = 0;
			if (cols.size() == 11) {
				String day = cols.get(columnCounter++).text(); //Column 1
				if (day.replace("\u00A0", "").isEmpty()) {
					entry.setDay(seasonScheduleEntries.get(i - 2).getDay());
				} else {
					entry.setDay(day);
				}

				String date = cols.get(columnCounter++).text(); //Column 2
				if (date.replace("\u00A0", "").isEmpty()) {
					entry.setDate(seasonScheduleEntries.get(i - 2).getDate());
				} else {
					entry.setDate(date);
				}

				String time = cols.get(columnCounter++).text().replace("\u00A0", "").trim(); //Column 3
				entry.setTime(time);
			} else {
				entry.setInactive(true);
				String inactiveReason = cols.get(columnCounter++).text(); //Column 1
				entry.setInactiveReason(inactiveReason);
				columnCounter++; //Skip column 2
			}
			@SuppressWarnings("unused")
			/*
				ignore Hallennr
				column index is different depending on case above
			 */
			String place = cols.get(columnCounter++).text();
			String matchId = cols.get(columnCounter++).text();
			entry.setMatchId(Long.parseLong(matchId));
			String homeTeam = cols.get(columnCounter++).text().replace("\u00A0", "").trim();
			entry.setHomeTeam(homeTeam);
			String guestTeam = cols.get(columnCounter++).text().replace("\u00A0", "").trim();
			entry.setGuestTeam(guestTeam);
			String result = cols.get(columnCounter++).text().replace("\u00A0", "").trim();
			entry.setResult(result);
			@SuppressWarnings({"unused", "UnusedAssignment"})
			String col9 = cols.get(columnCounter++).text(); //ignore emtpy cell

			seasonScheduleEntries.add(entry);

			em.persist(entry);
			em.flush();
		}
	}

	private void loadTeamStandings(Team team, Document document) throws IOException {
		logger.info("Updating standings data for team={} with id={}", team.getName(), team.getId());
		List<Element> tables = document.select("table");
		Element standingTable = tables.get(0);
		Elements rows = standingTable.select("tr");

		for (int i = 1; i < rows.size(); i++) { //first row is the col names so skip it.
			Element row = rows.get(i);
			Elements cols = row.select("td");
{'fraction_non_alphanumeric': 0.1489085545722714, 'fraction_numerical': 0.008377581120943953, 'mean_word_length': 4.511053315994799, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6651784', 'n_tokens_mistral': 2981, 'n_tokens_neox': 2632, 'n_words': 572}