<commit_before>package models

func MaintKillMails() error { // Broken into smaller chunks so we have a chance of it getting completed.
	// Delete stuff older than 90 days, we do not care...
	if err := RetryExecTillNoRows(`
				DELETE LOW_PRIORITY A.* FROM evedata.killmailAttackers A
		            JOIN (SELECT id FROM evedata.killmails WHERE killTime < DATE_SUB(UTC_TIMESTAMP, INTERVAL 365 DAY) LIMIT 50000) K ON A.id = K.id;
		            `); err != nil {
		return err
	}
	if err := RetryExecTillNoRows(`
				DELETE LOW_PRIORITY A.* FROM evedata.killmailItems A
		        JOIN (SELECT id FROM evedata.killmails WHERE killTime < DATE_SUB(UTC_TIMESTAMP, INTERVAL 365 DAY) LIMIT 50000) K ON A.id = K.id;
		            `); err != nil {
		return err
	}
	if err := RetryExecTillNoRows(`
				DELETE LOW_PRIORITY FROM evedata.killmails
		        WHERE killTime < DATE_SUB(UTC_TIMESTAMP, INTERVAL 365 DAY) LIMIT 50000;
		            `); err != nil {
		return err
	}

	// Remove any invalid items
	if err := RetryExecTillNoRows(`
	        DELETE LOW_PRIORITY D.* FROM evedata.killmailAttackers D 
            JOIN (SELECT A.id FROM evedata.killmailAttackers A
				 LEFT OUTER JOIN evedata.killmails K ON A.id = K.id
	             WHERE K.id IS NULL LIMIT 500) S ON D.id = S.id;
	               `); err != nil {
		return err
	}
	if err := RetryExecTillNoRows(`
			DELETE LOW_PRIORITY D.* FROM evedata.killmailItems D 
            JOIN (SELECT A.id FROM evedata.killmailItems A
				 LEFT OUTER JOIN evedata.killmails K ON A.id = K.id
	             WHERE K.id IS NULL LIMIT 500) S ON D.id = S.id;
	               `); err != nil {
		return err
	}

	// Prefill stats for known entities that may have no kills
	if _, err := RetryExec(`
        INSERT IGNORE INTO evedata.entityKillStats (id)
	    (SELECT corporationID AS id FROM evedata.corporations); 
            `); err != nil {
		return err
	}
    
	if _, err := RetryExec(`
        INSERT IGNORE INTO evedata.entityKillStats (id)
	    (SELECT allianceID AS id FROM evedata.alliances); 
            `); err != nil {
		return err
	}

	// Build entity stats
	if _, err := RetryExec(`
        INSERT INTO evedata.entityKillStats (id, losses)
            (SELECT 
                victimCorporationID AS id,
                COUNT(DISTINCT K.id) AS losses
            FROM evedata.killmails K
            WHERE K.killTime > DATE_SUB(UTC_TIMESTAMP, INTERVAL 30 DAY)
            GROUP BY victimCorporationID
            ) ON DUPLICATE KEY UPDATE losses = values(losses);
            `); err != nil {
		return err
	}
	if _, err := RetryExec(`
        INSERT INTO evedata.entityKillStats (id, losses)
            (SELECT 
                victimAllianceID AS id,
                COUNT(DISTINCT K.id) AS losses
            FROM evedata.killmails K
            WHERE K.killTime > DATE_SUB(UTC_TIMESTAMP, INTERVAL 30 DAY)
            GROUP BY victimAllianceID
            ) ON DUPLICATE KEY UPDATE losses = values(losses);
            `); err != nil {
		return err
	}

	if _, err := RetryExec(`
        INSERT INTO evedata.entityKillStats (id, kills)
            (SELECT 
                corporationID AS id,
                COUNT(DISTINCT K.id) AS kills
            FROM evedata.killmails K
            INNER JOIN evedata.killmailAttackers A ON A.id = K.id
            WHERE K.killTime > DATE_SUB(UTC_TIMESTAMP, INTERVAL 30 DAY)
            GROUP BY A.corporationID
            ) ON DUPLICATE KEY UPDATE kills = values(kills);
            `); err != nil {
		return err
	}
	if _, err := RetryExec(`
        INSERT INTO evedata.entityKillStats (id, kills)
            (SELECT 
                allianceID AS id,
                COUNT(DISTINCT K.id) AS kills
            FROM evedata.killmails K
            INNER JOIN evedata.killmailAttackers A ON A.id = K.id
            WHERE K.killTime > DATE_SUB(UTC_TIMESTAMP, INTERVAL 30 DAY)
            GROUP BY A.allianceID
            ) ON DUPLICATE KEY UPDATE kills = values(kills);
            `); err != nil {
		return err
	}

	// Update everyone efficiency
	if _, err := RetryExec(`
        UPDATE evedata.entityKillStats SET efficiency = IF(losses+kills, (kills/(kills+losses)) , 1.0000);
            `); err != nil {
		return err
	}

	return nil
}
<commit_msg>increase kill stats to a half a year,<commit_after>package models

func MaintKillMails() error { // Broken into smaller chunks so we have a chance of it getting completed.
	// Delete stuff older than 90 days, we do not care...
	if err := RetryExecTillNoRows(`
				DELETE LOW_PRIORITY A.* FROM evedata.killmailAttackers A
		            JOIN (SELECT id FROM evedata.killmails WHERE killTime < DATE_SUB(UTC_TIMESTAMP, INTERVAL 365 DAY) LIMIT 50000) K ON A.id = K.id;
		            `); err != nil {
		return err
	}
	if err := RetryExecTillNoRows(`
				DELETE LOW_PRIORITY A.* FROM evedata.killmailItems A
		        JOIN (SELECT id FROM evedata.killmails WHERE killTime < DATE_SUB(UTC_TIMESTAMP, INTERVAL 365 DAY) LIMIT 50000) K ON A.id = K.id;
		            `); err != nil {
		return err
	}
	if err := RetryExecTillNoRows(`
				DELETE LOW_PRIORITY FROM evedata.killmails
		        WHERE killTime < DATE_SUB(UTC_TIMESTAMP, INTERVAL 365 DAY) LIMIT 50000;
		            `); err != nil {
		return err
	}

	// Remove any invalid items
	if err := RetryExecTillNoRows(`
	        DELETE LOW_PRIORITY D.* FROM evedata.killmailAttackers D 
            JOIN (SELECT A.id FROM evedata.killmailAttackers A
				 LEFT OUTER JOIN evedata.killmails K ON A.id = K.id
	             WHERE K.id IS NULL LIMIT 500) S ON D.id = S.id;
	               `); err != nil {
		return err
	}
	if err := RetryExecTillNoRows(`
			DELETE LOW_PRIORITY D.* FROM evedata.killmailItems D 
            JOIN (SELECT A.id FROM evedata.killmailItems A
				 LEFT OUTER JOIN evedata.killmails K ON A.id = K.id
	             WHERE K.id IS NULL LIMIT 500) S ON D.id = S.id;
	               `); err != nil {
		return err
	}

	// Prefill stats for known entities that may have no kills
	if _, err := RetryExec(`
        INSERT IGNORE INTO evedata.entityKillStats (id)
	    (SELECT corporationID AS id FROM evedata.corporations); 
            `); err != nil {
		return err
	}

	if _, err := RetryExec(`
        INSERT IGNORE INTO evedata.entityKillStats (id)
	    (SELECT allianceID AS id FROM evedata.alliances); 
            `); err != nil {
		return err
	}

	// Build entity stats
	if _, err := RetryExec(`
        INSERT INTO evedata.entityKillStats (id, losses)
            (SELECT 
                victimCorporationID AS id,
                COUNT(DISTINCT K.id) AS losses
            FROM evedata.killmails K
            WHERE K.killTime > DATE_SUB(UTC_TIMESTAMP, INTERVAL 180 DAY)
            GROUP BY victimCorporationID
            ) ON DUPLICATE KEY UPDATE losses = values(losses);
            `); err != nil {
		return err
	}
	if _, err := RetryExec(`
        INSERT INTO evedata.entityKillStats (id, losses)
            (SELECT 
                victimAllianceID AS id,
                COUNT(DISTINCT K.id) AS losses
            FROM evedata.killmails K
            WHERE K.killTime > DATE_SUB(UTC_TIMESTAMP, INTERVAL 180 DAY)
            GROUP BY victimAllianceID
            ) ON DUPLICATE KEY UPDATE losses = values(losses);
            `); err != nil {
		return err
	}

	if _, err := RetryExec(`
        INSERT INTO evedata.entityKillStats (id, kills)
            (SELECT 
                corporationID AS id,
                COUNT(DISTINCT K.id) AS kills
            FROM evedata.killmails K
            INNER JOIN evedata.killmailAttackers A ON A.id = K.id
            WHERE K.killTime > DATE_SUB(UTC_TIMESTAMP, INTERVAL 180 DAY)
            GROUP BY A.corporationID
            ) ON DUPLICATE KEY UPDATE kills = values(kills);
            `); err != nil {
		return err
	}
	if _, err := RetryExec(`
        INSERT INTO evedata.entityKillStats (id, kills)
            (SELECT 
                allianceID AS id,
                COUNT(DISTINCT K.id) AS kills
            FROM evedata.killmails K
            INNER JOIN evedata.killmailAttackers A ON A.id = K.id
            WHERE K.killTime > DATE_SUB(UTC_TIMESTAMP, INTERVAL 180 DAY)
            GROUP BY A.allianceID
            ) ON DUPLICATE KEY UPDATE kills = values(kills);
            `); err != nil {
		return err
	}

	// Update everyone efficiency
	if _, err := RetryExec(`
        UPDATE evedata.entityKillStats SET efficiency = IF(losses+kills, (kills/(kills+losses)) , 1.0000);
            `); err != nil {
		return err
	}

	return nil
}

{'fraction_non_alphanumeric': 0.08203860640301318, 'fraction_numerical': 0.011064030131826741, 'mean_word_length': 2.0152590489709015, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3179807', 'n_tokens_mistral': 3020, 'n_tokens_neox': 2617, 'n_words': 845}