<commit_before>-- DELETE spawnpoints
CREATE TEMPORARY TABLE IF NOT EXISTS spawnpoints_tmp AS (
  SELECT sp.id
  FROM spawnpoints sp
  LEFT JOIN sightings s ON s.spawn_id = sp.spawn_id
  LEFT JOIN mystery_sightings ms ON ms.spawn_id = sp.spawn_id
  WHERE (s.id IS NULL AND ms.id IS NULL)
  ORDER BY sp.id ASC
  LIMIT 500000
);
DELETE sp.* FROM spawnpoints sp INNER JOIN spawnpoints_tmp spt ON sp.id=spt.id;
<commit_msg>Do NOT delete spawnpoints if last update more than 1 hour ago<commit_after>
SELECT @lastupdate = (SELECT MAX(last_modified) last_updated from fort_sightings);

-- DELETE spawnpoints
CREATE TEMPORARY TABLE IF NOT EXISTS spawnpoints_tmp AS (
  SELECT sp.id
  FROM spawnpoints sp
  LEFT JOIN sightings s ON s.spawn_id = sp.spawn_id
  LEFT JOIN mystery_sightings ms ON ms.spawn_id = sp.spawn_id
  WHERE (s.id IS NULL AND ms.id IS NULL)
	AND @lastupdate > (UNIX_TIMESTAMP() - 3600)
  ORDER BY sp.id ASC
  LIMIT 500000
);
DELETE sp.* FROM spawnpoints sp INNER JOIN spawnpoints_tmp spt ON sp.id=spt.id;

{'fraction_non_alphanumeric': 0.0650887573964497, 'fraction_numerical': 0.016765285996055226, 'mean_word_length': 4.024752475247524, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7192132', 'n_tokens_mistral': 382, 'n_tokens_neox': 346, 'n_words': 132}