<commit_before>-- remove duplicates, if any
DELETE FROM hzrdr.ses_rupture AS x USING (
   SELECT ses_id, tag FROM hzrdr.ses_rupture
   GROUP BY ses_id, tag HAVING count(*) > 1) AS y
WHERE x.ses_id=y.ses_id AND x.tag=y.tag;

CREATE UNIQUE INDEX hzrdr_ses_rupture_tag_uniq_idx
ON hzrdr.ses_rupture(ses_id, tag);<commit_msg>Added a comment explaining the need for the DELETE duplicates query<commit_after>/*
A bug was introduced in march 2014 with the refactoring to remove
the ruptures as pickled objects. This is why duplicated rupture
tags may appear in the ses_rupture table. If this upgrade script
fails, just remove the duplicates with the following query:

DELETE FROM hzrdr.ses_rupture AS x USING (
   SELECT ses_id, tag FROM hzrdr.ses_rupture
   GROUP BY ses_id, tag HAVING count(*) > 1) AS y
WHERE x.ses_id=y.ses_id AND x.tag=y.tag;

This is the only viable approach, since if you have duplicated tags
in the database the risk results are not reliable, depending on the
random order the ruptures are extracted from the database.

Since we not want to delete data from the user database without
warning, the query is commented and must be executed manually.
*/

CREATE UNIQUE INDEX hzrdr_ses_rupture_tag_uniq_idx
ON hzrdr.ses_rupture(ses_id, tag);
{'fraction_non_alphanumeric': 0.052673583399840386, 'fraction_numerical': 0.004788507581803671, 'mean_word_length': 4.752293577981652, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1374274', 'n_tokens_mistral': 428, 'n_tokens_neox': 393, 'n_words': 174}