<commit_before>#!/usr/bin/env python

"""
Script to validate certain conventions of FLAC releases

It does checks at different level. For example, the tracktotal check
is done at the disc level. This means that the property has to be there,
but it also has to be the same for every track on that disc. Properties
at the album level have to be the same for every track in the album.

Checks:
 - The FLAC files:
   - checks that files aren't corrupted by verifying the STREAMINFO MD5
   - checks the path length of each file
   - [TODO] recommed re-encoding to add the md5 if one doesn't exist

 - The extra info:
   - checks if a cue and log file are provided at the disc level
   - checks a cover image is provided at the album level
   - checks if an m3u file shoudl be deleted
   - [TODO] check a folder with additional art is provided at the album level

 - The folder/file names:
   - [TODO] validate a naming scheme
   - [TODO] validate the name against vorbis information

 - vorbis information:
   - album, date, albumartist, and disctotal are at the album level
   - discnumber, tracktotal are at the disc level
   - artist, tracknumber, title are at the track level
   - checks for totaldiscs and totaltracks metadata
   - checks that disctotal is equal to the amount of discs
   - checks that tracktotal is equal to the number of tracks
   - checks for duplicate tags
   - [TODO] warn if TRACKNUMBER is the "tracknum/totaltracks" style
   - [TODO] warn if album art is embedded

 - replaygain information:
   - checks reference loudness, album gain, album peak are at the disc level
   - checks track gain and track peak are at the track level
"""

import argparse
import itertools
import subprocess
import os

import taglib

MAX_PATH_LENGTH = 180
COVER_FILENAME = "cover.jpg"
REQUIRED_TAGS_ALBUM = {"ALBUM", "DATE", "ALBUMARTIST", "DISCTOTAL"}
REQUIRED_TAGS_DISC = {"DISCNUMBER", "TRACKTOTAL"}
REQUIRED_TAGS_TRACK = {"ARTIST", "TRACKNUMBER", "TITLE"}
REPLAYGAIN_TAGS_DISC = {"REPLAYGAIN_REFERENCE_LOUDNESS", "REPLAYGAIN_ALBUM_GAIN",
                        "REPLAYGAIN_ALBUM_PEAK"}
REPLAYGAIN_TAGS_TRACK = {"REPLAYGAIN_TRACK_GAIN", "REPLAYGAIN_TRACK_PEAK"}


def has_ext(path, ext):
    return path.rsplit(".", 1)[-1].lower() == ext.lower()


def files_by_ext(files, ext):
    return [x for x in files if has_ext(x, ext)]


class ValidatorBase(object):

    def validate_all_same(self, tag):
        temp = self.get_tag(tag, placeholder=True)
        tags = set(temp)
        if None in tags:
            tags.remove(None)
            if len(tags) == 0:
                print("Problem with {}-level tag {} - missing from all items".format(self.level, tag))
                return
            else:
                print("Problem with {}-level tag {} - missing from {}/{} items"
                      "".format(self.level, tag, temp.count(None), len(temp)))
        if len(tags) > 1:
            print("Problem with {}-level tag {} - multiple values: {}".format(self.level, tag, tags))

    def validate_number_metadata(self):
        # Check for invalid [type]TOTAL metadata
        if self.__class__ is Album:<commit_msg>Pull apart tag checking and message printing<commit_after>#!/usr/bin/env python

"""
Script to validate certain conventions of FLAC releases

It does checks at different level. For example, the tracktotal check
is done at the disc level. This means that the property has to be there,
but it also has to be the same for every track on that disc. Properties
at the album level have to be the same for every track in the album.

Checks:
 - The FLAC files:
   - checks that files aren't corrupted by verifying the STREAMINFO MD5
   - checks the path length of each file
   - [TODO] recommed re-encoding to add the md5 if one doesn't exist

 - The extra info:
   - checks if a cue and log file are provided at the disc level
   - checks a cover image is provided at the album level
   - checks if an m3u file shoudl be deleted
   - [TODO] check a folder with additional art is provided at the album level

 - The folder/file names:
   - [TODO] validate a naming scheme
     - folder: [<ALBUMARTIST> - ]<ALBUM> (<YEAR>) \[{CD,WEB}-FLAC\] {<anything}}
     - file: <TRACKNUMBER> - [<ARTIST> - ]<TITLE>.flac
   - [TODO] validate the name against vorbis information

 - vorbis information:
   - album, date, albumartist, and disctotal are at the album level
   - discnumber, tracktotal are at the disc level
   - artist, tracknumber, title are at the track level
   - checks for totaldiscs and totaltracks metadata
   - checks that disctotal is equal to the amount of discs
   - checks that tracktotal is equal to the number of tracks
   - checks for duplicate tags
   - [TODO] warn if TRACKNUMBER is the "tracknum/totaltracks" style
   - [TODO] warn if album art is embedded

 - replaygain information:
   - checks reference loudness, album gain, album peak are at the disc level
   - checks track gain and track peak are at the track level
"""

import argparse
import enum
import itertools
import subprocess
import os

import taglib

MAX_PATH_LENGTH = 180
COVER_FILENAME = "cover.jpg"
REQUIRED_TAGS_ALBUM = {"ALBUM", "DATE", "ALBUMARTIST", "DISCTOTAL"}
REQUIRED_TAGS_DISC = {"DISCNUMBER", "TRACKTOTAL"}
REQUIRED_TAGS_TRACK = {"ARTIST", "TRACKNUMBER", "TITLE"}
REPLAYGAIN_TAGS_DISC = {"REPLAYGAIN_REFERENCE_LOUDNESS", "REPLAYGAIN_ALBUM_GAIN",
                        "REPLAYGAIN_ALBUM_PEAK"}
REPLAYGAIN_TAGS_TRACK = {"REPLAYGAIN_TRACK_GAIN", "REPLAYGAIN_TRACK_PEAK"}


def has_ext(path, ext):
    return path.rsplit(".", 1)[-1].lower() == ext.lower()


def files_by_ext(files, ext):
    return [x for x in files if has_ext(x, ext)]


class Missing(enum.Enum):
    NONE = 0
    SOME = 1
    ALL = 2

class ValidatorBase(object):

    def _check_all_same(self, tag):
        """Check and generate messages but don't print them

        returns (missing code, multiple, msg)
        """
        code = Missing.NONE
        multiple = False
        msgs = []

        temp = self.get_tag(tag, placeholder=True)
        tags = set(temp)
        if None in tags:
            tags.remove(None)
            if len(tags) == 0:
                code = Missing.ALL
                msgs.append("missing from all items")
            else:
                code = Missing.SOME
                msgs.append("missing from {}/{} items".format(temp.count(None),
                                                             len(temp)))
        if len(tags) > 1:
            multiple = True
            msgs.append("multiple values: {}".format(tags))

        return code, multiple, msgs

    def validate_all_same(self, tag):
        code, multiple, msgs = self._check_all_same(tag)
        if code != Missing.NONE or multiple:
            print("Problem with tag {}: {}".format(tag, ", ".join(msgs)))

    def validate_number_metadata(self):
        # Check for invalid [type]TOTAL metadata
        if self.__class__ is Album:
{'fraction_non_alphanumeric': 0.08773448773448773, 'fraction_numerical': 0.003318903318903319, 'mean_word_length': 2.874231414197876, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4637695', 'n_tokens_mistral': 2139, 'n_tokens_neox': 2048, 'n_words': 795}