<commit_before>/**
 * Copyright (C) 2008-2011 by ProFUSION embedded systems
 * Copyright (C) 2008 by INdT
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public License
 * as published by the Free Software Foundation; either version 2.1 of
 * the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
 * 02110-1301 USA
 *
 * @author <NAME> <<EMAIL>>
 * @author <NAME> <<EMAIL>>
 */

/**
 * @brief
 *
 * id3 file parser.
 */

#include <lightmediascanner_plugin.h>
#include <lightmediascanner_db.h>
#include <lightmediascanner_charset_conv.h>
#include <shared/util.h>

#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <ctype.h>

#define ID3V2_HEADER_SIZE       10
#define ID3V2_FOOTER_SIZE       10

/* We parse only the first 4 bytes, which are the interesting ones */
#define MPEG_HEADER_SIZE 4

enum mpeg_audio_version {
    MPEG_AUDIO_VERSION_1,
    MPEG_AUDIO_VERSION_2,
    MPEG_AUDIO_VERSION_2_5,
    MPEG_AUDIO_VERSION_4,
};

enum mpeg_audio_layer {
    MPEG_AUDIO_LAYER_1,
    MPEG_AUDIO_LAYER_2,
    MPEG_AUDIO_LAYER_3,
    MPEG_AUDIO_LAYER_AAC,
};

struct mpeg_header {
    enum mpeg_audio_version version;
    enum mpeg_audio_layer layer;

    uint8_t channels;
    uint8_t sampling_rate_idx;
    uint8_t codec_idx;
};

static const struct lms_string_size _codecs[] = {
    /* mp3 */
    [0] = LMS_STATIC_STRING_SIZE("mpeg1layer1"),
    [1] = LMS_STATIC_STRING_SIZE("mpeg1layer2"),
    [2] = LMS_STATIC_STRING_SIZE("mpeg1layer3"),
    [3] = LMS_STATIC_STRING_SIZE("mpeg2layer1"),
    [4] = LMS_STATIC_STRING_SIZE("mpeg2layer2"),
    [5] = LMS_STATIC_STRING_SIZE("mpeg2layer3"),
    [6] = LMS_STATIC_STRING_SIZE("mpeg2.5layer1"),
    [7] = LMS_STATIC_STRING_SIZE("mpeg2.5layer2"),
    [8] = LMS_STATIC_STRING_SIZE("mpeg2.5layer3"),

    /* aac */
#define MPEG_CODEC_AAC_START 9
    [9] = LMS_STATIC_STRING_SIZE("mpeg2aac-main"),
    [10] = LMS_STATIC_STRING_SIZE("mpeg2aac-lc"),
    [11] = LMS_STATIC_STRING_SIZE("mpeg2aac-ssr"),
    [12] = LMS_STATIC_STRING_SIZE("mpeg2aac-ltp"),

    [13] = LMS_STATIC_STRING_SIZE("mpeg4aac-main"),
    [14] = LMS_STATIC_STRING_SIZE("mpeg4aac-lc"),
    [15] = LMS_STATIC_STRING_SIZE("mpeg4aac-ssr"),
    [16] = LMS_STATIC_STRING_SIZE("mpeg4aac-ltp"),
    { }
};

/* Ordered according to AAC index, take care with mp3 */
static int _sample_rates[16] = {
    96000, 88200, 64000,

    /* Frequencies available on mp3, */
    48000, 44100, 32000,
    24000, 22050, 16000,
    12000, 11025, 8000,

    7350, /* reserved, zeroed */
};

enum ID3Encodings {
    ID3_ENCODING_LATIN1 = 0,
    ID3_ENCODING_UTF16,
    ID3_ENCODING_UTF16BE,
    ID3_ENCODING_UTF8,
    ID3_ENCODING_UTF16LE,
    ID3_ENCODING_LAST
};
#define ID3_NUM_ENCODINGS ID3_ENCODING_LAST


#include "id3v1_genres.c"

struct id3_info {
    struct lms_string_size title;
    struct lms_string_size artist;
    struct lms_string_size album;
    struct lms_string_size genre;
    int trackno;
    int cur_artist_priority;
};

struct id3v2_frame_header {
    char frame_id[4];
    unsigned int frame_size;
    int compression;
    int data_length_indicator;
};

struct id3v1_tag {
    char title[30];
    char artist[30];
    char album[30];
    char year[4];
    char comments[30];
    char genre;
} __attribute__((packed));

struct plugin {
    struct lms_plugin plugin;
    lms_db_audio_t *audio_db;
    lms_charset_conv_t *cs_convs[ID3_NUM_ENCODINGS];
};

static const char _name[] = "id3";
static const struct lms_string_size _exts[] = {
    LMS_STATIC_STRING_SIZE(".mp3"),
    LMS_STATIC_STRING_SIZE(".aac"),
    LMS_STATIC_STRING_SIZE(".adts"),
};
static const char *_cats[] = {
    "multimedia",
    "audio",
    NULL
};
static const char *_authors[] = {
    "<NAME>",
    "<NAME>",
    NULL
};

static unsigned int
_to_uint(const char *data, int data_size)
{
    unsigned int sum = 0;
    unsigned int last, i;

    last = data_size > 4 ? 3 : data_size - 1;

    for (i = 0; i <= last; i++)
        sum |= ((unsigned char) data[i]) << ((last - i) * 8);

    return sum;
}

static inline int
_is_id3v2_second_synch_byte(unsigned char byte)
{
    if (byte == 0xff)
        return 0;
    if ((byte & 0xE0) == 0xE0)
        return 1;
    return 0;
}

static inline int
_fill_mp3_header(struct mpeg_header *hdr, const uint8_t b[4])
{
    hdr->sampling_rate_idx = (b[2] & 0x0C) >> 2;
    if (hdr->sampling_rate_idx == 0x3)
        return -1;
    /*
     * Sampling rate frequency index
     * bits     MPEG1           MPEG2           MPEG2.5
     * 00       44100 Hz        22050 Hz        11025 Hz
     * 01       48000 Hz        24000 Hz        12000 Hz
     * 10       32000 Hz        16000 Hz        8000 Hz
     * 11       reserv.         reserv.         reserv.
     */

    /* swap 0x1 and 0x0 */
    if (hdr->sampling_rate_idx < 0x2)
        hdr->sampling_rate_idx = !hdr->sampling_rate_idx;
    hdr->sampling_rate_idx += 3 * hdr->version + 3;

    hdr->codec_idx = hdr->version * 3 + hdr->layer;

    hdr->channels = (b[3] & 0xC0) >> 6;
    hdr->channels = hdr->channels == 0x3 ? 1 : 2;
    return 0;
}

static inline int
_fill_aac_header(struct mpeg_header *hdr, const uint8_t b[4])
{
    unsigned int profile;

    hdr->sampling_rate_idx = (b[2] & 0x3C) >> 2;

    profile = (b[2] & 0xC0) >> 6;
    hdr->codec_idx = MPEG_CODEC_AAC_START + profile;
    if (hdr->version == MPEG_AUDIO_VERSION_4)
        hdr->codec_idx += 4;

    hdr->channels = ((b[2] & 0x1) << 2) | ((b[3] & 0xC0) >> 6);
    return 0;
}

static inline int
_fill_mpeg_header(struct mpeg_header *hdr, const uint8_t b[4])
{
    unsigned int version = (b[1] & 0x18) >>  3;
    unsigned int layer = (b[1] & 0x06) >> 1;

    switch (layer) {
    case 0x0:
        if (version == 0x2 || version == 0x3)
            hdr->layer = MPEG_AUDIO_LAYER_AAC;
        else
            return -1;
        break;
    case 0x1:
        hdr->layer = MPEG_AUDIO_LAYER_3;
        break;
    case 0x2:
        hdr->layer = MPEG_AUDIO_LAYER_2;
        break;
    case 0x3:
        hdr->layer = MPEG_AUDIO_LAYER_1;
        break;
    }

    switch (version) {
    case 0x0:
        hdr->version = MPEG_AUDIO_VERSION_2_5;
        break;
    case 0x1:
        return -1;
    case 0x2:
        if (layer == 0x0)
            hdr->version = MPEG_AUDIO_VERSION_4;
        else
            hdr->version = MPEG_AUDIO_VERSION_2;
        break;
    case 0x3:
        if (layer == 0x0)
            hdr->version = MPEG_AUDIO_VERSION_2;
        else
            hdr->version = MPEG_AUDIO_VERSION_1;
    }

    if (hdr->layer == MPEG_AUDIO_LAYER_AAC)
        return _fill_aac_header(hdr, b);
    else
        return _fill_mp3_header(hdr, b);

    return 0;
}

static int
_parse_mpeg_header(int fd, off_t off, struct lms_audio_info *audio_info)
{
    uint8_t buffer[32];
    const uint8_t *p, *p_end;
    unsigned int prev_read;
    struct mpeg_header hdr;

    lseek(fd, off, SEEK_SET);

    /* Find sync word */
    prev_read = 0;
    do {
        int nread = read(fd, buffer + prev_read, sizeof(buffer) - prev_read);
        if (nread < MPEG_HEADER_SIZE)
            return -1;

        p = buffer;
        p_end = buffer + nread;
        while (p < p_end && (p = memchr(p, 0xFF, p_end - p))) {
            /* poor man's ring buffer since the the needle is small (4 bytes) */
            if (p > p_end - MPEG_HEADER_SIZE) {
                memcpy(buffer, p, p_end - p);
                break;
            }

            if (_is_id3v2_second_synch_byte(*(p + 1)))
                goto found;

            p++;
        }
        prev_read = p ? p_end - p : 0;
    } while(1);

found:

    if (_fill_mpeg_header(&hdr, p) < 0) {
        fprintf(stderr, "Invalid field in file, ignoring.\n");
        return 0;<commit_msg>plugins/id3: Add references and fix typo<commit_after>/**
 * Copyright (C) 2008-2011 by ProFUSION embedded systems
 * Copyright (C) 2008 by INdT
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public License
 * as published by the Free Software Foundation; either version 2.1 of
 * the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
 * 02110-1301 USA
 *
 * @author <NAME> <<EMAIL>>
 * @author <NAME> <<EMAIL>>
 */

/**
 * @brief
 *
 * id3 file parser.
 *
 * Reference:
 *   http://www.mp3-tech.org/programmer/frame_header.html
 *   http://www.mpgedit.org/mpgedit/mpeg_format/MP3Format.html
 */

#include <lightmediascanner_plugin.h>
#include <lightmediascanner_db.h>
#include <lightmediascanner_charset_conv.h>
#include <shared/util.h>

#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <ctype.h>

#define ID3V2_HEADER_SIZE       10
#define ID3V2_FOOTER_SIZE       10

/* We parse only the first 4 bytes, which are the interesting ones */
#define MPEG_HEADER_SIZE 4

enum mpeg_audio_version {
    MPEG_AUDIO_VERSION_1,
    MPEG_AUDIO_VERSION_2,
    MPEG_AUDIO_VERSION_2_5,
    MPEG_AUDIO_VERSION_4,
};

enum mpeg_audio_layer {
    MPEG_AUDIO_LAYER_1,
    MPEG_AUDIO_LAYER_2,
    MPEG_AUDIO_LAYER_3,
    MPEG_AUDIO_LAYER_AAC,
};

struct mpeg_header {
    enum mpeg_audio_version version;
    enum mpeg_audio_layer layer;

    uint8_t channels;
    uint8_t sampling_rate_idx;
    uint8_t codec_idx;
};

static const struct lms_string_size _codecs[] = {
    /* mp3 */
    [0] = LMS_STATIC_STRING_SIZE("mpeg1layer1"),
    [1] = LMS_STATIC_STRING_SIZE("mpeg1layer2"),
    [2] = LMS_STATIC_STRING_SIZE("mpeg1layer3"),
    [3] = LMS_STATIC_STRING_SIZE("mpeg2layer1"),
    [4] = LMS_STATIC_STRING_SIZE("mpeg2layer2"),
    [5] = LMS_STATIC_STRING_SIZE("mpeg2layer3"),
    [6] = LMS_STATIC_STRING_SIZE("mpeg2.5layer1"),
    [7] = LMS_STATIC_STRING_SIZE("mpeg2.5layer2"),
    [8] = LMS_STATIC_STRING_SIZE("mpeg2.5layer3"),

    /* aac */
#define MPEG_CODEC_AAC_START 9
    [9] = LMS_STATIC_STRING_SIZE("mpeg2aac-main"),
    [10] = LMS_STATIC_STRING_SIZE("mpeg2aac-lc"),
    [11] = LMS_STATIC_STRING_SIZE("mpeg2aac-ssr"),
    [12] = LMS_STATIC_STRING_SIZE("mpeg2aac-ltp"),

    [13] = LMS_STATIC_STRING_SIZE("mpeg4aac-main"),
    [14] = LMS_STATIC_STRING_SIZE("mpeg4aac-lc"),
    [15] = LMS_STATIC_STRING_SIZE("mpeg4aac-ssr"),
    [16] = LMS_STATIC_STRING_SIZE("mpeg4aac-ltp"),
    { }
};

/* Ordered according to AAC index, take care with mp3 */
static int _sample_rates[16] = {
    96000, 88200, 64000,

    /* Frequencies available on mp3, */
    48000, 44100, 32000,
    24000, 22050, 16000,
    12000, 11025, 8000,

    7350, /* reserved, zeroed */
};

enum ID3Encodings {
    ID3_ENCODING_LATIN1 = 0,
    ID3_ENCODING_UTF16,
    ID3_ENCODING_UTF16BE,
    ID3_ENCODING_UTF8,
    ID3_ENCODING_UTF16LE,
    ID3_ENCODING_LAST
};
#define ID3_NUM_ENCODINGS ID3_ENCODING_LAST


#include "id3v1_genres.c"

struct id3_info {
    struct lms_string_size title;
    struct lms_string_size artist;
    struct lms_string_size album;
    struct lms_string_size genre;
    int trackno;
    int cur_artist_priority;
};

struct id3v2_frame_header {
    char frame_id[4];
    unsigned int frame_size;
    int compression;
    int data_length_indicator;
};

struct id3v1_tag {
    char title[30];
    char artist[30];
    char album[30];
    char year[4];
    char comments[30];
    char genre;
} __attribute__((packed));

struct plugin {
    struct lms_plugin plugin;
    lms_db_audio_t *audio_db;
    lms_charset_conv_t *cs_convs[ID3_NUM_ENCODINGS];
};

static const char _name[] = "id3";
static const struct lms_string_size _exts[] = {
    LMS_STATIC_STRING_SIZE(".mp3"),
    LMS_STATIC_STRING_SIZE(".aac"),
    LMS_STATIC_STRING_SIZE(".adts"),
};
static const char *_cats[] = {
    "multimedia",
    "audio",
    NULL
};
static const char *_authors[] = {
    "<NAME>",
    "<NAME>",
    NULL
};

static unsigned int
_to_uint(const char *data, int data_size)
{
    unsigned int sum = 0;
    unsigned int last, i;

    last = data_size > 4 ? 3 : data_size - 1;

    for (i = 0; i <= last; i++)
        sum |= ((unsigned char) data[i]) << ((last - i) * 8);

    return sum;
}

static inline int
_is_id3v2_second_synch_byte(unsigned char byte)
{
    if (byte == 0xff)
        return 0;
    if ((byte & 0xE0) == 0xE0)
        return 1;
    return 0;
}

static inline int
_fill_mp3_header(struct mpeg_header *hdr, const uint8_t b[4])
{
    hdr->sampling_rate_idx = (b[2] & 0x0C) >> 2;
    if (hdr->sampling_rate_idx == 0x3)
        return -1;
    /*
     * Sampling rate frequency index
     * bits     MPEG1           MPEG2           MPEG2.5
     * 00       44100 Hz        22050 Hz        11025 Hz
     * 01       48000 Hz        24000 Hz        12000 Hz
     * 10       32000 Hz        16000 Hz        8000 Hz
     * 11       reserv.         reserv.         reserv.
     */

    /* swap 0x1 and 0x0 */
    if (hdr->sampling_rate_idx < 0x2)
        hdr->sampling_rate_idx = !hdr->sampling_rate_idx;
    hdr->sampling_rate_idx += 3 * hdr->version + 3;

    hdr->codec_idx = hdr->version * 3 + hdr->layer;

    hdr->channels = (b[3] & 0xC0) >> 6;
    hdr->channels = hdr->channels == 0x3 ? 1 : 2;
    return 0;
}

static inline int
_fill_aac_header(struct mpeg_header *hdr, const uint8_t b[4])
{
    unsigned int profile;

    hdr->sampling_rate_idx = (b[2] & 0x3C) >> 2;

    profile = (b[2] & 0xC0) >> 6;
    hdr->codec_idx = MPEG_CODEC_AAC_START + profile;
    if (hdr->version == MPEG_AUDIO_VERSION_4)
        hdr->codec_idx += 4;

    hdr->channels = ((b[2] & 0x1) << 2) | ((b[3] & 0xC0) >> 6);
    return 0;
}

static inline int
_fill_mpeg_header(struct mpeg_header *hdr, const uint8_t b[4])
{
    unsigned int version = (b[1] & 0x18) >>  3;
    unsigned int layer = (b[1] & 0x06) >> 1;

    switch (layer) {
    case 0x0:
        if (version == 0x2 || version == 0x3)
            hdr->layer = MPEG_AUDIO_LAYER_AAC;
        else
            return -1;
        break;
    case 0x1:
        hdr->layer = MPEG_AUDIO_LAYER_3;
        break;
    case 0x2:
        hdr->layer = MPEG_AUDIO_LAYER_2;
        break;
    case 0x3:
        hdr->layer = MPEG_AUDIO_LAYER_1;
        break;
    }

    switch (version) {
    case 0x0:
        hdr->version = MPEG_AUDIO_VERSION_2_5;
        break;
    case 0x1:
        return -1;
    case 0x2:
        if (layer == 0x0)
            hdr->version = MPEG_AUDIO_VERSION_4;
        else
            hdr->version = MPEG_AUDIO_VERSION_2;
        break;
    case 0x3:
        if (layer == 0x0)
            hdr->version = MPEG_AUDIO_VERSION_2;
        else
            hdr->version = MPEG_AUDIO_VERSION_1;
    }

    if (hdr->layer == MPEG_AUDIO_LAYER_AAC)
        return _fill_aac_header(hdr, b);
    else
        return _fill_mp3_header(hdr, b);

    return 0;
}

static int
_parse_mpeg_header(int fd, off_t off, struct lms_audio_info *audio_info)
{
    uint8_t buffer[32];
    const uint8_t *p, *p_end;
    unsigned int prev_read;
    struct mpeg_header hdr;

    lseek(fd, off, SEEK_SET);

    /* Find sync word */
    prev_read = 0;
    do {
        int nread = read(fd, buffer + prev_read, sizeof(buffer) - prev_read);
        if (nread < MPEG_HEADER_SIZE)
            return -1;

        p = buffer;
        p_end = buffer + nread;
        while (p < p_end && (p = memchr(p, 0xFF, p_end - p))) {
            /* poor man's ring buffer since the needle is small (4 bytes) */
            if (p > p_end - MPEG_HEADER_SIZE) {
                memcpy(buffer, p, p_end - p);
                break;
            }

            if (_is_id3v2_second_synch_byte(*(p + 1)))
                goto found;

            p++;
        }
        prev_read = p ? p_end - p : 0;
    } while(1);

found:
    if (_fill_mpeg_header(&hdr, p) < 0) {
        fprintf(stderr, "Invalid field in file, ignoring.\n");
        return 0;
{'fraction_non_alphanumeric': 0.11511350059737156, 'fraction_numerical': 0.0470131421744325, 'mean_word_length': 2.666447656592203, 'pattern_counts': {'":': 0, '<': 61, '<?xml version=': 0, '>': 131, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7182476', 'n_tokens_mistral': 6774, 'n_tokens_neox': 6185, 'n_words': 1500}