<commit_before>/*
 *  Copyright 2006 The National Library of New Zealand
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */

package nz.govt.natlib.adapter.mp3;

import java.io.File;
import java.io.IOException;
import java.util.HashMap;

import nz.govt.natlib.adapter.DataAdapter;
import nz.govt.natlib.fx.BitFieldUtil;
import nz.govt.natlib.fx.CompoundElement;
import nz.govt.natlib.fx.DataSource;
import nz.govt.natlib.fx.Element;
import nz.govt.natlib.fx.EnumeratedElement;
import nz.govt.natlib.fx.FXUtil;
import nz.govt.natlib.fx.FileDataSource;
import nz.govt.natlib.fx.FixedLengthStringElement;
import nz.govt.natlib.fx.IntegerElement;
import nz.govt.natlib.fx.ParserContext;

/**
 * Adapter for MP3 Audio Files.
 * 
 * @author unascribed
 * @version 1.0
 */

public class MP3Adapter extends DataAdapter {

	private Element genre = new EnumeratedElement(new IntegerElement(
			IntegerElement.BYTE_SIZE, false, IntegerElement.DECIMAL_FORMAT),
			new String[] { "0", "1", "2", "3", "4", "5", "6", "7", "8", "9",
					"10", "11", "12", "13", "14", "15", "16", "17", "18", "19",
					"20", "21", "22", "23", "24", "25", "26", "27", "28", "29",
					"30", "31", "32", "33", "34", "35", "36", "37", "38", "39",
					"40", "41", "42", "43", "44", "45", "46", "47", "48", "49",
					"50", "51", "52", "53", "54", "55", "56", "57", "58", "59",
					"60", "61", "62", "63", "64", "65", "66", "67", "68", "69",
					"70", "71", "72", "73", "74", "75", "76", "77", "78", "79",
					"80", "81", "82", "83", "84", "85", "86", "87", "88", "89",
					"90", "91", "92", "93", "94", "95", "96", "97", "98", "99",
					"100", "101", "102", "103", "104", "105", "106", "107",
					"108", "109", "110", "111", "112", "113", "114", "115",
					"116", "117", "118", "119", "120", "121", "122", "123",
					"124", "125" }, new String[] { "Blues", "Classic Rock",
					"Country", "Dance", "Disco", "Funk", "Grunge", "Hip-Hop",
					"Jazz", "Metal", "New Age", "Oldies", "Other", "Pop",
					"R&B", "Rap", "Reggae", "Rock", "Techno", "Industrial",
					"Alternative", "Ska", "Death Metal", "Pranks",
					"Soundtrack", "Euro-Techno", "Ambient", "Trip-Hop",
					"Vocal", "Jazz+Funk", "Fusion", "Trance", "Classical",
					"Instrumental", "Acid", "House", "Game", "Sound Clip",
					"Gospel", "Noise", "AlternRock", "Bass", "Soul", "Punk",
					"Space", "Meditative", "Instrumental Pop",
					"Instrumental Rock", "Ethnic", "Gothic", "Darkwave",
					"Techno-Industrial", "Electronic", "Pop-Folk", "Eurodance",
					"Dream", "Southern Rock", "Comedy", "Cult", "Gangsta",
					"Top 40", "Christian Rap", "Pop/Funk", "Jungle",
					"Native American", "Cabaret", "New Wave", "Psychadelic",
					"Rave", "Showtunes", "Trailer", "Lo-Fi", "Tribal",
					"Acid Punk", "Acid Jazz", "Polka", "Retro", "Musical",
					"Rock & Roll", "Hard Rock", "Folk", "Folk-Rock",
					"National Folk", "Swing", "Fast Fusion", "Bebob", "Latin",
					"Revival", "Celtic", "Bluegrass", "Avantgarde",
					"Gothic Rock", "Progressive Rock", "Psychedelic Rock",
					"Symphonic Rock", "Slow Rock", "Big Band", "Chorus",
					"Easy Listening", "Acoustic", "Humour", "Speech",
					"Chanson", "Opera", "Chamber Music", "Sonata", "Symphony",
					"Booty Brass", "Primus", "Porn Groove", "Satire",
					"Slow Jam", "Club", "Tango", "Samba", "Folklore", "Ballad",
					"Poweer Ballad", "Rhytmic Soul", "Freestyle", "Duet",
					"Punk Rock", "Drum Solo", "A Capela", "Euro-House",
					"Dance Hall" }, "unknown");

	// ID3v1 Tag
	private Element id3v1Element = new CompoundElement(new String[] { "title",
			"artist", "album", "year", "comment", "genre" }, new Element[] {
			new FixedLengthStringElement(30), new FixedLengthStringElement(30),
			new FixedLengthStringElement(30), new FixedLengthStringElement(4),
			new FixedLengthStringElement(30), genre, });

	// ID3v1.1 Tag
	private Element id3v11Element = new CompoundElement(new String[] { "title",
			"artist", "album", "year", "comment", "track", "genre" },
			new Element[] {
					new FixedLengthStringElement(30),
					new FixedLengthStringElement(30),
					new FixedLengthStringElement(30),
					new FixedLengthStringElement(4),
					new FixedLengthStringElement(29),
					new IntegerElement(IntegerElement.BYTE_SIZE, false,
							IntegerElement.DECIMAL_FORMAT), genre, });

	public boolean acceptsFile(File file) {
		String name = file.getName().toLowerCase();
		if (ignoreFileExtension || name.endsWith(".mp3")) {
			return true;
		}

		return false;
	}

	public String getVersion() {
		return "1.0";
	}

	public String getOutputType() {
		return "mp3.dtd";
	}

	public String getInputType() {
		return "audio/mpeg";
	}

	public String getName() {
		return "Mp3 Audio Adapter";
	}

	public String getDescription() {
		return "Adapts Mp3 Audio files including ID3v1 and ID3v2 tags";
	}

	public void adapt(File file, ParserContext ctx) throws IOException {
		long SECS = 1000;
		long MINS = 60 * SECS;
		long HRS = 60 * MINS;
		// Add the MetaData to the tree!
		DataSource ftk = new FileDataSource(file);
		ctx.fireStartParseEvent("mp3");
		writeFileInfo(file, ctx);
		try {
			boolean headFound = false;
			int sizeOfID3Metadata = 0;
			// read the ID3v2.x tag...
			ftk.setPosition(0);
			String id3 = FXUtil.getFixedStringValue(ftk, 3);
			if (id3.equalsIgnoreCase("ID3")) {
				headFound = true;
				ctx.fireStartParseEvent("header");
				byte[] ver = ftk.getData(1);
				byte[] subVer = ftk.getData(1);
				byte[] fg = ftk.getData(1);
				byte[] s = ftk.getData(4);
				int version = (int) FXUtil.getNumericalValue(ver, true);
				int subVersion = (int) FXUtil.getNumericalValue(subVer, true);
				int flags = (int) FXUtil.getNumericalValue(fg, true);
				int size = (int) FXUtil.getNumericalValue(s, true);
				boolean unsync = BitFieldUtil.isSet(flags, 64);
				boolean ext = BitFieldUtil.isSet(flags, 32);
				boolean exp = BitFieldUtil.isSet(flags, 16);
				boolean foot = BitFieldUtil.isSet(flags, 8);
				size = getSyncsafeInt(size);

				ctx.fireParseEvent("tag-version", "ID3v2." + version + "."
						+ subVersion);

				// if there is an extended header then that needs to be read...
				if (ext) {
					byte[] extS = ftk.getData(4);
					int extSize = (int) FXUtil.getNumericalValue(extS, true);
					extSize = getSyncsafeInt(extSize);
					// move past the crap...
					ftk.setPosition(ftk.getPosition() + extSize - 4);
				}

				// allow for the footer to be present
				if (foot) {
					size -= 10;
				}

				// now read the frames...
				int read = 0;
				while (read < size) {
					String frameType = FXUtil.getFixedStringValue(ftk, 4);
					byte[] fS = ftk.getData(4);
					byte[] frameFlags = ftk.getData(2);
					int frameSize = (int) FXUtil.getNumericalValue(fS, true);
					
					/*
					 * The Sync Safe Int logic doesn't seem to work for an embedded
					 * picture (frame with "APIC" frame type). 
					 */
					if ("APIC".equalsIgnoreCase(frameType) == false)
						frameSize = getSyncsafeInt(frameSize);
					String data = FXUtil.getFixedStringValue(ftk, frameSize);
					ID3v2Tag v2tag = MP3Util.getTag(frameType, data);
					if (v2tag != null) {
						HashMap map = v2tag.getMap();
						if (map != null) {
							ctx.fireParseEvent(v2tag.getName(), v2tag
									.getValue(), false, map);
						} else {
							ctx.fireParseEvent(v2tag.getName(), v2tag
									.getValue());
						}
					}
					read += frameSize + 10;
				}<commit_msg>Frame size Sync Safe Int conversion on certain ID3 frame types caused NegativeArraySizeException. Fixed this problem.<commit_after>/*
 *  Copyright 2006 The National Library of New Zealand
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */

package nz.govt.natlib.adapter.mp3;

import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;

import nz.govt.natlib.adapter.DataAdapter;
import nz.govt.natlib.fx.BitFieldUtil;
import nz.govt.natlib.fx.CompoundElement;
import nz.govt.natlib.fx.DataSource;
import nz.govt.natlib.fx.Element;
import nz.govt.natlib.fx.EnumeratedElement;
import nz.govt.natlib.fx.FXUtil;
import nz.govt.natlib.fx.FileDataSource;
import nz.govt.natlib.fx.FixedLengthStringElement;
import nz.govt.natlib.fx.IntegerElement;
import nz.govt.natlib.fx.ParserContext;

/**
 * Adapter for MP3 Audio Files.
 * 
 * @author unascribed
 * @version 1.0
 */

public class MP3Adapter extends DataAdapter {

	private Element genre = new EnumeratedElement(new IntegerElement(
			IntegerElement.BYTE_SIZE, false, IntegerElement.DECIMAL_FORMAT),
			new String[] { "0", "1", "2", "3", "4", "5", "6", "7", "8", "9",
					"10", "11", "12", "13", "14", "15", "16", "17", "18", "19",
					"20", "21", "22", "23", "24", "25", "26", "27", "28", "29",
					"30", "31", "32", "33", "34", "35", "36", "37", "38", "39",
					"40", "41", "42", "43", "44", "45", "46", "47", "48", "49",
					"50", "51", "52", "53", "54", "55", "56", "57", "58", "59",
					"60", "61", "62", "63", "64", "65", "66", "67", "68", "69",
					"70", "71", "72", "73", "74", "75", "76", "77", "78", "79",
					"80", "81", "82", "83", "84", "85", "86", "87", "88", "89",
					"90", "91", "92", "93", "94", "95", "96", "97", "98", "99",
					"100", "101", "102", "103", "104", "105", "106", "107",
					"108", "109", "110", "111", "112", "113", "114", "115",
					"116", "117", "118", "119", "120", "121", "122", "123",
					"124", "125" }, new String[] { "Blues", "Classic Rock",
					"Country", "Dance", "Disco", "Funk", "Grunge", "Hip-Hop",
					"Jazz", "Metal", "New Age", "Oldies", "Other", "Pop",
					"R&B", "Rap", "Reggae", "Rock", "Techno", "Industrial",
					"Alternative", "Ska", "Death Metal", "Pranks",
					"Soundtrack", "Euro-Techno", "Ambient", "Trip-Hop",
					"Vocal", "Jazz+Funk", "Fusion", "Trance", "Classical",
					"Instrumental", "Acid", "House", "Game", "Sound Clip",
					"Gospel", "Noise", "AlternRock", "Bass", "Soul", "Punk",
					"Space", "Meditative", "Instrumental Pop",
					"Instrumental Rock", "Ethnic", "Gothic", "Darkwave",
					"Techno-Industrial", "Electronic", "Pop-Folk", "Eurodance",
					"Dream", "Southern Rock", "Comedy", "Cult", "Gangsta",
					"Top 40", "Christian Rap", "Pop/Funk", "Jungle",
					"Native American", "Cabaret", "New Wave", "Psychadelic",
					"Rave", "Showtunes", "Trailer", "Lo-Fi", "Tribal",
					"Acid Punk", "Acid Jazz", "Polka", "Retro", "Musical",
					"Rock & Roll", "Hard Rock", "Folk", "Folk-Rock",
					"National Folk", "Swing", "Fast Fusion", "Bebob", "Latin",
					"Revival", "Celtic", "Bluegrass", "Avantgarde",
					"Gothic Rock", "Progressive Rock", "Psychedelic Rock",
					"Symphonic Rock", "Slow Rock", "Big Band", "Chorus",
					"Easy Listening", "Acoustic", "Humour", "Speech",
					"Chanson", "Opera", "Chamber Music", "Sonata", "Symphony",
					"Booty Brass", "Primus", "Porn Groove", "Satire",
					"Slow Jam", "Club", "Tango", "Samba", "Folklore", "Ballad",
					"Poweer Ballad", "Rhytmic Soul", "Freestyle", "Duet",
					"Punk Rock", "Drum Solo", "A Capela", "Euro-House",
					"Dance Hall" }, "unknown");

	// ID3v1 Tag
	private Element id3v1Element = new CompoundElement(new String[] { "title",
			"artist", "album", "year", "comment", "genre" }, new Element[] {
			new FixedLengthStringElement(30), new FixedLengthStringElement(30),
			new FixedLengthStringElement(30), new FixedLengthStringElement(4),
			new FixedLengthStringElement(30), genre, });

	// ID3v1.1 Tag
	private Element id3v11Element = new CompoundElement(new String[] { "title",
			"artist", "album", "year", "comment", "track", "genre" },
			new Element[] {
					new FixedLengthStringElement(30),
					new FixedLengthStringElement(30),
					new FixedLengthStringElement(30),
					new FixedLengthStringElement(4),
					new FixedLengthStringElement(29),
					new IntegerElement(IntegerElement.BYTE_SIZE, false,
							IntegerElement.DECIMAL_FORMAT), genre, });

	/*
	 * The Sync Safe Int logic doesn't seem to work for certain frame types,
	 * such as an embedded picture (frame with "APIC" frame type) or 
	 * the Music CD info ("MCDI" frame type). When we do a sync safe int
	 * conversion on the frame size for these frames, the frame size becomes 
	 * negative and a NegativeArraySizeException is thrown from the application.
	 * 
	 * The list below defines the frame types that shouldn't do a SyncsafeInt 
	 * conversion on the frame size.
	 * 
	 * Note: If certain MP3 files fail the metadata extraction giving a
	 * NegativeArraySizeException or a NullPointerException, it may be due the
	 * fact that certain frame types in the MP3 file don't require/support the
	 * sync safe int conversion of frame size. If so, add those frame types 
	 * to the list below.
	 */
	private final List frameTypesToIgnoreSyncSafeInt = Arrays.asList(new String[]{
		"APIC",
		"MCDI",
	});
	
	public boolean acceptsFile(File file) {
		String name = file.getName().toLowerCase();
		if (ignoreFileExtension || name.endsWith(".mp3")) {
			return true;
		}

		return false;
	}

	public String getVersion() {
		return "1.0";
	}

	public String getOutputType() {
		return "mp3.dtd";
	}

	public String getInputType() {
		return "audio/mpeg";
	}

	public String getName() {
		return "Mp3 Audio Adapter";
	}

	public String getDescription() {
		return "Adapts Mp3 Audio files including ID3v1 and ID3v2 tags";
	}

	public void adapt(File file, ParserContext ctx) throws IOException {
		long SECS = 1000;
		long MINS = 60 * SECS;
		long HRS = 60 * MINS;
		// Add the MetaData to the tree!
		DataSource ftk = new FileDataSource(file);
		ctx.fireStartParseEvent("mp3");
		writeFileInfo(file, ctx);
		try {
			boolean headFound = false;
			int sizeOfID3Metadata = 0;
			// read the ID3v2.x tag...
			ftk.setPosition(0);
			String id3 = FXUtil.getFixedStringValue(ftk, 3);
			if (id3.equalsIgnoreCase("ID3")) {
				headFound = true;
				ctx.fireStartParseEvent("header");
				byte[] ver = ftk.getData(1);
				byte[] subVer = ftk.getData(1);
				byte[] fg = ftk.getData(1);
				byte[] s = ftk.getData(4);
				int version = (int) FXUtil.getNumericalValue(ver, true);
				int subVersion = (int) FXUtil.getNumericalValue(subVer, true);
				int flags = (int) FXUtil.getNumericalValue(fg, true);
				int size = (int) FXUtil.getNumericalValue(s, true);
				boolean unsync = BitFieldUtil.isSet(flags, 64);
				boolean ext = BitFieldUtil.isSet(flags, 32);
				boolean exp = BitFieldUtil.isSet(flags, 16);
				boolean foot = BitFieldUtil.isSet(flags, 8);
				size = getSyncsafeInt(size);

				ctx.fireParseEvent("tag-version", "ID3v2." + version + "."
						+ subVersion);

				// if there is an extended header then that needs to be read...
				if (ext) {
					byte[] extS = ftk.getData(4);
					int extSize = (int) FXUtil.getNumericalValue(extS, true);
					extSize = getSyncsafeInt(extSize);
					// move past the crap...
					ftk.setPosition(ftk.getPosition() + extSize - 4);
				}

				// allow for the footer to be present
				if (foot) {
					size -= 10;
				}

				// now read the frames...
				int read = 0;
				while (read < size) {
					String frameType = FXUtil.getFixedStringValue(ftk, 4);
					byte[] fS = ftk.getData(4);
					byte[] frameFlags = ftk.getData(2);
					int frameSize = (int) FXUtil.getNumericalValue(fS, true);
					
					/*
					 * Check whether a sync-safe int conversion to be done on the
					 * frame size. Do the conversion only if the frame type is not
					 * defined in the frameTypesToIgnoreSyncSafeInt list.
					 */
					if (frameTypesToIgnoreSyncSafeInt.contains(frameType) == false)
						frameSize = getSyncsafeInt(frameSize);
					String data = FXUtil.getFixedStringValue(ftk, frameSize);
					ID3v2Tag v2tag = MP3Util.getTag(frameType, data);
					if (v2tag != null) {
						HashMap map = v2tag.getMap();
						if (map != null) {
							ctx.fireParseEvent(v2tag.getName(), v2tag
									.getValue(), false, map);
						} else {
							ctx.fireParseEvent(v2tag.getName(), v2tag
									.getValue());
						}
					}
					read += frameSize + 10;
				}
{'fraction_non_alphanumeric': 0.1723528371928584, 'fraction_numerical': 0.04401626303694538, 'mean_word_length': 3.568506056527591, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7053880', 'n_tokens_mistral': 7358, 'n_tokens_neox': 5874, 'n_words': 1692}