<commit_before>/*
jMimeMagic(TM) is a Java library for determining the content type of files or
streams.

Copyright (C) 2004 <NAME>

This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.

This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
Lesser General Public License for more details.

You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

For more information, please email <EMAIL>
*/
package net.sf.jmimemagic;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.oro.text.perl.Perl5Util;

import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;

import java.nio.ByteBuffer;
import java.nio.ByteOrder;

import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;


/**
 * This class represents a single match test
 *
 * @author $Author: arimus $
 * @version $Revision: 1.1 $
 */
public class MagicMatcher implements Cloneable
{
    private static Log log = LogFactory.getLog(MagicMatcher.class);
    private ArrayList subMatchers = new ArrayList(0);
    private MagicMatch match = null;

    /** 
     * constructor 
     */
    public MagicMatcher()
    {
        log.debug("instantiated");
    }

    /**
     * DOCUMENT ME!
     *
     * @param match DOCUMENT ME!
     */
    public void setMatch(MagicMatch match)
    {
        log.debug("setMatch()");
        this.match = match;
    }

    /**
     * DOCUMENT ME!
     *
     * @return DOCUMENT ME!
     */
    public MagicMatch getMatch()
    {
        log.debug("getMatch()");

        return this.match;
    }

    /**
     * test to see if everything is in order for this match
     *
     * @return whether or not this match has enough data to be valid
     */
    public boolean isValid()
    {
        log.debug("isValid()");

        if ((match == null) || (match.getTest() == null)) {
            return false;
        }

        String type = new String(match.getTest().array());
        char comparator = match.getComparator();
        String description = match.getDescription();
        String test = new String(match.getTest().array());

        if ((type != null) && !type.equals("") && (comparator != '\0') &&
                ((comparator == '=') || (comparator == '!') || (comparator == '>') ||
                (comparator == '<')) && (description != null) && !description.equals("") &&
                (test != null) && !test.equals("")) {
            return true;
        }

        return false;
    }

    /**
     * add a submatch to this magic match
     *
     * @param m a magic match
     */
    public void addSubMatcher(MagicMatcher m)
    {
        log.debug("addSubMatcher()");
        subMatchers.add(m);
    }

    /**
     * set all submatches
     *
     * @param a a collection of submatches
     */
    public void setSubMatchers(Collection a)
    {
        log.debug("setSubMatchers(): for match '" + match.getDescription() + "'");
        subMatchers.clear();
        subMatchers.addAll(a);
    }

    /**
     * get all submatches for this magic match
     *
     * @return a collection of submatches
     */
    public Collection getSubMatchers()
    {
        log.debug("getSubMatchers()");

        return subMatchers;
    }

    /**
     * test to see if this match or any submatches match
     *
     * @param f the file that should be used to test the match
     * @param onlyMimeMatch DOCUMENT ME!
     *
     * @return the deepest magic match object that matched
     *
     * @throws IOException DOCUMENT ME!
     * @throws UnsupportedTypeException DOCUMENT ME!
     */
    public MagicMatch test(File f, boolean onlyMimeMatch)
        throws IOException, UnsupportedTypeException
    {
        log.debug("test(File)");

        int offset = match.getOffset();
        String description = match.getDescription();
        String type = match.getType();
        String mimeType = match.getMimeType();

        log.debug("test(File): testing '" + f.getName() + "' for '" + description + "'");

        log.debug("test(File): \n=== BEGIN MATCH INFO ==");
        log.debug(match.print());
        log.debug("test(File): \n=== END MATCH INFO ====\n");

        RandomAccessFile file = null;
        file = new RandomAccessFile(f, "r");

        try {
            int length = 0;

            if (type.equals("byte")) {
                length = 1;
            } else if (type.equals("short") || type.equals("leshort") || type.equals("beshort")) {
                length = 4;
            } else if (type.equals("long") || type.equals("lelong") || type.equals("belong")) {
                length = 8;
            } else if (type.equals("string")) {
                length = match.getTest().capacity();
            } else if (type.equals("regex")) {
                length = (int) file.length() - offset;

                if (length < 0) {
                    length = 0;
                }
            } else if (type.equals("detector")) {
                length = (int) file.length() - offset;

                if (length < 0) {
                    length = 0;
                }
            } else {
                throw new UnsupportedTypeException("unsupported test type '" + type + "'");
            }

            // we know this match won't work since there isn't enough data for the test
            if (length > (file.length() - offset)) {
                return null;
            }

            byte[] buf = new byte[length];
            file.seek(offset);

            int bytesRead = 0;
            int size = 0;<commit_msg>Fix that a regular expression can be configured with a lenght. I started adding matching movie files (mp4) etc and those are huge, so I dont want them in the memory :)<commit_after>/*
jMimeMagic(TM) is a Java library for determining the content type of files or
streams.

Copyright (C) 2004 <NAME>

This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.

This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
Lesser General Public License for more details.

You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

For more information, please email <EMAIL>
*/
package net.sf.jmimemagic;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.oro.text.perl.Perl5Util;

import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;

import java.nio.ByteBuffer;
import java.nio.ByteOrder;

import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;


/**
 * This class represents a single match test
 *
 * @author $Author: arimus $
 * @version $Revision: 1.1 $
 */
public class MagicMatcher implements Cloneable
{
    private static Log log = LogFactory.getLog(MagicMatcher.class);
    private ArrayList subMatchers = new ArrayList(0);
    private MagicMatch match = null;

    /** 
     * constructor 
     */
    public MagicMatcher()
    {
        log.debug("instantiated");
    }

    /**
     * DOCUMENT ME!
     *
     * @param match DOCUMENT ME!
     */
    public void setMatch(MagicMatch match)
    {
        log.debug("setMatch()");
        this.match = match;
    }

    /**
     * DOCUMENT ME!
     *
     * @return DOCUMENT ME!
     */
    public MagicMatch getMatch()
    {
        log.debug("getMatch()");

        return this.match;
    }

    /**
     * test to see if everything is in order for this match
     *
     * @return whether or not this match has enough data to be valid
     */
    public boolean isValid()
    {
        log.debug("isValid()");

        if ((match == null) || (match.getTest() == null)) {
            return false;
        }

        String type = new String(match.getTest().array());
        char comparator = match.getComparator();
        String description = match.getDescription();
        String test = new String(match.getTest().array());

        if ((type != null) && !type.equals("") && (comparator != '\0') &&
                ((comparator == '=') || (comparator == '!') || (comparator == '>') ||
                (comparator == '<')) && (description != null) && !description.equals("") &&
                (test != null) && !test.equals("")) {
            return true;
        }

        return false;
    }

    /**
     * add a submatch to this magic match
     *
     * @param m a magic match
     */
    public void addSubMatcher(MagicMatcher m)
    {
        log.debug("addSubMatcher()");
        subMatchers.add(m);
    }

    /**
     * set all submatches
     *
     * @param a a collection of submatches
     */
    public void setSubMatchers(Collection a)
    {
        log.debug("setSubMatchers(): for match '" + match.getDescription() + "'");
        subMatchers.clear();
        subMatchers.addAll(a);
    }

    /**
     * get all submatches for this magic match
     *
     * @return a collection of submatches
     */
    public Collection getSubMatchers()
    {
        log.debug("getSubMatchers()");

        return subMatchers;
    }

    /**
     * test to see if this match or any submatches match
     *
     * @param f the file that should be used to test the match
     * @param onlyMimeMatch DOCUMENT ME!
     *
     * @return the deepest magic match object that matched
     *
     * @throws IOException DOCUMENT ME!
     * @throws UnsupportedTypeException DOCUMENT ME!
     */
    public MagicMatch test(File f, boolean onlyMimeMatch)
        throws IOException, UnsupportedTypeException
    {
        log.debug("test(File)");

        int offset = match.getOffset();
        String description = match.getDescription();
        String type = match.getType();
        String mimeType = match.getMimeType();

        log.debug("test(File): testing '" + f.getName() + "' for '" + description + "'");

        log.debug("test(File): \n=== BEGIN MATCH INFO ==");
        log.debug(match.print());
        log.debug("test(File): \n=== END MATCH INFO ====\n");

        RandomAccessFile file = null;
        file = new RandomAccessFile(f, "r");

        try {
            int length = 0;

            if (type.equals("byte")) {
                length = 1;
            } else if (type.equals("short") || type.equals("leshort") || type.equals("beshort")) {
                length = 4;
            } else if (type.equals("long") || type.equals("lelong") || type.equals("belong")) {
                length = 8;
            } else if (type.equals("string")) {
                length = match.getTest().capacity();
            } else if (type.equals("regex")) {
                
                
                final int matchLength = match.getLength();
                length = (matchLength == 0) ? (int) file.length() - offset : matchLength;

                if (length < 0) {
                    length = 0;
                }
            } else if (type.equals("detector")) {
                length = (int) file.length() - offset;

                if (length < 0) {
                    length = 0;
                }
            } else {
                throw new UnsupportedTypeException("unsupported test type '" + type + "'");
            }

            // we know this match won't work since there isn't enough data for the test
            if (length > (file.length() - offset)) {
                return null;
            }

            byte[] buf = new byte[length];
            file.seek(offset);

            int bytesRead = 0;
            int size = 0;
{'fraction_non_alphanumeric': 0.11922515804830605, 'fraction_numerical': 0.005835629761711785, 'mean_word_length': 2.079361118043424, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4885808', 'n_tokens_mistral': 3507, 'n_tokens_neox': 3336, 'n_words': 1087}