<commit_before>/* This file is part of Strigi Desktop Search
 *
 * Copyright (C) 2007 <NAME> <<EMAIL>>
 *           (C) 2006 <NAME> <<EMAIL>>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Library General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public License
 * along with this library; see the file COPYING.LIB.  If not, write to
 * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
 * Boston, MA 02110-1301, USA.
 */
#include "mimeeventanalyzer.h"
#include "fieldtypes.h"
#include "textutils.h"
#include "fileinputstream.h"
#include "analysisresult.h"
#include <config.h>

// http://standards.freedesktop.org/shared-mime-info-spec/shared-mime-info-spec-0.12.html

using namespace Strigi;
using namespace std;

class MimeRule {
public:
    uint32_t offset;
    uint32_t range;
    unsigned char* value;
    unsigned char* mask;
    uint16_t length;
    unsigned char indent;
    MimeRule() :range(1), indent(0) {}
    bool matches(const char* data, int32_t length) const;
};
bool
MimeRule::matches(const char* data, int32_t len) const {
    // TODO: handle range
    data += offset;
    len -= offset;
    bool match = false;
    for (uint32_t o = offset; !match && o <= range; ++o) {
        if (len < (int32_t)length) {
            return false;
        }
        if (mask) {
            match = true;
            for (uint16_t i = 0; match && i<length; ++i) {
                match = match && (data[i] & mask[i]) == value[i];
            }
        } else {
            match = memcmp(data, value, length) == 0;
        }
        data++;
        len--;
    }
    return match;
}
class Mime {
public:
    string mimetype;
    vector<MimeRule> rules;
    int32_t priority;

    bool matches(const char* data, int32_t length) const;
};
bool
Mime::matches(const char* data, int32_t length) const {
    vector<MimeRule>::const_iterator i;
    bool match = false;
    for (i = rules.begin(); i < rules.end(); ++i) {
        if (i->indent == 0) {
            if (match) return true;
            match = true;
        }
        match = match && i->matches(data, length);
    }
    return match;
}
class MimeEventAnalyzer::Private {
public:
    bool parsed;
    vector<Mime> mimes;
    AnalysisResult* analysisResult;
    const MimeEventAnalyzerFactory* const factory;

    Private(const MimeEventAnalyzerFactory* f)
        :parsed(false), factory(f) {}
    ~Private();

    void parseFile(const string&);
    void parseFiles();
};
MimeEventAnalyzer::Private::~Private() {
    vector<Mime>::const_iterator i;
    vector<MimeRule>::const_iterator j;
    for (i=mimes.begin(); i!=mimes.end(); ++i) {
        for (j=i->rules.begin(); j!=i->rules.end(); ++j) {
            free(j->value);
            if (j->mask) free(j->mask);
        }
    }
}
void
MimeEventAnalyzer::Private::parseFiles() {
    vector<string> files;
    files.push_back("/usr/share/mime/magic");
    //When we install kde into a patch different from /usr
    files.push_back(MIMEINSTALLDIR "/magic");

    vector<string>::const_iterator i;
    for (i = files.begin(); i< files.end(); ++i) {
        parseFile(*i);
    }
}

#ifndef __BIG_ENDIAN__
// endianness conversion is copied from kde's kmimetypefactory.cpp
#define MAKE_LITTLE_ENDIAN16(val) val = (uint16_t)(((uint16_t)(val) << 8)|((uint16_t)(val) >> 8))
#define MAKE_LITTLE_ENDIAN32(val) \
   val = (((uint32_t)(val) & 0xFF000000U) >> 24) | \
         (((uint32_t)(val) & 0x00FF0000U) >> 8) | \
         (((uint32_t)(val) & 0x0000FF00U) << 8) | \
         (((uint32_t)(val) & 0x000000FFU) << 24)
void
makeLittleEndian16(void* data, uint32_t len) {
    for (uint32_t i = 0; i < len; i += 2) {
       MAKE_LITTLE_ENDIAN16(*((uint16_t*)data+i));
    }
}
void
makeLittleEndian32(void* data, uint32_t len) {
    for (uint32_t i = 0; i < len; i += 4) {
       MAKE_LITTLE_ENDIAN32(*((uint32_t*)data+i));
    }
}
#endif

void
MimeEventAnalyzer::Private::parseFile(const string& file) {
    FileInputStream f(file.c_str());
    const char* data;
    int32_t nread = f.read(data, 12, 12);
    if (nread != 12 || memcmp(data, "MIME-Magic\0\n", 12) != 0) {
        // cannot read this magic file
        fprintf(stderr, "'%s' is not a valid magic file.\n", file.c_str());
        return;
    }
    while (true) {
        // find \n
        int64_t startpos = f.position();
        nread = f.read(data, 10000, 0);
        const char* end = data+nread;
        const char* pos = data;
        while (pos < end && *pos != '\n') pos++;
        if (pos >= end) {<commit_msg>do not print an error message if the mime file does not exist<commit_after>/* This file is part of Strigi Desktop Search
 *
 * Copyright (C) 2007 <NAME> <<EMAIL>>
 *           (C) 2006 <NAME> <<EMAIL>>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Library General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public License
 * along with this library; see the file COPYING.LIB.  If not, write to
 * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
 * Boston, MA 02110-1301, USA.
 */
#include "mimeeventanalyzer.h"
#include "fieldtypes.h"
#include "textutils.h"
#include "fileinputstream.h"
#include "analysisresult.h"
#include <config.h>

// http://standards.freedesktop.org/shared-mime-info-spec/shared-mime-info-spec-0.12.html

using namespace Strigi;
using namespace std;

class MimeRule {
public:
    uint32_t offset;
    uint32_t range;
    unsigned char* value;
    unsigned char* mask;
    uint16_t length;
    unsigned char indent;
    MimeRule() :range(1), indent(0) {}
    bool matches(const char* data, int32_t length) const;
};
bool
MimeRule::matches(const char* data, int32_t len) const {
    // TODO: handle range
    data += offset;
    len -= offset;
    bool match = false;
    for (uint32_t o = offset; !match && o <= range; ++o) {
        if (len < (int32_t)length) {
            return false;
        }
        if (mask) {
            match = true;
            for (uint16_t i = 0; match && i<length; ++i) {
                match = match && (data[i] & mask[i]) == value[i];
            }
        } else {
            match = memcmp(data, value, length) == 0;
        }
        data++;
        len--;
    }
    return match;
}
class Mime {
public:
    string mimetype;
    vector<MimeRule> rules;
    int32_t priority;

    bool matches(const char* data, int32_t length) const;
};
bool
Mime::matches(const char* data, int32_t length) const {
    vector<MimeRule>::const_iterator i;
    bool match = false;
    for (i = rules.begin(); i < rules.end(); ++i) {
        if (i->indent == 0) {
            if (match) return true;
            match = true;
        }
        match = match && i->matches(data, length);
    }
    return match;
}
class MimeEventAnalyzer::Private {
public:
    bool parsed;
    vector<Mime> mimes;
    AnalysisResult* analysisResult;
    const MimeEventAnalyzerFactory* const factory;

    Private(const MimeEventAnalyzerFactory* f)
        :parsed(false), factory(f) {}
    ~Private();

    void parseFile(const string&);
    void parseFiles();
};
MimeEventAnalyzer::Private::~Private() {
    vector<Mime>::const_iterator i;
    vector<MimeRule>::const_iterator j;
    for (i=mimes.begin(); i!=mimes.end(); ++i) {
        for (j=i->rules.begin(); j!=i->rules.end(); ++j) {
            free(j->value);
            if (j->mask) free(j->mask);
        }
    }
}
void
MimeEventAnalyzer::Private::parseFiles() {
    vector<string> files;
    files.push_back("/usr/share/mime/magic");
    //When we install kde into a patch different from /usr
    files.push_back(MIMEINSTALLDIR "/magic");

    vector<string>::const_iterator i;
    for (i = files.begin(); i< files.end(); ++i) {
        parseFile(*i);
    }
}

#ifndef __BIG_ENDIAN__
// endianness conversion is copied from kde's kmimetypefactory.cpp
#define MAKE_LITTLE_ENDIAN16(val) val = (uint16_t)(((uint16_t)(val) << 8)|((uint16_t)(val) >> 8))
#define MAKE_LITTLE_ENDIAN32(val) \
   val = (((uint32_t)(val) & 0xFF000000U) >> 24) | \
         (((uint32_t)(val) & 0x00FF0000U) >> 8) | \
         (((uint32_t)(val) & 0x0000FF00U) << 8) | \
         (((uint32_t)(val) & 0x000000FFU) << 24)
void
makeLittleEndian16(void* data, uint32_t len) {
    for (uint32_t i = 0; i < len; i += 2) {
       MAKE_LITTLE_ENDIAN16(*((uint16_t*)data+i));
    }
}
void
makeLittleEndian32(void* data, uint32_t len) {
    for (uint32_t i = 0; i < len; i += 4) {
       MAKE_LITTLE_ENDIAN32(*((uint32_t*)data+i));
    }
}
#endif

void
MimeEventAnalyzer::Private::parseFile(const string& file) {
    FileInputStream f(file.c_str());
    const char* data;
    int32_t nread = f.read(data, 12, 12);
    if (nread <= 0) return; // file does not exist or contains no data
    if (nread != 12 || memcmp(data, "MIME-Magic\0\n", 12) != 0) {
        // cannot read this magic file
        fprintf(stderr, "'%s' is not a valid magic file.\n", file.c_str());
        return;
    }
    while (true) {
        // find \n
        int64_t startpos = f.position();
        nread = f.read(data, 10000, 0);
        const char* end = data+nread;
        const char* pos = data;
        while (pos < end && *pos != '\n') pos++;
        if (pos >= end) {
{'fraction_non_alphanumeric': 0.13962488835963086, 'fraction_numerical': 0.02947305745757666, 'mean_word_length': 2.7929996236356796, 'pattern_counts': {'":': 0, '<': 60, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5513467', 'n_tokens_mistral': 3587, 'n_tokens_neox': 3317, 'n_words': 986}