<commit_before>/* This file is part of Strigi Desktop Search
 *
 * Copyright (C) 2006 <NAME> <<EMAIL>>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Library General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public License
 * along with this library; see the file COPYING.LIB.  If not, write to
 * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
 * Boston, MA 02110-1301, USA.
 */
#include "arendanalyzer.h"
#include <strigi/strigiconfig.h>
#include "arinputstream.h"
#include "analysisresult.h"
#include "fieldtypes.h"
#include "subinputstream.h"
using namespace Strigi;

void
ArEndAnalyzerFactory::registerFields(FieldRegister& reg) {
    typeField = reg.typeField;
}

bool
ArEndAnalyzer::checkHeader(const char* header, int32_t headersize) const {
    return ArInputStream::checkHeader(header, headersize);
}
signed char
ArEndAnalyzer::analyze(AnalysisResult& idx, InputStream* in) {
    char result = staticAnalyze(idx, in);
    idx.addValue(factory->typeField, "http://freedesktop.org/standards/xesam/1.0/core#Archive");
    return  result;
}
signed char
ArEndAnalyzer::staticAnalyze(AnalysisResult& idx,
        InputStream* in) {
    if(!in)
        return -1;

    ArInputStream ar(in);
    InputStream *s = ar.nextEntry();
    while (s) {
        idx.indexChild(ar.entryInfo().filename, ar.entryInfo().mtime, s);
        s = ar.nextEntry();
    }
    if (ar.status() == Error) {
        return -1;
//        printf("Error: %s\n", ar.error());
    } else {
//        printf("finished ok\n");
    }
    return 0;
}
<commit_msg>Make ar and deb analyzer more well-behaved.<commit_after>/* This file is part of Strigi Desktop Search
 *
 * Copyright (C) 2006 <NAME> <<EMAIL>>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Library General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public License
 * along with this library; see the file COPYING.LIB.  If not, write to
 * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
 * Boston, MA 02110-1301, USA.
 */
#include "arendanalyzer.h"
#include <strigi/strigiconfig.h>
#include "arinputstream.h"
#include "analysisresult.h"
#include "fieldtypes.h"
#include "subinputstream.h"
#include "analyzerconfiguration.h"
using namespace Strigi;

void
ArEndAnalyzerFactory::registerFields(FieldRegister& reg) {
    typeField = reg.typeField;
}

bool
ArEndAnalyzer::checkHeader(const char* header, int32_t headersize) const {
    return ArInputStream::checkHeader(header, headersize);
}
signed char
ArEndAnalyzer::analyze(AnalysisResult& idx, InputStream* in) {
    char result = staticAnalyze(idx, in);
    idx.addValue(factory->typeField,
        "http://freedesktop.org/standards/xesam/1.0/core#Archive");
    return  result;
}
signed char
ArEndAnalyzer::staticAnalyze(AnalysisResult& idx,
        InputStream* in) {
    if(!in)
        return -1;

    ArInputStream ar(in);
    // if the first two files are called 'debian-binary' and 'control.tar.gz'
    // those are analyzed regardless, since they signal that this file is a 
    // debian archive
    InputStream *s = ar.nextEntry();
    if (s && ar.entryInfo().filename.compare("debian-binary") == 0) {
        idx.indexChild(ar.entryInfo().filename, ar.entryInfo().mtime, s);
        s = ar.nextEntry();
    }
    if (s && ar.entryInfo().filename.compare("control.tar.gz") == 0) {
        idx.indexChild(ar.entryInfo().filename, ar.entryInfo().mtime, s);
        s = ar.nextEntry();
    }
    if (idx.config().indexArchiveContents()) {
        while (s) {
            // check if we're done
            int64_t max = idx.config().maximalStreamReadLength(idx);
            if (max != -1 && in->position() > max) {
                return 0;
            }
            // check if the analysis has been aborted
            if (!idx.config().indexMore()) {
                return 0;
            }
            idx.indexChild(ar.entryInfo().filename, ar.entryInfo().mtime, s);
            s = ar.nextEntry();
        }
    }
    if (ar.status() == Error) {
        return -1;
//        printf("Error: %s\n", ar.error());
    } else {
//        printf("finished ok\n");
    }
    return 0;
}

{'fraction_non_alphanumeric': 0.11589925226288862, 'fraction_numerical': 0.01042896497441952, 'mean_word_length': 3.169811320754717, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1209963', 'n_tokens_mistral': 1539, 'n_tokens_neox': 1479, 'n_words': 501}