<commit_before>/* This file is part of the KDE project
 * Copyright (C) 2007 <NAME> <<EMAIL>>
 * Copyright (C) 2004 <NAME> <<EMAIL>>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public
 * License as published by the Free Software Foundation version 2.
 *
 * This program is distributed in the hope that
t it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; see the file COPYING.  If not, write to
 * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
 * Boston, MA 02110-1301, USA.
 *
 */

#include "icsendanalyzer.h"

#include <kcal/calendarlocal.h>
#include <kcal/icalformat.h>
#include <kcal/vcalformat.h>
#include <kcal/todo.h>

#include <strigi/fieldtypes.h>
#include <strigi/analysisresult.h>
#include <strigi/streamendanalyzer.h>

//#include <KDebug>

using namespace KCal;

IcsEndAnalyzer::IcsEndAnalyzer( const IcsEndAnalyzerFactory* f )
  : m_factory( f )
{
}

bool IcsEndAnalyzer::checkHeader( const char* header, int32_t headersize ) const
{
  const char* magic = "BEGIN:VCALENDAR";
  int32_t magicLength = strlen( magic );

  return headersize >= magicLength && !strncmp( magic, header, magicLength );
}

/*
I chose to use libkcal instead of reading the calendar manually. It's easier to
maintain this way.
*/
STRIGI_ENDANALYZER_RETVAL IcsEndAnalyzer::analyze( Strigi::AnalysisResult& idx, Strigi::InputStream* in )
{
  CalendarLocal cal( QString::fromLatin1( "UTC" ) );

  const char* data;
  //FIXME: large calendars will exhaust memory; incremental loading would be
  // nice
  int32_t nread = in->read( data, 1, in->size() );
  if ( nread <= 0 ) {
    //kDebug() <<"Reading data from input stream failed";
    return Strigi::Error;
  }

  ICalFormat ical;
  if ( !ical.fromRawString( &cal, QByteArray::fromRawData(data, nread) ) ) {
    VCalFormat vcal;
    if ( !vcal.fromRawString( &cal, data ) ) {
      //kDebug() <<"Could not load calendar";
      return Strigi::Error;
    }
  }

  idx.addValue( m_factory->field( ProductId ), cal.productId().toUtf8().data() );
  idx.addValue( m_factory->field( Events ), (uint32_t)cal.events().count() );
  idx.addValue( m_factory->field( Journals ), (uint32_t)cal.journals().count() );
  Todo::List todos = cal.todos();

  // count completed and overdue
  int completed = 0;
  int overdue = 0;
  foreach ( const Todo* todo, todos ) {
    if ( todo->isCompleted() ) {
      ++completed;
    } else if ( todo->hasDueDate() && todo->dtDue().date() < QDate::currentDate() ) {
      ++overdue;
    }
  }

  idx.addValue( m_factory->field( Todos ), (uint32_t)todos.count() );
  idx.addValue( m_factory->field( TodosCompleted ), (uint32_t)completed );
  idx.addValue( m_factory->field( TodosOverdue ), (uint32_t)overdue );

  cal.close();

  return Strigi::Ok;
}

const Strigi::RegisteredField* IcsEndAnalyzerFactory::field( IcsEndAnalyzer::Field f ) const
{
  switch ( f ) {
    case IcsEndAnalyzer::ProductId:
      return productIdField;
      break;
    case IcsEndAnalyzer::Events:
      return eventsField;
      break;
    case IcsEndAnalyzer::Journals:
      return journalsField;
      break;
    case IcsEndAnalyzer::Todos:
      return todosField;
      break;
    case IcsEndAnalyzer::TodosCompleted:
      return todosCompletedField;
      break;
    case IcsEndAnalyzer::TodosOverdue:
    default:
      return todosOverdueField;
      break;
  }
}

void IcsEndAnalyzerFactory::registerFields( Strigi::FieldRegister& reg ) {
  productIdField = reg.registerField("Product Id", Strigi::FieldRegister::stringType, 1, 0 );
  eventsField = reg.registerField("Events", Strigi::FieldRegister::integerType, 1, 0 );
  journalsField = reg.registerField("Journals", Strigi::FieldRegister::integerType, 1, 0 );
  todosField = reg.registerField("Todos", Strigi::FieldRegister::integerType, 1, 0 );
  todosCompletedField = reg.registerField("Todos Completed", Strigi::FieldRegister::integerType, 1, 0 );
  todosOverdueField = reg.registerField("Todos Overdue", Strigi::FieldRegister::integerType, 1, 0 );
}

<commit_msg>Make it work for ICS files over 1KB.<commit_after>/* This file is part of the KDE project
 * Copyright (C) 2007 <NAME> <<EMAIL>>
 * Copyright (C) 2004 <NAME> <<EMAIL>>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public
 * License as published by the Free Software Foundation version 2.
 *
 * This program is distributed in the hope that
t it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; see the file COPYING.  If not, write to
 * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
 * Boston, MA 02110-1301, USA.
 *
 */

#include "icsendanalyzer.h"

#include <kcal/calendarlocal.h>
#include <kcal/icalformat.h>
#include <kcal/vcalformat.h>
#include <kcal/todo.h>

#include <strigi/fieldtypes.h>
#include <strigi/analysisresult.h>
#include <strigi/streamendanalyzer.h>

//#include <KDebug>

using namespace KCal;

IcsEndAnalyzer::IcsEndAnalyzer( const IcsEndAnalyzerFactory* f )
  : m_factory( f )
{
}

bool IcsEndAnalyzer::checkHeader( const char* header, int32_t headersize ) const
{
  const char* magic = "BEGIN:VCALENDAR";
  int32_t magicLength = strlen( magic );

  return headersize >= magicLength && !strncmp( magic, header, magicLength );
}

/*
I chose to use libkcal instead of reading the calendar manually. It's easier to
maintain this way.
*/
STRIGI_ENDANALYZER_RETVAL IcsEndAnalyzer::analyze( Strigi::AnalysisResult& idx, Strigi::InputStream* in )
{
  CalendarLocal cal( QString::fromLatin1( "UTC" ) );

  const char* data;
  //FIXME: large calendars will exhaust memory; incremental loading would be
  // nice
  int32_t nread = in->read( data, in->size(), in->size() );
  if ( nread <= 0 ) {
    //kDebug() <<"Reading data from input stream failed";
    return Strigi::Error;
  }

  ICalFormat ical;
  if ( !ical.fromRawString( &cal, QByteArray::fromRawData(data, nread) ) ) {
    VCalFormat vcal;
    if ( !vcal.fromRawString( &cal, data ) ) {
      //kDebug() <<"Could not load calendar";
      return Strigi::Error;
    }
  }

  idx.addValue( m_factory->field( ProductId ), cal.productId().toUtf8().data() );
  idx.addValue( m_factory->field( Events ), (uint32_t)cal.events().count() );
  idx.addValue( m_factory->field( Journals ), (uint32_t)cal.journals().count() );
  Todo::List todos = cal.todos();

  // count completed and overdue
  int completed = 0;
  int overdue = 0;
  foreach ( const Todo* todo, todos ) {
    if ( todo->isCompleted() ) {
      ++completed;
    } else if ( todo->hasDueDate() && todo->dtDue().date() < QDate::currentDate() ) {
      ++overdue;
    }
  }

  idx.addValue( m_factory->field( Todos ), (uint32_t)todos.count() );
  idx.addValue( m_factory->field( TodosCompleted ), (uint32_t)completed );
  idx.addValue( m_factory->field( TodosOverdue ), (uint32_t)overdue );

  cal.close();

  return Strigi::Ok;
}

const Strigi::RegisteredField* IcsEndAnalyzerFactory::field( IcsEndAnalyzer::Field f ) const
{
  switch ( f ) {
    case IcsEndAnalyzer::ProductId:
      return productIdField;
      break;
    case IcsEndAnalyzer::Events:
      return eventsField;
      break;
    case IcsEndAnalyzer::Journals:
      return journalsField;
      break;
    case IcsEndAnalyzer::Todos:
      return todosField;
      break;
    case IcsEndAnalyzer::TodosCompleted:
      return todosCompletedField;
      break;
    case IcsEndAnalyzer::TodosOverdue:
    default:
      return todosOverdueField;
      break;
  }
}

void IcsEndAnalyzerFactory::registerFields( Strigi::FieldRegister& reg ) {
  productIdField = reg.registerField("Product Id", Strigi::FieldRegister::stringType, 1, 0 );
  eventsField = reg.registerField("Events", Strigi::FieldRegister::integerType, 1, 0 );
  journalsField = reg.registerField("Journals", Strigi::FieldRegister::integerType, 1, 0 );
  todosField = reg.registerField("Todos", Strigi::FieldRegister::integerType, 1, 0 );
  todosCompletedField = reg.registerField("Todos Completed", Strigi::FieldRegister::integerType, 1, 0 );
  todosOverdueField = reg.registerField("Todos Overdue", Strigi::FieldRegister::integerType, 1, 0 );
}


{'fraction_non_alphanumeric': 0.12479759426324312, 'fraction_numerical': 0.012491325468424705, 'mean_word_length': 4.128706998813761, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 56, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '916966', 'n_tokens_mistral': 2901, 'n_tokens_neox': 2780, 'n_words': 777}