<commit_before>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using N2.Engine;

namespace N2.Persistence.Search
{
	/// <summary>
	/// Extracts text from content.
	/// </summary>
	[Service]
	public class TextExtractor
	{
		ITextExtractor[] extractors;

		public TextExtractor(params ITextExtractor[] extractors)
		{
			this.extractors = extractors;
		}

		public virtual bool IsIndexable(ContentItem item)
		{
			if (item.GetContentType().GetCustomAttributes(false).OfType<IIndexableType>().Any(it => !it.IsIndexable))
				return false;

			return true;
		}

		public virtual IEnumerable<IndexableContent> Extract(ContentItem item)
		{
			return extractors.SelectMany(e => e.Extract(item)).Where(ic => !string.IsNullOrEmpty(ic.TextContent));
		}

		public virtual string Join(IEnumerable<IndexableContent> content)
		{
			StringBuilder sb = new StringBuilder();
			foreach (var c in content)
				sb.AppendLine(c.TextContent);
			return sb.ToString();
		}<commit_msg>Changed text extractor to search for indexable=false attribute on base types<commit_after>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using N2.Engine;

namespace N2.Persistence.Search
{
	/// <summary>
	/// Extracts text from content.
	/// </summary>
	[Service]
	public class TextExtractor
	{
		ITextExtractor[] extractors;

		public TextExtractor(params ITextExtractor[] extractors)
		{
			this.extractors = extractors;
		}

		public virtual bool IsIndexable(ContentItem item)
		{
			if (item.GetContentType().GetCustomAttributes(true).OfType<IIndexableType>().Any(it => !it.IsIndexable))
				return false;

			return true;
		}

		public virtual IEnumerable<IndexableContent> Extract(ContentItem item)
		{
			return extractors.SelectMany(e => e.Extract(item)).Where(ic => !string.IsNullOrEmpty(ic.TextContent));
		}

		public virtual string Join(IEnumerable<IndexableContent> content)
		{
			StringBuilder sb = new StringBuilder();
			foreach (var c in content)
				sb.AppendLine(c.TextContent);
			return sb.ToString();
		}
{'fraction_non_alphanumeric': 0.11534749034749035, 'fraction_numerical': 0.0019305019305019305, 'mean_word_length': 4.469656992084433, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2008590', 'n_tokens_mistral': 713, 'n_tokens_neox': 637, 'n_words': 146}