<commit_before>﻿using System;
using System.Collections.Generic;
using Spinvoice.Domain.Pdf;

namespace Spinvoice.Infrastructure.Pdf
{
    internal class BricksToSentensesTranslator
    {
        private readonly double _xDelta;
        private readonly double _yDelta;
        private readonly bool _addSpaces;

        public BricksToSentensesTranslator(double xDelta, double yDelta, bool addSpaces)
        {
            _xDelta = xDelta;
            _yDelta = yDelta;
            _addSpaces = addSpaces;
        }
        
        public List<List<SentenceModel>> Translate(IEnumerable<IBrick[]> brickLists)
        {
            var blockSentences = new List<List<SentenceModel>>();
            foreach (var brickList in brickLists)
            {
                var sentences = new List<SentenceModel>();
                var builder = new SentenceModelBuilder(_addSpaces);

                for (var i = 0; i < brickList.Length; i++)
                {
                    var brick = brickList[i];
                    if (builder.IsEmpty)
                    {
                        Append(builder, brick);
                    }
                    else
                    {
                        var prevBrick = brickList[i - 1];
                        if (RoughEqual(brick.Y, prevBrick.Y, _yDelta)
                            && RoughEqual(prevBrick.X + prevBrick.Width, brick.X, _xDelta))
                        {
                            Append(builder, brick);
                        }
                        else
                        {
                            sentences.Add(builder.Build());
                            builder = new SentenceModelBuilder(_addSpaces);
                            Append(builder, brick);
                        }
                    }
                    if (i == brickList.Length - 1)
                    {
                        sentences.Add(builder.Build());
                    }
                }
                blockSentences.Add(sentences);
            }
            return blockSentences;
        }

        private static void Append(SentenceModelBuilder builder, IBrick brick)
        {
            builder.Append(
                brick.Text,
                brick.X,
                brick.Y,
                brick.Width,
                brick.Height);
        }

        private static bool RoughEqual(double currentRectY, double prevRectX, double delta)
        {
            return Math.Abs(currentRectY - prevRectX) < delta;
        }
    }
}<commit_msg>Parse PDF files with images #7 - Ignoring empty sentences<commit_after>﻿using System;
using System.Collections.Generic;
using System.Linq;
using Spinvoice.Domain.Pdf;

namespace Spinvoice.Infrastructure.Pdf
{
    internal class BricksToSentensesTranslator
    {
        private readonly double _xDelta;
        private readonly double _yDelta;
        private readonly bool _addSpaces;

        public BricksToSentensesTranslator(double xDelta, double yDelta, bool addSpaces)
        {
            _xDelta = xDelta;
            _yDelta = yDelta;
            _addSpaces = addSpaces;
        }
        
        public List<List<SentenceModel>> Translate(IEnumerable<IBrick[]> brickLists)
        {
            var blockSentences = new List<List<SentenceModel>>();
            foreach (var brickList in brickLists)
            {
                var sentences = new List<SentenceModel>();
                var builder = new SentenceModelBuilder(_addSpaces);

                for (var i = 0; i < brickList.Length; i++)
                {
                    var brick = brickList[i];
                    if (builder.IsEmpty)
                    {
                        Append(builder, brick);
                    }
                    else
                    {
                        var prevBrick = brickList[i - 1];
                        if (RoughEqual(brick.Y, prevBrick.Y, _yDelta)
                            && RoughEqual(prevBrick.X + prevBrick.Width, brick.X, _xDelta))
                        {
                            Append(builder, brick);
                        }
                        else
                        {
                            sentences.Add(builder.Build());
                            builder = new SentenceModelBuilder(_addSpaces);
                            Append(builder, brick);
                        }
                    }
                    if (i == brickList.Length - 1)
                    {
                        sentences.Add(builder.Build());
                    }
                }
                if (sentences.Any())
                {
                    blockSentences.Add(sentences);
                }
            }
            return blockSentences;
        }

        private static void Append(SentenceModelBuilder builder, IBrick brick)
        {
            builder.Append(
                brick.Text,
                brick.X,
                brick.Y,
                brick.Width,
                brick.Height);
        }

        private static bool RoughEqual(double currentRectY, double prevRectX, double delta)
        {
            return Math.Abs(currentRectY - prevRectX) < delta;
        }
    }
}
{'fraction_non_alphanumeric': 0.07433730311179408, 'fraction_numerical': 0.0013446023818670765, 'mean_word_length': 1.1123732251521299, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2558283', 'n_tokens_mistral': 1296, 'n_tokens_neox': 1216, 'n_words': 266}