<commit_before><!-- Get filenames of files ound in service directory and put it in array -->
<% files = Dir.entries("data/contentful/service").reject {|f| File.directory?(f) || f[0].include?('.')} %>
<!-- Remove file extension from returned array-->
<% files.map!{|x| x.split('.').first} %>

<!-- Collect all types from service folder -->
<% types = [] %>
<% files.each do |type| %>
<%  types << data.contentful.service["#{type}"]['type'] %>
<% end %>

<!-- remove duplicates from types array -->
<% types = types.uniq %>

<% sectors = [] %>
<% files.each do |sector| %>
<% sectors << data.contentful.service["#{sector}"]['sector'] %>
<% end %>

<!-- remove duplicates from sectors array -->
<% sectors = sectors.uniq %>



<div style="height: 30px"></div>
<div class="container">
    <div class="row">
        <% sectors.each do |s| %>
        <div class="col-md-8">
            <h3><%= "#{s}".humanize %> Sector Training</h3>
            <ul>
                <% types.each do |t| %>
                <li>
                    <h2><%= "#{t}" %></h2>
                    <% files.each do |m| %>
                    <% if  data.contentful.service["#{m}"]['type'] == "#{t}" %>
                        <% if data.contentful.service["#{m}"]['description'] %>
                        <ul>
                            <li>
                                <%= data.contentful.service["#{m}"]['name'] %>
                                <ul>
                                    <li>
                                        <%= data.contentful.service["#{m}"]['description'] %>
                                    </li>
                                </ul>
                            </li>
                        </ul>
                        <% else %>
                        <ul>
                            <li><%= data.contentful.service["#{m}"]['name'] %></li>
                        </ul>
                        <% end %>
                    <% end %>
                    <% end %>
                </li>
                <% end %>
            </ul>
        </div>
        <% end %>
    </div>
</div>

<div style="height: 80px"></div><commit_msg>added breaking long descriptions into sentences<commit_after><!-- Get filenames of files ound in service directory and put it in array -->
<% files = Dir.entries("data/contentful/service").reject {|f| File.directory?(f) || f[0].include?('.')} %>
<!-- Remove file extension from returned array-->
<% files.map!{|x| x.split('.').first} %>

<!-- Collect all types from service folder -->
<% types = [] %>
<% files.each do |type| %>
<%  types << data.contentful.service["#{type}"]['type'] %>
<% end %>

<!-- remove duplicates from types array -->
<% types = types.uniq %>

<% sectors = [] %>
<% files.each do |sector| %>
<% sectors << data.contentful.service["#{sector}"]['sector'] %>
<% end %>

<!-- remove duplicates from sectors array -->
<% sectors = sectors.uniq %>



<div style="height: 30px"></div>
<div class="container">
    <div class="row">
        <% sectors.each do |s| %>
        <div class="col-md-8">
            <h3><%= "#{s}".humanize %> Sector Training</h3>
            <ul>
                <% types.each do |t| %>
                <li>
                    <h2><%= "#{t}" %></h2>
                    <% files.each do |m| %>
                    <% if  data.contentful.service["#{m}"]['type'] == "#{t}" %>
                        <% if data.contentful.service["#{m}"]['description'] %>
                            <% if data.contentful.service["#{m}"]['description'].to_s.length >= 350 %>
                                <ul>
                                <% s = data.contentful.service["#{m}"]['description'].to_s %>
                                    <% s.scan(/[^\.!?]+[\.!?]/).map(&:strip).each do |sentence|  %>
                                    <li><%= sentence %></li>
                                    <% end %>
                                </ul>
                            <% else %>
                        <ul>
                            <li>
                                <%= data.contentful.service["#{m}"]['name'] %>
                                <ul>
                                    <li>
                                        <%= data.contentful.service["#{m}"]['description'] %>
                                    </li>
                                </ul>
                            </li>
                        </ul>
                            <%end %>
                        <% else %>
                        <ul>
                            <li><%= data.contentful.service["#{m}"]['name'] %></li>
                        </ul>
                        <% end %>
                    <% end %>
                    <% end %>
                </li>
                <% end %>
            </ul>
        </div>
        <% end %>
    </div>
</div>

<div style="height: 80px"></div>
{'fraction_non_alphanumeric': 0.19561205659216732, 'fraction_numerical': 0.004716013942997745, 'mean_word_length': 1.1357267950963224, 'pattern_counts': {'":': 0, '<': 148, '<?xml version=': 0, '>': 141, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '507790', 'n_tokens_mistral': 1465, 'n_tokens_neox': 1278, 'n_words': 264}