<commit_before>#pragma once

#include <cstdint>
#include <utility>
#include <vector>
#include <string>

#include "sequence/Common.hpp"
#include "sequence/details/StringView.hpp"

namespace sequence {

// An Item represents a sequence entry.
// It can be of the following types :
// - INVALID : the item is not in a valid state.
// - SINGLE  : it's either a single file or directory.
// - INDICED : it's a sequence with a set of numbers attached to it.
// - PACKED  : it's a contiguous sequence going from start to end inclusive.
struct Item {
  enum : char { PADDING_CHAR = '#' };
  enum Type { INVALID, SINGLE, INDICED, PACKED };

  std::string filename;
  Index start = -1, end = -1;
  char padding = -1, step = -1;<commit_msg>Remove supplementary PADDING_CHAR definition<commit_after>#pragma once

#include <cstdint>
#include <utility>
#include <vector>
#include <string>

#include "sequence/Common.hpp"
#include "sequence/details/StringView.hpp"

namespace sequence {

// An Item represents a sequence entry.
// It can be of the following types :
// - INVALID : the item is not in a valid state.
// - SINGLE  : it's either a single file or directory.
// - INDICED : it's a sequence with a set of numbers attached to it.
// - PACKED  : it's a contiguous sequence going from start to end inclusive.
struct Item {
  enum Type { INVALID, SINGLE, INDICED, PACKED };

  std::string filename;
  Index start = -1, end = -1;
  char padding = -1, step = -1;
{'fraction_non_alphanumeric': 0.11440385940730531, 'fraction_numerical': 0.005513439007580979, 'mean_word_length': 3.4539877300613497, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5492411', 'n_tokens_mistral': 474, 'n_tokens_neox': 455, 'n_words': 183}