<commit_before><?php

namespace STC;

class Files
{
  private $files;

  private $data_path;

  public function __construct()
  {
    $this->data_path = '/' . Config::site()->get('pages_data');
    $this->files = [];
  }

  private function read_dir($folder = '')
  {
    return array_diff(scandir($folder), array('..', '.'));
  }

  public function load($data_folder = '')
  {
    $files = $this->read_dir($data_folder . $this->data_path);

    $pattern = '/(.+).json$/';

    $data_loader = new DataLoader();

    foreach($files as $file) {
      if (preg_match($pattern, $file, $match)) {
        $f = $data_loader->load(
          $data_folder . $this->data_path,
          $file
        );
        $f['file'] = $file;
        $this->files[] = $f;
      }
    }
  }

  public function get_all()
  {
    return $this->files;
  }

  public function filter_by($fn)
  {
    return array_filter($this->files, $fn);
  }

  public function count()
  {
    return count($this->files);
  }
}<commit_msg>documented Files class.<commit_after><?php

namespace STC;

class Files
{
  private $files;

  private $data_path;

  public function __construct()
  {
    $this->data_path = '/' . Config::site()->get('pages_data');
    $this->files = [];
  }

  /**
   * Read a directory.
   * @param $folder string | The folder name.
   * @return array
   */
  private function read_dir($folder = '')
  {
    return array_diff(scandir($folder), array('..', '.'));
  }

  /**
   * Load files from a given directory.
   * @param $data_folder string | The folder name.
   * @return array
   */
  public function load($data_folder = '')
  {
    $files = $this->read_dir($data_folder . $this->data_path);

    $pattern = '/(.+).json$/';

    $data_loader = new DataLoader();

    foreach($files as $file) {
      if (preg_match($pattern, $file, $match)) {
        $f = $data_loader->load(
          $data_folder . $this->data_path,
          $file
        );
        $f['file'] = $file;
        $this->files[] = $f;
      }
    }
  }

  /**
   * Get all loaded files.
   * @return array
   */
  public function get_all() { return $this->files; }

  /**
   * Execute a predicate and get back a list of files.
   * @param $fn Function | A function for the filter.
   * @return array
   */
  public function filter_by($fn) { return array_filter($this->files, $fn); }

  /**
   * Get the number of loaded files.
   * @return int
   */
  public function count() { return count($this->files); }
}
{'fraction_non_alphanumeric': 0.1746353322528363, 'fraction_numerical': 0.0, 'mean_word_length': 2.4725738396624473, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2710210', 'n_tokens_mistral': 848, 'n_tokens_neox': 822, 'n_words': 189}