<commit_before><?php

namespace Gizmo;

use Symfony\Component\Finder\Finder;

class Cache
{
    protected
        $store = array();
    
    public function addFolder($dir)
    {
        $dir = rtrim($dir, '/');
        $it = Finder::create()
          ->depth(0)
          ->in($dir);
        
        foreach ($it as $item) {
            $filename = $item->getBasename();
            if (!$item->isReadable() || '.' === $filename{0}) {
                continue;
            }
            if ($item->isDir()) {
                $this->addFolder((string) $item);
            }
            $this->store[$dir][$item->getBasename()] = array(
                'path' => (string) $item,
                'is_folder' => (int) $item->isDir(),
                'mtime' => $item->getMTime(),
            );
        }
    }

    public function getFilesOrFolders($dir, $regex = null, $with_files = true, $with_folders = true)
    {
        $dir = rtrim($dir, '/');
        if (!isset($this->store[$dir])) {
            return array();
        }
        $items = array();
        foreach ($this->store[$dir] as $name => $item) {
            if (!$regex || preg_match($regex, $name)) {
                if ($item['is_folder']) {
                    if ($with_folders) $items[] = $item['path'];
                } else {
                    if ($with_files) $items[] = $item['path'];
                }
            }
        }
        natsort($items);
        return $items;
    }

    public function getFiles($dir, $regex = null)
    {
        return $this->getFilesOrFolders($dir, $regex, true, false);
    }
    
    public function getFolders($dir, $regex = null)
    {
        return $this->getFilesOrFolders($dir, $regex, false, true);
    }<commit_msg>[Cache] Properly resets sorted array keys<commit_after><?php

namespace Gizmo;

use Symfony\Component\Finder\Finder;

class Cache
{
    protected
        $store = array();
    
    public function addFolder($dir)
    {
        $dir = rtrim($dir, '/');
        $it = Finder::create()
          ->depth(0)
          ->in($dir);
        
        foreach ($it as $item) {
            $filename = $item->getBasename();
            if (!$item->isReadable() || '.' === $filename{0}) {
                continue;
            }
            if ($item->isDir()) {
                $this->addFolder((string) $item);
            }
            $this->store[$dir][$item->getBasename()] = array(
                'path' => (string) $item,
                'is_folder' => (int) $item->isDir(),
                'mtime' => $item->getMTime(),
            );
        }
    }

    public function getFilesOrFolders($dir, $regex = null, $with_files = true, $with_folders = true)
    {
        $dir = rtrim($dir, '/');
        if (!isset($this->store[$dir])) {
            return array();
        }
        $items = array();
        foreach ($this->store[$dir] as $name => $item) {
            if (!$regex || preg_match($regex, $name)) {
                if ($item['is_folder']) {
                    if ($with_folders) $items[] = $item['path'];
                } else {
                    if ($with_files) $items[] = $item['path'];
                }
            }
        }
        natcasesort($items);
        $items = array_values($items);
        return $items;
    }

    public function getFiles($dir, $regex = null)
    {
        return $this->getFilesOrFolders($dir, $regex, true, false);
    }
    
    public function getFolders($dir, $regex = null)
    {
        return $this->getFilesOrFolders($dir, $regex, false, true);
    }
{'fraction_non_alphanumeric': 0.17178259645170374, 'fraction_numerical': 0.001126443255421008, 'mean_word_length': 1.5945945945945945, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1743243', 'n_tokens_mistral': 1080, 'n_tokens_neox': 1063, 'n_words': 213}