<commit_before><?php

namespace Asoc\Dadatata;

use Asoc\Dadatata\Metadata\ExaminerInterface;
use Asoc\Dadatata\Model\FilePathFragments;
use Asoc\Dadatata\Model\ModelProviderInterface;
use Asoc\Dadatata\Model\ThingInterface;

abstract class AbstractLibrary implements LibraryInterface {

    public function identify($data, ThingInterface $thing = null) {
        if($data === null || !isset($data)) {
            throw new \InvalidArgumentException('Cannot identify empty data');
        }

        $examiner = $this->getIdentifier();

        // by default, there's one file fragment
        $fragments = 1;

        // if there are multiple fragments, use the first for metadata. all others are assumed to be of the same type.
        if($data instanceof FilePathFragments) {
            // first fragment is responsible for determining all the infos
            $file = $data->getFileInfos()[0];
            $path = $file->getPathname();
            list($category, $mime) = $examiner->categorize($path);
            $fragments = $data->getNum();
        }
        else if($data instanceof \SplFileInfo) {
            $path = $data->getPathname();
            list($category, $mime) = $examiner->categorize($path);
        }
        // check if a blob or something different has been passed here
        else {
            // not a file, assuming arbitrary data. we need to store the data in a temporary file so all the tools
            // can operate on it
            if(is_string($data) && ctype_print($data) === true && !is_file($data)) {
                $path = tempnam(sys_get_temp_dir(), 'dadatata');
                file_put_contents($path, $data);
            }
            else if(is_file($data)) {
                $path = realpath($data);
            }
            else {
                throw new \InvalidArgumentException(sprintf('Cannot itentify data, unsupported type: %s', gettype($data)));
            }

            if($path !== null && file_exists($path)) {
                list($category, $mime) = $examiner->categorize($data);
            }
            else {
                throw new \InvalidArgumentException(sprintf('Cannot identify data, file does not exist: %s', gettype($data)));
            }
        }

        $modelProvider = $this->getModelProvider();

        if($thing === null) {<commit_msg>don't support arbitrary data in identify because it is not supported anywhere else. this is the responsibility of another layer.<commit_after><?php

namespace Asoc\Dadatata;

use Asoc\Dadatata\Exception\FileNotFoundException;
use Asoc\Dadatata\Metadata\ExaminerInterface;
use Asoc\Dadatata\Model\FilePathFragments;
use Asoc\Dadatata\Model\ModelProviderInterface;
use Asoc\Dadatata\Model\ThingInterface;

abstract class AbstractLibrary implements LibraryInterface {

    public function identify($data, ThingInterface $thing = null) {
        if($data === null || !isset($data)) {
            throw new \InvalidArgumentException('Cannot identify empty data');
        }

        $examiner = $this->getIdentifier();

        // by default, there's one file fragment
        $fragments = 1;

        // if there are multiple fragments, use the first for metadata. all others are assumed to be of the same type.
        if($data instanceof FilePathFragments) {
            // first fragment is responsible for determining all the infos
            $file = $data->getFileInfos()[0];
            $path = $file->getPathname();
            list($category, $mime) = $examiner->categorize($path);
            $fragments = $data->getNum();
        }
        else if($data instanceof \SplFileInfo) {
            $path = $data->getPathname();
            list($category, $mime) = $examiner->categorize($path);
        }
        else if(is_string($data) && ctype_print($data) === true) {
            $path = $data;
            list($category, $mime) = $examiner->categorize($data);
        }
        else {
            throw new FileNotFoundException(sprintf('Cannot itentify data, unsupported type: %s', gettype($data)));
        }

        $modelProvider = $this->getModelProvider();

        if($thing === null) {
{'fraction_non_alphanumeric': 0.11925495887760039, 'fraction_numerical': 0.0009675858732462506, 'mean_word_length': 2.4230132450331126, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2749060', 'n_tokens_mistral': 1141, 'n_tokens_neox': 1099, 'n_words': 307}