<commit_before><?php namespace NpmWeb\PackageUsage\Services;

use NpmWeb\GitApiClient\GitApiClientInterface;
use NpmWeb\ComposerService\GitRepoComposerService;
use NpmWeb\ComposerService\CompositeComposerService;

class PackageUsageService implements PackageUsageServiceInterface {

    protected $git;
    protected $gitComposer;
    protected $compositeComposer;

    public function __construct( GitApiClientInterface $git,
        GitRepoComposerService $gitComposer,
        CompositeComposerService $compositeComposer )
    {
        $this->git = $git;
        $this->gitComposer = $gitComposer;
        $this->compositeComposer = $compositeComposer;
    }

    /**
     * Recalculates all usages with the latest information in the repos.
     */
    public function updateUsage() {
        $owner = 'npmweb';
        $packageUsage = [];

        $repoNames = $this->git->getReposOwnedBy($owner,'php');

        foreach( $repoNames as $repoName ) {
            $composerConfig = $this->gitComposer->getPackageInfo( "$owner/$repoName", array(
                'repo' => "<EMAIL>:$owner/$repoName"
            ));

            // TODO cdn-helper not getting here
            if( isset($composerConfig->type) && 'library' == $composerConfig->type ) {
                echo 'it\'s a library; skipping'."\n";
                continue;
            }

            if(!isset($composerConfig->require)) {
                continue;
            }
            foreach( $composerConfig->require as $packageName => $version ) {
                // if it's php itself, skip it
                if( 'php' == $packageName ) {
                    continue;
                }

                if( !array_key_exists($packageName, $packageUsage) ) {
                    $composer2Config = $this->compositeComposer->getPackageInfo($packageName, array(
                        'repos' => isset($composerConfig->repositories)?$composerConfig->repositories:array(),
                    ));
                    // TODO not getting config for npm ones
                    if( !$composer2Config ) {
                        continue;
                    }
                    $packageUsage[$packageName] = [
                        'name' => $packageName,
                        'description' => $composer2Config->description,
                        'usages' => [],
                    ];
                }
                $packageUsage[$packageName]['usages'][] = $repoName;
            }
        }

        // output
        $dir = storage_path().'/usage/';
        if( !file_exists($dir) ) { mkdir($dir); }
        $fh = fopen( $dir.'usage.json', 'w' );
        fwrite( $fh, json_encode($packageUsage) );
        fclose($fh);
    }

    /**
     * Retrieves the usage results.
     */
    public function getUsage() {
        $dir = storage_path().'/usage/';
        return json_decode(file_get_contents($dir.'usage.json'));
    }

}<commit_msg>sorts results<commit_after><?php namespace NpmWeb\PackageUsage\Services;

use NpmWeb\GitApiClient\GitApiClientInterface;
use NpmWeb\ComposerService\GitRepoComposerService;
use NpmWeb\ComposerService\CompositeComposerService;

class PackageUsageService implements PackageUsageServiceInterface {

    protected $git;
    protected $gitComposer;
    protected $compositeComposer;

    public function __construct( GitApiClientInterface $git,
        GitRepoComposerService $gitComposer,
        CompositeComposerService $compositeComposer )
    {
        $this->git = $git;
        $this->gitComposer = $gitComposer;
        $this->compositeComposer = $compositeComposer;
    }

    /**
     * Recalculates all usages with the latest information in the repos.
     */
    public function updateUsage() {
        $owner = 'npmweb';
        $packageUsage = [];

        $repoNames = $this->git->getReposOwnedBy($owner,'php');

        foreach( $repoNames as $repoName ) {
            $composerConfig = $this->gitComposer->getPackageInfo( "$owner/$repoName", array(
                'repo' => "<EMAIL>:$owner/$repoName"
            ));

            // TODO cdn-helper not getting here
            if( isset($composerConfig->type) && 'library' == $composerConfig->type ) {
                echo 'it\'s a library; skipping'."\n";
                continue;
            }

            if(!isset($composerConfig->require)) {
                continue;
            }
            foreach( $composerConfig->require as $packageName => $version ) {
                // if it's php itself, skip it
                if( 'php' == $packageName ) {
                    continue;
                }

                if( !array_key_exists($packageName, $packageUsage) ) {
                    $composer2Config = $this->compositeComposer->getPackageInfo($packageName, array(
                        'repos' => isset($composerConfig->repositories)?$composerConfig->repositories:array(),
                    ));
                    // TODO not getting config for npm ones
                    if( !$composer2Config ) {
                        continue;
                    }
                    $packageUsage[$packageName] = [
                        'name' => $packageName,
                        'description' => $composer2Config->description,
                        'usages' => [],
                    ];
                }
                $packageUsage[$packageName]['usages'][] = $repoName;
            }
        }

        // sort results
        ksort( $packageUsage, SORT_STRING | SORT_FLAG_CASE );
        foreach( $packageUsage as &$package ) {
            sort($package['usages'], SORT_STRING | SORT_FLAG_CASE); // alphabetical
        }

        // save to disk
        $dir = storage_path().'/usage/';
        if( !file_exists($dir) ) { mkdir($dir); }
        $fh = fopen( $dir.'usage.json', 'w' );
        fwrite( $fh, json_encode($packageUsage) );
        fclose($fh);
    }

    /**
     * Retrieves the usage results.
     */
    public function getUsage() {
        $dir = storage_path().'/usage/';
        return json_decode(file_get_contents($dir.'usage.json'));
    }

}
{'fraction_non_alphanumeric': 0.12382831771090282, 'fraction_numerical': 0.000986679822397632, 'mean_word_length': 1.8144377602961592, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6270205', 'n_tokens_mistral': 1693, 'n_tokens_neox': 1602, 'n_words': 307}