<commit_before>#!/usr/bin/env perl

use strict;
use warnings;

use JSON;
use LWP::Simple;
use Data::Dumper;

sub parse_players {
    my $json = shift;
    my @athletes = $json->{sports}[0]{leagues}[0]{athletes};
    map { print lc "  '$_->{displayName}' => $_->{id},\n" } @{$athletes[0]};
    return $json;
}

sub parse_teams {
    my $json = shift;
    if (exists $json->{sports}[0]{leagues}[0]{teams}){
      my @teams = $json->{sports}[0]{leagues}[0]{teams};
      map {
        print lc "  '$_->{location} $_->{name}' => $_->{id},\n";
        print lc "  'the $_->{location} $_->{name}' => $_->{id},\n";
      } @{$teams[0]};
    }
    return $json;
}

sub parse {
    my ($resource, $sub) = @_;
    my %subs = (
        'players' => \&parse_players,
        'teams'   => \&parse_teams,
    );

    my $json = $subs{$sub}(decode_json get $resource);

    my $resultsCount  = $json->{resultsCount};
    my $resultsLimit  = $json->{resultsLimit};
    my $resultsOffset = 0;<commit_msg>rm single quotes from names - temporary solution<commit_after>#!/usr/bin/env perl

use strict;
use warnings;

use JSON;
use LWP::Simple;
use Data::Dumper;

sub parse_players {
    my $json = shift;
    my @athletes = $json->{sports}[0]{leagues}[0]{athletes};
    map {
        my $name = lc $_->{displayName};
        $name =~ s/'//g;
        print "  '$name' => $_->{id},\n"
    } @{$athletes[0]};
    return $json;
}

sub parse_teams {
    my $json = shift;
    if (exists $json->{sports}[0]{leagues}[0]{teams}){
      my @teams = $json->{sports}[0]{leagues}[0]{teams};
      map {
        print lc "  '$_->{location} $_->{name}' => $_->{id},\n";
        print lc "  'the $_->{location} $_->{name}' => $_->{id},\n";
      } @{$teams[0]};
    }
    return $json;
}

sub parse {
    my ($resource, $sub) = @_;
    my %subs = (
        'players' => \&parse_players,
        'teams'   => \&parse_teams,
    );

    my $json = $subs{$sub}(decode_json get $resource);

    my $resultsCount  = $json->{resultsCount};
    my $resultsLimit  = $json->{resultsLimit};
    my $resultsOffset = 0;
{'fraction_non_alphanumeric': 0.2195945945945946, 'fraction_numerical': 0.008687258687258687, 'mean_word_length': 2.735135135135135, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2463698', 'n_tokens_mistral': 808, 'n_tokens_neox': 792, 'n_words': 168}