<commit_before>package DataDownloader::Source::RedFly;

use Moose;
use Web::Scraper;
extends 'DataDownloader::Source::ABC';

use constant {
    TITLE       => 
        'RedFly GFF',
    DESCRIPTION => 
        "Transcriptional CRMs and TFBSs from RedFly",
    SOURCE_LINK => 
        "http://redfly.ccr.buffalo.edu",
    SOURCE_DIR => "flymine/redfly",
    METADATA_URL => "http://redfly.ccr.buffalo.edu/index.php",
};

sub BUILD {
    my $self = shift;
    $self->set_sources([
        {
            SERVER  => 'http://redfly.ccr.buffalo.edu/datadumps',
            FILE    => "tfbs_dump.gff",
            SUB_DIR => ["tfbs"],
        },
        {
            SERVER  => 'http://redfly.ccr.buffalo.edu/datadumps',
            FILE    => "crm_dump.gff",
            SUB_DIR => ["crm"],
        },
    ]);
}

sub generate_version {
    my $self = shift;
    my $scraper = scraper {
        process 'div.lastupdate', version => 'TEXT';
    };
    my $ua = LWP::UserAgent->new(agent => 'Mozilla/5.0');
    my $response = $ua->get(METADATA_URL);
    confess $response->status_line unless $response->is_success;
    my $scraps = $scraper->scrape($response);
    my ($version) = $scraps->{version} =~ /(\d+-\S*-\d{4})/g;
    die "Could not determine RedFly version" unless $version;
    return $version;
}
1;
<commit_msg>ignore redfly version -- there isn't a good way to get it, just use date.<commit_after>package DataDownloader::Source::RedFly;

use Moose;
use Web::Scraper;
extends 'DataDownloader::Source::ABC';

use constant {
    TITLE       => 
        'RedFly GFF',
    DESCRIPTION => 
        "Transcriptional CRMs and TFBSs from RedFly",
    SOURCE_LINK => 
        "http://redfly.ccr.buffalo.edu",
    SOURCE_DIR => "flymine/redfly",
};

sub BUILD {
    my $self = shift;
    $self->set_sources([
        {
            SERVER  => 'http://redfly.ccr.buffalo.edu/datadumps',
            FILE    => "tfbs_dump.gff",
            SUB_DIR => ["tfbs"],
        },
        {
            SERVER  => 'http://redfly.ccr.buffalo.edu/datadumps',
            FILE    => "crm_dump.gff",
            SUB_DIR => ["crm"],
        },
    ]);
}


1;

{'fraction_non_alphanumeric': 0.16151685393258428, 'fraction_numerical': 0.0023408239700374533, 'mean_word_length': 2.1800595238095237, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2764003', 'n_tokens_mistral': 772, 'n_tokens_neox': 738, 'n_words': 141}