<commit_before>#!/usr/bin/env perl

use warnings;
use strict;


package Gwybodaeth::Read;

# Methods for reading in data from either a file or URL;
use LWP::UserAgent;
use HTTP::Request;
use Carp qw(croak);

=head1 NAME

Read - input data reader class for gwybodaeth

=head1 SYNOPSIS

    use Read;

    my $r = Read->new();

    $r->get_file_data("/home/foo/bar.csv");
    $r->get_url_data("www.example.org/bar.csv");

    $r->get_input_data();


=head1 DESCRIPTION

This module imports data from the URIs given to it.

=over

=item new()

Create a new instance of Read.

$r = Read->new();

=cut

sub new {
    my $class = shift;
    my $self = {'Data' => [] };
    bless $self, $class;
    return $self;
}

=item get_file_data($filename)

This function gets data from $filename.

=cut

# Open a file and store its contents
# Returns length of file data
sub get_file_data {
    ref(my $self = shift) or croak "instance variable needed";
    my $file = shift;

    my $fh;    

    if (fileno($file)) { # Check if its a file handle
        $fh = $file;
    } else {             # If it's just a file open a file handle
        # Return if file doesn't exist<commit_msg>Fixed comment<commit_after>#!/usr/bin/env perl

use warnings;
use strict;


package Gwybodaeth::Read;

# Methods for reading in data from either a file or URL;
use LWP::UserAgent;
use HTTP::Request;
use Carp qw(croak);

=head1 NAME

Read - input data reader class for gwybodaeth

=head1 SYNOPSIS

    use Read;

    my $r = Read->new();

    $r->get_file_data("/home/foo/bar.csv");
    $r->get_url_data("www.example.org/bar.csv");

    $r->get_input_data();


=head1 DESCRIPTION

This module imports data from the URIs given to it.

=over

=item new()

Create a new instance of Read.

$r = Read->new();

=cut

sub new {
    my $class = shift;
    my $self = {'Data' => [] };
    bless $self, $class;
    return $self;
}

=item get_file_data($filename)

This function gets data from $filename.

=cut

# Open a file and store its contents
# Returns length of file data
sub get_file_data {
    ref(my $self = shift) or croak "instance variable needed";
    my $file = shift;

    my $fh;    

    if (fileno($file)) { # Check if its a file handle
        $fh = $file;
    } else {             # If it's just a file name open a file handle
        # Return if file doesn't exist
{'fraction_non_alphanumeric': 0.1203585147247119, 'fraction_numerical': 0.002560819462227913, 'mean_word_length': 2.9001663893510816, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4295601', 'n_tokens_mistral': 841, 'n_tokens_neox': 804, 'n_words': 287}