<commit_before>package SQL::Tidy;
use strict;
use warnings;

use Carp();
use Data::Dumper;

use SQL::Tidy::Comment;
use SQL::Tidy::Dialect;
use SQL::Tidy::DDL;
use SQL::Tidy::DML;
use SQL::Tidy::PL;
use SQL::Tidy::String;
use SQL::Tidy::Tokenize;

=head1 NAME

SQL::Tidy

=head1 SYNOPSIS

=head1 DESCRIPTION

=head1 METHODS

=over

=cut

=item new

Create, and return, a new instance of this

=cut

sub new {
    my ( $this, $args ) = @_;
    my $class = ref($this) || $this;
    my $self = {};
    bless $self, $class;

    # Set defaults if not otherwise set
    $args->{use_tabs}       ||= 0;            # Spaces all the way
    $args->{tab_size}       ||= 4;
    $args->{max_line_width} ||= 120;
    $args->{min_line_width} ||= 40;
    $args->{case_folding}   ||= 'upper';      # It is in the SQL standard after all
    $args->{dialect}        ||= 'Default';    # Oracle, PostgreSQL, ...

    if ( not defined $args->{uppercase_keywords} ) {
        $args->{uppercase_keywords} = 1;
    }

    # Convert Oracle functions:
    $args->{convert_nvl}    ||= 0;            # Convert nvl to coalesce
    $args->{convert_nvl2}   ||= 0;            # Convert nvl2 to CASE structure
    $args->{convert_decode} ||= 0;            # Convert decode to CASE structure

    # Regexp for "Safe-to-unquote" identifiers
    # Not that this will probably ever make sense as a config option...
    $args->{stu_ident} ||= '[A-Z0-9_]+';
    my $stu_ident = $args->{stu_ident};
    $args->{stu_re} = qr/^"$stu_ident"$/;

    foreach my $key ( keys %{$args} ) {
        unless ( exists $args->{$key} ) {
            $self->{$_} = $args->{$_};
        }
    }

    $args->{space_re} = qr/^[ \t]+$/;

    $self->{tokenizer} = SQL::Tidy::Tokenize->new($args);
    $self->{comments}  = SQL::Tidy::Comment->new($args);
    $self->{strings}   = SQL::Tidy::String->new($args);
    $self->{dialect}   = SQL::Tidy::Dialect->new($args);
    $self->{ddl}       = SQL::Tidy::DDL->new($args);
    $self->{dmls}      = SQL::Tidy::DML->new($args);
    $self->{pls}       = SQL::Tidy::PL->new($args);

    return $self;
}

=item tidy ( code )

Takes a string of code, formats it, and returns the result

=cut

sub tidy {
    my ( $self, $code ) = @_;

    my $comments;
    my $strings;
    my $dml;
    my $pl;
    my @tokens = $self->{tokenizer}->tokenize_sql($code);

    ( $comments, @tokens ) = $self->{comments}->tag_comments(@tokens);
    ( $strings,  @tokens ) = $self->{strings}->tag_strings(@tokens);

    @tokens = $self->normalize(@tokens);
    ( $dml, @tokens ) = $self->{dmls}->tag_dml(@tokens);

    # TODO: format pieces (to include unquoting identifiers/capitalizing key-words)

    ( $pl, @tokens ) = $self->{pls}->tag_pl(@tokens);

    @tokens = $self->{ddl}->format_ddl( $comments, @tokens );

    $pl = $self->{pls}->format_pl( $comments, $pl );

    $dml = $self->{dmls}->format_dml( $comments, $dml );

    @tokens = $self->{pls}->untag_pl( $pl, @tokens );
    @tokens = $self->{dmls}->untag_dml( $dml, @tokens );

    # TODO: Fix line-wrapping here? Or as part of format DDL/DML/PL?
    @tokens = $self->fix_spacing(@tokens);

    @tokens = $self->{strings}->untag_strings( $strings, @tokens );
    @tokens = $self->{comments}->untag_comments( $comments, @tokens );

    # TODO: any cleanup?

    $code = join( '', @tokens );

    $code =~ s/^\n+//;
    $code =~ s/RETURN - /RETURN -/g;
    $code =~ s/\( \)/()/g;
    $code =~ s/\( \* \)/(*)/g;

    return $code;
}

=item normalize ( tokens )

Remove white-space and empty tokens from the supplied list of tokens

Returns the modified list

=cut

sub normalize {<commit_msg>Some cleanup<commit_after>package SQL::Tidy;
use strict;
use warnings;

use Carp();
use Data::Dumper;

use SQL::Tidy::Comment;
use SQL::Tidy::Dialect;
use SQL::Tidy::DDL;
use SQL::Tidy::DML;
use SQL::Tidy::PL;
use SQL::Tidy::String;
use SQL::Tidy::Tokenize;

=head1 NAME

SQL::Tidy

=head1 SYNOPSIS

=head1 DESCRIPTION

=head1 METHODS

=over

=cut

my $Tokenizer;
my $Comment;
my $String;
my $dialect;
my $DDL;
my $DML;
my $PL;

=item new

Create, and return, a new instance of this

=cut

sub new {
    my ( $this, $args ) = @_;
    my $class = ref($this) || $this;
    my $self = {};
    bless $self, $class;

    # Set defaults if not otherwise set
    $args->{use_tabs}       ||= 0;            # Spaces all the way
    $args->{tab_size}       ||= 4;
    $args->{max_line_width} ||= 120;
    $args->{min_line_width} ||= 40;
    $args->{case_folding}   ||= 'upper';      # It is in the SQL standard after all
    $args->{dialect}        ||= 'Default';    # Oracle, PostgreSQL, ...

    if ( not defined $args->{uppercase_keywords} ) {
        $args->{uppercase_keywords} = 1;
    }

    # Convert Oracle functions:
    $args->{convert_nvl}    ||= 0;            # Convert nvl to coalesce
    $args->{convert_nvl2}   ||= 0;            # Convert nvl2 to CASE structure
    $args->{convert_decode} ||= 0;            # Convert decode to CASE structure

    # Regexp for "Safe-to-unquote" identifiers
    # Not that this will probably ever make sense as a config option...
    $args->{stu_ident} ||= '[A-Z0-9_]+';
    my $stu_ident = $args->{stu_ident};
    $args->{stu_re} = qr/^"$stu_ident"$/;

    foreach my $key ( keys %{$args} ) {
        unless ( exists $args->{$key} ) {
            $self->{$_} = $args->{$_};
        }
    }

    $args->{space_re} = qr/^[ \t]+$/;

    $Tokenizer = SQL::Tidy::Tokenize->new($args);
    $Comment   = SQL::Tidy::Comment->new($args);
    $String    = SQL::Tidy::String->new($args);
    $dialect   = SQL::Tidy::Dialect->new($args);
    $DDL       = SQL::Tidy::DDL->new($args);
    $DML       = SQL::Tidy::DML->new($args);
    $PL        = SQL::Tidy::PL->new($args);

    return $self;
}

=item tidy ( code )

Takes a string of code, formats it, and returns the result

=cut

sub tidy {
    my ( $self, $code ) = @_;

    my $comments;
    my $strings;
    my $dml;
    my $pl;
    my @tokens = $Tokenizer->tokenize_sql($code);

    ( $comments, @tokens ) = $Comment->tag_comments(@tokens);
    ( $strings,  @tokens ) = $String->tag_strings(@tokens);

    @tokens = $self->normalize(@tokens);
    ( $dml, @tokens ) = $DML->tag_dml(@tokens);

    # TODO: format pieces (to include unquoting identifiers/capitalizing key-words)

    ( $pl, @tokens ) = $PL->tag_pl(@tokens);

    @tokens = $DDL->format_ddl( $comments, @tokens );

    $pl = $PL->format_pl( $comments, $pl );

    $dml = $DML->format_dml( $comments, $dml );

    @tokens = $PL->untag_pl( $pl, @tokens );
    @tokens = $DML->untag_dml( $dml, @tokens );

    # TODO: Fix line-wrapping here? Or as part of format DDL/DML/PL?
    @tokens = $self->fix_spacing(@tokens);

    @tokens = $String->untag_strings( $strings, @tokens );
    @tokens = $Comment->untag_comments( $comments, @tokens );

    # TODO: any cleanup?

    $code = join( '', @tokens );

    $code =~ s/^\n+//;
    $code =~ s/RETURN - /RETURN -/g;
    $code =~ s/\( \)/()/g;
    $code =~ s/\( \* \)/(*)/g;

    return $code;
}

=item normalize ( tokens )

Remove white-space and empty tokens from the supplied list of tokens

Returns the modified list

=cut

sub normalize {
{'fraction_non_alphanumeric': 0.19347613047739046, 'fraction_numerical': 0.005319893602127958, 'mean_word_length': 2.8040468583599574, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 100, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '456325', 'n_tokens_mistral': 2771, 'n_tokens_neox': 2649, 'n_words': 629}