<commit_before>#!/usr/bin/perl -w

# $Id: dudl_storhave.pl,v 1.1 2001-12-13 14:44:06 bj Exp $

use strict;
use Dudl;

my @attribs = qw( dir fname );
my $filesql = "TRIM( both ' ' FROM collection ) || ".
	"colnum || '/' || dir || '/' || fname";

my $what=lc shift || die "missing 'what' argument: any|file|sql|..";
my $pattern = join( " ", @ARGV ) || die "missing pattern";

my $dudl = Dudl->new;
my $db = $dudl->db;

$pattern = $db->quote( $pattern, DBI::SQL_CHAR );

my $where;
if( $what eq "all" || $what eq "any" ){
	foreach( @attribs ){
		$where .= " OR " if $where;
		$where .= "( $_ ~* $pattern )";
	}

} elsif( $what eq "file" ){
	$where .= "($filesql) ~* $pattern";

} elsif( $what eq "sql" || $what eq "select" ){
	$where = join " ", @ARGV;

} else {
	my $found = 0;
	foreach( @attribs ){
		if( $_ eq $what ){
			$where .= "$what ~* $pattern";
			$found++;
			last;
		}
	}
	if( $found ){
		die "invalid 'what' argument";
	}
}


print STDERR "SELECT ... WHERE $where\n";

my $query = 
"SELECT ".
	"$filesql as file ".
"FROM ".
	"stor_file f INNER JOIN ".
		"stor_unit u ".
		"ON u.id = f.unitid ".
"WHERE ".
	"$where ".
"ORDER BY ".
	"u.collection, ".
	"u.colnum, ".
	"f.dir, ".
	"f.fname";

my $sth = $db->prepare( $query );
if( ! $sth ){
	die $db->errstr ."\nquery: $query\n";
}

my $res = $sth->execute;
if( ! $res ){
	die $sth->errstr ."\nquery: $query\n";
}

my $file;
$sth->bind_columns( \$file );<commit_msg>ignore broken files<commit_after>#!/usr/bin/perl -w

# $Id: dudl_storhave.pl,v 1.2 2001-12-13 14:49:36 bj Exp $

use strict;
use Dudl;

my @attribs = qw( dir fname );
my $filesql = "TRIM( both ' ' FROM collection ) || ".
	"colnum || '/' || dir || '/' || fname";

my $what=lc shift || die "missing 'what' argument: any|file|sql|..";
my $pattern = join( " ", @ARGV ) || die "missing pattern";

my $dudl = Dudl->new;
my $db = $dudl->db;

$pattern = $db->quote( $pattern, DBI::SQL_CHAR );

my $where;
if( $what eq "all" || $what eq "any" ){
	foreach( @attribs ){
		$where .= " OR " if $where;
		$where .= "( $_ ~* $pattern )";
	}

} elsif( $what eq "file" ){
	$where .= "($filesql) ~* $pattern";

} elsif( $what eq "sql" || $what eq "select" ){
	$where = join " ", @ARGV;

} else {
	my $found = 0;
	foreach( @attribs ){
		if( $_ eq $what ){
			$where .= "$what ~* $pattern";
			$found++;
			last;
		}
	}
	if( $found ){
		die "invalid 'what' argument";
	}
}


print STDERR "SELECT ... WHERE $where\n";

my $query = 
"SELECT ".
	"$filesql as file ".
"FROM ".
	"stor_file f INNER JOIN ".
		"stor_unit u ".
		"ON u.id = f.unitid ".
"WHERE ".
	"NOT broken AND ".
	"( $where ) ".
"ORDER BY ".
	"u.collection, ".
	"u.colnum, ".
	"f.dir, ".
	"f.fname";

my $sth = $db->prepare( $query );
if( ! $sth ){
	die $db->errstr ."\nquery: $query\n";
}

my $res = $sth->execute;
if( ! $res ){
	die $sth->errstr ."\nquery: $query\n";
}

my $file;
$sth->bind_columns( \$file );
{'fraction_non_alphanumeric': 0.2402642559109875, 'fraction_numerical': 0.011821974965229486, 'mean_word_length': 2.979253112033195, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5475628', 'n_tokens_mistral': 1322, 'n_tokens_neox': 1252, 'n_words': 321}