<commit_before>#!/usr/bin/perl -w

# check if files marked as unavailable somehow became abailable

use strict;
use Dudl::DB;


my $dudl = Dudl::DB->new;
my $db = $dudl->db;
my $topdir = $dudl->conf("cdpath");

my $res = $db->select( "f.id, stor_filename(collection,colnum,dir,fname) AS file ".
	"FROM stor_file f INNER JOIN stor_unit u ON f.unit_id = u.id ".
	"WHERE not available" );

my( $id, $fname );
$res->bind_columns( \$id, \$fname );

while( $res->fetch ){
	if( -e "$topdir/$fname" ){
		print "reapeared: $fname\n";
		$db->update( "stor_file", { available => "true" }, 
			"id = $id" );
	}
}

$db->commit;
$db->rollback;<commit_msg>also mark vanished files as unavailable<commit_after>#!/usr/bin/perl -w

# check if files marked as unavailable somehow became abailable

use strict;
use Dudl::DB;


my $dudl = Dudl::DB->new;
my $db = $dudl->db;
my $topdir = $dudl->conf("cdpath");

my( $res, $id, $fname );

print "\nstep 1: find files that reapeared...\n";
$res = $db->select( "f.id, stor_filename(collection,colnum,dir,fname) AS file ".
	"FROM stor_file f INNER JOIN stor_unit u ON f.unit_id = u.id ".
	"WHERE not available" );

$res->bind_columns( \$id, \$fname );

while( $res->fetch ){
	if( -e "$topdir/$fname" ){
		print "reapeared: $fname\n";
		$db->update( "stor_file", { available => "true" }, 
			"id = $id" );
	}
}

print "\nstep 2: find files that vanished...\n";
$res = $db->select( "f.id, stor_filename(collection,colnum,dir,fname) AS file ".
	"FROM stor_file f INNER JOIN stor_unit u ON f.unit_id = u.id ".
	"WHERE available OR available ISNULL" );

$res->bind_columns( \$id, \$fname );

while( $res->fetch ){
	if( ! -e "$topdir/$fname" ){
		print "vanished: $fname\n";
		$db->update( "stor_file", { available => "false" }, 
			"id = $id" );
	}
}


$db->commit;
$db->rollback;
{'fraction_non_alphanumeric': 0.20177876598110062, 'fraction_numerical': 0.0011117287381878821, 'mean_word_length': 3.8517520215633425, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5959420', 'n_tokens_mistral': 766, 'n_tokens_neox': 722, 'n_words': 188}