<commit_before>#! /usr/bin/perl -w


use strict;
use File::Find;
use Getopt::Long;
use Dudl;

# Dudl::File elements to update
my @want;
#my @want = qw{ FSUM DSUM };

my $dir;
my $dev;
my $cd;

my $disc;
my @files;

sub usage {
	print "usage: ". $0 ." [opts] <cdpath> \"<collection><discid>\" ...\n";
	print " scan one CD mounted at <topdir> for mp3s\n";
 	print " get IDtag and add infos to database\n";
	print " options:\n";
	print "  --eject   open tray when done with CD\n";
	print "  --unit    do not scan files\n";
	print "  --mp3     scan mp3 information\n";
	print "  --sum     calculate md5 sum\n";
	print " if none is specified, --mp3 --sum is assumed.\n";
}


my $opt_eject;
my $opt_mp3;
my $opt_sum;
my $opt_unit;

my $result = GetOptions(
	"eject"		=> \$opt_eject,
	"mp3!"		=> \$opt_mp3,
	"sum!"		=> \$opt_sum,
	"unit!"		=> \$opt_unit,
	);
if( ! $result ){
	&usage();
	exit 1;
}

if( !( $opt_mp3 || $opt_sum || $opt_unit )){
	$opt_mp3=1;
	$opt_sum=1;
}

if( $opt_mp3 ){
	push @want, Dudl::File::acquires( "base" );
	push @want, Dudl::File::acquires( "file" );
	push @want, Dudl::File::acquires( "info" );
	push @want, Dudl::File::acquires( "tag" );
}

if( $opt_sum ){
	push @want, Dudl::File::acquires( "sum" );
}


$cd = shift;
if( ! $cd ){
	print STDERR "missing device/cdpath\n";
	&usage();
	exit 1;
}

if( $#ARGV < 0 ){
	print STDERR "missing collection name(s)\n";
	&usage();
	exit 1;
}

	
my $dudl = Dudl->new;

( $dev, $dir ) = $dudl->get_fstab( $cd );
if( ! $dir ){
	$dir = $cd;
	if( $#ARGV > 0 ){
		print STDERR 
			"cannot scan multiple units when $cd is no CD-ROM\n";
		$dudl->done;
		exit 1;
	}
}

$dir .= "/";
$dir =~ s:/+:/:g;

print "going to scan ". ($#ARGV +1) ." CDs in \"". $dir ."\"\n";



&cd_umount( $dev, 0 );

foreach $disc ( @ARGV ){
	&cd_mount( $dev, $disc );

	&scan( $dev, $dir, $disc, ! $opt_unit );
	$dudl->commit || die;

	&cd_umount( $dev, $opt_eject );
}

$dudl->done;





sub scan {
	my $dev		= shift;
	my $dir		= shift;
	my $disc	= shift;
	my $dofiles	= shift;

	my $collection;
	my $discid;

	if( 2 != ( ( $collection, $discid ) = ($disc =~ m/(\D+)(\d+)/ ))){
		print STDERR "invalid collection or id: $disc\n";
		return;
	}




	my $unit = $dudl->newunit;
	$unit->get_collection( $collection, $discid );

	if( $dev ){
		$unit->acquire( $dev );
	}

	my $id;
	if( $unit->id  ){
		$id = $unit->update;
	} else {
		$id = $unit->insert;
	}

	if( ! $id ){
		die;
	}
	print "unit id: ". $id ."\n";
	


	if( $dofiles ){
		my $dlen = length($dir);
		my $file = $unit->newfile;
		$file->want( \@want );


		print "searching for mp3s in \"$dir\"\n";
		#@files = ();
		#&finddepth(\&want_file, $dir );
		@files = `find $dir -type f `;


		# TODO: scan for filenames not ending in .mp3

		print "analyzing mp3s\n";
		foreach (@files){
			chomp;

			my $relpath = substr($_, $dlen );
			print "$relpath ...";

			$file->get_path( $relpath );
			if( ! $file->acquire( $dir, $relpath ) ){
				die "cannot get file details";
			}

			if( $file->id ){
				print " updating";
				$id = $file->update;
			} else {
				print " adding";
				$id = $file->insert;
			}

			if( ! $id ){
				die;
			}
			print " $id\n";

			$file->clean;
		}
	}
}

sub cd_mount {<commit_msg>by default don't add files to already existing units<commit_after>#! /usr/bin/perl -w


use strict;
use File::Find;
use Getopt::Long;
use Dudl;

# Dudl::File elements to update
my @want;
#my @want = qw{ FSUM DSUM };

my $dir;
my $dev;
my $cd;

my $disc;
my @files;

sub usage {
	print "usage: ". $0 ." [opts] <cdpath> \"<collection><discid>\" ...\n";
	print " scan one CD mounted at <topdir> for mp3s\n";
 	print " get IDtag and add infos to database\n";
	print " options:\n";
	print "  --eject   open tray when done with CD\n";
	print "  --unit    do not scan files\n";
	print "  --mp3     scan mp3 information\n";
	print "  --sum     calculate md5 sum\n";
	print "  --add     add missing files although CD is already in DB\n";
	print " if none is specified, --mp3 --sum is assumed.\n";
}


my $opt_eject;
my $opt_mp3;
my $opt_sum;
my $opt_unit;
my $opt_add;

my $result = GetOptions(
	"eject"		=> \$opt_eject,
	"mp3!"		=> \$opt_mp3,
	"sum!"		=> \$opt_sum,
	"unit!"		=> \$opt_unit,
	"add|a"		=> \$opt_add;
	);
if( ! $result ){
	&usage();
	exit 1;
}

if( !( $opt_mp3 || $opt_sum || $opt_unit )){
	$opt_mp3=1;
	$opt_sum=1;
}

if( $opt_mp3 ){
	push @want, Dudl::File::acquires( "base" );
	push @want, Dudl::File::acquires( "file" );
	push @want, Dudl::File::acquires( "info" );
	push @want, Dudl::File::acquires( "tag" );
}

if( $opt_sum ){
	push @want, Dudl::File::acquires( "sum" );
}


$cd = shift;
if( ! $cd ){
	print STDERR "missing device/cdpath\n";
	&usage();
	exit 1;
}

if( $#ARGV < 0 ){
	print STDERR "missing collection name(s)\n";
	&usage();
	exit 1;
}

	
my $dudl = Dudl->new;

( $dev, $dir ) = $dudl->get_fstab( $cd );
if( ! $dir ){
	$dir = $cd;
	if( $#ARGV > 0 ){
		print STDERR 
			"cannot scan multiple units when $cd is no CD-ROM\n";
		$dudl->done;
		exit 1;
	}
}

$dir .= "/";
$dir =~ s:/+:/:g;

print "going to scan ". ($#ARGV +1) ." CDs in \"". $dir ."\"\n";



&cd_umount( $dev, 0 );

foreach $disc ( @ARGV ){
	&cd_mount( $dev, $disc );

	&scan( $dev, $dir, $disc, ! $opt_unit );
	$dudl->commit || die;

	&cd_umount( $dev, $opt_eject );
}

$dudl->done;





sub scan {
	my $dev		= shift;
	my $dir		= shift;
	my $disc	= shift;
	my $dofiles	= shift;

	my $collection;
	my $discid;

	if( 2 != ( ( $collection, $discid ) = ($disc =~ m/(\D+)(\d+)/ ))){
		print STDERR "invalid collection or id: $disc\n";
		return;
	}




	my $unit = $dudl->newunit;
	$unit->get_collection( $collection, $discid );

	if( $dev ){
		$unit->acquire( $dev );
	}

	my $id;
	my $add_files = 0;
	if( $unit->id  ){
		$id = $unit->update;
	} else {
		$id = $unit->insert;
		$add_files ++;
	}

	if( ! $id ){
		die;
	}
	print "unit id: ". $id ."\n";
	

	$add_file ++ if $opt_add;

	if( $dofiles ){
		my $dlen = length($dir);
		my $file = $unit->newfile;
		$file->want( \@want );


		print "searching for mp3s in \"$dir\"\n";
		#@files = ();
		#&finddepth(\&want_file, $dir );
		@files = `find $dir -type f -iname "*.mp3"`;


		# TODO: scan for filenames not ending in .mp3

		print "analyzing mp3s\n";
		foreach (@files){
			chomp;

			my $relpath = substr($_, $dlen );
			print "$relpath ...";

			$file->get_path( $relpath );
			if( ! $file->acquire( $dir, $relpath ) ){
				die "cannot get file details";
			}

			if( $file->id ){
				print " updating";
				$id = $file->update;
			} else {
				die "not adding file to existing unit" 
					unless $add_file;

				print " adding";
				$id = $file->insert;
			}

			if( ! $id ){
				die;
			}
			print " $id\n";

			$file->clean;
		}
	}
}

sub cd_mount {
{'fraction_non_alphanumeric': 0.21371992228366463, 'fraction_numerical': 0.00807054251980272, 'mean_word_length': 2.8393574297188753, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 60, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 27, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1249142', 'n_tokens_mistral': 3201, 'n_tokens_neox': 2907, 'n_words': 771}