<commit_before>#!/usr/bin/perl
# Copyright (c) 2009, Kulag <<EMAIL>>
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#
# KADR was forked from ADBREN v4 Copyright (c) 2008, clip9 <<EMAIL>>

use v5.10;
use common::sense;
use Config::YAML;
use Data::Types qw(:int);
use DBI::SpeedySimple;
use Digest::ED2K;
use Encode;
use Expect;
use File::Copy;
use File::Basename;
use File::HomeDir;
use File::Find;
use Getopt::Long;
use PortIO;
use Parse::TitleSyntax;
use Parse::TitleSyntax::Functions::Regexp;
use Readonly;
use Term::StatusLine::Freeform;
use Term::StatusLine::XofX;

$|++;
$SIG{INT} = "cleanup";
binmode STDIN, ':encoding(UTF-8)';
binmode STDOUT, ':encoding(UTF-8)';

# Some debug options
Readonly my $appdir => $ENV{KADR_DIR} // File::HomeDir->my_home . '/.kadr';
Readonly my $dont_move => 0; # Does everything short of moving/renaming the files when true.
Readonly my $dont_expire_cache => 0; # Leaves expired cache entries untouched.

if(!is_file("$appdir/config")) {
	if(!is_dir($appdir)) {
		mkpath($appdir);
	}
	close(file_open('>', "$appdir/config"));
}
my $conf = Config::YAML->new(
	config => "$appdir/config",
	output => "$appdir/config",
	anidb => {
		cache_timeout => {
			mylist_unwatched => 7200,
			mylist_watched => 1036800,
			file => 1036800,
		},
		password => <PASSWORD>,
		time_to_sleep_when_busy => 10*60, # How long (in seconds) to sleep if AniDB informs us it's too busy to talk to us.
		update_records_for_deleted_files => 1,
		username => undef,
	},
	avdump => undef, # Commandline to run avdump.
	avdump_timeout => 30, # How many seconds to wait for avdump to contact AniDB before retrying.
	dirs => {
		delete_empty_dirs_in_scanned => 1,
		to_put_unwatched_eps => undef,
		to_put_watched_eps => undef,
		to_scan => [],
		valid_for_unwatched_eps => [],
		valid_for_watched_eps => [],
	},
	file_naming_scheme => <<'EOF',
$if(%only_episode_in_folder%,,%anime_romaji_name%/)%anime_romaji_name%
$if($rematch(%episode_english_name%,'^(Complete Movie|OVA|Special|TV Special)$'),,
 - %episode_number%$ifgreater(%file_version%,1,v%file_version%,) - %episode_english_name%)
$if($not($strcmp(%group_short_name%,raw)), '['%group_short_name%']').%file_type%
EOF
	load_local_cache_into_memory => 1,
	show_hashing_progress => 1, # Only disable if you think that printing the hashing progess is taking up a significant amount of CPU time when hashing a file.
	use_windows_compatible_filenames => 0, # Off by default since not having to do this produces nicer filenames.
);
$conf->read("$appdir/config");
$conf->write;

my $parsets = Parse::TitleSyntax->new($conf->{file_naming_scheme});
$parsets->AddFunctions(Parse::TitleSyntax::Functions::Regexp->new($parsets));

# A cache to speed up in_list calls.
my $in_list_cache = {};

my $db = DBI::SpeedySimple->new("dbi:SQLite:$appdir/db");
$db->{dbh}->do(q{CREATE TABLE IF NOT EXISTS known_files (`filename` TEXT, `size` INT, `ed2k` TEXT PRIMARY KEY, `avdumped` INT);}) and
$db->{dbh}->do(q{CREATE TABLE IF NOT EXISTS anidb_mylist_file (`lid` INT, `fid` INTEGER PRIMARY KEY, `eid` INT, `aid` INT, `gid` INT,
				 `date` INT, `state` INT, `viewdate` INT, `storage` TEXT, `source` TEXT, `other` TEXT, `filestate` TEXT, `updated` INT);}) and
$db->{dbh}->do(q{CREATE TABLE IF NOT EXISTS anidb_mylist_anime (`aid` INTEGER PRIMARY KEY, `anime_title` TEXT, `episodes` INT,
				 `eps_with_state_unknown` TEXT, `eps_with_state_on_hdd` TEXT, `eps_with_state_on_cd` TEXT, `eps_with_state_deleted` TEXT,
				 `watched_eps` TEXT, `updated` INT);}) and
$db->{dbh}->do(q{CREATE TABLE IF NOT EXISTS adbcache_file (`fid` INTEGER PRIMARY KEY, `aid` INT, `eid` INT, `gid` INT, `lid` INT,
				 `other_episodes` TEXT, `is_deprecated` INT, `status` INT, `size` INT, `ed2k` TEXT, `md5` TEXT, `sha1` TEXT, `crc32` TEXT,
				 `quality` TEXT, `source` TEXT, `audio_codec` TEXT, `audio_bitrate` INT, `video_codec` TEXT, `video_bitrate` INT, `video_resolution` TEXT,
				 `file_type` TEXT, `dub_language` TEXT, `sub_language` TEXT, `length` INT, `description` TEXT, `air_date` INT,
				 `anime_total_episodes` INT, `anime_highest_episode_number` INT, `anime_year` INT, `anime_type` INT, `anime_related_aids` TEXT,
				 `anime_related_aid_types` TEXT, `anime_categories` TEXT, `anime_romaji_name` TEXT, `anime_kanji_name` TEXT, `anime_english_name` TEXT,
				 `anime_other_name` TEXT, `anime_short_names` TEXT, `anime_synonyms` TEXT, `episode_number` TEXT, `episode_english_name` TEXT,
				 `episode_romaji_name` TEXT, `episode_kanji_name` TEXT, `episode_rating` TEXT, `episode_vote_count` TEXT, `group_name` TEXT,
				 `group_short_name` TEXT, `updated` INT)}) or die "Could not initialize the database";

unless($dont_expire_cache) {
	my $timeout = $conf->{anidb}->{cache_timeout};
	$db->{dbh}->do("DELETE FROM adbcache_file WHERE updated < " . (time - $timeout->{file}));
	$db->{dbh}->do("DELETE FROM anidb_mylist_anime WHERE updated < " . (time - $timeout->{mylist_unwatched}) . " AND watched_eps = eps_with_state_on_hdd");
	$db->{dbh}->do("DELETE FROM anidb_mylist_anime WHERE updated < " . (time - $timeout->{mylist_watched}) . " AND watched_eps != eps_with_state_on_hdd");
}

if($conf->{load_local_cache_into_memory}) {
	$db->cache([
		{table => 'known_files', indices => ['filename', 'size']},
		{table => 'adbcache_file', indices => ['ed2k', 'size']},
		{table => 'anidb_mylist_file', indices => ['fid']},
		{table => 'anidb_mylist_anime', indices => ['aid']},
	]);
}

my $a = AniDB::UDPClient->new({
	username => $conf->{anidb}->{username},
	password => $conf->{anidb}->{password},
	db => $db,
	port => 3700,
});

my @files = sort(find_files(@{$conf->{dirs}->{to_scan}}));
my @ed2k_of_processed_files;
my $sl = Term::StatusLine::XofX->new(total_item_count => scalar(@files));
for(@files) {
	$sl->update('++', $_);
	next if !valid_file($_);
	if(my $ed2k = process_file($_)) {
		push @ed2k_of_processed_files, $ed2k;
	}
}<commit_msg>Fix a logic error in cache expiration + asthetics<commit_after>#!/usr/bin/perl
# Copyright (c) 2009, Kulag <<EMAIL>>
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#
# KADR was forked from ADBREN v4 Copyright (c) 2008, clip9 <<EMAIL>>

use v5.10;
use common::sense;
use Config::YAML;
use Data::Types qw(:int);
use DBI::SpeedySimple;
use Digest::ED2K;
use Encode;
use Expect;
use File::Copy;
use File::Basename;
use File::HomeDir;
use File::Find;
use Getopt::Long;
use PortIO;
use Parse::TitleSyntax;
use Parse::TitleSyntax::Functions::Regexp;
use Readonly;
use Term::StatusLine::Freeform;
use Term::StatusLine::XofX;

$|++;
$SIG{INT} = "cleanup";
binmode STDIN, ':encoding(UTF-8)';
binmode STDOUT, ':encoding(UTF-8)';

# Some debug options
Readonly my $appdir => $ENV{KADR_DIR} // File::HomeDir->my_home . '/.kadr';
Readonly my $dont_move => 0; # Does everything short of moving/renaming the files when true.
Readonly my $dont_expire_cache => 0; # Leaves expired cache entries untouched.

if(!is_file("$appdir/config")) {
	if(!is_dir($appdir)) {
		mkpath($appdir);
	}
	close(file_open('>', "$appdir/config"));
}
my $conf = Config::YAML->new(
	config => "$appdir/config",
	output => "$appdir/config",
	anidb => {
		cache_timeout => {
			mylist_unwatched => 7200,
			mylist_watched => 1036800,
			file => 1036800,
		},
		password => <PASSWORD>,
		time_to_sleep_when_busy => 10*60, # How long (in seconds) to sleep if AniDB informs us it's too busy to talk to us.
		update_records_for_deleted_files => 1,
		username => undef,
	},
	avdump => undef, # Commandline to run avdump.
	avdump_timeout => 30, # How many seconds to wait for avdump to contact AniDB before retrying.
	dirs => {
		delete_empty_dirs_in_scanned => 1,
		to_put_unwatched_eps => undef,
		to_put_watched_eps => undef,
		to_scan => [],
		valid_for_unwatched_eps => [],
		valid_for_watched_eps => [],
	},
	file_naming_scheme => <<'EOF',
$if(%only_episode_in_folder%,,%anime_romaji_name%/)%anime_romaji_name%
$if($rematch(%episode_english_name%,'^(Complete Movie|OVA|Special|TV Special)$'),,
 - %episode_number%$ifgreater(%file_version%,1,v%file_version%,) - %episode_english_name%)
$if($not($strcmp(%group_short_name%,raw)), '['%group_short_name%']').%file_type%
EOF
	load_local_cache_into_memory => 1,
	show_hashing_progress => 1, # Only disable if you think that printing the hashing progess is taking up a significant amount of CPU time when hashing a file.
	use_windows_compatible_filenames => 0, # Off by default since not having to do this produces nicer filenames.
);
$conf->read("$appdir/config");
$conf->write;

my $parsets = Parse::TitleSyntax->new($conf->{file_naming_scheme});
$parsets->AddFunctions(Parse::TitleSyntax::Functions::Regexp->new($parsets));

# A cache to speed up in_list calls.
my $in_list_cache = {};

my $db = DBI::SpeedySimple->new("dbi:SQLite:$appdir/db");
$db->{dbh}->do(q{CREATE TABLE IF NOT EXISTS known_files (`filename` TEXT, `size` INT, `ed2k` TEXT PRIMARY KEY, `avdumped` INT);}) and
$db->{dbh}->do(q{CREATE TABLE IF NOT EXISTS anidb_mylist_file (`lid` INT, `fid` INTEGER PRIMARY KEY, `eid` INT, `aid` INT, `gid` INT,
				 `date` INT, `state` INT, `viewdate` INT, `storage` TEXT, `source` TEXT, `other` TEXT, `filestate` TEXT, `updated` INT);}) and
$db->{dbh}->do(q{CREATE TABLE IF NOT EXISTS anidb_mylist_anime (`aid` INTEGER PRIMARY KEY, `anime_title` TEXT, `episodes` INT,
				 `eps_with_state_unknown` TEXT, `eps_with_state_on_hdd` TEXT, `eps_with_state_on_cd` TEXT, `eps_with_state_deleted` TEXT,
				 `watched_eps` TEXT, `updated` INT);}) and
$db->{dbh}->do(q{CREATE TABLE IF NOT EXISTS adbcache_file (`fid` INTEGER PRIMARY KEY, `aid` INT, `eid` INT, `gid` INT, `lid` INT,
				 `other_episodes` TEXT, `is_deprecated` INT, `status` INT, `size` INT, `ed2k` TEXT, `md5` TEXT, `sha1` TEXT, `crc32` TEXT,
				 `quality` TEXT, `source` TEXT, `audio_codec` TEXT, `audio_bitrate` INT, `video_codec` TEXT, `video_bitrate` INT, `video_resolution` TEXT,
				 `file_type` TEXT, `dub_language` TEXT, `sub_language` TEXT, `length` INT, `description` TEXT, `air_date` INT,
				 `anime_total_episodes` INT, `anime_highest_episode_number` INT, `anime_year` INT, `anime_type` INT, `anime_related_aids` TEXT,
				 `anime_related_aid_types` TEXT, `anime_categories` TEXT, `anime_romaji_name` TEXT, `anime_kanji_name` TEXT, `anime_english_name` TEXT,
				 `anime_other_name` TEXT, `anime_short_names` TEXT, `anime_synonyms` TEXT, `episode_number` TEXT, `episode_english_name` TEXT,
				 `episode_romaji_name` TEXT, `episode_kanji_name` TEXT, `episode_rating` TEXT, `episode_vote_count` TEXT, `group_name` TEXT,
				 `group_short_name` TEXT, `updated` INT)}) or die "Could not initialize the database";

unless($dont_expire_cache) {
	my $timeout = $conf->{anidb}->{cache_timeout};
	$db->{dbh}->do('DELETE FROM adbcache_file WHERE updated < ' . (time - $timeout->{file}));
	$db->{dbh}->do('DELETE FROM anidb_mylist_anime WHERE updated < ' . (time - $timeout->{mylist_unwatched}) . ' AND watched_eps != eps_with_state_on_hdd');
	$db->{dbh}->do('DELETE FROM anidb_mylist_anime WHERE updated < ' . (time - $timeout->{mylist_watched}) . ' AND watched_eps = eps_with_state_on_hdd');
}

if($conf->{load_local_cache_into_memory}) {
	$db->cache([
		{table => 'known_files', indices => ['filename', 'size']},
		{table => 'adbcache_file', indices => ['ed2k', 'size']},
		{table => 'anidb_mylist_file', indices => ['fid']},
		{table => 'anidb_mylist_anime', indices => ['aid']},
	]);
}

my $a = AniDB::UDPClient->new({
	username => $conf->{anidb}->{username},
	password => $conf->{anidb}->{password},
	db => $db,
	port => 3700,
});

my @files = sort(find_files(@{$conf->{dirs}->{to_scan}}));
my @ed2k_of_processed_files;
my $sl = Term::StatusLine::XofX->new(total_item_count => scalar(@files));
for(@files) {
	$sl->update('++', $_);
	next if !valid_file($_);
	if(my $ed2k = process_file($_)) {
		push @ed2k_of_processed_files, $ed2k;
	}
}
{'fraction_non_alphanumeric': 0.1569926873857404, 'fraction_numerical': 0.009597806215722121, 'mean_word_length': 5.757076685537828, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 173, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1602832', 'n_tokens_mistral': 5134, 'n_tokens_neox': 4780, 'n_words': 1253}