Commit e70080e0 authored by David Cameron's avatar David Cameron

make cache scripts stand-alone with own conf parsing and logging

parent 7be8ecee
......@@ -2253,9 +2253,7 @@ AC_CONFIG_FILES([Makefile
src/hed/libs/crypto/Makefile
src/hed/libs/cryptomod/Makefile
src/hed/libs/data/Makefile
src/hed/libs/data/cache-clean
src/hed/libs/data/cache-clean.1
src/hed/libs/data/cache-list
src/hed/libs/data/cache-list.1
src/hed/libs/data/test/Makefile
src/hed/libs/data/examples/Makefile
......
......@@ -19,17 +19,11 @@ use Getopt::Std;
use Fcntl ':mode';
use DirHandle;
use File::Basename;
use POSIX;
use strict;
use warnings;
BEGIN {
unshift @INC, dirname($0).'/@pkgdatadir_rel_to_pkglibexecdir@';
}
use ConfigCentral;
# Set the variable $File::Find::dont_use_nlink if you're using AFS,
# since AFS cheats.
......@@ -46,7 +40,7 @@ sub diskspace;
my(%opts);
my $configfile;
my $configfile = '/etc/arc.conf';
# max/min used percentage
my $maxusedpercent = 0;
......@@ -60,10 +54,6 @@ my $totlocksize = 0;
my $totlockfiles = 0;
my $debuglevel = 'INFO';
LogUtils::level($debuglevel);
LogUtils::timestamps(1);
my $log = LogUtils->getLogger('cache-clean');
my $currenttime = time();
getopts('hsSc:m:M:E:f:D:', \%opts);
......@@ -79,11 +69,9 @@ if(defined($opts{m})) {
}
if(defined($opts{D})) {
$debuglevel = $opts{D};
LogUtils::level($debuglevel);
}
if(defined($opts{s})) {
$debuglevel = 'ERROR';
LogUtils::level($debuglevel);
}
if($maxusedpercent < 0 || $maxusedpercent > 100) {
......@@ -135,7 +123,7 @@ sub usage {
Default is INFO
Caches are given by dir1, dir2.. or taken from the config file specified
by -c or ARC_CONFIG
by -c, ARC_CONFIG or the default /etc/arc.conf.
EOH
exit 1;
......@@ -147,43 +135,82 @@ if (!$configfile && $ENV{'ARC_CONFIG'} && -e $ENV{'ARC_CONFIG'}) {
$configfile = $ENV{'ARC_CONFIG'};
}
$log->info('Cache cleaning started');
my %loglevels = (FATAL => 0, ERROR => 1, WARNING => 2, INFO => 3, VERBOSE => 4, DEBUG => 5);
die "Bad debug level $debuglevel\n" unless $loglevels{$debuglevel};
my $loglevel = $loglevels{$debuglevel};
# Prints message with required format
sub p($) {
my $s = shift;
my $t = strftime "%F %T", localtime;
print STDERR "[$t] [cache-clean] $s\n";
}
# logging functions
sub error($) {
if ($loglevel >= 1) {
my $s = shift;
p("[ERROR] $s");
}
}
sub warning($) {
if ($loglevel >= 2) {
my $s = shift;
p("[WARNING] $s");
}
}
sub info($) {
if ($loglevel >= 3) {
my $s = shift;
p("[INFO] $s");
}
}
sub verbose($) {
if ($loglevel >= 4) {
my $s = shift;
p("[VERBOSE] $s");
}
}
info('Cache cleaning started');
my @caches = @ARGV;
if (!@caches) {
die "No config file found and no caches specified\n" unless $configfile;
my $config = ConfigCentral::parseConfig($configfile);
die "Failed parsing A-REX config file '$configfile'\n" unless $config;
die "No users set up in config file '$configfile'\n"
unless $config->{control} and ref $config->{control} eq 'HASH';
for my $control (values %{$config->{control}}) {
next unless ref $control eq 'HASH';
next unless $control->{cachedir} and ref $control->{cachedir} eq 'ARRAY';
for (@{$control->{cachedir}}) {
$log->warning("\n Warning: cache-clean cannot deal with substitutions - $_") and next if /%/;
$log->warning("\n Warning: ignoring malformed cache location - $_\n") and next unless m{^(/\S+)};
push @caches, $1;
die "No such configuration file $configfile\n" unless -e $configfile;
open CONFIG, $configfile;
my $cacheblock = 0;
while (my $line = <CONFIG>) {
if ($line =~ /^\[/) { $cacheblock = 0; }
if ($line =~ /^\[arex\/cache\]/) { $cacheblock = 1; }
next unless $cacheblock;
if ($line =~ /^cachedir/) {
$line =~ /\S+\s*=\s*([^\s]+)/;
my $cache = $1;
push @caches, $cache;
}
}
close CONFIG;
die "No caches found in config file '$configfile'\n" unless @caches;
}
# ConfigCentral sets debug level to level in conf file, so we have to reset it here
LogUtils::level($debuglevel);
foreach my $filesystem (@caches) {
$filesystem =~ s|/+$|| unless $filesystem eq "/";
next if $filesystem eq "";
if ($filesystem =~ /%/) {
$log->warning("$filesystem: Warning: cache-clean cannot deal with substitutions");
warning("$filesystem: Warning: cache-clean cannot deal with substitutions");
next;
}
if (! -d $filesystem || ! -d $filesystem."/data") {
$log->info("$filesystem: Cache is empty");
info("$filesystem: Cache is empty");
next;
}
......@@ -198,7 +225,7 @@ foreach my $filesystem (@caches) {
my $fsvalues = diskspace($filesystem);
if(!($fsvalues)) {
$log->warning("Unable to stat $filesystem");
warning("Unable to stat $filesystem");
next;
}
......@@ -206,14 +233,14 @@ foreach my $filesystem (@caches) {
my $fsused = $fsvalues->{used};
my $maxfbytes=$fssize*$maxusedpercent/100;
$log->info(join("", "$filesystem: used space ", printsize($fsused), " / ", printsize($fssize), " (", sprintf("%.2f",100*$fsused/$fssize), "%)"));
info(join("", "$filesystem: used space ", printsize($fsused), " / ", printsize($fssize), " (", sprintf("%.2f",100*$fsused/$fssize), "%)"));
if ($expirytime == 0 && $fsused < $maxfbytes && !$opts{'s'}) {
$log->info(join("", "Used space is lower than upper limit (", $maxusedpercent, "%)"));
info(join("", "Used space is lower than upper limit (", $maxusedpercent, "%)"));
next;
}
my $minfbytes=$fssize*$minusedpercent/100;
%files = ();
%expiredfiles = ();
$totsize = 0;
......@@ -284,13 +311,13 @@ foreach my $filesystem (@caches) {
my @lines = <FILE>;
close FILE;
my @values = split(' ', $lines[0]);
$log->verbose(join("", "Deleting expired file: $fil atime: ", scalar localtime($expiredfiles{$fil}{atime}), " size: $expiredfiles{$fil}{size} url: $values[0]"));
verbose(join("", "Deleting expired file: $fil atime: ", scalar localtime($expiredfiles{$fil}{atime}), " size: $expiredfiles{$fil}{size} url: $values[0]"));
}
else {
$log->verbose(join("", "Deleting expired file: $fil atime: ", scalar localtime($expiredfiles{$fil}{atime}), " size: $expiredfiles{$fil}{size}"));
verbose(join("", "Deleting expired file: $fil atime: ", scalar localtime($expiredfiles{$fil}{atime}), " size: $expiredfiles{$fil}{size}"));
}
} else {
$log->warning("Error deleting file '$fil': $!");
warning("Error deleting file '$fil': $!");
}
# not critical if this fails
......@@ -298,14 +325,14 @@ foreach my $filesystem (@caches) {
if ( unlink "$fil.meta" ) {
my $lastslash = rindex($fil, "/");
if ( rmdir(substr($fil, 0, $lastslash))) {
$log->verbose("Deleting directory ".substr($fil, 0, $lastslash));
verbose("Deleting directory ".substr($fil, 0, $lastslash));
}
} else {
$log->warning("Error deleting file '$fil.meta': $!");
warning("Error deleting file '$fil.meta': $!");
}
}
}
#Are we still exceding limit after deleting expired files.
#Are we still exceding limit after deleting expired files.
if ($fsused > $maxfbytes) {
# delete in order of access
foreach my $fil (sort { $files{$a}{atime} <=> $files{$b}{atime} } keys %files) {
......@@ -324,27 +351,27 @@ foreach my $filesystem (@caches) {
my $line = <FILE>;
close FILE;
chomp($line);
$log->verbose(join("", "Deleting file: $fil atime: ", scalar localtime($files{$fil}{atime}), " size: $files{$fil}{size} url: $line"));
verbose(join("", "Deleting file: $fil atime: ", scalar localtime($files{$fil}{atime}), " size: $files{$fil}{size} url: $line"));
}
else {
$log->verbose(join("","Deleting file: $fil atime: ", scalar localtime($files{$fil}{atime}), " size: $files{$fil}{size}"));
verbose(join("","Deleting file: $fil atime: ", scalar localtime($files{$fil}{atime}), " size: $files{$fil}{size}"));
}
} else {
$log->warning("Error deleting file '$fil': $!");
warning("Error deleting file '$fil': $!");
}
next if (! -e "$fil.meta");
# not critical if this fails
if ( unlink "$fil.meta" ) {
my $lastslash = rindex($fil, "/");
if ( rmdir(substr($fil, 0, $lastslash))) {
$log->verbose("Deleting directory ".substr($fil, 0, $lastslash));
verbose("Deleting directory ".substr($fil, 0, $lastslash));
}
} else {
$log->warning("Error deleting file '$fil.meta': $!");
warning("Error deleting file '$fil.meta': $!");
}
}
}
$log->info(join("", "Cleaning finished, used space now ", printsize($fsused), " / ", printsize($fssize), " (", sprintf("%.2f",100*$fsused/$fssize),"%)"));
info(join("", "Cleaning finished, used space now ", printsize($fsused), " / ", printsize($fssize), " (", sprintf("%.2f",100*$fsused/$fssize),"%)"));
}
exit 0;
......@@ -360,13 +387,13 @@ sub wanted {
return unless defined $atime;
return unless !(-d _) || -f _;
if ($links != 1) {
$totlocksize += 512 * $blocks;
$totlockfiles++;
return;
}
if (-e "$name.lock") {
# check if lock is still valid
my $lockatime = (stat("$name.lock"))[8];
......@@ -428,30 +455,30 @@ sub diskspace ($) {
my $spacecmd = $opts{f};
my @output= `$spacecmd $path`;
if ($? != 0) {
$log->warning("Failed running $spacecmd");
warning("Failed running $spacecmd");
} elsif ($output[0] =~ m/(\d+) (\d+)/) {
$disktotal = $1;
$diskused = $2;
} else {
$log->warning("Bad output from $spacecmd: @output");
warning("Bad output from $spacecmd: @output");
}
}
}
# check if on afs
elsif ($path =~ m#/afs/#) {
my @dfstring =`fs listquota $path 2>/dev/null`;
if ($? != 0) {
$log->warning("Failed running: fs listquota $path");
warning("Failed running: fs listquota $path");
} elsif ($dfstring[-1] =~ /\s+(\d+)\s+(\d+)\s+\d+%\s+\d+%/) {
$disktotal = $1*1024;
$diskused = $2*1024;
} else {
$log->warning("Failed interpreting output of: fs listquota $path");
warning("Failed interpreting output of: fs listquota $path");
}
# "ordinary" disk
} else {
my @dfstring =`df -k $path 2>/dev/null`;
if ($? != 0) {
$log->warning("Failed running: df -k $path");
warning("Failed running: df -k $path");
# The first column may be printed on a separate line.
# The relevant numbers are always on the last line.
......@@ -459,7 +486,7 @@ sub diskspace ($) {
$disktotal = $1*1024;
$diskused = $2*1024;
} else {
$log->warning("Failed interpreting output of: df -k $path");
warning("Failed interpreting output of: df -k $path");
}
}
# get actual used disk for caches on shared partitions if configured
......@@ -467,15 +494,15 @@ sub diskspace ($) {
$diskused = undef;
my @dustring =`du -ks $path 2>/dev/null`;
if ($? != 0) {
$log->warning("Failed running: du -ks $path");
warning("Failed running: du -ks $path");
} elsif ($dustring[-1] =~ /(\d+)\s+[\w\/]+/) {
$diskused = $1*1024;
} else {
$log->warning("Failed interpreting output of: du -ks $path");
warning("Failed interpreting output of: du -ks $path");
}
}
} else {
$log->warning("Not a directory: $path");
warning("Not a directory: $path");
}
return undef unless defined($disktotal) and defined($diskused);
......
......@@ -7,12 +7,8 @@ use File::Find ();
use File::Basename;
use Digest::SHA1 qw/sha1_hex/;
BEGIN {
unshift @INC, dirname($0).'/@pkgdatadir_rel_to_pkglibexecdir@';
}
use ConfigCentral;
use strict;
use warnings;
# for the convenience of &wanted calls, including -eval statements:
use vars qw/*name *dir *prune/;
......@@ -22,13 +18,10 @@ use vars qw/*name *dir *prune/;
sub wanted;
my $conffile;
sub usage {
print <<EOH;
usage: $0 [-c arex_config_file] [url1 [url2 [...]]]
ARC_CONFIG can also be used to specify a config file
ARC_CONFIG can also be used to specify a config file. Default is /etc/arc.conf
EOH
exit 1;
}
......@@ -47,27 +40,28 @@ my %ports = (
);
my @files;
my $conffile = "/etc/arc.conf";
if (@ARGV > 0) {
$file = $ARGV[0];
my $file = $ARGV[0];
if ($file eq '-h') {
usage();
usage();
}
if ($file eq '-c') {
shift (@ARGV);
$conffile = shift (@ARGV);
shift (@ARGV);
$conffile = shift (@ARGV);
}
# for each file add default port if necessary
foreach $file (@ARGV) {
push (@files, $file);
next if $file !~ m|(\w+)://(\S+?)/\S*|;
next if ! defined( $ports{$1} );
$protocol = $1;
$host = $2;
next if index($host, ':') != -1;
# no port so try the default
$file =~ s|$host|$host:$ports{$protocol}|;
push (@files, $file);
foreach my $f (@ARGV) {
push (@files, $f);
next if $f !~ m|(\w+)://(\S+?)/\S*|;
next if ! defined( $ports{$1} );
my $protocol = $1;
my $host = $2;
next if index($host, ':') != -1;
# no port so try the default
$f =~ s|$host|$host:$ports{$protocol}|;
push (@files, $f);
}
}
......@@ -80,46 +74,49 @@ usage() unless $conffile;
# parse to find cache dirs
my @caches;
my $config = ConfigCentral::parseConfig($conffile);
die "Failed parsing A-REX config file '$conffile'" unless $config;
die "No users set up in config file '$conffile'"
unless $config->{control} and ref $config->{control} eq 'HASH';
for my $control (values %{$config->{control}}) {
next unless ref $control eq 'HASH';
next unless $control->{cachedir} and ref $control->{cachedir} eq 'ARRAY';
for (@{$control->{cachedir}}) {
print "\n Warning: cache-list cannot deal with substitutions - $_\n" and next if /%/;
print "\n Warning: ignoring malformed cache location - $_\n" and next unless m{^(/\S+)};
push @caches, $1;
die "No config file found and no caches specified\n" unless $conffile;
die "No such configuration file $conffile\n" unless -e $conffile;
open CONFIG, $conffile;
my $cacheblock = 0;
while (my $line = <CONFIG>) {
if ($line =~ /^\[/) { $cacheblock = 0; }
if ($line =~ /^\[arex\/cache\]/) { $cacheblock = 1; }
next unless $cacheblock;
if ($line =~ /^cachedir/) {
$line =~ /\S+\s*=\s*([^\s]+)/;
my $cache = $1;
push @caches, $cache;
}
}
die "No caches found in config file '$conffile'" unless @caches;
close CONFIG;
die "No caches found in config file '$conffile'\n" unless @caches;
# list all files
if (@files == 0) {
foreach $cache (@caches) {
print "Cache: $cache\n";
if (! -d $cache || ! -d $cache."/data") { print " Cache is empty\n"; }
foreach my $cache (@caches) {
print "Cache: $cache\n";
if (! -d $cache || ! -d $cache."/data") { print " Cache is empty\n"; }
else { File::Find::find({wanted => \&wanted}, $cache."/data"); }
}
}
# list files given as arguments
else {
foreach $file (@files) {
$hash = sha1_hex($file);
if (length($hash) != 40) {
print "Error in hash calculation for file $file\n";
next;
}
# look for this file in the caches
foreach $cache (@caches) {
$cachefile = $cache.'/data/'.substr($hash, 0, 2).'/'.substr($hash, 2);
if (-e $cachefile) {
print " $file $cachefile";
print ' (locked)' if -e "$cachefile.lock";
print "\n";
}
}
foreach my $file (@files) {
my $hash = sha1_hex($file);
if (length($hash) != 40) {
print "Error in hash calculation for file $file\n";
next;
}
# look for this file in the caches
foreach my $cache (@caches) {
my $cachefile = $cache.'/data/'.substr($hash, 0, 2).'/'.substr($hash, 2);
if (-e $cachefile) {
print " $file $cachefile";
print ' (locked)' if -e "$cachefile.lock";
print "\n";
}
}
}
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment