Commit f5927fe4 authored by David Cameron's avatar David Cameron

implement session dir cleaning in common function

parent a1d6fd13
Pipeline #5966 passed with stage
in 58 minutes and 39 seconds
......@@ -453,6 +453,56 @@ bool DirDelete(const std::string& path, bool recursive) {
return true;
}
bool DirDeleteExcl(const std::string& path, const std::list<std::string>& files, bool excl) {
return DirDeleteExcl(path, files, excl, 0, 0);
}
bool DirDeleteExcl(const std::string& path, const std::list<std::string>& files, bool excl, uid_t uid, gid_t gid) {
// First check special case of "/" in files (keep or delete everything)
for (std::list<std::string>::const_iterator f = files.begin(); f != files.end(); ++f) {
if (*f == "/") {
if (excl) return true; // delete nothing
return DirDelete(path, true, uid, gid); // delete everything
}
}
std::list<std::string> dirlisting;
if (!DirList(path, dirlisting, false, uid, gid)) return false;
for (std::list<std::string>::const_iterator d = dirlisting.begin(); d != dirlisting.end(); ++d) {
// Check for file or dir
struct stat st;
if (!FileStat(*d, &st, uid, gid, false)) return false;
if (S_ISDIR(st.st_mode)) {
// Check for any files in this dir
std::list<std::string> newfiles;
for (std::list<std::string>::const_iterator f = files.begin(); f != files.end(); ++f) {
std::string fullpath(path + '/' + *f);
if (fullpath.substr(0, d->size()) == *d && fullpath.size() > d->size() && fullpath[d->size()] == '/') {
newfiles.push_back(f->substr(f->find('/')+1));
}
}
if (!newfiles.empty()) {
if (!DirDeleteExcl(*d, newfiles, excl, uid, gid)) return false;
if (excl) continue;
}
}
bool del = excl;
for (std::list<std::string>::const_iterator f = files.begin(); f != files.end(); ++f) {
std::string fullpath(path + '/' + *f);
if (fullpath == *d) {
del = !del;
break;
}
}
if (del) {
if (S_ISDIR(st.st_mode)) DirDelete(*d, true);
else FileDelete(*d);
}
}
return true;
}
static bool list_recursive(FileAccess* fa,const std::string& path,std::list<std::string>& entries,bool recursive) {
std::string curpath = path;
while (curpath.rfind('/') == curpath.length()-1) curpath.erase(curpath.length()-1);
......
......@@ -99,6 +99,18 @@ namespace Arc {
Specified uid and gid are used for accessing filesystem. */
bool DirDelete(const std::string& path, bool recursive, uid_t uid, gid_t gid);
/// Delete a directory, including or excluding certain files
/** If excl is true, all files except those specified in files will be
deleted. If excl is false, only the files in files will be deleted.
files is a list of paths relative to path.*/
bool DirDeleteExcl(const std::string& path, const std::list<std::string>& files, bool excl);
/// Delete a directory, including or excluding certain files, using the specified uid and gid
/** If excl is true, all files except those specified in files will be
deleted. If excl is false, only the files in files will be deleted.
files is a list of paths relative to path.*/
bool DirDeleteExcl(const std::string& path, const std::list<std::string>& files, bool excl, uid_t uid, gid_t gid);
/// List all entries in a directory.
/** If path is not a directory or cannot be listed then false is returned. On
success entries is filled with the list of entries in the directory. The
......
......@@ -131,20 +131,54 @@ void FileUtilsTest::TestFileCreateAndRead() {
void FileUtilsTest::TestMakeAndDeleteDir() {
// create a few subdirs and files then recursively delete
struct stat st;
std::string file1("file1");
std::string file2("file2");
std::string file3("dir1"+sep+"file3");
std::string file4("dir1"+sep+"dir3"+sep+"dir4"+sep+"file4");
std::string dir1("dir1");
std::string dir2("dir1"+sep+"dir2");
std::string dir3("dir1"+sep+"dir3"+sep+"dir4");
std::string link1("dir1"+sep+"dir2"+sep+"link1");
CPPUNIT_ASSERT(stat(testroot.c_str(), &st) == 0);
CPPUNIT_ASSERT(_createFile(testroot+sep+"file1"));
CPPUNIT_ASSERT(Arc::DirCreate(std::string(testroot+sep+"dir1"), S_IRUSR | S_IWUSR | S_IXUSR));
CPPUNIT_ASSERT(Arc::DirCreate(std::string(testroot+sep+"dir1"), S_IRUSR | S_IWUSR | S_IXUSR));
CPPUNIT_ASSERT(stat(std::string(testroot+sep+"dir1").c_str(), &st) == 0);
CPPUNIT_ASSERT(_createFile(testroot+sep+file1));
CPPUNIT_ASSERT(Arc::DirCreate(std::string(testroot+sep+dir1), S_IRUSR | S_IWUSR | S_IXUSR));
CPPUNIT_ASSERT(Arc::DirCreate(std::string(testroot+sep+dir1), S_IRUSR | S_IWUSR | S_IXUSR));
CPPUNIT_ASSERT(stat(std::string(testroot+sep+dir1).c_str(), &st) == 0);
CPPUNIT_ASSERT(S_ISDIR(st.st_mode));
CPPUNIT_ASSERT(_createFile(testroot+sep+"dir1"+sep+"file2"));
CPPUNIT_ASSERT(_createFile(testroot+sep+file2));
CPPUNIT_ASSERT(_createFile(testroot+sep+file3));
// should fail if with_parents is set to false
CPPUNIT_ASSERT(!Arc::DirCreate(std::string(testroot+sep+"dir1"+sep+"dir2"+sep+"dir3"), S_IRUSR | S_IWUSR | S_IXUSR, false));
CPPUNIT_ASSERT(Arc::DirCreate(std::string(testroot+sep+"dir1"+sep+"dir2"+sep+"dir3"), S_IRUSR | S_IWUSR | S_IXUSR, true));
CPPUNIT_ASSERT(stat(std::string(testroot+sep+"dir1"+sep+"dir2"+sep+"dir3").c_str(), &st) == 0);
CPPUNIT_ASSERT(!Arc::DirCreate(std::string(testroot+sep+dir3), S_IRUSR | S_IWUSR | S_IXUSR, false));
CPPUNIT_ASSERT(Arc::DirCreate(std::string(testroot+sep+dir3), S_IRUSR | S_IWUSR | S_IXUSR, true));
CPPUNIT_ASSERT(stat(std::string(testroot+sep+dir3).c_str(), &st) == 0);
CPPUNIT_ASSERT(S_ISDIR(st.st_mode));
CPPUNIT_ASSERT(_createFile(testroot+sep+"dir1"+sep+"dir2"+sep+"dir3"+sep+"file4"));
CPPUNIT_ASSERT(symlink(std::string(testroot+sep+"dir1"+sep+"dir2").c_str(), std::string(testroot+sep+"dir1"+sep+"dir2"+sep+"link1").c_str()) == 0);
CPPUNIT_ASSERT(_createFile(testroot+sep+file4));
CPPUNIT_ASSERT(Arc::DirCreate(std::string(testroot+sep+dir2), S_IRUSR | S_IWUSR | S_IXUSR));
CPPUNIT_ASSERT(symlink(std::string(testroot+sep+file3).c_str(), std::string(testroot+sep+link1).c_str()) == 0);
std::list<std::string> files;
files.push_back("/");
CPPUNIT_ASSERT(Arc::DirDeleteExcl(testroot, files, true));
CPPUNIT_ASSERT(stat(std::string(testroot+sep+dir2).c_str(), &st) == 0);
CPPUNIT_ASSERT(stat(std::string(testroot+sep+link1).c_str(), &st) == 0);
files.clear();
files.push_back(file1);
files.push_back(dir2);
// Delete everything except file1 and dir2
CPPUNIT_ASSERT(Arc::DirDeleteExcl(testroot, files, true));
CPPUNIT_ASSERT(stat(std::string(testroot+sep+dir3).c_str(), &st) != 0);
CPPUNIT_ASSERT(stat(std::string(testroot+sep+dir2).c_str(), &st) == 0);
CPPUNIT_ASSERT(stat(std::string(testroot+sep+link1).c_str(), &st) != 0);
CPPUNIT_ASSERT(stat(std::string(testroot+sep+file1).c_str(), &st) == 0);
CPPUNIT_ASSERT(stat(std::string(testroot+sep+file2).c_str(), &st) != 0);
files.pop_back();
// Delete only file 1
CPPUNIT_ASSERT(Arc::DirDeleteExcl(testroot, files, false));
CPPUNIT_ASSERT(stat(std::string(testroot+sep+dir2).c_str(), &st) == 0);
CPPUNIT_ASSERT(stat(std::string(testroot+sep+file1).c_str(), &st) != 0);
CPPUNIT_ASSERT(!Arc::DirDelete(testroot, false));
CPPUNIT_ASSERT(Arc::DirDelete(testroot, true));
......
......@@ -4,10 +4,11 @@ else
MYSQL_WRAPPER_TEST =
endif
TESTS = URLTest LoggerTest RunTest XMLNodeTest FileAccessTest FileUtilsTest \
ProfileTest ArcRegexTest FileLockTest EnvTest UserConfigTest \
StringConvTest CheckSumTest WatchdogTest UserTest $(MYSQL_WRAPPER_TEST) \
Base64Test
TESTS = FileUtilsTest
#TESTS = URLTest LoggerTest RunTest XMLNodeTest FileAccessTest FileUtilsTest \
# ProfileTest ArcRegexTest FileLockTest EnvTest UserConfigTest \
# StringConvTest CheckSumTest WatchdogTest UserTest $(MYSQL_WRAPPER_TEST) \
# Base64Test
check_PROGRAMS = $(TESTS) ThreadTest
......
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <cstring>
#include <cstdio>
#include <cstdlib>
#include <string>
#include <glibmm.h>
#include <arc/FileUtils.h>
#include "../files/ControlFileContent.h"
#include "Delete.h"
namespace ARex {
struct FL_p {
const char* s;
FL_p* next;
FL_p* prev;
};
/* return values: 0 - empty, 1 - has files, 2 - failed */
static int delete_all_recur(const std::string &dir_base,
const std::string &dir_cur,
FL_p **fl_list,bool excl,
uid_t uid,gid_t gid) {
/* take corresponding members of fl_list */
FL_p* fl_new = NULL; /* new list */
FL_p* fl_cur = (*fl_list); /* pointer in old list */
int l = dir_cur.length();
/* extract suitable files from list */
for(;;) {
if(fl_cur == NULL) break;
FL_p* tmp = fl_cur->next;
if(!strncmp(fl_cur->s,dir_cur.c_str(),l)) {
if(fl_cur->s[l] == '/') {
/* remove from list */
if(fl_cur->prev == NULL) { (*fl_list)=fl_cur->next; }
else { fl_cur->prev->next=fl_cur->next; };
if(fl_cur->next == NULL) { }
else { fl_cur->next->prev=fl_cur->prev; };
/* add to list */
fl_cur->prev=NULL; fl_cur->next=fl_new;
if(fl_new == NULL) { fl_new=fl_cur; }
else { fl_new->prev = fl_cur; fl_new=fl_cur; };
};
};
fl_cur=tmp;
};
/* go through directory and remove files */
std::string file;
std::string dir_s = dir_base+dir_cur;
int files = 0;
try {
Glib::Dir dir(dir_s);
for(;;) {
file=dir.read_name();
if(file.empty()) break;
if(file == ".") continue;
if(file == "..") continue;
fl_cur = fl_new;
for(;;) {
if(fl_cur == NULL) break;
if(!strcmp(file.c_str(),(fl_cur->s)+(l+1))) {
/* do not delete or delete */
break;
};
fl_cur=fl_cur->next;
};
if(excl) {
if(fl_cur == NULL) {
/* delete */
struct stat f_st;
std::string fname=dir_s+'/'+file;
if(!Arc::FileStat(fname.c_str(),&f_st,uid,gid,false)) { files++; }
else if(S_ISDIR(f_st.st_mode)) {
if(delete_all_recur(dir_base,
dir_cur+'/'+file,&fl_new,excl,uid,gid) != 0) {
files++;
}
else {
if(!Arc::DirDelete(fname, false, uid, gid)) { files++; };
};
}
else {
if(!Arc::FileDelete(fname, uid, gid)) { files++; };
};
}
else { files++; };
}
else {
struct stat f_st;
std::string fname=dir_s+'/'+file;
if(!Arc::FileStat(fname.c_str(),&f_st,uid,gid,false)) { files++; }
else if(S_ISDIR(f_st.st_mode)) {
if(fl_cur != NULL) { /* MUST delete it */
if(!Arc::DirDelete(fname, true, uid, gid)) { files++; };
}
else { /* CAN delete if empty, and maybe files inside */
if(delete_all_recur(dir_base,
dir_cur+'/'+file,&fl_new,excl,uid,gid) != 0) {
files++;
}
else {
if(!Arc::DirDelete(fname, false, uid, gid)) { files++; };
};
};
}
else {
if(fl_cur != NULL) { /* MUST delete this file */
if(!Arc::FileDelete(fname, uid, gid)) { files++; };
}
else { files++; };
};
};
};
} catch(Glib::FileError& e) { return 2; };
if(files) return 1;
return 0;
}
/* filenames should start from / and not to have / at end */
int delete_all_files(const std::string &dir_base,const std::list<FileData> &files,
bool excl,uid_t uid, gid_t gid) {
int n = files.size();
FL_p* fl_list = NULL;
if(n != 0) {
if((fl_list=(FL_p*)malloc(sizeof(FL_p)*n)) == NULL) { return 2; };
std::list<FileData>::const_iterator file=files.begin();
// fl_list[0].s=file->pfn.c_str();
int i;
for(i=0;i<n;) {
if(excl || file->lfn.find(':') != std::string::npos) {
if(excl) {
if(file->pfn == "/") { /* keep all requested */
free(fl_list); return 0;
};
};
fl_list[i].s=file->pfn.c_str();
if(i) { fl_list[i].prev=fl_list+(i-1); fl_list[i-1].next=fl_list+i; }
else { fl_list[i].prev=NULL; };
fl_list[i].next=NULL;
i++;
};
++file; if(file == files.end()) break;
};
if(i==0) { free(fl_list); fl_list=NULL; };
};
std::string dir_cur("");
FL_p* fl_list_tmp = fl_list;
int res=delete_all_recur(dir_base,dir_cur,&fl_list_tmp,excl,uid,gid);
if(fl_list) free(fl_list);
return res;
}
} // namespace ARex
#ifndef __ARC_GM_DELETE_H__
#define __ARC_GM_DELETE_H__
#include <string>
#include <list>
#include "../files/ControlFileContent.h"
namespace ARex {
/**
Delete all files and subdirectories in 'dir_base' which are or are not
present in 'files' list.
Accepts:
dir_base - path to directory.
files - list of files to delete/keep. Paths are relative to 'dir_base'.
excl - if set to true all files excluding those in 'files' will be
deleted. Otherwise - only files in 'files' which have LFN information
will be deleted. If some of 'files' correspond to directories - whole
directory will be deleted.
uid - uid under which to perform file system operations
gid - gid under which to perform file system operations
*/
int delete_all_files(const std::string &dir_base, const std::list<FileData> &files,
bool excl, uid_t uid = 0, gid_t gid = 0);
} // namespace ARex
#endif
noinst_LTLIBRARIES = libfiles.la
libfiles_la_SOURCES = \
Delete.cpp ControlFileHandling.cpp ControlFileContent.cpp JobLogFile.cpp \
Delete.h ControlFileHandling.h ControlFileContent.h JobLogFile.h
ControlFileHandling.cpp ControlFileContent.cpp JobLogFile.cpp \
ControlFileHandling.h ControlFileContent.h JobLogFile.h
libfiles_la_CXXFLAGS = -I$(top_srcdir)/include \
$(LIBXML2_CFLAGS) $(GLIBMM_CFLAGS) $(AM_CXXFLAGS)
......@@ -9,7 +9,6 @@
#include <arc/FileAccess.h>
#include <arc/data/FileCache.h>
#include "../files/Delete.h"
#include "../conf/UrlMapConfig.h"
#include "../files/ControlFileHandling.h"
#include "../conf/StagingConfig.h"
......@@ -53,6 +52,10 @@ bool compare_job_description(GMJobRef const& first, GMJobRef const& second) {
return priority_first > priority_second;
}
std::string filedata_pfn(FileData const& fd) {
return fd.pfn;
}
void DTRGenerator::main_thread(void* arg) {
((DTRGenerator*)arg)->thread();
}
......@@ -136,14 +139,14 @@ void DTRGenerator::thread() {
}
DTRGenerator::DTRGenerator(const GMConfig& config, JobsList& jobs) :
jobs_received(JobsList::ProcessingQueuePriority+1, "DTR received", *this),
jobs_processing(JobsList::ProcessingQueuePriority+2, "DTR processing", *this),
generator_state(DataStaging::INITIATED),
config(config),
central_dtr_log(NULL),
staging_conf(config),
info(config),
jobs(jobs),
jobs_received(JobsList::ProcessingQueuePriority+1, "DTR received", *this),
jobs_processing(JobsList::ProcessingQueuePriority+2, "DTR processing", *this) {
jobs(jobs) {
if (!staging_conf) return;
// Set log level for DTR in job.id.errors files
......@@ -585,8 +588,11 @@ bool DTRGenerator::processReceivedDTR(DataStaging::DTR_ptr dtr) {
}
}
}
if (delete_all_files(job->SessionDir(), files, true, job_uid, job_gid) == 2)
std::list<std::string> tokeep;
std::transform(files.begin(), files.end(), tokeep.begin(), filedata_pfn);
if (!Arc::DirDeleteExcl(job->SessionDir(), tokeep, true, job_uid, job_gid)) {
logger.msg(Arc::WARNING, "%s: Failed to clean up session dir", jobid);
}
}
// clean up cache joblinks
CleanCacheJobLinks(config, job);
......@@ -596,8 +602,13 @@ bool DTRGenerator::processReceivedDTR(DataStaging::DTR_ptr dtr) {
std::list<FileData> files;
if (!job_input_read_file(jobid, config, files))
logger.msg(Arc::WARNING, "%s: Failed to read list of input files, can't clean up session dir", jobid);
else if (delete_all_files(job->SessionDir(), files, false, job_uid, job_gid) == 2)
logger.msg(Arc::WARNING, "%s: Failed to clean up session dir", jobid);
else {
std::list<std::string> todelete;
std::transform(files.begin(), files.end(), todelete.begin(), filedata_pfn);
if (!Arc::DirDeleteExcl(job->SessionDir(), todelete, false, job_uid, job_gid)) {
logger.msg(Arc::WARNING, "%s: Failed to clean up session dir", jobid);
}
}
}
// add to finished jobs (without overwriting existing error) and finally
// remove from active
......@@ -709,7 +720,9 @@ bool DTRGenerator::processReceivedJob(GMJobRef& job) {
}
}
// pre-clean session dir before downloading
if (delete_all_files(job->SessionDir(), files, false, job_uid, job_gid) == 2) {
std::list<std::string> todelete;
std::transform(files.begin(), files.end(), todelete.begin(), filedata_pfn);
if (Arc::DirDeleteExcl(job->SessionDir(), todelete, false, job_uid, job_gid)) {
logger.msg(Arc::ERROR, "%s: Failed to clean up session dir", jobid);
dtrs_lock.lock();
finished_jobs[jobid] = std::string("Failed to clean up session dir before downloading inputs");
......@@ -787,7 +800,7 @@ bool DTRGenerator::processReceivedJob(GMJobRef& job) {
it = files.erase(it);
continue;
}
// Remove trailing slashes otherwise it will be cleaned in delete_all_files
// Remove trailing slashes otherwise it will be cleaned in DirDeleteExcl
std::string::size_type pos = it->pfn.find_last_not_of('/');
it->pfn.resize((pos == std::string::npos)?1:(pos+1));
}
......@@ -829,7 +842,9 @@ bool DTRGenerator::processReceivedJob(GMJobRef& job) {
break;
}
// pre-clean session dir before uploading
if (delete_all_files(job->SessionDir(), files, true, job_uid, job_gid) == 2) {
std::list<std::string> tokeep;
std::transform(files.begin(), files.end(), tokeep.begin(), filedata_pfn);
if (Arc::DirDeleteExcl(job->SessionDir(), tokeep, true, job_uid, job_gid)) {
logger.msg(Arc::ERROR, "%s: Failed to clean up session dir", jobid);
dtrs_lock.lock();
finished_jobs[jobid] = std::string("Failed to clean up session dir before uploading outputs");
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment