Commit e90c986d authored by Maiken's avatar Maiken
Browse files

Add new file - jobid_collector.py

parent 5d47d267
#!/usr/bin/env python
"""
Can be used e.g. in statecallout in arc.conf
statecallout=FINISHED timeout=10,onfailure=pass <full-path-to-this-script> -i %I -o '/mnt'
"""
import glob, os, sys, re
import shutil
import argparse
import datetime
import json
from arc.utils import config
from arc.paths import *
class JobIDCollector():
def __init__(self):
self.parser = argparse.ArgumentParser()
self.parser.add_argument('-i', '--arcid', required=True, default=False, help='ARC ID')
self.parser.add_argument('-o', '--outputpath', required=True, default=False, help='Path for storing output - e.g. /scratch. The files with jobid information will then be found in /scratch/jobids.json and /scratch/jobids.txt')
#self.parser.parse_args(namespace=self)
self.args = self.parser.parse_args()
""" TO-DO fix for proper check of runtime and placement of arc.conf like in ControlCommon.py """
arcctl_runtime_config = ARC_RUN_DIR + '/arcctl.runtime.conf'
conf_f = '/etc/arc.conf'
config.parse_arc_conf(conf_f)
if arcctl_runtime_config is not None:
config.save_run_config(arcctl_runtime_config)
self.arcconfig = config
self.txtfile = self.args.outputpath + '/jobids.out'
self.jsonfile = self.args.outputpath + '/jobids.json'
# controldir is mandatory
self.control_dir = self.arcconfig.get_value('controldir', 'arex').rstrip('/')
if self.control_dir is None:
print('JobIDCollector is not possible without controldir.')
sys.exit(1)
def extract_jobids(self):
gramifile = self.control_dir + '/job.' + self.args.arcid + '.grami'
jobids = {}
jsondict = {}
pandaid = 'N/A'
lrmsid = 'N/A'
try:
with open(gramifile,'r') as this_file:
for line in this_file:
line = line.strip()
""" Experiment specific GTAG is used for ATLAS monitoring """
if 'GTAG' in line:
pandaid = (line.split('/')[-1]).split('.')[0]
elif 'joboption_jobid' in line:
lrmsid = line.split('=')[-1]
except:
print( 'Problem opening grami file. No slurm or pandaids will be available. Error was:\n' + str(sys.exc_info()))
print(pandaid, lrmsid,self.args.arcid)
jobids[self.args.arcid]={'lrmsid':lrmsid,'pandaid':pandaid}
if os.path.exists(self.jsonfile):
try:
with open(self.jsonfile) as f:
jsondict = json.load(f)
jsondict.update(jobids)
with open(self.jsonfile,'w') as f:
json.dump(jsondict,f)
except:
""" In case the json file is corrupted in some way - just make a new one """
now = datetime.datetime.now()
oldfile=self.jsonfile+'.'+str(now)
shutil.copyfile(self.jsonfile,oldfile)
with open(self.jsonfile,'w') as f:
json.dump(jobids,f)
else:
with open(self.jsonfile,'w') as f:
json.dump(jobids,f)
now = datetime.datetime.now()
now_str = now.strftime("%m/%d/%YT%H:%M:%S")
if not os.path.exists(self.txtfile):
with open(self.txtfile,'w') as f:
f.write('{0:<23}\t{1:<70}\t{2:10}\t{3:10}\n'.format('TIME','ARCID','LRMSID','PANDAID'))
with open(self.txtfile,'a+') as f:
for key,val in jobids.iteritems():
f.write('{0:<23}\t{1:<70}\t{2:10}\t{3:10}\n'.format(now_str,key,val['lrmsid'],val['pandaid']))
if __name__ == '__main__':
jobidcollector = JobIDCollector()
jobidcollector.extract_jobids()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment