Commit caae2cde authored by Maiken's avatar Maiken

Merge branch 'python-fixes-centos81' into 'master'

Fixes for CentOS 8 deployment

See merge request !939
parents 62d80682 a7e0209b
Pipeline #7421 passed with stages
in 143 minutes and 27 seconds
......@@ -2446,7 +2446,7 @@
## will cause errors.
## These entries will be shown in all GLUE2 AccessPolicy and MappingPolicy
## objects, that is, they will apply for all Endpoints(Interfaces) and all
## Shares(currently queues). You can also add additional advertisedvos to queues.
## Shares(currently queues). You can override the advertisedvos per queue.
## The information is also published in the NorduGrid schema.
## NOTE that it is IMPORTANT to understand that this parameter is NOT enforcing any
## access control, it is just for information publishing!
......@@ -2787,12 +2787,11 @@
## they will apply for the Shares that corresponds to the queue.
## The information is also published in the NorduGrid schema.
## NOTE that if you have also configured "advertisedvo" in the [infosys/cluster] block,
## the result advertised VOs per queue is the union set of what is contained in the
## [infosys/cluster] and in this [queue:name] block!
## the result advertised VOs per queue will override whatever is defined in [infosys/cluster] block!
## NOTE that it is IMPORTANT to understand that this parameter is NOT enforcing any
## access control, it is just for information publishing!
## multivalued
## default: undefined
## default: $VAR{[infosys/cluster]advertisedvo}
#advertisedvo=atlas
#advertisedvo=community.nordugrid.org
## CHANGE: renamed it as advertisedvo
......
......@@ -24,5 +24,5 @@ community_software_prepare () {
community_software_environment () {
# skip if this is not a community-defined RTE
[ -e "${rte_params_path}.community" ] || return
echo "RUNTIME_JOB_SWDIR=\"${joboption_directory}/${COMMUNITY_RTES_SW_SUBDIR}\""
echo "RUNTIME_JOB_SWDIR=\"\${RUNTIME_JOB_DIR}/${COMMUNITY_RTES_SW_SUBDIR}\""
}
......@@ -1806,9 +1806,9 @@ def _raw_dns_fetch(dnsdata, domain, nameserver=None):
if rdns['rdata']:
if isinstance(rdns['rdata'], list):
for rdata in rdns['rdata']:
rdns['config_data'].add(rdata.encode())
rdns['config_data'].add(rdata)
else:
rdns['config_data'].add(rdns['rdata'].encode())
rdns['config_data'].add(rdns['rdata'])
# fetch and construct server data
rdns['server_data'] = set()
resolve_name = domain
......@@ -1822,17 +1822,19 @@ def _raw_dns_fetch(dnsdata, domain, nameserver=None):
if not rrs.response.answer:
for rr in rrs.response.authority:
for lrr in rr.to_text().split('\n'):
rdns['server_data'].add(lrr.split(' ')[-1].encode())
rdns['server_data'].add(lrr.split(' ')[-1])
except dns.resolver.NXDOMAIN as e:
logger.debug('NXDOMAIN received for %s DNS query for domain %s.',
rdns['type'], resolve_name)
pass
except dns.resolver.NoAnswer as e:
logger.warning('No answer for %s DNS query for domain %s. Error: %s',
rdns['type'], resolve_name, str(e))
except dns.resolver.NoNameservers as e:
logger.warning('No namservers received for %s DNS query for domain %s. Error: %s',
rdns['type'], resolve_name, str(e))
else:
for rr in rrs:
rdns['server_data'].add(rr.to_text().encode())
rdns['server_data'].add(rr.to_text().strip('"'))
#
# HANDLE DDNS UPDATE
......
......@@ -84,13 +84,13 @@ class AccountingControl(ComponentControl):
s_ids += s['GLUE2ServiceID']
self.logger.debug('Running LDAP query over %s to find service endpoint URLs', args.top_bdii)
s_filter = reduce(lambda x, y: x + '(GLUE2EndpointServiceForeignKey={0})'.format(y), s_ids, '')
s_filter = reduce(lambda x, y: x + '(GLUE2EndpointServiceForeignKey={0})'.format(y.decode()), s_ids, '')
endpoints = ldap_conn.search_st('o=glue', ldap.SCOPE_SUBTREE, attrlist=['GLUE2EndpointURL'], timeout=30,
filterstr='(&(objectClass=Glue2Endpoint)(|{0}))'.format(s_filter))
for (_, e) in endpoints:
if 'GLUE2EndpointURL' in e:
for url in e['GLUE2EndpointURL']:
print(url.replace('stomp+ssl://', 'https://').replace('stomp://', 'http://'))
print(url.decode().replace('stomp+ssl://', 'https://').replace('stomp://', 'http://'))
ldap_conn.unbind()
except ldap.LDAPError as err:
self.logger.error('Failed to query Top-BDII %s. Error: %s.', args.top_bdii, err.message['desc'])
......@@ -461,6 +461,7 @@ class AccountingControl(ComponentControl):
accounting_actions = job_accounting_ctl.add_subparsers(title='Job Accounting Actions', dest='jobaction',
metavar='ACTION', help='DESCRIPTION')
accounting_actions.required = True
accounting_job = accounting_actions.add_parser('info', help='Show job accounting data')
accounting_job.add_argument('-o', '--output', default='all',
......@@ -482,6 +483,7 @@ class AccountingControl(ComponentControl):
accounting_actions = accounting_ctl.add_subparsers(title='Accounting Actions', dest='action',
metavar='ACTION', help='DESCRIPTION')
accounting_actions.required = True
# add legacy accounting control as a sub-parser
LegacyAccountingControl.register_parser(accounting_actions)
......
......@@ -207,6 +207,7 @@ class LegacyAccountingControl(ComponentControl):
accounting_actions = accounting_ctl.add_subparsers(title='Legacy Accounting Actions', dest='legacyaction',
metavar='ACTION', help='DESCRIPTION')
accounting_actions.required = True
# republish
accounting_republish = accounting_actions.add_parser('republish',
......
......@@ -100,6 +100,7 @@ class CacheControl(ComponentControl):
cache_actions = cache_ctl.add_subparsers(title='A-REX Cache Actions', dest='action',
metavar='ACTION', help='DESCRIPTION')
cache_actions.required = True
cache_stats = cache_actions.add_parser('stats', help='Show cache usage statistics')
......
......@@ -279,7 +279,7 @@ class CommunityRTEControl(ComponentControl):
self.logger.debug('Fetching data from %s', url)
req = Request(url)
try:
with open(path, 'w') as dest_f:
with open(path, 'wb') as dest_f:
url_data = urlopen(req)
data_len = 0
if 'content-length' in url_data.headers:
......@@ -317,7 +317,7 @@ class CommunityRTEControl(ComponentControl):
self.logger.error('Failed to fetch community public key.')
return None
keyfile = os.path.join(cdir, 'pubkey.gpg')
with open(keyfile, 'w') as key_f:
with open(keyfile, 'wb') as key_f:
key_f.write(keydata)
return keyfile
......@@ -330,7 +330,10 @@ class CommunityRTEControl(ComponentControl):
def __ask_yes_no(self, question, default_yes=False):
"""Interactively ask for confirmation"""
yes_no = ' (YES/no): ' if default_yes else ' (yes/NO): '
try:
reply = str(raw_input(question + yes_no)).lower().strip()
except NameError:
reply = str(input(question + yes_no)).lower().strip()
if reply == 'yes':
return True
if reply == 'no':
......@@ -373,7 +376,7 @@ class CommunityRTEControl(ComponentControl):
try:
decoded_keydata = base64.b64decode(key_data['pubkey']['keydata'])
keyfile = os.path.join(cdir, 'pubkey.gpg')
with open(keyfile, 'w') as key_f:
with open(keyfile, 'wb') as key_f:
key_f.write(decoded_keydata)
key_data['pubkey']['keyfile'] = keyfile
except TypeError as e:
......@@ -536,18 +539,16 @@ class CommunityRTEControl(ComponentControl):
self.__cdir_cleanup(c, gpgproc.returncode)
# verify fingerprint
gpgcmd = ['gpg', '--homedir', gpgdir, '--fingerprint']
gpgcmd = ['gpg', '--with-colons', '--homedir', gpgdir, '--fingerprint']
gpgproc = subprocess.Popen(gpgcmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
__fingerprint_re = re.compile(r'^\s*Key fingerprint =\s*(.*)\s*$')
vfp = None
all_messages = []
keyuid = ''
for rline in iter(gpgproc.stdout.readline, b''):
line = rline.decode('utf-8').rstrip()
all_messages.append(line)
fp_match = __fingerprint_re.match(line)
if fp_match:
vfp = fp_match.group(1)
vfp = vfp.replace(' ', '').upper()
if line.startswith('fpr:'):
vfp = line.split(':')[-2].upper()
elif line.startswith('uid:'):
keyuid = line.split(':')[9]
gpgproc.wait()
if gpgproc.returncode == 0:
if vfp is None:
......@@ -565,8 +566,8 @@ class CommunityRTEControl(ComponentControl):
else:
# fingerprint should be confirmed interactively
print('The imported community public key data is:')
for m in all_messages[2:]:
print(' ' + m)
print(' ' + keyuid)
print(' Fingerprint: ' + ' '.join([vfp[i:i+4] for i in range(0, len(vfp), 4)]))
if not self.__ask_yes_no('Is the community key fingerprint correct?'):
self.logger.info('Removing community %s from trusted.', c)
self.__cdir_cleanup(c)
......@@ -775,7 +776,7 @@ class CommunityRTEControl(ComponentControl):
(gpg_stdout, gpg_stderr) = gpgproc.communicate(input=sigrtedata)
if gpgproc.returncode == 0:
self.logger.info('Showing the content of RTE %s fetched from the registry.', rtename)
print(gpg_stdout)
print(gpg_stdout.decode())
else:
self.logger.error('Failed to decrypt signed RTE %s. GPG errors are: %s', rtename, gpg_stderr)
sys.exit(1)
......@@ -825,7 +826,7 @@ class CommunityRTEControl(ComponentControl):
if not os.path.exists(signed_rtes):
os.makedirs(signed_rtes, mode=0o755)
signed_rte_file = os.path.join(signed_rtes, rtename.replace('/', '-') + '.signed')
with open(signed_rte_file, 'w') as srte_f:
with open(signed_rte_file, 'wb') as srte_f:
srte_f.write(sigrtedata)
# verify
......@@ -925,7 +926,7 @@ class CommunityRTEControl(ComponentControl):
if 'checksum_type' in d:
try:
h = hashlib.new(d['checksum_type'])
with open(swfile, 'r') as h_f:
with open(swfile, 'rb') as h_f:
while True:
buf = h_f.read(4096)
if not buf:
......@@ -1065,6 +1066,7 @@ class CommunityRTEControl(ComponentControl):
crte_actions = crte_ctl.add_subparsers(title='Community RTE Actions', dest='communityaction',
metavar='ACTION', help='DESCRIPTION')
crte_actions.required = True
cadd = crte_actions.add_parser('add', help='Add new trusted community to ARC CE')
cadd.add_argument('-f', '--fingerprint', help='Fingerprint of the community key', action='store')
......
......@@ -137,6 +137,7 @@ class ConfigControl(ComponentControl):
config_actions = config_ctl.add_subparsers(title='Config Actions', dest='action',
metavar='ACTION', help='DESCRIPTION')
config_actions.required = True
config_dump = config_actions.add_parser('dump', help='Dump ARC CE running configuration')
......
from __future__ import absolute_import
import sys
import os
import logging
import datetime
import argparse
......@@ -93,6 +94,7 @@ def get_parsed_arcconf(conf_f):
'Using /etc/arc.conf that exists.', def_conf_f)
runconf_load = True
else:
if arcctl_ce_mode():
logger.error('Cannot find ARC configuration file in the default location.')
return None
......
......@@ -476,6 +476,8 @@ class JobsControl(ComponentControl):
jobs_actions = jobs_ctl.add_subparsers(title='Jobs Control Actions', dest='action',
metavar='ACTION', help='DESCRIPTION')
jobs_actions.required = True
jobs_list = jobs_actions.add_parser('list', help='List available A-REX jobs')
jobs_list.add_argument('-l', '--long', help='Detailed listing of jobs', action='store_true')
jobs_list.add_argument('-s', '--state', help='Filter jobs by state', action='append', choices=__JOB_STATES)
......
......@@ -616,6 +616,7 @@ class RTEControl(ComponentControl):
rte_actions = rte_ctl.add_subparsers(title='RunTime Environments Actions', dest='action',
metavar='ACTION', help='DESCRIPTION')
rte_actions.required = True
rte_enable = rte_actions.add_parser('enable', help='Enable RTE to be used by A-REX')
rte_enable.add_argument('rte', nargs='+', help='RTE name').completer = complete_rte_name
......
......@@ -247,6 +247,7 @@ class ServicesControl(ComponentControl):
services_actions = services_ctl.add_subparsers(title='Services Actions', dest='action',
metavar='ACTION', help='DESCRIPTION')
services_actions.required = True
services_enable = services_actions.add_parser('enable', help='Enable ARC CE services')
services_enable.add_argument('--now', help='Start the services just after enable', action='store_true')
......
......@@ -298,6 +298,7 @@ class TestCAControl(ComponentControl):
testca_actions = testca_ctl.add_subparsers(title='Test CA Actions', dest='action',
metavar='ACTION', help='DESCRIPTION')
testca_actions.required = True
testca_init = testca_actions.add_parser('init', help='Generate self-signed TestCA files')
add_parser_digest_validity(testca_init)
......
......@@ -183,13 +183,15 @@ class ThirdPartyControl(ComponentControl):
break
return vomses
def __get_ssl_cert_openssl(self, url):
def __get_ssl_cert_openssl(self, url, compat=False):
# parse connection parameters
(hostname, port) = self.__get_socket_from_url(url)
# try to connect using openssl
try:
s_client = subprocess.Popen(['openssl', 's_client', '-connect'] + ['{0}:{1}'.format(hostname, port)],
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
cmd = ['openssl', 's_client', '-connect'] + ['{0}:{1}'.format(hostname, port)]
if compat:
cmd += ['-nameopt', 'compat']
s_client = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
dn = None
ca = None
for line in iter(s_client.stdout.readline, ''):
......@@ -200,7 +202,11 @@ class ThirdPartyControl(ComponentControl):
ca = line.replace('issuer=', '')
break
if dn and ca:
if dn.startswith('/'):
return {hostname: {'dn': dn.rstrip(), 'ca': ca.rstrip()}}
elif not compat:
self.logger.debug('Seams we are on the newer OpenSSL version, retrying with compat DN output')
return self.__get_ssl_cert_openssl(url, compat=True)
self.logger.error('Failed to get DN and CA with OpenSSL SSL/TLS bind to %s:%s.', hostname, port)
sys.exit(1)
except OSError:
......@@ -495,6 +501,7 @@ deb http://dist.eugridpma.info/distribution/igtf/current igtf accredited
deploy_actions = deploy_ctl.add_subparsers(title='Deployment Actions', dest='action',
metavar='ACTION', help='DESCRIPTION')
deploy_actions.required = True
igtf_ca = deploy_actions.add_parser('igtf-ca', help='Deploy IGTF CA certificates')
igtf_ca.add_argument('bundle', help='IGTF CA bundle name', nargs='+',
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment