Commit d40c777e authored by Maiken's avatar Maiken
Browse files

In middle or rewrite from ansible to python only with jinja template.

parent 4c2553f0
......@@ -2,245 +2,291 @@
import os, sys, re
import subprocess
import argparse
from jinja2 import Environment, FileSystemLoader
import yaml
class ConfigureARC:
def __init__(self):
pass
self.create_host_cert = False
self.descr = {}
self.descr['host_cert'] = 'hostcert path'
self.descr['host_key'] = 'hostkey path'
self.descr['hostname'] = ''
self.descr['conf_path'] = 'arc.conf path'
self.descr['logdir'] = 'logfile path'
self.descr['controldir'] = 'controldir'
self.descr['sessiondir'] = 'sessiondir'
self.descr['grid_security_path'] = 'grid-security folder'
def check_for_ansible(self):
self.descr['lrms_type'] = 'lrms type'
self.descr['lrms_path'] = 'lrms bin path'
req = 'ansible'
proc = subprocess.Popen('which ' + req, stdout=subprocess.PIPE, stderr=subprocess.PIPE,shell=True)
stdout, stderr = proc.communicate()
if 'which: no ' + req +' in' in stderr or not stdout:
print('===> Ansible is required, and I can not find ansible in your path. Please install ansible and try again.')
return False
return True
self.descr['grid_user'] = 'linux grid user'
self.descr['grid_group'] = 'linux grid group'
def true_false(self,ask):
_in = str(raw_input('{0:<80}'.format(ask)).lower())
if 'n' in _in:
return False
else:
return True
self.descr['enable_emies'] = 'enable emies submission'
self.descr['enable_gridftpd'] = 'enable gridftpd submission'
def text_answ(self,ask):
return str(raw_input('{0:<80}'.format(ask)).lower())
def configure(self):
create_host_cert = True
host_cert = ''
host_key = ''
hostname = ''
self.settings = {}
self.settings['host_cert'] = '/etc/grid-security/hostcert.pem'
self.settings['host_key'] = '/etc/grid-security/hostkey.pem'
self.settings['hostname'] = ''
conf_path = '/etc'
logdir = '/var/log/arc'
grid_security_path = '/etc/grid-security'
lrms_path = '/usr/bin'
grid_user = 'griduser'
grid_group = 'grid'
controldir = '/var/spool/arc/control'
sessiondir = '/var/spool/arc/session'
enable_emies = False
enable_gridftpd = False
use_all_defaults = False
self.settings['conf_path'] = '/etc'
self.settings['logdir'] = '/var/log/arc'
self.settings['controldir'] = '/var/spool/arc/control'
self.settings['sessiondir'] = '/var/spool/arc/session'
self.settings['grid_security_path'] = '/etc/grid-security'
self.settings['lrms_type'] = 'fork'
self.settings['lrms_path'] = '/usr/bin'
self.settings['grid_user'] = 'griduser'
self.settings['grid_group'] = 'grid'
self.settings['enable_emies'] = False
self.settings['enable_gridftpd'] = False
pass
def run_command(self,cmd):
proc = subprocess.Popen(cmd, stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell=True)
stdout, stderr = proc.communicate()
return stdout, stderr
def print_summary(self,prepend_txt='',append_txt=''):
__print_order = ['conf_path',
'host_cert',
'host_key',
'logdir',
'controldir',
'sessiondir',
'grid_security_path',
'lrms_type',
'lrms_path',
'grid_user',
'grid_group',
'enable_emies',
'enable_gridftpd']
print('\n')
print '='*90
print('A minimal configuration setup.')
print('You can choose to use the default values requiring root priveleges. These are:')
print('* {0:<30s} {1:<20s}'.format('arc.conf path:',conf_path))
print('* {0:<30s} {1:<20s}'.format('sessiondir:',sessiondir))
print('* {0:<30s} {1:<20s}'.format('controldir:',controldir))
print('* {0:<30s} {1:<20s}'.format('log files:',logdir))
print('* {0:<30s} {1:<20s}'.format('linux user:',grid_user))
print('* {0:<30s} {1:<20s}'.format('linux group:',grid_group))
print('* {0:<30s} {1:<20s}'.format('grid-security folder:', grid_security_path))
print('* {0:<30s} {1:<20s}'.format('path to lrms executable:',lrms_path))
print('\n\n[NOTE:]: sudo is required to check that linux user and group exists or should be created.\n')
print('')
print('Uses ansible to:')
print('* Construct an arc.conf placed in default location or location you specify with some minimum required contents.')
print('* Create a linux griduser and gridgroup if such does not exist')
print('* Create grid-security folder if such does not exits')
print('* Create a test-host certificate if no real host certificate exist')
print '='*90 + '\n'
if not self.check_for_ansible():
sys.exit()
if prepend_txt:
print(prepend_txt +'\n')
for key in __print_order:
print('* {0:<30s} {1:<20s}'.format(self.descr[key] + ' :',str(self.settings[key])))
if append_txt:
print(append_txt + '\n')
print '='*90
remote = self.true_false('Running this script remotely? [y/n]:...........................-> ')
#sudo = self.true_false('Need sudo to install? [y/n]')
lrms_type = self.text_answ('Which lrms [slurm/condor/fork]: ')
def true_false(self,ask, default_val=''):
if default_val:
ask = '\n' + ask + ' (ENTER for default value: ' + default_val + ')'
else:
ask = '\n' + ask
_in = None
while _in is None:
try:
_in = str(raw_input('{0:<80}\n==> '.format(ask)).lower())
except KeyboardInterrupt:
break
except:
print('Sorry, something went wrong, try again')
try:
if 'y' in _in:
return True
else:
return False
except:
print('Something went wrong, (or got KeyboardInterrupt), exiting...')
sys.exit(0)
""" HOSTCERTIFICATE
For a production server a host certificate should actually exist.
However, for quick testing, before a real host certificate is installed, a test host certificate can be generated."""
exists = self.true_false('Do you have a host certificate installed? [y/n]: ')
if exists:
host_cert = self.text_answ('Path to your host certificate: ')
host_key = self.text_answ('Path to your host key: ')
create_host_cert = False
else:
create_host_cert = True
def text_answ(self,ask,default_val=''):
""" HOSTCERTIFICATE
If a host certificate exists, the hostname in arc.conf must match the hostname used in the host certificate """
if not create_host_cert:
hostname = self.text_answ('\nARC CE hostname fqdns or ip (must match host certificate).')
if default_val:
ask = '\n' + ask + ' (ENTER for default value: ' + default_val + ')'
else:
if remote:
""" If running this script remotely, and you do not have a host certificate,
a host certificate will generate one for you based on the machine ip,
and this hostname will just be used to ssh """
hostname = self.text_answ('\nARC CE hostname fqdns or ip: ')
ask = '\n' + ask
while True:
try:
return str(raw_input('{0:<80}\n==> '.format(ask)).lower())
except KeyboardInterrupt:
break
except:
print('Sorry, something went wrong, try again')
continue
else:
break
#if sudo:
use_all_defaults = self.true_false('Use all defaults for rest? [y/n]: ')
def get_user_input(self):
use_all_defaults = self.true_false('Use all defaults? [y/n]: ')
if not use_all_defaults:
non_default = self.text_answ('Path to log dir (ENTER for default): ')
""" HOSTCERTIFICATE
For a production server a host certificate should actually exist.
However, for quick testing, before a real host certificate is installed, a test host certificate can be generated."""
self.create_host_cert = self.true_false('Do you need to create a test host certificate? [y/n]')
if not self.create_host_cert:
non_default = self.text_answ('Full path to your host certificate',self.settings['host_cert'])
if non_default:
self.settings['host_cert'] = non_default
non_default = self.text_answ('Full path to your host key',self.settings['host_key'])
if non_default:
self.settings['host_key'] = non_default
lrms_type = self.text_answ('Which lrms [slurm/condor/fork]: ',self.settings['lrms_type'])
non_default = self.text_answ('Path to log dir',self.settings['logdir'])
if non_default:
logdir = non_default
self.settings['logdir'] = non_default
non_default = self.text_answ('Path to arc.conf (ENTER for default): ')
non_default = self.text_answ('Path to arc.conf',self.settings['conf_path'])
if non_default:
conf_path = non_default
self.settings['conf_path'] = non_default
non_default = self.text_answ('Path to sessiondir (ENTER for default): ')
non_default = self.text_answ('Path to sessiondir',self.settings['sessiondir'])
if non_default:
sessiondir = non_default
self.settings['sessiondir'] = non_default
non_default = self.text_answ('Path to controldir (ENTER for default): ')
non_default = self.text_answ('Path to controldir', self.settings['controldir'])
if non_default:
controldir = non_default
self.settings['controldir'] = non_default
non_default = self.text_answ('LRMS bin path (ENTER for default): ')
if 'fork' not in self.settings['lrms_type']:
non_default = self.text_answ('LRMS bin path',self.settings['lrms_path'])
if non_default:
lrms_path = non_default
self.settings['lrms_path'] = non_default
non_default = self.text_answ('Path to grid-security folder (ENTER for default): ')
non_default = self.text_answ('Path to grid-security folder',self.settings['grid_security_path'])
if non_default:
grid_security_path = non_default
self.settings['grid_security_path'] = non_default
non_default = self.text_answ('What linux user to map grid jobs to (ENTER for default): ')
non_default = self.text_answ('What linux user to map grid jobs to',self.settings['grid_user'])
if non_default:
grid_user = non_default
self.settings['grid_user'] = non_default
non_default= self.text_answ('What linux group to map grid users to (ENTER for default): ')
non_default = self.text_answ('What linux group to map grid jobs to',self.settings['grid_group'])
if non_default:
grid_group = non_default
self.settings['grid_group'] = non_default
enable = self.true_false('Enable emies job submission [y/n]: ')
enable = self.true_false('Enable emies job submission [y/n]')
if enable:
enable_emies = True
self.settings['enable_emies'] = True
print('==> Assuming default port 443. See man arc.conf for defaults and how to change.\n')
enable = self.true_false('Enable gridftpd job submission [y/n]: ')
enable = self.true_false('Enable gridftpd job submission [y/n]')
if enable:
enable_gridftpd = True
self.settings['enable_gridftpd'] = True
print('==> Assuming default port 2811. See man arc.conf for defaults and how to change.')
print('==> Using port ranges 9000-10000, change in arc.conf if you want other port ranges.\n')
def create_conf(self):
if create_host_cert:
print('==> Will produce temporary test host certificate, key and CAs for now, and place them in: ' + grid_security_path)
j2_env = Environment(loader=FileSystemLoader('/home/centos/fork-contrib/setupscript/templates'), trim_blocks=True)
template = j2_env.get_template('arc.conf.j2')
rendered = template.render(**self.settings)
print(rendered)
""" Create host file """
host_file = open('./hosts','w+')
if not remote:
host_file.write('localhost ansible_connection=local')
else:
host_file.write(hostname)
host_file.close()
""" Print out a summary """
print '\n'
print '='*90
print('Using following values')
print('* {0:<30s} {1:<20s}'.format('arc.conf path:',conf_path))
print('* {0:<30s} {1:<20s}'.format('sessiondir:',sessiondir))
print('* {0:<30s} {1:<20s}'.format('controldir:',controldir))
print('* {0:<30s} {1:<20s}'.format('log files:',logdir))
print('* {0:<30s} {1:<20s}'.format('linux user:',grid_user))
print('* {0:<30s} {1:<20s}'.format('linux group:',grid_group))
print('* {0:<30s} {1:<20s}'.format('grid-security folder:', grid_security_path))
print('* {0:<30s} {1:<20s}'.format('lrms type:',lrms_type))
print('* {0:<30s} {1:<20s}'.format('path to lrms executable:',lrms_path))
if create_host_cert:
print('* You did not specify any host certificate:')
print('\t* {0:<30s} {1:<20s}'.format('A test host certificate will be created and placed in:',grid_security_path))
print('\t* Please copy the CA and softlinks to your client machines grid-security folder.')
print '='*90
def create_griduser(self):
import pwd
""" Construct ansible command """
cmd = ''
pass_string = ''
#if sudo:
pass_string = ' --ask-become-pass '
cmd = 'ansible-playbook configure_arc.yml -i hosts' + pass_string + ' --extra-vars=' \
+ '"' \
+ ' conf_path={}'.format(conf_path) \
+ ' controldir={}'.format(controldir) \
+ ' sessiondir={}'.format(sessiondir) \
+ ' lrms_type={}'.format(lrms_type) \
+ ' lrms_path={}'.format(lrms_path) \
+ ' grid_user={}'.format(grid_user) \
+ ' grid_group={}'.format(grid_group) \
+ ' grid_security_path={}'.format(grid_security_path) \
+ ' enable_emies={}'.format(enable_emies) \
+ ' enable_gridftpd={}'.format(enable_gridftpd) \
+ ' host_cert={}'.format(host_cert) \
+ ' host_key={}'.format(host_key) \
+ ' hostname={}'.format(hostname) \
+ ' create_host_cert={}'.format(create_host_cert) \
+ '"'
stdout, stderr = self.run_command('getent group ' + self.settings['grid_user'])
if stdout:
pass
else:
print('Group ' + self.settings['grid_group'] + ' does not exist, creating it')
stdout, stderr = self.run_command('sudo groupadd ' + self.settings['grid_group'])
try:
pwd.getpwnam(self.settings['grid_user'])
except KeyError:
print(self.settings['grid_user']+' does not exist, creating user')
stdout, stderr = self.run_command('sudo useradd -g ' + self.settings['grid_group'] self.settings['grid_user'])
def create_logdirs(self):
pass
def create_testCERT(self):
pass
def control(self):
""" Print out some general info """
prepend_txt = 'A minimal configuration setup.\nYou can choose to use the default values requiring root priveleges. Default values are:'
append_txt = '\n\n[NOTE:]: sudo rights is required to check that linux user and group exists or should be created.\n' \
'Will:\n' \
'* Construct an arc.conf placed in default location or location you specify with some minimum required contents.\n' \
'* Create a linux griduser and gridgroup if such does not exist\n' \
'* Create grid-security folder if such does not exits\n' \
'* Create a test-host certificate if no real host certificate exist\n'
self.print_summary(prepend_txt,append_txt)
""" Ask user for custom values for arc.conf """
self.get_user_input()
""" Print out a summary of values selected by user """
prepend_txt='Your chosen configuration values are:'
append_txt=''
if self.create_host_cert:
append_txt='\n* A test host certificate will be created and placed in: ' \
+ self.settings['grid_security_path'] \
+ '\n* Please copy the CA and softlinks to your client machines grid-security folder (ignore if client and server are same machine).'
self.print_summary(prepend_txt,append_txt)
""" Call ansible command """
print '\n ===>Will now run:'
print cmd
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,shell=True)
stdout, stderr = proc.communicate()
print stdout, stderr
""" Create arc.conf """
self.create_conf()
""" Create grid user and group """
self.create_griduser()
""" Ensure log-dir exists """
self.create_logdirs()
""" Create test-CA and host certificate """
self.create_testCERT()
if create_host_cert:
print('\n')
print '='*150
print('A temporary test host certificate has been created for you.\nTo be able to submit jobs from a client,\nplease copy ' + grid_security_path + '/certificates/tempCA.pem from ' + hostname + ' to your clients grid-security folder.')
print '='*150
......@@ -9,12 +9,12 @@ if __name__ == '__main__':
if len(sys.argv) != 2:
sys.exit('Syntax: %s COMMAND' % sys.argv[0])
cmd = sys.argv[1].lower()
if cmd == 'deploy-config':
action = (sys.argv[1].lower())[2:]
print action
if action == 'installwizard':
config = ConfigureARC()
config.configure()
config.control()
else:
sys.exit('Unknown command "%s".' % cmd)
sys.exit('Unknown action "%s".' % action)
......@@ -28,7 +28,7 @@
state: present
createhome: no
- name: Ensure /etc/grid-security folder exists
- name: Ensure grid-security folder exists
file:
name: "{{ item }}"
state: directory
......@@ -38,33 +38,28 @@
- name: Download CertificateGenerator.py
get_url:
url: http://svn.nordugrid.org/trac/nordugrid/export/32500/contrib/certificate_generator/CertificateGenerator.py
url: https://source.coderefinery.org/nordugrid/contrib/raw/master/certificate_generator/CertificateGenerator.py
dest: /tmp
mode: 444
when: create_host_cert
- name: checks hostcert exists on remote path
stat:
path: "/tmp/host-{{ frontend_ip }}-cert.pem"
register: cert
- name: checks hostkey on remote path
stat:
path: "/tmp/host-{{ frontend_ip }}-key.pem"
register: key
- name: checks tempCA on remote path
stat:
path: "/tmp/tempCA.pem"
register: ca
- name: checks tempCA.srl on remote path
stat:
path: "/tmp/tempCA.srl"
register: srl
- name: checks tempCA.signing_policy on remote path
stat:
path: "/tmp/tempCA.signing_policy"
register: policy
- name: Run CertificateGenerator.py to create host certificate
- name: Check if hostcerts and testCA files already exist
stat:
path: "/tmp/{{ item.name }}"
loop:
- "{{ HOST_CERT }}"
- "{{ TESTCA }}"
register: stat_var
- name: debug
debug:
msg: "{{item.reg}}"
with_items: "{{HOST_CERT}}"
- name: Run CertificateGenerator.py to create host and CA certificates (only if they do not exist)
command: python CertificateGenerator.py --CA tempCA --host {{ frontend_ip }}
when: "not( cert.stat.exists or key.stat.exists or ca.stat.exists or srl.stat.exists or policy.stat.exists) and create_host_cert"
args:
......@@ -73,25 +68,12 @@
ignore_errors: yes
- name: move host certificate key file if it exists
command: mv /tmp/host-{{ frontend_ip }}-key.pem {{ grid_security_path }}
when: create_host_cert and run_cert.changed
- name: move host certificate file if it exists
command: mv /tmp/host-{{ frontend_ip }}-cert.pem {{ grid_security_path }}
when: create_host_cert and run_cert.changed
- name: move CA to /etc/grid-security/certificates
command: mv /tmp/tempCA.pem {{ grid_security_path }}/certificates
when: create_host_cert and run_cert.changed
- name: move ca signing policy file if it exists
command: mv /tmp/tempCA.signing_policy {{ grid_security_path }}/certificates
when: create_host_cert and run_cert.changed
- name: move softlinks it exists
shell: mv /tmp/*.0 {{ grid_security_path }}/certificates
- name: Move host certifcate and testCA to specified folders
shell: mv /tmp/{{ item.name }} {{ item.dest }}
when: create_host_cert and run_cert.changed
with_items:
- '{{HOST_CERT}}'
- '{{TESTCA}}'
ignore_errors: yes
......
......@@ -18,3 +18,24 @@ arc_frontend_grid_queue: main
#enable_gridftpd: no
##To-do: where to place key?
HOST_CERT:
- name: "host-{{ frontend_ip }}-cert.pem"
dest: "{{ grid_security_path }}"
- name: "host-{{ frontend_ip }}-key.pem"
dest: "{{ grid_security_path }}"
##To-do: where to place key and srl?
TESTCA:
- name: tempCA.pem
dest: "{{ grid_security_path }}/certificates"
- name: tempCA.signing_policy
dest: "{{ grid_security_path }}/certificates"
- name: "*.0"
dest: "{{ grid_security_path }}/certificates"
- name: tempCA-key.pem
dest: "{{ grid_security_path }}"
- name: tempCA.srl
dest: "{{ grid_security_path }}"
......@@ -20,6 +20,9 @@ x509_host_key = {{ host_key}}
{% endif %}
[mapping]
unixmap={{ grid_user }}:{{ grid_group }} all