Commit d99acadc authored by Mattias Ellert's avatar Mattias Ellert
Browse files

Port to Python 3 (still works with Python 2.6+)

parent 7f88abb0
......@@ -28,6 +28,8 @@ output will be written (the file will be clobbered).
#
######################################################################
from __future__ import print_function
from xml.dom import minidom
import re
import textwrap
......@@ -50,7 +52,7 @@ def my_open_write(dest):
return open(dest, 'w')
class Doxy2SWIG:
class Doxy2SWIG(object):
"""Converts Doxygen generated XML files into a file containing
docstrings that can be used by SWIG-1.3.x that have support for
feature("docstring"). Once the data is parsed it is stored in
......
......@@ -4,6 +4,8 @@ Create a JobSelector class in order to specify a custom selection to be used
with the JobSupervisor class.
'''
from __future__ import print_function
import arc, sys
# Extend the arc.compute.JobSelector class and the select method.
......@@ -55,7 +57,7 @@ selector = ThreeDaysOldJobSelector()
js.Select(selector)
for j in js.GetSelectedJobs():
print (j.JobID)
print(j.JobID)
# Make operation on selected jobs. E.g.:
#js.Clean()
......@@ -12,4 +12,4 @@ jobs.push_back(arc.Job())
itJobs = jobs.__iter__()
itJobs.next()
next(itJobs)
#!/usr/bin/env python
#import sys
#sys.path=sys.path+['../', '../.libs/' ]
from __future__ import print_function
import arc
#f = open('soap.xml').read()
#e = arc.SOAPEnvelope(f)
......@@ -12,7 +13,7 @@ import arc
ns = arc.NS()
ns['echo'] = 'urn:echo'
outpayload = arc.PayloadSOAP(ns)
print type(outpayload)
print(type(outpayload))
outpayload.NewChild('echo:echoResponse').NewChild('echo:hear').Set('foo')
# n = outpayload.NewChild('echo:echoResponse')
# print type(n)
......@@ -20,5 +21,5 @@ outmsg = arc.SOAPMessage()
outmsg.Payload(outpayload)
s = 'a'
ss = outpayload.GetXML()
print s, ss
print str(outmsg)
print(s, ss)
print(str(outmsg))
class A:
from __future__ import print_function
class A(object):
def __init__(self):
pass
def funca(self):
print 'funca'
print('funca')
class B:
class B(object):
def __init__(self):
pass
def funcb(self):
print 'funcb'
print('funcb')
class C(A,B):
def __init__(self):
......@@ -17,7 +19,7 @@ class C(A,B):
def funcc(self):
self.funca()
self.funcb()
print 'funcc'
print('funcc')
c = C()
print type(c), dir(c), c.__class__
print(type(c), dir(c), c.__class__)
......@@ -2,8 +2,13 @@
Python client for the HED cache service.
"""
from __future__ import print_function
import sys
import httplib
try:
import http.client as httplib
except ImportError:
import httplib
import re
import time
import os
......@@ -16,7 +21,7 @@ except ImportError:
# python 2.4 version
from elementtree import ElementTree as ET
except ImportError:
print "Error importing ElementTree. Please install it or use python 2.5 or greater"
print("Error importing ElementTree. Please install it or use python 2.5 or greater")
sys.exit(1)
class CacheException(Exception):
......@@ -88,7 +93,7 @@ def cacheCheck(service, proxy, urls):
try:
conn.request('POST', path, request)
resp = conn.getresponse()
except Exception, e:
except Exception as e:
raise CacheException('Error connecting to service at ' + host + ': ' + str(e))
# On SOAP fault 500 is returned - this is caught in checkSOAPFault
......@@ -170,7 +175,7 @@ def cacheLink(service, proxy, user, jobid, urls, dostage):
try:
conn.request('POST', path, request)
resp = conn.getresponse()
except Exception, e:
except Exception as e:
raise CacheException('Error connecting to service at ' + host + ': ' + str(e))
# On SOAP fault 500 is returned - this is caught in checkSOAPFault
......@@ -227,7 +232,7 @@ def cacheLink(service, proxy, user, jobid, urls, dostage):
try:
conn.request('POST', path, request)
resp = conn.getresponse()
except Exception, e:
except Exception as e:
raise CacheException('Error connecting to service at ' + host + ': ' + str(e))
# On SOAP fault 500 is returned - this is caught in checkSOAPFault
......@@ -249,7 +254,7 @@ def cacheLink(service, proxy, user, jobid, urls, dostage):
if link_result_code == '1':
# still staging
print "still staging"
print("still staging")
# poll for final link appearing in local dir, for one minute, then
# check service again. It is assumed that files will appear in the
......@@ -257,7 +262,7 @@ def cacheLink(service, proxy, user, jobid, urls, dostage):
for i in range(60):
time.sleep(1)
for stagingfile in stagingfiles:
print "Checking for", stagingfiles[stagingfile]
print("Checking for", stagingfiles[stagingfile])
if not os.path.exists(stagingfiles[stagingfile]):
break
else:
......@@ -295,7 +300,7 @@ def echo(service, proxy, say):
conn = httplib.HTTPSConnection(host, port, proxy, proxy)
try:
conn.request('POST', path, request)
except Exception, e:
except Exception as e:
raise CacheException('Error connecting to service at ' + host + ': ' + str(e))
resp = conn.getresponse()
......@@ -328,7 +333,7 @@ if __name__ == '__main__':
try:
cacheurls = cacheCheck(endpoint, proxy, urls_to_check)
except CacheException, e:
except CacheException as e:
print('Error calling cacheCheck: ' + str(e))
sys.exit(1)
......@@ -339,7 +344,7 @@ if __name__ == '__main__':
jobid = '1'
try:
cacheurls = cacheLink(endpoint, proxy, os.getlogin(), jobid, urls_to_link, True)
except CacheException, e:
except CacheException as e:
print('Error calling cacheLink: ' + str(e))
sys.exit(1)
......
......@@ -12,6 +12,8 @@ Limitations:
* Unable to handle templates.
'''
from __future__ import print_function
import sys, re
from os.path import isfile
......@@ -48,16 +50,16 @@ for line in f:
namespaceName, className, _, templateParameters, _, methodName = regMatch.groups()
if templateParameters:
#print "Found template: %s::%s<%s>::%s" % (namespaceName, className, templateParameters, methodName)
print "Error: Unable to handle template signatures %s" % ignoredName
print("Error: Unable to handle template signatures %s" % ignoredName)
continue
#print " Ignoring method '%s' in class '%s' in Arc namespace." % (methodName, className)
sdkFNOfIgnoredInstance = sdkDocumentationLocation + '/class' + namespaceName + '_1_1' + className + '.html'
if not expressionsFound.has_key(sdkFNOfIgnoredInstance):
if sdkFNOfIgnoredInstance not in expressionsFound:
expressionsFound[sdkFNOfIgnoredInstance] = []
ignoreScope = ["Python"] if "SWIGPYTHON" in inIfdef else ["Python"]
expressionsFound[sdkFNOfIgnoredInstance].append({"text" : "Method is unavailable", "scope" : ignoreScope, "name" : methodName})
continue
print "Error: Couldn't parse ignore signature %s" % ignoredName
print("Error: Couldn't parse ignore signature %s" % ignoredName)
continue
regMatch = re.match('%rename\(([^)]+)\)\s+([^;]+)', line)
......@@ -69,24 +71,24 @@ for line in f:
namespaceName, className, _, templateParameters, _, methodName = regMatch.groups()
if templateParameters:
#print "Found template: %s::%s<%s>::%s" % (namespaceName, className, templateParameters, methodName)
print "Error: Unable to handle template signatures %s" % renameFullName
print("Error: Unable to handle template signatures %s" % renameFullName)
continue
#print " Ignoring method '%s' in class '%s' in Arc namespace." % (methodName, className)
sdkFNOfRenamedInstance = sdkDocumentationLocation + '/class' + namespaceName + '_1_1' + className + '.html'
if not expressionsFound.has_key(sdkFNOfRenamedInstance):
if sdkFNOfRenamedInstance not in expressionsFound:
expressionsFound[sdkFNOfRenamedInstance] = []
renameScope = ["Python"] if "SWIGPYTHON" in inIfdef else ["Python"]
expressionsFound[sdkFNOfRenamedInstance].append({"text" : "Renamed to <tt>" + toName + "</tt>", "scope" : renameScope, "name" : methodName})
continue
print "Error: Couldn't parse rename signature %s" % renameFullName
print("Error: Couldn't parse rename signature %s" % renameFullName)
continue
f.close()
#print expressionsFound
for filename, v in expressionsFound.iteritems():
for filename, v in expressionsFound.items():
if not isfile(filename):
print "Error: No such file %s" % filename
print("Error: No such file %s" % filename)
continue
doxHTMLFile = open(filename, "r")
doxHTMLFileLines = doxHTMLFile.readlines()
......@@ -142,16 +144,16 @@ for filename, v in expressionsFound.iteritems():
break
else:
print "Error: Unable to parse method signature %s" % entry["name"]
print("Error: Unable to parse method signature %s" % entry["name"])
i += 1
doxHTMLFile.close()
if v:
print "Error: The following methods was not found in the HTML file '%s':" % filename
print("Error: The following methods was not found in the HTML file '%s':" % filename)
for entry in v:
print " %s" % entry["name"]
print "??? => Is there a API description in the corresponding header file for these?"
print(" %s" % entry["name"])
print("??? => Is there a API description in the corresponding header file for these?")
......@@ -19,6 +19,7 @@
# \mapattr <specialisation-attribute> {->|<-} <library-attribute> ["<note>"]
# \mapnote <note>
from __future__ import print_function
import sys, re
......@@ -52,14 +53,14 @@ for line in sourcefile:
elif line[0:12] == "\mapdefattr ":
regMatch = re.match("([^\s]+)\s+([^\s]+)", line[12:].lstrip())
if not regMatch:
print "ERROR: Wrong format of the \mapdefattr attribute in '%s' file on line %d" % (sourcefilename, i)
print("ERROR: Wrong format of the \mapdefattr attribute in '%s' file on line %d" % (sourcefilename, i))
sys.exit(1)
mapdef["attributes"].append(regMatch.group(1))
mapdef["attributeprefixes"].append(regMatch.group(2))
elif line[0:8] == "\mapdef ":
regMatch = re.match("(\w+)\s+(.+)", line[8:].lstrip())
if not regMatch:
print "ERROR: Wrong format of the \mapdef attribute in '%s' file on line %d" % (sourcefilename, i)
print("ERROR: Wrong format of the \mapdef attribute in '%s' file on line %d" % (sourcefilename, i))
sys.exit(1)
mapdef["id"] = regMatch.group(1)
mapdef["name"] = regMatch.group(2)
......@@ -94,7 +95,7 @@ for filename in mapfiles:
if line[0:9] == "\mapname ":
regMatch = re.match("(\w+)\s+(.+)", line[9:].lstrip())
if not regMatch:
print "ERROR: Wrong format of the \mapname command in '%s' file on line %d" % (filename, i)
print("ERROR: Wrong format of the \mapname command in '%s' file on line %d" % (filename, i))
sys.exit(1)
m["id"] = regMatch.group(1)
......@@ -108,8 +109,8 @@ for filename in mapfiles:
# <specialisation-attr> -> <lib. attr> ["<note>"]
regMatch = re.match("(.+)\s+->\s+([^\s]+)(?:\s+\"([^\"]+)\")?", line[9:])
if regMatch:
if not m["attributes"].has_key(regMatch.group(2)):
print "ERROR: The '%s' attribute present in file '%s' on line %d is not defined in file '%s'" % (regMatch.group(2), filename, i, sourcefilename)
if regMatch.group(2) not in m["attributes"]:
print("ERROR: The '%s' attribute present in file '%s' on line %d is not defined in file '%s'" % (regMatch.group(2), filename, i, sourcefilename))
sys.exit(1)
m["attributes"][regMatch.group(2)]["in"].append(regMatch.group(1))
if regMatch.group(3):
......@@ -118,8 +119,8 @@ for filename in mapfiles:
regMatch = re.match("(.+)\s+<-\s+([^\s]+)(?:\s+\"([^\"]+)\")?", line[9:])
if regMatch:
if not m["attributes"].has_key(regMatch.group(2)):
print "ERROR: The '%s' attribute present in file '%s' on line %d is not defined in file '%s'" % (regMatch.group(2), filename, i, sourcefilename)
if regMatch.group(2) not in m["attributes"]:
print("ERROR: The '%s' attribute present in file '%s' on line %d is not defined in file '%s'" % (regMatch.group(2), filename, i, sourcefilename))
sys.exit(1)
m["attributes"][regMatch.group(2)]["out"].append(regMatch.group(1))
if regMatch.group(3):
......@@ -193,7 +194,7 @@ for m in mappings:
outfile.write('<dd>' + note+ '</dd>\n')
outfile.write('</dl>\n')
has_input = has_output = False
for attr, m_attrs in m["attributes"].iteritems():
for attr, m_attrs in m["attributes"].items():
has_input = has_input or bool(m_attrs["in"])
has_output = has_output or bool(m_attrs["out"])
table_header = "| Input " if has_input else ""
......@@ -203,7 +204,7 @@ for m in mappings:
outfile.write(re.sub(r'[. \w]', '-', table_header) + "\n")
notes = []
for attr, m_attrs in m["attributes"].iteritems():
for attr, m_attrs in m["attributes"].items():
if not m_attrs["in"] and not m_attrs["out"]:
continue
line = ""
......
......@@ -35,7 +35,10 @@ The default level is WARNING.
'''
import random
import httplib
try:
import http.client as httplib
except ImportError:
import httplib
import re
import logging
......@@ -47,7 +50,7 @@ except ImportError:
import arc
class ACIXBroker:
class ACIXBroker(object):
def __init__(self, usercfg):
'''
Set up internal fields and get information from UserConfig.
......@@ -181,13 +184,13 @@ class ACIXBroker:
conn = httplib.HTTPSConnection(host, port)
try:
conn.request('GET', path)
except Exception, e:
except Exception as e:
logging.error('Error connecting to service at %s: %s', host, str(e))
return
try:
resp = conn.getresponse()
except httplib.HTTPException, e:
except httplib.HTTPException as e:
logging.error('Bad response from ACIX: %s', str(e))
return
......
......@@ -5,10 +5,12 @@ Invoke the broker with:
arcsub -b PythonBroker:SampleBroker.MyBroker:brokerargs
'''
from __future__ import print_function
import arc
import random
class MyBroker:
class MyBroker(object):
def __init__(self, usercfg):
# Extract some useful information from the broker configuration
self.uc = usercfg
......@@ -28,34 +30,34 @@ class MyBroker:
# needed.
self.job = job
# Only printing below for clarity.
print 'JobName:', job.Identification.JobName
print 'Executable:', job.Application.Executable.Path
print('JobName:', job.Identification.JobName)
print('Executable:', job.Application.Executable.Path)
for i in range(job.Application.Executable.Argument.size()):
print 'Argument', i, ':', job.Application.Executable.Argument[i]
print('Argument', i, ':', job.Application.Executable.Argument[i])
def match(self, target):
# Some printouts - only as an example
print 'Proxy Path:', self.proxypath
print 'Certificate Path:', self.certificatepath
print 'Key Path:', self.keypath
print 'CA Certificates Dir:', self.cacertificatesdir
print('Proxy Path:', self.proxypath)
print('Certificate Path:', self.certificatepath)
print('Key Path:', self.keypath)
print('CA Certificates Dir:', self.cacertificatesdir)
print 'Broker arguments:', self.args
print('Broker arguments:', self.args)
# Broker implementation starts here
print 'Targets before brokering:'
print target.ComputingEndpoint.URLString
print('Targets before brokering:')
print(target.ComputingEndpoint.URLString)
# Do generic matchmaking
if not arc.Broker.genericMatch(target, self.job, self.uc):
print 'Target', target.ComputingEndpoint.URLString, 'rejected'
print('Target', target.ComputingEndpoint.URLString, 'rejected')
return False
# Accept target
print 'Target', target.ComputingEndpoint.URLString, 'accepted'
print('Target', target.ComputingEndpoint.URLString, 'accepted')
return True
def lessthan(self, lhs, rhs):
print 'Randomizing...'
print('Randomizing...')
return random.randint(0, 1)
......@@ -18,6 +18,9 @@
Class to interact with a BDII LDAP server to retrieve information about
the stomp brokers specified in a network.
'''
from __future__ import print_function
import ldap
import logging
......@@ -140,12 +143,12 @@ if __name__ == '__main__':
def print_brokers(text, service, network):
brokers = BG.get_broker_hosts_and_ports(service, network)
# Print section heading
print '==', text, '=='
print('==', text, '==')
# Print brokers in form 'host:port'
for broker in brokers:
print '%s:%i' % (broker[0], broker[1])
print('%s:%i' % (broker[0], broker[1]))
# Leave space between sections
print
print()
print_brokers('SSL production brokers', STOMP_SSL_SERVICE, 'PROD')
print_brokers('Production brokers', STOMP_SERVICE, 'PROD')
......
......@@ -57,7 +57,7 @@ def check_cert_key(certpath, keypath):
try:
cert = _from_file(certpath)
key = _from_file(keypath)
except IOError, e:
except IOError as e:
log.error('Could not find cert or key file: %s', e)
return False
......@@ -101,7 +101,7 @@ def sign(text, certpath, keypath):
return signed_msg
except OSError, e:
except OSError as e:
log.error('Failed to sign message: %s', e)
raise CryptoException('Message signing failed. Check cert and key permissions.')
......
......@@ -16,6 +16,8 @@
@author: Will Rogers
'''
from __future__ import absolute_import
# It's possible for SSM to be used without SSL, and the ssl module isn't in the
# standard library until 2.6, so this makes it safe for earlier Python versions.
try:
......@@ -24,7 +26,7 @@ except ImportError:
# ImportError is raised later on if SSL is actually requested.
ssl = None
import crypto
from . import crypto
from dirq.QueueSimple import QueueSimple
from dirq.queue import Queue
......@@ -165,7 +167,7 @@ class Ssm2(stomp.ConnectionListener):
self._inq.add({'body': raw_msg,
'signer':signer,
'empaid': headers['empa-id']})
except OSError, e:
except OSError as e:
log.error('Failed to read or write file: %s', e)
def on_error(self, unused_headers, body):
......@@ -218,14 +220,14 @@ class Ssm2(stomp.ConnectionListener):
if 'application/pkcs7-mime' in text or 'application/x-pkcs7-mime' in text:
try:
text = crypto.decrypt(text, self._cert, self._key)
except crypto.CryptoException, e:
except crypto.CryptoException as e:
log.error('Failed to decrypt message: %s', e)
return None, None
# always signed
try:
message, signer = crypto.verify(text, self._capath, self._check_crls)
except crypto.CryptoException, e:
except crypto.CryptoException as e:
log.error('Failed to verify message: %s', e)
return None, None
......@@ -303,7 +305,7 @@ class Ssm2(stomp.ConnectionListener):
try:
# Remove empty dirs and unlock msgs older than 5 min (default)
self._outq.purge()
except OSError, e:
except OSError as e:
log.warn('OSError raised while purging message queue: %s', e)
############################################################################
......@@ -365,10 +367,10 @@ class Ssm2(stomp.ConnectionListener):
try:
self.start_connection()
break
except ConnectFailedException, e:
except ConnectFailedException as e:
# ConnectFailedException doesn't provide a message.
log.warn('Failed to connect to %s:%s.', host, port)
except Ssm2Exception, e:
except Ssm2Exception as e:
log.warn('Failed to connect to %s:%s: %s', host, port, e)
if not self.connected:
......@@ -455,7 +457,7 @@ class Ssm2(stomp.ConnectionListener):
f.write(str(os.getpid()))
f.write('\n')
f.close()
except IOError, e:
except IOError as e:
log.warn('Failed to create pidfile %s: %s', self._pidfile, e)
self.handle_connect()
......@@ -471,7 +473,7 @@ class Ssm2(stomp.ConnectionListener):
os.remove(self._pidfile)
else:
log.warn('pidfile %s not found.', self._pidfile)
except IOError, e:
except IOError as e:
log.warn('Failed to remove pidfile %s: %e', self._pidfile, e)
log.warn('SSM may not start again until it is removed.')
......
......@@ -4,9 +4,11 @@ Subpackage with helper functions for the batch system specific modules.
Logging is setup here.
"""
from __future__ import absolute_import
import sys
import arc
import cancel, config, files, log, parse, proc, scan, ssh, submit
from . import cancel, config, files, log, parse, proc, scan, ssh, submit
_logstream = arc.LogStream(sys.stderr)
_logstream.setFormat(arc.EmptyFormat)
......
......@@ -2,10 +2,12 @@
Module for job cancelling.
"""
from config import Config
from log import debug, error
from ssh import ssh_connect
from proc import *
from __future__ import absolute_import
from .config import Config
from .log import debug, error
from .ssh import ssh_connect
from .proc import *
def cancel(cmd, jobid):
"""
......
......@@ -2,10 +2,13 @@
Provides the ``Config`` object, with each arc.conf option as an attribute.
"""
from __future__ import absolute_import