Skip to content

GitLab

Commit fc683811 authored by Andrey Kutuzov's avatar Andrey Kutuzov
Browse files

Numpy testing

parent b6dbef27
Hide whitespace changes
Inline Side-by-side
README.md
View file @ fc683811
......@@ -48,3 +48,8 @@ If needed, you can change the `EASYBUILD_INSTALLPATH_MODULES` and `EASYBUILD_INS
(or `eb --robot nlpl-nvidia-bert-tf-20.06.08-foss-2019b-Python3.7.4.eb` if your CPU architecture is not Intel)
- After the process is finished, your modules will be visible along with the system-provided ones via `module avail`, and can be loaded with `module load`.
- Approximate building time for all the required modules is several hours (from 3 to 5).
## Testing
It is important to make sure that Numpy is built with accelerated linear algebra libraries (OpenBLAS or MKL).
To test this, we provide the `test/numpy/numpy_test.py` script.
Its execution time with our `nlpl-numpy*` modules should not be more than 10 minutes (usually much less).
tests/mkl.inof.test deleted 100644 → 0
View file @ b6dbef27
import numpy as pk
from numpy.distutils.system_info import system_info, ConfigParser, mkl_info
# Check that the '[ALL]' section does not override
# missing values from other sections
info = mkl_info()
print("0")
print(info)
print(info.keys)
lib_dirs = info.cp['ALL']['library_dirs'].split(os.pathsep)
assert info.get_lib_dirs() != lib_dirs
# But if we copy the values to a '[mkl]' section the value
# is correct
with open(cfg, 'r') as fid:
print("1")
print(fid.read())
mkl = fid.read().replace('ALL', 'mkl')
print("2")
print(mkl)
with open(cfg, 'w') as fid:
fid.write(mkl)
info = mkl_info()
assert info.get_lib_dirs() == lib_dirs
tests/numpy/numpy_test.py
View file @ fc683811
......@@ -12,7 +12,7 @@ start = time.time()
for n in range(50):
matrix0 = np.random.randn(4096, 1024)
matrix1 = np.random.randn(1024, 4096)
_ = matrix0 @ matrix1
_ = matrix0 @ matrix1
end = time.time()
print(f"Multiplication took {int(end-start)} seconds.")
......@@ -23,7 +23,7 @@ start = time.time()
for n in range(50):
matrix = np.random.randn(1024, 1024)
u, _, v = np.linalg.svd(matrix)
ortho = u @ v
_ = u @ v
end = time.time()
print(f"SVD took {int(end-start)} seconds.")
......
tests/numpy/slurm-1872518.out deleted 100644 → 0
View file @ b6dbef27
Starting job 1872518 on c1-24 at Mon Jan 18 23:16:13 CET 2021
The following modules were not unloaded:
(Use "module --force purge" to unload all):
1) StdEnv
blas_mkl_info:
libraries = ['mkl_rt', 'pthread']
library_dirs = ['/cluster/shared/nlpl/software/easybuild_ak/easybuild/install/software/imkl/2019.1.144-gompi-2019b/mkl/lib/intel64']
define_macros = [('SCIPY_MKL_H', None), ('HAVE_CBLAS', None)]
include_dirs = ['/cluster/shared/nlpl/software/easybuild_ak/easybuild/install/software/imkl/2019.1.144-gompi-2019b/mkl', '/cluster/shared/nlpl/software/easybuild_ak/easybuild/install/software/imkl/2019.1.144-gompi-2019b/mkl/include', '/cluster/shared/nlpl/software/easybuild_ak/easybuild/install/software/imkl/2019.1.144-gompi-2019b/mkl/lib']
blas_opt_info:
libraries = ['mkl_rt', 'pthread']
library_dirs = ['/cluster/shared/nlpl/software/easybuild_ak/easybuild/install/software/imkl/2019.1.144-gompi-2019b/mkl/lib/intel64']
define_macros = [('SCIPY_MKL_H', None), ('HAVE_CBLAS', None)]
include_dirs = ['/cluster/shared/nlpl/software/easybuild_ak/easybuild/install/software/imkl/2019.1.144-gompi-2019b/mkl', '/cluster/shared/nlpl/software/easybuild_ak/easybuild/install/software/imkl/2019.1.144-gompi-2019b/mkl/include', '/cluster/shared/nlpl/software/easybuild_ak/easybuild/install/software/imkl/2019.1.144-gompi-2019b/mkl/lib']
lapack_mkl_info:
libraries = ['mkl_rt', 'pthread']
library_dirs = ['/cluster/shared/nlpl/software/easybuild_ak/easybuild/install/software/imkl/2019.1.144-gompi-2019b/mkl/lib/intel64']
define_macros = [('SCIPY_MKL_H', None), ('HAVE_CBLAS', None)]
include_dirs = ['/cluster/shared/nlpl/software/easybuild_ak/easybuild/install/software/imkl/2019.1.144-gompi-2019b/mkl', '/cluster/shared/nlpl/software/easybuild_ak/easybuild/install/software/imkl/2019.1.144-gompi-2019b/mkl/include', '/cluster/shared/nlpl/software/easybuild_ak/easybuild/install/software/imkl/2019.1.144-gompi-2019b/mkl/lib']
lapack_opt_info:
libraries = ['mkl_rt', 'pthread']
library_dirs = ['/cluster/shared/nlpl/software/easybuild_ak/easybuild/install/software/imkl/2019.1.144-gompi-2019b/mkl/lib/intel64']
define_macros = [('SCIPY_MKL_H', None), ('HAVE_CBLAS', None)]
include_dirs = ['/cluster/shared/nlpl/software/easybuild_ak/easybuild/install/software/imkl/2019.1.144-gompi-2019b/mkl', '/cluster/shared/nlpl/software/easybuild_ak/easybuild/install/software/imkl/2019.1.144-gompi-2019b/mkl/include', '/cluster/shared/nlpl/software/easybuild_ak/easybuild/install/software/imkl/2019.1.144-gompi-2019b/mkl/lib']
None
Multiplication took 52 seconds.
SVD took 42 seconds.
Task and CPU usage stats:
JobID JobName AllocCPUS NTasks MinCPU MinCPUTask AveCPU Elapsed ExitCode
------------ ---------- ---------- -------- ---------- ---------- ---------- ---------- --------
1872518 test_numpy 8 00:01:42 0:0
1872518.bat+ batch 8 1 00:01:26 0 00:01:26 00:01:42 0:0
1872518.ext+ extern 8 1 00:00:00 0 00:00:00 00:01:42 0:0
Memory usage stats:
JobID MaxRSS MaxRSSTask AveRSS MaxPages MaxPagesTask AvePages
------------ ---------- ---------- ---------- -------- -------------- ----------
1872518
1872518.bat+ 271792K 0 271792K 0 0 0
1872518.ext+ 0 0 0 0 0 0
Disk usage stats:
JobID MaxDiskRead MaxDiskReadTask AveDiskRead MaxDiskWrite MaxDiskWriteTask AveDiskWrite
------------ ------------ --------------- -------------- ------------ ---------------- --------------
1872518
1872518.bat+ 9.45M 0 9.45M 0.05M 0 0.05M
1872518.ext+ 0.00M 0 0.00M 0 0 0
Job 1872518 completed at Mon Jan 18 23:17:55 CET 2021
tests/numpy/test_numpy_mkl.slurm
View file @ fc683811
#!/bin/bash
#SBATCH --job-name=test_numpy
#SBATCH --partition=small
#SBATCH --account=project_2000582
#SBATCH --account=YOUR_PROJECT_NUMBER
#SBATCH --time=1:00:00
#SBATCH --mem-per-cpu=4G
#SBATCH --ntasks=8
#SBATCH --cpus-per-task=8
#SBATCH --nodes=1
set -o errexit # Recommended for easier debugging
# Replace with your module repository directory:
module use -a /projappl/nlpl/software/andreku/modules/all
module use -a DIRECTORY
module purge
module load nlpl-scipy-bundle/2021.01-gomkl-2019b-Python-3.7.4
module load nlpl-numpy/1.18.1-gomkl-2019b-Python-3.7.4
python3 numpy_test.py
tests/numpy/test_numpy_openblas.slurm
View file @ fc683811
#!/bin/bash
#SBATCH --job-name=test_numpy
#SBATCH --mail-type=FAIL
#SBATCH --account=n9851k
#SBATCH --time=1:00:00 # Max walltime is 14 days.
#SBATCH --account=YOUR_PROJECT_NUMBER
#SBATCH --time=1:00:00
#SBATCH --mem-per-cpu=4G
#SBATCH --ntasks=8
#SBATCH --cpus-per-task=8
#SBATCH --nodes=1
set -o errexit # Recommended for easier debugging
# Replace with your module repository directory:
module use -a /cluster/shared/nlpl/software/easybuild_ak/easybuild/install/modules/all/
module use -a DIRECTORY
module purge
module purge
module load NLPL-numpy/1.18.1-foss-2019b-Python-3.7.4
module load nlpl-numpy/1.18.1-foss-2019b-Python-3.7.4
python3 tests/numpy/numpy_test.py
python3 numpy_test.py
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment