CP2K 2023.1

Webpage

https://www.cp2k.org/

Version

2023.1

Build Environment

  • GCC 11.2.1 (gcc-toolset-11)
  • HPC-X 2.11 (Open MPI 4.1.4)

Files Required

  • cp2k-2023.1.tar.gz
  • patch-openblas.diff

--- scripts/stage2/install_openblas.sh.org      2023-01-17 10:19:31.000000000 +0900
+++ scripts/stage2/install_openblas.sh  2023-01-17 10:20:21.000000000 +0900
@@ -47,6 +47,7 @@
       [ -d OpenBLAS-${openblas_ver} ] && rm -rf OpenBLAS-${openblas_ver}
       tar -zxf ${openblas_pkg}
       cd OpenBLAS-${openblas_ver}
+      sed -i -e "/NOTPARALLEL/s/$/ shared/" Makefile
 
       # First attempt to make openblas using auto detected
       # TARGET, if this fails, then make with forced

  • patch-plumed.diff

--- scripts/stage6/install_plumed.sh.org        2023-04-04 15:56:54.000000000 +0900
+++ scripts/stage6/install_plumed.sh    2023-04-04 15:57:28.000000000 +0900
@@ -90,7 +90,7 @@
 esac
 
 if [ "$with_plumed" != "__DONTUSE__" ]; then
-  PLUMED_LIBS='-lplumed -ldl -lstdc++ -lz -ldl'
+  PLUMED_LIBS='-lplumedKernel -lplumed -ldl -lstdc++ -lz -ldl'
   if [ "$with_plumed" != "__SYSTEM__" ]; then
     cat << EOF > "${BUILDDIR}/setup_plumed"
 prepend_path LD_LIBRARY_PATH "$pkg_install_dir/lib"

  • patch-spla-nogpu.diff

--- scripts/stage8/install_spla.sh.org  2023-04-04 15:31:42.000000000 +0900
+++ scripts/stage8/install_spla.sh      2023-04-04 15:37:09.000000000 +0900
@@ -204,6 +204,11 @@
 export CP_LDFLAGS="\${CP_LDFLAGS} IF_CUDA(${SPLA_CUDA_LDFLAGS}|${SPLA_LDFLAGS})"
 EOF
   fi
+  if [ "$ENABLE_CUDA" != "__TRUE__" -a "$ENABLE_HIP" != "__TRUE__" ]; then
+    cat << EOF >> "${BUILDDIR}/setup_spla"
+export CP_LDFLAGS="\${CP_LDFLAGS} ${SPLA_LDFLAGS}"
+EOF
+  fi
   cat "${BUILDDIR}/setup_spla" >> $SETUPFILE
 fi

Build Procedure

#!/bin/sh

VERSION=2023.1
DBCSR_VERSION=v2.5.0

INSTDIR=/apl/cp2k/${VERSION}

SOURCE_ROOT=/home/users/${USER}/Software/CP2K/${VERSION}
TARBALL=${SOURCE_ROOT}/cp2k-${VERSION}.tar.bz2

TC_PATCH_2_1=${SOURCE_ROOT}/patch-openblas.diff
TC_PATCH_6_1=${SOURCE_ROOT}/patch-plumed.diff
TC_PATCH_8_1=${SOURCE_ROOT}/patch-spla-nogpu.diff

PARALLEL=32

# ------------------------------------------------------------------------
umask 0022
export LANG=C
export LC_ALL=C
ulimit -s unlimited

module -s purge
module -s load gcc-toolset/11
module -s load openmpi/4.1.4-hpcx/gcc11

cd $INSTDIR
if [ -d cp2k-${VERSION} ]; then
  mv cp2k-${VERSION} cp2k-erase
  rm -rf cp2k-erase &
fi
tar jxf ${TARBALL}
sleep 5
mv cp2k-${VERSION}/* .
sleep 5
rm -rf cp2k-${VERSION}/.dockerignore
rmdir cp2k-${VERSION}

cd ${INSTDIR}/tools/toolchain

# apply patches
patch -p0 < ${TC_PATCH_2_1}
patch -p0 < ${TC_PATCH_6_1}
patch -p0 < ${TC_PATCH_8_1}
sed -i -e  "/PARMETISLIB=FALSE/"a'-DCMAKE_C_COMPILER=${MPICC} -DCMAKE_CXX_COMPILER=${MPICXX} \\' scripts/stage5/install_superlu.sh

export CC=gcc
export CXX=g++
export FC=gfortran
export MPICC=mpicc
export MPICXX=mpicxx
export MPIFC=mpif90

./install_cp2k_toolchain.sh --mpi-mode=openmpi \
                            --math-mode=openblas \
                            --with-gcc=system \
                            --with-cmake=system \
                            --with-openmpi=system \
                            --with-mpich=no \
                            --with-intelmpi=no \
                            --with-libxc=install \
                            --with-libint=install \
                            --with-fftw=install \
                            --with-acml=no \
                            --with-mkl=no \
                            --with-openblas=install \
                            --with-scalapack=install \
                            --with-libxsmm=install \
                            --with-elpa=install \
                            --with-ptscotch=install \
                            --with-superlu=install \
                            --with-pexsi=install \
                            --with-quip=install \
                            --with-plumed=install \
                            --with-sirius=install \
                            --with-gsl=install \
                            --with-libvdwxc=install \
                            --with-spglib=install \
                            --with-hdf5=install \
                            --with-spfft=install \
                            --with-spla=install \
                            --with-cosma=install \
                            --with-libvori=install \
                            --with-libtorch=install

cp install/arch/local.psmp ../../arch/rccs.psmp
cd ${INSTDIR}

# dbscr source code is already available

make -j ${PARALLEL} ARCH=rccs VERSION=psmp
make -j ${PARALLEL} ARCH=rccs VERSION=psmp libcp2k

Tests

Following job script was employed.

#!/bin/sh
#PBS -l select=1:ncpus=16:mpiprocs=16:ompthreads=1
#PBS -l walltime=12:00:00

export LC_ALL=C
export LANG=""
export OMP_STACKSIZE=64M

module -s purge
module -s load gcc-toolset/11
module -s load openmpi/4.1.4-hpcx/gcc11

CP2K=/apl/cp2k/2023.1

CP2K_ARCH=rccs
CP2K_VER=psmp
TIMEOUT=600
PARALLEL=16

ulimit -s unlimited

cd ${CP2K}/regtesting/${CP2K_ARCH}/${CP2K_VER}
rm -rf LAST-${CP2K_ARCH}-${CP2K_VER}

# serial test
../../../tools/regtesting/do_regtest \
        -nobuild \
        -arch ${CP2K_ARCH} \
        -version ${CP2K_VER} \
        -mpiranks 1 \
        -ompthreads 1 \
        -jobmaxtime ${TIMEOUT} \
        -cp2kdir ../../../ \
        -maxtasks ${PARALLEL} >& regtest_mpi1_omp1.log
rm -rf LAST-${CP2K_ARCH}-${CP2K_VER}

# omp test
../../../tools/regtesting/do_regtest \
        -nobuild \
        -arch ${CP2K_ARCH} \
        -version ${CP2K_VER} \
        -mpiranks 1 \
        -ompthreads 2 \
        -jobmaxtime ${TIMEOUT} \
        -cp2kdir ../../../ \
        -maxtasks ${PARALLEL} >& regtest_mpi1_omp2.log
rm -rf LAST-${CP2K_ARCH}-${CP2K_VER}

# mpi test
../../../tools/regtesting/do_regtest \
        -nobuild \
        -arch ${CP2K_ARCH} \
        -version ${CP2K_VER} \
        -mpiranks 2 \
        -ompthreads 1 \
        -jobmaxtime ${TIMEOUT} \
        -cp2kdir ../../../ \
        -maxtasks ${PARALLEL} >& regtest_mpi2_omp1.log
rm -rf LAST-${CP2K_ARCH}-${CP2K_VER}

# mpi/openmp test
../../../tools/regtesting/do_regtest \
        -nobuild \
        -arch ${CP2K_ARCH} \
        -version ${CP2K_VER} \
        -mpiranks 2 \
        -ompthreads 2 \
        -jobmaxtime ${TIMEOUT} \
        -cp2kdir ../../../ \
        -maxtasks ${PARALLEL} >& regtest_mpi2_omp2.log
rm -rf LAST-${CP2K_ARCH}-${CP2K_VER}

# yet another mpi test
../../../tools/regtesting/do_regtest \
        -nobuild \
        -arch ${CP2K_ARCH} \
        -version ${CP2K_VER} \
        -mpiranks 8 \
        -ompthreads 1 \
        -jobmaxtime ${TIMEOUT} \
        -cp2kdir ../../../ \
        -maxtasks ${PARALLEL} >& regtest_mpi8_omp1.log
rm -rf LAST-${CP2K_ARCH}-${CP2K_VER}

# yet another mpi/openmp test
../../../tools/regtesting/do_regtest \
        -nobuild \
        -arch ${CP2K_ARCH} \
        -version ${CP2K_VER} \
        -mpiranks 8 \
        -ompthreads 2 \
        -jobmaxtime ${TIMEOUT} \
        -cp2kdir ../../../ \
        -maxtasks ${PARALLEL} >& regtest_mpi8_omp2.log
rm -rf LAST-${CP2K_ARCH}-${CP2K_VER}

Results:

regtest_mpi1_omp1.log:GREPME 0 0 3868 0 3868 X
regtest_mpi1_omp2.log:GREPME 0 0 3868 0 3868 X
regtest_mpi2_omp1.log:GREPME 0 0 3926 0 3926 X
regtest_mpi2_omp2.log:GREPME 0 0 3926 0 3926 X
regtest_mpi8_omp1.log:GREPME 10 15 3908 0 3933 X
regtest_mpi8_omp2.log:GREPME 10 15 3908 0 3933 X

Notes

  • Please check /apl/cp2k/2023.1/regtesting/rccs/psmp and its sub-directory for details about tests.
    • summary.txt, error_summary, timings.txt may be informative.
  • Superlu is successfully activated by passing compilers explicitly. (This was failed in the previous build.)
  • Cosma is enabled this time. There are no noticeable problems in the build/performance.
  • GPU version is not released this time. Although performance is improved by GPUs in some cases, GPU version seems bit unstable.
    • Some investigations may be required to get good performance. (More difficult than CPU version.)
    • Memory usage level seems tobe very high. Mpirun command often needs "-mca coll_hcoll_enable 0" to run correctly.
    • If GPUs are available only to toolchain part, the overall calculations and performances become stable. However, performance improvement is limited (performance is often worse than cpu only version).
  • Following message may be harmless since the cp2k is built with "-mtune=native" option.
    • Most probably, this message is related to -march option in the build time. (-march is not specified)

 *** HINT in environment.F:884 :: The compiler target flags (generic) used ***
 *** to build this binary cannot exploit all extensions of this CPU model  ***
 *** (x86_avx2). Consider compiler target flags as part of FCFLAGS and     ***

  • (Jul 26, 2024) Runtime library of HPC-X 2.11 or 2.13.1 cause significant performance loss for calculation of H20-256 system with 128 MPI processes. This problem can be removed by switching to Open MPI 4.1.5, 4.1.6 or HPC-X 2.16 runtime libraries.