|
|
### Install story of TenStream/PALM on cirrus (gcc, the GNU Fortran compiler)
|
|
|
|
|
|
Here is a story of how I installed TenStream/PALM and configured the needed packages on cirrus cluster at HUB.
|
|
|
|
|
|
This story is for installation of TenStream/PALM model using the GNU Compiler Collection (GCC). The Intel based installation story will be described in a separate story.
|
|
|
|
|
|
**Loading the required packages**
|
|
|
|
|
|
The GCC based modules of FORTRAN, NetCDF, [PETSc](https://petsc.org/release/) as well as cmake3 should be loaded first in order to configure and install TenStream.
|
|
|
|
|
|
These module load commands should be performed:
|
|
|
|
|
|
```bash
|
|
|
module load gcc_openmp
|
|
|
module load netcdf_fortran
|
|
|
module load petsc_single/3.15.4
|
|
|
module load cmake/3.21.2
|
|
|
```
|
|
|
**Cloning TenStream**
|
|
|
|
|
|
TenStream can be cloned from the github repository [here](https://github.com/tenstream/tenstream). Clone the model and create the build directory, as following
|
|
|
|
|
|
```bash
|
|
|
cd $HOME/work
|
|
|
git clone https://github.com/tenstream/tenstream.git
|
|
|
cd tenstream
|
|
|
mkdir build
|
|
|
cd build
|
|
|
```
|
|
|
**Configure and compile TenStream**
|
|
|
|
|
|
First, create a configurations file (e.g. cirrus.gcc.cmake) for compiling TenStream on cirrus as follow and save it to $HOME/tenstream/config/ folder:
|
|
|
|
|
|
```bash
|
|
|
message(STATUS "### USING GCC CONFIG ###")
|
|
|
|
|
|
set(CMAKE_C_COMPILER "mpicc")
|
|
|
set(CMAKE_Fortran_COMPILER "mpif90")
|
|
|
set(Fortran_COMPILER_WRAPPER "mpif90")
|
|
|
|
|
|
set(USER_C_FLAGS "-W -Wall")
|
|
|
set(USER_Fortran_FLAGS "-W -Wall")
|
|
|
set(USER_Fortran_FLAGS_RELEASE "-fno-backtrace -fno-range-check -O3 -mtune=native")
|
|
|
set(USER_Fortran_FLAGS_DEBUG "-ggdb -fbacktrace -fcheck=all -fbounds-check -pedantic -Wsurprising -Werror -Wuninitialized -ffpe-trap=invalid")
|
|
|
|
|
|
set(NETCDF_DIR "$ENV{NETCDF_C_DIR}")
|
|
|
set(NETCDF_LIBRARIES "$ENV{NETCDF_C_LIBDIR}")
|
|
|
set(NETCDF_DIR_F90 "$ENV{NETCDF_FORTRAN_DIR}")
|
|
|
```
|
|
|
Note: One can add the module load commands above in the configuration file to skip the loading modules step above (not tested).
|
|
|
|
|
|
Then, run the configuration using cmake based on the configuration file (e.g. cirrus.gcc.cmake) and then run the compilation using make using these commands:
|
|
|
|
|
|
```bash
|
|
|
PKG_CONFIG_ALLOW_SYSTEM_CFLAGS=1 cmake .. -DSYST=cirrus.gcc
|
|
|
make
|
|
|
```
|
|
|
Note: the environmental variable PKG_CONFIG_ALLOW_SYSTEM_CFLAGS is explicitly set here to enable the pkg-config. Also one can use make -j8 to parallelize the compilation.
|
|
|
|
|
|
**PALM configuration file**
|
|
|
|
|
|
Create a configuration file for PALM, i.e. .palm.config.TS_gf, to configure PALM with TenStream. The following points should be considered:
|
|
|
1. Add the cpp directive -D__tenstream to the %cpp_options
|
|
|
2. Add the required modules to the load module %module_commands line, e.g. module load gcc_openmpi; module load netcdf_fortran rrtmg/1.0a; module load petsc_single
|
|
|
3. Add the include and the lib directories of TenStream to the compiler and linker options
|
|
|
4. The lib variable PETSC_WITH_EXTERNAL_LIB, which can be found in /programs/gcc_openmpi-4.1/petsc_single/3.15.4/lib/petsc/conf/petscvariables, should be added to the linker options
|
|
|
|
|
|
An example of the palm configuration file is shown below.
|
|
|
|
|
|
```bash
|
|
|
# PALM configuration file for Cirrus using Intel Compiler 2019_update5
|
|
|
#column 1 column 2
|
|
|
#name of variable value of variable (~ must not be used)
|
|
|
#----------------------------------------------------------------------------
|
|
|
|
|
|
%base_data $SCRATCH/PALM_JOBS/TENSTREAM
|
|
|
%base_directory $HOME/palm/current_version
|
|
|
%source_path $HOME/palm_model_system/packages/palm/model/src
|
|
|
%user_source_path $base_data/$run_identifier/USER_CODE
|
|
|
%fast_io_catalog $SCRATCH/PALM_TMP
|
|
|
%local_jobcatalog $SCRATCH/PALM_TMP/job_logfiles
|
|
|
%restart_data_path $fast_io_catalog
|
|
|
%output_data_path $base_data
|
|
|
#
|
|
|
%local_username $(whoami)
|
|
|
%local_hostname lcgeohu
|
|
|
%compiler_name mpifort
|
|
|
%compiler_name_ser gfortran
|
|
|
|
|
|
%tenstream_include $HOME/tenstream/build/include
|
|
|
%tenstream_lib $HOME/tenstream/build/lib
|
|
|
%PETSC_LIBS -L/programs/gcc_openmpi-4.1/petsc_single/3.15.4/lib -Wl,-rpath,/programs/gcc_openmpi-4.1/petsc_single/3.15.4/lib -L/programs/gcc_openmpi-4.1/petsc_single/3.15.4/lib -Wl,-rpath,/progr
|
|
|
ams/gcc_openmpi-4.1/hdf5/1.12.0/lib -L/programs/gcc_openmpi-4.1/hdf5/1.12.0/lib -Wl,-rpath,/programs/gcc_openmpi-4.1/openmpi/4.1.1/lib -L/programs/gcc_openmpi-4.1/openmpi/4.1.1/lib -Wl,-rpath,/usr/lib/gc
|
|
|
c/x86_64-redhat-linux/8 -L/usr/lib/gcc/x86_64-redhat-linux/8 -lpetsc -lopenblas -lptesmumps -lptscotchparmetis -lptscotch -lptscotcherr -lesmumps -lscotch -lscotcherr -lhdf5hl_fortran -lhdf5_fortran -lhd
|
|
|
f5_hl -lhdf5 -lparmetis -lmetis -lm -lX11 -lstdc++ -ldl -lmpi_usempif08 -lmpi_usempi_ignore_tkr -lmpi_mpifh -lmpi -lgfortran -lm -lgfortran -lm -lgcc_s -lquadmath -lpthread -lrt -lquadmath -lstdc++ -ldl
|
|
|
|
|
|
%module_commands module load gcc_openmpi; module load netcdf_fortran rrtmg/1.0a; module load petsc_single
|
|
|
%cpp_options -cpp -D__gfortran -D__parallel -DMPI_REAL=MPI_DOUBLE_PRECISION -DMPI_2REAL=MPI_2DOUBLE_PRECISION -D__netcdf -D__netcdf4 -D__netcdf4_parallel -D__fftw -D__tenstream -D__rrtmg
|
|
|
%make_options -j 8
|
|
|
%compiler_options -Ofast -ffree-line-length-none -I /programs/gcc_openmpi-4.1/netcdf_fortran/4.5.3/include -I /usr/include -I /programs/gcc_openmpi-4.1/rrtmg/1.0a/include -I /programs/gcc_openmpi-4.1/
|
|
|
petsc_single/3.15.4/include -I $tenstream_include
|
|
|
%linker_options -Ofast -ffree-line-length-none /programs/gcc_openmpi-4.1/netcdf_fortran/4.5.3/lib/libnetcdff.so /usr/lib64/libfftw3.so /programs/gcc_openmpi-4.1/rrtmg/1.0a/lib/librrtmg.so $PETSC_LIB
|
|
|
S $tenstream_lib/libtenstream.a
|
|
|
%hostfile auto
|
|
|
%execute_command srun -n {{mpi_tasks}} palm
|
|
|
%submit_command sbatch
|
|
|
%memory 2300
|
|
|
#BATCH-directives to be used for batch jobs
|
|
|
BD:#!/bin/bash
|
|
|
BD:#SBATCH --job-name={{job_id}}
|
|
|
#BD:#SBATCH --nodes={{nodes}}
|
|
|
#BD:#SBATCH --ntasks={{tasks_per_node}}
|
|
|
#BD:#SBATCH --ntasks-per-node={{tasks_per_node}}
|
|
|
BD:#SBATCH --time={{cpu_hours}}:{{cpu_minutes}}:{{cpu_seconds}}
|
|
|
BD:#SBATCH --output={{job_protocol_file}}.out
|
|
|
BD:#SBATCH --error={{job_protocol_file}}.err
|
|
|
BD:#SBATCH --qos={{queue}}
|
|
|
BD:#SBATCH --ntasks={{cores}}
|
|
|
|
|
|
IC: module load gcc_openmpi; module load netcdf_fortran rrtmg/1.0a
|
|
|
#
|
|
|
#
|
|
|
#
|
|
|
#----------------------------------------------------------------------------
|
|
|
# INPUT-commands, executed before running PALM - lines must start with "IC:"
|
|
|
export I_MPI_SCALABLE_OPTIMIZATION=off
|
|
|
export I_MPI_DEBUG=5
|
|
|
#----------------------------------------------------------------------------
|
|
|
#IC:
|
|
|
#
|
|
|
#----------------------------------------------------------------------------
|
|
|
# ERROR-commands - executed when program terminates abnormally
|
|
|
#----------------------------------------------------------------------------
|
|
|
EC:[[ \$locat = execution ]] && cat RUN_CONTROL
|
|
|
EC:[[ \$locat = execution ]] && cat PARTICLE_INFOS/*
|
|
|
#
|
|
|
#----------------------------------------------------------------------------
|
|
|
# OUTPUT-commands - executed when program terminates normally
|
|
|
#----------------------------------------------------------------------------
|
|
|
#
|
|
|
# Combine 1D- and 3D-profile output (these files are not usable for plotting)
|
|
|
OC:[[ -f LIST_PROFIL_1D ]] && cat LIST_PROFIL_1D >> LIST_PROFILE
|
|
|
OC:[[ -f LIST_PROFIL ]] && cat LIST_PROFIL >> LIST_PROFILE
|
|
|
#
|
|
|
# Combine all particle information files
|
|
|
OC:[[ -f PARTICLE_INFOS/_0000 ]] && cat PARTICLE_INFOS/* >> PARTICLE_INFO
|
|
|
``` |
|
|
\ No newline at end of file |