vasp 5.2.2 on Cray XT3
Posted: Tue Jul 07, 2009 1:14 pm
Dear all,
I have some troubles compiling Vasp 5.2.2 on a Cray XT3 (AMD Opteron CPUs with UNICOS/lc operating system, implementing the Linux kernel and the Catamount lightweight kernel).
The compiler has to be called by the ftn wrapper. ftn -V gives:
pgf90 7.2-4 64-bit target on x86-64 Linux -tp k8-64e
The makefile.cray doesn't work out of the box. After changing FC=f90 for
FC=ftn -target=catamount
the flags -f free or -r2 are not recognised.
The 4.6 vasp version could be compiled with Makefiles similar to those reproduced below (NB: the rules are those given by the makefile.cray of the 5.2.2 version). With the 5.2.2 version the compilation stops with the following message:
eccp_nl_:
1666, Unrolled inner loop 4 times
Generated 5 prefetch instructions for this loop
make: *** No rule to make target `xcgrad.f90', needed by `xcgrad.o'. Stop.
I would be grateful for any pointers !
Kind regards,
Anne-Christine
*********************************************
vasp.5.2 Makefile
*********************************************
#-----------------------------------------------------------------------
# Makefile for CRAY
#-----------------------------------------------------------------------
# all CPP processed fortran files have the extension .f
SUFFIX=.f90
# fortran compiler
FC=ftn -target=catamount
CC=cc -target=catamount
FCL=$(FC)
# C-preprocessor
# preprocessor define any of the flags given below
# single_precission single precission BLAS/LAPACK calls
# vector
# essl use ESSL call sequence for DSYGV
# NGXhalf charge density reduced in X direction
# NGZhalf charge density reduced in Z direction
# wNGXhalf gamma point only reduced in X direction
# wNGZhalf gamma point only reduced in Z direction
#CPP = -/lib/cpp -C -P -DHOST=\"Cray\" \
# -Dsingle_BLAS -Dkind8 -DNGXhalf -Dvector -DUSE_ERF $*.F >$*.f
CPP_ = ./preprocess <$*.F | /usr/bin/cpp -P -C -traditional >$*$(SUFFIX)
CPP = $(CPP_) -DHOST=\"Cray_XT3\" -DMPI -Duse_collective -Dkind8 \
-DCACHE_SIZE=32768 -DPGF90 -Davoidalloc -DscaLAPACK
# general fortran flags
FFLAGS = -tp k8-64e -Mextend -Mfree -Mcray=pointer -I/apps/fftw/fftw-3.1.2_PE1.5.47/include/
# optimization
OFLAG = -fastsse -Mipa=fast -Minfo=opt,inline,ipa
OFLAG_HIGH = $(OFLAG)
OBJ_HIGH =
OBJ_NOOPT =
DEBUG =
INLINE = $(OFLAG)
#INLINE = -O3,inline3
# options for linking
#BLAS = -lsci
BLAS = -lacml
LINK = -tp k8-64e
#LIB = ../vasp.5.lib/linpack_single.o ../vasp.5.lib/lapack_single.o -L../vasp.5.lib -ldmy -lsci
LIB = -L../vasp.4.lib -ldmy \
../vasp.4.lib/linpack_double.o $(LAPACK) \
$(BLAS)
FFT3D = fft3dfurth.o fft3dlib.o
#FFT3D = fft3dcray.o fftlib.o
FFT3D = fft3dfurth.o fft3dlib.o
FFT3D = fftw3d.o fft3dlib.o
FFT3D = fftmpi.o fftmpi_map.o fft3dlib.o
#-----------------------------------------------------------------------
# general rules and compile lines
#-----------------------------------------------------------------------
BASIC= symmetry.o symlib.o lattlib.o random.o
SOURCE= base.o mpi.o smart_allocate.o xml.o \
constant.o jacobi.o main_mpi.o scala.o \
asa.o lattice.o poscar.o ini.o xclib.o xclib_grad.o \
radial.o pseudo.o mgrid.o gridq.o ebs.o \
mkpoints.o wave.o wave_mpi.o wave_high.o \
$(BASIC) nonl.o nonlr.o nonl_high.o dfast.o choleski2.o \
mix.o hamil.o xcgrad.o xcspin.o potex1.o potex2.o \
metagga.o constrmag.o cl_shift.o relativistic.o LDApU.o \
paw_base.o egrad.o pawsym.o pawfock.o pawlhf.o paw.o \
mkpoints_full.o charge.o dipol.o pot.o \
dos.o elf.o tet.o tetweight.o hamil_rot.o \
steep.o chain.o dyna.o sphpro.o us.o core_rel.o \
aedens.o wavpre.o wavpre_noio.o broyden.o \
dynbr.o rmm-diis.o reader.o writer.o tutor.o xml_writer.o \
brent.o stufak.o fileio.o opergrid.o stepver.o \
chgloc.o fast_aug.o fock.o mkpoints_change.o sym_grad.o \
mymath.o internals.o dimer_heyden.o dvvtrajectory.o vdwforcefield.o \
hamil_high.o nmr.o force.o \
pead.o subrot.o subrot_scf.o pwlhf.o gw_model.o optreal.o davidson.o \
electron.o rot.o electron_all.o shm.o pardens.o paircorrection.o \
optics.o constr_cell_relax.o stm.o finite_diff.o elpol.o \
hamil_lr.o rmm-diis_lr.o subrot_cluster.o subrot_lr.o \
lr_helper.o hamil_lrf.o elinear_response.o ilinear_response.o \
linear_optics.o linear_response.o \
setlocalpp.o wannier.o electron_OEP.o electron_lhf.o twoelectron4o.o \
ratpol.o screened_2e.o wave_cacher.o chi_base.o wpot.o local_field.o \
ump2.o bse.o acfdt.o chi.o sydmat.o
INC=
vasp: $(SOURCE) $(FFT3D) $(INC) main.o
rm -f vasp
$(FCL) -o vasp main.o $(SOURCE) $(FFT3D) $(LIB) $(LINK)
makeparam: $(SOURCE) $(FFT3D) makeparam.o main.F $(INC)
$(FCL) -o makeparam $(LINK) makeparam.o $(SOURCE) $(FFT3D) $(LIB)
zgemmtest: zgemmtest.o base.o random.o $(INC)
$(FCL) -o zgemmtest $(LINK) zgemmtest.o random.o base.o $(LIB)
dgemmtest: dgemmtest.o base.o random.o $(INC)
$(FCL) -o dgemmtest $(LINK) dgemmtest.o random.o base.o $(LIB)
ffttest: base.o smart_allocate.o mpi.o mgrid.o random.o ffttest.o $(FFT3D) $(INC)
$(FCL) -o ffttest $(LINK) ffttest.o mpi.o mgrid.o random.o smart_allocate.o base.o $(FFT3D) $(LIB)
kpoints: $(SOURCE) $(FFT3D) makekpoints.o main.F $(INC)
$(FCL) -o kpoints $(LINK) makekpoints.o $(SOURCE) $(FFT3D) $(LIB)
clean:
-rm -f *.g *.f *.o *.L *.mod ; touch *.F
main.o: main$(SUFFIX)
$(FC) $(FFLAGS)$(DEBUG) $(INCS) -c main$(SUFFIX)
xcgrad.o: xcgrad$(SUFFIX)
$(FC) $(FFLAGS) $(INLINE) $(INCS) -c xcgrad$(SUFFIX)
xcspin.o: xcspin$(SUFFIX)
$(FC) $(FFLAGS) $(INLINE) $(INCS) -c xcspin$(SUFFIX)
makeparam.o: makeparam$(SUFFIX)
$(FC) $(FFLAGS)$(DEBUG) $(INCS) -c makeparam$(SUFFIX)
makeparam$(SUFFIX): makeparam.F main.F
#
# MIND: I do not have a full dependency list for the include
# and MODULES: here are only the minimal basic dependencies
# if one strucuture is changed then touch_dep must be called
# with the corresponding name of the structure
#
base.o: base.inc base.F
mgrid.o: mgrid.inc mgrid.F
constant.o: constant.inc constant.F
lattice.o: lattice.inc lattice.F
setex.o: setexm.inc setex.F
pseudo.o: pseudo.inc pseudo.F
poscar.o: poscar.inc poscar.F
mkpoints.o: mkpoints.inc mkpoints.F
wave.o: wave.inc wave.F
nonl.o: nonl.inc nonl.F
nonlr.o: nonlr.inc nonlr.F
$(OBJ_HIGH):
$(CPP)
$(FC) $(FFLAGS) $(OFLAG_HIGH) $(INCS) -c $*$(SUFFIX)
$(OBJ_NOOPT):
$(CPP)
$(FC) $(FFLAGS) $(INCS) -c $*$(SUFFIX)
fft3dlib_f77.o: fft3dlib_f77.F
$(CPP)
$(F77) $(FFLAGS_F77) -c $*$(SUFFIX)
.F.o:
$(CPP)
$(FC) $(FFLAGS) $(OFLAG) $(INCS) -c $*$(SUFFIX)
.F$(SUFFIX):
$(CPP)
$(SUFFIX).o:
$(FC) $(FFLAGS) $(OFLAG) $(INCS) -c $*$(SUFFIX)
*********************************************
vasp.5.lib Makefile (compiles fine)
*********************************************
.SUFFIXES: .inc .f .F
#-----------------------------------------------------------------------
# Makefile for CRAY
#-----------------------------------------------------------------------
# fortran compiler
FC=ftn -target=catamount
# C-preprocessor
CPP= gcc -E -P -C $*.F >$*.f
CC= cc -target=catamount
CFLAGS = -O2
FFLAGS = -fastsse -tp k8-64e -Minfo=opt,inline,ipa -Mcray=pointer
FREE = -Mfree
#DOBJ = preclib.o timing.fujitsu.o diolib.o dlexlib.o drdatab.o crayerrf.o \
# sclock_cray.o
DOBJ = preclib.o timing_.o derrf_.o dclock_.o diolib.o dlexlib.o drdatab.o
#-----------------------------------------------------------------------
# general rules
#-----------------------------------------------------------------------
libdmy.a: $(DOBJ) lapack_double.o linpack_double.o
-rm libdmy.a
ar vq libdmy.a $(DOBJ)
# files which do not require autodouble
lapack_double.o: lapack_double.f
$(FC) $(FFLAGS) $(NOFREE) -c lapack_double.f
lapack_single.o: lapack_single.f
$(FC) $(FFLAGS) $(NOFREE) -c lapack_single.f
linpack_double.o: linpack_double.f
$(FC) $(FFLAGS) $(NOFREE) -c linpack_double.f
linpack_single.o: linpack_single.f
$(FC) $(FFLAGS) $(NOFREE) -c linpack_single.f
.c.o:
$(CC) $(CFLAGS) -c $*.c
.F.o:
$(CPP)
$(FC) $(FFLAGS) $(FREE) $(INCS) -c $*.f
.F.f:
$(CPP)
.f.o:
$(FC) $(FFLAGS) $(FREE) $(INCS) -c $*.f
I have some troubles compiling Vasp 5.2.2 on a Cray XT3 (AMD Opteron CPUs with UNICOS/lc operating system, implementing the Linux kernel and the Catamount lightweight kernel).
The compiler has to be called by the ftn wrapper. ftn -V gives:
pgf90 7.2-4 64-bit target on x86-64 Linux -tp k8-64e
The makefile.cray doesn't work out of the box. After changing FC=f90 for
FC=ftn -target=catamount
the flags -f free or -r2 are not recognised.
The 4.6 vasp version could be compiled with Makefiles similar to those reproduced below (NB: the rules are those given by the makefile.cray of the 5.2.2 version). With the 5.2.2 version the compilation stops with the following message:
eccp_nl_:
1666, Unrolled inner loop 4 times
Generated 5 prefetch instructions for this loop
make: *** No rule to make target `xcgrad.f90', needed by `xcgrad.o'. Stop.
I would be grateful for any pointers !
Kind regards,
Anne-Christine
*********************************************
vasp.5.2 Makefile
*********************************************
#-----------------------------------------------------------------------
# Makefile for CRAY
#-----------------------------------------------------------------------
# all CPP processed fortran files have the extension .f
SUFFIX=.f90
# fortran compiler
FC=ftn -target=catamount
CC=cc -target=catamount
FCL=$(FC)
# C-preprocessor
# preprocessor define any of the flags given below
# single_precission single precission BLAS/LAPACK calls
# vector
# essl use ESSL call sequence for DSYGV
# NGXhalf charge density reduced in X direction
# NGZhalf charge density reduced in Z direction
# wNGXhalf gamma point only reduced in X direction
# wNGZhalf gamma point only reduced in Z direction
#CPP = -/lib/cpp -C -P -DHOST=\"Cray\" \
# -Dsingle_BLAS -Dkind8 -DNGXhalf -Dvector -DUSE_ERF $*.F >$*.f
CPP_ = ./preprocess <$*.F | /usr/bin/cpp -P -C -traditional >$*$(SUFFIX)
CPP = $(CPP_) -DHOST=\"Cray_XT3\" -DMPI -Duse_collective -Dkind8 \
-DCACHE_SIZE=32768 -DPGF90 -Davoidalloc -DscaLAPACK
# general fortran flags
FFLAGS = -tp k8-64e -Mextend -Mfree -Mcray=pointer -I/apps/fftw/fftw-3.1.2_PE1.5.47/include/
# optimization
OFLAG = -fastsse -Mipa=fast -Minfo=opt,inline,ipa
OFLAG_HIGH = $(OFLAG)
OBJ_HIGH =
OBJ_NOOPT =
DEBUG =
INLINE = $(OFLAG)
#INLINE = -O3,inline3
# options for linking
#BLAS = -lsci
BLAS = -lacml
LINK = -tp k8-64e
#LIB = ../vasp.5.lib/linpack_single.o ../vasp.5.lib/lapack_single.o -L../vasp.5.lib -ldmy -lsci
LIB = -L../vasp.4.lib -ldmy \
../vasp.4.lib/linpack_double.o $(LAPACK) \
$(BLAS)
FFT3D = fft3dfurth.o fft3dlib.o
#FFT3D = fft3dcray.o fftlib.o
FFT3D = fft3dfurth.o fft3dlib.o
FFT3D = fftw3d.o fft3dlib.o
FFT3D = fftmpi.o fftmpi_map.o fft3dlib.o
#-----------------------------------------------------------------------
# general rules and compile lines
#-----------------------------------------------------------------------
BASIC= symmetry.o symlib.o lattlib.o random.o
SOURCE= base.o mpi.o smart_allocate.o xml.o \
constant.o jacobi.o main_mpi.o scala.o \
asa.o lattice.o poscar.o ini.o xclib.o xclib_grad.o \
radial.o pseudo.o mgrid.o gridq.o ebs.o \
mkpoints.o wave.o wave_mpi.o wave_high.o \
$(BASIC) nonl.o nonlr.o nonl_high.o dfast.o choleski2.o \
mix.o hamil.o xcgrad.o xcspin.o potex1.o potex2.o \
metagga.o constrmag.o cl_shift.o relativistic.o LDApU.o \
paw_base.o egrad.o pawsym.o pawfock.o pawlhf.o paw.o \
mkpoints_full.o charge.o dipol.o pot.o \
dos.o elf.o tet.o tetweight.o hamil_rot.o \
steep.o chain.o dyna.o sphpro.o us.o core_rel.o \
aedens.o wavpre.o wavpre_noio.o broyden.o \
dynbr.o rmm-diis.o reader.o writer.o tutor.o xml_writer.o \
brent.o stufak.o fileio.o opergrid.o stepver.o \
chgloc.o fast_aug.o fock.o mkpoints_change.o sym_grad.o \
mymath.o internals.o dimer_heyden.o dvvtrajectory.o vdwforcefield.o \
hamil_high.o nmr.o force.o \
pead.o subrot.o subrot_scf.o pwlhf.o gw_model.o optreal.o davidson.o \
electron.o rot.o electron_all.o shm.o pardens.o paircorrection.o \
optics.o constr_cell_relax.o stm.o finite_diff.o elpol.o \
hamil_lr.o rmm-diis_lr.o subrot_cluster.o subrot_lr.o \
lr_helper.o hamil_lrf.o elinear_response.o ilinear_response.o \
linear_optics.o linear_response.o \
setlocalpp.o wannier.o electron_OEP.o electron_lhf.o twoelectron4o.o \
ratpol.o screened_2e.o wave_cacher.o chi_base.o wpot.o local_field.o \
ump2.o bse.o acfdt.o chi.o sydmat.o
INC=
vasp: $(SOURCE) $(FFT3D) $(INC) main.o
rm -f vasp
$(FCL) -o vasp main.o $(SOURCE) $(FFT3D) $(LIB) $(LINK)
makeparam: $(SOURCE) $(FFT3D) makeparam.o main.F $(INC)
$(FCL) -o makeparam $(LINK) makeparam.o $(SOURCE) $(FFT3D) $(LIB)
zgemmtest: zgemmtest.o base.o random.o $(INC)
$(FCL) -o zgemmtest $(LINK) zgemmtest.o random.o base.o $(LIB)
dgemmtest: dgemmtest.o base.o random.o $(INC)
$(FCL) -o dgemmtest $(LINK) dgemmtest.o random.o base.o $(LIB)
ffttest: base.o smart_allocate.o mpi.o mgrid.o random.o ffttest.o $(FFT3D) $(INC)
$(FCL) -o ffttest $(LINK) ffttest.o mpi.o mgrid.o random.o smart_allocate.o base.o $(FFT3D) $(LIB)
kpoints: $(SOURCE) $(FFT3D) makekpoints.o main.F $(INC)
$(FCL) -o kpoints $(LINK) makekpoints.o $(SOURCE) $(FFT3D) $(LIB)
clean:
-rm -f *.g *.f *.o *.L *.mod ; touch *.F
main.o: main$(SUFFIX)
$(FC) $(FFLAGS)$(DEBUG) $(INCS) -c main$(SUFFIX)
xcgrad.o: xcgrad$(SUFFIX)
$(FC) $(FFLAGS) $(INLINE) $(INCS) -c xcgrad$(SUFFIX)
xcspin.o: xcspin$(SUFFIX)
$(FC) $(FFLAGS) $(INLINE) $(INCS) -c xcspin$(SUFFIX)
makeparam.o: makeparam$(SUFFIX)
$(FC) $(FFLAGS)$(DEBUG) $(INCS) -c makeparam$(SUFFIX)
makeparam$(SUFFIX): makeparam.F main.F
#
# MIND: I do not have a full dependency list for the include
# and MODULES: here are only the minimal basic dependencies
# if one strucuture is changed then touch_dep must be called
# with the corresponding name of the structure
#
base.o: base.inc base.F
mgrid.o: mgrid.inc mgrid.F
constant.o: constant.inc constant.F
lattice.o: lattice.inc lattice.F
setex.o: setexm.inc setex.F
pseudo.o: pseudo.inc pseudo.F
poscar.o: poscar.inc poscar.F
mkpoints.o: mkpoints.inc mkpoints.F
wave.o: wave.inc wave.F
nonl.o: nonl.inc nonl.F
nonlr.o: nonlr.inc nonlr.F
$(OBJ_HIGH):
$(CPP)
$(FC) $(FFLAGS) $(OFLAG_HIGH) $(INCS) -c $*$(SUFFIX)
$(OBJ_NOOPT):
$(CPP)
$(FC) $(FFLAGS) $(INCS) -c $*$(SUFFIX)
fft3dlib_f77.o: fft3dlib_f77.F
$(CPP)
$(F77) $(FFLAGS_F77) -c $*$(SUFFIX)
.F.o:
$(CPP)
$(FC) $(FFLAGS) $(OFLAG) $(INCS) -c $*$(SUFFIX)
.F$(SUFFIX):
$(CPP)
$(SUFFIX).o:
$(FC) $(FFLAGS) $(OFLAG) $(INCS) -c $*$(SUFFIX)
*********************************************
vasp.5.lib Makefile (compiles fine)
*********************************************
.SUFFIXES: .inc .f .F
#-----------------------------------------------------------------------
# Makefile for CRAY
#-----------------------------------------------------------------------
# fortran compiler
FC=ftn -target=catamount
# C-preprocessor
CPP= gcc -E -P -C $*.F >$*.f
CC= cc -target=catamount
CFLAGS = -O2
FFLAGS = -fastsse -tp k8-64e -Minfo=opt,inline,ipa -Mcray=pointer
FREE = -Mfree
#DOBJ = preclib.o timing.fujitsu.o diolib.o dlexlib.o drdatab.o crayerrf.o \
# sclock_cray.o
DOBJ = preclib.o timing_.o derrf_.o dclock_.o diolib.o dlexlib.o drdatab.o
#-----------------------------------------------------------------------
# general rules
#-----------------------------------------------------------------------
libdmy.a: $(DOBJ) lapack_double.o linpack_double.o
-rm libdmy.a
ar vq libdmy.a $(DOBJ)
# files which do not require autodouble
lapack_double.o: lapack_double.f
$(FC) $(FFLAGS) $(NOFREE) -c lapack_double.f
lapack_single.o: lapack_single.f
$(FC) $(FFLAGS) $(NOFREE) -c lapack_single.f
linpack_double.o: linpack_double.f
$(FC) $(FFLAGS) $(NOFREE) -c linpack_double.f
linpack_single.o: linpack_single.f
$(FC) $(FFLAGS) $(NOFREE) -c linpack_single.f
.c.o:
$(CC) $(CFLAGS) -c $*.c
.F.o:
$(CPP)
$(FC) $(FFLAGS) $(FREE) $(INCS) -c $*.f
.F.f:
$(CPP)
.f.o:
$(FC) $(FFLAGS) $(FREE) $(INCS) -c $*.f