Skip to content

In integrate test add force threshold of 0.0001 and stress threhold 0.001… #3088

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
hongriTianqi merged 9 commits into from
Oct 24, 2023
Merged

In integrate test add force threshold of 0.0001 and stress threhold 0.001… #3088

Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
a7bd9cc
In integrate test add force threshold of 0.0001 and stress threhold o…
Zhuxuegang2022 Oct 22, 2023
3536e46
just add space before ' ]'
Zhuxuegang2022 Oct 23, 2023
6ab6b99
add some tab to make the bash script align
Zhuxuegang2022 Oct 23, 2023
f939e52
delet unnecessary 'if ' with 'if...elif...else' is already work
Zhuxuegang2022 Oct 23, 2023
cfd2082
Change tab to space in the 'Autotest.sh'
Zhuxuegang2022 Oct 23, 2023
157c5a6
Merge branch 'develop' into intergratetest
hongriTianqi Oct 24, 2023
ac16c4b
using correct formation of ash
Zhuxuegang2022 Oct 24, 2023
c0ef441
Merge branch 'deepmodeling:develop' into intergratetest
Zhuxuegang2022 Oct 24, 2023
7f2fa04
Merge branch 'intergratetest' of github.com:Zhuxuegang2022/abacus-dev…
Zhuxuegang2022 Oct 24, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
286 changes: 151 additions & 135 deletions tests/integrate/Autotest.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ abacus=abacus
np=4
# threshold with unit: eV
threshold=0.0000001
force_threshold=0.0001
stress_threshold=0.001
# check accuracy
ca=8
# regex of case name
Expand All @@ -18,11 +20,11 @@ do
case "${flag}" in
a) abacus=${OPTARG};;
n) np=${OPTARG};;
t) threshold=${OPTARG};;
c) ca=${OPTARG};;
s) sanitize=${OPTARG};;
r) case=${OPTARG};;
g) g=true;; #generate test reference
t) threshold=${OPTARG};;
c) ca=${OPTARG};;
s) sanitize=${OPTARG};;
r) case=${OPTARG};;
g) g=true;; #generate test reference
esac
done

Expand All @@ -37,7 +39,9 @@ echo "-----AUTO TESTS OF ABACUS ------"
echo "ABACUS path: $abacus"
echo "Number of processes: $np"
echo "Number of threads: $nt"
echo "Test accuracy: $threshold eV"
echo "Test accuracy totenergy: $threshold eV"
echo "Test accuracy force: $force_threshold"
echo "Test accuracy stress: $stress_threshold"
echo "Check accuaracy: $ca"
echo "Test cases: $case"
echo "Generate reference: $g"
Expand All @@ -48,86 +52,98 @@ echo ""
# define a function named 'check_out'
#----------------------------------------------------------
check_out(){
#------------------------------------------------------
# input file $1 is 'result.out' in each test directory
#------------------------------------------------------
outfile=$1

#------------------------------------------------------
# outfile = result.out
#------------------------------------------------------
properties=`awk '{print $1}' $outfile`

#------------------------------------------------------
# jd = job description
#------------------------------------------------------
if test -e "jd"; then
jd=`cat jd`
echo "[----------] $jd"
fi

#------------------------------------------------------
# check every 'key' word
#------------------------------------------------------
for key in $properties; do

#--------------------------------------------------
# calculated value
#--------------------------------------------------
cal=`grep "$key" result.out | awk '{printf "%.'$ca'f\n",$2}'`

#--------------------------------------------------
# reference value
#--------------------------------------------------
ref=`grep "$key" result.ref | awk '{printf "%.'$ca'f\n",$2}'`

#--------------------------------------------------
# computed the deviation between the calculated
# and reference value
#--------------------------------------------------
deviation=`awk 'BEGIN {x='$ref';y='$cal';printf "%.'$ca'f\n",x-y}'`

if [ $key == "totaltimeref" ]; then
# echo "time=$cal ref=$ref"
break
fi


#--------------------------------------------------
# If deviation < threshold, then the test passes,
# otherwise, the test prints out warning
# Daye Zheng found bug on 2021-06-20,
# deviation should be positively defined
#--------------------------------------------------
if [ ! -n "$deviation" ]; then
#------------------------------------------------------
# input file $1 is 'result.out' in each test directory
#------------------------------------------------------
outfile=$1

#------------------------------------------------------
# outfile = result.out
#------------------------------------------------------
properties=`awk '{print $1}' $outfile`

#------------------------------------------------------
# jd = job description
#------------------------------------------------------
if test -e "jd"; then
jd=`cat jd`
echo "[----------] $jd"
fi

#------------------------------------------------------
# check every 'key' word
#------------------------------------------------------
for key in $properties; do

#--------------------------------------------------
# calculated value
#--------------------------------------------------
cal=`grep "$key" result.out | awk '{printf "%.'$ca'f\n",$2}'`

#--------------------------------------------------
# reference value
#--------------------------------------------------
ref=`grep "$key" result.ref | awk '{printf "%.'$ca'f\n",$2}'`

#--------------------------------------------------
# computed the deviation between the calculated
# and reference value
#--------------------------------------------------
deviation=`awk 'BEGIN {x='$ref';y='$cal';printf "%.'$ca'f\n",x-y}'`

if [ $key == "totaltimeref" ]; then
# echo "time=$cal ref=$ref"
break
fi


#--------------------------------------------------
# If deviation < threshold, then the test passes,
# otherwise, the test prints out warning
# Daye Zheng found bug on 2021-06-20,
# deviation should be positively defined
#--------------------------------------------------
if [ ! -n "$deviation" ]; then
echo -e "\e[0;31m[ERROR ] Fatal Error: key $key not found in output.\e[0m"
let fatal++
fatal_case_list+=$dir'\n'
break
let fatal++
fatal_case_list+=$dir'\n'
break
else
if [ $(echo "sqrt($deviation*$deviation) < $threshold"|bc) = 0 ]; then
echo -e "[WARNING ] "\
"$key cal=$cal ref=$ref deviation=$deviation"
let failed++
failed_case_list+=$dir'\n'
if [ $(echo "sqrt($deviation*$deviation) < $fatal_threshold"|bc) = 0 ]; then
let fatal++
fatal_case_list+=$dir
echo -e "\e[0;31m[ERROR ] \e[0m"\
"An unacceptable deviation occurs."
calculation=`grep calculation INPUT | awk '{print $2}' | sed s/[[:space:]]//g`
running_path=`echo "OUT.autotest/running_$calculation"".log"`
cat $running_path
fi
break
else
#echo "$key cal=$cal ref=$ref deviation=$deviation"
#echo "[ PASS ] $key"
echo -e "\e[0;32m[ OK ] \e[0m $key"
fi
fi
let ok++
done
if [ $(echo "sqrt($deviation*$deviation) < $threshold"|bc) = 0 ]; then
if [[ $key == "totalforceref" && $(echo "sqrt($deviation*$deviation) < $force_threshold"|bc) = 0 ]]; then
echo -e "[WARNING ] "\
"$key cal=$cal ref=$ref deviation=$deviation"
let failed++
failed_case_list+=$dir'\n'
elif [[ $key == "totalstressref" && $(echo "sqrt($deviation*$deviation) < $stress_threshold"|bc) = 0 ]]; then
echo -e "[WARNING ] "\
"$key cal=$cal ref=$ref deviation=$deviation"
let failed++
failed_case_list+=$dir'\n'
else
echo -e "[WARNING ] "\
"$key cal=$cal ref=$ref deviation=$deviation"
let failed++
failed_case_list+=$dir'\n'
fi
if [ $(echo "sqrt($deviation*$deviation) < $fatal_threshold"|bc) = 0 ]; then
let fatal++
fatal_case_list+=$dir
echo -e "\e[0;31m[ERROR ] \e[0m"\
"An unacceptable deviation occurs."
calculation=`grep calculation INPUT | awk '{print $2}' | sed s/[[:space:]]//g`
running_path=`echo "OUT.autotest/running_$calculation"".log"`
cat $running_path
fi
break
else
#echo "$key cal=$cal ref=$ref deviation=$deviation"
#echo "[ PASS ] $key"
echo -e "\e[0;32m[ OK ] \e[0m $key"
fi
fi
let ok++
done
}

#---------------------------------------------
Expand All @@ -148,77 +164,77 @@ report=""
repo="$(realpath ..)/"

if [ "$sanitize" == true ]; then
echo "Testing with Address Sanitizer..."
mkdir ../html
echo -e "# Address Sanitizer Diagnostics\n" > ../html/README.md
report=$(realpath ../html/README.md)
echo "Testing with Address Sanitizer..."
mkdir ../html
echo -e "# Address Sanitizer Diagnostics\n" > ../html/README.md
report=$(realpath ../html/README.md)
export ASAN_OPTIONS="log_path=asan"
fi

for dir in $testdir; do
cd $dir
echo -e "\e[0;32m[ RUN ]\e[0m $dir"
TIMEFORMAT='[----------] Time elapsed: %R seconds'
#parallel test
time {
cd $dir
echo -e "\e[0;32m[ RUN ]\e[0m $dir"
TIMEFORMAT='[----------] Time elapsed: %R seconds'
#parallel test
time {
if [ "$case" = "282_NO_RPA" -o "$dir" = "102_PW_BPCG" ]; then
mpirun -np 1 $abacus > log.txt
else
mpirun -np $np $abacus > log.txt
fi

if [ "$sanitize" == true ]; then
echo -e "## Test case ${dir}\n" >> ${report}
for diagnostic in asan.*; do
echo -e "### On process id ${diagnostic}\n" >> ${report}
echo -e "\`\`\`bash" >> ${report}
cat ${diagnostic} >> ${report}
echo -e "\`\`\`\n" >> ${report}
done
fi
#$abacus > log.txt
test -d OUT.autotest || (echo "No 'OUT.autotest' dir presented. Some errors may happened in ABACUS." && exit 1)
if test -z $g
then
../tools/catch_properties.sh result.out
if [ $? == 1 ]; then
echo -e "\e[0;31m [ERROR ] Fatal Error in catch_properties.sh \e[0m"
let fatal++
fatal_case_list+=$dir'\n'
break
else
check_out result.out
fi
else
../tools/catch_properties.sh result.ref
fi
}
echo ""
cd ../
if [ "$sanitize" == true ]; then
echo -e "## Test case ${dir}\n" >> ${report}
for diagnostic in asan.*; do
echo -e "### On process id ${diagnostic}\n" >> ${report}
echo -e "\`\`\`bash" >> ${report}
cat ${diagnostic} >> ${report}
echo -e "\`\`\`\n" >> ${report}
done
fi
#$abacus > log.txt
test -d OUT.autotest || (echo "No 'OUT.autotest' dir presented. Some errors may happened in ABACUS." && exit 1)
if test -z $g
then
../tools/catch_properties.sh result.out
if [ $? == 1 ]; then
echo -e "\e[0;31m [ERROR ] Fatal Error in catch_properties.sh \e[0m"
let fatal++
fatal_case_list+=$dir'\n'
break
else
check_out result.out
fi
else
../tools/catch_properties.sh result.ref
fi
}
echo ""
cd ../
done

if [ "$sanitize" == true ]; then
if [[ `uname` == "Darwin" ]]; then
sed -i '' "s,${repo},,g" ${report}
else
sed -i "s,${repo},,g" ${report}
fi
if [[ `uname` == "Darwin" ]]; then
sed -i '' "s,${repo},,g" ${report}
else
sed -i "s,${repo},,g" ${report}
fi
fi

if [ -z $g ]
then
if [ $failed -eq 0 ]
then
echo -e "\e[0;32m[ PASSED ] \e[0m $ok test cases passed."
echo -e "\e[0;32m[ PASSED ] \e[0m $ok test cases passed."
else
echo -e "[WARNING]\e[0m $failed test cases out of $[ $failed + $ok ] failed."
echo -e $failed_case_list
if [ $fatal -gt 0 ]
then
echo -e "\e[0;31m[ERROR ]\e[0m $fatal test cases out of $[ $failed + $ok ] produced fatal error."
echo -e $fatal_case_list
exit 1
fi
echo -e "[WARNING]\e[0m $failed test cases out of $[ $failed + $ok ] failed."
echo -e $failed_case_list
if [ $fatal -gt 0 ]
then
echo -e "\e[0;31m[ERROR ]\e[0m $fatal test cases out of $[ $failed + $ok ] produced fatal error."
echo -e $fatal_case_list
exit 1
fi
fi
else
echo "Generate test cases complete."
Expand Down