From eec734ae2e43f8d48e02d4ce3700c9f43c10e252 Mon Sep 17 00:00:00 2001 From: amartin Date: Thu, 21 Oct 2021 14:28:57 +1100 Subject: [PATCH 01/13] Generating sysimage in Github actions in order to reduce MPI tests time --- .github/workflows/ci.yml | 3 +- test/mpi/compile/compile.jl | 62 +++++++++++++++++++++++++++++++++++++ test/mpi/compile/compile.sh | 21 +++++++++++++ test/mpi/runtests.jl | 21 ++++++++++--- 4 files changed, 101 insertions(+), 6 deletions(-) create mode 100644 test/mpi/compile/compile.jl create mode 100755 test/mpi/compile/compile.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4d3ebc81..95f6a3a4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,7 +66,8 @@ jobs: version: ${{ matrix.version }} arch: ${{ matrix.arch }} - uses: julia-actions/julia-buildpkg@v1 - - run: julia --project=. --color=yes --check-bounds=yes test/mpi/runtests.jl + - run: cd test/mpi/compile; ./compile.sh PoissonTests.jl + - run: julia --project=. --color=yes --check-bounds=yes test/mpi/runtests.jl test/mpi/compile/GridapDistributedMPIBackend.so - uses: julia-actions/julia-processcoverage@v1 - uses: codecov/codecov-action@v1 with: diff --git a/test/mpi/compile/compile.jl b/test/mpi/compile/compile.jl new file mode 100644 index 00000000..2e141e92 --- /dev/null +++ b/test/mpi/compile/compile.jl @@ -0,0 +1,62 @@ +using Pkg +Pkg.add("PackageCompiler") +using PackageCompiler + +@assert length(ARGS)==1 +test_script=ARGS[1] +@assert isfile("../../"*test_script) +modulename=split(test_script,".")[1] + +function generate_precompile_execution_file(modulename) + source_code=""" + module sysimagegenerator + + using PartitionedArrays + const PArrays = PartitionedArrays + using MPI + + if ! MPI.Initialized() + MPI.Init() + end + + include(\"../../$(modulename).jl\") + + @assert MPI.Comm_size(MPI.COMM_WORLD) == 1 + parts = get_part_ids(mpi,(1,1)) + + display(parts) + + t = PArrays.PTimer(parts,verbose=true) + PArrays.tic!(t) + + eval(Meta.parse(\"$(modulename).main(parts)\")) + PArrays.toc!(t,\"$(modulename)\") + display(t) + + MPI.Finalize() + + end #module + """ + open("sysimagegenerator.jl","w") do io + println(io,source_code) + end +end + +@assert length(ARGS)==1 +test_script=ARGS[1] +@assert isfile("../../"*test_script) +modulename=split(test_script,".")[1] +generate_precompile_execution_file(modulename) + +pkgs = Symbol[] +push!(pkgs, :GridapDistributed) + +if VERSION >= v"1.4" + append!(pkgs, [Symbol(v.name) for v in values(Pkg.dependencies()) if v.is_direct_dep],) +else + append!(pkgs, [Symbol(name) for name in keys(Pkg.installed())]) +end + +create_sysimage(pkgs, + sysimage_path=joinpath(@__DIR__,"GridapDistributedMPIBackend.so"), + precompile_execution_file=joinpath(@__DIR__,"sysimagegenerator.jl")) diff --git a/test/mpi/compile/compile.sh b/test/mpi/compile/compile.sh new file mode 100755 index 00000000..1ce2a825 --- /dev/null +++ b/test/mpi/compile/compile.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +if [ "$#" -ne 1 ]; then + test_paths=$(ls ../../*Tests.jl) + test_names="" + for i in $test_paths + do + name=$(basename $i) + test_names="$test_names $name" + done + echo "Illegal number of parameters" + echo "Usage: $0 TESTNAME.jl, where TESTNAME.jl=$test_names" + exit 1 +fi + +# This script is to be executed from this folder (compile/) +julia --project=../../.. --color=yes -e 'using Pkg; Pkg.instantiate()' +# See https://juliaparallel.github.io/MPI.jl/latest/knownissues/#Julia-module-precompilation-1 +# for a justification of this line +julia --project=../../.. --color=yes -e 'using Pkg; pkg"precompile"' +julia --project=../../.. -O3 --check-bounds=no --color=yes compile.jl $1 diff --git a/test/mpi/runtests.jl b/test/mpi/runtests.jl index 8e150ba0..6f6450ba 100644 --- a/test/mpi/runtests.jl +++ b/test/mpi/runtests.jl @@ -3,19 +3,30 @@ module MPITests using MPI using Test +#Sysimage +sysimage=nothing +if length(ARGS)==1 + @assert isfile(ARGS[1]) "$(ARGS[1]) must be a valid Julia sysimage file" + sysimage=ARGS[1] +end + mpidir = @__DIR__ testdir = joinpath(mpidir,"..") repodir = joinpath(testdir,"..") - -function run_driver(procs,file) +function run_driver(procs,file,sysimage) mpiexec() do cmd - run(`$cmd -n $procs $(Base.julia_cmd()) --project=$repodir $(joinpath(mpidir,file))`) + if sysimage!=nothing + extra_args="-J$(sysimage)" + run(`$cmd -n $procs $(Base.julia_cmd()) $(extra_args) --project=$repodir $(joinpath(mpidir,file))`) + else + run(`$cmd -n $procs $(Base.julia_cmd()) --project=$repodir $(joinpath(mpidir,file))`) + end @test true end end -run_driver(4,"runtests_np4.jl") -run_driver(1,"runtests_np4.jl") # Check that the degenerated case works +run_driver(4,"runtests_np4.jl",sysimage) +run_driver(1,"runtests_np4.jl",sysimage) # Check that the degenerated case works end # module From 519ee363729219a424eaa7a77ababe5435793d49 Mon Sep 17 00:00:00 2001 From: amartin Date: Thu, 21 Oct 2021 17:17:17 +1100 Subject: [PATCH 02/13] Generated Project.toml and Manifest.toml in TestApp --- test/TestApp/Manifest.toml | 512 +++++++++++++++++++++++++++++++++++++ test/TestApp/Project.toml | 4 + 2 files changed, 516 insertions(+) create mode 100644 test/TestApp/Manifest.toml create mode 100644 test/TestApp/Project.toml diff --git a/test/TestApp/Manifest.toml b/test/TestApp/Manifest.toml new file mode 100644 index 00000000..def66519 --- /dev/null +++ b/test/TestApp/Manifest.toml @@ -0,0 +1,512 @@ +# This file is machine-generated - editing it directly is not advised + +[[AbstractTrees]] +git-tree-sha1 = "03e0550477d86222521d254b741d470ba17ea0b5" +uuid = "1520ce14-60c1-5f80-bbc7-55ef81b5835c" +version = "0.3.4" + +[[ArgTools]] +uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" + +[[ArrayInterface]] +deps = ["Compat", "IfElse", "LinearAlgebra", "Requires", "SparseArrays", "Static"] +git-tree-sha1 = "1d6835607e9f214cb4210310868f8cf07eb0facc" +uuid = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9" +version = "3.1.34" + +[[ArrayLayouts]] +deps = ["FillArrays", "LinearAlgebra", "SparseArrays"] +git-tree-sha1 = "7a92ea1dd16472d18ca1ffcbb7b3cc67d7e78a3f" +uuid = "4c555306-a7a7-4459-81d9-ec55ddd5c99a" +version = "0.7.7" + +[[Artifacts]] +uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" + +[[BSON]] +git-tree-sha1 = "ebcd6e22d69f21249b7b8668351ebf42d6dc87a1" +uuid = "fbb218c0-5317-5bc6-957e-2ee96dd4b1f0" +version = "0.3.4" + +[[Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" + +[[BlockArrays]] +deps = ["ArrayLayouts", "FillArrays", "LinearAlgebra"] +git-tree-sha1 = "5524e27323cf4c4505699c3fb008c3f772269945" +uuid = "8e7c35d0-a365-5155-bbbb-fb81a777f24e" +version = "0.16.9" + +[[ChainRulesCore]] +deps = ["Compat", "LinearAlgebra", "SparseArrays"] +git-tree-sha1 = "d9e40e3e370ee56c5b57e0db651d8f92bce98fea" +uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" +version = "1.10.1" + +[[CodecZlib]] +deps = ["TranscodingStreams", "Zlib_jll"] +git-tree-sha1 = "ded953804d019afa9a3f98981d99b33e3db7b6da" +uuid = "944b1d66-785c-5afd-91f1-9de20f533193" +version = "0.7.0" + +[[Combinatorics]] +git-tree-sha1 = "08c8b6831dc00bfea825826be0bc8336fc369860" +uuid = "861a8166-3701-5b0c-9a16-15d98fcdc6aa" +version = "1.0.2" + +[[CommonSubexpressions]] +deps = ["MacroTools", "Test"] +git-tree-sha1 = "7b8a93dba8af7e3b42fecabf646260105ac373f7" +uuid = "bbf7d656-a473-5ed7-a52c-81e309532950" +version = "0.3.0" + +[[Compat]] +deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "SHA", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"] +git-tree-sha1 = "31d0151f5716b655421d9d75b7fa74cc4e744df2" +uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" +version = "3.39.0" + +[[CompilerSupportLibraries_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" + +[[DataStructures]] +deps = ["Compat", "InteractiveUtils", "OrderedCollections"] +git-tree-sha1 = "7d9d316f04214f7efdbb6398d545446e246eff02" +uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" +version = "0.18.10" + +[[Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" + +[[DelimitedFiles]] +deps = ["Mmap"] +uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab" + +[[DiffResults]] +deps = ["StaticArrays"] +git-tree-sha1 = "c18e98cba888c6c25d1c3b048e4b3380ca956805" +uuid = "163ba53b-c6d8-5494-b064-1a9d43ac40c5" +version = "1.0.3" + +[[DiffRules]] +deps = ["NaNMath", "Random", "SpecialFunctions"] +git-tree-sha1 = "7220bc21c33e990c14f4a9a319b1d242ebc5b269" +uuid = "b552c78f-8df3-52c6-915a-8e097449b14b" +version = "1.3.1" + +[[Distances]] +deps = ["LinearAlgebra", "Statistics", "StatsAPI"] +git-tree-sha1 = "9f46deb4d4ee4494ffb5a40a27a2aced67bdd838" +uuid = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7" +version = "0.10.4" + +[[Distributed]] +deps = ["Random", "Serialization", "Sockets"] +uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" + +[[DocStringExtensions]] +deps = ["LibGit2"] +git-tree-sha1 = "a32185f5428d3986f47c2ab78b1f216d5e6cc96f" +uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" +version = "0.8.5" + +[[Downloads]] +deps = ["ArgTools", "LibCURL", "NetworkOptions"] +uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" + +[[FastGaussQuadrature]] +deps = ["LinearAlgebra", "SpecialFunctions", "StaticArrays"] +git-tree-sha1 = "5829b25887e53fb6730a9df2ff89ed24baa6abf6" +uuid = "442a2c76-b920-505d-bb47-c5924d526838" +version = "0.4.7" + +[[FileIO]] +deps = ["Pkg", "Requires", "UUIDs"] +git-tree-sha1 = "3c041d2ac0a52a12a27af2782b34900d9c3ee68c" +uuid = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" +version = "1.11.1" + +[[FillArrays]] +deps = ["LinearAlgebra", "Random", "SparseArrays", "Statistics"] +git-tree-sha1 = "8756f9935b7ccc9064c6eef0bff0ad643df733a3" +uuid = "1a297f60-69ca-5386-bcde-b61e274b549b" +version = "0.12.7" + +[[FiniteDiff]] +deps = ["ArrayInterface", "LinearAlgebra", "Requires", "SparseArrays", "StaticArrays"] +git-tree-sha1 = "8b3c09b56acaf3c0e581c66638b85c8650ee9dca" +uuid = "6a86dc24-6348-571c-b903-95158fe2bd41" +version = "2.8.1" + +[[ForwardDiff]] +deps = ["CommonSubexpressions", "DiffResults", "DiffRules", "LinearAlgebra", "NaNMath", "Preferences", "Printf", "Random", "SpecialFunctions", "StaticArrays"] +git-tree-sha1 = "63777916efbcb0ab6173d09a658fb7f2783de485" +uuid = "f6369f11-7733-5829-9624-2563aa707210" +version = "0.10.21" + +[[Gridap]] +deps = ["AbstractTrees", "BSON", "BlockArrays", "Combinatorics", "DocStringExtensions", "FastGaussQuadrature", "FileIO", "FillArrays", "ForwardDiff", "JLD2", "JSON", "LineSearches", "LinearAlgebra", "NLsolve", "NearestNeighbors", "QuadGK", "Random", "SparseArrays", "SparseMatricesCSR", "StaticArrays", "Test", "WriteVTK"] +git-tree-sha1 = "1c5c6b7cbdc5895f690d731e3653d01636eaa243" +repo-rev = "gridap_distributed" +repo-url = "https://github.com/gridap/Gridap.jl" +uuid = "56d4f2e9-7ea1-5844-9cf6-b9c51ca7ce8e" +version = "0.17.0" + +[[GridapDistributed]] +deps = ["FillArrays", "Gridap", "LinearAlgebra", "MPI", "PartitionedArrays", "SparseArrays", "WriteVTK"] +path = "../.." +uuid = "f9701e48-63b3-45aa-9a63-9bc6c271f355" +version = "0.2.0" + +[[IfElse]] +git-tree-sha1 = "28e837ff3e7a6c3cdb252ce49fb412c8eb3caeef" +uuid = "615f187c-cbe4-4ef1-ba3b-2fcf58d6d173" +version = "0.1.0" + +[[InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" + +[[InverseFunctions]] +deps = ["Test"] +git-tree-sha1 = "f0c6489b12d28fb4c2103073ec7452f3423bd308" +uuid = "3587e190-3f89-42d0-90ee-14403ec27112" +version = "0.1.1" + +[[IrrationalConstants]] +git-tree-sha1 = "7fd44fd4ff43fc60815f8e764c0f352b83c49151" +uuid = "92d709cd-6900-40b7-9082-c6be49f344b6" +version = "0.1.1" + +[[IterativeSolvers]] +deps = ["LinearAlgebra", "Printf", "Random", "RecipesBase", "SparseArrays"] +git-tree-sha1 = "1a8c6237e78b714e901e406c096fc8a65528af7d" +uuid = "42fd0dbc-a981-5370-80f2-aaf504508153" +version = "0.9.1" + +[[JLD2]] +deps = ["DataStructures", "FileIO", "MacroTools", "Mmap", "Pkg", "Printf", "Reexport", "TranscodingStreams", "UUIDs"] +git-tree-sha1 = "46b7834ec8165c541b0b5d1c8ba63ec940723ffb" +uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819" +version = "0.4.15" + +[[JLLWrappers]] +deps = ["Preferences"] +git-tree-sha1 = "642a199af8b68253517b80bd3bfd17eb4e84df6e" +uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" +version = "1.3.0" + +[[JSON]] +deps = ["Dates", "Mmap", "Parsers", "Unicode"] +git-tree-sha1 = "8076680b162ada2a031f707ac7b4953e30667a37" +uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +version = "0.21.2" + +[[LibCURL]] +deps = ["LibCURL_jll", "MozillaCACerts_jll"] +uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" + +[[LibCURL_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] +uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" + +[[LibGit2]] +deps = ["Base64", "NetworkOptions", "Printf", "SHA"] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" + +[[LibSSH2_jll]] +deps = ["Artifacts", "Libdl", "MbedTLS_jll"] +uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" + +[[Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" + +[[Libiconv_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "42b62845d70a619f063a7da093d995ec8e15e778" +uuid = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531" +version = "1.16.1+1" + +[[LightXML]] +deps = ["Libdl", "XML2_jll"] +git-tree-sha1 = "e129d9391168c677cd4800f5c0abb1ed8cb3794f" +uuid = "9c8b4983-aa76-5018-a973-4c85ecc9e179" +version = "0.9.0" + +[[LineSearches]] +deps = ["LinearAlgebra", "NLSolversBase", "NaNMath", "Parameters", "Printf"] +git-tree-sha1 = "f27132e551e959b3667d8c93eae90973225032dd" +uuid = "d3d80556-e9d4-5f37-9878-2ab0fcc64255" +version = "7.1.1" + +[[LinearAlgebra]] +deps = ["Libdl"] +uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" + +[[LogExpFunctions]] +deps = ["ChainRulesCore", "DocStringExtensions", "InverseFunctions", "IrrationalConstants", "LinearAlgebra"] +git-tree-sha1 = "6193c3815f13ba1b78a51ce391db8be016ae9214" +uuid = "2ab3a3ac-af41-5b50-aa03-7779005ae688" +version = "0.3.4" + +[[Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" + +[[MPI]] +deps = ["Distributed", "DocStringExtensions", "Libdl", "MPICH_jll", "MicrosoftMPI_jll", "OpenMPI_jll", "Pkg", "Random", "Requires", "Serialization", "Sockets"] +git-tree-sha1 = "340d8dc89e1c85a846d3f38ee294bfdd1684055a" +uuid = "da04e1cc-30fd-572f-bb4f-1f8673147195" +version = "0.19.1" + +[[MPICH_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "c6cafe3f9747c0a0740611e2dffc4d37248fb691" +uuid = "7cb0a576-ebde-5e09-9194-50597f1243b4" +version = "3.4.2+0" + +[[MacroTools]] +deps = ["Markdown", "Random"] +git-tree-sha1 = "5a5bc6bf062f0f95e62d0fe0a2d99699fed82dd9" +uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" +version = "0.5.8" + +[[Markdown]] +deps = ["Base64"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" + +[[MbedTLS_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" + +[[MicrosoftMPI_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "e5c90234b3967684c9c6f87b4a54549b4ce21836" +uuid = "9237b28f-5490-5468-be7b-bb81f5f5e6cf" +version = "10.1.3+0" + +[[Mmap]] +uuid = "a63ad114-7e13-5084-954f-fe012c677804" + +[[MozillaCACerts_jll]] +uuid = "14a3606d-f60d-562e-9121-12d972cd8159" + +[[NLSolversBase]] +deps = ["DiffResults", "Distributed", "FiniteDiff", "ForwardDiff"] +git-tree-sha1 = "144bab5b1443545bc4e791536c9f1eacb4eed06a" +uuid = "d41bc354-129a-5804-8e4c-c37616107c6c" +version = "7.8.1" + +[[NLsolve]] +deps = ["Distances", "LineSearches", "LinearAlgebra", "NLSolversBase", "Printf", "Reexport"] +git-tree-sha1 = "019f12e9a1a7880459d0173c182e6a99365d7ac1" +uuid = "2774e3e8-f4cf-5e23-947b-6d7e65073b56" +version = "4.5.1" + +[[NaNMath]] +git-tree-sha1 = "bfe47e760d60b82b66b61d2d44128b62e3a369fb" +uuid = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3" +version = "0.3.5" + +[[NearestNeighbors]] +deps = ["Distances", "StaticArrays"] +git-tree-sha1 = "16baacfdc8758bc374882566c9187e785e85c2f0" +uuid = "b8a86587-4115-5ab1-83bc-aa920d37bbce" +version = "0.4.9" + +[[NetworkOptions]] +uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" + +[[OpenLibm_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "05823500-19ac-5b8b-9628-191a04bc5112" + +[[OpenMPI_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "a784e5133fc7e204c900f2cf38ed37a92ff9248d" +uuid = "fe0851c0-eecd-5654-98d4-656369965a5c" +version = "4.1.1+2" + +[[OpenSpecFun_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "13652491f6856acfd2db29360e1bbcd4565d04f1" +uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e" +version = "0.5.5+0" + +[[OrderedCollections]] +git-tree-sha1 = "85f8e6578bf1f9ee0d11e7bb1b1456435479d47c" +uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +version = "1.4.1" + +[[Parameters]] +deps = ["OrderedCollections", "UnPack"] +git-tree-sha1 = "34c0e9ad262e5f7fc75b10a9952ca7692cfc5fbe" +uuid = "d96e819e-fc66-5662-9728-84c9c7592b0a" +version = "0.12.3" + +[[Parsers]] +deps = ["Dates"] +git-tree-sha1 = "98f59ff3639b3d9485a03a72f3ab35bab9465720" +uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" +version = "2.0.6" + +[[PartitionedArrays]] +deps = ["Distances", "IterativeSolvers", "LinearAlgebra", "MPI", "Printf", "SparseArrays", "SparseMatricesCSR"] +git-tree-sha1 = "4b79fc86fd30f5063d97d9c40291976ede6fd4d5" +repo-rev = "gridap_distributed" +repo-url = "https://github.com/fverdugo/PartitionedArrays.jl" +uuid = "5a9dfac6-5c52-46f7-8278-5e2210713be9" +version = "0.2.2" + +[[Pkg]] +deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" + +[[Preferences]] +deps = ["TOML"] +git-tree-sha1 = "00cfd92944ca9c760982747e9a1d0d5d86ab1e5a" +uuid = "21216c6a-2e73-6563-6e65-726566657250" +version = "1.2.2" + +[[Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" + +[[QuadGK]] +deps = ["DataStructures", "LinearAlgebra"] +git-tree-sha1 = "78aadffb3efd2155af139781b8a8df1ef279ea39" +uuid = "1fd47b50-473d-5c70-9696-f719f8f3bcdc" +version = "2.4.2" + +[[REPL]] +deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" + +[[Random]] +deps = ["Serialization"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" + +[[RecipesBase]] +git-tree-sha1 = "44a75aa7a527910ee3d1751d1f0e4148698add9e" +uuid = "3cdcf5f2-1ef4-517c-9805-6587b60abb01" +version = "1.1.2" + +[[Reexport]] +git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b" +uuid = "189a3867-3050-52da-a836-e630ba90ab69" +version = "1.2.2" + +[[Requires]] +deps = ["UUIDs"] +git-tree-sha1 = "4036a3bd08ac7e968e27c203d45f5fff15020621" +uuid = "ae029012-a4dd-5104-9daa-d747884805df" +version = "1.1.3" + +[[SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" + +[[Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" + +[[SharedArrays]] +deps = ["Distributed", "Mmap", "Random", "Serialization"] +uuid = "1a1011a3-84de-559e-8e89-a11a2f7dc383" + +[[Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" + +[[SparseArrays]] +deps = ["LinearAlgebra", "Random"] +uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" + +[[SparseMatricesCSR]] +deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse"] +git-tree-sha1 = "ad906b39ce5e05ec509495dfc7b38d11ce9ab40b" +uuid = "a0a7dd2c-ebf4-11e9-1f05-cf50bc540ca1" +version = "0.6.5" + +[[SpecialFunctions]] +deps = ["ChainRulesCore", "IrrationalConstants", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"] +git-tree-sha1 = "2d57e14cd614083f132b6224874296287bfa3979" +uuid = "276daf66-3868-5448-9aa4-cd146d93841b" +version = "1.8.0" + +[[Static]] +deps = ["IfElse"] +git-tree-sha1 = "a8f30abc7c64a39d389680b74e749cf33f872a70" +uuid = "aedffcd0-7271-4cad-89d0-dc628f76c6d3" +version = "0.3.3" + +[[StaticArrays]] +deps = ["LinearAlgebra", "Random", "Statistics"] +git-tree-sha1 = "3c76dde64d03699e074ac02eb2e8ba8254d428da" +uuid = "90137ffa-7385-5640-81b9-e52037218182" +version = "1.2.13" + +[[Statistics]] +deps = ["LinearAlgebra", "SparseArrays"] +uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" + +[[StatsAPI]] +git-tree-sha1 = "1958272568dc176a1d881acb797beb909c785510" +uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0" +version = "1.0.0" + +[[SuiteSparse]] +deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"] +uuid = "4607b0f0-06f3-5cda-b6b1-a6196a1729e9" + +[[TOML]] +deps = ["Dates"] +uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" + +[[Tar]] +deps = ["ArgTools", "SHA"] +uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" + +[[Test]] +deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[[TranscodingStreams]] +deps = ["Random", "Test"] +git-tree-sha1 = "216b95ea110b5972db65aa90f88d8d89dcb8851c" +uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" +version = "0.9.6" + +[[UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" + +[[UnPack]] +git-tree-sha1 = "387c1f73762231e86e0c9c5443ce3b4a0a9a0c2b" +uuid = "3a884ed6-31ef-47d7-9d2a-63182c4928ed" +version = "1.0.2" + +[[Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" + +[[WriteVTK]] +deps = ["Base64", "CodecZlib", "FillArrays", "LightXML", "TranscodingStreams"] +git-tree-sha1 = "c3403143cecb391ea51fc133be82b024e4ce720b" +uuid = "64499a7a-5c06-52f2-abe2-ccb03c286192" +version = "1.11.0" + +[[XML2_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Libiconv_jll", "Pkg", "Zlib_jll"] +git-tree-sha1 = "1acf5bdf07aa0907e0a37d3718bb88d4b687b74a" +uuid = "02c8fc9c-b97f-50b9-bbe4-9be30ff0a78a" +version = "2.9.12+0" + +[[Zlib_jll]] +deps = ["Libdl"] +uuid = "83775a58-1f1d-513f-b197-d71354ab007a" + +[[nghttp2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" + +[[p7zip_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" diff --git a/test/TestApp/Project.toml b/test/TestApp/Project.toml new file mode 100644 index 00000000..828f2937 --- /dev/null +++ b/test/TestApp/Project.toml @@ -0,0 +1,4 @@ +[deps] +Gridap = "56d4f2e9-7ea1-5844-9cf6-b9c51ca7ce8e" +GridapDistributed = "f9701e48-63b3-45aa-9a63-9bc6c271f355" +PartitionedArrays = "5a9dfac6-5c52-46f7-8278-5e2210713be9" From 33d00130031232e8c5084c7e267f3723d09fd832 Mon Sep 17 00:00:00 2001 From: amartin Date: Thu, 21 Oct 2021 17:42:33 +1100 Subject: [PATCH 03/13] More work in TestApp --- src/Visualization.jl | 1 - test/TestApp/Manifest.toml | 4 +++- test/TestApp/Project.toml | 6 ++++++ test/{ => TestApp/src}/CellDataTests.jl | 0 test/{ => TestApp/src}/FESpacesTests.jl | 0 test/{ => TestApp/src}/GeometryTests.jl | 0 test/{ => TestApp/src}/MultiFieldTests.jl | 0 test/{ => TestApp/src}/PLaplacianTests.jl | 0 test/{ => TestApp/src}/PoissonTests.jl | 0 test/TestApp/src/TestApp.jl | 8 ++++++++ test/sequential/CellDataTests.jl | 5 ++--- test/sequential/FESpacesTests.jl | 6 ++---- test/sequential/GeometryTests.jl | 7 +++---- test/sequential/MultiFieldTests.jl | 6 ++---- test/sequential/PLaplacianTests.jl | 5 ++--- test/sequential/PoissonTests.jl | 7 ++----- 16 files changed, 30 insertions(+), 25 deletions(-) rename test/{ => TestApp/src}/CellDataTests.jl (100%) rename test/{ => TestApp/src}/FESpacesTests.jl (100%) rename test/{ => TestApp/src}/GeometryTests.jl (100%) rename test/{ => TestApp/src}/MultiFieldTests.jl (100%) rename test/{ => TestApp/src}/PLaplacianTests.jl (100%) rename test/{ => TestApp/src}/PoissonTests.jl (100%) create mode 100644 test/TestApp/src/TestApp.jl diff --git a/src/Visualization.jl b/src/Visualization.jl index bf2b2e62..75ab41d7 100644 --- a/src/Visualization.jl +++ b/src/Visualization.jl @@ -146,4 +146,3 @@ function Visualization.create_vtk_file( create_vtk_file(g,f;celldata=c,nodaldata=n) end end - diff --git a/test/TestApp/Manifest.toml b/test/TestApp/Manifest.toml index def66519..bd57975d 100644 --- a/test/TestApp/Manifest.toml +++ b/test/TestApp/Manifest.toml @@ -489,7 +489,9 @@ uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" [[WriteVTK]] deps = ["Base64", "CodecZlib", "FillArrays", "LightXML", "TranscodingStreams"] -git-tree-sha1 = "c3403143cecb391ea51fc133be82b024e4ce720b" +git-tree-sha1 = "9345fe8d0bf4f789bb1305f248595ceb7dd97d1b" +repo-rev = "master" +repo-url = "https://github.com/gridap/WriteVTK.jl" uuid = "64499a7a-5c06-52f2-abe2-ccb03c286192" version = "1.11.0" diff --git a/test/TestApp/Project.toml b/test/TestApp/Project.toml index 828f2937..480bcae0 100644 --- a/test/TestApp/Project.toml +++ b/test/TestApp/Project.toml @@ -1,4 +1,10 @@ +name = "TestApp" +uuid = "3ba29202-0f57-4e69-8cbd-5c57d4c4860a" +version = "0.1.0" + [deps] Gridap = "56d4f2e9-7ea1-5844-9cf6-b9c51ca7ce8e" GridapDistributed = "f9701e48-63b3-45aa-9a63-9bc6c271f355" PartitionedArrays = "5a9dfac6-5c52-46f7-8278-5e2210713be9" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +WriteVTK = "64499a7a-5c06-52f2-abe2-ccb03c286192" diff --git a/test/CellDataTests.jl b/test/TestApp/src/CellDataTests.jl similarity index 100% rename from test/CellDataTests.jl rename to test/TestApp/src/CellDataTests.jl diff --git a/test/FESpacesTests.jl b/test/TestApp/src/FESpacesTests.jl similarity index 100% rename from test/FESpacesTests.jl rename to test/TestApp/src/FESpacesTests.jl diff --git a/test/GeometryTests.jl b/test/TestApp/src/GeometryTests.jl similarity index 100% rename from test/GeometryTests.jl rename to test/TestApp/src/GeometryTests.jl diff --git a/test/MultiFieldTests.jl b/test/TestApp/src/MultiFieldTests.jl similarity index 100% rename from test/MultiFieldTests.jl rename to test/TestApp/src/MultiFieldTests.jl diff --git a/test/PLaplacianTests.jl b/test/TestApp/src/PLaplacianTests.jl similarity index 100% rename from test/PLaplacianTests.jl rename to test/TestApp/src/PLaplacianTests.jl diff --git a/test/PoissonTests.jl b/test/TestApp/src/PoissonTests.jl similarity index 100% rename from test/PoissonTests.jl rename to test/TestApp/src/PoissonTests.jl diff --git a/test/TestApp/src/TestApp.jl b/test/TestApp/src/TestApp.jl new file mode 100644 index 00000000..18165e57 --- /dev/null +++ b/test/TestApp/src/TestApp.jl @@ -0,0 +1,8 @@ +module TestApp + include("CellDataTests.jl") + include("FESpacesTests.jl") + include("GeometryTests.jl") + include("MultiFieldTests.jl") + include("PLaplacianTests.jl") + include("PoissonTests.jl") +end diff --git a/test/sequential/CellDataTests.jl b/test/sequential/CellDataTests.jl index e3475d02..64678837 100644 --- a/test/sequential/CellDataTests.jl +++ b/test/sequential/CellDataTests.jl @@ -1,10 +1,9 @@ module CellDataTestsSeq using PartitionedArrays - -include("../CellDataTests.jl") +using TestApp parts = get_part_ids(sequential,(2,2)) -CellDataTests.main(parts) +TestApp.CellDataTests.main(parts) end # module diff --git a/test/sequential/FESpacesTests.jl b/test/sequential/FESpacesTests.jl index b604fc30..919c4e11 100644 --- a/test/sequential/FESpacesTests.jl +++ b/test/sequential/FESpacesTests.jl @@ -1,11 +1,9 @@ module FESpacesTestsSeq using PartitionedArrays - -include("../FESpacesTests.jl") +using TestApp parts = get_part_ids(sequential,(2,2)) -FESpacesTests.main(parts) +TestApp.FESpacesTests.main(parts) end # module - diff --git a/test/sequential/GeometryTests.jl b/test/sequential/GeometryTests.jl index f3c641cd..d53efa41 100644 --- a/test/sequential/GeometryTests.jl +++ b/test/sequential/GeometryTests.jl @@ -1,13 +1,12 @@ module GeometryTestsSeq using PartitionedArrays - -include("../GeometryTests.jl") +using TestApp parts = get_part_ids(sequential,(2,2)) -GeometryTests.main(parts) +TestApp.GeometryTests.main(parts) parts = get_part_ids(sequential,(2,2,2)) -GeometryTests.main(parts) +TestApp.GeometryTests.main(parts) end diff --git a/test/sequential/MultiFieldTests.jl b/test/sequential/MultiFieldTests.jl index 7967dfa8..8efdf8c0 100644 --- a/test/sequential/MultiFieldTests.jl +++ b/test/sequential/MultiFieldTests.jl @@ -1,11 +1,9 @@ module MultiFieldTestsSeq using PartitionedArrays - -include("../MultiFieldTests.jl") +using TestApp parts = get_part_ids(sequential,(2,2)) -MultiFieldTests.main(parts) +TestApp.MultiFieldTests.main(parts) end # module - diff --git a/test/sequential/PLaplacianTests.jl b/test/sequential/PLaplacianTests.jl index ced09577..e314ab5f 100644 --- a/test/sequential/PLaplacianTests.jl +++ b/test/sequential/PLaplacianTests.jl @@ -1,10 +1,9 @@ module PLaplacianTestsSeq using PartitionedArrays - -include("../PLaplacianTests.jl") +using TestApp parts = get_part_ids(sequential,(2,2)) -PLaplacianTests.main(parts) +PTestApp.LaplacianTests.main(parts) end # module diff --git a/test/sequential/PoissonTests.jl b/test/sequential/PoissonTests.jl index 2ada4780..d8c11b59 100644 --- a/test/sequential/PoissonTests.jl +++ b/test/sequential/PoissonTests.jl @@ -1,12 +1,9 @@ module PoissonTestsSeq +using TestApp using PartitionedArrays -include("../PoissonTests.jl") - parts = get_part_ids(sequential,(2,2)) -PoissonTests.main(parts) +TestApp.PoissonTests.main(parts) end # module - - From d1d7e29d7054ec6540ba3f815cb06e47936cc5c8 Mon Sep 17 00:00:00 2001 From: amartin Date: Thu, 21 Oct 2021 17:47:18 +1100 Subject: [PATCH 04/13] Removing WriteVTK from Project.toml TestAPP --- test/TestApp/Project.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/test/TestApp/Project.toml b/test/TestApp/Project.toml index 480bcae0..d0bde848 100644 --- a/test/TestApp/Project.toml +++ b/test/TestApp/Project.toml @@ -7,4 +7,3 @@ Gridap = "56d4f2e9-7ea1-5844-9cf6-b9c51ca7ce8e" GridapDistributed = "f9701e48-63b3-45aa-9a63-9bc6c271f355" PartitionedArrays = "5a9dfac6-5c52-46f7-8278-5e2210713be9" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" -WriteVTK = "64499a7a-5c06-52f2-abe2-ccb03c286192" From a911f23fe90dbfac314be8abd49a75c54e886ff9 Mon Sep 17 00:00:00 2001 From: amartin Date: Thu, 21 Oct 2021 18:12:43 +1100 Subject: [PATCH 05/13] Preliminary version of TestApp (without image generation yet in Github Actions CI) --- .github/workflows/ci.yml | 8 ++-- test/TestApp/Manifest.toml | 24 +++++++++++- test/TestApp/Project.toml | 2 + test/TestApp/compile/compile.jl | 63 ++++++++++++++++++++++++++++++ test/TestApp/compile/compile.sh | 8 ++++ test/mpi/compile/compile.jl | 62 ----------------------------- test/mpi/compile/compile.sh | 21 ---------- test/mpi/runtests_np4.jl | 20 ++++------ test/sequential/PLaplacianTests.jl | 2 +- 9 files changed, 108 insertions(+), 102 deletions(-) create mode 100644 test/TestApp/compile/compile.jl create mode 100755 test/TestApp/compile/compile.sh delete mode 100644 test/mpi/compile/compile.jl delete mode 100755 test/mpi/compile/compile.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 95f6a3a4..417fb6ae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,7 +32,7 @@ jobs: ${{ runner.os }}-test- ${{ runner.os }}- - uses: julia-actions/julia-buildpkg@v1 - - run: julia --project=. --color=yes --check-bounds=yes test/sequential/runtests.jl + - run: julia --project=test/TestApp/ --color=yes --check-bounds=yes test/sequential/runtests.jl - uses: julia-actions/julia-processcoverage@v1 - uses: codecov/codecov-action@v1 with: @@ -66,9 +66,9 @@ jobs: version: ${{ matrix.version }} arch: ${{ matrix.arch }} - uses: julia-actions/julia-buildpkg@v1 - - run: cd test/mpi/compile; ./compile.sh PoissonTests.jl - - run: julia --project=. --color=yes --check-bounds=yes test/mpi/runtests.jl test/mpi/compile/GridapDistributedMPIBackend.so + #- run: cd test/mpi/compile; ./compile.sh PoissonTests.jl + - run: julia --project=test/TestApp/ --color=yes --check-bounds=yes test/mpi/runtests.jl - uses: julia-actions/julia-processcoverage@v1 - uses: codecov/codecov-action@v1 with: - file: lcov.info \ No newline at end of file + file: lcov.info diff --git a/test/TestApp/Manifest.toml b/test/TestApp/Manifest.toml index bd57975d..3f842cd7 100644 --- a/test/TestApp/Manifest.toml +++ b/test/TestApp/Manifest.toml @@ -155,7 +155,7 @@ uuid = "56d4f2e9-7ea1-5844-9cf6-b9c51ca7ce8e" version = "0.17.0" [[GridapDistributed]] -deps = ["FillArrays", "Gridap", "LinearAlgebra", "MPI", "PartitionedArrays", "SparseArrays", "WriteVTK"] +deps = ["FillArrays", "Gridap", "LinearAlgebra", "MPI", "PartitionedArrays", "SparseArrays"] path = "../.." uuid = "f9701e48-63b3-45aa-9a63-9bc6c271f355" version = "0.2.0" @@ -204,6 +204,10 @@ git-tree-sha1 = "8076680b162ada2a031f707ac7b4953e30667a37" uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" version = "0.21.2" +[[LazyArtifacts]] +deps = ["Artifacts", "Pkg"] +uuid = "4af54fe1-eca0-43a8-85a7-787d91b784e3" + [[LibCURL]] deps = ["LibCURL_jll", "MozillaCACerts_jll"] uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" @@ -339,6 +343,12 @@ git-tree-sha1 = "85f8e6578bf1f9ee0d11e7bb1b1456435479d47c" uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" version = "1.4.1" +[[PackageCompiler]] +deps = ["Artifacts", "LazyArtifacts", "Libdl", "Pkg", "RelocatableFolders", "UUIDs"] +git-tree-sha1 = "a965dd53ccaa69010d62851ab73d4e0c3d098314" +uuid = "9b87118b-4619-50d2-8e1e-99f35a4d4d9d" +version = "1.7.6" + [[Parameters]] deps = ["OrderedCollections", "UnPack"] git-tree-sha1 = "34c0e9ad262e5f7fc75b10a9952ca7692cfc5fbe" @@ -397,6 +407,12 @@ git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b" uuid = "189a3867-3050-52da-a836-e630ba90ab69" version = "1.2.2" +[[RelocatableFolders]] +deps = ["SHA", "Scratch"] +git-tree-sha1 = "df2be5142a2a3db2da37b21d87c9fa7973486bfd" +uuid = "05181044-ff0b-4ac5-8273-598c1e38db00" +version = "0.1.2" + [[Requires]] deps = ["UUIDs"] git-tree-sha1 = "4036a3bd08ac7e968e27c203d45f5fff15020621" @@ -406,6 +422,12 @@ version = "1.1.3" [[SHA]] uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +[[Scratch]] +deps = ["Dates"] +git-tree-sha1 = "0b4b7f1393cff97c33891da2a0bf69c6ed241fda" +uuid = "6c6a2e73-6563-6170-7368-637461726353" +version = "1.1.0" + [[Serialization]] uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" diff --git a/test/TestApp/Project.toml b/test/TestApp/Project.toml index d0bde848..6eed60fb 100644 --- a/test/TestApp/Project.toml +++ b/test/TestApp/Project.toml @@ -5,5 +5,7 @@ version = "0.1.0" [deps] Gridap = "56d4f2e9-7ea1-5844-9cf6-b9c51ca7ce8e" GridapDistributed = "f9701e48-63b3-45aa-9a63-9bc6c271f355" +MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195" +PackageCompiler = "9b87118b-4619-50d2-8e1e-99f35a4d4d9d" PartitionedArrays = "5a9dfac6-5c52-46f7-8278-5e2210713be9" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/test/TestApp/compile/compile.jl b/test/TestApp/compile/compile.jl new file mode 100644 index 00000000..1b180f89 --- /dev/null +++ b/test/TestApp/compile/compile.jl @@ -0,0 +1,63 @@ +function generate_precompile_execution_file() + source_code=""" + module sysimagegenerator + using TestApp + using PartitionedArrays + const PArrays = PartitionedArrays + using MPI + + if ! MPI.Initialized() + MPI.Init() + end + + parts = get_part_ids(mpi,(1,1)) + + display(parts) + + t = PArrays.PTimer(parts,verbose=true) + PArrays.tic!(t) + + TestApp.GeometryTests.main(parts) + PArrays.toc!(t,"Geometry") + + TestApp.CellDataTests.main(parts) + PArrays.toc!(t,"CellData") + + TestApp.FESpacesTests.main(parts) + PArrays.toc!(t,"FESpaces") + + TestApp.MultiFieldTests.main(parts) + PArrays.toc!(t,"MultiField") + + TestApp.PoissonTests.main(parts) + PArrays.toc!(t,"Poisson") + + TestApp.PLaplacianTests.main(parts) + PArrays.toc!(t,"PLaplacian") + + display(t) + + MPI.Finalize() + + end #module + """ + open("sysimagegenerator.jl","w") do io + println(io,source_code) + end +end + +using Pkg +using PackageCompiler +generate_precompile_execution_file() +pkgs = Symbol[] +push!(pkgs, :TestApp) + +if VERSION >= v"1.4" + append!(pkgs, [Symbol(v.name) for v in values(Pkg.dependencies()) if v.is_direct_dep],) +else + append!(pkgs, [Symbol(name) for name in keys(Pkg.installed())]) +end + +create_sysimage(pkgs, + sysimage_path=joinpath(@__DIR__,"TestApp.so"), + precompile_execution_file=joinpath(@__DIR__,"sysimagegenerator.jl")) diff --git a/test/TestApp/compile/compile.sh b/test/TestApp/compile/compile.sh new file mode 100755 index 00000000..0d2f9f19 --- /dev/null +++ b/test/TestApp/compile/compile.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# This script is to be executed from this folder (compile/) +julia --project=../ --color=yes -e 'using Pkg; Pkg.instantiate()' +# See https://juliaparallel.github.io/MPI.jl/latest/knownissues/#Julia-module-precompilation-1 +# for a justification of this line +julia --project=../ --color=yes -e 'using Pkg; pkg"precompile"' +julia --project=../ -O3 --check-bounds=no --color=yes compile.jl $1 diff --git a/test/mpi/compile/compile.jl b/test/mpi/compile/compile.jl deleted file mode 100644 index 2e141e92..00000000 --- a/test/mpi/compile/compile.jl +++ /dev/null @@ -1,62 +0,0 @@ -using Pkg -Pkg.add("PackageCompiler") -using PackageCompiler - -@assert length(ARGS)==1 -test_script=ARGS[1] -@assert isfile("../../"*test_script) -modulename=split(test_script,".")[1] - -function generate_precompile_execution_file(modulename) - source_code=""" - module sysimagegenerator - - using PartitionedArrays - const PArrays = PartitionedArrays - using MPI - - if ! MPI.Initialized() - MPI.Init() - end - - include(\"../../$(modulename).jl\") - - @assert MPI.Comm_size(MPI.COMM_WORLD) == 1 - parts = get_part_ids(mpi,(1,1)) - - display(parts) - - t = PArrays.PTimer(parts,verbose=true) - PArrays.tic!(t) - - eval(Meta.parse(\"$(modulename).main(parts)\")) - PArrays.toc!(t,\"$(modulename)\") - display(t) - - MPI.Finalize() - - end #module - """ - open("sysimagegenerator.jl","w") do io - println(io,source_code) - end -end - -@assert length(ARGS)==1 -test_script=ARGS[1] -@assert isfile("../../"*test_script) -modulename=split(test_script,".")[1] -generate_precompile_execution_file(modulename) - -pkgs = Symbol[] -push!(pkgs, :GridapDistributed) - -if VERSION >= v"1.4" - append!(pkgs, [Symbol(v.name) for v in values(Pkg.dependencies()) if v.is_direct_dep],) -else - append!(pkgs, [Symbol(name) for name in keys(Pkg.installed())]) -end - -create_sysimage(pkgs, - sysimage_path=joinpath(@__DIR__,"GridapDistributedMPIBackend.so"), - precompile_execution_file=joinpath(@__DIR__,"sysimagegenerator.jl")) diff --git a/test/mpi/compile/compile.sh b/test/mpi/compile/compile.sh deleted file mode 100755 index 1ce2a825..00000000 --- a/test/mpi/compile/compile.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -if [ "$#" -ne 1 ]; then - test_paths=$(ls ../../*Tests.jl) - test_names="" - for i in $test_paths - do - name=$(basename $i) - test_names="$test_names $name" - done - echo "Illegal number of parameters" - echo "Usage: $0 TESTNAME.jl, where TESTNAME.jl=$test_names" - exit 1 -fi - -# This script is to be executed from this folder (compile/) -julia --project=../../.. --color=yes -e 'using Pkg; Pkg.instantiate()' -# See https://juliaparallel.github.io/MPI.jl/latest/knownissues/#Julia-module-precompilation-1 -# for a justification of this line -julia --project=../../.. --color=yes -e 'using Pkg; pkg"precompile"' -julia --project=../../.. -O3 --check-bounds=no --color=yes compile.jl $1 diff --git a/test/mpi/runtests_np4.jl b/test/mpi/runtests_np4.jl index e7cbd800..d7d32fb3 100644 --- a/test/mpi/runtests_np4.jl +++ b/test/mpi/runtests_np4.jl @@ -1,6 +1,7 @@ module NP4 # All test running on 4 procs here +using TestApp using PartitionedArrays const PArrays = PartitionedArrays using MPI @@ -9,13 +10,6 @@ if ! MPI.Initialized() MPI.Init() end -include("../GeometryTests.jl") -include("../CellDataTests.jl") -include("../FESpacesTests.jl") -include("../MultiFieldTests.jl") -include("../PoissonTests.jl") -include("../PLaplacianTests.jl") - if MPI.Comm_size(MPI.COMM_WORLD) == 4 parts = get_part_ids(mpi,(2,2)) elseif MPI.Comm_size(MPI.COMM_WORLD) == 1 @@ -29,22 +23,22 @@ display(parts) t = PArrays.PTimer(parts,verbose=true) PArrays.tic!(t) -GeometryTests.main(parts) +TestApp.GeometryTests.main(parts) PArrays.toc!(t,"Geometry") -CellDataTests.main(parts) +TestApp.CellDataTests.main(parts) PArrays.toc!(t,"CellData") -FESpacesTests.main(parts) +TestApp.FESpacesTests.main(parts) PArrays.toc!(t,"FESpaces") -MultiFieldTests.main(parts) +TestApp.MultiFieldTests.main(parts) PArrays.toc!(t,"MultiField") -PoissonTests.main(parts) +TestApp.PoissonTests.main(parts) PArrays.toc!(t,"Poisson") -PLaplacianTests.main(parts) +TestApp.PLaplacianTests.main(parts) PArrays.toc!(t,"PLaplacian") display(t) diff --git a/test/sequential/PLaplacianTests.jl b/test/sequential/PLaplacianTests.jl index e314ab5f..fe60ac84 100644 --- a/test/sequential/PLaplacianTests.jl +++ b/test/sequential/PLaplacianTests.jl @@ -4,6 +4,6 @@ using PartitionedArrays using TestApp parts = get_part_ids(sequential,(2,2)) -PTestApp.LaplacianTests.main(parts) +TestApp.LaplacianTests.main(parts) end # module From 3380ba7f2953208c77ab362875f5cef3669e4a06 Mon Sep 17 00:00:00 2001 From: amartin Date: Thu, 21 Oct 2021 18:15:41 +1100 Subject: [PATCH 06/13] Separated the body of runtests_np4.jl in a separate file with reuse in mind --- test/mpi/runtests_np4.jl | 25 +------------------------ test/mpi/runtests_np4_body.jl | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 24 deletions(-) create mode 100644 test/mpi/runtests_np4_body.jl diff --git a/test/mpi/runtests_np4.jl b/test/mpi/runtests_np4.jl index d7d32fb3..7ca2595f 100644 --- a/test/mpi/runtests_np4.jl +++ b/test/mpi/runtests_np4.jl @@ -18,30 +18,7 @@ else error() end -display(parts) - -t = PArrays.PTimer(parts,verbose=true) -PArrays.tic!(t) - -TestApp.GeometryTests.main(parts) -PArrays.toc!(t,"Geometry") - -TestApp.CellDataTests.main(parts) -PArrays.toc!(t,"CellData") - -TestApp.FESpacesTests.main(parts) -PArrays.toc!(t,"FESpaces") - -TestApp.MultiFieldTests.main(parts) -PArrays.toc!(t,"MultiField") - -TestApp.PoissonTests.main(parts) -PArrays.toc!(t,"Poisson") - -TestApp.PLaplacianTests.main(parts) -PArrays.toc!(t,"PLaplacian") - -display(t) +include("runtests_np4_body.jl") MPI.Finalize() diff --git a/test/mpi/runtests_np4_body.jl b/test/mpi/runtests_np4_body.jl new file mode 100644 index 00000000..d3f5db14 --- /dev/null +++ b/test/mpi/runtests_np4_body.jl @@ -0,0 +1,24 @@ +display(parts) + +t = PArrays.PTimer(parts,verbose=true) +PArrays.tic!(t) + +TestApp.GeometryTests.main(parts) +PArrays.toc!(t,"Geometry") + +TestApp.CellDataTests.main(parts) +PArrays.toc!(t,"CellData") + +TestApp.FESpacesTests.main(parts) +PArrays.toc!(t,"FESpaces") + +TestApp.MultiFieldTests.main(parts) +PArrays.toc!(t,"MultiField") + +TestApp.PoissonTests.main(parts) +PArrays.toc!(t,"Poisson") + +TestApp.PLaplacianTests.main(parts) +PArrays.toc!(t,"PLaplacian") + +display(t) From fe3f36a5b3c8218273fa0d0041a5784017bb59dc Mon Sep 17 00:00:00 2001 From: amartin Date: Thu, 21 Oct 2021 18:22:06 +1100 Subject: [PATCH 07/13] Instantiating TestApp in ci.yml --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 417fb6ae..da0b82f4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,6 +32,7 @@ jobs: ${{ runner.os }}-test- ${{ runner.os }}- - uses: julia-actions/julia-buildpkg@v1 + - run: julia --project=test/TestApp/ -e 'using Pkg; Pkg.instantiate()' - run: julia --project=test/TestApp/ --color=yes --check-bounds=yes test/sequential/runtests.jl - uses: julia-actions/julia-processcoverage@v1 - uses: codecov/codecov-action@v1 @@ -67,6 +68,7 @@ jobs: arch: ${{ matrix.arch }} - uses: julia-actions/julia-buildpkg@v1 #- run: cd test/mpi/compile; ./compile.sh PoissonTests.jl + - run: julia --project=test/TestApp/ -e 'using Pkg; Pkg.instantiate()' - run: julia --project=test/TestApp/ --color=yes --check-bounds=yes test/mpi/runtests.jl - uses: julia-actions/julia-processcoverage@v1 - uses: codecov/codecov-action@v1 From 953b9a67e4f28c3cb41f70509e2f64dd89b565d5 Mon Sep 17 00:00:00 2001 From: amartin Date: Thu, 21 Oct 2021 18:32:46 +1100 Subject: [PATCH 08/13] Fixing paths to --project in runtests.jl mpi --- test/mpi/runtests.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/mpi/runtests.jl b/test/mpi/runtests.jl index 6f6450ba..122ab7d1 100644 --- a/test/mpi/runtests.jl +++ b/test/mpi/runtests.jl @@ -8,11 +8,11 @@ sysimage=nothing if length(ARGS)==1 @assert isfile(ARGS[1]) "$(ARGS[1]) must be a valid Julia sysimage file" sysimage=ARGS[1] -end +end mpidir = @__DIR__ testdir = joinpath(mpidir,"..") -repodir = joinpath(testdir,"..") +repodir = joinpath(testdir,"TestApp") function run_driver(procs,file,sysimage) mpiexec() do cmd if sysimage!=nothing From 97c007eed23bcf04e6c93013468f305ab3b8346a Mon Sep 17 00:00:00 2001 From: amartin Date: Thu, 21 Oct 2021 18:43:11 +1100 Subject: [PATCH 09/13] * Deactivating package build, do it manually in CI.yml * Addint automated image generation in CI.yml --- .github/workflows/ci.yml | 8 ++++---- test/TestApp/compile/compile.jl | 29 +++-------------------------- 2 files changed, 7 insertions(+), 30 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index da0b82f4..f6457955 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,7 +31,7 @@ jobs: ${{ runner.os }}-test-${{ env.cache-name }}- ${{ runner.os }}-test- ${{ runner.os }}- - - uses: julia-actions/julia-buildpkg@v1 + #- uses: julia-actions/julia-buildpkg@v1 - run: julia --project=test/TestApp/ -e 'using Pkg; Pkg.instantiate()' - run: julia --project=test/TestApp/ --color=yes --check-bounds=yes test/sequential/runtests.jl - uses: julia-actions/julia-processcoverage@v1 @@ -66,10 +66,10 @@ jobs: with: version: ${{ matrix.version }} arch: ${{ matrix.arch }} - - uses: julia-actions/julia-buildpkg@v1 - #- run: cd test/mpi/compile; ./compile.sh PoissonTests.jl + #- uses: julia-actions/julia-buildpkg@v1 + - run: cd test/TestApp/compile; ./compile.sh - run: julia --project=test/TestApp/ -e 'using Pkg; Pkg.instantiate()' - - run: julia --project=test/TestApp/ --color=yes --check-bounds=yes test/mpi/runtests.jl + - run: julia --project=test/TestApp/ --color=yes --check-bounds=yes test/mpi/runtests.jl test/TestApp/TestApp.so - uses: julia-actions/julia-processcoverage@v1 - uses: codecov/codecov-action@v1 with: diff --git a/test/TestApp/compile/compile.jl b/test/TestApp/compile/compile.jl index 1b180f89..b85e847b 100644 --- a/test/TestApp/compile/compile.jl +++ b/test/TestApp/compile/compile.jl @@ -1,6 +1,6 @@ function generate_precompile_execution_file() source_code=""" - module sysimagegenerator + module sysimagegenerator using TestApp using PartitionedArrays const PArrays = PartitionedArrays @@ -12,30 +12,7 @@ function generate_precompile_execution_file() parts = get_part_ids(mpi,(1,1)) - display(parts) - - t = PArrays.PTimer(parts,verbose=true) - PArrays.tic!(t) - - TestApp.GeometryTests.main(parts) - PArrays.toc!(t,"Geometry") - - TestApp.CellDataTests.main(parts) - PArrays.toc!(t,"CellData") - - TestApp.FESpacesTests.main(parts) - PArrays.toc!(t,"FESpaces") - - TestApp.MultiFieldTests.main(parts) - PArrays.toc!(t,"MultiField") - - TestApp.PoissonTests.main(parts) - PArrays.toc!(t,"Poisson") - - TestApp.PLaplacianTests.main(parts) - PArrays.toc!(t,"PLaplacian") - - display(t) + include("../../mpi/runtests_np4_body.jl") MPI.Finalize() @@ -44,7 +21,7 @@ function generate_precompile_execution_file() open("sysimagegenerator.jl","w") do io println(io,source_code) end -end +end using Pkg using PackageCompiler From cb416f3d46972807fa247a9cbf5c922def55617d Mon Sep 17 00:00:00 2001 From: amartin Date: Thu, 21 Oct 2021 18:44:36 +1100 Subject: [PATCH 10/13] Fixing typo in seq tests --- test/sequential/PLaplacianTests.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/sequential/PLaplacianTests.jl b/test/sequential/PLaplacianTests.jl index fe60ac84..427b6855 100644 --- a/test/sequential/PLaplacianTests.jl +++ b/test/sequential/PLaplacianTests.jl @@ -4,6 +4,6 @@ using PartitionedArrays using TestApp parts = get_part_ids(sequential,(2,2)) -TestApp.LaplacianTests.main(parts) +TestApp.PLaplacianTests.main(parts) end # module From 953eef75a3c37e5ab643f58db245d1d7dc73ffbb Mon Sep 17 00:00:00 2001 From: amartin Date: Thu, 21 Oct 2021 21:16:12 +1100 Subject: [PATCH 11/13] Debugging image generation --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f6457955..1a893086 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,6 +68,7 @@ jobs: arch: ${{ matrix.arch }} #- uses: julia-actions/julia-buildpkg@v1 - run: cd test/TestApp/compile; ./compile.sh + - run: find . -name "*.so" - run: julia --project=test/TestApp/ -e 'using Pkg; Pkg.instantiate()' - run: julia --project=test/TestApp/ --color=yes --check-bounds=yes test/mpi/runtests.jl test/TestApp/TestApp.so - uses: julia-actions/julia-processcoverage@v1 From 4fb646b23c0a6bdeb18bf314f86dd4402ba28851 Mon Sep 17 00:00:00 2001 From: "Alberto F. Martin" <38347633+amartinhuertas@users.noreply.github.com> Date: Fri, 22 Oct 2021 00:15:19 +1100 Subject: [PATCH 12/13] Update ci.yml --- .github/workflows/ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1a893086..1dac13ea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,9 +68,8 @@ jobs: arch: ${{ matrix.arch }} #- uses: julia-actions/julia-buildpkg@v1 - run: cd test/TestApp/compile; ./compile.sh - - run: find . -name "*.so" - run: julia --project=test/TestApp/ -e 'using Pkg; Pkg.instantiate()' - - run: julia --project=test/TestApp/ --color=yes --check-bounds=yes test/mpi/runtests.jl test/TestApp/TestApp.so + - run: julia --project=test/TestApp/ --color=yes --check-bounds=yes test/mpi/runtests.jl test/TestApp/compile/TestApp.so - uses: julia-actions/julia-processcoverage@v1 - uses: codecov/codecov-action@v1 with: From 120e0a4c59fe2cf7598558aa8934a4e659aa57e2 Mon Sep 17 00:00:00 2001 From: amartin Date: Fri, 22 Oct 2021 10:26:45 +1100 Subject: [PATCH 13/13] Fixing TestApp Manifest.toml --- test/TestApp/Manifest.toml | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/test/TestApp/Manifest.toml b/test/TestApp/Manifest.toml index 3f842cd7..0e8cda9d 100644 --- a/test/TestApp/Manifest.toml +++ b/test/TestApp/Manifest.toml @@ -98,9 +98,9 @@ version = "1.3.1" [[Distances]] deps = ["LinearAlgebra", "Statistics", "StatsAPI"] -git-tree-sha1 = "9f46deb4d4ee4494ffb5a40a27a2aced67bdd838" +git-tree-sha1 = "09d9eaef9ef719d2cd5d928a191dc95be2ec8059" uuid = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7" -version = "0.10.4" +version = "0.10.5" [[Distributed]] deps = ["Random", "Serialization", "Sockets"] @@ -148,14 +148,14 @@ version = "0.10.21" [[Gridap]] deps = ["AbstractTrees", "BSON", "BlockArrays", "Combinatorics", "DocStringExtensions", "FastGaussQuadrature", "FileIO", "FillArrays", "ForwardDiff", "JLD2", "JSON", "LineSearches", "LinearAlgebra", "NLsolve", "NearestNeighbors", "QuadGK", "Random", "SparseArrays", "SparseMatricesCSR", "StaticArrays", "Test", "WriteVTK"] -git-tree-sha1 = "1c5c6b7cbdc5895f690d731e3653d01636eaa243" +git-tree-sha1 = "d3bfeb769259b08f7385bdad91681f38c50fc519" repo-rev = "gridap_distributed" repo-url = "https://github.com/gridap/Gridap.jl" uuid = "56d4f2e9-7ea1-5844-9cf6-b9c51ca7ce8e" version = "0.17.0" [[GridapDistributed]] -deps = ["FillArrays", "Gridap", "LinearAlgebra", "MPI", "PartitionedArrays", "SparseArrays"] +deps = ["FillArrays", "Gridap", "LinearAlgebra", "MPI", "PartitionedArrays", "SparseArrays", "WriteVTK"] path = "../.." uuid = "f9701e48-63b3-45aa-9a63-9bc6c271f355" version = "0.2.0" @@ -357,9 +357,9 @@ version = "0.12.3" [[Parsers]] deps = ["Dates"] -git-tree-sha1 = "98f59ff3639b3d9485a03a72f3ab35bab9465720" +git-tree-sha1 = "f19e978f81eca5fd7620650d7dbea58f825802ee" uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" -version = "2.0.6" +version = "2.1.0" [[PartitionedArrays]] deps = ["Distances", "IterativeSolvers", "LinearAlgebra", "MPI", "Printf", "SparseArrays", "SparseMatricesCSR"] @@ -511,9 +511,7 @@ uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" [[WriteVTK]] deps = ["Base64", "CodecZlib", "FillArrays", "LightXML", "TranscodingStreams"] -git-tree-sha1 = "9345fe8d0bf4f789bb1305f248595ceb7dd97d1b" -repo-rev = "master" -repo-url = "https://github.com/gridap/WriteVTK.jl" +git-tree-sha1 = "c3403143cecb391ea51fc133be82b024e4ce720b" uuid = "64499a7a-5c06-52f2-abe2-ccb03c286192" version = "1.11.0"