Skip to content

Rhs assembly alone #40

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
fverdugo merged 17 commits into from
Oct 21, 2021
Merged

Rhs assembly alone #40

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
f8e383d
First approach to implement rhs assemble + SubAssembledRows()
amartinhuertas Oct 15, 2021
ce94cf5
Separated assembly strategies for symbolic and numerical loops no longer
amartinhuertas Oct 16, 2021
70bd475
Reconsidered strategy followed in
amartinhuertas Oct 18, 2021
0f047cd
Merge branch 'release-0.2' of github.com:gridap/GridapDistributed.jl …
amartinhuertas Oct 18, 2021
8f17a66
Fixing Manifest
amartinhuertas Oct 18, 2021
bcc8946
* Optimized assemble_matrix_and_vector using the new generalized
amartinhuertas Oct 19, 2021
becf6b1
* Adding broken tests for assemble_XXX! functions
amartinhuertas Oct 19, 2021
442e180
Adding Gridap compact to Project.toml
amartinhuertas Oct 19, 2021
300ec1f
Merge branch 'release-0.2' of github.com:gridap/GridapDistributed.jl …
amartinhuertas Oct 20, 2021
0033b58
Fixed all @test_broken tests in FESpacesTests.jl
amartinhuertas Oct 20, 2021
5544855
Removing symbolic_loop_vector!
amartinhuertas Oct 20, 2021
8317378
Adding tests for assemble_matrix/assemble_matrix!/allocate_matrix
amartinhuertas Oct 20, 2021
ecf6a38
* Bug fixed local_views(a::PVectorAllocationTrackTouchedAndValues,rows)
amartinhuertas Oct 21, 2021
4f1cc79
Merge branch 'release-0.2' of github.com:gridap/GridapDistributed.jl …
amartinhuertas Oct 21, 2021
862af4a
Merge branch 'release-0.2' of github.com:gridap/GridapDistributed.jl …
amartinhuertas Oct 21, 2021
0a5776b
Minor fixes in Project.toml and FESpacesTests.jl
fverdugo Oct 21, 2021
f9fca06
Merge branch 'release-0.2' into rhs_assembly
fverdugo Oct 21, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 41 additions & 31 deletions Manifest.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,38 +10,38 @@ uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f"

[[ArrayInterface]]
deps = ["Compat", "IfElse", "LinearAlgebra", "Requires", "SparseArrays", "Static"]
git-tree-sha1 = "b8d49c34c3da35f220e7295659cd0bab8e739fed"
git-tree-sha1 = "1d6835607e9f214cb4210310868f8cf07eb0facc"
uuid = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9"
version = "3.1.33"
version = "3.1.34"

[[ArrayLayouts]]
deps = ["FillArrays", "LinearAlgebra", "SparseArrays"]
git-tree-sha1 = "a745df3fdce8aff885e6f4a3b7427cc27ce5f86c"
git-tree-sha1 = "7a92ea1dd16472d18ca1ffcbb7b3cc67d7e78a3f"
uuid = "4c555306-a7a7-4459-81d9-ec55ddd5c99a"
version = "0.7.6"
version = "0.7.7"

[[Artifacts]]
uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"

[[BSON]]
git-tree-sha1 = "92b8a8479128367aaab2620b8e73dff632f5ae69"
git-tree-sha1 = "ebcd6e22d69f21249b7b8668351ebf42d6dc87a1"
uuid = "fbb218c0-5317-5bc6-957e-2ee96dd4b1f0"
version = "0.3.3"
version = "0.3.4"

[[Base64]]
uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"

[[BlockArrays]]
deps = ["ArrayLayouts", "FillArrays", "LinearAlgebra"]
git-tree-sha1 = "782362509cf50a51092f513e92783c18ab0b6d51"
git-tree-sha1 = "5524e27323cf4c4505699c3fb008c3f772269945"
uuid = "8e7c35d0-a365-5155-bbbb-fb81a777f24e"
version = "0.16.8"
version = "0.16.9"

[[ChainRulesCore]]
deps = ["Compat", "LinearAlgebra", "SparseArrays"]
git-tree-sha1 = "a325370b9dd0e6bf5656a6f1a7ae80755f8ccc46"
git-tree-sha1 = "d9e40e3e370ee56c5b57e0db651d8f92bce98fea"
uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
version = "1.7.2"
version = "1.10.1"

[[CodecZlib]]
deps = ["TranscodingStreams", "Zlib_jll"]
Expand Down Expand Up @@ -130,9 +130,9 @@ version = "1.11.1"

[[FillArrays]]
deps = ["LinearAlgebra", "Random", "SparseArrays", "Statistics"]
git-tree-sha1 = "29890dfbc427afa59598b8cfcc10034719bd7744"
git-tree-sha1 = "8756f9935b7ccc9064c6eef0bff0ad643df733a3"
uuid = "1a297f60-69ca-5386-bcde-b61e274b549b"
version = "0.12.6"
version = "0.12.7"

[[FiniteDiff]]
deps = ["ArrayInterface", "LinearAlgebra", "Requires", "SparseArrays", "StaticArrays"]
Expand All @@ -141,16 +141,16 @@ uuid = "6a86dc24-6348-571c-b903-95158fe2bd41"
version = "2.8.1"

[[ForwardDiff]]
deps = ["CommonSubexpressions", "DiffResults", "DiffRules", "LinearAlgebra", "NaNMath", "Printf", "Random", "SpecialFunctions", "StaticArrays"]
git-tree-sha1 = "c4203b60d37059462af370c4f3108fb5d155ff13"
deps = ["CommonSubexpressions", "DiffResults", "DiffRules", "LinearAlgebra", "NaNMath", "Preferences", "Printf", "Random", "SpecialFunctions", "StaticArrays"]
git-tree-sha1 = "63777916efbcb0ab6173d09a658fb7f2783de485"
uuid = "f6369f11-7733-5829-9624-2563aa707210"
version = "0.10.20"
version = "0.10.21"

[[Gridap]]
deps = ["AbstractTrees", "BSON", "BlockArrays", "Combinatorics", "DocStringExtensions", "FastGaussQuadrature", "FileIO", "FillArrays", "ForwardDiff", "JLD2", "JSON", "LineSearches", "LinearAlgebra", "NLsolve", "NearestNeighbors", "QuadGK", "Random", "SparseArrays", "SparseMatricesCSR", "StaticArrays", "Test", "WriteVTK"]
git-tree-sha1 = "d3bfeb769259b08f7385bdad91681f38c50fc519"
repo-rev = "gridap_distributed"
repo-url = "https://github.com/gridap/Gridap.jl.git"
repo-url = "https://github.com/gridap/Gridap.jl"
uuid = "56d4f2e9-7ea1-5844-9cf6-b9c51ca7ce8e"
version = "0.17.0"

Expand All @@ -163,10 +163,16 @@ version = "0.1.0"
deps = ["Markdown"]
uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"

[[InverseFunctions]]
deps = ["Test"]
git-tree-sha1 = "f0c6489b12d28fb4c2103073ec7452f3423bd308"
uuid = "3587e190-3f89-42d0-90ee-14403ec27112"
version = "0.1.1"

[[IrrationalConstants]]
git-tree-sha1 = "f76424439413893a832026ca355fe273e93bce94"
git-tree-sha1 = "7fd44fd4ff43fc60815f8e764c0f352b83c49151"
uuid = "92d709cd-6900-40b7-9082-c6be49f344b6"
version = "0.1.0"
version = "0.1.1"

[[IterativeSolvers]]
deps = ["LinearAlgebra", "Printf", "Random", "RecipesBase", "SparseArrays"]
Expand All @@ -176,9 +182,9 @@ version = "0.9.1"

[[JLD2]]
deps = ["DataStructures", "FileIO", "MacroTools", "Mmap", "Pkg", "Printf", "Reexport", "TranscodingStreams", "UUIDs"]
git-tree-sha1 = "192934b3e2a94e897ce177423fd6cf7bdf464bce"
git-tree-sha1 = "46b7834ec8165c541b0b5d1c8ba63ec940723ffb"
uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
version = "0.4.14"
version = "0.4.15"

[[JLLWrappers]]
deps = ["Preferences"]
Expand Down Expand Up @@ -234,10 +240,10 @@ deps = ["Libdl"]
uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"

[[LogExpFunctions]]
deps = ["ChainRulesCore", "DocStringExtensions", "IrrationalConstants", "LinearAlgebra"]
git-tree-sha1 = "34dc30f868e368f8a17b728a1238f3fcda43931a"
deps = ["ChainRulesCore", "DocStringExtensions", "InverseFunctions", "IrrationalConstants", "LinearAlgebra"]
git-tree-sha1 = "6193c3815f13ba1b78a51ce391db8be016ae9214"
uuid = "2ab3a3ac-af41-5b50-aa03-7779005ae688"
version = "0.3.3"
version = "0.3.4"

[[Logging]]
uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
Expand Down Expand Up @@ -335,13 +341,13 @@ version = "0.12.3"

[[Parsers]]
deps = ["Dates"]
git-tree-sha1 = "a8709b968a1ea6abc2dc1967cb1db6ac9a00dfb6"
git-tree-sha1 = "98f59ff3639b3d9485a03a72f3ab35bab9465720"
uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
version = "2.0.5"
version = "2.0.6"

[[PartitionedArrays]]
deps = ["Distances", "IterativeSolvers", "LinearAlgebra", "MPI", "Printf", "SparseArrays", "SparseMatricesCSR"]
git-tree-sha1 = "763146e768b343fec44c521f5c7822116adc24ad"
git-tree-sha1 = "4b79fc86fd30f5063d97d9c40291976ede6fd4d5"
repo-rev = "gridap_distributed"
repo-url = "https://github.com/fverdugo/PartitionedArrays.jl.git"
uuid = "5a9dfac6-5c52-46f7-8278-5e2210713be9"
Expand Down Expand Up @@ -409,16 +415,16 @@ deps = ["LinearAlgebra", "Random"]
uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"

[[SparseMatricesCSR]]
deps = ["LinearAlgebra", "SparseArrays"]
git-tree-sha1 = "dd6dda1484d4031a47d27614ceeb89f511cd9ca4"
deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse"]
git-tree-sha1 = "ad906b39ce5e05ec509495dfc7b38d11ce9ab40b"
uuid = "a0a7dd2c-ebf4-11e9-1f05-cf50bc540ca1"
version = "0.6.1"
version = "0.6.5"

[[SpecialFunctions]]
deps = ["ChainRulesCore", "IrrationalConstants", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"]
git-tree-sha1 = "793793f1df98e3d7d554b65a107e9c9a6399a6ed"
git-tree-sha1 = "2d57e14cd614083f132b6224874296287bfa3979"
uuid = "276daf66-3868-5448-9aa4-cd146d93841b"
version = "1.7.0"
version = "1.8.0"

[[Static]]
deps = ["IfElse"]
Expand All @@ -441,6 +447,10 @@ git-tree-sha1 = "1958272568dc176a1d881acb797beb909c785510"
uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0"
version = "1.0.0"

[[SuiteSparse]]
deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"]
uuid = "4607b0f0-06f3-5cda-b6b1-a6196a1729e9"

[[TOML]]
deps = ["Dates"]
uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
Expand Down
2 changes: 2 additions & 0 deletions Project.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,9 @@ SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
WriteVTK = "64499a7a-5c06-52f2-abe2-ccb03c286192"

[compat]
FillArrays = "0.8.4, 0.9, 0.10, 0.11, 0.12"
Gridap = "0.17"
Comment thread
amartinhuertas marked this conversation as resolved.
MPI = "0.16, 0.17, 0.18, 0.19"
PartitionedArrays = "0.2"
julia = "1.3"

Expand Down
158 changes: 145 additions & 13 deletions src/Algebra.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -380,37 +380,47 @@ function local_views(a::PVectorCounter,rows)
a.counters
end

function Arrays.nz_allocation(a::PVectorCounter)
function Arrays.nz_allocation(a::PVectorCounter{<:FullyAssembledRows})
dofs = a.rows
values = map_parts(nz_allocation,a.counters)
PVectorAllocation(a.par_strategy,values,dofs)
PVectorAllocationTrackOnlyValues(a.par_strategy,values,dofs)
end

struct PVectorAllocation{A,B,C}
struct PVectorAllocationTrackOnlyValues{A,B,C}
par_strategy::A
values::B
rows::C
end

function local_views(a::PVectorAllocation)
function local_views(a::PVectorAllocationTrackOnlyValues)
a.values
end

function local_views(a::PVectorAllocation,rows)
function local_views(a::PVectorAllocationTrackOnlyValues,rows)
@check rows === a.rows
a.values
end

function Algebra.create_from_nz(a::PVectorAllocation{<:FullyAssembledRows})
@notimplemented
function Algebra.create_from_nz(a::PVectorAllocationTrackOnlyValues{<:FullyAssembledRows})
# Create PRange for the rows of the linear system
parts = get_part_ids(a.values)
ngdofs = length(a.rows)
nodofs = map_parts(num_oids,a.rows.partition)
first_grdof = map_parts(first_gdof_from_ids,a.rows.partition)

# This one has no ghost rows
rows = PRange(parts,ngdofs,nodofs,first_grdof)

_rhs_callback(a,rows)
end

function Algebra.create_from_nz(a::PVectorAllocation{<:SubAssembledRows})
@notimplemented
function Algebra.create_from_nz(a::PVectorAllocationTrackOnlyValues{<:SubAssembledRows})
# This point MUST NEVER be reached. If reached there is an inconsistency
# in the parallel code in charge of vector assembly
@assert false
end

function _rhs_callback(c_fespace,rows)

# The ghost values in b_fespace are aligned with the FESpace
# but not with the ghost values in the rows of A
b_fespace = PVector(c_fespace.values,c_fespace.rows)
Expand Down Expand Up @@ -442,14 +452,14 @@ end

function Algebra.create_from_nz(a::PVector)
# For FullyAssembledRows the underlying Exchanger should
# not have ghost layer making asseble! do nothing (TODO check)
# not have ghost layer making assemble! do nothing (TODO check)
assemble!(a)
a
end

function Algebra.create_from_nz(
a::DistributedAllocationCOO{<:FullyAssembledRows},
c_fespace::PVectorAllocation{<:FullyAssembledRows})
c_fespace::PVectorAllocationTrackOnlyValues{<:FullyAssembledRows})

function callback(rows)
_rhs_callback(c_fespace,rows)
Expand All @@ -459,9 +469,15 @@ function Algebra.create_from_nz(
A,b
end

struct PVectorAllocationTrackTouchedAndValues{A,B,C}
allocations::A
values::B
rows::C
end

function Algebra.create_from_nz(
a::DistributedAllocationCOO{<:SubAssembledRows},
c_fespace::PVectorAllocation{<:SubAssembledRows})
c_fespace::PVectorAllocationTrackOnlyValues{<:SubAssembledRows})

function callback(rows)
_rhs_callback(c_fespace,rows)
Expand All @@ -476,3 +492,119 @@ function Algebra.create_from_nz(
A,b
end

struct ArrayAllocationTrackTouchedAndValues{A}
touched::Vector{Bool}
values::A
end

Gridap.Algebra.LoopStyle(::Type{<:ArrayAllocationTrackTouchedAndValues}) = Gridap.Algebra.Loop()


function local_views(a::PVectorAllocationTrackTouchedAndValues,rows)
@check rows === a.rows
a.allocations
end

@inline function Arrays.add_entry!(c::Function,a::ArrayAllocationTrackTouchedAndValues,v,i,j)
@notimplemented
end
@inline function Arrays.add_entry!(c::Function,a::ArrayAllocationTrackTouchedAndValues,v,i)
if i>0
if !(a.touched[i])
a.touched[i]=true
end
if v!=nothing
a.values[i]=c(v,a.values[i])
end
end
nothing
end
@inline function Arrays.add_entries!(c::Function,a::ArrayAllocationTrackTouchedAndValues,v,i,j)
@notimplemented
end
@inline function Arrays.add_entries!(c::Function,a::ArrayAllocationTrackTouchedAndValues,v,i)
if v != nothing
for (ve,ie) in zip(v,i)
Arrays.add_entry!(c,a,ve,ie)
end
else
for ie in i
Arrays.add_entry!(c,a,nothing,ie)
end
end
nothing
end

function Arrays.nz_allocation(a::DistributedCounterCOO{<:SubAssembledRows},
b::PVectorCounter{<:SubAssembledRows})
A = nz_allocation(a)
dofs = b.rows
values = map_parts(nz_allocation,b.counters)
B=PVectorAllocationTrackOnlyValues(b.par_strategy,values,dofs)
A,B
end

function Arrays.nz_allocation(a::PVectorCounter{<:SubAssembledRows})
dofs = a.rows
values = map_parts(nz_allocation,a.counters)
touched = map_parts(values) do values
fill!(Vector{Bool}(undef,length(values)),false)
end
allocations=map_parts(values,touched) do values,touched
ArrayAllocationTrackTouchedAndValues(touched,values)
end
PVectorAllocationTrackTouchedAndValues(allocations,values,dofs)
end

function local_views(a::PVectorAllocationTrackTouchedAndValues)
a.allocations
end

function Algebra.create_from_nz(a::PVectorAllocationTrackTouchedAndValues)
parts = get_part_ids(a.values)
rdofs = a.rows # dof ids of the test space
ngrdofs = length(rdofs)
nordofs = map_parts(num_oids,rdofs.partition)
first_grdof = map_parts(first_gdof_from_ids,rdofs.partition)
rneigs_snd = rdofs.exchanger.parts_snd
rneigs_rcv = rdofs.exchanger.parts_rcv

# Find the ghost rows
hrow_to_hrdof=map_parts(local_views(a.allocations),rdofs.partition) do allocation, indices
lids_touched=findall(allocation.touched)
nhlids = count((x)->indices.lid_to_ohid[x]<0,lids_touched)
hlids = Vector{Int32}(undef,nhlids)
cur=1
for lid in lids_touched
hlid=indices.lid_to_ohid[lid]
if hlid<0
hlids[cur]=-hlid
cur=cur+1
end
end
hlids
end
hrow_to_gid, hrow_to_part = map_parts(
find_gid_and_part,hrow_to_hrdof,rdofs.partition)

# Create the range for rows
rows = PRange(
parts,
ngrdofs,
nordofs,
first_grdof,
hrow_to_gid,
hrow_to_part,
rneigs_snd,
rneigs_rcv)

b = _rhs_callback(a,rows)
t2 = async_assemble!(b)

# Wait the transfer to finish
if t2 !== nothing
map_parts(schedule,t2)
map_parts(wait,t2)
end
b
end
Loading