Skip to content

[hotfix] Fix typos for supporting distributed training #2415

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
haofanwang wants to merge 10 commits into from
Closed

[hotfix] Fix typos for supporting distributed training #2415

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
93b45a3
Update initializer_1d.py
haofanwang Jan 10, 2023
ad20ab3
Update initializer_2d.py
haofanwang Jan 10, 2023
e8a5f0f
Update initializer_2p5d.py
haofanwang Jan 10, 2023
5542756
Update initializer_3d.py
haofanwang Jan 10, 2023
2704ccc
Update initializer_data.py
haofanwang Jan 10, 2023
ce0edc3
Update initializer_model.py
haofanwang Jan 10, 2023
46292dd
Update initializer_pipeline.py
haofanwang Jan 10, 2023
ce76f98
Update initializer_sequence.py
haofanwang Jan 10, 2023
9fa9f5d
Update initializer_tensor.py
haofanwang Jan 10, 2023
957de2f
Merge branch 'hpcaitech:main' into main
haofanwang Jan 11, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion colossalai/context/process_group_initializer/initializer_1d.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ def init_dist_group(self):
for i in range(self.num_group):
ranks = [i * self.tensor_parallel_size + j for j in range(self.tensor_parallel_size)]
group = dist.new_group(ranks)
group_cpu = dist.new_group(ranks, backend='gloo') if dist.get_backend() != 'gloo' else group
group_cpu = dist.new_group(ranks, backend='gloo') if dist.get_backend() == 'gloo' else group

if self.rank in ranks:
local_rank = ranks.index(self.rank)
Expand Down
4 changes: 2 additions & 2 deletions colossalai/context/process_group_initializer/initializer_2d.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ def init_dist_group(self):
for j in range(self.summa_dim):
ranks = [i * self.tensor_parallel_size + j * self.summa_dim + k for k in range(self.summa_dim)]
group = dist.new_group(ranks)
group_cpu = dist.new_group(ranks, backend='gloo') if dist.get_backend() != 'gloo' else group
group_cpu = dist.new_group(ranks, backend='gloo') if dist.get_backend() == 'gloo' else group

if self.rank in ranks:
local_rank = ranks.index(self.rank)
Expand Down Expand Up @@ -105,7 +105,7 @@ def init_dist_group(self):
for j in range(self.summa_dim):
ranks = [i * self.tensor_parallel_size + j + k * self.summa_dim for k in range(self.summa_dim)]
group = dist.new_group(ranks)
group_cpu = dist.new_group(ranks, backend='gloo') if dist.get_backend() != 'gloo' else group
group_cpu = dist.new_group(ranks, backend='gloo') if dist.get_backend() == 'gloo' else group

if self.rank in ranks:
local_rank = ranks.index(self.rank)
Expand Down
8 changes: 4 additions & 4 deletions colossalai/context/process_group_initializer/initializer_2p5d.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ def init_dist_group(self):
for i in range(self.tesseract_dim)
]
group = dist.new_group(ranks)
group_cpu = dist.new_group(ranks, backend='gloo') if dist.get_backend() != 'gloo' else group
group_cpu = dist.new_group(ranks, backend='gloo') if dist.get_backend() == 'gloo' else group

if self.rank in ranks:
local_rank = ranks.index(self.rank)
Expand Down Expand Up @@ -128,7 +128,7 @@ def init_dist_group(self):
for j in range(self.tesseract_dim)
]
group = dist.new_group(ranks)
group_cpu = dist.new_group(ranks, backend='gloo') if dist.get_backend() != 'gloo' else group
group_cpu = dist.new_group(ranks, backend='gloo') if dist.get_backend() == 'gloo' else group

if self.rank in ranks:
local_rank = ranks.index(self.rank)
Expand Down Expand Up @@ -182,7 +182,7 @@ def init_dist_group(self):
for k in range(self.tesseract_dep)
]
group = dist.new_group(ranks)
group_cpu = dist.new_group(ranks, backend='gloo') if dist.get_backend() != 'gloo' else group
group_cpu = dist.new_group(ranks, backend='gloo') if dist.get_backend() == 'gloo' else group

if self.rank in ranks:
local_rank = ranks.index(self.rank)
Expand Down Expand Up @@ -237,7 +237,7 @@ def init_dist_group(self):
for j in range(self.tesseract_dim)
]
group = dist.new_group(ranks)
group_cpu = dist.new_group(ranks, backend='gloo') if dist.get_backend() != 'gloo' else group
group_cpu = dist.new_group(ranks, backend='gloo') if dist.get_backend() == 'gloo' else group

if self.rank in ranks:
local_rank = ranks.index(self.rank)
Expand Down
10 changes: 5 additions & 5 deletions colossalai/context/process_group_initializer/initializer_3d.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ def init_dist_group(self):
for k in range(self.depth):
ranks = [h * self.depth**3 + i + self.depth * (j + self.depth * k) for j in range(self.depth)]
group = dist.new_group(ranks)
group_cpu = dist.new_group(ranks, backend='gloo') if dist.get_backend() != 'gloo' else group
group_cpu = dist.new_group(ranks, backend='gloo') if dist.get_backend() == 'gloo' else group

if self.rank in ranks:
local_rank = ranks.index(self.rank)
Expand Down Expand Up @@ -113,7 +113,7 @@ def init_dist_group(self):
for j in range(self.depth):
ranks = [h * self.depth**3 + i + self.depth * (j + self.depth * k) for i in range(self.depth)]
group = dist.new_group(ranks)
group_cpu = dist.new_group(ranks, backend='gloo') if dist.get_backend() != 'gloo' else group
group_cpu = dist.new_group(ranks, backend='gloo') if dist.get_backend() == 'gloo' else group

if self.rank in ranks:
local_rank = ranks.index(self.rank)
Expand Down Expand Up @@ -164,7 +164,7 @@ def init_dist_group(self):
for j in range(self.depth):
ranks = [h * self.depth**3 + i + self.depth * (j + self.depth * k) for k in range(self.depth)]
group = dist.new_group(ranks)
group_cpu = dist.new_group(ranks, backend='gloo') if dist.get_backend() != 'gloo' else group
group_cpu = dist.new_group(ranks, backend='gloo') if dist.get_backend() == 'gloo' else group

if self.rank in ranks:
local_rank = ranks.index(self.rank)
Expand Down Expand Up @@ -217,7 +217,7 @@ def init_dist_group(self):
for i in range(self.depth)
]
group = dist.new_group(ranks)
group_cpu = dist.new_group(ranks, backend='gloo') if dist.get_backend() != 'gloo' else group
group_cpu = dist.new_group(ranks, backend='gloo') if dist.get_backend() == 'gloo' else group

if self.rank in ranks:
local_rank = ranks.index(self.rank)
Expand Down Expand Up @@ -270,7 +270,7 @@ def init_dist_group(self):
for i in range(self.depth)
]
group = dist.new_group(ranks)
group_cpu = dist.new_group(ranks, backend='gloo') if dist.get_backend() != 'gloo' else group
group_cpu = dist.new_group(ranks, backend='gloo') if dist.get_backend() == 'gloo' else group

if self.rank in ranks:
local_rank = ranks.index(self.rank)
Expand Down
2 changes: 1 addition & 1 deletion colossalai/context/process_group_initializer/initializer_data.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ def init_dist_group(self):
for i in range(self.num_data_parallel_group):
ranks = [i + j * self.num_data_parallel_group for j in range(self.data_parallel_size)]
group = dist.new_group(ranks)
group_cpu = dist.new_group(ranks, backend='gloo') if dist.get_backend() != 'gloo' else group
group_cpu = dist.new_group(ranks, backend='gloo') if dist.get_backend() == 'gloo' else group

if self.rank in ranks:
local_rank = ranks.index(self.rank)
Expand Down
2 changes: 1 addition & 1 deletion colossalai/context/process_group_initializer/initializer_model.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ def init_dist_group(self):
for i in range(self.num_group):
ranks = [i * self.model_parallel_size + j for j in range(self.model_parallel_size)]
group = dist.new_group(ranks)
group_cpu = dist.new_group(ranks, backend='gloo') if dist.get_backend() != 'gloo' else group
group_cpu = dist.new_group(ranks, backend='gloo') if dist.get_backend() == 'gloo' else group

if self.rank in ranks:
local_rank = ranks.index(self.rank)
Expand Down
2 changes: 1 addition & 1 deletion colossalai/context/process_group_initializer/initializer_pipeline.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ def init_dist_group(self):
range(i * self.data_group_size + j, (i + 1) * self.data_group_size, self.pipeline_stage_size))
pipe_group_size = len(pipe_ranks)
pipe_group = dist.new_group(pipe_ranks)
group_cpu = dist.new_group(pipe_ranks, backend='gloo') if dist.get_backend() != 'gloo' else pipe_group
group_cpu = dist.new_group(pipe_ranks, backend='gloo') if dist.get_backend() == 'gloo' else pipe_group

if self.rank in pipe_ranks:
local_rank = pipe_ranks.index(self.rank)
Expand Down
2 changes: 1 addition & 1 deletion colossalai/context/process_group_initializer/initializer_sequence.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ def init_dist_group(self):
for i in range(self.num_group):
ranks = [i * self.dp_size + j for j in range(self.dp_size)]
group = dist.new_group(ranks)
group_cpu = dist.new_group(ranks, backend='gloo') if dist.get_backend() != 'gloo' else group
group_cpu = dist.new_group(ranks, backend='gloo') if dist.get_backend() == 'gloo' else group

if self.rank in ranks:
local_rank = ranks.index(self.rank)
Expand Down
2 changes: 1 addition & 1 deletion colossalai/context/process_group_initializer/initializer_tensor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ def init_dist_group(self):
for i in range(self.num_tensor_parallel_group):
ranks = [i * self.tensor_parallel_size + j for j in range(self.tensor_parallel_size)]
group = dist.new_group(ranks)
group_cpu = dist.new_group(ranks, backend='gloo') if dist.get_backend() != 'gloo' else group
group_cpu = dist.new_group(ranks, backend='gloo') if dist.get_backend() == 'gloo' else group

if self.rank in ranks:
local_rank = ranks.index(self.rank)
Expand Down