Skip to content

[Optimizer] Remove useless ColoOptimizer #1312

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
feifeibear merged 11 commits into from
Jul 14, 2022
Merged

[Optimizer] Remove useless ColoOptimizer #1312

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
86be744
make it faster
feifeibear Jul 6, 2022
4b2333e
Merge branch 'main' of github.com:hpcaitech/ColossalAI into main
feifeibear Jul 7, 2022
5078a72
Merge branch 'main' of github.com:hpcaitech/ColossalAI into main
feifeibear Jul 7, 2022
73069e2
Merge branch 'main' of github.com:hpcaitech/ColossalAI into main
feifeibear Jul 11, 2022
371aa27
Merge branch 'main' of github.com:hpcaitech/ColossalAI into main
feifeibear Jul 12, 2022
3ba19e1
Merge branch 'main' of github.com:hpcaitech/ColossalAI into main
feifeibear Jul 13, 2022
2bde9d8
Merge branch 'main' of github.com:hpcaitech/ColossalAI into main
feifeibear Jul 13, 2022
098c24a
Merge branch 'main' of github.com:hpcaitech/ColossalAI into main
feifeibear Jul 14, 2022
0e5210e
Merge branch 'main' of github.com:hpcaitech/ColossalAI into main
feifeibear Jul 14, 2022
7e994dc
Merge branch 'main' of github.com:hpcaitech/ColossalAI into main
feifeibear Jul 14, 2022
75cda1e
remove useless ColoOptimizer
feifeibear Jul 14, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 1 addition & 3 deletions colossalai/nn/optimizer/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,7 @@
from .lars import Lars
from .cpu_adam import CPUAdam
from .hybrid_adam import HybridAdam
from .colo_optimizer import ColoOptimizer

__all__ = [
'ColossalaiOptimizer', 'FusedLAMB', 'FusedAdam', 'FusedSGD', 'Lamb', 'Lars', 'CPUAdam', 'HybridAdam',
'CPU_ADAM_CNT', 'ColoOptimizer'
'ColossalaiOptimizer', 'FusedLAMB', 'FusedAdam', 'FusedSGD', 'Lamb', 'Lars', 'CPUAdam', 'HybridAdam', 'CPU_ADAM_CNT'
]
80 changes: 0 additions & 80 deletions colossalai/nn/optimizer/colo_optimizer.py
View file
Open in desktop

This file was deleted.

1 change: 0 additions & 1 deletion colossalai/tensor/colo_parameter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import torch

from typing import Optional
from copy import copy

from colossalai.tensor.colo_tensor import ColoTensor
from colossalai.tensor.const import TensorType
Expand Down
9 changes: 5 additions & 4 deletions tests/test_tensor/test_model.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
from colossalai.utils import free_port
from colossalai.utils.model.colo_init_context import ColoInitContext
from colossalai.tensor import ColoTensor, ProcessGroup
from colossalai.nn.optimizer import ColoOptimizer
from colossalai.nn.optimizer import ColossalaiOptimizer

from tests.components_to_test.registry import non_distributed_component_funcs
from _utils import split_param_row_tp1d, split_param_col_tp1d
Expand All @@ -33,7 +33,8 @@ def run_1d_hybrid_tp(model_name):
if rank == 0:
model_torch = model_builder(checkpoint=True)
model_torch = model_torch.cuda()
optimizer_torch = ColoOptimizer(model_torch.named_parameters(), torch.optim.SGD, lr=0.1)

optimizer_torch = ColossalaiOptimizer(torch.optim.SGD(model_torch.parameters(), lr=0.1))

# Make two models have the same init params
for p1, p2 in zip(model.parameters(), model_torch.parameters()):
Expand Down Expand Up @@ -80,7 +81,7 @@ def run_1d_hybrid_tp(model_name):
if rank == 0:
model_torch.train()

colo_optimizer = ColoOptimizer(model.named_parameters(), torch.optim.SGD, lr=0.1)
colo_optimizer = ColossalaiOptimizer(torch.optim.SGD(model.parameters(), lr=0.1))

for i, (data, label) in enumerate(train_dataloader):

Expand Down Expand Up @@ -170,7 +171,7 @@ def test_colo_optimizer():
with ColoInitContext(lazy_memory_allocate=False, device=get_current_device()):
model = model_builder(checkpoint=True)

colo_optimizer = ColoOptimizer(model.named_parameters(), torch.optim.SGD, lr=0.1)
colo_optimizer = ColossalaiOptimizer(torch.optim.SGD(model.parameters(), lr=0.1))
for i, (data, label) in enumerate(train_dataloader):
colo_optimizer.zero_grad()
data = data.to(get_current_device())
Expand Down
4 changes: 2 additions & 2 deletions tests/test_utils/test_colo_checkpoint.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
from colossalai.tensor import ComputePattern, ComputeSpec, ColoTensor, ShardSpec, ProcessGroup, DistSpecManager, ReplicaSpec
from colossalai.nn.parallel.data_parallel import ColoDDP
from colossalai.utils.checkpoint import save_checkpoint, load_checkpoint
from colossalai.nn.optimizer import ColoOptimizer
from colossalai.nn.optimizer import ColossalaiOptimizer

from tests.components_to_test.registry import non_distributed_component_funcs

Expand Down Expand Up @@ -117,7 +117,7 @@ def _run_checkpoint(model_name, init_spec_func, use_ddp, use_mp_reload, test_sch
model_reload = model_reload.cuda()
model_reload.train()

colo_optimizer = ColoOptimizer(model.named_parameters(), torch.optim.SGD, lr=0.1)
colo_optimizer = ColossalaiOptimizer(torch.optim.SGD(model.named_parameters(), r=0.1))

for i, (data, label) in enumerate(train_dataloader):

Expand Down