From 667f47969f748798a9c912a46f54bd4983fd3ba1 Mon Sep 17 00:00:00 2001 From: 1SAA Date: Mon, 9 Jan 2023 16:10:02 +0800 Subject: [PATCH 1/4] change the benchmark config file --- examples/language/gpt/gemini/benchmark_gemini.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/language/gpt/gemini/benchmark_gemini.sh b/examples/language/gpt/gemini/benchmark_gemini.sh index 464ea03da7eb..a6bafaa10611 100644 --- a/examples/language/gpt/gemini/benchmark_gemini.sh +++ b/examples/language/gpt/gemini/benchmark_gemini.sh @@ -1,12 +1,12 @@ for MODEL_TYPE in "gpt2_medium"; do - for DISPAN in "colossalai"; do - for BATCH_SIZE in 16; do + for DISPAN in "zero1" "zero2"; do + for BATCH_SIZE in 8; do for GPUNUM in 1 2 4 8; do - for TPDEGREE in 1 2 4 8; do + for TPDEGREE in 1; do if [ ${TPDEGREE} -gt ${GPUNUM} ]; then continue fi - for PLACEMENT in "cpu" "auto"; do + for PLACEMENT in "auto"; do echo "****************** Begin ***************************" echo "+ benchmrking MODEL ${MODEL_TYPE} DISPAN ${DISPAN} GPU ${GPUNUM} BS ${BATCH_SIZE} TP ${TPDEGREE} POLICY ${PLACEMENT}" MODEL_TYPE=${MODEL_TYPE} DISPAN=${DISPAN} BATCH_SIZE=${BATCH_SIZE} GPUNUM=${GPUNUM} TPDEGREE=${TPDEGREE} PLACEMENT=${PLACEMENT} \ From 31ce17fea7ab691436709830a2f4c2a90f2aa178 Mon Sep 17 00:00:00 2001 From: 1SAA Date: Mon, 9 Jan 2023 17:24:06 +0800 Subject: [PATCH 2/4] change config --- examples/language/gpt/gemini/benchmark_gemini.sh | 10 +++++----- examples/language/gpt/gemini/train_gpt_demo.py | 2 ++ 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/examples/language/gpt/gemini/benchmark_gemini.sh b/examples/language/gpt/gemini/benchmark_gemini.sh index a6bafaa10611..9c7f911ebe6a 100644 --- a/examples/language/gpt/gemini/benchmark_gemini.sh +++ b/examples/language/gpt/gemini/benchmark_gemini.sh @@ -1,6 +1,6 @@ -for MODEL_TYPE in "gpt2_medium"; do - for DISPAN in "zero1" "zero2"; do - for BATCH_SIZE in 8; do +for MODEL_TYPE in "gpt2_10b"; do + for DISTPAN in "zero1" "zero2"; do + for BATCH_SIZE in 8 16 32; do for GPUNUM in 1 2 4 8; do for TPDEGREE in 1; do if [ ${TPDEGREE} -gt ${GPUNUM} ]; then @@ -8,8 +8,8 @@ for MODEL_TYPE in "gpt2_medium"; do fi for PLACEMENT in "auto"; do echo "****************** Begin ***************************" - echo "+ benchmrking MODEL ${MODEL_TYPE} DISPAN ${DISPAN} GPU ${GPUNUM} BS ${BATCH_SIZE} TP ${TPDEGREE} POLICY ${PLACEMENT}" - MODEL_TYPE=${MODEL_TYPE} DISPAN=${DISPAN} BATCH_SIZE=${BATCH_SIZE} GPUNUM=${GPUNUM} TPDEGREE=${TPDEGREE} PLACEMENT=${PLACEMENT} \ + echo "+ benchmrking MODEL ${MODEL_TYPE} DISTPAN ${DISTPAN} GPU ${GPUNUM} BS ${BATCH_SIZE} TP ${TPDEGREE} POLICY ${PLACEMENT}" + MODEL_TYPE=${MODEL_TYPE} DISTPAN=${DISTPAN} BATCH_SIZE=${BATCH_SIZE} GPUNUM=${GPUNUM} TPDEGREE=${TPDEGREE} PLACEMENT=${PLACEMENT} \ bash ./run_gemini.sh echo "****************** Finished ***************************" echo "" diff --git a/examples/language/gpt/gemini/train_gpt_demo.py b/examples/language/gpt/gemini/train_gpt_demo.py index 891b1de15af1..92cb7393c37b 100644 --- a/examples/language/gpt/gemini/train_gpt_demo.py +++ b/examples/language/gpt/gemini/train_gpt_demo.py @@ -290,9 +290,11 @@ def main(): from torch.distributed.optim import ZeroRedundancyOptimizer optimizer = ZeroRedundancyOptimizer(model.parameters(), optimizer_class=torch.optim.Adam, lr=0.01) elif args.distplan.startswith("zero"): + model = model.half() partition_flag = args.distplan == "zero2" optimizer = torch.optim.Adam(model.parameters(), lr=0.01) optimizer = LowLevelZeroOptimizer(optimizer, + reduce_bucket_size=12 * 1024 * 1024, overlap_communication=True, partition_grad=partition_flag, verbose=True) From ea0cb1f48760b1bb8ec06b629185ab05e1e540de Mon Sep 17 00:00:00 2001 From: 1SAA Date: Tue, 10 Jan 2023 11:23:40 +0800 Subject: [PATCH 3/4] revert config file --- examples/language/gpt/gemini/benchmark_gemini.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/language/gpt/gemini/benchmark_gemini.sh b/examples/language/gpt/gemini/benchmark_gemini.sh index 9c7f911ebe6a..78393756a0a5 100644 --- a/examples/language/gpt/gemini/benchmark_gemini.sh +++ b/examples/language/gpt/gemini/benchmark_gemini.sh @@ -1,12 +1,12 @@ -for MODEL_TYPE in "gpt2_10b"; do - for DISTPAN in "zero1" "zero2"; do - for BATCH_SIZE in 8 16 32; do +for MODEL_TYPE in "gpt2_medium"; do + for DISTPAN in "colossalai"; do + for BATCH_SIZE in 16; do for GPUNUM in 1 2 4 8; do - for TPDEGREE in 1; do + for TPDEGREE in 1 2 4 8; do if [ ${TPDEGREE} -gt ${GPUNUM} ]; then continue fi - for PLACEMENT in "auto"; do + for PLACEMENT in "cpu" "auto"; do echo "****************** Begin ***************************" echo "+ benchmrking MODEL ${MODEL_TYPE} DISTPAN ${DISTPAN} GPU ${GPUNUM} BS ${BATCH_SIZE} TP ${TPDEGREE} POLICY ${PLACEMENT}" MODEL_TYPE=${MODEL_TYPE} DISTPAN=${DISTPAN} BATCH_SIZE=${BATCH_SIZE} GPUNUM=${GPUNUM} TPDEGREE=${TPDEGREE} PLACEMENT=${PLACEMENT} \ From 7b7f8091b41118afc756a4f229d534b6e011c61f Mon Sep 17 00:00:00 2001 From: 1SAA Date: Tue, 10 Jan 2023 11:38:24 +0800 Subject: [PATCH 4/4] rename distpan to distplan --- examples/language/gpt/gemini/benchmark_gemini.sh | 6 +++--- examples/language/gpt/gemini/run_gemini.sh | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/examples/language/gpt/gemini/benchmark_gemini.sh b/examples/language/gpt/gemini/benchmark_gemini.sh index 78393756a0a5..9a630b2ffe23 100644 --- a/examples/language/gpt/gemini/benchmark_gemini.sh +++ b/examples/language/gpt/gemini/benchmark_gemini.sh @@ -1,5 +1,5 @@ for MODEL_TYPE in "gpt2_medium"; do - for DISTPAN in "colossalai"; do + for DISTPLAN in "colossalai"; do for BATCH_SIZE in 16; do for GPUNUM in 1 2 4 8; do for TPDEGREE in 1 2 4 8; do @@ -8,8 +8,8 @@ for MODEL_TYPE in "gpt2_medium"; do fi for PLACEMENT in "cpu" "auto"; do echo "****************** Begin ***************************" - echo "+ benchmrking MODEL ${MODEL_TYPE} DISTPAN ${DISTPAN} GPU ${GPUNUM} BS ${BATCH_SIZE} TP ${TPDEGREE} POLICY ${PLACEMENT}" - MODEL_TYPE=${MODEL_TYPE} DISTPAN=${DISTPAN} BATCH_SIZE=${BATCH_SIZE} GPUNUM=${GPUNUM} TPDEGREE=${TPDEGREE} PLACEMENT=${PLACEMENT} \ + echo "+ benchmrking MODEL ${MODEL_TYPE} DISTPLAN ${DISTPLAN} GPU ${GPUNUM} BS ${BATCH_SIZE} TP ${TPDEGREE} POLICY ${PLACEMENT}" + MODEL_TYPE=${MODEL_TYPE} DISTPLAN=${DISTPLAN} BATCH_SIZE=${BATCH_SIZE} GPUNUM=${GPUNUM} TPDEGREE=${TPDEGREE} PLACEMENT=${PLACEMENT} \ bash ./run_gemini.sh echo "****************** Finished ***************************" echo "" diff --git a/examples/language/gpt/gemini/run_gemini.sh b/examples/language/gpt/gemini/run_gemini.sh index ad577c350d39..0c2ea660f1e0 100644 --- a/examples/language/gpt/gemini/run_gemini.sh +++ b/examples/language/gpt/gemini/run_gemini.sh @@ -1,8 +1,8 @@ set -x # distplan in ["colossalai", "zero1", "zero2", "torch_ddp", "torch_zero"] -export DISTPAN=${DISTPAN:-"colossalai"} +export DISTPLAN=${DISTPLAN:-"colossalai"} -# The following options only valid when DISTPAN="colossalai" +# The following options only valid when DISTPLAN="colossalai" export GPUNUM=${GPUNUM:-1} export TPDEGREE=${TPDEGREE:-1} export PLACEMENT=${PLACEMENT:-"cpu"} @@ -20,5 +20,5 @@ torchrun --standalone --nproc_per_node=${GPUNUM} ./train_gpt_demo.py \ --batch_size=${BATCH_SIZE} \ --placement=${PLACEMENT} \ --shardinit=${USE_SHARD_INIT} \ ---distplan=${DISTPAN} \ -2>&1 | tee ./gemini_logs/${MODEL_TYPE}_${DISTPAN}_gpu_${GPUNUM}_bs_${BATCH_SIZE}_tp_${TPDEGREE}_${PLACEMENT}.log +--distplan=${DISTPLAN} \ +2>&1 | tee ./gemini_logs/${MODEL_TYPE}_${DISTPLAN}_gpu_${GPUNUM}_bs_${BATCH_SIZE}_tp_${TPDEGREE}_${PLACEMENT}.log