From bccfc905451ebb3d0b2f5c80df81a391c653073c Mon Sep 17 00:00:00 2001 From: binmakeswell Date: Tue, 23 Apr 2024 17:50:44 +0800 Subject: [PATCH 1/5] release llama3 --- examples/language/{llama2 => llama}/README.md | 0 examples/language/{llama2 => llama}/benchmark.py | 0 examples/language/{llama2 => llama}/requirements.txt | 0 examples/language/{llama2 => llama}/scripts/benchmark_70B/3d.sh | 0 .../language/{llama2 => llama}/scripts/benchmark_70B/gemini.sh | 0 .../{llama2 => llama}/scripts/benchmark_70B/gemini_auto.sh | 0 .../language/{llama2 => llama}/scripts/benchmark_7B/gemini.sh | 0 .../{llama2 => llama}/scripts/benchmark_7B/gemini_auto.sh | 0 examples/language/{llama2 => llama}/test_ci.sh | 0 9 files changed, 0 insertions(+), 0 deletions(-) rename examples/language/{llama2 => llama}/README.md (100%) rename examples/language/{llama2 => llama}/benchmark.py (100%) rename examples/language/{llama2 => llama}/requirements.txt (100%) rename examples/language/{llama2 => llama}/scripts/benchmark_70B/3d.sh (100%) rename examples/language/{llama2 => llama}/scripts/benchmark_70B/gemini.sh (100%) rename examples/language/{llama2 => llama}/scripts/benchmark_70B/gemini_auto.sh (100%) rename examples/language/{llama2 => llama}/scripts/benchmark_7B/gemini.sh (100%) rename examples/language/{llama2 => llama}/scripts/benchmark_7B/gemini_auto.sh (100%) rename examples/language/{llama2 => llama}/test_ci.sh (100%) diff --git a/examples/language/llama2/README.md b/examples/language/llama/README.md similarity index 100% rename from examples/language/llama2/README.md rename to examples/language/llama/README.md diff --git a/examples/language/llama2/benchmark.py b/examples/language/llama/benchmark.py similarity index 100% rename from examples/language/llama2/benchmark.py rename to examples/language/llama/benchmark.py diff --git a/examples/language/llama2/requirements.txt b/examples/language/llama/requirements.txt similarity index 100% rename from examples/language/llama2/requirements.txt rename to examples/language/llama/requirements.txt diff --git a/examples/language/llama2/scripts/benchmark_70B/3d.sh b/examples/language/llama/scripts/benchmark_70B/3d.sh similarity index 100% rename from examples/language/llama2/scripts/benchmark_70B/3d.sh rename to examples/language/llama/scripts/benchmark_70B/3d.sh diff --git a/examples/language/llama2/scripts/benchmark_70B/gemini.sh b/examples/language/llama/scripts/benchmark_70B/gemini.sh similarity index 100% rename from examples/language/llama2/scripts/benchmark_70B/gemini.sh rename to examples/language/llama/scripts/benchmark_70B/gemini.sh diff --git a/examples/language/llama2/scripts/benchmark_70B/gemini_auto.sh b/examples/language/llama/scripts/benchmark_70B/gemini_auto.sh similarity index 100% rename from examples/language/llama2/scripts/benchmark_70B/gemini_auto.sh rename to examples/language/llama/scripts/benchmark_70B/gemini_auto.sh diff --git a/examples/language/llama2/scripts/benchmark_7B/gemini.sh b/examples/language/llama/scripts/benchmark_7B/gemini.sh similarity index 100% rename from examples/language/llama2/scripts/benchmark_7B/gemini.sh rename to examples/language/llama/scripts/benchmark_7B/gemini.sh diff --git a/examples/language/llama2/scripts/benchmark_7B/gemini_auto.sh b/examples/language/llama/scripts/benchmark_7B/gemini_auto.sh similarity index 100% rename from examples/language/llama2/scripts/benchmark_7B/gemini_auto.sh rename to examples/language/llama/scripts/benchmark_7B/gemini_auto.sh diff --git a/examples/language/llama2/test_ci.sh b/examples/language/llama/test_ci.sh similarity index 100% rename from examples/language/llama2/test_ci.sh rename to examples/language/llama/test_ci.sh From e68014ac6b87ffe85faf217015d20095f6abc331 Mon Sep 17 00:00:00 2001 From: binmakeswell Date: Tue, 23 Apr 2024 17:54:01 +0800 Subject: [PATCH 2/5] [release] llama3 --- examples/language/llama/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/examples/language/llama/README.md b/examples/language/llama/README.md index 11b2ee511a6e..fa0c6dc07156 100644 --- a/examples/language/llama/README.md +++ b/examples/language/llama/README.md @@ -1,4 +1,10 @@ # Pretraining LLaMA-1/2/3: best practices for building LLaMA-1/2/3-like base models +### LLaMA3 +

+ +

+ +- 70 billion parameter LLaMA3 model training accelerated by 18% ### LLaMA2

From 5bca7fea4c0365478edd5c793a3f14a30d0af42b Mon Sep 17 00:00:00 2001 From: binmakeswell Date: Tue, 23 Apr 2024 17:54:44 +0800 Subject: [PATCH 3/5] [release] llama3 --- README.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 26776bdf6d9f..816da7537bdd 100644 --- a/README.md +++ b/README.md @@ -270,13 +270,21 @@ Acceleration of [AlphaFold Protein Structure](https://alphafold.ebi.ac.uk/)

(back to top)

## Parallel Training Demo +### LLaMA3 +

+ +

+ +- 70 billion parameter LLaMA3 model training accelerated by 18% +[[code]](https://github.com/hpcaitech/ColossalAI/tree/main/examples/language/llama) + ### LLaMA2

- 70 billion parameter LLaMA2 model training accelerated by 195% -[[code]](https://github.com/hpcaitech/ColossalAI/tree/main/examples/language/llama2) +[[code]](https://github.com/hpcaitech/ColossalAI/tree/main/examples/language/llama) [[blog]](https://www.hpc-ai.tech/blog/70b-llama2-training) ### LLaMA1 @@ -285,7 +293,7 @@ Acceleration of [AlphaFold Protein Structure](https://alphafold.ebi.ac.uk/)

- 65-billion-parameter large model pretraining accelerated by 38% -[[code]](https://github.com/hpcaitech/ColossalAI/tree/example/llama/examples/language/llama) +[[code]](https://github.com/hpcaitech/ColossalAI/tree/main/examples/language/llama) [[blog]](https://www.hpc-ai.tech/blog/large-model-pretraining) ### MoE From 27e0b0058f0feac9cc8e3d46b8fe8b8596c98327 Mon Sep 17 00:00:00 2001 From: binmakeswell Date: Tue, 23 Apr 2024 17:59:26 +0800 Subject: [PATCH 4/5] [release] llama3 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 816da7537bdd..c1e2da0d406f 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@
  • Parallel Training Demo
      -
    • LLaMA 1/2
      • +
    • LLaMA 1/2/3
    • MoE
    • GPT-3
    • GPT-2
      • From 5a483473adcd164290d88d35f74b19d5761f0968 Mon Sep 17 00:00:00 2001 From: binmakeswell Date: Tue, 23 Apr 2024 18:01:06 +0800 Subject: [PATCH 5/5] [release] llama3 --- docs/README-zh-Hans.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/README-zh-Hans.md b/docs/README-zh-Hans.md index 6d243a80852d..7e0ed07fec16 100644 --- a/docs/README-zh-Hans.md +++ b/docs/README-zh-Hans.md @@ -51,7 +51,7 @@
    • 并行训练样例展示
        -
      • LLaMA 1/2
        • +
      • LLaMA 1/2/3
      • MoE
      • GPT-3
      • GPT-2
        • @@ -261,6 +261,14 @@ Colossal-AI 为您提供了一系列并行组件。我们的目标是让您的

        (返回顶端)

        ## 并行训练样例展示 +### LLaMA3 +

        + +

        + +- 700亿参数LLaMA3训练加速18% +[[code]](https://github.com/hpcaitech/ColossalAI/tree/main/examples/language/llama) + ### LLaMA2