Transformers v4.12.0 compatible

README.md

@@ -16,17 +16,18 @@ Below shows the generation speed gain by using FastSeq.
 
 | Model            | W/O FastSeq (in samples/s) | W/ FastSeq (in samples/s) | Speedup |
 |------------------|:--------------------------:|:-------------------------:|:-----:|
-| [ProphetNet](examples/prophetnet/README.md)       | 2.8 | 11.9  | 4.3  |
+| [ProphetNet (`fs`)](examples/prophetnet/README.md)       | 2.8 | 11.9  | 4.3  |
 | [Bart (`fs`)](examples/bart/README.md)              | 3.3  | 25.1 | 7.7x  |
-| [Bart (`hf`)](examples/bart/README.md#speedup-bart-huggingface-transformers-version-by-using-fastseq) | 2.5 | 12.4 | 5.0x  |
-| [DistilBart (`hf`)](examples/distilbart/README.md)    | 3.4  | 18.5  | 5.4x  |
-| [T5 (`hf`)](examples/t5/README.md)                  | 8.7  | 31.3  | 3.6x  |
+| [Bart (`hf`)](examples/bart/README.md#speedup-bart-huggingface-transformers-version-by-using-fastseq) | 4.5 | 12.4 | 2.8x  |
+| [DistilBart (`hf`)](examples/distilbart/README.md)    | 5.5  | 19.1  | 3.5x  |
+| [T5 (`hf`)](examples/t5/README.md)                  | 9.5  | 31.7  | 3.3x  |
 | [WMT16 En-De (`fs`)](examples/wmt/README.md)        | 144.5   | 422.8  | 2.9x  |
-| [GPT2 (`hf`)](examples/gpt2/README.md)        | 3.0   | 16.7  | 5.5x  |
-| [UniLM (`hf`)](examples/unilm/README.md)        | 1.7   | 16.4  | 9.6x  |
+| [GPT2 (`hf`)](examples/gpt2/README.md)        | 3.9   | 21.8  | 5.6x  |
+| [ProphetNet (`hf`)](examples/prophetnet/README.md)        | 3.4   | 6.2  | 1.8x  |
 
 - All benchmarking experiments run on NVIDIA-V100-16GB with [docker](docker/Dockerfile). Highest speed recorded for each model by tuning batch size. For parameter setting details, click link of corresponding model.
-- `fs` stands for [Fairseq](https://github.com/pytorch/fairseq) 0.10.2 version, `hf` stands for [Huggingface Transformers](https://github.com/huggingface/transformers) 3.0.2 version.
+- The baseline (W/O Fastseq) for [ProphetNet (`fs`)](examples/prophetnet/README.md) is run with fairseq 0.9.0, as it has not yet been updated for compatibility with version 0.10.2
+- `fs` stands for [Fairseq](https://github.com/pytorch/fairseq) 0.10.2 version, `hf` stands for [Huggingface Transformers](https://github.com/huggingface/transformers) 4.12.0 version.
 - Optimizations were automatically applied to all generation/sequence models in Fairseq & Huggingface Transformers. Above only lists a subset of them.
 
 ## How it works?
@@ -39,7 +40,7 @@ FastSeq develops multiple speedup techniques, including an attention cache optim
 - Python version >= 3.6
 - [torch](http://pytorch.org/) >= 1.4.0
 - [fairseq](https://github.com/pytorch/fairseq) >= 0.10.0
-- [transformers](https://github.com/huggingface/transformers) == 3.0.2
+- [transformers](https://github.com/huggingface/transformers) >= 4.12.0
 - [requests](https://pypi.org/project/requests/) >= 2.24.0
 - [absl-py](https://pypi.org/project/absl-py/) >= 0.9.0
 - [rouge-score](https://pypi.org/project/rouge-score/) >= 0.0.4

azure-pipelines.yml

@@ -15,21 +15,21 @@ jobs:
       demands:
         - agent.name -equals gpu3
     container:
-      image: adsbrainwestus2.azurecr.io/fastseq:dev-py3 
-      endpoint: fastseq-acr
+      image: huggingface/transformers-pytorch-gpu:latest
       options: --gpus device=3
     steps:
     - script: |
         #install fastseq
-        which pip
-        which python 
+        pip install --upgrade pip
+        pip install sentencepiece==0.1.96
+        pip install torch==1.10.0
 
         echo "******* Installing fairseq *******"
         pip install fairseq==0.10.2
         pip show fairseq
 
         echo "******* Installing transformers *******"
-        pip install transformers
+        pip install transformers==4.12.0
         pip show transformers
 
         echo "******* Installing fastseq *******"
@@ -39,10 +39,6 @@ jobs:
         echo "******* Adding local bin to path *******"
         export PATH="$HOME/bin:$HOME/.local/bin:$PATH"
 
-        echo "******* Running fastseq unittests *******"
-        pip install pytorch-transformers==1.0.0
-        bash tests/run_fastseq_tests.sh
-
         #cd benchmarks/
         #bash run_all_benchmarks.sh
 
@@ -53,11 +49,16 @@ jobs:
         python -c "import torch; print('torch:', torch.__version__, torch)"
         python -c "import torch; print('CUDA available:', torch.cuda.is_available())"
 
+        echo "******* Running transformers unittests *******"
+        bash tests/run_transformers_tests.sh
+
         echo "******* Running fairseq unittests *******"
+        pip install apex==0.9.10.dev0
         bash tests/run_fairseq_tests.sh
 
-        echo "******* Running transformers unittests *******"
-        bash tests/run_transformers_tests.sh
+        echo "******* Running fastseq unittests *******"
+        pip install pytorch-transformers==1.0.0
+        bash tests/run_fastseq_tests.sh
 
       displayName: 'run fastseq unit tests'
     - task: PublishTestResults@2

benchmarks/hf.sh

@@ -1,10 +1,9 @@
 #!/bin/bash
 source utils.sh
 if [[ $SKIP_BASELINE -eq 0 ]]; then
-    export BASELINE_REPO=$CACHE_DIR/transformers_v3.0.2
-    #https://github.com/huggingface/transformers.git \
+    export BASELINE_REPO=$CACHE_DIR/transformers_v4.12.0
     git_clone_if_not_in_cache \
-	https://github.com/JiushengChen/transformers.git \
+	https://github.com/huggingface/transformers.git \
         $BASELINE_REPO \
-        v3.0.2-ngram
+        v4.12.0
 fi

benchmarks/models/hf_bart.sh

@@ -15,7 +15,7 @@ source hf.sh
     facebook/bart-large-cnn \
     cnn_dm/raw \
     val \
-    32 \
+    32/64 \
     --task summarization \
     --no_repeat_ngram_size 3
 ./benchmark.sh \
@@ -33,16 +33,16 @@ grep "facebook/bart-large-cnn cnn_dm/raw val " perf \
 	| awk -F'|' '{if($1!="NA"){c+=1;s+=$1}}END{print s/c}' \
 	| ./range.sh 0.447 0.448
 # Speed on V100 16GB 250W
-grep -E "transformers_v3.0.2 facebook/bart-large-cnn cnn_dm/raw val 32 " perf \
+grep -E "transformers_v4.12.0 facebook/bart-large-cnn cnn_dm/raw val 64 " perf \
 	| awk '{s+=$13}END{if(NR==0) print -1; else print s/NR}' \
-	| ./range.sh 2 3
-grep -E "transformers_v3.0.2\+fastseq_v.* facebook/bart-large-cnn cnn_dm/raw val 32 " perf \
+	| ./range.sh 4 5
+grep -E "transformers_v4.12.0\+fastseq_v.* facebook/bart-large-cnn cnn_dm/raw val 32 " perf \
 	| awk '{s+=$13}END{print s/NR}' \
-	| ./range.sh 7 100
-grep -E "transformers_v3.0.2\+fastseq_v.* facebook/bart-large-cnn cnn_dm/raw val 64 " perf \
+	| ./range.sh 10 100
+grep -E "transformers_v4.12.0\+fastseq_v.* facebook/bart-large-cnn cnn_dm/raw val 64 " perf \
 	| awk '{s+=$13}END{print s/NR}' \
 	| ./range.sh 11 100
-grep -E "transformers_v3.0.2\+fastseq_v.* facebook/bart-large-cnn cnn_dm/raw val 128 " perf \
+grep -E "transformers_v4.12.0\+fastseq_v.* facebook/bart-large-cnn cnn_dm/raw val 128 " perf \
 	| awk '{s+=$13}END{print s/NR}' \
 	| ./range.sh 12 100
 

benchmarks/models/hf_distibart.sh

@@ -32,12 +32,12 @@ grep "hf.sshleifer.distilbart-cnn-12-6.tar.gz cnn_dm/raw val " perf \
 	| awk -F'|' '{if($1!="NA"){c+=1;s+=$1}}END{print s/c}' \
 	| ./range.sh 0.45 0.452
 # Speed on V100 16GB 250W
-grep -E "transformers_v3.0.2 hf.sshleifer.distilbart-cnn-12-6.tar.gz cnn_dm/raw val 64 " perf \
+grep -E "transformers_v4.12.0 hf.sshleifer.distilbart-cnn-12-6.tar.gz cnn_dm/raw val 128 " perf \
 	| awk '{s+=$13}END{if(NR==0) print -1; else print s/NR}' \
-	| ./range.sh 3 4
-grep -E "transformers_v3.0.2\+fastseq_v.* hf.sshleifer.distilbart-cnn-12-6.tar.gz cnn_dm/raw val 64 " perf \
+	| ./range.sh 5 6
+grep -E "transformers_v4.12.0\+fastseq_v.* hf.sshleifer.distilbart-cnn-12-6.tar.gz cnn_dm/raw val 64 " perf \
 	| awk '{s+=$13}END{print s/NR}' \
-	| ./range.sh 16.5 100
-grep -E "transformers_v3.0.2\+fastseq_v.* hf.sshleifer.distilbart-cnn-12-6.tar.gz cnn_dm/raw val 128 " perf \
+	| ./range.sh 17 100
+grep -E "transformers_v4.12.0\+fastseq_v.* hf.sshleifer.distilbart-cnn-12-6.tar.gz cnn_dm/raw val 128 " perf \
 	| awk '{s+=$13}END{print s/NR}' \
-	| ./range.sh 18.3 100
+	| ./range.sh 18 100

benchmarks/models/hf_gpt2.sh

@@ -7,7 +7,6 @@
 #   <split> # train/val/test (text) or train/valid/test (binary)
 #   <batch-sizes>
 source hf.sh
-
 # MODEL - bart large cnn from transformer
 # TASK - cnn dm val full set
 
@@ -16,7 +15,7 @@ source hf.sh
     gpt2 \
     cnn_dm/raw \
     val \
-    64/128 \
+    64/128/256 \
     --task summarization \
     --no_repeat_ngram_size 3 \
     --max_tokenizer_length 512 \
@@ -27,7 +26,7 @@ source hf.sh
     gpt2 \
     cnn_dm/raw \
     val \
-    64 \
+    64/128 \
     --task summarization \
     --no_repeat_ngram_size 3 \
     --max_tokenizer_length 512 \
@@ -37,14 +36,18 @@ source hf.sh
 grep "gpt2 cnn_dm/raw val " perf \
 	| awk '{print $9}' \
 	| awk -F'|' '{if($1!="NA"){c+=1;s+=$1}}END{print s/c}' \
-	| ./range.sh 0.155 0.156
+	| ./range.sh 0.160 0.162
 # Speed on V100 16GB 250W
-grep -E "transformers_v3.0.2 gpt2 cnn_dm/raw val 64 " perf \
+grep -E "transformers_v4.12.0 gpt2 cnn_dm/raw val 64 " perf \
 	| awk '{s+=$13}END{if(NR==0) print -1; else print s/NR}' \
-	| ./range.sh 2.9 3.2
-grep -E "transformers_v3.0.2\+fastseq_v.* gpt2 cnn_dm/raw val 64 " perf \
+	| ./range.sh 3.5 4.5
+grep -E "transformers_v4.12.0\+fastseq_v.* gpt2 cnn_dm/raw val 64 " perf \
+	| awk '{s+=$13}END{print s/NR}' \
+	| ./range.sh 16 100
+grep -E "transformers_v4.12.0\+fastseq_v.* gpt2 cnn_dm/raw val 128 " perf \
 	| awk '{s+=$13}END{print s/NR}' \
-	| ./range.sh 10.8 11.3
-grep -E "transformers_v3.0.2\+fastseq_v.* gpt2 cnn_dm/raw val 128 " perf \
+	| ./range.sh 20 100
+grep -E "transformers_v4.12.0\+fastseq_v.* gpt2 cnn_dm/raw val 256 " perf \
 	| awk '{s+=$13}END{print s/NR}' \
-	| ./range.sh 16.4 16.8
+	| ./range.sh 21 100
+

benchmarks/models/hf_mbart.sh

@@ -26,11 +26,11 @@ source hf.sh
 # Accuracy
 grep "facebook/mbart-large-en-ro wmt_en_ro/raw val " perf \
 	| awk '{if($8!="NA"){c+=1;s+=$8}}END{print s/c}' \
-	| ./range.sh 56.1 56.3
+	| ./range.sh 56.1 56.4
 # Speed on V100 16GB 250W
-grep -E "transformers_v3.0.2 facebook/mbart-large-en-ro wmt_en_ro/raw val 64 " perf \
+grep -E "transformers_v4.12.0 facebook/mbart-large-en-ro wmt_en_ro/raw val 64 " perf \
 	| awk '{s+=$13}END{if(NR==0) print -1; else print s/NR}' \
 	| ./range.sh 6.0 100
-grep -E "transformers_v3.0.2\+fastseq_v.* facebook/mbart-large-en-ro wmt_en_ro/raw val 64 " perf \
+grep -E "transformers_v4.12.0\+fastseq_v.* facebook/mbart-large-en-ro wmt_en_ro/raw val 64 " perf \
 	| awk '{s+=$13}END{print s/NR}' \
 	| ./range.sh 9 100

benchmarks/models/hf_prophetnet.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+# Run it at its parent folder, and check result at ../perf.
+# USAGE - ./benchmark.sh
+#   [fairseq|fairseq+fastseq|transformers|transformers+fastseq]
+#   <model>
+#   <task>
+#   <split> # train/val/test (text) or train/valid/test (binary)
+#   <batch-sizes>
+source hf.sh
+
+# MODEL - prophetnet from transformer
+# TASK - cnn dm val full set
+./benchmark.sh \
+    transformers \
+    microsoft/prophetnet-large-uncased \
+    cnn_dm_bert/raw \
+    val \
+    128 \
+    --task summarization \
+    --no_repeat_ngram_size 3
+./benchmark.sh \
+    transformers+fastseq \
+    microsoft/prophetnet-large-uncased \
+    cnn_dm_bert/raw \
+    val \
+    128 \
+    --task summarization \
+    --no_repeat_ngram_size 3
+
+# Accuracy
+grep "microsoft/prophetnet-large-uncased cnn_dm_bert/raw val " perf \
+	| awk '{print $9}' \
+	| awk -F'|' '{if($1!="NA"){c+=1;s+=$1}}END{print s/c}' \
+	| ./range.sh 0.230 0.232
+# Speed on V100 16GB 250W
+grep -E "transformers_v4.12.0 microsoft/prophetnet-large-uncased cnn_dm_bert/raw val 128 " perf \
+	| awk '{s+=$13}END{if(NR==0) print -1; else print s/NR}' \
+	| ./range.sh 3 4
+grep -E "transformers_v4.12.0+fastseq_v.* microsoft/prophetnet-large-uncased cnn_dm_bert/raw val 128 " perf \
+	| awk '{s+=$13}END{if(NR==0) print -1; else print s/NR}' \
+	| ./range.sh 6 100

benchmarks/models/hf_t5.sh

@@ -14,28 +14,30 @@ source hf.sh
     wmt_en_ro/raw \
     val \
     64 \
-    --task translation_en_to_ro 
-#    --no_repeat_ngram_size 3	# baseline don't support this arg now.
+    --task translation_en_to_ro \
+    --no_repeat_ngram_size 3
+
 ./benchmark.sh \
     transformers+fastseq \
     t5-base \
     wmt_en_ro/raw \
     val \
     64/128 \
     --task translation_en_to_ro \
-    --postprocess_workers 3
-#    --no_repeat_ngram_size 3
-# Accuracy
+    --postprocess_workers 3 \
+    --no_repeat_ngram_size 3
+
+# # Accuracy
 grep "t5-base wmt_en_ro/raw val " perf \
 	| awk '{if($8!="NA"){c+=1;s+=$8}}END{print s/c}' \
-	| ./range.sh 57.8 57.9
+	| ./range.sh 58.0 59.0
 # Speed on V100 16GB 250W
-grep -E "transformers_v3.0.2 t5-base wmt_en_ro/raw val 64 " perf \
+grep -E "transformers_v4.12.0 t5-base wmt_en_ro/raw val 64 " perf \
 	| awk '{s+=$13}END{if(NR==0) print -1; else print s/NR}' \
-	| ./range.sh 8 10
-grep -E "transformers_v3.0.2\+fastseq_v.* t5-base wmt_en_ro/raw val 64 " perf \
+	| ./range.sh 12 17
+grep -E "transformers_v4.12.0\+fastseq_v.* t5-base wmt_en_ro/raw val 64 " perf \
 	| awk '{s+=$13}END{print s/NR}' \
-	| ./range.sh 19 100
-grep -E "transformers_v3.0.2\+fastseq_v.* t5-base wmt_en_ro/raw val 128 " perf \
+	| ./range.sh 23 100
+grep -E "transformers_v4.12.0\+fastseq_v.* t5-base wmt_en_ro/raw val 128 " perf \
 	| awk '{s+=$13}END{print s/NR}' \
 	| ./range.sh 30 100

benchmarks/run_eval_hf.py

@@ -39,7 +39,7 @@ def calculate_bleu_score(output_lns, refs_lns, **kwargs) -> dict:
     return {"bleu": corpus_bleu(output_lns, [refs_lns], **kwargs).score}
 
 def trim_batch(
-    input_ids, pad_token_id, attention_mask=None,
+    input_ids, pad_token_id, attention_mask=None, **kwargs
 ):
     """Remove columns that are populated exclusively by pad_token_id"""
     keep_column_mask = input_ids.ne(pad_token_id).any(dim=0)

docker/Dockerfile

@@ -43,15 +43,16 @@ RUN cd ${STAGE_DIR} && \
 # Installation/NLP Utilities
 ##############################################################################
 RUN pip install --upgrade pip && \
-	pip install yapf>=v0.30.0 && \
-	pip install absl-py>=v0.9.0 && \
-	pip install filelock>=v3.0.12 && \
-	pip install requests>=v2.24.0 && \
-	pip install gitpython>=v3.1.7 && \
-	pip install rouge_score==v0.0.4 && \
-	pip install fairseq==v0.10.2 && \
-	pip install transformers==v3.0.2 && \
-	pip install pytorch-transformers==1.0.0
+    pip install yapf>=v0.30.0 && \
+    pip install absl-py>=v0.9.0 && \
+    pip install filelock>=v3.0.12 && \
+    pip install requests>=v2.24.0 && \
+    pip install gitpython>=v3.1.7 && \
+    pip install rouge_score==v0.0.4 && \
+    pip install fairseq==v0.10.2 && \
+    pip install transformers==v4.12.0 && \
+    pip install pytorch-transformers==1.0.0 && \
+    pip install sentencepiece==0.1.90
 
 
 ##############################################################################