diff --git a/.ci/scripts/gather_test_models.py b/.ci/scripts/gather_test_models.py index 44ae9cb1116..59f078944cb 100644 --- a/.ci/scripts/gather_test_models.py +++ b/.ci/scripts/gather_test_models.py @@ -10,7 +10,7 @@ from typing import Any from examples.models import MODEL_NAME_TO_MODEL -from examples.recipes.xnnpack_optimization import MODEL_NAME_TO_OPTIONS +from examples.recipes.xnnpack import MODEL_NAME_TO_OPTIONS BUILD_TOOLS = [ "buck2", @@ -51,7 +51,8 @@ def export_models_for_ci() -> None: for name in MODEL_NAME_TO_MODEL.keys(): quantization_configs = { False, - name in MODEL_NAME_TO_OPTIONS and MODEL_NAME_TO_OPTIONS[name].quantization, + name in MODEL_NAME_TO_OPTIONS + and MODEL_NAME_TO_OPTIONS[name].xnnpack_quantization, } delegation_configs = { False, diff --git a/.ci/scripts/test.sh b/.ci/scripts/test.sh index af50faa1c47..e65300fb872 100755 --- a/.ci/scripts/test.sh +++ b/.ci/scripts/test.sh @@ -53,11 +53,11 @@ build_cmake_executor_runner() { } test_model() { - "${PYTHON_EXECUTABLE}" -m examples.export.export_example --model_name="${MODEL_NAME}" + "${PYTHON_EXECUTABLE}" -m examples.export.portable --model_name="${MODEL_NAME}" # Run test model if [[ "${BUILD_TOOL}" == "buck2" ]]; then - buck2 run //examples/executor_runner:executor_runner -- --model_path "./${MODEL_NAME}.pte" + buck2 run //examples/runtime/portable:executor_runner -- --model_path "./${MODEL_NAME}.pte" elif [[ "${BUILD_TOOL}" == "cmake" ]]; then if [[ ! -f ${CMAKE_OUTPUT_DIR}/executor_runner ]]; then build_cmake_executor_runner @@ -92,24 +92,24 @@ test_model_with_xnnpack() { # Quantization-only if [[ ${WITH_QUANTIZATION} == true ]] && [[ ${WITH_DELEGATION} == false ]]; then - bash examples/quantization/test_quantize.sh "${BUILD_TOOL}" "${MODEL_NAME}" + bash examples/quantization/quant_flow/test_quantize.sh "${BUILD_TOOL}" "${MODEL_NAME}" exit 0 fi # Delegation if [[ ${WITH_QUANTIZATION} == true ]]; then SUFFIX="q8" - "${PYTHON_EXECUTABLE}" -m examples.backend.xnnpack_examples --model_name="${MODEL_NAME}" --delegate --quantize + "${PYTHON_EXECUTABLE}" -m examples.recipes.xnnpack.aot_compiler --model_name="${MODEL_NAME}" --delegate --quantize else SUFFIX="fp32" - "${PYTHON_EXECUTABLE}" -m examples.backend.xnnpack_examples --model_name="${MODEL_NAME}" --delegate + "${PYTHON_EXECUTABLE}" -m examples.recipes.xnnpack.aot_compiler --model_name="${MODEL_NAME}" --delegate fi OUTPUT_MODEL_PATH="${MODEL_NAME}_xnnpack_${SUFFIX}.pte" # Run test model if [[ "${BUILD_TOOL}" == "buck2" ]]; then - buck2 run //examples/backend:xnn_executor_runner -- --model_path "${OUTPUT_MODEL_PATH}" + buck2 run //examples/runtime/xnnpack:xnn_executor_runner -- --model_path "${OUTPUT_MODEL_PATH}" elif [[ "${BUILD_TOOL}" == "cmake" ]]; then if [[ ! -f ${CMAKE_OUTPUT_DIR}/backends/xnnpack/xnn_executor_runner ]]; then build_cmake_xnn_executor_runner @@ -123,15 +123,15 @@ test_model_with_xnnpack() { test_demo_backend_delegation() { echo "Testing demo backend delegation on AddMul" - "${PYTHON_EXECUTABLE}" -m examples.export.export_and_delegate --option "composite" - "${PYTHON_EXECUTABLE}" -m examples.export.export_and_delegate --option "partition" - "${PYTHON_EXECUTABLE}" -m examples.export.export_and_delegate --option "whole" + "${PYTHON_EXECUTABLE}" -m examples.recipes.export_and_delegate --option "composite" + "${PYTHON_EXECUTABLE}" -m examples.recipes.export_and_delegate --option "partition" + "${PYTHON_EXECUTABLE}" -m examples.recipes.export_and_delegate --option "whole" # Run test model if [[ "${BUILD_TOOL}" == "buck2" ]]; then - buck2 run //examples/executor_runner:executor_runner -- --model_path "./composite_model.pte" - buck2 run //examples/executor_runner:executor_runner -- --model_path "./partition_lowered_model.pte" - buck2 run //examples/executor_runner:executor_runner -- --model_path "./whole.pte" + buck2 run //examples/runtime/portable:executor_runner -- --model_path "./composite_model.pte" + buck2 run //examples/runtime/portable:executor_runner -- --model_path "./partition_lowered_model.pte" + buck2 run //examples/runtime/portable:executor_runner -- --model_path "./whole.pte" elif [[ "${BUILD_TOOL}" == "cmake" ]]; then if [[ ! -f ${CMAKE_OUTPUT_DIR}/executor_runner ]]; then build_cmake_executor_runner diff --git a/.ci/scripts/utils.sh b/.ci/scripts/utils.sh index f0681d969a0..00780bdadc7 100644 --- a/.ci/scripts/utils.sh +++ b/.ci/scripts/utils.sh @@ -70,7 +70,7 @@ install_flatc_from_source() { build_executorch_runner_buck2() { # Build executorch runtime with retry as this step is flaky on macos CI - retry buck2 build //examples/executor_runner:executor_runner + retry buck2 build //examples/runtime/portable:executor_runner } build_executorch_runner_cmake() { diff --git a/build/cmake_deps.toml b/build/cmake_deps.toml index afc538b7ab6..14d1b6888d7 100644 --- a/build/cmake_deps.toml +++ b/build/cmake_deps.toml @@ -13,7 +13,7 @@ excludes = [ [targets.executor_runner] buck_targets = [ - "//examples/executor_runner:executor_runner", + "//examples/runtime/portable:executor_runner", ] filters = [ ".cpp$", @@ -93,7 +93,7 @@ deps = [ [targets.xnn_executor_runner] buck_targets = [ - "//examples/backend:xnn_executor_runner", + "//examples/runtime/xnnpack:xnn_executor_runner", ] filters = [ ".cpp$", diff --git a/docs/source/getting-started-setup.md b/docs/source/getting-started-setup.md index 39df51febc4..7a61690c896 100644 --- a/docs/source/getting-started-setup.md +++ b/docs/source/getting-started-setup.md @@ -105,17 +105,17 @@ how to do it, we will generate an ExecuTorch program file from an `nn.Module`. You can generate an ExecuTorch program by using a sample script or by using the Python interpreter. -We have created the `export_example.py` script that demonstrates a simple model +We have created the `portable.py` script that demonstrates a simple model export to flatbuffer. This script is available in the [pytorch/executorch](https://github.com/pytorch/executorch/tree/main/examples/export) repository. To generate a sample program, complete the following steps: -1. Run the `export_example.py` script: +1. Run the `portable.py` script: ```bash - python3 -m examples.export.export_example --model_name="add" + python3 -m examples.export.portable --model_name="add" ``` :::{dropdown} Output @@ -193,7 +193,7 @@ Complete the following steps: 3. Build a binary: ```bash - /tmp/buck2 build //examples/executor_runner:executor_runner --show-output + /tmp/buck2 build //examples/runtime/portable:executor_runner --show-output ``` :::{dropdown} Output @@ -227,7 +227,7 @@ the `buck run` command to run our program. * To run the `add.pte` program: ```bash - /tmp/buck2 run //examples/executor_runner:executor_runner -- --model_path add.pte + /tmp/buck2 run //examples/runtime/portable:executor_runner -- --model_path add.pte ``` :::{dropdown} Sample Output diff --git a/docs/website/docs/tutorials/00_setting_up_executorch.md b/docs/website/docs/tutorials/00_setting_up_executorch.md index 5c241b2f92d..11ccebe6a0e 100644 --- a/docs/website/docs/tutorials/00_setting_up_executorch.md +++ b/docs/website/docs/tutorials/00_setting_up_executorch.md @@ -58,10 +58,10 @@ corresponsing version of the repo. Via python script: ```bash # Creates the file `add.pte` -python3 -m examples.export.export_example --model_name="add" +python3 -m examples.export.portable --model_name="add" # Creates the delegated program `composite_model.pte`, other options are "whole" and "partition" -python3 -m examples.export.export_and_delegate --option "composite" +python3 -m examples.recipes.export_and_delegate --option "composite" ``` Or via python interpreter: @@ -98,7 +98,7 @@ You may want to copy the `buck2` binary into your `$PATH` so you can run it as ` `executor_runner` is an example wrapper around executorch runtime which includes all the operators and backends ```bash -/tmp/buck2 build //examples/executor_runner:executor_runner --show-output +/tmp/buck2 build //examples/runtime/portable:executor_runner --show-output ``` The `--show-output` flag will print the path to the executable if you want to run it directly. @@ -112,10 +112,10 @@ conda install -c conda-forge lld ```bash # add.pte is the program generated from export_example.py during AOT Setup Step 3 -/tmp/buck2 run //examples/executor_runner:executor_runner -- --model_path add.pte +/tmp/buck2 run //examples/runtime/portable:executor_runner -- --model_path add.pte # To run a delegated model -/tmp/buck2 run //examples/executor_runner:executor_runner -- --model_path composite_model.pte +/tmp/buck2 run //examples/runtime/portable:executor_runner -- --model_path composite_model.pte ``` or execute the binary directly from the `--show-output` path shown when building. diff --git a/docs/website/docs/tutorials/profiling.md b/docs/website/docs/tutorials/profiling.md index 34ce7341f80..23ed7f7947c 100644 --- a/docs/website/docs/tutorials/profiling.md +++ b/docs/website/docs/tutorials/profiling.md @@ -56,7 +56,7 @@ Here is an example of what a ExecuTorch run + profile + post-procesing workflow This runs the sample program with profiling enabled ```bash cd executorch -buck2 run -c executorch.prof_enabled=true examples/executor_runner:executor_runner -- --model_path add.pte +buck2 run -c executorch.prof_enabled=true examples/runtime/portable:executor_runner -- --model_path add.pte ``` Run the post-processing CLI tool that calls into the same API's listed above and prints out the profiling results in a tabulated format in the terminal. diff --git a/examples/README.md b/examples/README.md index d85f76aa0fa..5c30b045647 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,6 +1,6 @@ # Examples -This dir contains scripts and other helper utilities to illustrate an end-to-end workflow to run a torch.nn.module on the ExecuTorch runtime. +This dir contains scripts and other helper utilities to illustrate an end-to-end workflow to run a torch.nn.module on the Executorch runtime. It also includes a list of modules, from a simple `Add` to a full model like `MobileNetv2` and `MobileNetv3`, with more to come. @@ -10,12 +10,12 @@ examples |── backend # Contains examples for exporting delegate models and running them using custom executor runners ├── custom_ops # Contains examples to register custom operators into PyTorch as well as register its kernels into ExecuTorch runtime ├── example_quantizer_and_delegate # Contains examples to to fully lowered a MobileNetV2 model to the example backend with an example quantizer -├── executor_runner # This is an example C++ wrapper around the ET runtime -├── export # Python helper scripts to illustrate export workflow +├── export # Contains scripts to illustrate export workflow in portable mode ├── ios_demo_apps # Contains iOS demo apps -├── models # Contains a set of simple to PyTorch models +├── models # Contains a set of out-of-box PyTorch models ├── quantization # Contains examples of quantization workflow -├── arm # Contains examples of the Arm TOSA and Ethos-U NPU flows +├── recipes # Contains recipes for a set of demos +├── runtime # Contains examples of C++ wrapper around the ET runtime └── README.md # This file ``` @@ -31,7 +31,7 @@ and executing previously exported binary file(s). 1. Following the setup guide in [Setting up ExecuTorch from GitHub](/docs/website/docs/tutorials/00_setting_up_executorch.md) you should be able to get the basic development environment for ExecuTorch working. -2. Using the script `export/export_example.py` generate a model binary file by selecting a +2. Using the example script `portable/aot_compiler.py` generate a model binary file by selecting a model name from the list of available models in the `models` dir. @@ -39,10 +39,10 @@ model name from the list of available models in the `models` dir. cd executorch # To the top level dir # To get a list of example models -python3 -m examples.export.export_example -h +python3 -m examples.export.portable -h # To generate a specific pte model -python3 -m examples.export.export_example --model_name="mv2" # for MobileNetv2 +python3 -m examples.export.portable --model_name="mv2" # for MobileNetv2 # This should generate ./mv2.pte file, if successful. ``` @@ -52,7 +52,7 @@ Use `-h` (or `--help`) to see all the supported models. 3. Once we have the model binary (pte) file, then let's run it with ExecuTorch runtime using the `executor_runner`. ```bash -buck2 run examples/executor_runner:executor_runner -- --model_path mv2.pte +buck2 run examples/runtime/portable:executor_runner -- --model_path mv2.pte ``` ## Quantization @@ -87,7 +87,7 @@ buck2 run executorch/examples/quantization:example -- --help Quantized model can be run via executor_runner, similar to floating point model, via, as shown above: ```bash -buck2 run examples/executor_runner:executor_runner -- --model_path mv2.pte +buck2 run examples/runtime/portable:executor_runner -- --model_path mv2.pte ``` Note that, running quantized model, requires various quantized/dequantize operators, available in [quantized kernel lib](/kernels/quantized). diff --git a/examples/backend/targets.bzl b/examples/backend/targets.bzl deleted file mode 100644 index e923e1d5d77..00000000000 --- a/examples/backend/targets.bzl +++ /dev/null @@ -1,44 +0,0 @@ -load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "get_oss_build_kwargs", "runtime") - -def define_common_targets(): - """Defines targets that should be shared between fbcode and xplat. - - The directory containing this targets.bzl file should also contain both - TARGETS and BUCK files that call this function. - """ - - runtime.python_binary( - name = "xnnpack_examples", - main_module = "executorch.examples.backend.xnnpack_examples", - deps = [ - ":xnnpack_examples_lib", - ], - ) - - runtime.python_library( - name = "xnnpack_examples_lib", - srcs = [ - "xnnpack_examples.py", - ], - deps = [ - "//executorch/backends/xnnpack/partition:xnnpack_partitioner", - "//executorch/examples/recipes/xnnpack_optimization:models", - "//executorch/examples/quantization:quant_utils", - "//executorch/examples/export:utils", - "//executorch/exir:lib", - "//executorch/exir/backend:backend_api", - ], - ) - - # executor_runner for XNNPACK Backend and portable kernels. - runtime.cxx_binary( - name = "xnn_executor_runner", - srcs = [], - deps = [ - "//executorch/examples/executor_runner:executor_runner_lib", - "//executorch/backends/xnnpack:xnnpack_backend", - "//executorch/kernels/portable:generated_lib_all_ops", - ], - define_static_target = True, - **get_oss_build_kwargs() - ) diff --git a/examples/custom_ops/custom_ops_1.py b/examples/custom_ops/custom_ops_1.py index b7f6ea76e29..1dd99abaef2 100644 --- a/examples/custom_ops/custom_ops_1.py +++ b/examples/custom_ops/custom_ops_1.py @@ -6,7 +6,7 @@ """Example of showcasing registering custom operator through torch library API.""" import torch -from examples.export.export_example import export_to_exec_prog, save_pte_program +from examples.export.utils import export_to_exec_prog, save_pte_program from executorch.exir import EdgeCompileConfig from torch.library import impl, Library diff --git a/examples/custom_ops/custom_ops_2.py b/examples/custom_ops/custom_ops_2.py index 5869f039a6d..a4fedda2454 100644 --- a/examples/custom_ops/custom_ops_2.py +++ b/examples/custom_ops/custom_ops_2.py @@ -11,7 +11,7 @@ import argparse import torch -from examples.export.export_example import export_to_exec_prog, save_pte_program +from examples.export.utils import export_to_exec_prog, save_pte_program from executorch.exir import EdgeCompileConfig diff --git a/examples/custom_ops/test_custom_ops.sh b/examples/custom_ops/test_custom_ops.sh index 10966aa9b80..65e0a69520d 100644 --- a/examples/custom_ops/test_custom_ops.sh +++ b/examples/custom_ops/test_custom_ops.sh @@ -21,7 +21,7 @@ test_buck2_custom_op_1() { # should save file custom_ops_1.pte echo 'Running executor_runner' - buck2 run //examples/executor_runner:executor_runner \ + buck2 run //examples/runtime/portable:executor_runner \ --config=executorch.register_custom_op=1 -- --model_path="./${model_name}.pte" # should give correct result @@ -58,7 +58,7 @@ test_buck2_custom_op_2() { ${PYTHON_EXECUTABLE} -m "examples.custom_ops.${model_name}" --so_library="$SO_LIB" # should save file custom_ops_2.pte - buck2 run //examples/executor_runner:executor_runner \ + buck2 run //examples/runtime/portable:executor_runner \ --config=executorch.register_custom_op=2 -- --model_path="./${model_name}.pte" # should give correct result echo "Removing ${model_name}.pte" diff --git a/examples/export/export_example.py b/examples/export/portable.py similarity index 99% rename from examples/export/export_example.py rename to examples/export/portable.py index 9c2a9d9362e..e92521bfd6c 100644 --- a/examples/export/export_example.py +++ b/examples/export/portable.py @@ -11,6 +11,7 @@ from ..models import MODEL_NAME_TO_MODEL from ..models.model_factory import EagerModelFactory + from .utils import export_to_exec_prog, save_pte_program diff --git a/examples/ios_demo_apps/README.md b/examples/ios_demo_apps/README.md index 7a8b4416b6e..8deae436800 100644 --- a/examples/ios_demo_apps/README.md +++ b/examples/ios_demo_apps/README.md @@ -20,7 +20,7 @@ and UI looks like ![](./executorch_mobilenet_ui.png) -Step 1. Export a MobileNetV2 model following example follwing executorch/examples/export/export_example.py. Instead of export mv2 directly, add a softmax at the end +Step 1. Export a MobileNetV2 model following example follwing executorch/examples/export/portable.py. Instead of export mv2 directly, add a softmax at the end ```python class MobileNetV2Wrapper(torch.nn.Module): def __init__(self): diff --git a/examples/quantization/quant_flow/TARGETS b/examples/quantization/quant_flow/TARGETS new file mode 100644 index 00000000000..ce01949c8df --- /dev/null +++ b/examples/quantization/quant_flow/TARGETS @@ -0,0 +1,26 @@ +load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") + +runtime.python_binary( + name = "example", + main_src = "example.py", + preload_deps = ["//executorch/kernels/quantized:aot_lib"], + deps = [ + ":quant_utils", + "//caffe2:torch", + "//executorch/examples/export:lib", + "//executorch/examples/recipes/xnnpack:models", + ], +) + +runtime.python_library( + name = "quant_utils", + srcs = [ + "utils.py", + ], + visibility = [ + "//executorch/examples/...", + ], + deps = [ + "//caffe2:torch", + ], +) diff --git a/examples/quantization/example.py b/examples/quantization/quant_flow/example.py similarity index 96% rename from examples/quantization/example.py rename to examples/quantization/quant_flow/example.py index e38b9814d22..d156638e516 100644 --- a/examples/quantization/example.py +++ b/examples/quantization/quant_flow/example.py @@ -28,10 +28,10 @@ XNNPACKQuantizer, ) -from ..export.utils import export_to_edge, save_pte_program -from ..models import MODEL_NAME_TO_MODEL -from ..models.model_factory import EagerModelFactory -from ..recipes.xnnpack_optimization import MODEL_NAME_TO_OPTIONS +from ...export.utils import export_to_edge, save_pte_program +from ...models import MODEL_NAME_TO_MODEL +from ...models.model_factory import EagerModelFactory +from ...recipes.xnnpack import MODEL_NAME_TO_OPTIONS from .utils import quantize @@ -176,8 +176,9 @@ def verify_xnnpack_quantizer_matching_fx_quant_model(model_name, model, example_ model = model.eval() # pre-autograd export. eventually this will become torch.export model = export.capture_pre_autograd_graph(model, example_inputs) + quantizer = XNNPACKQuantizer() start = time.perf_counter() - quantized_model = quantize(model, example_inputs) + quantized_model = quantize(model, example_inputs, quantizer) end = time.perf_counter() logging.info(f"Quantize time: {end - start}s") diff --git a/examples/quantization/test_quantize.sh b/examples/quantization/quant_flow/test_quantize.sh similarity index 85% rename from examples/quantization/test_quantize.sh rename to examples/quantization/quant_flow/test_quantize.sh index 488c2632433..19dda52cf7a 100644 --- a/examples/quantization/test_quantize.sh +++ b/examples/quantization/quant_flow/test_quantize.sh @@ -10,7 +10,7 @@ set -e # shellcheck source=/dev/null -source "$(dirname "${BASH_SOURCE[0]}")/../../.ci/scripts/utils.sh" +source "$(dirname "${BASH_SOURCE[0]}")/../../../.ci/scripts/utils.sh" get_shared_lib_ext() { UNAME=$(uname) @@ -32,10 +32,10 @@ test_buck2_quantization() { SO_LIB=$($BUCK build //kernels/quantized:aot_lib --show-output | grep "buck-out" | cut -d" " -f2) echo "Run example.py" - ${PYTHON_EXECUTABLE} -m "examples.quantization.example" --so_library="$SO_LIB" --model_name="$1" + ${PYTHON_EXECUTABLE} -m "examples.quantization.quant_flow.example" --so_library="$SO_LIB" --model_name="$1" echo 'Running executor_runner' - $BUCK run //examples/executor_runner:executor_runner -- --model_path="./${1}_quantized.pte" + $BUCK run //examples/runtime/portable:executor_runner -- --model_path="./${1}_quantized.pte" # should give correct result echo "Removing ${1}_quantized.pte" @@ -62,7 +62,7 @@ test_cmake_quantization() { SO_LIB="cmake-out/kernels/quantized/libquantized_ops_aot_lib$EXT" echo "Run example.py, shared library $SO_LIB" - ${PYTHON_EXECUTABLE} -m "examples.quantization.example" --so_library="$SO_LIB" --model_name="$1" + ${PYTHON_EXECUTABLE} -m "examples.quantization.quant_flow.example" --so_library="$SO_LIB" --model_name="$1" echo 'Running executor_runner' cmake-out/executor_runner --model_path="./${1}_quantized.pte" diff --git a/examples/quantization/utils.py b/examples/quantization/quant_flow/utils.py similarity index 83% rename from examples/quantization/utils.py rename to examples/quantization/quant_flow/utils.py index 6f8aa3913f3..75c2e634ed6 100644 --- a/examples/quantization/utils.py +++ b/examples/quantization/quant_flow/utils.py @@ -9,14 +9,14 @@ from torch.ao.quantization.quantize_pt2e import convert_pt2e, prepare_pt2e from torch.ao.quantization.quantizer.xnnpack_quantizer import ( get_symmetric_quantization_config, - XNNPACKQuantizer, ) -def quantize(model, example_inputs): - """This is the official recommended flow for quantization in pytorch 2.0 export""" +def quantize(model, example_inputs, quantizer): + """ + This is the official recommended flow for quantization in pytorch 2.0 export + """ logging.info(f"Original model: {model}") - quantizer = XNNPACKQuantizer() # if we set is_per_channel to True, we also need to add out_variant of quantize_per_channel/dequantize_per_channel operator_config = get_symmetric_quantization_config(is_per_channel=False) quantizer.set_global(operator_config) diff --git a/examples/recipes/TARGETS b/examples/recipes/TARGETS new file mode 100644 index 00000000000..5594354f115 --- /dev/null +++ b/examples/recipes/TARGETS @@ -0,0 +1,13 @@ +load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") + +runtime.python_binary( + name = "export_bundled_program", + main_src = "export_bundled_program.py", + deps = [ + "//executorch/bundled_program:config", + "//executorch/bundled_program:core", + "//executorch/bundled_program/serialize:lib", + "//executorch/examples/export:utils", + "//executorch/examples/models:models", + ], +) diff --git a/examples/export/export_and_delegate.py b/examples/recipes/export_and_delegate.py similarity index 99% rename from examples/export/export_and_delegate.py rename to examples/recipes/export_and_delegate.py index 6e4f57becd7..14b762fe066 100644 --- a/examples/export/export_and_delegate.py +++ b/examples/recipes/export_and_delegate.py @@ -17,11 +17,11 @@ ) from executorch.exir.backend.test.op_partitioner_demo import AddMulPartitionerDemo +from ..export.utils import export_to_edge + from ..models import MODEL_NAME_TO_MODEL from ..models.model_factory import EagerModelFactory -from .utils import export_to_edge - FORMAT = "[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s" logging.basicConfig(level=logging.INFO, format=FORMAT) diff --git a/examples/export/export_bundled_program.py b/examples/recipes/export_bundled_program.py similarity index 92% rename from examples/export/export_bundled_program.py rename to examples/recipes/export_bundled_program.py index 45d0f0f0b64..757f9704bb6 100644 --- a/examples/export/export_bundled_program.py +++ b/examples/recipes/export_bundled_program.py @@ -8,16 +8,21 @@ import argparse +import logging + from executorch.bundled_program.config import BundledConfig from executorch.bundled_program.core import create_bundled_program from executorch.bundled_program.serialize import ( serialize_from_bundled_program_to_flatbuffer, ) +from ..export.utils import export_to_exec_prog, save_pte_program + from ..models import MODEL_NAME_TO_MODEL from ..models.model_factory import EagerModelFactory -from .utils import export_to_exec_prog, save_pte_program +FORMAT = "[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s" +logging.basicConfig(level=logging.INFO, format=FORMAT) def save_bundled_program( @@ -62,7 +67,7 @@ def export_to_pte(model_name, model, example_inputs): # create a list with the example_inputs tuple used twice. Each instance of example_inputs # is a Tuple[Union[torch.tenor, int, bool]] which represents one test set for the model. bundled_inputs = [example_inputs, example_inputs] - print(f"Saving exported program to {model_name}_bundled.pte") + logging.info(f"Saving exported program to {model_name}_bundled.pte") save_bundled_program(bundled_inputs, exec_prog, model, f"{model_name}_bundled.pte") diff --git a/examples/backend/README.md b/examples/recipes/xnnpack/README.md similarity index 75% rename from examples/backend/README.md rename to examples/recipes/xnnpack/README.md index 535565dd3e7..a63fb5a2f7e 100644 --- a/examples/backend/README.md +++ b/examples/recipes/xnnpack/README.md @@ -10,26 +10,26 @@ The following command will produce an floating-point XNNPACK delegated model `mv ```bash # For MobileNet V2 -python3 -m examples.backend.xnnpack_examples --model_name="mv2" --delegate +python3 -m examples.recipes.xnnpack.aot_compiler --model_name="mv2" --delegate ``` Once we have the model binary (pte) file, then let's run it with ExecuTorch runtime using the `xnn_executor_runner`. ```bash -buck2 run examples/backend:xnn_executor_runner -- --model_path ./mv2_xnnpack_fp32.pte +buck2 run examples/runtime/xnnpack:xnn_executor_runner -- --model_path ./mv2_xnnpack_fp32.pte ``` ## XNNPACK quantization + delegation The following command will produce an XNNPACK quantized and delegated model `mv2_xnnpack_q8.pte` that can be run using XNNPACK's operators. It will also print out the lowered graph, showing what parts of the models have been lowered to XNNPACK via `executorch_call_delegate`. ```bash -python3 -m examples.backend.xnnpack_examples --model_name="mv2" --quantize --delegate +python3 -m examples.recipes.xnnpack.aot_compiler --model_name="mv2" --quantize --delegate ``` Once we have the model binary (pte) file, then let's run it with ExecuTorch runtime using the `xnn_executor_runner`. ```bash -buck2 run examples/backend:xnn_executor_runner -- --model_path ./mv2_xnnpack_q8.pte +buck2 run examples/runtime/xnnpack:xnn_executor_runner -- --model_path ./mv2_xnnpack_q8.pte ``` ## XNNPACK performance gain @@ -40,14 +40,14 @@ We tested the performance for MobileNet V2 and MobileNet V3 on Linux x86 and Mac For each model, we export three variations: portable (without any optimization), xnnpack fp32 (exported for XNNPACK delegation without quantization), xnnpack q8 (exported for XNNPACK delegation with qint8 delegation). -We build the benchmarking binary (will be released in the near future, but it is similar to `examples/backend:xnn_executor_runner`). Benchmarking binary, by default, runs 10 iterations of warmup and 50 iterations of benchmarking. Number reported here are average measured latency, in ms, across 50 runs. The first iteration is slower due to warm up, and the performance is is stable on subsequent iterations, so we also report the execution time for the first iteration for reference. Below is the model execution time for first iteration and subsequent iterations (average after warmup), in milliseconds. We use a single thread to test the models. Details about the methodology and repro steps are below the tables. +We build the benchmarking binary (will be released in the near future, but it is similar to `examples/runtime/xnnpack:xnn_executor_runner`). Benchmarking binary, by default, runs 10 iterations of warmup and 50 iterations of benchmarking. Number reported here are average measured latency, in ms, across 50 runs. The first iteration is slower due to warm up, and the performance is is stable on subsequent iterations, so we also report the execution time for the first iteration for reference. Below is the model execution time for first iteration and subsequent iterations (average after warmup), in milliseconds. We use a single thread to test the models. Details about the methodology and repro steps are below the tables. ### Methodology -Models are exported with the steps above for XNNPACK delegation, and with `examples/export:export_example` for portable backend without any optimization. Then use `//examples/backend:xnn_executor_runner` with profiler (command listed below); or in the future, use the runtime in `//sdk/runners:executor_runner` since it gives more options such as number of iterations after build rules for OSS is added. +Models are exported with the steps above for XNNPACK delegation, and with `examples/export/portable.py` for portable backend without any optimization. Then use `//examples/runtime/xnnpack:xnn_executor_runner` with profiler (command listed below); or in the future, use the runtime in `//sdk/runners:executor_runner` since it gives more options such as number of iterations after build rules for OSS is added. ``` -buck run -c executorch.prof_enabled=true -c executorch.prof_buf_size=8096 -c executorch.num_prof_blocks=61 //examples/backend:xnn_executor_runner -- --model_path mv3.pte +buck run -c executorch.prof_enabled=true -c executorch.prof_buf_size=8096 -c executorch.num_prof_blocks=61 //examples/runtime/xnnpack:xnn_executor_runner -- --model_path mv3.pte ``` A rough number of execution time can be obtained via the log timestamp. The profiler result can be analyzed with `profiler:profiler_results_cli`. diff --git a/examples/backend/TARGETS b/examples/recipes/xnnpack/TARGETS similarity index 100% rename from examples/backend/TARGETS rename to examples/recipes/xnnpack/TARGETS diff --git a/examples/recipes/xnnpack_optimization/models.py b/examples/recipes/xnnpack/__init__.py similarity index 93% rename from examples/recipes/xnnpack_optimization/models.py rename to examples/recipes/xnnpack/__init__.py index 60bd3568dcc..887e84502be 100644 --- a/examples/recipes/xnnpack_optimization/models.py +++ b/examples/recipes/xnnpack/__init__.py @@ -4,12 +4,13 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. + from dataclasses import dataclass @dataclass class OptimizationOptions(object): - quantization: bool + xnnpack_quantization: bool xnnpack_delegation: bool @@ -29,3 +30,5 @@ class OptimizationOptions(object): "edsr": OptimizationOptions(True, False), "mobilebert": OptimizationOptions(True, False), } + +__all__ = [MODEL_NAME_TO_OPTIONS] diff --git a/examples/backend/xnnpack_examples.py b/examples/recipes/xnnpack/aot_compiler.py similarity index 87% rename from examples/backend/xnnpack_examples.py rename to examples/recipes/xnnpack/aot_compiler.py index 537072dfff0..180069ef7ac 100644 --- a/examples/backend/xnnpack_examples.py +++ b/examples/recipes/xnnpack/aot_compiler.py @@ -15,12 +15,14 @@ from executorch.exir import EdgeCompileConfig from executorch.exir.backend.backend_api import to_backend -from ..export.utils import export_to_edge, save_pte_program +from torch.ao.quantization.quantizer.xnnpack_quantizer import XNNPACKQuantizer -from ..models import MODEL_NAME_TO_MODEL -from ..models.model_factory import EagerModelFactory -from ..quantization.utils import quantize -from ..recipes.xnnpack_optimization import MODEL_NAME_TO_OPTIONS +from ...export.utils import export_to_edge, save_pte_program + +from ...models import MODEL_NAME_TO_MODEL +from ...models.model_factory import EagerModelFactory +from ...quantization.quant_flow.utils import quantize +from . import MODEL_NAME_TO_OPTIONS FORMAT = "[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s" @@ -77,7 +79,8 @@ if args.quantize: logging.info("Quantizing Model...") - model = quantize(model, example_inputs) + quantizer = XNNPACKQuantizer() + model = quantize(model, example_inputs, quantizer) edge = export_to_edge( model, diff --git a/examples/recipes/xnnpack/targets.bzl b/examples/recipes/xnnpack/targets.bzl new file mode 100644 index 00000000000..d06199d75dc --- /dev/null +++ b/examples/recipes/xnnpack/targets.bzl @@ -0,0 +1,36 @@ +load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "get_oss_build_kwargs", "runtime") + +def define_common_targets(): + """Defines targets that should be shared between fbcode and xplat. + + The directory containing this targets.bzl file should also contain both + TARGETS and BUCK files that call this function. + """ + runtime.python_library( + name = "models", + srcs = [ + "__init__.py", + ], + deps = [ + "//executorch/examples/models:models", # @manual + ], + visibility = [ + "//executorch/examples/...", + ], + ) + + runtime.python_binary( + name = "aot_compiler", + srcs = [ + "aot_compiler.py", + ], + main_module = "executorch.examples.recipes.xnnpack.aot_compiler", + deps = [ + "//executorch/backends/xnnpack/partition:xnnpack_partitioner", + "//executorch/examples/recipes/xnnpack:models", + "//executorch/examples/quantization/quant_flow:quant_utils", + "//executorch/examples/export:utils", + "//executorch/exir:lib", + "//executorch/exir/backend:backend_api", + ], + ) diff --git a/examples/recipes/xnnpack_optimization/TARGETS b/examples/recipes/xnnpack_optimization/TARGETS deleted file mode 100644 index 8b25df013bf..00000000000 --- a/examples/recipes/xnnpack_optimization/TARGETS +++ /dev/null @@ -1,12 +0,0 @@ -load("@fbcode_macros//build_defs:python_library.bzl", "python_library") - -python_library( - name = "models", - srcs = [ - "__init__.py", - "models.py", - ], - deps = [ - "//executorch/examples/models:models", # @manual - ], -) diff --git a/examples/recipes/xnnpack_optimization/__init__.py b/examples/recipes/xnnpack_optimization/__init__.py deleted file mode 100644 index 95a9180bd55..00000000000 --- a/examples/recipes/xnnpack_optimization/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. - -from .models import MODEL_NAME_TO_OPTIONS - -__all__ = [MODEL_NAME_TO_OPTIONS] diff --git a/examples/bundled_executor_runner/TARGETS b/examples/runtime/bundled/TARGETS similarity index 100% rename from examples/bundled_executor_runner/TARGETS rename to examples/runtime/bundled/TARGETS diff --git a/examples/bundled_executor_runner/bundled_executor_runner.cpp b/examples/runtime/bundled/bundled_executor_runner.cpp similarity index 100% rename from examples/bundled_executor_runner/bundled_executor_runner.cpp rename to examples/runtime/bundled/bundled_executor_runner.cpp diff --git a/examples/bundled_executor_runner/targets.bzl b/examples/runtime/bundled/targets.bzl similarity index 100% rename from examples/bundled_executor_runner/targets.bzl rename to examples/runtime/bundled/targets.bzl diff --git a/examples/executor_runner/TARGETS b/examples/runtime/portable/TARGETS similarity index 100% rename from examples/executor_runner/TARGETS rename to examples/runtime/portable/TARGETS diff --git a/examples/executor_runner/executor_runner.cpp b/examples/runtime/portable/executor_runner.cpp similarity index 100% rename from examples/executor_runner/executor_runner.cpp rename to examples/runtime/portable/executor_runner.cpp diff --git a/examples/executor_runner/targets.bzl b/examples/runtime/portable/targets.bzl similarity index 100% rename from examples/executor_runner/targets.bzl rename to examples/runtime/portable/targets.bzl diff --git a/examples/runtime/xnnpack/TARGETS b/examples/runtime/xnnpack/TARGETS new file mode 100644 index 00000000000..2341af9282f --- /dev/null +++ b/examples/runtime/xnnpack/TARGETS @@ -0,0 +1,8 @@ +# Any targets that should be shared between fbcode and xplat must be defined in +# targets.bzl. This file can contain fbcode-only targets. + +load(":targets.bzl", "define_common_targets") + +oncall("executorch") + +define_common_targets() diff --git a/examples/runtime/xnnpack/targets.bzl b/examples/runtime/xnnpack/targets.bzl new file mode 100644 index 00000000000..a00ecd82575 --- /dev/null +++ b/examples/runtime/xnnpack/targets.bzl @@ -0,0 +1,21 @@ +load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "get_oss_build_kwargs", "runtime") + +def define_common_targets(): + """Defines targets that should be shared between fbcode and xplat. + + The directory containing this targets.bzl file should also contain both + TARGETS and BUCK files that call this function. + """ + + # executor_runner for XNNPACK Backend and portable kernels. + runtime.cxx_binary( + name = "xnn_executor_runner", + srcs = [], + deps = [ + "//executorch/examples/runtime/portable:executor_runner_lib", + "//executorch/backends/xnnpack:xnnpack_backend", + "//executorch/kernels/portable:generated_lib_all_ops", + ], + define_static_target = True, + **get_oss_build_kwargs() + ) diff --git a/examples/selective_build/targets.bzl b/examples/selective_build/targets.bzl index 73d3ff81795..372a0b0780f 100644 --- a/examples/selective_build/targets.bzl +++ b/examples/selective_build/targets.bzl @@ -73,7 +73,7 @@ def define_common_targets(): name = "selective_build_test", srcs = [], deps = [ - "//executorch/examples/executor_runner:executor_runner_lib", + "//executorch/examples/runtime/portable:executor_runner_lib", ] + lib, define_static_target = True, **get_oss_build_kwargs() diff --git a/examples/selective_build/test_selective_build.sh b/examples/selective_build/test_selective_build.sh index 4a818dc9872..ffe2f26d808 100644 --- a/examples/selective_build/test_selective_build.sh +++ b/examples/selective_build/test_selective_build.sh @@ -17,7 +17,7 @@ source "$(dirname "${BASH_SOURCE[0]}")/../../.ci/scripts/utils.sh" test_buck2_select_all_ops() { echo "Exporting MobilenetV3" - ${PYTHON_EXECUTABLE} -m examples.export.export_example --model_name="mv3" + ${PYTHON_EXECUTABLE} -m examples.export.portable --model_name="mv3" echo "Running selective build test" $BUCK run //examples/selective_build:selective_build_test \ @@ -29,7 +29,7 @@ test_buck2_select_all_ops() { test_buck2_select_ops_in_list() { echo "Exporting add_mul" - ${PYTHON_EXECUTABLE} -m examples.export.export_example --model_name="add_mul" + ${PYTHON_EXECUTABLE} -m examples.export.portable --model_name="add_mul" echo "Running selective build test" # set max_kernel_num=17: 14 primops, add, mul @@ -55,7 +55,7 @@ test_buck2_select_ops_from_yaml() { test_cmake_select_all_ops() { echo "Exporting MobilenetV3" - ${PYTHON_EXECUTABLE} -m examples.export.export_example --model_name="mv3" + ${PYTHON_EXECUTABLE} -m examples.export.portable --model_name="mv3" (rm -rf cmake-out \ && mkdir cmake-out \ @@ -78,7 +78,7 @@ test_cmake_select_all_ops() { test_cmake_select_ops_in_list() { echo "Exporting MobilenetV2" - ${PYTHON_EXECUTABLE} -m examples.export.export_example --model_name="mv2" + ${PYTHON_EXECUTABLE} -m examples.export.portable --model_name="mv2" # set MAX_KERNEL_NUM=17: 14 primops, add, mul (rm -rf cmake-out \