From 89c950cc3137109b662e8a822af3ac5b90091a43 Mon Sep 17 00:00:00 2001 From: Jens Scheffler Date: Sat, 20 May 2023 17:32:34 +0200 Subject: [PATCH 1/3] Add multi-select, proposals and labels for trigger form --- .../example_params_ui_tutorial.py | 58 ++++++++++++++++++- airflow/www/static/js/trigger.js | 34 +++++++++-- airflow/www/templates/airflow/trigger.html | 33 ++++++++++- docs/apache-airflow/core-concepts/params.rst | 10 +++- 4 files changed, 125 insertions(+), 10 deletions(-) diff --git a/airflow/example_dags/example_params_ui_tutorial.py b/airflow/example_dags/example_params_ui_tutorial.py index 485817f137080..93608cd21ae71 100644 --- a/airflow/example_dags/example_params_ui_tutorial.py +++ b/airflow/example_dags/example_params_ui_tutorial.py @@ -55,7 +55,7 @@ "most_loved_number": Param( 42, type="integer", - title="You favorite number", + title="Your favorite number", description_html="""Everybody should have a favorite number. Not only math teachers. If you can not think of any at the moment please think of the 42 which is very famous because of the book @@ -71,6 +71,62 @@ description="You can use JSON schema enum's to generate drop down selection boxes.", enum=[f"value {i}" for i in range(1, 42)], ), + # You can also label the selected values via values_display attribute + "pick_with_label": Param( + 3, + type="number", + title="Select one Number", + description="With drop down selections you can also have nice display labels for the values.", + enum=[*range(1, 10)], + values_display={ + 1: "One", + 2: "Two", + 3: "Three", + 4: "Four - is like you take three and get one for free!", + 5: "Five", + 6: "Six", + 7: "Seven", + 8: "Eight", + 9: "Nine", + }, + ), + # If you want to have a list box with proposals but not enforcing a fixed list + # then you can use the examples feature of JSON schema + "proposals": Param( + "some value", + type="string", + title="Field with proposals", + description="You can use JSON schema examples's to generate drop down selection boxes " + "but allow also to enter custom values. Try typing an 'a' and see options.", + examples=( + "Alpha,Bravo,Charlie,Delta,Echo,Foxtrot,Golf,Hotel,India,Juliett,Kilo,Lima,Mike,November,Oscar,Papa," + "Quebec,Romeo,Sierra,Tango,Uniform,Victor,Whiskey,X-ray,Yankee,Zulu" + ).split(","), + ), + # If you want to select multiple items from a fixed list JSON schema des not allow to use enum + # In this case the type "array" is being used together with "examples" as pick list + "multi_select": Param( + ["two", "three"], + "Select from the list of options.", + type="array", + title="Multi Select", + examples=["one", "two", "three", "four", "five"], + ), + # A multiple options selection can also be combined with values_display + "multi_select_with_label": Param( + ["2", "3"], + "Select from the list of options. See that options can have nicer text and still technical values" + "are propagated as values during trigger to the DAG.", + type="array", + title="Multi Select with Labels", + examples=["1", "2", "3", "4", "5"], + values_display={ + "1": "One box of choccolate", + "2": "Two bananas", + "3": "Three apples", + # Note: Value display mapping does not need to be complete.s + }, + ), # Boolean as proper parameter with description "bool": Param( True, diff --git a/airflow/www/static/js/trigger.js b/airflow/www/static/js/trigger.js index 27fcc099726cf..3c12efe255cbd 100644 --- a/airflow/www/static/js/trigger.js +++ b/airflow/www/static/js/trigger.js @@ -17,7 +17,7 @@ * under the License. */ -/* global document, CodeMirror, window */ +/* global document, CodeMirror, window, $ */ let jsonForm; const objectFields = new Map(); @@ -46,7 +46,19 @@ function updateJSONconf() { values[values.length] = lines[j].trim(); } } - params[keyName] = values.length === 0 ? params[keyName] : values; + params[keyName] = values.length === 0 ? null : values; + } else if ( + elements[i].attributes.valuetype && + elements[i].attributes.valuetype.value === "multiselect" + ) { + const { options } = elements[i]; + const values = []; + for (let j = 0; j < options.length; j += 1) { + if (options[j].selected) { + values[values.length] = options[j].value; + } + } + params[keyName] = values.length === 0 ? null : values; } else if (elements[i].value.length === 0) { params[keyName] = null; } else if ( @@ -104,7 +116,7 @@ function initForm() { jsonForm.setSize(null, height); if (formHasFields) { - // Apply JSON formatting and linting to all object fields in the form + // Initialize jQuery and Chakra fields const elements = document.getElementById("trigger_form"); for (let i = 0; i < elements.length; i += 1) { if (elements[i].name && elements[i].name.startsWith("element_")) { @@ -112,6 +124,7 @@ function initForm() { elements[i].attributes.valuetype && elements[i].attributes.valuetype.value === "object" ) { + // Apply JSON formatting and linting to all object fields in the form const field = CodeMirror.fromTextArea(elements[i], { lineNumbers: true, mode: { name: "javascript", json: true }, @@ -120,6 +133,17 @@ function initForm() { }); field.on("blur", updateJSONconf); objectFields.set(elements[i].name, field); + } else if ( + elements[i].attributes.valuetype && + elements[i].attributes.valuetype.value === "multiselect" + ) { + // Activate select2 multi select boxes + const elementId = `#${elements[i].name}`; + $(elementId).select2({ + placeholder: "Select Values", + allowClear: true, + }); + elements[i].addEventListener("blur", updateJSONconf); } else if (elements[i].type === "checkbox") { elements[i].addEventListener("change", updateJSONconf); } else { @@ -174,7 +198,9 @@ function initForm() { setTimeout(updateJSONconf, 100); } } -initForm(); +$(document).ready(() => { + initForm(); +}); window.updateJSONconf = updateJSONconf; diff --git a/airflow/www/templates/airflow/trigger.html b/airflow/www/templates/airflow/trigger.html index f98e31b0c9b46..d5c7df2518d2d 100644 --- a/airflow/www/templates/airflow/trigger.html +++ b/airflow/www/templates/airflow/trigger.html @@ -72,12 +72,29 @@ {% elif "enum" in form_details.schema and form_details.schema.enum %} {% elif form_details.schema and "array" in form_details.schema.type %} + {% if "examples" in form_details.schema and form_details.schema.examples %} + + {% else %} + {% endif %} {% elif form_details.schema and "object" in form_details.schema.type %} {% elif form_details.schema and ("integer" in form_details.schema.type or "number" in form_details.schema.type) %} - {% else %} - + {% if "examples" in form_details.schema and form_details.schema.examples %} + + {% for option in form_details.schema.examples -%} + + {% endfor -%} + + {% endif %} {% endif %} {% if form_details.description -%} {{ form_details.description }} diff --git a/docs/apache-airflow/core-concepts/params.rst b/docs/apache-airflow/core-concepts/params.rst index eacaef3c5d2fd..7619d56171d46 100644 --- a/docs/apache-airflow/core-concepts/params.rst +++ b/docs/apache-airflow/core-concepts/params.rst @@ -189,13 +189,19 @@ The following features are supported in the Trigger UI Form: You can add the parameters ``minimum`` and ``maximum`` to restrict number range accepted. - ``boolean``: Generates a toggle button to be used as ``True`` or ``False``. - ``date``, ``datetime`` and ``time``: Generate date and/or time picker - - ``array``: Generates a HTML multi line text field, every line edited will be made into a string array as value + - ``array``: Generates a HTML multi line text field, every line edited will be made into a string array as value. + if you add the attribute ``example`` with a list, a multi-value select option will be generated. - ``object``: Generates a JSON entry field - Note: Per default if you specify a type, a field will be made required with input - because of JSON validation. If you want to have a field value being added optional only, you must allow JSON schema validation allowing null values via: ``type=["null", "string"]`` -- The Param attribute ``enum`` generates a drop-down select list. As of JSON validation, a value must be selected. +- The Param attribute ``enum`` generates a drop-down select list for scalar values. As of JSON validation, a value must be selected or + the field must be marked as optional explicit. +- If you want to present proposals for scalar values (not restricting the user to a fixed ``enum`` as above) you can make use of + ``examples`` which is a list of items. +- For select drop-downs generated via ``enum`` or multi-value selects you can add the attribute ``values_display`` with a dict and + map data values to display labels. - If a form field is left empty, it is passed as ``None`` value to the params dict. - Form fields are rendered in the order of definition. - If you want to add sections to the Form, add the parameter ``section`` to each field. The text will be used as section label. From 734ee013bac23bf133318c2c2d1b569917da33bf Mon Sep 17 00:00:00 2001 From: Jens Scheffler Date: Sun, 21 May 2023 20:49:32 +0200 Subject: [PATCH 2/3] Merge remote-tracking branch 'origin/main' into feature/31440-multi-select-and-labels-for-trigger-forms --- INTHEWILD.md | 3 +- .../example_params_ui_tutorial.py | 4 +-- airflow/www/templates/airflow/trigger.html | 28 +++++++++++-------- .../run_provider_yaml_files_check.py | 17 +++++++++++ scripts/in_container/verify_providers.py | 15 ++++++++++ tests/always/test_providers_manager.py | 5 ++++ 6 files changed, 58 insertions(+), 14 deletions(-) diff --git a/INTHEWILD.md b/INTHEWILD.md index 2e3ccd3891e18..da58858618d4a 100644 --- a/INTHEWILD.md +++ b/INTHEWILD.md @@ -102,7 +102,8 @@ Currently, **officially** using Airflow: 1. [Braintree](https://www.braintreepayments.com) [[@coopergillan](https://github.com/coopergillan), [@curiousjazz77](https://github.com/curiousjazz77), [@raymondberg](https://github.com/raymondberg)] 1. [Branch](https://branch.io) [[@sdebarshi](https://github.com/sdebarshi), [@dmitrig01](https://github.com/dmitrig01)] 1. [Breezeline (formerly Atlantic Broadband)](https://www.breezeline.com/) [[@IanDoarn](https://github.com/IanDoarn), [@willsims14](https://github.com/willsims14)] -1. [Bwtech](https://www.bwtech.com/) [[@wolvery](https://github.com/wolvery) +1. [BWGI](https://www.bwgi.com.br/) [[@jgmarcel](https://github.com/jgmarcel)] +1. [Bwtech](https://www.bwtech.com/) [[@wolvery](https://github.com/wolvery)] 1. [C2FO](https://www.c2fo.com/) 1. [Caesars Entertainment](https://www.caesars.com) 1. [Cafe Bazaar](https://cafebazaar.ir/about?l=en) [[@cafebazaar](https://github.com/cafebazaar)] diff --git a/airflow/example_dags/example_params_ui_tutorial.py b/airflow/example_dags/example_params_ui_tutorial.py index 93608cd21ae71..c07f9fc7c2bd9 100644 --- a/airflow/example_dags/example_params_ui_tutorial.py +++ b/airflow/example_dags/example_params_ui_tutorial.py @@ -65,11 +65,11 @@ ), # If you want to have a selection list box then you can use the enum feature of JSON schema "pick_one": Param( - "value 1", + "value 42", type="string", title="Select one Value", description="You can use JSON schema enum's to generate drop down selection boxes.", - enum=[f"value {i}" for i in range(1, 42)], + enum=[f"value {i}" for i in range(16, 64)], ), # You can also label the selected values via values_display attribute "pick_with_label": Param( diff --git a/airflow/www/templates/airflow/trigger.html b/airflow/www/templates/airflow/trigger.html index d5c7df2518d2d..56ad656883660 100644 --- a/airflow/www/templates/airflow/trigger.html +++ b/airflow/www/templates/airflow/trigger.html @@ -71,15 +71,18 @@ {% elif "enum" in form_details.schema and form_details.schema.enum %} {% elif form_details.schema and "array" in form_details.schema.type %} @@ -87,11 +90,14 @@ {% else %} diff --git a/scripts/in_container/run_provider_yaml_files_check.py b/scripts/in_container/run_provider_yaml_files_check.py index 40bf38e50f1e5..1f1dd675ace3d 100755 --- a/scripts/in_container/run_provider_yaml_files_check.py +++ b/scripts/in_container/run_provider_yaml_files_check.py @@ -306,6 +306,22 @@ def check_hook_classes(yaml_files: dict[str, dict]): ) +def check_trigger_classes(yaml_files: dict[str, dict]): + print("Checking triggers classes belong to package, exist and are classes") + resource_type = "triggers" + for yaml_file_path, provider_data in yaml_files.items(): + provider_package = pathlib.Path(yaml_file_path).parent.as_posix().replace("/", ".") + trigger_classes = { + name + for trigger_class in provider_data.get(resource_type, {}) + for name in trigger_class["class-names"] + } + if trigger_classes: + check_if_objects_exist_and_belong_to_package( + trigger_classes, provider_package, yaml_file_path, resource_type, ObjectType.CLASS + ) + + def check_plugin_classes(yaml_files: dict[str, dict]): print("Checking plugin classes belong to package, exist and are classes") resource_type = "plugins" @@ -509,6 +525,7 @@ def check_providers_have_all_documentation_files(yaml_files: dict[str, dict]): check_hook_classes(all_parsed_yaml_files) check_plugin_classes(all_parsed_yaml_files) check_extra_link_classes(all_parsed_yaml_files) + check_trigger_classes(all_parsed_yaml_files) check_correctness_of_list_of_sensors_operators_hook_modules(all_parsed_yaml_files) check_unique_provider_name(all_parsed_yaml_files) check_providers_have_all_documentation_files(all_parsed_yaml_files) diff --git a/scripts/in_container/verify_providers.py b/scripts/in_container/verify_providers.py index 4da365b9165c7..741850f1737aa 100755 --- a/scripts/in_container/verify_providers.py +++ b/scripts/in_container/verify_providers.py @@ -52,6 +52,7 @@ class EntityType(Enum): Sensors = "Sensors" Hooks = "Hooks" Secrets = "Secrets" + Trigger = "Trigger" class EntityTypeSummary(NamedTuple): @@ -82,6 +83,7 @@ class ProviderPackageDetails(NamedTuple): EntityType.Sensors: "Sensors", EntityType.Hooks: "Hooks", EntityType.Secrets: "Secrets", + EntityType.Trigger: "Trigger", } TOTALS: dict[EntityType, int] = { @@ -90,6 +92,7 @@ class ProviderPackageDetails(NamedTuple): EntityType.Sensors: 0, EntityType.Transfers: 0, EntityType.Secrets: 0, + EntityType.Trigger: 0, } OPERATORS_PATTERN = r".*Operator$" @@ -98,6 +101,7 @@ class ProviderPackageDetails(NamedTuple): SECRETS_PATTERN = r".*Backend$" TRANSFERS_PATTERN = r".*To[A-Z0-9].*Operator$" WRONG_TRANSFERS_PATTERN = r".*Transfer$|.*TransferOperator$" +TRIGGER_PATTERN = r".*Trigger$" ALL_PATTERNS = { OPERATORS_PATTERN, @@ -106,6 +110,7 @@ class ProviderPackageDetails(NamedTuple): SECRETS_PATTERN, TRANSFERS_PATTERN, WRONG_TRANSFERS_PATTERN, + TRIGGER_PATTERN, } EXPECTED_SUFFIXES: dict[EntityType, str] = { @@ -114,6 +119,7 @@ class ProviderPackageDetails(NamedTuple): EntityType.Sensors: "Sensor", EntityType.Secrets: "Backend", EntityType.Transfers: "Operator", + EntityType.Trigger: "Trigger", } @@ -549,6 +555,7 @@ def get_package_class_summary( from airflow.models.baseoperator import BaseOperator from airflow.secrets import BaseSecretsBackend from airflow.sensors.base import BaseSensorOperator + from airflow.triggers.base import BaseTrigger all_verified_entities: dict[EntityType, VerifiedEntities] = { EntityType.Operators: find_all_entities( @@ -601,6 +608,14 @@ def get_package_class_summary( expected_class_name_pattern=TRANSFERS_PATTERN, unexpected_class_name_patterns=ALL_PATTERNS - {OPERATORS_PATTERN, TRANSFERS_PATTERN}, ), + EntityType.Trigger: find_all_entities( + imported_classes=imported_classes, + base_package=full_package_name, + sub_package_pattern_match=r".*\.triggers\..*", + ancestor_match=BaseTrigger, + expected_class_name_pattern=TRIGGER_PATTERN, + unexpected_class_name_patterns=ALL_PATTERNS - {TRIGGER_PATTERN}, + ), } for entity in EntityType: print_wrong_naming(entity, all_verified_entities[entity].wrong_entities) diff --git a/tests/always/test_providers_manager.py b/tests/always/test_providers_manager.py index b18eb220715cf..a4f8acf0d86fa 100644 --- a/tests/always/test_providers_manager.py +++ b/tests/always/test_providers_manager.py @@ -341,6 +341,11 @@ def test_auth_backends(self): auth_backend_module_names = list(provider_manager.auth_backend_module_names) assert len(auth_backend_module_names) > 0 + def test_trigger(self): + provider_manager = ProvidersManager() + trigger_class_names = list(provider_manager.trigger) + assert len(trigger_class_names) > 10 + @patch("airflow.providers_manager.import_string") def test_optional_feature_no_warning(self, mock_importlib_import_string): with self._caplog.at_level(logging.WARNING): From a20606d8d1fa593b80cc56cabf45243611b9338d Mon Sep 17 00:00:00 2001 From: Jens Scheffler Date: Sun, 21 May 2023 21:45:41 +0200 Subject: [PATCH 3/3] Fix re-population of data for dropdowns when recent config applied --- airflow/www/static/js/trigger.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/airflow/www/static/js/trigger.js b/airflow/www/static/js/trigger.js index 3c12efe255cbd..6ada3d615eaed 100644 --- a/airflow/www/static/js/trigger.js +++ b/airflow/www/static/js/trigger.js @@ -133,10 +133,7 @@ function initForm() { }); field.on("blur", updateJSONconf); objectFields.set(elements[i].name, field); - } else if ( - elements[i].attributes.valuetype && - elements[i].attributes.valuetype.value === "multiselect" - ) { + } else if (elements[i].nodeName === "SELECT") { // Activate select2 multi select boxes const elementId = `#${elements[i].name}`; $(elementId).select2({ @@ -238,6 +235,8 @@ function setRecentConfig(e) { objectFields .get(`element_${keys[i]}`) .setValue(JSON.stringify(newValue, null, 4)); + } else if (element.nodeName === "SELECT") { + $(`#${element.name}`).select2("val", [newValue]); } else { element.value = newValue; }