From feb4b831bd5c91aa2cd1b1483580af845702380e Mon Sep 17 00:00:00 2001 From: Vinay Date: Sat, 27 Jun 2020 19:07:31 +0530 Subject: [PATCH 1/9] YAML file supports extra json parameters --- airflow/secrets/local_filesystem.py | 5 ++ .../howto/use-alternative-secrets-backend.rst | 6 ++- tests/secrets/test_local_filesystem.py | 48 ++++++++++++++++++- 3 files changed, 56 insertions(+), 3 deletions(-) diff --git a/airflow/secrets/local_filesystem.py b/airflow/secrets/local_filesystem.py index 90f565c86b373..755e9f8df9473 100644 --- a/airflow/secrets/local_filesystem.py +++ b/airflow/secrets/local_filesystem.py @@ -101,6 +101,11 @@ def _parse_yaml_file(file_path: str) -> Tuple[Dict[str, List[str]], List[FileSyn return {}, [FileSyntaxError(line_no=1, message="The file is empty.")] try: secrets = yaml.safe_load(content) + for key in list(secrets.keys()): + secret_values = secrets[key] + if(isinstance(secret_values, dict) and 'extra' in secret_values.keys()): + secrets[key]['extra'] = json.dumps(secrets[key]['extra']) + except yaml.MarkedYAMLError as e: return {}, [FileSyntaxError(line_no=e.problem_mark.line, message=str(e))] if not isinstance(secrets, dict): diff --git a/docs/howto/use-alternative-secrets-backend.rst b/docs/howto/use-alternative-secrets-backend.rst index 10ca35e1b02a2..8297b00f634ce 100644 --- a/docs/howto/use-alternative-secrets-backend.rst +++ b/docs/howto/use-alternative-secrets-backend.rst @@ -117,7 +117,7 @@ The following is a sample JSON file. } The YAML file structure is similar to that of a JSON. The key-value pair of connection ID and the definitions of one or more connections. -The connection can be defined as a URI (string) or JSON object. +The connection can be defined as a URI (string) or JSON object. Any extra json parameters can be provided with the key extra. For a guide about defining a connection as a URI, see:: :ref:`generating_connection_uri`. For a description of the connection object parameters see :class:`~airflow.models.connection.Connection`. The following is a sample YAML file. @@ -137,6 +137,10 @@ The following is a sample YAML file. login: Login password: None port: 1234 + extra: + a: b + nestedblock_dict: + x: y You can also define connections using a ``.env`` file. Then the key is the connection ID, and the value should describe the connection using the URI. If the connection ID is repeated, all values will diff --git a/tests/secrets/test_local_filesystem.py b/tests/secrets/test_local_filesystem.py index 4dbbf3bb5a3dd..d6ec317ff640f 100644 --- a/tests/secrets/test_local_filesystem.py +++ b/tests/secrets/test_local_filesystem.py @@ -226,9 +226,15 @@ def test_missing_file(self, mock_exists): schema: lschema login: Login password: None - port: 1234""", + port: 1234 + extra: + extra__google_cloud_platform__keyfile_dict: + a: b + extra__google_cloud_platform__keyfile_path: asaa""", {"conn_a": ["mysql://hosta"], "conn_b": ["mysql://hostb", "mysql://hostc"], - "conn_c": ["scheme://Login:None@host:1234/lschema"]}), + "conn_c": [''.join("""scheme://Login:None@host:1234/lschema? + extra__google_cloud_platform__keyfile_dict=%7B%27a%27%3A+%27b%27%7D + &extra__google_cloud_platform__keyfile_path=asaa""".split())]}), ) ) def test_yaml_file_should_load_connection(self, file_content, expected_connection_uris): @@ -241,6 +247,44 @@ def test_yaml_file_should_load_connection(self, file_content, expected_connectio self.assertEqual(expected_connection_uris, connection_uris_by_conn_id) + @parameterized.expand( + ( + ("""conn_c: + conn_type: scheme + host: host + schema: lschema + login: Login + password: None + port: 1234 + extra: + aws_conn_id: bbb + region_name: ccc + """, {"conn_c": [{"aws_conn_id": "bbb", "region_name": "ccc"}]}), + ("""conn_d: + conn_type: scheme + host: host + schema: lschema + login: Login + password: None + port: 1234 + extra: + extra__google_cloud_platform__keyfile_dict: + a: b + extra__google_cloud_platform__key_path: xxx + """, {"conn_d": [{"extra__google_cloud_platform__keyfile_dict": {"a": "b"}, + "extra__google_cloud_platform__key_path": "xxx"}]}), + + ) + ) + def test_yaml_file_should_load_connection_extras(self, file_content, expected_extras): + with mock_local_file(file_content): + connections_by_conn_id = local_filesystem.load_connections("a.yaml") + connection_uris_by_conn_id = { + conn_id: [connection.extra_dejson for connection in connections] + for conn_id, connections in connections_by_conn_id.items() + } + self.assertEqual(expected_extras, connection_uris_by_conn_id) + class TestLocalFileBackend(unittest.TestCase): def test_should_read_variable(self): From afdbb6c8dfead3cfc911b4292f549389ea6ea634 Mon Sep 17 00:00:00 2001 From: Vinay Date: Tue, 30 Jun 2020 14:43:55 +0530 Subject: [PATCH 2/9] Adding extra_dejson key for json params --- airflow/secrets/local_filesystem.py | 14 ++++++---- .../howto/use-alternative-secrets-backend.rst | 4 +-- tests/secrets/test_local_filesystem.py | 28 +++++++++++++++++-- 3 files changed, 36 insertions(+), 10 deletions(-) diff --git a/airflow/secrets/local_filesystem.py b/airflow/secrets/local_filesystem.py index 755e9f8df9473..0c246afc4054e 100644 --- a/airflow/secrets/local_filesystem.py +++ b/airflow/secrets/local_filesystem.py @@ -101,10 +101,6 @@ def _parse_yaml_file(file_path: str) -> Tuple[Dict[str, List[str]], List[FileSyn return {}, [FileSyntaxError(line_no=1, message="The file is empty.")] try: secrets = yaml.safe_load(content) - for key in list(secrets.keys()): - secret_values = secrets[key] - if(isinstance(secret_values, dict) and 'extra' in secret_values.keys()): - secrets[key]['extra'] = json.dumps(secrets[key]['extra']) except yaml.MarkedYAMLError as e: return {}, [FileSyntaxError(line_no=e.problem_mark.line, message=str(e))] @@ -185,7 +181,7 @@ def _create_connection(conn_id: str, value: Any): if isinstance(value, str): return Connection(conn_id=conn_id, uri=value) if isinstance(value, dict): - connection_parameter_names = get_connection_parameter_names() + connection_parameter_names = get_connection_parameter_names() | {"extra_dejson"} current_keys = set(value.keys()) if not current_keys.issubset(connection_parameter_names): illegal_keys = current_keys - connection_parameter_names @@ -194,6 +190,14 @@ def _create_connection(conn_id: str, value: Any): f"The object have illegal keys: {illegal_keys_list}. " f"The dictionary can only contain the following keys: {connection_parameter_names}" ) + if "extra" in value and "extra_dejson" in value: + raise AirflowException( + "The extra and extra_dejson parameters are mutually exclusive. " + "Please provide only one parameter." + ) + if "extra_dejson" in value: + value["extra"] = json.dumps(value["extra_dejson"]) + del value["extra_dejson"] if "conn_id" in current_keys and conn_id != value["conn_id"]: raise AirflowException( diff --git a/docs/howto/use-alternative-secrets-backend.rst b/docs/howto/use-alternative-secrets-backend.rst index 8297b00f634ce..b693249a4150e 100644 --- a/docs/howto/use-alternative-secrets-backend.rst +++ b/docs/howto/use-alternative-secrets-backend.rst @@ -117,7 +117,7 @@ The following is a sample JSON file. } The YAML file structure is similar to that of a JSON. The key-value pair of connection ID and the definitions of one or more connections. -The connection can be defined as a URI (string) or JSON object. Any extra json parameters can be provided with the key extra. +The connection can be defined as a URI (string) or JSON object. Any extra json parameters can be provided with the key extra_dejson (Keys extra and extra_dejson are mutually exclusive). For a guide about defining a connection as a URI, see:: :ref:`generating_connection_uri`. For a description of the connection object parameters see :class:`~airflow.models.connection.Connection`. The following is a sample YAML file. @@ -137,7 +137,7 @@ The following is a sample YAML file. login: Login password: None port: 1234 - extra: + extra_dejson: a: b nestedblock_dict: x: y diff --git a/tests/secrets/test_local_filesystem.py b/tests/secrets/test_local_filesystem.py index d6ec317ff640f..44167db3fda3f 100644 --- a/tests/secrets/test_local_filesystem.py +++ b/tests/secrets/test_local_filesystem.py @@ -227,7 +227,7 @@ def test_missing_file(self, mock_exists): login: Login password: None port: 1234 - extra: + extra_dejson: extra__google_cloud_platform__keyfile_dict: a: b extra__google_cloud_platform__keyfile_path: asaa""", @@ -256,7 +256,7 @@ def test_yaml_file_should_load_connection(self, file_content, expected_connectio login: Login password: None port: 1234 - extra: + extra_dejson: aws_conn_id: bbb region_name: ccc """, {"conn_c": [{"aws_conn_id": "bbb", "region_name": "ccc"}]}), @@ -267,7 +267,7 @@ def test_yaml_file_should_load_connection(self, file_content, expected_connectio login: Login password: None port: 1234 - extra: + extra_dejson: extra__google_cloud_platform__keyfile_dict: a: b extra__google_cloud_platform__key_path: xxx @@ -285,6 +285,28 @@ def test_yaml_file_should_load_connection_extras(self, file_content, expected_ex } self.assertEqual(expected_extras, connection_uris_by_conn_id) + @parameterized.expand( + ( + ("""conn_c: + conn_type: scheme + host: host + schema: lschema + login: Login + password: None + port: 1234 + extra: + abc: xyz + extra_dejson: + aws_conn_id: bbb + region_name: ccc + """, "The extra and extra_dejson parameters are mutually exclusive."), + ) + ) + def test_yaml_invalid_extra(self, file_content, expected_message): + with mock_local_file(file_content): + with self.assertRaisesRegex(AirflowException, re.escape(expected_message)): + local_filesystem.load_connections("a.yaml") + class TestLocalFileBackend(unittest.TestCase): def test_should_read_variable(self): From 9a9547cedb4a78a9d9364aece1815f0a50483319 Mon Sep 17 00:00:00 2001 From: Vinay Date: Thu, 2 Jul 2020 02:19:57 +0530 Subject: [PATCH 3/9] Updating tests and docs --- docs/howto/use-alternative-secrets-backend.rst | 6 +++++- tests/secrets/test_local_filesystem.py | 9 +++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/howto/use-alternative-secrets-backend.rst b/docs/howto/use-alternative-secrets-backend.rst index b693249a4150e..427400e457539 100644 --- a/docs/howto/use-alternative-secrets-backend.rst +++ b/docs/howto/use-alternative-secrets-backend.rst @@ -94,6 +94,7 @@ The file can be defined in ``JSON``, ``YAML`` or ``env`` format. The JSON file must contain an object where the key contains the connection ID and the value contains the definitions of one or more connections. The connection can be defined as a URI (string) or JSON object. + For a guide about defining a connection as a URI, see:: :ref:`generating_connection_uri`. For a description of the connection object parameters see :class:`~airflow.models.connection.Connection`. The following is a sample JSON file. @@ -117,7 +118,10 @@ The following is a sample JSON file. } The YAML file structure is similar to that of a JSON. The key-value pair of connection ID and the definitions of one or more connections. -The connection can be defined as a URI (string) or JSON object. Any extra json parameters can be provided with the key extra_dejson (Keys extra and extra_dejson are mutually exclusive). +The connection can be defined as a URI (string) or JSON object. Any extra json parameters can be provided using keys like ``extra_dejson`` and ``extra``. +The key ``extra_dejson`` can be used to provide parameters as JSON object where as the key ``extra`` can be used in case of a JSON string. +The keys ``extra`` and ``extra_dejson`` are mutually exclusive. + For a guide about defining a connection as a URI, see:: :ref:`generating_connection_uri`. For a description of the connection object parameters see :class:`~airflow.models.connection.Connection`. The following is a sample YAML file. diff --git a/tests/secrets/test_local_filesystem.py b/tests/secrets/test_local_filesystem.py index 44167db3fda3f..6f58850f9e089 100644 --- a/tests/secrets/test_local_filesystem.py +++ b/tests/secrets/test_local_filesystem.py @@ -273,6 +273,15 @@ def test_yaml_file_should_load_connection(self, file_content, expected_connectio extra__google_cloud_platform__key_path: xxx """, {"conn_d": [{"extra__google_cloud_platform__keyfile_dict": {"a": "b"}, "extra__google_cloud_platform__key_path": "xxx"}]}), + ("""conn_d: + conn_type: scheme + host: host + schema: lschema + login: Login + password: None + port: 1234 + extra: '{\"extra__google_cloud_platform__keyfile_dict\": {\"a\": \"b\"}}'""", {"conn_d": [ + {"extra__google_cloud_platform__keyfile_dict": {"a": "b"}}]}) ) ) From 39643fd9c08c29e4d61550a8ce2d7b2aa7d53c0a Mon Sep 17 00:00:00 2001 From: Vinay Date: Thu, 2 Jul 2020 12:15:27 +0530 Subject: [PATCH 4/9] Updating docs --- docs/howto/use-alternative-secrets-backend.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/howto/use-alternative-secrets-backend.rst b/docs/howto/use-alternative-secrets-backend.rst index 427400e457539..b2276e27cbb80 100644 --- a/docs/howto/use-alternative-secrets-backend.rst +++ b/docs/howto/use-alternative-secrets-backend.rst @@ -92,6 +92,10 @@ file ``/files/my_conn.json`` when it looks for connections. The file can be defined in ``JSON``, ``YAML`` or ``env`` format. +Any extra json parameters can be provided using keys like ``extra_dejson`` and ``extra``. +The key ``extra_dejson`` can be used to provide parameters as JSON object where as the key ``extra`` can be used in case of a JSON string. +The keys ``extra`` and ``extra_dejson`` are mutually exclusive. + The JSON file must contain an object where the key contains the connection ID and the value contains the definitions of one or more connections. The connection can be defined as a URI (string) or JSON object. @@ -118,9 +122,7 @@ The following is a sample JSON file. } The YAML file structure is similar to that of a JSON. The key-value pair of connection ID and the definitions of one or more connections. -The connection can be defined as a URI (string) or JSON object. Any extra json parameters can be provided using keys like ``extra_dejson`` and ``extra``. -The key ``extra_dejson`` can be used to provide parameters as JSON object where as the key ``extra`` can be used in case of a JSON string. -The keys ``extra`` and ``extra_dejson`` are mutually exclusive. +The connection can be defined as a URI (string) or JSON object. For a guide about defining a connection as a URI, see:: :ref:`generating_connection_uri`. For a description of the connection object parameters see :class:`~airflow.models.connection.Connection`. From 25506d59e3261bbfd8272cefd931a9a68f25e536 Mon Sep 17 00:00:00 2001 From: Vinay Date: Thu, 2 Jul 2020 12:57:41 +0530 Subject: [PATCH 5/9] Updating docs --- docs/howto/use-alternative-secrets-backend.rst | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/docs/howto/use-alternative-secrets-backend.rst b/docs/howto/use-alternative-secrets-backend.rst index b2276e27cbb80..a2062b0495493 100644 --- a/docs/howto/use-alternative-secrets-backend.rst +++ b/docs/howto/use-alternative-secrets-backend.rst @@ -68,6 +68,9 @@ This backend is especially useful in the following use cases: To use variable and connection from local file, specify :py:class:`~airflow.secrets.local_filesystem.LocalFilesystemBackend` as the ``backend`` in ``[secrets]`` section of ``airflow.cfg``. +For a guide about defining a connection as a URI, see:: :ref:`generating_connection_uri`. +For a description of the connection object parameters see :class:`~airflow.models.connection.Connection`. + Available parameters to ``backend_kwargs``: * ``variables_file_path``: File location with variables data. @@ -98,9 +101,6 @@ The keys ``extra`` and ``extra_dejson`` are mutually exclusive. The JSON file must contain an object where the key contains the connection ID and the value contains the definitions of one or more connections. The connection can be defined as a URI (string) or JSON object. - -For a guide about defining a connection as a URI, see:: :ref:`generating_connection_uri`. -For a description of the connection object parameters see :class:`~airflow.models.connection.Connection`. The following is a sample JSON file. .. code-block:: json @@ -124,10 +124,6 @@ The following is a sample JSON file. The YAML file structure is similar to that of a JSON. The key-value pair of connection ID and the definitions of one or more connections. The connection can be defined as a URI (string) or JSON object. -For a guide about defining a connection as a URI, see:: :ref:`generating_connection_uri`. -For a description of the connection object parameters see :class:`~airflow.models.connection.Connection`. -The following is a sample YAML file. - .. code-block:: yaml CONN_A: 'mysq://host_a' From b84622181a550d2d7a52f1c53a5e26355beecc3d Mon Sep 17 00:00:00 2001 From: Vinay G B Date: Tue, 7 Jul 2020 19:10:28 +0530 Subject: [PATCH 6/9] Update docs/howto/use-alternative-secrets-backend.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Kamil Breguła --- docs/howto/use-alternative-secrets-backend.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/howto/use-alternative-secrets-backend.rst b/docs/howto/use-alternative-secrets-backend.rst index a2062b0495493..c6e37afc4592c 100644 --- a/docs/howto/use-alternative-secrets-backend.rst +++ b/docs/howto/use-alternative-secrets-backend.rst @@ -93,7 +93,7 @@ Storing and Retrieving Connections If you have set ``connections_file_path`` as ``/files/my_conn.json``, then the backend will read the file ``/files/my_conn.json`` when it looks for connections. -The file can be defined in ``JSON``, ``YAML`` or ``env`` format. +The file can be defined in ``JSON``, ``YAML`` or ``env`` format. Depending on the format, the data should be saved as a URL or as a connection object. For a guide about defining a connection as a URI, see:: :ref:`generating_connection_uri`. For a description of the connection object parameters see :class:`~airflow.models.connection.Connection`. Any extra json parameters can be provided using keys like ``extra_dejson`` and ``extra``. The key ``extra_dejson`` can be used to provide parameters as JSON object where as the key ``extra`` can be used in case of a JSON string. From 981b106bae7e35daae2b5ff0cbd08b32300e246f Mon Sep 17 00:00:00 2001 From: Vinay G B Date: Tue, 7 Jul 2020 19:13:14 +0530 Subject: [PATCH 7/9] Update docs/howto/use-alternative-secrets-backend.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Kamil Breguła --- docs/howto/use-alternative-secrets-backend.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/howto/use-alternative-secrets-backend.rst b/docs/howto/use-alternative-secrets-backend.rst index c6e37afc4592c..af95ebf50a763 100644 --- a/docs/howto/use-alternative-secrets-backend.rst +++ b/docs/howto/use-alternative-secrets-backend.rst @@ -100,7 +100,7 @@ The key ``extra_dejson`` can be used to provide parameters as JSON object where The keys ``extra`` and ``extra_dejson`` are mutually exclusive. The JSON file must contain an object where the key contains the connection ID and the value contains -the definitions of one or more connections. The connection can be defined as a URI (string) or JSON object. +the definitions of one or more connections. In this format, the connection can be defined as a URI (string) or JSON object. The following is a sample JSON file. .. code-block:: json From ef32fc9fb8f2f45f10992296a304802eff633aed Mon Sep 17 00:00:00 2001 From: Vinay G B Date: Tue, 7 Jul 2020 19:13:26 +0530 Subject: [PATCH 8/9] Update docs/howto/use-alternative-secrets-backend.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Kamil Breguła --- docs/howto/use-alternative-secrets-backend.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/howto/use-alternative-secrets-backend.rst b/docs/howto/use-alternative-secrets-backend.rst index af95ebf50a763..e35d534165064 100644 --- a/docs/howto/use-alternative-secrets-backend.rst +++ b/docs/howto/use-alternative-secrets-backend.rst @@ -122,7 +122,7 @@ The following is a sample JSON file. } The YAML file structure is similar to that of a JSON. The key-value pair of connection ID and the definitions of one or more connections. -The connection can be defined as a URI (string) or JSON object. +In this format, the connection can be defined as a URI (string) or JSON object. .. code-block:: yaml From 6fe7af9b7c1d25f3a28c4b8c4bfa0096f0f826a9 Mon Sep 17 00:00:00 2001 From: Vinay Date: Tue, 7 Jul 2020 19:31:45 +0530 Subject: [PATCH 9/9] Removing repeated docs --- docs/howto/use-alternative-secrets-backend.rst | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/docs/howto/use-alternative-secrets-backend.rst b/docs/howto/use-alternative-secrets-backend.rst index e35d534165064..33e766ef99dc5 100644 --- a/docs/howto/use-alternative-secrets-backend.rst +++ b/docs/howto/use-alternative-secrets-backend.rst @@ -68,9 +68,6 @@ This backend is especially useful in the following use cases: To use variable and connection from local file, specify :py:class:`~airflow.secrets.local_filesystem.LocalFilesystemBackend` as the ``backend`` in ``[secrets]`` section of ``airflow.cfg``. -For a guide about defining a connection as a URI, see:: :ref:`generating_connection_uri`. -For a description of the connection object parameters see :class:`~airflow.models.connection.Connection`. - Available parameters to ``backend_kwargs``: * ``variables_file_path``: File location with variables data. @@ -93,8 +90,7 @@ Storing and Retrieving Connections If you have set ``connections_file_path`` as ``/files/my_conn.json``, then the backend will read the file ``/files/my_conn.json`` when it looks for connections. -The file can be defined in ``JSON``, ``YAML`` or ``env`` format. Depending on the format, the data should be saved as a URL or as a connection object. For a guide about defining a connection as a URI, see:: :ref:`generating_connection_uri`. For a description of the connection object parameters see :class:`~airflow.models.connection.Connection`. - +The file can be defined in ``JSON``, ``YAML`` or ``env`` format. Depending on the format, the data should be saved as a URL or as a connection object. Any extra json parameters can be provided using keys like ``extra_dejson`` and ``extra``. The key ``extra_dejson`` can be used to provide parameters as JSON object where as the key ``extra`` can be used in case of a JSON string. The keys ``extra`` and ``extra_dejson`` are mutually exclusive.