DMS: Update to commons-codec 0.0.24

CHANGES.md

@@ -1,6 +1,7 @@
 # Changelog
 
 ## in progress
+- DMS: Updated to commons-codec 0.0.24
 
 ## 2025-06-23 v0.0.8
 - DMS: Improved stack configuration and replication settings

lorrystream/process/kinesis_cratedb_lambda.py

@@ -36,8 +36,8 @@
 
 import sqlalchemy as sa
 from commons_codec.exception import UnknownOperationError
-from commons_codec.model import ColumnTypeMapStore
-from commons_codec.transform.aws_dms import DMSTranslatorCrateDB
+from commons_codec.model import ColumnMappingStrategy, ColumnTypeMapStore
+from commons_codec.transform.aws_dms import DMSTranslatorCrateDB, DMSTranslatorCrateDBRecordFactory
 from commons_codec.transform.dynamodb import DynamoDBCDCTranslator
 from sqlalchemy.util import asbool
 
@@ -76,7 +76,9 @@
 
 # TODO: Automatically create destination table.
 # TODO: Propagate mapping definitions and other settings.
+# TODO: Propagate column mapping strategy.
 if MESSAGE_FORMAT == "dms":
+    DMSTranslatorCrateDBRecordFactory.DEFAULT_MAPPING_STRATEGY = ColumnMappingStrategy.UNIVERSAL
     cdc = DMSTranslatorCrateDB(column_types=column_types)
 elif MESSAGE_FORMAT == "dynamodb":
     cdc = DynamoDBCDCTranslator(table_name=SINK_TABLE)

pyproject.toml

@@ -88,7 +88,7 @@ dependencies = [
   "click<9",
   "colorama<1",
   "colorlog",
-  "commons-codec==0.0.22",
+  "commons-codec>=0.0.24",
   "dask",
   "funcy",
   "influxdb",

tests/test_process.py

@@ -120,7 +120,7 @@ def test_kinesis_dms_cratedb_lambda_basic(mocker, cratedb, reset_handler):
     column_types = ColumnTypeMapStore().add(
         table=TableAddress(schema="public", table="foo"),
         column="attributes",
-        type_=ColumnType.MAP,
+        type_=ColumnType.OBJECT,
     )
 
     # Configure environment variables.
@@ -142,5 +142,7 @@ def test_kinesis_dms_cratedb_lambda_basic(mocker, cratedb, reset_handler):
 
     records = cratedb.database.run_sql('SELECT * FROM "public"."foo";', records=True)
     assert records[0] == {
-        "data": {"id": 46, "name": "Jane", "age": 31, "attributes": {"baz": "qux"}},
+        "pk": {"id": 46},
+        "data": {"name": "Jane", "age": 31, "attributes": {"baz": "qux"}},
+        "aux": {},
     }