Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
95fc8dd
[temp] remove Django tests for faster CI
timgraham Nov 8, 2025
3e4e244
INTPYTHON-527 Add Queryable Encryption support
aclark4life Jun 25, 2025
d4a555c
use shared library instead of mongocryptd
timgraham Nov 5, 2025
6e53f6d
polish howto
timgraham Nov 11, 2025
b5a02cd
doc query limitations + docs polish + todos
timgraham Nov 12, 2025
f33d0a4
edit "Dynamic library path configuration"
timgraham Nov 12, 2025
b1ad80e
combine topic guide with howto
timgraham Nov 13, 2025
7404fa9
Add "start csfle servers" func to evergreen config
aclark4life Nov 15, 2025
6fc5f8d
remove support for multiple kms providers
timgraham Nov 15, 2025
c4bb896
Add tests-8-qe to evergreen buildvariants
aclark4life Nov 17, 2025
abe29b7
fix less than lookup on encrypted fields
timgraham Nov 18, 2025
c157f04
simplify "Configuring the Automatic Encryption Shared Library" to rem…
timgraham Nov 20, 2025
483784f
reorder "Configuring the Automatic Encryption Shared Library" to make…
timgraham Nov 20, 2025
8fcc3c3
update docs/tests for $facet removal
timgraham Nov 22, 2025
aaa3d4f
Combine crypt shared w/installation & db setup
aclark4life Dec 3, 2025
6ebc3a8
Update versionadded to 6.0.1
aclark4life Dec 3, 2025
e9a748c
Configure AWS KMS for testing on evergreen
aclark4life Dec 5, 2025
10014f4
doc edits
aclark4life Dec 7, 2025
1caf632
Address review
aclark4life Dec 10, 2025
be1d3cb
Address UAT feedback
aclark4life Dec 10, 2025
417f1ef
Address UAT feedback
aclark4life Dec 11, 2025
070c2a6
add encryption-compatible aggregation wrap
timgraham Dec 17, 2025
47e064f
remove KMS_CREDENTIALS for "local" provider
timgraham Dec 17, 2025
2e87e1d
Relax "cannot aggregate encrypted fields" error message, possibly for…
timgraham Dec 17, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 49 additions & 1 deletion .evergreen/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,17 @@ post_error_fails_task: true
post_timeout_secs: 1800 # 5 minutes

functions:
"start csfle servers":
- command: ec2.assume_role
params:
role_arn: ${aws_test_secrets_role}
- command: subprocess.exec
params:
binary: bash
include_expansions_in_env: ["AWS_SECRET_ACCESS_KEY", "AWS_ACCESS_KEY_ID", "AWS_SESSION_TOKEN"]
args:
- ${DRIVERS_TOOLS}/.evergreen/csfle/setup.sh

"setup":
- command: git.get_project
params:
Expand Down Expand Up @@ -44,19 +55,32 @@ functions:
params:
binary: bash
working_dir: "src"
include_expansions_in_env: ["DRIVERS_TOOLS", "MONGODB_URI"]
include_expansions_in_env: ["DRIVERS_TOOLS", "MONGODB_URI", "DJANGO_SETTINGS_MODULE", "CRYPT_SHARED_LIB_PATH"]
args:
- ./.evergreen/run-tests.sh

"run encryption tests":
- command: subprocess.exec
type: test
params:
binary: bash
working_dir: "src"
include_expansions_in_env: ["DRIVERS_TOOLS", "MONGODB_URI", "DJANGO_SETTINGS_MODULE", "CRYPT_SHARED_LIB_PATH"]
args:
- ./.evergreen/run-tests.sh
- encryption

"teardown":
- command: subprocess.exec
params:
binary: bash
args:
- ${DRIVERS_TOOLS}/.evergreen/teardown.sh
- ${DRIVERS_TOOLS}/.evergreen/csfle/teardown.sh

pre:
- func: setup
- func: start csfle servers
- func: bootstrap mongo-orchestration

post:
Expand All @@ -67,6 +91,10 @@ tasks:
commands:
- func: "run unit tests"

- name: run-encryption-tests
commands:
- func: "run encryption tests"

buildvariants:
- name: tests-7-noauth-nossl
display_name: Run Tests 7.0 NoAuth NoSSL
Expand Down Expand Up @@ -111,3 +139,23 @@ buildvariants:
SSL: "ssl"
tasks:
- name: run-tests

- name: tests-8-qe-local
display_name: Run Tests 8.2 QE local KMS
run_on: rhel87-small
expansions:
MONGODB_VERSION: "8.2"
Copy link
Collaborator

@timgraham timgraham Dec 9, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My impression is that we only want to test with dot zero MongoDB's, but lets get clarification on this.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we use 8 then we get this:


2025/12/09 22:29:18.949] ======================================================================
[2025/12/09 22:29:18.949] ERROR: test_join (encryption_.test_fields.QueryTests)
[2025/12/09 22:29:18.949] ----------------------------------------------------------------------
[2025/12/09 22:29:18.949] Traceback (most recent call last):
[2025/12/09 22:29:18.949]   File "/data/mci/85bf7b204919d8a8575368e5b4761ad6/src/venv/lib/python3.10/site-packages/pymongo/synchronous/encryption.py", line 124, in _wrap_encryption_errors
[2025/12/09 22:29:18.949]     yield
[2025/12/09 22:29:18.949]   File "/data/mci/85bf7b204919d8a8575368e5b4761ad6/src/venv/lib/python3.10/site-packages/pymongo/synchronous/encryption.py", line 467, in encrypt
[2025/12/09 22:29:18.949]     encrypted_cmd = self._auto_encrypter.encrypt(database, encoded_cmd)
[2025/12/09 22:29:18.949]   File "/data/mci/85bf7b204919d8a8575368e5b4761ad6/src/venv/lib/python3.10/site-packages/pymongocrypt/synchronous/auto_encrypter.py", line 43, in encrypt
[2025/12/09 22:29:18.949]     with self.mongocrypt.encryption_context(database, cmd) as ctx:
[2025/12/09 22:29:18.949]   File "/data/mci/85bf7b204919d8a8575368e5b4761ad6/src/venv/lib/python3.10/site-packages/pymongocrypt/mongocrypt.py", line 228, in encryption_context
[2025/12/09 22:29:18.949]     return EncryptionContext(
[2025/12/09 22:29:18.949]   File "/data/mci/85bf7b204919d8a8575368e5b4761ad6/src/venv/lib/python3.10/site-packages/pymongocrypt/mongocrypt.py", line 426, in __init__
[2025/12/09 22:29:18.949]     self._raise_from_status()
[2025/12/09 22:29:18.949]   File "/data/mci/85bf7b204919d8a8575368e5b4761ad6/src/venv/lib/python3.10/site-packages/pymongocrypt/mongocrypt.py", line 355, in _raise_from_status
[2025/12/09 22:29:18.949]     raise exc
[2025/12/09 22:29:18.949] pymongocrypt.errors.MongoCryptError: Encrypting 'aggregate' requires multiple schemas. Detected crypt_shared with version mongo_crypt_v1-dev-8.0.16, but need 8.1. Upgrade crypt_shared to 8.1 or newer.

Presumably because mongo-orchestration installs the corresponding crypt shared version to match the server version. We could --skip-crypt-shared and manually install crypt shared 8.2 similar to what is in GitHub Actions, but I'm not sure if it's worth the effort.

TOPOLOGY: replica_set
DJANGO_SETTINGS_MODULE: "local_kms_encrypted_settings"
tasks:
- name: run-encryption-tests

- name: tests-8-qe-aws
display_name: Run Tests 8.2 QE aws KMS
run_on: rhel87-small
expansions:
MONGODB_VERSION: "8.2"
TOPOLOGY: replica_set
DJANGO_SETTINGS_MODULE: "aws_kms_encrypted_settings"
tasks:
- name: run-encryption-tests
15 changes: 13 additions & 2 deletions .evergreen/run-tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,22 @@

set -eux

# Install django-mongodb-backend
# Export secrets as environment variables
if [[ "${1:-}" == "encryption" ]]; then
. ../secrets-export.sh
fi

# Set up virtual environment
/opt/python/3.12/bin/python3 -m venv venv
. venv/bin/activate
python -m pip install -U pip
pip install -e .

# Conditionally install encryption extra if "encryption" arg is passed
if [[ "${1:-}" == "encryption" ]]; then
pip install -e '.[encryption]'
else
pip install -e .
fi

# Install django and test dependencies
git clone --branch mongodb-6.0.x https://github.com/mongodb-forks/django django_repo
Expand Down
8 changes: 4 additions & 4 deletions .evergreen/setup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ DRIVERS_TOOLS="$(dirname "$(pwd)")/drivers-tools"
PROJECT_DIRECTORY="$(pwd)"

if [ "Windows_NT" = "${OS:-}" ]; then
DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS)
PROJECT_DIRECTORY=$(cygpath -m $PROJECT_DIRECTORY)
DRIVERS_TOOLS=$(cygpath -m "$DRIVERS_TOOLS")
PROJECT_DIRECTORY=$(cygpath -m "$PROJECT_DIRECTORY")
fi
export PROJECT_DIRECTORY
export DRIVERS_TOOLS
Expand All @@ -37,8 +37,8 @@ PROJECT_DIRECTORY: "$PROJECT_DIRECTORY"
EOT

# Set up drivers-tools with a .env file.
git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git ${DRIVERS_TOOLS}
cat <<EOT > ${DRIVERS_TOOLS}/.env
git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git "${DRIVERS_TOOLS}"
cat <<EOT > "${DRIVERS_TOOLS}/.env"
CURRENT_VERSION="$CURRENT_VERSION"
DRIVERS_TOOLS="$DRIVERS_TOOLS"
MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME"
Expand Down
26 changes: 26 additions & 0 deletions .github/workflows/aws_kms_encrypted_settings.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
from local_kms_encrypted_settings import * # noqa: F403

DATABASES["encrypted"] = { # noqa: F405
"ENGINE": "django_mongodb_backend",
"NAME": "djangotests_encrypted",
"OPTIONS": {
"auto_encryption_opts": AutoEncryptionOpts( # noqa: F405
key_vault_namespace="djangotests_encrypted.__keyVault",
kms_providers={
"aws": {
"accessKeyId": os.environ.get("FLE_AWS_KEY"), # noqa: F405
"secretAccessKey": os.environ.get("FLE_AWS_SECRET"), # noqa: F405
}
},
crypt_shared_lib_path=os.environ["CRYPT_SHARED_LIB_PATH"], # noqa: F405
crypt_shared_lib_required=True,
),
"directConnection": True,
},
"KMS_CREDENTIALS": {
"aws": {
"key": "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0",
"region": "us-east-1",
}
},
}
43 changes: 43 additions & 0 deletions .github/workflows/local_kms_encrypted_settings.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
# Settings for django_mongodb_backend/tests when encryption is supported.
import os
from pathlib import Path

from mongodb_settings import * # noqa: F403
from pymongo.encryption import AutoEncryptionOpts

os.environ["LD_LIBRARY_PATH"] = str(Path(os.environ["CRYPT_SHARED_LIB_PATH"]).parent)

DATABASES["encrypted"] = { # noqa: F405
"ENGINE": "django_mongodb_backend",
"NAME": "djangotests_encrypted",
"OPTIONS": {
"auto_encryption_opts": AutoEncryptionOpts(
key_vault_namespace="djangotests_encrypted.__keyVault",
kms_providers={"local": {"key": os.urandom(96)}},
crypt_shared_lib_path=os.environ["CRYPT_SHARED_LIB_PATH"],
),
"directConnection": True,
},
}


class EncryptedRouter:
def db_for_read(self, model, **hints):
if model._meta.app_label == "encryption_":
return "encrypted"
return None

db_for_write = db_for_read

def allow_migrate(self, db, app_label, model_name=None, **hints):
# The encryption_ app's models are only created in the encrypted
# database.
if app_label == "encryption_":
return db == "encrypted"
# Don't create other app's models in the encrypted database.
if db == "encrypted":
return False
return None


DATABASE_ROUTERS.append(EncryptedRouter()) # noqa: F405
1 change: 1 addition & 0 deletions .github/workflows/mongodb_settings.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
# Settings for django_mongodb_backend/tests.
from django_settings import * # noqa: F403

DATABASES["encrypted"] = {} # noqa: F405
DATABASE_ROUTERS = ["django_mongodb_backend.routers.MongoRouter"]
145 changes: 0 additions & 145 deletions .github/workflows/runtests.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,151 +6,6 @@
from django.core.exceptions import ImproperlyConfigured

test_apps = [
"admin_changelist",
"admin_checks",
"admin_custom_urls",
"admin_docs",
"admin_filters",
"admin_inlines",
"admin_ordering",
"admin_scripts",
"admin_utils",
"admin_views",
"admin_widgets",
"aggregation",
"aggregation_regress",
"annotations",
"apps",
"async",
"auth_tests",
"backends",
"basic",
"bulk_create",
"cache",
"check_framework",
"constraints",
"contenttypes_tests",
"context_processors",
"custom_columns",
"custom_lookups",
"custom_managers",
"custom_pk",
"datatypes",
"dates",
"datetimes",
"db_functions",
"defer",
"defer_regress",
"delete",
"delete_regress",
"empty",
"empty_models",
"expressions",
"expressions_case",
"field_defaults",
"file_storage",
"file_uploads",
"fixtures",
"fixtures_model_package",
"fixtures_regress",
"flatpages_tests",
"force_insert_update",
"foreign_object",
"forms_tests",
"from_db_value",
"generic_inline_admin",
"generic_relations",
"generic_relations_regress",
"generic_views",
"get_earliest_or_latest",
"get_object_or_404",
"get_or_create",
"i18n",
"indexes",
"inline_formsets",
"introspection",
"invalid_models_tests",
"known_related_objects",
"lookup",
"m2m_and_m2o",
"m2m_intermediary",
"m2m_multiple",
"m2m_recursive",
"m2m_regress",
"m2m_signals",
"m2m_through",
"m2m_through_regress",
"m2o_recursive",
"managers_regress",
"many_to_many",
"many_to_one",
"many_to_one_null",
"max_lengths",
"messages_tests",
"migrate_signals",
"migration_test_data_persistence",
"migrations",
"model_fields",
"model_forms",
"model_formsets",
"model_formsets_regress",
"model_indexes",
"model_inheritance",
"model_inheritance_regress",
"model_options",
"model_package",
"model_regress",
"model_utils",
"modeladmin",
"multiple_database",
"mutually_referential",
"nested_foreign_keys",
"null_fk",
"null_fk_ordering",
"null_queries",
"one_to_one",
"or_lookups",
"order_with_respect_to",
"ordering",
"pagination",
"prefetch_related",
"proxy_model_inheritance",
"proxy_models",
"queries",
"queryset_pickle",
"redirects_tests",
"reserved_names",
"reverse_lookup",
"save_delete_hooks",
"schema",
"select_for_update",
"select_related",
"select_related_onetoone",
"select_related_regress",
"serializers",
"servers",
"sessions_tests",
"shortcuts",
"signals",
"sitemaps_tests",
"sites_framework",
"sites_tests",
"string_lookup",
"swappable_models",
"syndication_tests",
"test_client",
"test_client_regress",
"test_runner",
"test_utils",
"timezones",
"transactions",
"unmanaged_models",
"update",
"update_only_fields",
"user_commands",
"validation",
"view_tests",
"xor_lookups",
# Add directories in django_mongodb_backend/tests
*sorted(
[
Expand Down
11 changes: 9 additions & 2 deletions .github/workflows/test-python-atlas.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ jobs:
- name: install django-mongodb-backend
run: |
pip3 install --upgrade pip
pip3 install -e .
pip3 install -e .[encryption]
- name: Checkout Django
uses: actions/checkout@v6
with:
Expand All @@ -51,8 +51,15 @@ jobs:
run: cp .github/workflows/runtests.py django_repo/tests/runtests_.py
- name: Start local Atlas
working-directory: .
run: bash .github/workflows/start_local_atlas.sh mongodb/mongodb-atlas-local:7
run: bash .github/workflows/start_local_atlas.sh mongodb/mongodb-atlas-local:8.0.15
- name: Download crypt shared
run: |
wget https://downloads.mongodb.com/linux/mongo_crypt_shared_v1-linux-x86_64-enterprise-ubuntu2404-8.2.1.tgz
tar -xvzf mongo_crypt_shared_v1-linux-x86_64-enterprise-ubuntu2404-8.2.1.tgz lib/mongo_crypt_v1.so
- name: Run tests
run: python3 django_repo/tests/runtests_.py
permissions:
contents: read
env:
DJANGO_SETTINGS_MODULE: "local_kms_encrypted_settings"
CRYPT_SHARED_LIB_PATH: "${{ github.workspace }}/lib/mongo_crypt_v1.so"
Loading