diff --git a/archery/settings.py b/archery/settings.py index be57e358..97725efe 100644 --- a/archery/settings.py +++ b/archery/settings.py @@ -26,14 +26,10 @@ ), # Reference: https://docs.djangoproject.com/en/4.0/ref/settings/#secret-key DATABASE_URL=(str, "mysql://root:@127.0.0.1:3306/archery"), CACHE_URL=(str, "redis://127.0.0.1:6379/0"), - # External authentication currently supports LDAP, OIDC, and DINGDING. - # Only one method should be enabled. If multiple are enabled, only one takes effect with priority LDAP > DINGDING > OIDC. + # External authentication currently supports LDAP and OIDC. + # Only one method should be enabled. If multiple are enabled, only one takes effect with priority LDAP > OIDC. ENABLE_LDAP=(bool, False), ENABLE_OIDC=(bool, False), - ENABLE_DINGDING=( - bool, - False, - ), # DingTalk authentication reference: https://open.dingtalk.com/document/orgapp/tutorial-obtaining-user-personal-information AUTH_LDAP_ALWAYS_UPDATE_USER=(bool, True), AUTH_LDAP_USER_ATTR_MAP=( dict, @@ -72,8 +68,6 @@ ENABLED_NOTIFIERS=( list, [ - "sql.notify:DingdingWebhookNotifier", - "sql.notify:DingdingPersonNotifier", "sql.notify:FeishuWebhookNotifier", "sql.notify:FeishuPersonNotifier", "sql.notify:QywxWebhookNotifier", @@ -84,6 +78,7 @@ ), CURRENT_AUDITOR=(str, "sql.utils.workflow_audit:AuditV2"), PASSWORD_MIXIN_PATH=(str, "sql.plugins.password:DummyMixin"), + FIELD_ENCRYPTION_KEYS=(str, ""), ) # SECURITY WARNING: keep the secret key used in production secret! @@ -129,6 +124,8 @@ PASSWORD_MIXIN_PATH = env("PASSWORD_MIXIN_PATH") +FIELD_ENCRYPTION_KEYS = env("FIELD_ENCRYPTION_KEYS") + # Application definition INSTALLED_APPS = ( "django.contrib.admin", @@ -343,20 +340,6 @@ LOGIN_REDIRECT_URL = "/" -# Dingding -ENABLE_DINGDING = env("ENABLE_DINGDING", False) -if ENABLE_DINGDING: - INSTALLED_APPS += ("django_auth_dingding",) - AUTHENTICATION_BACKENDS = ( - "common.authenticate.dingding_auth.DingdingAuthenticationBackend", - "django.contrib.auth.backends.ModelBackend", - ) - AUTH_DINGDING_AUTHENTICATION_CALLBACK_URL = env( - "AUTH_DINGDING_AUTHENTICATION_CALLBACK_URL" - ) - AUTH_DINGDING_APP_KEY = env("AUTH_DINGDING_APP_KEY") - AUTH_DINGDING_APP_SECRET = env("AUTH_DINGDING_APP_SECRET") - # LDAP ENABLE_LDAP = env("ENABLE_LDAP", False) if ENABLE_LDAP: @@ -421,7 +404,6 @@ SUPPORTED_AUTHENTICATION = [ ("LDAP", ENABLE_LDAP), - ("DINGDING", ENABLE_DINGDING), ("OIDC", ENABLE_OIDC), ("CAS", ENABLE_CAS), ] @@ -432,7 +414,7 @@ if ENABLE_AUTHENTICATION_COUNT > 0: if ENABLE_AUTHENTICATION_COUNT > 1: logger.warning( - "External authentication currently supports LDAP, DINGDING, OIDC, and CAS. Only one method should be enabled. If multiple are enabled, only one takes effect with priority LDAP > DINGDING > OIDC > CAS." + "External authentication currently supports LDAP, OIDC, and CAS. Only one method should be enabled. If multiple are enabled, only one takes effect with priority LDAP > OIDC > CAS." ) authentication = "" # Empty by default for name, enabled in SUPPORTED_AUTHENTICATION: diff --git a/archery/urls.py b/archery/urls.py index 03d8f556..2bbb206c 100644 --- a/archery/urls.py +++ b/archery/urls.py @@ -25,11 +25,6 @@ path("oidc/", include("mozilla_django_oidc.urls")), ] -if settings.ENABLE_DINGDING: # pragma: no cover - urlpatterns += [ - path("dingding/", include("django_auth_dingding.urls")), - ] - handler400 = views.bad_request handler403 = views.permission_denied handler404 = views.page_not_found diff --git a/common/auth.py b/common/auth.py index 7c248a43..59d1c4cc 100644 --- a/common/auth.py +++ b/common/auth.py @@ -11,9 +11,7 @@ from django.http import HttpResponse, HttpResponseRedirect from django.urls import reverse -from django.conf import settings from common.config import SysConfig -from common.utils.ding_api import get_ding_user_id from sql.models import Users, ResourceGroup, TwoFactorAuthConfig logger = logging.getLogger("default") @@ -162,11 +160,6 @@ def authenticate_entry(request): else: # No 2FA configured; log in directly login(request, authenticated_user) - # Fetch DingTalk user ID for direct notifications - if SysConfig().get( - "ding_to_person" - ) is True and "admin" not in request.POST.get("username"): - get_ding_user_id(request.POST.get("username")) result = {"status": 0, "msg": "ok", "data": None} return HttpResponse(json.dumps(result), content_type="application/json") @@ -221,11 +214,5 @@ def sign_up(request): # Sign out def sign_out(request): - user = request.user logout(request) - # If DingTalk auth is enabled, redirect to DingTalk logout page - if user.ding_user_id and settings.ENABLE_DINGDING: - return HttpResponseRedirect( - redirect_to="https://login.dingtalk.com/oauth2/logout" - ) return HttpResponseRedirect(reverse("sql:login")) diff --git a/common/authenticate/dingding_auth.py b/common/authenticate/dingding_auth.py deleted file mode 100644 index 091ba80b..00000000 --- a/common/authenticate/dingding_auth.py +++ /dev/null @@ -1,10 +0,0 @@ -from django_auth_dingding import auth -from common.auth import init_user - - -class DingdingAuthenticationBackend(auth.DingdingAuthenticationBackend): - def create_user(self, claims): - """Return object for a newly created user account.""" - user = super().create_user(claims) - init_user(user) - return user diff --git a/common/encryption.py b/common/encryption.py new file mode 100644 index 00000000..b028bb47 --- /dev/null +++ b/common/encryption.py @@ -0,0 +1,131 @@ +import base64 +import os +import re +from functools import lru_cache + +from cryptography.fernet import Fernet, MultiFernet +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import padding +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes +from django.conf import settings +from django.utils.encoding import force_bytes, force_str + +from common.utils.aes_decryptor import Prpcrypt + +ENCRYPTED_VALUE_PREFIX = "enc1:" +HEX_RE = re.compile(r"^[0-9a-fA-F]+$") + + +class DecryptionError(ValueError): + pass + + +def generate_field_encryption_key(): + return Fernet.generate_key().decode("ascii") + + +def is_encrypted_value(value): + return isinstance(value, str) and value.startswith(ENCRYPTED_VALUE_PREFIX) + + +def encrypt_value(value): + if value is None: + return None + if not _load_field_encryption_keys(): + raise ValueError( + "FIELD_ENCRYPTION_KEYS must be configured before writing encrypted values." + ) + token = get_multi_fernet().encrypt(force_bytes(value)) + return f"{ENCRYPTED_VALUE_PREFIX}{force_str(token)}" + + +def decrypt_value(value): + if value is None: + return None + if not isinstance(value, str): + return value + if is_encrypted_value(value): + token = value[len(ENCRYPTED_VALUE_PREFIX) :] + return force_str(get_multi_fernet().decrypt(force_bytes(token))) + return decrypt_legacy_value(value) + + +def decrypt_legacy_value(value): + if value in ("", None): + return value + + mirage_value = _try_decrypt_mirage(value) + if mirage_value is not None: + return mirage_value + + old_value = _try_decrypt_old_hex(value) + if old_value is not None: + return old_value + + raise DecryptionError( + "Unable to decrypt legacy value. " f"marker={value[:16]!r} length={len(value)}" + ) + + +@lru_cache(maxsize=1) +def get_multi_fernet(): + keys = _load_field_encryption_keys() + if not keys: + raise ValueError( + "FIELD_ENCRYPTION_KEYS must be configured before writing encrypted values." + ) + return MultiFernet([Fernet(key) for key in keys]) + + +def _load_field_encryption_keys(): + raw_keys = getattr(settings, "FIELD_ENCRYPTION_KEYS", "") or os.environ.get( + "FIELD_ENCRYPTION_KEYS", "" + ) + keys = [] + for raw_key in raw_keys.split(","): + key = raw_key.strip() + if not key: + continue + try: + Fernet(key) + except Exception as exc: + raise ValueError("Invalid FIELD_ENCRYPTION_KEYS entry.") from exc + keys.append(key.encode("ascii")) + return keys + + +def _try_decrypt_mirage(value): + try: + key = getattr(settings, "MIRAGE_SECRET_KEY", None) or getattr( + settings, "SECRET_KEY" + ) + if not key: + return None + cipher_key = base64.urlsafe_b64encode(force_bytes(key))[:32] + if len(cipher_key) not in (16, 24, 32): + return None + cipher_mode = getattr(settings, "MIRAGE_CIPHER_MODE", "ECB") + iv = force_bytes(getattr(settings, "MIRAGE_CIPHER_IV", "1234567890abcdef")) + encrypted = base64.urlsafe_b64decode(force_bytes(value)) + if cipher_mode == "CBC": + mode = modes.CBC(iv) + else: + mode = modes.ECB() + decryptor = Cipher( + algorithms.AES(cipher_key), mode, default_backend() + ).decryptor() + unpadder = padding.PKCS7(algorithms.AES(cipher_key).block_size).unpadder() + plaintext = decryptor.update(encrypted) + decryptor.finalize() + unpadded = unpadder.update(plaintext) + unpadder.finalize() + return force_str(unpadded) + except Exception: + return None + + +def _try_decrypt_old_hex(value): + if len(value) < 32 or len(value) % 2 != 0 or HEX_RE.fullmatch(value) is None: + return None + try: + return Prpcrypt().decrypt(value) + except Exception: + return None diff --git a/common/fields/__init__.py b/common/fields/__init__.py new file mode 100644 index 00000000..f51f468b --- /dev/null +++ b/common/fields/__init__.py @@ -0,0 +1,3 @@ +from common.fields.encrypted import EncryptedCharField, EncryptedTextField + +__all__ = ["EncryptedCharField", "EncryptedTextField"] diff --git a/common/fields/encrypted.py b/common/fields/encrypted.py new file mode 100644 index 00000000..16fda63f --- /dev/null +++ b/common/fields/encrypted.py @@ -0,0 +1,36 @@ +from django.core.validators import MaxLengthValidator +from django.db import models + +from common.encryption import decrypt_value, encrypt_value + + +class EncryptedMixin(models.Field): + def __init__(self, *args, **kwargs): + self._plaintext_max_length = kwargs.get("max_length") + super().__init__(*args, **kwargs) + if self._plaintext_max_length: + self.validators.append(MaxLengthValidator(self._plaintext_max_length)) + + def get_prep_value(self, value): + value = super().get_prep_value(value) + if value is None: + return None + return encrypt_value(value) + + def from_db_value(self, value, expression, connection): + if value is None: + return None + if value == "": + return value + return decrypt_value(value) + + def to_python(self, value): + return super().to_python(value) + + +class EncryptedTextField(EncryptedMixin, models.TextField): + pass + + +class EncryptedCharField(EncryptedMixin, models.TextField): + pass diff --git a/common/middleware/check_login_middleware.py b/common/middleware/check_login_middleware.py index b298df91..718f2964 100644 --- a/common/middleware/check_login_middleware.py +++ b/common/middleware/check_login_middleware.py @@ -12,8 +12,6 @@ "/oidc/callback/", "/oidc/authenticate/", "/oidc/logout/", - "/dingding/callback/", - "/dingding/authenticate/", "/cas/authenticate/", ] diff --git a/common/templates/config.html b/common/templates/config.html index 1ecbdfa6..9e51b0a8 100755 --- a/common/templates/config.html +++ b/common/templates/config.html @@ -804,95 +804,6 @@
Workflow Notifications
-
- -
-
- -
-
-
-
- -
-
- -
-
-
- -
@@ -1578,15 +1489,13 @@
Current Approval Flow: Current Approval Flow: Current Approval Flow: Current Approval Flow: Login To Archery{{ custom_title_suffix }}
{% endif %} - {% if dingding_enabled or oidc_enabled %} + {% if oidc_enabled %} diff --git a/common/test_encryption.py b/common/test_encryption.py new file mode 100644 index 00000000..ba644bb4 --- /dev/null +++ b/common/test_encryption.py @@ -0,0 +1,76 @@ +import os +from unittest.mock import patch + +from django.test import SimpleTestCase, override_settings + +from common.encryption import ( + DecryptionError, + ENCRYPTED_VALUE_PREFIX, + decrypt_value, + encrypt_value, + generate_field_encryption_key, + get_multi_fernet, +) +from common.test_fixtures import LEGACY_MIRAGE_CIPHERTEXTS, LEGACY_MIRAGE_SECRET_KEY +from common.utils.aes_decryptor import Prpcrypt + + +def _encrypt_legacy_mirage(value, secret_key): + assert secret_key == LEGACY_MIRAGE_SECRET_KEY + return LEGACY_MIRAGE_CIPHERTEXTS[value] + + +class EncryptionHelpersTest(SimpleTestCase): + def tearDown(self): + get_multi_fernet.cache_clear() + + @override_settings(FIELD_ENCRYPTION_KEYS=generate_field_encryption_key()) + def test_round_trip_encrypts_with_prefix(self): + get_multi_fernet.cache_clear() + + encrypted = encrypt_value("top-secret") + + self.assertTrue(encrypted.startswith(ENCRYPTED_VALUE_PREFIX)) + self.assertEqual(decrypt_value(encrypted), "top-secret") + + @override_settings(FIELD_ENCRYPTION_KEYS=generate_field_encryption_key()) + def test_prefix_looking_plaintext_is_still_encrypted(self): + get_multi_fernet.cache_clear() + + encrypted = encrypt_value("enc1:not-a-token") + + self.assertNotEqual(encrypted, "enc1:not-a-token") + self.assertEqual(decrypt_value(encrypted), "enc1:not-a-token") + + @override_settings(FIELD_ENCRYPTION_KEYS="") + def test_encrypt_requires_configured_keys(self): + get_multi_fernet.cache_clear() + + with patch.dict(os.environ, {"FIELD_ENCRYPTION_KEYS": ""}, clear=False): + with self.assertRaises(ValueError): + encrypt_value("top-secret") + + @override_settings(FIELD_ENCRYPTION_KEYS=generate_field_encryption_key()) + def test_decrypts_legacy_hex_ciphertext(self): + get_multi_fernet.cache_clear() + + legacy = Prpcrypt().encrypt("legacy-password") + + self.assertEqual(decrypt_value(legacy), "legacy-password") + + @override_settings( + FIELD_ENCRYPTION_KEYS=generate_field_encryption_key(), + SECRET_KEY=LEGACY_MIRAGE_SECRET_KEY, + ) + def test_decrypts_legacy_mirage_ciphertext(self): + get_multi_fernet.cache_clear() + + legacy = _encrypt_legacy_mirage("legacy-user", LEGACY_MIRAGE_SECRET_KEY) + + self.assertEqual(decrypt_value(legacy), "legacy-user") + + def test_decrypt_legacy_value_raises_for_unknown_ciphertext(self): + get_multi_fernet.cache_clear() + + with self.assertRaises(DecryptionError): + decrypt_value("not-a-legacy-ciphertext") diff --git a/common/test_fixtures.py b/common/test_fixtures.py new file mode 100644 index 00000000..e81ff6c8 --- /dev/null +++ b/common/test_fixtures.py @@ -0,0 +1,18 @@ +LEGACY_MIRAGE_SECRET_KEY = "test-mirage-secret-key-1234567890" +LEGACY_MIRAGE_CBC_IV = "fedcba0987654321" + +# These ciphertexts were generated from the pre-migration Mirage AES/ECB format +# with LEGACY_MIRAGE_SECRET_KEY and are pinned here to keep tests deterministic. +LEGACY_MIRAGE_CIPHERTEXTS = { + "legacy-user": "_Z3ltHqLM3ElYkNjsFGCMA==", + "legacy-root": "MVDwR1ka9b_2PNsblc75QQ==", + "legacy-password": "whY6rTtCeZicLTJmkbkXhg==", + "legacy-ak": "lqN38b1I1guLDVHUFmQZ4A==", + "legacy-sk": "xDwJ96BefaSfmGXSH_mThQ==", +} + +# This ciphertext was generated from the pre-migration Mirage AES/CBC format +# with LEGACY_MIRAGE_SECRET_KEY and LEGACY_MIRAGE_CBC_IV. +LEGACY_MIRAGE_CBC_CIPHERTEXTS = { + "legacy-cbc-user": "DvylVMJdhBSwATSUJMZiOg==", +} diff --git a/common/tests.py b/common/tests.py index aa161adb..411f64c6 100644 --- a/common/tests.py +++ b/common/tests.py @@ -16,9 +16,12 @@ SqlWorkflowContent, QueryLog, ResourceGroup, + TwoFactorAuthConfig, ) from common.utils.chart_dao import ChartDao from common.auth import init_user +from common.twofa.sms import SMS +from common.twofa.totp import TOTP from common.utils.extend_json_encoder import ExtendJSONEncoderFTime User = get_user_model() @@ -164,31 +167,6 @@ def tearDown(self): archer_config.set("mail_ssl", "") -class DingTest(TestCase): - def setUp(self): - self.url = "some_url" - self.content = "some_content" - - @patch("requests.post") - def testDing(self, post): - sender = MsgSender() - post.return_value.json.return_value = {"errcode": 0} - with self.assertLogs("default", level="DEBUG") as lg: - sender.send_ding(self.url, self.content) - post.assert_called_once_with( - url=self.url, - json={"msgtype": "text", "text": {"content": self.content}}, - ) - self.assertIn("DingTalk webhook sent successfully", lg.output[0]) - post.return_value.json.return_value = {"errcode": 1, "errmsg": "test_error"} - with self.assertLogs("default", level="ERROR") as lg: - sender.send_ding(self.url, self.content) - self.assertIn("test_error", lg.output[0]) - - def tearDown(self): - pass - - class GlobalInfoTest(TestCase): def setUp(self): self.u1 = User(username="test_user", display="Chinese display", is_active=True) @@ -559,6 +537,35 @@ def test_init_user(self): self.assertEqual(self.u1, self.resource_group1.users_set.get(pk=self.u1.pk)) +class TestTwoFactorAuth(TestCase): + def setUp(self): + self.user = User.objects.create( + username="twofa_user", + display="TwoFA User", + is_active=True, + ) + + def tearDown(self): + TwoFactorAuthConfig.objects.all().delete() + self.user.delete() + + def test_sms_verify_returns_controlled_error_when_config_missing(self): + result = SMS(user=self.user).verify("123456") + + self.assertEqual( + result, + {"status": 1, "msg": "SMS 2FA is not configured for this account."}, + ) + + def test_totp_verify_returns_controlled_error_when_config_missing(self): + result = TOTP(user=self.user).verify("123456") + + self.assertEqual( + result, + {"status": 1, "msg": "TOTP 2FA is not configured for this account."}, + ) + + class PermissionTest(TestCase): def setUp(self) -> None: self.user = User.objects.create( diff --git a/common/twofa/sms.py b/common/twofa/sms.py index 7eb58620..6c89ae25 100644 --- a/common/twofa/sms.py +++ b/common/twofa/sms.py @@ -62,7 +62,20 @@ def verify(self, otp, phone=None): if phone: phone = phone else: - phone = TwoFactorAuthConfig.objects.get(username=self.user.username).phone + try: + phone = TwoFactorAuthConfig.objects.get( + user=self.user, auth_type=self.auth_type + ).phone + except TwoFactorAuthConfig.DoesNotExist: + logger.warning( + "Missing SMS 2FA config for user=%s auth_type=%s", + getattr(self.user, "username", None), + self.auth_type, + ) + return { + "status": 1, + "msg": "SMS 2FA is not configured for this account.", + } r = get_redis_connection("default") data = r.get(f"captcha-{phone}") @@ -86,7 +99,6 @@ def save(self, phone): self.disable(self.auth_type) # Create new 2FA config TwoFactorAuthConfig.objects.create( - username=self.user.username, auth_type=self.auth_type, phone=phone, user=self.user, diff --git a/common/twofa/totp.py b/common/twofa/totp.py index 150ce638..779f4510 100644 --- a/common/twofa/totp.py +++ b/common/twofa/totp.py @@ -24,9 +24,20 @@ def verify(self, otp, key=None): if key: secret_key = key else: - secret_key = TwoFactorAuthConfig.objects.get( - username=self.user.username, auth_type=self.auth_type - ).secret_key + try: + secret_key = TwoFactorAuthConfig.objects.get( + user=self.user, auth_type=self.auth_type + ).secret_key + except TwoFactorAuthConfig.DoesNotExist: + logger.warning( + "Missing TOTP 2FA config for user=%s auth_type=%s", + getattr(self.user, "username", None), + self.auth_type, + ) + return { + "status": 1, + "msg": "TOTP 2FA is not configured for this account.", + } t = pyotp.TOTP(secret_key) status = t.verify(otp) result["status"] = 0 if status else 1 @@ -53,7 +64,6 @@ def save(self, secret_key): self.disable(self.auth_type) # Create new 2FA config TwoFactorAuthConfig.objects.create( - username=self.user.username, auth_type=self.auth_type, secret_key=secret_key, user=self.user, diff --git a/common/utils/ding_api.py b/common/utils/ding_api.py deleted file mode 100644 index 512901af..00000000 --- a/common/utils/ding_api.py +++ /dev/null @@ -1,129 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import logging -import requests -from django.http import JsonResponse -from django_redis import get_redis_connection -from common.config import SysConfig -from common.utils.permission import superuser_required -from sql.models import Users -from sql.utils.tasks import add_sync_ding_user_schedule - -logger = logging.getLogger("default") -rs = get_redis_connection("default") - - -def get_access_token(): - """Get DingTalk access token: https://ding-doc.dingtalk.com/doc#/serverapi2/eev437""" - # Read from cache first - try: - access_token = rs.execute_command(f"get ding_access_token") - except Exception as e: - logger.error(f"Failed to read DingTalk access_token from cache: {e}") - access_token = None - if access_token: - return access_token.decode() - # Request from DingTalk API - sys_config = SysConfig() - app_key = sys_config.get("ding_app_key") - app_secret = sys_config.get("ding_app_secret") - url = f"https://oapi.dingtalk.com/gettoken?appkey={app_key}&appsecret={app_secret}" - resp = requests.get(url, timeout=3).json() - if resp.get("errcode") == 0: - access_token = resp.get("access_token") - expires_in = resp.get("expires_in") - rs.execute_command(f"SETEX ding_access_token {expires_in-60} {access_token}") - return access_token - else: - logger.error(f"Failed to fetch DingTalk access_token: {resp}") - return None - - -def get_ding_user_id(username): - """Update user's ding_user_id.""" - try: - ding_user_id = rs.execute_command("GET {}".format(username.lower())) - if ding_user_id: - user = Users.objects.get(username=username) - if user.ding_user_id != str(ding_user_id, encoding="utf8"): - user.ding_user_id = str(ding_user_id, encoding="utf8") - user.save(update_fields=["ding_user_id"]) - except Exception as e: - logger.error(f"Failed to update user ding_user_id: {e}") - - -def get_dept_list_id_fetch_child(token, parent_dept_id): - """Get all child department IDs recursively.""" - ids = [int(parent_dept_id)] - url = ( - "https://oapi.dingtalk.com/department/list_ids?id={0}&access_token={1}".format( - parent_dept_id, token - ) - ) - resp = requests.get(url, timeout=3).json() - if resp.get("errcode") == 0: - for dept_id in resp.get("sub_dept_id_list"): - ids.extend(get_dept_list_id_fetch_child(token, dept_id)) - return list(set(ids)) - - -def sync_ding_user_id(): - """ - Archery users log in with employee ID (`username`), which maps to DingTalk's - `jobnumber` field. Use `jobnumber` to find and cache user `ding_user_id`. - """ - sys_config = SysConfig() - ding_dept_ids = sys_config.get("ding_dept_ids", "") - username2ding = sys_config.get("ding_archery_username") - token = get_access_token() - if not token: - return False - # Fetch all department IDs - sub_dept_id_list = [] - for dept_id in list(set(ding_dept_ids.split(","))): - sub_dept_id_list.extend(get_dept_list_id_fetch_child(token, dept_id)) - # Iterate users in each department - user_ids = [] - for sdi in sub_dept_id_list: - url = f"https://oapi.dingtalk.com/user/getDeptMember?access_token={token}&deptId={sdi}" - try: - resp = requests.get(url, timeout=3).json() - if resp.get("errcode") == 0: - user_ids.extend(resp.get("userIds")) - else: - raise Exception(f"Failed to fetch department users: {resp}") - except Exception as e: - raise Exception(f"Failed to fetch department users: {e}") - # Fetch user details and cache mappings - for user_id in list(set(user_ids)): - url = ( - f"https://oapi.dingtalk.com/user/get?access_token={token}&userid={user_id}" - ) - try: - resp = requests.get(url, timeout=3).json() - if resp.get("errcode") == 0: - if not resp.get(username2ding): - raise Exception( - f"DingTalk user payload does not include `{username2ding}`. " - f"Please check `ding_archery_username` config: {resp}" - ) - rs.execute_command( - f"SETEX {resp.get(username2ding).lower()} 86400 {resp.get('userid')}" - ) - else: - raise Exception(f"Failed to fetch user info: {resp}") - except Exception as e: - raise Exception(f"Failed to fetch user info: {e}") - return True - - -@superuser_required -def sync_ding_user(request): - """Trigger manual sync and also register daily schedule sync.""" - try: - # Add schedule and trigger sync - add_sync_ding_user_schedule() - return JsonResponse({"status": 0, "msg": "Sync triggered successfully"}) - except Exception as e: - return JsonResponse({"status": 1, "msg": f"Sync trigger failed: {e}"}) diff --git a/common/utils/sendmsg.py b/common/utils/sendmsg.py index 371162ec..84f9164d 100755 --- a/common/utils/sendmsg.py +++ b/common/utils/sendmsg.py @@ -11,7 +11,6 @@ from email.utils import formataddr from common.config import SysConfig -from common.utils.ding_api import get_access_token from common.utils.wx_api import get_wx_access_token from common.utils.feishu_api import * @@ -34,8 +33,6 @@ def __init__(self, **kwargs): self.MAIL_SSL = sys_config.get("mail_ssl") self.MAIL_REVIEW_FROM_ADDR = sys_config.get("mail_smtp_user") self.MAIL_REVIEW_FROM_PASSWORD = sys_config.get("mail_smtp_password") - # DingTalk settings - self.ding_agent_id = sys_config.get("ding_agent_id") # WeCom settings self.wx_agent_id = sys_config.get("wx_agent_id") # Feishu settings @@ -136,54 +133,6 @@ def send_email(self, subject, body, to, **kwargs): logger.error(errmsg) return errmsg - @staticmethod - def send_ding(url, content): - """ - Send DingTalk webhook message. - :param url: - :param content: - :return: - """ - data = { - "msgtype": "text", - "text": {"content": "{}".format(content)}, - } - r = requests.post(url=url, json=data) - r_json = r.json() - if r_json["errcode"] == 0: - logger.debug( - f"DingTalk webhook sent successfully\nTarget:{url}\nContent:{content}" - ) - else: - logger.error( - f"DingTalk webhook failed\nRequest url:{url}\nRequest data:{data}\nResponse:{r_json}" - ) - - def send_ding2user(self, userid_list, content): - """ - Send DingTalk message to specific users. - :param userid_list: - :param content: - :return: - """ - access_token = get_access_token() - send_url = f"https://oapi.dingtalk.com/topapi/message/corpconversation/asyncsend_v2?access_token={access_token}" - data = { - "userid_list": ",".join(list(set(userid_list))), - "agent_id": self.ding_agent_id, - "msg": {"msgtype": "text", "text": {"content": f"{content}"}}, - } - r = requests.post(url=send_url, json=data, timeout=5) - r_json = r.json() - if r_json["errcode"] == 0: - logger.debug( - f"DingTalk message sent successfully\nTargets:{userid_list}\nContent:{content}" - ) - else: - logger.error( - f"DingTalk message failed\nRequest url:{send_url}\nRequest data:{data}\nResponse:{r_json}" - ) - def send_wx2user(self, msg, user_list): if not user_list: logger.error("WeCom push failed: unable to determine target users.") diff --git a/conftest.py b/conftest.py index bf12eeab..7092a16a 100644 --- a/conftest.py +++ b/conftest.py @@ -1,5 +1,10 @@ +import os import datetime +os.environ.setdefault( + "FIELD_ENCRYPTION_KEYS", "9R_Jxat_be2SV-UbCS0dAYQ0SGjZVf0JyN-VPkVNyi0=" +) + import pytest from pytest_mock import MockFixture from django.contrib.auth.models import Group @@ -167,7 +172,6 @@ def create_resource_group(db): is_deleted=False, qywx_webhook="https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=xxx", feishu_webhook="https://open.feishu.cn/open-apis/bot/v2/hook/xxx", - ding_webhook="https://oapi.dingtalk.com/robot/send?access_token=xxx", ) yield resource_group resource_group.delete() diff --git a/mirage/__init__.py b/mirage/__init__.py new file mode 100644 index 00000000..18688ab8 --- /dev/null +++ b/mirage/__init__.py @@ -0,0 +1,3 @@ +from . import fields + +__all__ = ["fields"] diff --git a/mirage/fields.py b/mirage/fields.py new file mode 100644 index 00000000..fa1ab35a --- /dev/null +++ b/mirage/fields.py @@ -0,0 +1,12 @@ +from django.db import models + + +class EncryptedCharField(models.CharField): + pass + + +class EncryptedTextField(models.TextField): + pass + + +__all__ = ["EncryptedCharField", "EncryptedTextField"] diff --git a/requirements.txt b/requirements.txt index 2682be8f..c9ca99bd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,7 @@ aliyun-python-sdk-rds==2.1.1 cx-Oracle==7.3.0 supervisor==4.1.0 phoenixdb==1.2.1 -django-mirage-field==1.4.0 +cryptography>=45.0.0,<46.0.0 schema-sync==0.9.7 parsedatetime==2.4 sshtunnel==0.1.5 @@ -39,7 +39,6 @@ django-environ==0.8.1 alibabacloud_dysmsapi20170525==2.0.9 tencentcloud-sdk-python==3.0.656 mozilla-django-oidc==3.0.0 -django-auth-dingding==0.0.3 django-cas-ng==4.3.0 cassandra-driver httpx diff --git a/sql/admin.py b/sql/admin.py index c6e927ae..85f7028f 100755 --- a/sql/admin.py +++ b/sql/admin.py @@ -61,7 +61,6 @@ class UsersAdmin(UserAdmin): "fields": ( "display", "email", - "ding_user_id", "wx_user_id", "feishu_open_id", ) @@ -91,7 +90,6 @@ class UsersAdmin(UserAdmin): "fields": ( "display", "email", - "ding_user_id", "wx_user_id", "feishu_open_id", ) @@ -118,7 +116,7 @@ class UsersAdmin(UserAdmin): # User 2FA management @admin.register(TwoFactorAuthConfig) class TwoFactorAuthConfigAdmin(admin.ModelAdmin): - list_display = ("id", "username", "auth_type", "phone", "secret_key", "user_id") + list_display = ("id", "user", "auth_type", "phone", "secret_key", "user_id") # Resource group management @@ -127,7 +125,6 @@ class ResourceGroupAdmin(admin.ModelAdmin): list_display = ( "group_id", "group_name", - "ding_webhook", "feishu_webhook", "qywx_webhook", "is_deleted", diff --git a/sql/local_demo.py b/sql/local_demo.py index afa5912a..fb986ebd 100644 --- a/sql/local_demo.py +++ b/sql/local_demo.py @@ -291,7 +291,6 @@ def _seed_resource_groups(log): "group_sort": index, "group_level": 1, "is_deleted": 0, - "ding_webhook": "", "feishu_webhook": "", "qywx_webhook": "", }, diff --git a/sql/management/commands/generate_field_encryption_key.py b/sql/management/commands/generate_field_encryption_key.py new file mode 100644 index 00000000..88f58d14 --- /dev/null +++ b/sql/management/commands/generate_field_encryption_key.py @@ -0,0 +1,10 @@ +from django.core.management.base import BaseCommand + +from common.encryption import generate_field_encryption_key + + +class Command(BaseCommand): + help = "Generate a FIELD_ENCRYPTION_KEYS Fernet key." + + def handle(self, *args, **options): + self.stdout.write(generate_field_encryption_key()) diff --git a/sql/management/commands/reencrypt_sensitive_fields.py b/sql/management/commands/reencrypt_sensitive_fields.py new file mode 100644 index 00000000..69f555b2 --- /dev/null +++ b/sql/management/commands/reencrypt_sensitive_fields.py @@ -0,0 +1,46 @@ +from django.core.management.base import BaseCommand + +from sql.models import ( + CloudAccessKey, + Config, + Instance, + InstanceAccount, + Tunnel, + TwoFactorAuthConfig, +) + +MODEL_FIELDS = ( + (Instance, ("user", "password")), + (Tunnel, ("user", "password", "pkey", "pkey_password")), + (InstanceAccount, ("password",)), + (Config, ("value",)), + (TwoFactorAuthConfig, ("phone", "secret_key")), + (CloudAccessKey, ("key_id", "key_secret")), +) + + +class Command(BaseCommand): + help = "Rewrite sensitive fields into the current encrypted format." + + def add_arguments(self, parser): + parser.add_argument("--batch-size", type=int, default=500) + + def handle(self, *args, **options): + batch_size = options["batch_size"] + total_rows = 0 + + for model, fields in MODEL_FIELDS: + queryset = model.objects.order_by("pk").iterator(chunk_size=batch_size) + model_count = 0 + for obj in queryset: + update_fields = [ + field for field in fields if getattr(obj, field) not in (None, "") + ] + if not update_fields: + continue + obj.save(update_fields=update_fields) + model_count += 1 + total_rows += model_count + self.stdout.write(f"{model.__name__}: rewritten {model_count} rows") + + self.stdout.write(self.style.SUCCESS(f"Rewrote {total_rows} rows in total")) diff --git a/sql/migrations/0003_remove_resourcegroup_ding_webhook_and_more.py b/sql/migrations/0003_remove_resourcegroup_ding_webhook_and_more.py new file mode 100644 index 00000000..03cf87b4 --- /dev/null +++ b/sql/migrations/0003_remove_resourcegroup_ding_webhook_and_more.py @@ -0,0 +1,229 @@ +# Generated by Django 4.1.13 on 2026-04-02 22:13 + +import base64 +import common.fields.encrypted +from Crypto.Cipher import AES +from binascii import a2b_hex +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import padding +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes +from django.db import migrations, models +from django.utils.encoding import force_bytes, force_str + + +def _decrypt_mirage(value, secret_key=None, cipher_mode=None, cipher_iv=None): + from django.conf import settings + + key = secret_key or getattr(settings, "MIRAGE_SECRET_KEY", None) + if not key: + return None + cipher_key = base64.urlsafe_b64encode(force_bytes(key))[:32] + if len(cipher_key) not in (16, 24, 32): + return None + cipher_mode = ( + cipher_mode or getattr(settings, "MIRAGE_CIPHER_MODE", "ECB") + ).upper() + iv = force_bytes( + cipher_iv or getattr(settings, "MIRAGE_CIPHER_IV", "1234567890abcdef") + ) + if cipher_mode == "CBC": + mode = modes.CBC(iv) + else: + mode = modes.ECB() + cipher = Cipher(algorithms.AES(cipher_key), mode, default_backend()).decryptor() + unpadder = padding.PKCS7(algorithms.AES(cipher_key).block_size).unpadder() + plaintext = cipher.update(base64.urlsafe_b64decode(force_bytes(value))) + plaintext += cipher.finalize() + return force_str(unpadder.update(plaintext) + unpadder.finalize()) + + +def _decrypt_old_hex(value): + if len(value) < 32 or len(value) % 2 != 0: + return None + + try: + cryptor = AES.new( + b"eCcGFZQj6PNoSSma31LR39rTzTbLkU8E", AES.MODE_CBC, b"0000000000000000" + ) + plain_text = cryptor.decrypt(a2b_hex(value)) + return plain_text.decode().rstrip("\0") + except Exception: + return None + + +def _decrypt_legacy_value(value, secret_key, cipher_mode=None, cipher_iv=None): + if value in ("", None): + return value + + for decryptor in ( + lambda ciphertext: _decrypt_mirage( + ciphertext, secret_key, cipher_mode, cipher_iv + ), + _decrypt_old_hex, + ): + try: + plaintext = decryptor(value) + except Exception: + plaintext = None + if plaintext is not None: + return plaintext + + raise ValueError( + "Unable to decrypt legacy InstanceAccount.user value " + f"marker={value[:16]!r} length={len(value)}" + ) + + +def decrypt_instanceaccount_user(apps, schema_editor): + InstanceAccount = apps.get_model("sql", "InstanceAccount") + from django.conf import settings + + secret_key = getattr(settings, "MIRAGE_SECRET_KEY", None) or settings.SECRET_KEY + cipher_mode = getattr(settings, "MIRAGE_CIPHER_MODE", "ECB") + cipher_iv = getattr(settings, "MIRAGE_CIPHER_IV", "1234567890abcdef") + + for account in InstanceAccount.objects.using(schema_editor.connection.alias).all(): + if account.user in ("", None): + continue + plaintext = _decrypt_legacy_value( + account.user, secret_key, cipher_mode, cipher_iv + ) + InstanceAccount.objects.using(schema_editor.connection.alias).filter( + pk=account.pk + ).update(user=plaintext) + + +def noop_reverse(apps, schema_editor): + return + + +class Migration(migrations.Migration): + + dependencies = [ + ("sql", "0002_permissionrequest_alter_workflowaudit_workflow_type_and_more"), + ] + + operations = [ + migrations.RemoveField( + model_name="resourcegroup", + name="ding_webhook", + ), + migrations.RemoveField( + model_name="twofactorauthconfig", + name="username", + ), + migrations.RemoveField( + model_name="users", + name="ding_user_id", + ), + migrations.AlterField( + model_name="cloudaccesskey", + name="key_id", + field=common.fields.encrypted.EncryptedCharField(max_length=200), + ), + migrations.AlterField( + model_name="cloudaccesskey", + name="key_secret", + field=common.fields.encrypted.EncryptedCharField(max_length=200), + ), + migrations.AlterField( + model_name="config", + name="value", + field=common.fields.encrypted.EncryptedCharField( + max_length=500, verbose_name="Config Value" + ), + ), + migrations.AlterField( + model_name="instance", + name="password", + field=common.fields.encrypted.EncryptedCharField( + blank=True, default="", max_length=300, verbose_name="Password" + ), + ), + migrations.AlterField( + model_name="instance", + name="user", + field=common.fields.encrypted.EncryptedCharField( + blank=True, default="", max_length=200, verbose_name="Username" + ), + ), + migrations.AlterField( + model_name="instanceaccount", + name="password", + field=common.fields.encrypted.EncryptedCharField( + blank=True, default="", max_length=128, verbose_name="Password" + ), + ), + migrations.RunPython(decrypt_instanceaccount_user, noop_reverse), + migrations.AlterField( + model_name="instanceaccount", + name="user", + field=models.CharField(max_length=128, verbose_name="Account"), + ), + migrations.AlterField( + model_name="tunnel", + name="password", + field=common.fields.encrypted.EncryptedCharField( + blank=True, + default="", + max_length=300, + null=True, + verbose_name="Password", + ), + ), + migrations.AlterField( + model_name="tunnel", + name="pkey", + field=common.fields.encrypted.EncryptedTextField( + blank=True, null=True, verbose_name="Private Key" + ), + ), + migrations.AlterField( + model_name="tunnel", + name="pkey_password", + field=common.fields.encrypted.EncryptedCharField( + blank=True, + default="", + max_length=300, + null=True, + verbose_name="Key Passphrase", + ), + ), + migrations.AlterField( + model_name="tunnel", + name="user", + field=common.fields.encrypted.EncryptedCharField( + blank=True, + default="", + max_length=200, + null=True, + verbose_name="Username", + ), + ), + migrations.AlterField( + model_name="twofactorauthconfig", + name="auth_type", + field=models.CharField( + choices=[ + ("totp", "Google Authenticator"), + ("sms", "SMS Verification Code"), + ], + max_length=128, + verbose_name="Authentication Type", + ), + ), + migrations.AlterField( + model_name="twofactorauthconfig", + name="phone", + field=common.fields.encrypted.EncryptedCharField( + default="", max_length=64, null=True, verbose_name="Phone Number" + ), + ), + migrations.AlterField( + model_name="twofactorauthconfig", + name="secret_key", + field=common.fields.encrypted.EncryptedCharField( + max_length=256, null=True, verbose_name="User Secret" + ), + ), + ] diff --git a/sql/models.py b/sql/models.py index 6965b823..2e9b043e 100755 --- a/sql/models.py +++ b/sql/models.py @@ -5,11 +5,10 @@ from django.db import models from django.contrib.auth.models import AbstractUser -from mirage import fields from django.utils.translation import gettext as _ from django.conf import settings -from mirage.crypto import Crypto +from common.fields import EncryptedCharField, EncryptedTextField from common.utils.const import WorkflowStatus, WorkflowType, WorkflowAction logger = logging.getLogger("default") @@ -38,7 +37,6 @@ class ResourceGroup(models.Model): group_parent_id = models.BigIntegerField("Parent ID", default=0) group_sort = models.IntegerField("Sort Order", default=1) group_level = models.IntegerField("Level", default=1) - ding_webhook = models.CharField("DingTalk webhook URL", max_length=255, blank=True) feishu_webhook = models.CharField("Feishu webhook URL", max_length=255, blank=True) qywx_webhook = models.CharField("WeCom webhook URL", max_length=255, blank=True) is_deleted = models.IntegerField( @@ -63,7 +61,6 @@ class Users(AbstractUser): """ display = models.CharField("Display Name", max_length=50, default="") - ding_user_id = models.CharField("DingTalk User ID", max_length=64, blank=True) wx_user_id = models.CharField("WeCom User ID", max_length=64, blank=True) feishu_open_id = models.CharField("Feishu Open ID", max_length=64, blank=True) failed_login_count = models.IntegerField("Failed Login Count", default=0) @@ -101,14 +98,13 @@ class TwoFactorAuthConfig(models.Model): ("sms", "SMS Verification Code"), ) - username = fields.EncryptedCharField(verbose_name="Username", max_length=200) - auth_type = fields.EncryptedCharField( + auth_type = models.CharField( verbose_name="Authentication Type", max_length=128, choices=auth_type_choice ) - phone = fields.EncryptedCharField( + phone = EncryptedCharField( verbose_name="Phone Number", max_length=64, null=True, default="" ) - secret_key = fields.EncryptedCharField( + secret_key = EncryptedCharField( verbose_name="User Secret", max_length=256, null=True ) user = models.ForeignKey(Users, on_delete=models.CASCADE) @@ -166,17 +162,17 @@ class Tunnel(models.Model): tunnel_name = models.CharField("Tunnel Name", max_length=50, unique=True) host = models.CharField("Tunnel Host", max_length=200) port = models.IntegerField("Port", default=0) - user = fields.EncryptedCharField( + user = EncryptedCharField( verbose_name="Username", max_length=200, default="", blank=True, null=True ) - password = fields.EncryptedCharField( + password = EncryptedCharField( verbose_name="Password", max_length=300, default="", blank=True, null=True ) - pkey = fields.EncryptedTextField(verbose_name="Private Key", blank=True, null=True) + pkey = EncryptedTextField(verbose_name="Private Key", blank=True, null=True) pkey_path = models.FileField( verbose_name="Key File Path", blank=True, null=True, upload_to="keys/" ) - pkey_password = fields.EncryptedCharField( + pkey_password = EncryptedCharField( verbose_name="Key Passphrase", max_length=300, default="", blank=True, null=True ) create_time = models.DateTimeField("Created Time", auto_now_add=True) @@ -219,10 +215,10 @@ class Instance(models.Model, PasswordMixin): ) host = models.CharField("Instance Host", max_length=200) port = models.IntegerField("Port", default=0) - user = fields.EncryptedCharField( + user = EncryptedCharField( verbose_name="Username", max_length=200, default="", blank=True ) - password = fields.EncryptedCharField( + password = EncryptedCharField( verbose_name="Password", max_length=300, default="", blank=True ) is_ssl = models.BooleanField("Enable SSL", default=False) @@ -862,14 +858,14 @@ class InstanceAccount(models.Model): """ instance = models.ForeignKey(Instance, on_delete=models.CASCADE) - user = fields.EncryptedCharField(verbose_name="Account", max_length=128) + user = models.CharField(verbose_name="Account", max_length=128) host = models.CharField( verbose_name="Host", max_length=64 ) # MySQL stores host info here. db_name = models.CharField( verbose_name="Database Name", max_length=128 ) # MongoDB stores database name here. - password = fields.EncryptedCharField( + password = EncryptedCharField( verbose_name="Password", max_length=128, default="", blank=True ) remark = models.CharField("Remark", max_length=255) @@ -1054,7 +1050,7 @@ class Config(models.Model): """ item = models.CharField("Config Item", max_length=100, unique=True) - value = fields.EncryptedCharField(verbose_name="Config Value", max_length=500) + value = EncryptedCharField(verbose_name="Config Value", max_length=500) description = models.CharField( "Description", max_length=200, default="", blank=True ) @@ -1071,28 +1067,19 @@ class CloudAccessKey(models.Model): cloud_type_choices = (("aliyun", "aliyun"),) type = models.CharField(max_length=20, default="", choices=cloud_type_choices) - key_id = models.CharField(max_length=200) - key_secret = models.CharField(max_length=200) + key_id = EncryptedCharField(max_length=200) + key_secret = EncryptedCharField(max_length=200) remark = models.CharField(max_length=50, default="", blank=True) - def __init__(self, *args, **kwargs): - self.c = Crypto() - super().__init__(*args, **kwargs) - @property def raw_key_id(self): """Return key ID in plaintext.""" - return self.c.decrypt(self.key_id) + return self.key_id @property def raw_key_secret(self): """Return key secret in plaintext.""" - return self.c.decrypt(self.key_secret) - - def save(self, *args, **kwargs): - self.key_id = self.c.encrypt(self.key_id) - self.key_secret = self.c.encrypt(self.key_secret) - super(CloudAccessKey, self).save(*args, **kwargs) + return self.key_secret def __str__(self): return f"{self.type}({self.remark})" diff --git a/sql/notify.py b/sql/notify.py index f0714645..cbcebf61 100755 --- a/sql/notify.py +++ b/sql/notify.py @@ -379,38 +379,6 @@ def render(self): self.render_m2sql() -class DingdingWebhookNotifier(LegacyRender): - name = "dingding_webhook" - sys_config_key: str = "ding" - - def send(self): - dingding_webhook = ResourceGroup.objects.get( - group_id=self.audit.group_id - ).ding_webhook - if not dingding_webhook: - return - msg_sender = MsgSender() - for m in self.messages: - msg_sender.send_ding(dingding_webhook, f"{m.msg_title}\n{m.msg_content}") - - -class DingdingPersonNotifier(LegacyRender): - name = "ding_to_person" - sys_config_key: str = "ding_to_person" - - def send(self): - msg_sender = MsgSender() - for m in self.messages: - ding_user_id_list = [ - user.ding_user_id - for user in chain(m.msg_to, m.msg_cc) - if user.ding_user_id - ] - msg_sender.send_ding2user( - ding_user_id_list, f"{m.msg_title}\n{m.msg_content}" - ) - - class FeishuWebhookNotifier(LegacyRender): name = "feishu_webhook" sys_config_key: str = "feishu_webhook" diff --git a/sql/resource_group.py b/sql/resource_group.py index 7891b09b..9bda3ad6 100644 --- a/sql/resource_group.py +++ b/sql/resource_group.py @@ -28,9 +28,7 @@ def group(request): # Filter search conditions. group_obj = ResourceGroup.objects.filter(group_name__icontains=search, is_deleted=0) group_count = group_obj.count() - group_list = group_obj[offset:limit].values( - "group_id", "group_name", "ding_webhook" - ) + group_list = group_obj[offset:limit].values("group_id", "group_name") # Serialize QuerySet. rows = [row for row in group_list] diff --git a/sql/test_notify.py b/sql/test_notify.py index a0138f3a..756f15b1 100644 --- a/sql/test_notify.py +++ b/sql/test_notify.py @@ -25,8 +25,6 @@ LegacyRender, GenericWebhookNotifier, My2SqlResult, - DingdingWebhookNotifier, - DingdingPersonNotifier, FeishuPersonNotifier, FeishuWebhookNotifier, QywxWebhookNotifier, @@ -144,7 +142,7 @@ def setUp(self): remark="Test query note", ) - self.rs = ResourceGroup.objects.create(group_id=1, ding_webhook="url") + self.rs = ResourceGroup.objects.create(group_id=1) self.archive_apply = ArchiveConfig.objects.create( title="Test archive", @@ -505,8 +503,6 @@ def test_general_webhook(self): @pytest.mark.parametrize( "notifier_to_test,method_assert_called", [ - (DingdingWebhookNotifier, "send_ding"), - (DingdingPersonNotifier, "send_ding2user"), (FeishuWebhookNotifier, "send_feishu_webhook"), (FeishuPersonNotifier, "send_feishu_user"), (QywxWebhookNotifier, "send_qywx_webhook"), diff --git a/sql/test_reencrypt_sensitive_fields.py b/sql/test_reencrypt_sensitive_fields.py new file mode 100644 index 00000000..0d0338c4 --- /dev/null +++ b/sql/test_reencrypt_sensitive_fields.py @@ -0,0 +1,116 @@ +import importlib + +from django.core.management import call_command +from django.db import connection +from django.test import SimpleTestCase, TestCase, override_settings + +from common.encryption import ( + ENCRYPTED_VALUE_PREFIX, + generate_field_encryption_key, + get_multi_fernet, +) +from common.test_fixtures import ( + LEGACY_MIRAGE_CBC_CIPHERTEXTS, + LEGACY_MIRAGE_CBC_IV, + LEGACY_MIRAGE_CIPHERTEXTS, + LEGACY_MIRAGE_SECRET_KEY, +) +from sql.models import CloudAccessKey, Instance + + +def _encrypt_legacy_mirage(value, secret_key): + # Pinned ciphertexts captured from the legacy Mirage wire format. + assert secret_key == LEGACY_MIRAGE_SECRET_KEY + return LEGACY_MIRAGE_CIPHERTEXTS[value] + + +@override_settings( + FIELD_ENCRYPTION_KEYS=generate_field_encryption_key(), + SECRET_KEY=LEGACY_MIRAGE_SECRET_KEY, +) +class ReencryptSensitiveFieldsCommandTest(TestCase): + def setUp(self): + get_multi_fernet.cache_clear() + self.instance = Instance.objects.create( + instance_name="legacy-instance", + type="master", + db_type="mysql", + host="127.0.0.1", + port=3306, + user="root", + password="plain-password", + ) + self.access_key = CloudAccessKey.objects.create( + type="aliyun", + key_id="plain-ak", + key_secret="plain-sk", + ) + + legacy_user = _encrypt_legacy_mirage("legacy-root", LEGACY_MIRAGE_SECRET_KEY) + legacy_password = _encrypt_legacy_mirage( + "legacy-password", LEGACY_MIRAGE_SECRET_KEY + ) + legacy_key_id = _encrypt_legacy_mirage("legacy-ak", LEGACY_MIRAGE_SECRET_KEY) + legacy_key_secret = _encrypt_legacy_mirage( + "legacy-sk", LEGACY_MIRAGE_SECRET_KEY + ) + + with connection.cursor() as cursor: + cursor.execute( + "UPDATE sql_instance SET user=%s, password=%s WHERE id=%s", + [legacy_user, legacy_password, self.instance.id], + ) + cursor.execute( + "UPDATE cloud_access_key SET key_id=%s, key_secret=%s WHERE id=%s", + [legacy_key_id, legacy_key_secret, self.access_key.id], + ) + + def tearDown(self): + get_multi_fernet.cache_clear() + + def test_command_rewrites_legacy_values_to_new_prefix(self): + call_command("reencrypt_sensitive_fields", batch_size=10) + + with connection.cursor() as cursor: + cursor.execute( + "SELECT user, password FROM sql_instance WHERE id=%s", + [self.instance.id], + ) + raw_user, raw_password = cursor.fetchone() + cursor.execute( + "SELECT key_id, key_secret FROM cloud_access_key WHERE id=%s", + [self.access_key.id], + ) + raw_key_id, raw_key_secret = cursor.fetchone() + + self.assertTrue(raw_user.startswith(ENCRYPTED_VALUE_PREFIX)) + self.assertTrue(raw_password.startswith(ENCRYPTED_VALUE_PREFIX)) + self.assertTrue(raw_key_id.startswith(ENCRYPTED_VALUE_PREFIX)) + self.assertTrue(raw_key_secret.startswith(ENCRYPTED_VALUE_PREFIX)) + + self.instance.refresh_from_db() + self.access_key.refresh_from_db() + + self.assertEqual(self.instance.user, "legacy-root") + self.assertEqual(self.instance.password, "legacy-password") + self.assertEqual(self.access_key.key_id, "legacy-ak") + self.assertEqual(self.access_key.key_secret, "legacy-sk") + + +MIGRATION_0003 = importlib.import_module( + "sql.migrations.0003_remove_resourcegroup_ding_webhook_and_more" +) + + +class Migration0003MirageDecryptTest(SimpleTestCase): + @override_settings( + MIRAGE_SECRET_KEY=LEGACY_MIRAGE_SECRET_KEY, + MIRAGE_CIPHER_MODE="CBC", + MIRAGE_CIPHER_IV=LEGACY_MIRAGE_CBC_IV, + ) + def test_decrypt_mirage_supports_configured_cbc_mode(self): + plaintext = MIGRATION_0003._decrypt_mirage( + LEGACY_MIRAGE_CBC_CIPHERTEXTS["legacy-cbc-user"] + ) + + self.assertEqual(plaintext, "legacy-cbc-user") diff --git a/sql/urls.py b/sql/urls.py index 66142fd6..20200643 100644 --- a/sql/urls.py +++ b/sql/urls.py @@ -26,7 +26,6 @@ offlinedownload, ) from sql.utils import tasks -from common.utils import ding_api urlpatterns = [ path("", views.index), @@ -163,7 +162,6 @@ path("archive/switch/", archiver.archive_switch), path("archive/once/", archiver.archive_once), path("archive/log/", archiver.archive_log), - path("4admin/sync_ding_user/", ding_api.sync_ding_user), path("audit/log/", audit_log.audit_log), path("audit/input/", audit_log.audit_input), path("user/list/", user.lists), diff --git a/sql/utils/tasks.py b/sql/utils/tasks.py index 6e2fb9c6..4fdceb13 100644 --- a/sql/utils/tasks.py +++ b/sql/utils/tasks.py @@ -41,18 +41,6 @@ def add_kill_conn_schedule(name, run_date, instance_id, thread_id): ) -def add_sync_ding_user_schedule(): - """Add a scheduled task to sync DingTalk user IDs.""" - del_schedule(name="Sync DingTalk User IDs") - schedule( - "common.utils.ding_api.sync_ding_user_id", - name="Sync DingTalk User IDs", - schedule_type="D", - repeats=-1, - timeout=-1, - ) - - def del_schedule(name): """Delete a schedule.""" try: diff --git a/sql/views.py b/sql/views.py index 8332c203..b73ba14c 100644 --- a/sql/views.py +++ b/sql/views.py @@ -64,7 +64,6 @@ def login(request): context={ "sign_up_enabled": SysConfig().get("sign_up_enabled"), "oidc_enabled": settings.ENABLE_OIDC, - "dingding_enabled": settings.ENABLE_DINGDING, "cas_enabled": settings.ENABLE_CAS, "oidc_btn_name": SysConfig().get("oidc_btn_name", "Log in with OIDC"), }, @@ -79,7 +78,7 @@ def twofa(request): username = request.session.get("user") if username: verify_mode = request.session.get("verify_mode") - twofa_enabled = TwoFactorAuthConfig.objects.filter(username=username) + twofa_enabled = TwoFactorAuthConfig.objects.filter(user__username=username) user_auth_types = [twofa.auth_type for twofa in twofa_enabled] auth_types = [] @@ -93,7 +92,7 @@ def twofa(request): auth_types.append(auth_type) if "sms" in user_auth_types: phone = TwoFactorAuthConfig.objects.get( - username=username, auth_type="sms" + user__username=username, auth_type="sms" ).phone else: phone = 0 diff --git a/sql_api/serializers.py b/sql_api/serializers.py index 556a6c2c..85e8f4b7 100644 --- a/sql_api/serializers.py +++ b/sql_api/serializers.py @@ -784,6 +784,10 @@ class CloudAccessKeySerializer(serializers.ModelSerializer): class Meta: model = CloudAccessKey fields = "__all__" + extra_kwargs = { + "key_id": {"write_only": True}, + "key_secret": {"write_only": True}, + } class AliyunRdsSerializer(serializers.ModelSerializer): diff --git a/sql_api/tests.py b/sql_api/tests.py index b70dd463..f7c524dd 100644 --- a/sql_api/tests.py +++ b/sql_api/tests.py @@ -663,7 +663,6 @@ def tearDown(self): def test_token_requires_2fa_when_user_has_totp(self): secret = pyotp.random_base32(32) TwoFactorAuthConfig.objects.create( - username=self.user.username, auth_type="totp", secret_key=secret, user=self.user, @@ -680,7 +679,6 @@ def test_token_requires_2fa_when_user_has_totp(self): def test_token_totp_success(self): secret = pyotp.random_base32(32) TwoFactorAuthConfig.objects.create( - username=self.user.username, auth_type="totp", secret_key=secret, user=self.user, @@ -784,7 +782,6 @@ def test_request_sms_login_otp_success( self, mock_get_authenticator, mock_get_redis ): TwoFactorAuthConfig.objects.create( - username=self.user.username, auth_type="sms", phone="13800138000", user=self.user, diff --git a/sql_api/urls.py b/sql_api/urls.py index 3115965f..957a3381 100644 --- a/sql_api/urls.py +++ b/sql_api/urls.py @@ -176,5 +176,4 @@ ), path("info", views.info), path("debug", views.debug), - path("do_once/mirage", views.mirage), ] diff --git a/sql_api/views.py b/sql_api/views.py index 80c956d6..0cb63325 100644 --- a/sql_api/views.py +++ b/sql_api/views.py @@ -14,11 +14,8 @@ from django_q.brokers import get_broker from django.utils import timezone -from common.utils.aes_decryptor import Prpcrypt from common.utils.permission import superuser_required import archery -from sql.models import Instance -from mirage.tools import Migrator def info(request): @@ -177,7 +174,6 @@ def debug(request): # Mask sensitive information. secret_keys = [ "inception_remote_backup_password", - "ding_app_secret", "feishu_app_secret", "mail_smtp_password", "go_inception_password", @@ -206,27 +202,3 @@ def debug(request): "packages": installed_packages_list, } return JsonResponse(system_info) - - -@superuser_required -def mirage(request): - """Migrate encrypted Instance data. Remove after enough versions.""" - try: - pc = Prpcrypt() - mg_user = Migrator(app="sql", model="Instance", field="user") - mg_password = Migrator(app="sql", model="Instance", field="password") - # Restore password first. - for ins in Instance.objects.all(): - # Ignore records that cannot be decrypted (already invalid data). - try: - Instance(pk=ins.pk, password=pc.decrypt(ins.password)).save( - update_fields=["password"] - ) - except: - pass - # Re-encrypt with django-mirage-field. - mg_user.encrypt() - mg_password.encrypt() - return JsonResponse({"msg": "ok"}) - except Exception as msg: - return JsonResponse({"msg": f"{msg}"}) diff --git a/src/docker-compose/.env b/src/docker-compose/.env index ae4910dc..f8db9829 100644 --- a/src/docker-compose/.env +++ b/src/docker-compose/.env @@ -3,6 +3,7 @@ NGINX_PORT=9123 # https://django-environ.readthedocs.io/en/latest/quickstart.html#usage # https://docs.djangoproject.com/zh-hans/4.1/ref/settings/ DEBUG=false +FIELD_ENCRYPTION_KEYS=9R_Jxat_be2SV-UbCS0dAYQ0SGjZVf0JyN-VPkVNyi0= DATABASE_URL=mysql://root:123456@mysql:3306/archery CACHE_URL=redis://redis:6379/0?PASSWORD=123456 @@ -21,4 +22,3 @@ Q_CLUISTER_SYNC=false # 在网站标题及登录页面追加此内容, 可用于多archery实例的区分。Archery后台也有相同配置，如都做了配置，以后台配置为准 CUSTOM_TITLE_SUFFIX="" - diff --git a/src/docker-compose/.env.local-arm b/src/docker-compose/.env.local-arm index 786420de..93a5aa4e 100644 --- a/src/docker-compose/.env.local-arm +++ b/src/docker-compose/.env.local-arm @@ -2,6 +2,7 @@ NGINX_PORT=9123 DEBUG=false SECRET_KEY=datamingle-local-secret-key-change-me +FIELD_ENCRYPTION_KEYS=9R_Jxat_be2SV-UbCS0dAYQ0SGjZVf0JyN-VPkVNyi0= DATABASE_URL=mysql://root:123456@mysql:3306/archery CACHE_URL=redis://redis:6379/0?PASSWORD=123456 CSRF_TRUSTED_ORIGINS=http://127.0.0.1:9123,http://localhost:9123