From d95dcb50a9d6d3716b1591eb63020127412b122e Mon Sep 17 00:00:00 2001 From: Julien Richard Date: Tue, 28 May 2024 18:22:50 +0200 Subject: [PATCH 01/17] [client] Limit stix_ids explosion by rewriting the standard_id in client python (#659) --- pycti/connector/opencti_connector_helper.py | 13 ++ pycti/entities/opencti_attack_pattern.py | 7 +- pycti/entities/opencti_campaign.py | 4 + pycti/entities/opencti_case_incident.py | 4 + pycti/entities/opencti_case_rfi.py | 4 + pycti/entities/opencti_case_rft.py | 4 + pycti/entities/opencti_channel.py | 4 + pycti/entities/opencti_course_of_action.py | 7 +- pycti/entities/opencti_data_component.py | 4 + pycti/entities/opencti_data_source.py | 4 + pycti/entities/opencti_event.py | 4 + pycti/entities/opencti_external_reference.py | 6 + pycti/entities/opencti_feedback.py | 4 + pycti/entities/opencti_grouping.py | 4 + pycti/entities/opencti_identity.py | 7 +- pycti/entities/opencti_incident.py | 4 + pycti/entities/opencti_indicator.py | 4 + pycti/entities/opencti_infrastructure.py | 4 + pycti/entities/opencti_intrusion_set.py | 4 + pycti/entities/opencti_kill_chain_phase.py | 4 + pycti/entities/opencti_language.py | 4 + pycti/entities/opencti_location.py | 26 ++- pycti/entities/opencti_malware.py | 4 + pycti/entities/opencti_malware_analysis.py | 12 +- pycti/entities/opencti_marking_definition.py | 6 + pycti/entities/opencti_narrative.py | 4 + pycti/entities/opencti_note.py | 16 +- pycti/entities/opencti_observed_data.py | 4 + pycti/entities/opencti_opinion.py | 22 ++- pycti/entities/opencti_report.py | 4 + pycti/entities/opencti_task.py | 7 +- pycti/entities/opencti_threat_actor.py | 13 +- pycti/entities/opencti_threat_actor_group.py | 4 + .../opencti_threat_actor_individual.py | 4 + pycti/entities/opencti_tool.py | 4 + pycti/entities/opencti_vocabulary.py | 15 ++ pycti/entities/opencti_vulnerability.py | 4 + pycti/utils/opencti_stix2.py | 150 ++++++++++++------ tests/01-unit/stix/__init__.py | 0 tests/01-unit/stix/test_bundle_ids_rewrite.py | 136 ++++++++++++++++ tests/data/bundle_ids_sample.json | 76 +++++++++ 41 files changed, 544 insertions(+), 71 deletions(-) create mode 100644 tests/01-unit/stix/__init__.py create mode 100644 tests/01-unit/stix/test_bundle_ids_rewrite.py create mode 100644 tests/data/bundle_ids_sample.json diff --git a/pycti/connector/opencti_connector_helper.py b/pycti/connector/opencti_connector_helper.py index 4ebdb36a0..b3657cf2b 100644 --- a/pycti/connector/opencti_connector_helper.py +++ b/pycti/connector/opencti_connector_helper.py @@ -849,6 +849,13 @@ def __init__(self, config: Dict, playbook_compatible=False) -> None: False, True, ) + self.keep_original_id = get_config_variable( + "CONNECTOR_KEEP_ORIGINAL_ID", + ["connector", "keep_original_id"], + config, + False, + False, + ) self.bundle_send_to_directory = get_config_variable( "CONNECTOR_SEND_TO_DIRECTORY", ["connector", "send_to_directory"], @@ -1571,6 +1578,7 @@ def send_stix2_bundle(self, bundle: str, **kwargs) -> list: file_name = kwargs.get("file_name", None) bundle_send_to_queue = kwargs.get("send_to_queue", self.bundle_send_to_queue) cleanup_inconsistent_bundle = kwargs.get("cleanup_inconsistent_bundle", False) + keep_original_id = kwargs.get("keep_original_id", self.keep_original_id) bundle_send_to_directory = kwargs.get( "send_to_directory", self.bundle_send_to_directory ) @@ -1581,6 +1589,11 @@ def send_stix2_bundle(self, bundle: str, **kwargs) -> list: "send_to_directory_retention", self.bundle_send_to_directory_retention ) + # Bundle ids must be rewritten + bundle = self.api.stix2.prepare_bundle_ids( + bundle=bundle, use_json=True, keep_original_id=keep_original_id + ) + # In case of enrichment ingestion, ensure the sharing if needed if self.enrichment_shared_organizations is not None: # Every element of the bundle must be enriched with the same organizations diff --git a/pycti/entities/opencti_attack_pattern.py b/pycti/entities/opencti_attack_pattern.py index 7f549b5b0..cce8dcfd1 100644 --- a/pycti/entities/opencti_attack_pattern.py +++ b/pycti/entities/opencti_attack_pattern.py @@ -222,15 +222,18 @@ def __init__(self, opencti): @staticmethod def generate_id(name, x_mitre_id=None): - name = name.lower().strip() if x_mitre_id is not None: data = {"x_mitre_id": x_mitre_id} else: - data = {"name": name} + data = {"name": name.lower().strip()} data = canonicalize(data, utf8=False) id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data)) return "attack-pattern--" + id + @staticmethod + def generate_id_from_data(data): + return AttackPattern.generate_id(data.get("name"), data.get("x_mitre_id")) + """ List Attack-Pattern objects diff --git a/pycti/entities/opencti_campaign.py b/pycti/entities/opencti_campaign.py index 3070a4805..f3212de8f 100644 --- a/pycti/entities/opencti_campaign.py +++ b/pycti/entities/opencti_campaign.py @@ -216,6 +216,10 @@ def generate_id(name): id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data)) return "campaign--" + id + @staticmethod + def generate_id_from_data(data): + return Campaign.generate_id(data["name"]) + """ List Campaign objects diff --git a/pycti/entities/opencti_case_incident.py b/pycti/entities/opencti_case_incident.py index e1969c22e..40cc75ba9 100644 --- a/pycti/entities/opencti_case_incident.py +++ b/pycti/entities/opencti_case_incident.py @@ -461,6 +461,10 @@ def generate_id(name, created): id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data)) return "case-incident--" + id + @staticmethod + def generate_id_from_data(data): + return CaseIncident.generate_id(data["name"], data["created"]) + """ List Case Incident objects diff --git a/pycti/entities/opencti_case_rfi.py b/pycti/entities/opencti_case_rfi.py index 62df2817a..afa4b5be6 100644 --- a/pycti/entities/opencti_case_rfi.py +++ b/pycti/entities/opencti_case_rfi.py @@ -457,6 +457,10 @@ def generate_id(name, created): id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data)) return "case-rfi--" + id + @staticmethod + def generate_id_from_data(data): + return CaseRfi.generate_id(data["name"], data["created"]) + """ List Case Rfi objects diff --git a/pycti/entities/opencti_case_rft.py b/pycti/entities/opencti_case_rft.py index 50969bf46..dddf601a3 100644 --- a/pycti/entities/opencti_case_rft.py +++ b/pycti/entities/opencti_case_rft.py @@ -457,6 +457,10 @@ def generate_id(name, created): id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data)) return "case-rft--" + id + @staticmethod + def generate_id_from_data(data): + return CaseRft.generate_id(data["name"], data["created"]) + """ List Case Rft objects diff --git a/pycti/entities/opencti_channel.py b/pycti/entities/opencti_channel.py index dd6f8f095..9fc1b280e 100644 --- a/pycti/entities/opencti_channel.py +++ b/pycti/entities/opencti_channel.py @@ -212,6 +212,10 @@ def generate_id(name): id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data)) return "channel--" + id + @staticmethod + def generate_id_from_data(data): + return Channel.generate_id(data["name"]) + """ List Channel objects diff --git a/pycti/entities/opencti_course_of_action.py b/pycti/entities/opencti_course_of_action.py index 43be22055..af65bbbb2 100644 --- a/pycti/entities/opencti_course_of_action.py +++ b/pycti/entities/opencti_course_of_action.py @@ -196,15 +196,18 @@ def __init__(self, opencti): @staticmethod def generate_id(name, x_mitre_id=None): - name = name.lower().strip() if x_mitre_id is not None: data = {"x_mitre_id": x_mitre_id} else: - data = {"name": name} + data = {"name": name.lower().strip()} data = canonicalize(data, utf8=False) id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data)) return "course-of-action--" + id + @staticmethod + def generate_id_from_data(data): + return CourseOfAction.generate_id(data.get("name"), data.get("x_mitre_id")) + """ List Course-Of-Action objects diff --git a/pycti/entities/opencti_data_component.py b/pycti/entities/opencti_data_component.py index a675c5edf..c34250d6c 100644 --- a/pycti/entities/opencti_data_component.py +++ b/pycti/entities/opencti_data_component.py @@ -246,6 +246,10 @@ def generate_id(name): id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data)) return "data-component--" + id + @staticmethod + def generate_id_from_data(data): + return DataComponent.generate_id(data["name"]) + """ List Data-Component objects diff --git a/pycti/entities/opencti_data_source.py b/pycti/entities/opencti_data_source.py index 8ef6b8a04..3935a5074 100644 --- a/pycti/entities/opencti_data_source.py +++ b/pycti/entities/opencti_data_source.py @@ -204,6 +204,10 @@ def generate_id(name): id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data)) return "data-source--" + id + @staticmethod + def generate_id_from_data(data): + return DataSource.generate_id(data["name"]) + """ List Data-Source objects diff --git a/pycti/entities/opencti_event.py b/pycti/entities/opencti_event.py index cf5824a4c..c07fcad06 100644 --- a/pycti/entities/opencti_event.py +++ b/pycti/entities/opencti_event.py @@ -216,6 +216,10 @@ def generate_id(name): id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data)) return "event--" + id + @staticmethod + def generate_id_from_data(data): + return Event.generate_id(data["name"]) + """ List Event objects diff --git a/pycti/entities/opencti_external_reference.py b/pycti/entities/opencti_external_reference.py index f8590313a..a81f828b8 100644 --- a/pycti/entities/opencti_external_reference.py +++ b/pycti/entities/opencti_external_reference.py @@ -68,6 +68,12 @@ def generate_id(url=None, source_name=None, external_id=None): id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data)) return "external-reference--" + id + @staticmethod + def generate_id_from_data(data): + return ExternalReference.generate_id( + data.get("url"), data.get("source_name"), data.get("external_id") + ) + """ List External-Reference objects diff --git a/pycti/entities/opencti_feedback.py b/pycti/entities/opencti_feedback.py index 9f2396070..0f8925dae 100644 --- a/pycti/entities/opencti_feedback.py +++ b/pycti/entities/opencti_feedback.py @@ -419,6 +419,10 @@ def generate_id(name): id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data)) return "feedback--" + id + @staticmethod + def generate_id_from_data(data): + return Feedback.generate_id(data["name"]) + """ List Feedback objects diff --git a/pycti/entities/opencti_grouping.py b/pycti/entities/opencti_grouping.py index 1fcd88d70..8284277a3 100644 --- a/pycti/entities/opencti_grouping.py +++ b/pycti/entities/opencti_grouping.py @@ -406,6 +406,10 @@ def generate_id(name, context, created): id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data)) return "grouping--" + id + @staticmethod + def generate_id_from_data(data): + return Grouping.generate_id(data["name"], data["context"]) + """ List Grouping objects diff --git a/pycti/entities/opencti_identity.py b/pycti/entities/opencti_identity.py index 4b048c471..dec283ad2 100644 --- a/pycti/entities/opencti_identity.py +++ b/pycti/entities/opencti_identity.py @@ -226,12 +226,15 @@ def __init__(self, opencti): @staticmethod def generate_id(name, identity_class): - name = name.lower().strip() - data = {"name": name, "identity_class": identity_class} + data = {"name": name.lower().strip(), "identity_class": identity_class.lower()} data = canonicalize(data, utf8=False) id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data)) return "identity--" + id + @staticmethod + def generate_id_from_data(data): + return Identity.generate_id(data["name"], data["identity_class"]) + """ List Identity objects diff --git a/pycti/entities/opencti_incident.py b/pycti/entities/opencti_incident.py index 4f56b98b2..6ada2f49f 100644 --- a/pycti/entities/opencti_incident.py +++ b/pycti/entities/opencti_incident.py @@ -225,6 +225,10 @@ def generate_id(name, created): id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data)) return "incident--" + id + @staticmethod + def generate_id_from_data(data): + return Incident.generate_id(data["name"], data["created"]) + """ List Incident objects diff --git a/pycti/entities/opencti_indicator.py b/pycti/entities/opencti_indicator.py index a85356fa0..6d209099f 100644 --- a/pycti/entities/opencti_indicator.py +++ b/pycti/entities/opencti_indicator.py @@ -29,6 +29,10 @@ def generate_id(pattern): id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data)) return "indicator--" + id + @staticmethod + def generate_id_from_data(data): + return Indicator.generate_id(data["pattern"]) + def list(self, **kwargs): """List Indicator objects diff --git a/pycti/entities/opencti_infrastructure.py b/pycti/entities/opencti_infrastructure.py index 957333e07..d55d45740 100644 --- a/pycti/entities/opencti_infrastructure.py +++ b/pycti/entities/opencti_infrastructure.py @@ -239,6 +239,10 @@ def generate_id(name): id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data)) return "infrastructure--" + id + @staticmethod + def generate_id_from_data(data): + return Infrastructure.generate_id(data["name"]) + def list(self, **kwargs): """List Infrastructure objects diff --git a/pycti/entities/opencti_intrusion_set.py b/pycti/entities/opencti_intrusion_set.py index f7f4954b3..8f72b2eca 100644 --- a/pycti/entities/opencti_intrusion_set.py +++ b/pycti/entities/opencti_intrusion_set.py @@ -222,6 +222,10 @@ def generate_id(name): id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data)) return "intrusion-set--" + id + @staticmethod + def generate_id_from_data(data): + return IntrusionSet.generate_id(data["name"]) + """ List Intrusion-Set objects diff --git a/pycti/entities/opencti_kill_chain_phase.py b/pycti/entities/opencti_kill_chain_phase.py index ce91a11f3..093dd372c 100644 --- a/pycti/entities/opencti_kill_chain_phase.py +++ b/pycti/entities/opencti_kill_chain_phase.py @@ -28,6 +28,10 @@ def generate_id(phase_name, kill_chain_name): phase_name=phase_name, kill_chain_name=kill_chain_name ) + @staticmethod + def generate_id_from_data(data): + return KillChainPhase.generate_id(data["phase_name"], data["kill_chain_name"]) + """ List Kill-Chain-Phase objects diff --git a/pycti/entities/opencti_language.py b/pycti/entities/opencti_language.py index fd39b6563..2790dbe6e 100644 --- a/pycti/entities/opencti_language.py +++ b/pycti/entities/opencti_language.py @@ -224,6 +224,10 @@ def generate_id(name): id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data)) return "language--" + id + @staticmethod + def generate_id_from_data(data): + return Language.generate_id(data["name"]) + """ List Language objects diff --git a/pycti/entities/opencti_location.py b/pycti/entities/opencti_location.py index 10b1d870b..12a921e6a 100644 --- a/pycti/entities/opencti_location.py +++ b/pycti/entities/opencti_location.py @@ -210,15 +210,33 @@ def __init__(self, opencti): @staticmethod def generate_id(name, x_opencti_location_type, latitude=None, longitude=None): - name = name.lower().strip() - if x_opencti_location_type == "position": - data = {"name": name, "latitude": latitude, "longitude": longitude} + if x_opencti_location_type == "Position": + if latitude is not None and longitude is None: + data = {"latitude": latitude} + elif latitude is None and longitude is not None: + data = {"longitude": longitude} + elif latitude is not None and longitude is not None: + data = {"latitude": latitude, "longitude": longitude} + else: + data = {"name": name.lower().strip()} else: - data = {"name": name, "x_opencti_location_type": x_opencti_location_type} + data = { + "name": name.lower().strip(), + "x_opencti_location_type": x_opencti_location_type, + } data = canonicalize(data, utf8=False) id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data)) return "location--" + id + @staticmethod + def generate_id_from_data(data): + return Location.generate_id( + data.get("name"), + data.get("x_opencti_location_type"), + data.get("latitude"), + data.get("longitude"), + ) + """ List Location objects diff --git a/pycti/entities/opencti_malware.py b/pycti/entities/opencti_malware.py index d1d89bd57..615c2fc2e 100644 --- a/pycti/entities/opencti_malware.py +++ b/pycti/entities/opencti_malware.py @@ -250,6 +250,10 @@ def generate_id(name): id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data)) return "malware--" + id + @staticmethod + def generate_id_from_data(data): + return Malware.generate_id(data["name"]) + """ List Malware objects diff --git a/pycti/entities/opencti_malware_analysis.py b/pycti/entities/opencti_malware_analysis.py index d2a02ddc4..5740f9f81 100644 --- a/pycti/entities/opencti_malware_analysis.py +++ b/pycti/entities/opencti_malware_analysis.py @@ -219,13 +219,21 @@ def __init__(self, opencti): """ @staticmethod - def generate_id(result_name): + def generate_id(result_name, product, submitted): result_name = result_name.lower().strip() - data = {"result_name": result_name} + data = {"result_name": result_name, "product": product} + if submitted is not None: + data = {**data, "submitted": submitted} data = canonicalize(data, utf8=False) id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data)) return "malware-analysis--" + id + @staticmethod + def generate_id_from_data(data): + return MalwareAnalysis.generate_id( + data["result_name"], data["product"], data.get("submitted") + ) + """ List Malware analysis objects diff --git a/pycti/entities/opencti_marking_definition.py b/pycti/entities/opencti_marking_definition.py index 571e9fd4b..cf01d5cac 100644 --- a/pycti/entities/opencti_marking_definition.py +++ b/pycti/entities/opencti_marking_definition.py @@ -31,6 +31,12 @@ def generate_id(definition, definition_type): id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data)) return "marking-definition--" + id + @staticmethod + def generate_id_from_data(data): + return MarkingDefinition.generate_id( + data["definition"], data["definition_type"] + ) + """ List Marking-Definition objects diff --git a/pycti/entities/opencti_narrative.py b/pycti/entities/opencti_narrative.py index eb03b18f1..32477cd36 100644 --- a/pycti/entities/opencti_narrative.py +++ b/pycti/entities/opencti_narrative.py @@ -202,6 +202,10 @@ def generate_id(name): id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data)) return "narrative--" + id + @staticmethod + def generate_id_from_data(data): + return Narrative.generate_id(data["name"]) + """ List Narrative objects diff --git a/pycti/entities/opencti_note.py b/pycti/entities/opencti_note.py index 0ef8ba782..7430a31ca 100644 --- a/pycti/entities/opencti_note.py +++ b/pycti/entities/opencti_note.py @@ -436,14 +436,22 @@ def __init__(self, opencti): @staticmethod def generate_id(created, content): - content = content.lower().strip() - if isinstance(created, datetime.datetime): - created = created.isoformat() - data = {"content": content, "created": created} + if content is None: + raise ValueError("content is required") + if created is not None: + if isinstance(created, datetime.datetime): + created = created.isoformat() + data = {"content": content, "created": created} + else: + data = {"content": content} data = canonicalize(data, utf8=False) id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data)) return "note--" + id + @staticmethod + def generate_id_from_data(data): + return Note.generate_id(data.get("created"), data["content"]) + """ List Note objects diff --git a/pycti/entities/opencti_observed_data.py b/pycti/entities/opencti_observed_data.py index a258f2b2d..dc1293a96 100644 --- a/pycti/entities/opencti_observed_data.py +++ b/pycti/entities/opencti_observed_data.py @@ -439,6 +439,10 @@ def generate_id(object_ids): id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data)) return "observed-data--" + id + @staticmethod + def generate_id_from_data(data): + return ObservedData.generate_id(data["object_refs"]) + """ List ObservedData objects diff --git a/pycti/entities/opencti_opinion.py b/pycti/entities/opencti_opinion.py index 7442d948a..6b435c7ea 100644 --- a/pycti/entities/opencti_opinion.py +++ b/pycti/entities/opencti_opinion.py @@ -1,8 +1,10 @@ # coding: utf-8 - +import datetime import json import uuid +from stix2.canonicalization.Canonicalize import canonicalize + class Opinion: def __init__(self, opencti): @@ -211,8 +213,22 @@ def __init__(self, opencti): """ @staticmethod - def generate_id(): - return "opinion--" + str(uuid.uuid4()) + def generate_id(created, opinion): + if opinion is None: + raise ValueError("opinion is required") + if created is not None: + if isinstance(created, datetime.datetime): + created = created.isoformat() + data = {"opinion": opinion, "created": created} + else: + data = {"opinion": opinion} + data = canonicalize(data, utf8=False) + id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data)) + return "opinion--" + id + + @staticmethod + def generate_id_from_data(data): + return Opinion.generate_id(data.get("created"), data["opinion"]) """ List Opinion objects diff --git a/pycti/entities/opencti_report.py b/pycti/entities/opencti_report.py index 2dd5cf563..2ab74fc23 100644 --- a/pycti/entities/opencti_report.py +++ b/pycti/entities/opencti_report.py @@ -479,6 +479,10 @@ def generate_fixed_fake_id(name, published=None): id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data)) return "report--" + id + @staticmethod + def generate_id_from_data(data): + return Report.generate_id(data["name"], data["published"]) + """ List Report objects diff --git a/pycti/entities/opencti_task.py b/pycti/entities/opencti_task.py index dab6f63a4..bdda9cea7 100644 --- a/pycti/entities/opencti_task.py +++ b/pycti/entities/opencti_task.py @@ -226,14 +226,17 @@ def __init__(self, opencti): @staticmethod def generate_id(name, created): - name = name.lower().strip() if isinstance(created, datetime.datetime): created = created.isoformat() - data = {"name": name, "created": created} + data = {"name": name.lower().strip(), "created": created} data = canonicalize(data, utf8=False) id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data)) return "task--" + id + @staticmethod + def generate_id_from_data(data): + return Task.generate_id(data["name"], data["created"]) + """ List Task objects diff --git a/pycti/entities/opencti_threat_actor.py b/pycti/entities/opencti_threat_actor.py index 7f8d0efe2..3e4002a40 100644 --- a/pycti/entities/opencti_threat_actor.py +++ b/pycti/entities/opencti_threat_actor.py @@ -142,13 +142,20 @@ def __init__(self, opencti): """ @staticmethod - def generate_id(name): - name = name.lower().strip() - data = {"name": name} + def generate_id(name, opencti_type): + data = {"name": name.lower().strip(), "opencti_type": opencti_type} data = canonicalize(data, utf8=False) id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data)) return "threat-actor--" + id + def generate_id_from_data(self, data): + data_type = "Threat-Actor-Group" + if "x_opencti_type" in data: + data_type = data["x_opencti_type"] + elif self.opencti.get_attribute_in_extension("type", data) is not None: + data_type = self.opencti.get_attribute_in_extension("type", data) + return ThreatActor.generate_id(data["name"], data_type) + def list(self, **kwargs) -> dict: """List Threat-Actor objects diff --git a/pycti/entities/opencti_threat_actor_group.py b/pycti/entities/opencti_threat_actor_group.py index b1e564963..af9d23c24 100644 --- a/pycti/entities/opencti_threat_actor_group.py +++ b/pycti/entities/opencti_threat_actor_group.py @@ -145,6 +145,10 @@ def generate_id(name): id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data)) return "threat-actor--" + id + @staticmethod + def generate_id_from_data(data): + return ThreatActorGroup.generate_id(data["name"]) + def list(self, **kwargs) -> dict: """List Threat-Actor-Group objects diff --git a/pycti/entities/opencti_threat_actor_individual.py b/pycti/entities/opencti_threat_actor_individual.py index 9fa17afc9..ff1590d26 100644 --- a/pycti/entities/opencti_threat_actor_individual.py +++ b/pycti/entities/opencti_threat_actor_individual.py @@ -145,6 +145,10 @@ def generate_id(name): id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data)) return "threat-actor--" + id + @staticmethod + def generate_id_from_data(data): + return ThreatActorIndividual.generate_id(data["name"]) + def list(self, **kwargs) -> dict: """List Threat-Actor-Individual objects diff --git a/pycti/entities/opencti_tool.py b/pycti/entities/opencti_tool.py index 73ae94674..8609f5db9 100644 --- a/pycti/entities/opencti_tool.py +++ b/pycti/entities/opencti_tool.py @@ -134,6 +134,10 @@ def generate_id(name): id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data)) return "tool--" + id + @staticmethod + def generate_id_from_data(data): + return Tool.generate_id(data["name"]) + """ List Tool objects diff --git a/pycti/entities/opencti_vocabulary.py b/pycti/entities/opencti_vocabulary.py index a7b2eda8c..c18710ba0 100644 --- a/pycti/entities/opencti_vocabulary.py +++ b/pycti/entities/opencti_vocabulary.py @@ -1,4 +1,7 @@ import json +import uuid + +from stix2.canonicalization.Canonicalize import canonicalize class Vocabulary: @@ -15,6 +18,18 @@ def __init__(self, opencti): } """ + @staticmethod + def generate_id(name, category): + name = name.lower().strip() + data = {"name": name, "category": category} + data = canonicalize(data, utf8=False) + id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data)) + return "vocabulary--" + id + + @staticmethod + def generate_id_from_data(data): + return Vocabulary.generate_id(data["name"], data["category"]) + def list(self, **kwargs): filters = kwargs.get("filters", None) self.opencti.app_logger.info( diff --git a/pycti/entities/opencti_vulnerability.py b/pycti/entities/opencti_vulnerability.py index a6133cf39..70b31acaa 100644 --- a/pycti/entities/opencti_vulnerability.py +++ b/pycti/entities/opencti_vulnerability.py @@ -131,6 +131,10 @@ def generate_id(name): id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data)) return "vulnerability--" + id + @staticmethod + def generate_id_from_data(data): + return Vulnerability.generate_id(data["name"]) + """ List Vulnerability objects diff --git a/pycti/utils/opencti_stix2.py b/pycti/utils/opencti_stix2.py index 126803b84..cbab7d6c6 100644 --- a/pycti/utils/opencti_stix2.py +++ b/pycti/utils/opencti_stix2.py @@ -852,6 +852,50 @@ def get_reader(self, entity_type: str): # endregion + def get_stix_helper(self): + # Import + return { + "attack-pattern": self.opencti.attack_pattern, + "campaign": self.opencti.campaign, + "note": self.opencti.note, + "observed-data": self.opencti.observed_data, + "opinion": self.opencti.opinion, + "report": self.opencti.report, + "course-of-action": self.opencti.course_of_action, + "identity": self.opencti.identity, + "infrastructure": self.opencti.infrastructure, + "intrusion-set": self.opencti.intrusion_set, + "location": self.opencti.location, + "malware": self.opencti.malware, + "threat-actor": self.opencti.threat_actor, + "tool": self.opencti.tool, + "vulnerability": self.opencti.vulnerability, + "incident": self.opencti.incident, + "marking-definition": self.opencti.marking_definition, + "case-rfi": self.opencti.case_rfi, + "x-opencti-case-rfi": self.opencti.case_rfi, + "case-rft": self.opencti.case_rft, + "x-opencti-case-rft": self.opencti.case_rft, + "case-incident": self.opencti.case_incident, + "x-opencti-case-incident": self.opencti.case_incident, + "feedback": self.opencti.feedback, + "x-opencti-feedback": self.opencti.feedback, + "channel": self.opencti.channel, + "data-component": self.opencti.data_component, + "x-mitre-data-component": self.opencti.data_component, + "data-source": self.opencti.data_source, + "x-mitre-data-source": self.opencti.data_source, + "event": self.opencti.event, + "grouping": self.opencti.grouping, + "indicator": self.opencti.indicator, + "language": self.opencti.language, + "malware-analysis": self.opencti.malware_analysis, + "narrative": self.opencti.narrative, + "task": self.opencti.task, + "x-opencti-task": self.opencti.task, + "vocabulary": self.opencti.vocabulary, + } + # region import def import_object( self, stix_object: Dict, update: bool = False, types: List = None @@ -898,53 +942,16 @@ def import_object( "sample_ids": sample_refs_ids, } - # Import - importer = { - "marking-definition": self.opencti.marking_definition.import_from_stix2, - "attack-pattern": self.opencti.attack_pattern.import_from_stix2, - "campaign": self.opencti.campaign.import_from_stix2, - "channel": self.opencti.channel.import_from_stix2, - "event": self.opencti.event.import_from_stix2, - "note": self.opencti.note.import_from_stix2, - "observed-data": self.opencti.observed_data.import_from_stix2, - "opinion": self.opencti.opinion.import_from_stix2, - "report": self.opencti.report.import_from_stix2, - "grouping": self.opencti.grouping.import_from_stix2, - "case-rfi": self.opencti.case_rfi.import_from_stix2, - "x-opencti-case-rfi": self.opencti.case_rfi.import_from_stix2, - "case-rft": self.opencti.case_rft.import_from_stix2, - "x-opencti-case-rft": self.opencti.case_rft.import_from_stix2, - "task": self.opencti.task.import_from_stix2, - "x-opencti-task": self.opencti.task.import_from_stix2, - "case-incident": self.opencti.case_incident.import_from_stix2, - "x-opencti-case-incident": self.opencti.case_incident.import_from_stix2, - "feedback": self.opencti.feedback.import_from_stix2, - "x-opencti-feedback": self.opencti.feedback.import_from_stix2, - "course-of-action": self.opencti.course_of_action.import_from_stix2, - "data-component": self.opencti.data_component.import_from_stix2, - "x-mitre-data-component": self.opencti.data_component.import_from_stix2, - "data-source": self.opencti.data_source.import_from_stix2, - "x-mitre-data-source": self.opencti.data_source.import_from_stix2, - "identity": self.opencti.identity.import_from_stix2, - "indicator": self.opencti.indicator.import_from_stix2, - "infrastructure": self.opencti.infrastructure.import_from_stix2, - "intrusion-set": self.opencti.intrusion_set.import_from_stix2, - "location": self.opencti.location.import_from_stix2, - "malware": self.opencti.malware.import_from_stix2, - "malware-analysis": self.opencti.malware_analysis.import_from_stix2, - "threat-actor": self.opencti.threat_actor.import_from_stix2, - "tool": self.opencti.tool.import_from_stix2, - "narrative": self.opencti.narrative.import_from_stix2, - "vulnerability": self.opencti.vulnerability.import_from_stix2, - "incident": self.opencti.incident.import_from_stix2, - } - do_import = importer.get( - stix_object["type"], - lambda **kwargs: self.unknown_type(stix_object), - ) - stix_object_results = do_import( - stixObject=stix_object, extras=extras, update=update - ) + stix_helper = self.get_stix_helper().get(stix_object["type"]) + if stix_helper: + stix_object_results = stix_helper.import_from_stix2( + stixObject=stix_object, extras=extras, update=update + ) + else: + stix_object_results = None + self.opencti.app_logger.error( + "Unknown object type, doing nothing...", {"type": stix_object["type"]} + ) if stix_object_results is None: return None @@ -2375,6 +2382,49 @@ def export_selected( return bundle + def prepare_bundle_ids(self, bundle, use_json=True, keep_original_id=False): + if use_json: + try: + bundle_data = json.loads(bundle) + except: + raise Exception("File data is not a valid JSON") + else: + bundle_data = bundle + cache_ids = {} + # First iteration to cache all entity ids + stix_helpers = self.get_stix_helper() + for item in bundle_data["objects"]: + if item["type"] != "relationship" and item["type"] != "sighting": + helper = stix_helpers.get(item["type"]) + if hasattr(helper, "generate_id_from_data"): + standard_id = helper.generate_id_from_data(item) + cache_ids[item["id"]] = standard_id + # Second iteration to replace and remap + for item in bundle_data["objects"]: + # For entities, try to replace the main id + # Keep the current one if needed + if item["type"] != "relationship" and item["type"] != "sighting": + if cache_ids.get(item["id"]): + original_id = item["id"] + item["id"] = cache_ids[original_id] + if keep_original_id: + item["x_opencti_stix_ids"] = item.get( + "x_opencti_stix_ids", [] + ) + [original_id] + # For all elements, replace all refs (source_ref, object_refs, ...) + ref_keys = list( + filter(lambda i: i.endswith("_ref") or i.endswith("_refs"), item.keys()) + ) + for ref_key in ref_keys: + if ref_key.endswith("_refs"): + item[ref_key] = list( + map(lambda id_ref: cache_ids.get(id_ref, id_ref), item[ref_key]) + ) + else: + item[ref_key] = cache_ids.get(item[ref_key], item[ref_key]) + + return json.dumps(bundle_data) if use_json else bundle_data + def import_item( self, item, @@ -2623,6 +2673,12 @@ def import_bundle( if "x_opencti_event_version" in stix_bundle else None ) + + # Bundle ids must be rewritten + stix_bundle = self.prepare_bundle_ids( + bundle=stix_bundle, use_json=False, keep_original_id=False + ) + stix2_splitter = OpenCTIStix2Splitter() _, bundles = stix2_splitter.split_bundle_with_expectations( stix_bundle, False, event_version diff --git a/tests/01-unit/stix/__init__.py b/tests/01-unit/stix/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/01-unit/stix/test_bundle_ids_rewrite.py b/tests/01-unit/stix/test_bundle_ids_rewrite.py new file mode 100644 index 000000000..df6cb338a --- /dev/null +++ b/tests/01-unit/stix/test_bundle_ids_rewrite.py @@ -0,0 +1,136 @@ +import json + +from pycti import OpenCTIApiClient, OpenCTIStix2 + + +def get_cti_helper(): + client = OpenCTIApiClient( + "http://fake:4000", "fake", ssl_verify=False, perform_health_check=False + ) + return OpenCTIStix2(client) + + +def load_test_file(): + with open("../../data/bundle_ids_sample.json", "r") as content_file: + content = content_file.read() + bundle_data = json.loads(content) + return bundle_data + + +def gen_id(data_type, data): + helper = get_cti_helper() + stix_helpers = helper.get_stix_helper() + helper = stix_helpers.get(data_type) + return helper.generate_id_from_data(data) + + +# fmt: off +def test_ids_generation(): + # attack-pattern + assert gen_id("attack-pattern", {"name": "attack"}) =='attack-pattern--25f21617-8de8-5d5e-8cd4-b7e88547ba76' + assert gen_id("attack-pattern", {"name": "attack", "x_mitre_id": 'MITREID'}) == 'attack-pattern--b74cfee2-7b14-585e-862f-fea45e802da9' + assert gen_id("attack-pattern", {"x_mitre_id": "MITREID"}) == 'attack-pattern--b74cfee2-7b14-585e-862f-fea45e802da9' + # campaign + assert gen_id("campaign", {"name": "attack"}) == 'campaign--25f21617-8de8-5d5e-8cd4-b7e88547ba76' + # note + assert gen_id("note", {"content": "My note content!"}) == "note--2b4ab5af-2307-58e1-8862-a6a269aae798" + assert gen_id("note", {"content": "My note content!", "created": "2022-11-25T19:00:05.000Z"}) == "note--10861e5c-049e-54f6-9736-81c106e39a0b" + # observed-data + assert gen_id("observed-data", {"object_refs": ["id"]}) == "observed-data--4765c523-81bc-54c8-b1af-ee81d961dad1" + # opinion + assert gen_id("opinion", {"opinion": "Good"}) == "opinion--0aef8829-207e-508b-b1f1-9da07f3379cb" + assert gen_id("opinion", {"opinion": "Good", "created": "2022-11-25T19:00:05.000Z"}) == "opinion--941dbd61-c6b1-5290-b63f-19a38983d7f7" + # report + assert gen_id("report", {"name": "Report", "published": "2022-11-25T19:00:05.000Z"}) == "report--761c6602-975f-5e5e-b220-7a2d41f33ce4" + # course-of-action + assert gen_id("course-of-action", {"x_mitre_id": "MITREID"}) == "course-of-action--b74cfee2-7b14-585e-862f-fea45e802da9" + assert gen_id("course-of-action", {"x_mitre_id": "MITREID", "name": "Name"}) == "course-of-action--b74cfee2-7b14-585e-862f-fea45e802da9" + assert gen_id("course-of-action", {"name": "Name"}) == "course-of-action--e6e2ee8d-e54d-50cd-b77c-df8c8eea7726" + # identity + assert gen_id("identity", {"name": "julien", "identity_class": "Individual"}) == "identity--d969b177-497f-598d-8428-b128c8f5f819" + assert gen_id("identity", {"name": "julien", "identity_class": "Sector"}) == "identity--14ffa2a4-e16a-522a-937a-784c0ac1fab0" + assert gen_id("identity", {"name": "julien", "identity_class": "System"}) == "identity--8af97482-121d-53f7-a533-9c48f06b5a38" + assert gen_id("identity", {"name": "organization", "identity_class": "individual"}) == "identity--00f7eb8c-6af2-5ed5-9ede-ede4c623de3b" + # infrastructure + assert gen_id("infrastructure", {"name": "infra"}) == "infrastructure--8a20116f-5a41-5508-ae4b-c293ac67c527" + # intrusion-set + assert gen_id("intrusion-set", {"name": "intrusion"}) == "intrusion-set--30757026-c4bd-574d-ae52-8d8503b4818e" + # location + assert gen_id("location", {"name": "Lyon", "x_opencti_location_type": "City"}) == "location--da430873-42c8-57ca-b08b-a797558c6cbd" + assert gen_id("location", {"latitude": 5.12, "name": "Position1", "x_opencti_location_type": "Position"}) == "location--56b3fc50-5091-5f2e-bd19-7b40ee3881e4" + assert gen_id("location", {"longitude": 5.12, "name": 'Position2', "x_opencti_location_type": "Position"}) == "location--dd2cf94c-1d58-58a1-b21f-0ede4059aaf0" + assert gen_id("location", {"latitude": 5.12, "longitude": 5.12, "x_opencti_location_type": "Position"}) == "location--57acef55-747a-55ef-9c49-06ca85f8d749" + assert gen_id("location", {"name": 'Position3', "x_opencti_location_type": "Position"}) == "location--a4152781-8721-5d44-ae2d-e492665bc35b" + # malware + assert gen_id("malware", {"name": "malware"}) == "malware--92ddf766-b27c-5159-8f46-27002bba2f04" + # threat-actor-group + assert gen_id("threat-actor", {"name": "CARD04"}) == "threat-actor--6d458783-df3b-5398-8e30-282655ad7b94" + assert gen_id("threat-actor", {"name": "CARD04", "x_opencti_type": "Threat-Actor-Group"}) == "threat-actor--6d458783-df3b-5398-8e30-282655ad7b94" + # tool + assert gen_id("tool", {"name": "my-tool"}) == "tool--41cd21d0-f50e-5e3d-83fc-447e0def97b7" + # vulnerability + assert gen_id("vulnerability", {"name": "vulnerability"}) == "vulnerability--2c690168-aec3-57f1-8295-adf53f4dc3da" + # incident + assert gen_id("incident", {"name": "incident", "created": "2022-11-25T19:00:05.000Z"}) == "incident--0e117c15-0a94-5ad3-b090-0395613f5b29" + # case-incident + assert gen_id("case-incident", {"name": "case", "created": "2022-11-25T19:00:05.000Z"}) == "case-incident--4838a141-bd19-542c-85d9-cce0382645b5" + # case-rfi + assert gen_id("case-rfi", {"name": "case", "created": "2022-11-25T19:00:05.000Z"}) == "case-rfi--4838a141-bd19-542c-85d9-cce0382645b5" + # case-rft + assert gen_id("case-rft", {"name": "case", "created": "2022-11-25T19:00:05.000Z"}) == "case-rft--4838a141-bd19-542c-85d9-cce0382645b5" + # feedback, not supported yet + # assert gen_id("case-feedback", {"name": "case", "created": "2022-11-25T19:00:05.000Z"}) == "feedback--4838a141-bd19-542c-85d9-cce0382645b5" + # channel + assert gen_id("channel", {"name": "channel"}) == "channel--4936cdd5-6b6a-5c92-a756-cae1f09dcd80" + # data-component + assert gen_id("data-component", {"name": "data-component"}) == "data-component--32fdc52a-b4c5-5268-af2f-cdf820271f0b" + # data-source + assert gen_id("data-source", {"name": "data-source"}) == "data-source--f0925972-35e1-5172-9161-4d7180908339" + # grouping + assert gen_id("grouping", {"name": "grouping", "context": "context"}) == "grouping--8462bd42-4cad-54ae-a261-efc1a762d83d" + # language + assert gen_id("language", {"name": "fr"}) == "language--0ef28873-9d49-5cdb-a53a-eb7613391ee9" + # malware-analysis + assert gen_id("malware-analysis", {"product": "linux", "result_name": "result"}) == "malware-analysis--3d501241-a4a5-574d-a503-301a6426f8c1" + assert gen_id("malware-analysis", {"product": "linux", "result_name": "result", "submitted": "2022-11-25T19:00:05.000Z"}) == "malware-analysis--d7ffe68a-0d5f-5fea-a375-3338ba4ea13c" + # narrative + assert gen_id("narrative", {"name": "narrative"}) == "narrative--804a7e40-d39c-59b6-9e3f-1ba1bc92b739" + # task + assert gen_id("task", {"name": "case", "created": "2022-11-25T19:00:05.000Z"}) == "task--4838a141-bd19-542c-85d9-cce0382645b5" + # Threat-actor-individual + assert gen_id("threat-actor", {"name": "CARD04", "x_opencti_type": "Threat-Actor-Individual"}) == "threat-actor--af15b6ae-a3dd-54d3-8fa0-3adfe0391d01" + # vocabulary + assert gen_id("vocabulary", {"name": "facebook", "category": "account_type_ov"}) == "vocabulary--85ae7185-ff6f-509b-a011-3069921614aa" +# fmt: on + + +def test_prepare_bundle_ids_keep_original(): + helper = get_cti_helper() + bundle_data = load_test_file() + malware_source = bundle_data["objects"][0] + assert malware_source["id"] == "malware--d650c5b9-4b43-5781-8576-ea52bd6c7ce5" + assert malware_source.get("x_opencti_stix_ids") is None + prepared_bundle = helper.prepare_bundle_ids( + bundle=bundle_data, use_json=False, keep_original_id=True + ) + print(json.dumps(prepared_bundle)) + malware_target = prepared_bundle["objects"][0] + assert malware_target["id"] == "malware--d650c5b9-4b43-5781-8576-ea52bd6c7ce0" + assert malware_target.get("x_opencti_stix_ids") == [ + "malware--d650c5b9-4b43-5781-8576-ea52bd6c7ce5" + ] + + +def test_prepare_bundle_ids(): + helper = get_cti_helper() + bundle_data = load_test_file() + malware_source = bundle_data["objects"][0] + assert malware_source["id"] == "malware--d650c5b9-4b43-5781-8576-ea52bd6c7ce5" + assert malware_source.get("x_opencti_stix_ids") is None + prepared_bundle = helper.prepare_bundle_ids( + bundle=bundle_data, use_json=False, keep_original_id=False + ) + print(json.dumps(prepared_bundle)) + malware_target = prepared_bundle["objects"][0] + assert malware_target["id"] == "malware--d650c5b9-4b43-5781-8576-ea52bd6c7ce0" + assert malware_target.get("x_opencti_stix_ids") is None diff --git a/tests/data/bundle_ids_sample.json b/tests/data/bundle_ids_sample.json new file mode 100644 index 000000000..7f26d27d2 --- /dev/null +++ b/tests/data/bundle_ids_sample.json @@ -0,0 +1,76 @@ +{ + "type": "bundle", + "id": "bundle--8c939929-688f-4a72-badb-3dd1bd6af0fa", + "objects": [ + { + "id": "malware--d650c5b9-4b43-5781-8576-ea52bd6c7ce5", + "spec_version": "2.1", + "revoked": false, + "confidence": 100, + "created": "2024-03-13T09:56:18.259Z", + "modified": "2024-03-13T09:56:18.259Z", + "name": "BasicMalware", + "is_family": false, + "x_opencti_id": "75f2a512-fcc6-4cbc-a2ef-52ca9c57df46", + "x_opencti_type": "Malware", + "type": "malware" + }, + { + "id": "identity--7b82b010-b1c0-4dae-981f-7756374a17da", + "type": "identity", + "spec_version": "2.1", + "name": "ANSSI", + "identity_class": "organization", + "labels": ["identity"], + "created": "2020-02-23T23:40:53.575Z", + "modified": "2020-02-27T08:45:39.351Z", + "x_opencti_organization_type": "CSIRT" + }, + { + "id": "marking-definition--78ca4366-f5b8-4764-83f7-34ce38198e27", + "type": "marking-definition", + "spec_version": "2.1", + "definition_type": "TLP", + "definition": { + "TLP": "TLP:TEST" + }, + "created": "2020-02-25T09:02:29.040Z", + "modified": "2020-02-25T09:02:29.040Z", + "created_by_ref": "marking-definition--78ca4366-f5b8-4764-83f7-34ce38198e27" + }, + { + "id": "report--a445d22a-db0c-4b5d-9ec8-e9ad0b6dbdd7", + "type": "report", + "spec_version": "2.1", + "name": "A demo report for testing purposes", + "labels": ["report"], + "description": "Report for testing purposes (random data).", + "published": "2020-03-01T14:02:48.111Z", + "created": "2020-03-01T14:02:55.327Z", + "modified": "2020-03-01T14:09:48.078Z", + "report_types": ["threat-report"], + "x_opencti_report_status": 2, + "confidence": 3, + "created_by_ref": "identity--7b82b010-b1c0-4dae-981f-7756374a17da", + "object_marking_refs": ["marking-definition--78ca4366-f5b8-4764-83f7-34ce38198e27"], + "object_refs": [ + "observed-data--7d258c31-9a26-4543-aecb-2abc5ed366be", + "malware--d650c5b9-4b43-5781-8576-ea52bd6c7ce5" + ] + }, + { + "id": "relationship--ba52fced-422a-4bee-816a-85aa21c9eaca", + "type": "relationship", + "spec_version": "2.1", + "relationship_type": "related-to", + "source_ref": "malware--d650c5b9-4b43-5781-8576-ea52bd6c7ce5", + "target_ref": "report--a445d22a-db0c-4b5d-9ec8-e9ad0b6dbdd7", + "created": "2020-03-01T14:07:14.316Z", + "modified": "2020-03-01T14:07:14.316Z", + "start_time": "1900-01-01T00:00:00.000Z", + "stop_time": "1900-01-01T00:00:00.000Z", + "created_by_ref": "identity--7b82b010-b1c0-4dae-981f-7756374a17da", + "object_marking_refs": ["marking-definition--78ca4366-f5b8-4764-83f7-34ce38198e27"] + } + ] +} From 8f9150f18e1d194296e7597a37e4de90285d3e56 Mon Sep 17 00:00:00 2001 From: Julien Richard Date: Tue, 28 May 2024 19:54:42 +0200 Subject: [PATCH 02/17] [client] Change file bundle test path (#659) --- tests/01-unit/stix/test_bundle_ids_rewrite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/01-unit/stix/test_bundle_ids_rewrite.py b/tests/01-unit/stix/test_bundle_ids_rewrite.py index df6cb338a..26d2f3092 100644 --- a/tests/01-unit/stix/test_bundle_ids_rewrite.py +++ b/tests/01-unit/stix/test_bundle_ids_rewrite.py @@ -11,7 +11,7 @@ def get_cti_helper(): def load_test_file(): - with open("../../data/bundle_ids_sample.json", "r") as content_file: + with open("tests/data/bundle_ids_sample.json", "r") as content_file: content = content_file.read() bundle_data = json.loads(content) return bundle_data From 28aa7b5ebab2b5c131cbd8e76ff71b94e46f6aff Mon Sep 17 00:00:00 2001 From: Julien Richard Date: Tue, 28 May 2024 21:44:24 +0200 Subject: [PATCH 03/17] [client] Improve testing (#659) --- pycti/utils/opencti_stix2.py | 5 + tests/01-unit/stix/test_bundle_ids_rewrite.py | 96 +++++++++---------- tests/02-integration/utils/test_stix_crud.py | 2 +- 3 files changed, 51 insertions(+), 52 deletions(-) diff --git a/pycti/utils/opencti_stix2.py b/pycti/utils/opencti_stix2.py index cbab7d6c6..a83014fc2 100644 --- a/pycti/utils/opencti_stix2.py +++ b/pycti/utils/opencti_stix2.py @@ -896,6 +896,11 @@ def get_stix_helper(self): "vocabulary": self.opencti.vocabulary, } + def generate_standard_id_from_stix(self, data): + stix_helpers = self.get_stix_helper() + helper = stix_helpers.get(data["type"]) + return helper.generate_id_from_data(data) + # region import def import_object( self, stix_object: Dict, update: bool = False, types: List = None diff --git a/tests/01-unit/stix/test_bundle_ids_rewrite.py b/tests/01-unit/stix/test_bundle_ids_rewrite.py index 26d2f3092..eef882143 100644 --- a/tests/01-unit/stix/test_bundle_ids_rewrite.py +++ b/tests/01-unit/stix/test_bundle_ids_rewrite.py @@ -17,90 +17,84 @@ def load_test_file(): return bundle_data -def gen_id(data_type, data): - helper = get_cti_helper() - stix_helpers = helper.get_stix_helper() - helper = stix_helpers.get(data_type) - return helper.generate_id_from_data(data) - - # fmt: off def test_ids_generation(): + gen_id = get_cti_helper().generate_standard_id_from_stix # attack-pattern - assert gen_id("attack-pattern", {"name": "attack"}) =='attack-pattern--25f21617-8de8-5d5e-8cd4-b7e88547ba76' - assert gen_id("attack-pattern", {"name": "attack", "x_mitre_id": 'MITREID'}) == 'attack-pattern--b74cfee2-7b14-585e-862f-fea45e802da9' - assert gen_id("attack-pattern", {"x_mitre_id": "MITREID"}) == 'attack-pattern--b74cfee2-7b14-585e-862f-fea45e802da9' + assert gen_id({"type": "attack-pattern", "name": "attack"}) =='attack-pattern--25f21617-8de8-5d5e-8cd4-b7e88547ba76' + assert gen_id({"type": "attack-pattern", "name": "attack", "x_mitre_id": 'MITREID'}) == 'attack-pattern--b74cfee2-7b14-585e-862f-fea45e802da9' + assert gen_id({"type": "attack-pattern", "x_mitre_id": "MITREID"}) == 'attack-pattern--b74cfee2-7b14-585e-862f-fea45e802da9' # campaign - assert gen_id("campaign", {"name": "attack"}) == 'campaign--25f21617-8de8-5d5e-8cd4-b7e88547ba76' + assert gen_id({"type": "campaign", "name": "attack"}) == 'campaign--25f21617-8de8-5d5e-8cd4-b7e88547ba76' # note - assert gen_id("note", {"content": "My note content!"}) == "note--2b4ab5af-2307-58e1-8862-a6a269aae798" - assert gen_id("note", {"content": "My note content!", "created": "2022-11-25T19:00:05.000Z"}) == "note--10861e5c-049e-54f6-9736-81c106e39a0b" + assert gen_id({"type": "note", "content": "My note content!"}) == "note--2b4ab5af-2307-58e1-8862-a6a269aae798" + assert gen_id({"type": "note", "content": "My note content!", "created": "2022-11-25T19:00:05.000Z"}) == "note--10861e5c-049e-54f6-9736-81c106e39a0b" # observed-data - assert gen_id("observed-data", {"object_refs": ["id"]}) == "observed-data--4765c523-81bc-54c8-b1af-ee81d961dad1" + assert gen_id({"type": "observed-data", "object_refs": ["id"]}) == "observed-data--4765c523-81bc-54c8-b1af-ee81d961dad1" # opinion - assert gen_id("opinion", {"opinion": "Good"}) == "opinion--0aef8829-207e-508b-b1f1-9da07f3379cb" - assert gen_id("opinion", {"opinion": "Good", "created": "2022-11-25T19:00:05.000Z"}) == "opinion--941dbd61-c6b1-5290-b63f-19a38983d7f7" + assert gen_id({"type": "opinion", "opinion": "Good"}) == "opinion--0aef8829-207e-508b-b1f1-9da07f3379cb" + assert gen_id({"type": "opinion", "opinion": "Good", "created": "2022-11-25T19:00:05.000Z"}) == "opinion--941dbd61-c6b1-5290-b63f-19a38983d7f7" # report - assert gen_id("report", {"name": "Report", "published": "2022-11-25T19:00:05.000Z"}) == "report--761c6602-975f-5e5e-b220-7a2d41f33ce4" + assert gen_id({"type": "report", "name": "Report", "published": "2022-11-25T19:00:05.000Z"}) == "report--761c6602-975f-5e5e-b220-7a2d41f33ce4" # course-of-action - assert gen_id("course-of-action", {"x_mitre_id": "MITREID"}) == "course-of-action--b74cfee2-7b14-585e-862f-fea45e802da9" - assert gen_id("course-of-action", {"x_mitre_id": "MITREID", "name": "Name"}) == "course-of-action--b74cfee2-7b14-585e-862f-fea45e802da9" - assert gen_id("course-of-action", {"name": "Name"}) == "course-of-action--e6e2ee8d-e54d-50cd-b77c-df8c8eea7726" + assert gen_id({"type": "course-of-action", "x_mitre_id": "MITREID"}) == "course-of-action--b74cfee2-7b14-585e-862f-fea45e802da9" + assert gen_id({"type": "course-of-action", "x_mitre_id": "MITREID", "name": "Name"}) == "course-of-action--b74cfee2-7b14-585e-862f-fea45e802da9" + assert gen_id({"type": "course-of-action", "name": "Name"}) == "course-of-action--e6e2ee8d-e54d-50cd-b77c-df8c8eea7726" # identity - assert gen_id("identity", {"name": "julien", "identity_class": "Individual"}) == "identity--d969b177-497f-598d-8428-b128c8f5f819" - assert gen_id("identity", {"name": "julien", "identity_class": "Sector"}) == "identity--14ffa2a4-e16a-522a-937a-784c0ac1fab0" - assert gen_id("identity", {"name": "julien", "identity_class": "System"}) == "identity--8af97482-121d-53f7-a533-9c48f06b5a38" - assert gen_id("identity", {"name": "organization", "identity_class": "individual"}) == "identity--00f7eb8c-6af2-5ed5-9ede-ede4c623de3b" + assert gen_id({"type": "identity", "name": "julien", "identity_class": "Individual"}) == "identity--d969b177-497f-598d-8428-b128c8f5f819" + assert gen_id({"type": "identity", "name": "julien", "identity_class": "Sector"}) == "identity--14ffa2a4-e16a-522a-937a-784c0ac1fab0" + assert gen_id({"type": "identity", "name": "julien", "identity_class": "System"}) == "identity--8af97482-121d-53f7-a533-9c48f06b5a38" + assert gen_id({"type": "identity", "name": "organization", "identity_class": "individual"}) == "identity--00f7eb8c-6af2-5ed5-9ede-ede4c623de3b" # infrastructure - assert gen_id("infrastructure", {"name": "infra"}) == "infrastructure--8a20116f-5a41-5508-ae4b-c293ac67c527" + assert gen_id({"type": "infrastructure", "name": "infra"}) == "infrastructure--8a20116f-5a41-5508-ae4b-c293ac67c527" # intrusion-set - assert gen_id("intrusion-set", {"name": "intrusion"}) == "intrusion-set--30757026-c4bd-574d-ae52-8d8503b4818e" + assert gen_id({"type": "intrusion-set", "name": "intrusion"}) == "intrusion-set--30757026-c4bd-574d-ae52-8d8503b4818e" # location - assert gen_id("location", {"name": "Lyon", "x_opencti_location_type": "City"}) == "location--da430873-42c8-57ca-b08b-a797558c6cbd" - assert gen_id("location", {"latitude": 5.12, "name": "Position1", "x_opencti_location_type": "Position"}) == "location--56b3fc50-5091-5f2e-bd19-7b40ee3881e4" - assert gen_id("location", {"longitude": 5.12, "name": 'Position2', "x_opencti_location_type": "Position"}) == "location--dd2cf94c-1d58-58a1-b21f-0ede4059aaf0" - assert gen_id("location", {"latitude": 5.12, "longitude": 5.12, "x_opencti_location_type": "Position"}) == "location--57acef55-747a-55ef-9c49-06ca85f8d749" - assert gen_id("location", {"name": 'Position3', "x_opencti_location_type": "Position"}) == "location--a4152781-8721-5d44-ae2d-e492665bc35b" + assert gen_id({"type": "location", "name": "Lyon", "x_opencti_location_type": "City"}) == "location--da430873-42c8-57ca-b08b-a797558c6cbd" + assert gen_id({"type": "location", "latitude": 5.12, "name": "Position1", "x_opencti_location_type": "Position"}) == "location--56b3fc50-5091-5f2e-bd19-7b40ee3881e4" + assert gen_id({"type": "location", "longitude": 5.12, "name": 'Position2', "x_opencti_location_type": "Position"}) == "location--dd2cf94c-1d58-58a1-b21f-0ede4059aaf0" + assert gen_id({"type": "location", "latitude": 5.12, "longitude": 5.12, "x_opencti_location_type": "Position"}) == "location--57acef55-747a-55ef-9c49-06ca85f8d749" + assert gen_id({"type": "location", "name": 'Position3', "x_opencti_location_type": "Position"}) == "location--a4152781-8721-5d44-ae2d-e492665bc35b" # malware - assert gen_id("malware", {"name": "malware"}) == "malware--92ddf766-b27c-5159-8f46-27002bba2f04" + assert gen_id({"type": "malware", "name": "malware"}) == "malware--92ddf766-b27c-5159-8f46-27002bba2f04" # threat-actor-group - assert gen_id("threat-actor", {"name": "CARD04"}) == "threat-actor--6d458783-df3b-5398-8e30-282655ad7b94" - assert gen_id("threat-actor", {"name": "CARD04", "x_opencti_type": "Threat-Actor-Group"}) == "threat-actor--6d458783-df3b-5398-8e30-282655ad7b94" + assert gen_id({"type": "threat-actor", "name": "CARD04"}) == "threat-actor--6d458783-df3b-5398-8e30-282655ad7b94" + assert gen_id({"type": "threat-actor", "name": "CARD04", "x_opencti_type": "Threat-Actor-Group"}) == "threat-actor--6d458783-df3b-5398-8e30-282655ad7b94" # tool - assert gen_id("tool", {"name": "my-tool"}) == "tool--41cd21d0-f50e-5e3d-83fc-447e0def97b7" + assert gen_id({"type": "tool", "name": "my-tool"}) == "tool--41cd21d0-f50e-5e3d-83fc-447e0def97b7" # vulnerability - assert gen_id("vulnerability", {"name": "vulnerability"}) == "vulnerability--2c690168-aec3-57f1-8295-adf53f4dc3da" + assert gen_id({"type": "vulnerability", "name": "vulnerability"}) == "vulnerability--2c690168-aec3-57f1-8295-adf53f4dc3da" # incident - assert gen_id("incident", {"name": "incident", "created": "2022-11-25T19:00:05.000Z"}) == "incident--0e117c15-0a94-5ad3-b090-0395613f5b29" + assert gen_id({"type": "incident", "name": "incident", "created": "2022-11-25T19:00:05.000Z"}) == "incident--0e117c15-0a94-5ad3-b090-0395613f5b29" # case-incident - assert gen_id("case-incident", {"name": "case", "created": "2022-11-25T19:00:05.000Z"}) == "case-incident--4838a141-bd19-542c-85d9-cce0382645b5" + assert gen_id({"type": "case-incident", "name": "case", "created": "2022-11-25T19:00:05.000Z"}) == "case-incident--4838a141-bd19-542c-85d9-cce0382645b5" # case-rfi - assert gen_id("case-rfi", {"name": "case", "created": "2022-11-25T19:00:05.000Z"}) == "case-rfi--4838a141-bd19-542c-85d9-cce0382645b5" + assert gen_id({"type": "case-rfi", "name": "case", "created": "2022-11-25T19:00:05.000Z"}) == "case-rfi--4838a141-bd19-542c-85d9-cce0382645b5" # case-rft - assert gen_id("case-rft", {"name": "case", "created": "2022-11-25T19:00:05.000Z"}) == "case-rft--4838a141-bd19-542c-85d9-cce0382645b5" + assert gen_id({"type": "case-rft", "name": "case", "created": "2022-11-25T19:00:05.000Z"}) == "case-rft--4838a141-bd19-542c-85d9-cce0382645b5" # feedback, not supported yet # assert gen_id("case-feedback", {"name": "case", "created": "2022-11-25T19:00:05.000Z"}) == "feedback--4838a141-bd19-542c-85d9-cce0382645b5" # channel - assert gen_id("channel", {"name": "channel"}) == "channel--4936cdd5-6b6a-5c92-a756-cae1f09dcd80" + assert gen_id({"type": "channel", "name": "channel"}) == "channel--4936cdd5-6b6a-5c92-a756-cae1f09dcd80" # data-component - assert gen_id("data-component", {"name": "data-component"}) == "data-component--32fdc52a-b4c5-5268-af2f-cdf820271f0b" + assert gen_id({"type": "data-component", "name": "data-component"}) == "data-component--32fdc52a-b4c5-5268-af2f-cdf820271f0b" # data-source - assert gen_id("data-source", {"name": "data-source"}) == "data-source--f0925972-35e1-5172-9161-4d7180908339" + assert gen_id({"type": "data-source", "name": "data-source"}) == "data-source--f0925972-35e1-5172-9161-4d7180908339" # grouping - assert gen_id("grouping", {"name": "grouping", "context": "context"}) == "grouping--8462bd42-4cad-54ae-a261-efc1a762d83d" + assert gen_id({"type": "grouping", "name": "grouping", "context": "context"}) == "grouping--8462bd42-4cad-54ae-a261-efc1a762d83d" # language - assert gen_id("language", {"name": "fr"}) == "language--0ef28873-9d49-5cdb-a53a-eb7613391ee9" + assert gen_id({"type": "language", "name": "fr"}) == "language--0ef28873-9d49-5cdb-a53a-eb7613391ee9" # malware-analysis - assert gen_id("malware-analysis", {"product": "linux", "result_name": "result"}) == "malware-analysis--3d501241-a4a5-574d-a503-301a6426f8c1" - assert gen_id("malware-analysis", {"product": "linux", "result_name": "result", "submitted": "2022-11-25T19:00:05.000Z"}) == "malware-analysis--d7ffe68a-0d5f-5fea-a375-3338ba4ea13c" + assert gen_id({"type": "malware-analysis", "product": "linux", "result_name": "result"}) == "malware-analysis--3d501241-a4a5-574d-a503-301a6426f8c1" + assert gen_id({"type": "malware-analysis", "product": "linux", "result_name": "result", "submitted": "2022-11-25T19:00:05.000Z"}) == "malware-analysis--d7ffe68a-0d5f-5fea-a375-3338ba4ea13c" # narrative - assert gen_id("narrative", {"name": "narrative"}) == "narrative--804a7e40-d39c-59b6-9e3f-1ba1bc92b739" + assert gen_id({"type": "narrative", "name": "narrative"}) == "narrative--804a7e40-d39c-59b6-9e3f-1ba1bc92b739" # task - assert gen_id("task", {"name": "case", "created": "2022-11-25T19:00:05.000Z"}) == "task--4838a141-bd19-542c-85d9-cce0382645b5" + assert gen_id({"type": "task", "name": "case", "created": "2022-11-25T19:00:05.000Z"}) == "task--4838a141-bd19-542c-85d9-cce0382645b5" # Threat-actor-individual - assert gen_id("threat-actor", {"name": "CARD04", "x_opencti_type": "Threat-Actor-Individual"}) == "threat-actor--af15b6ae-a3dd-54d3-8fa0-3adfe0391d01" + assert gen_id({"type": "threat-actor", "name": "CARD04", "x_opencti_type": "Threat-Actor-Individual"}) == "threat-actor--af15b6ae-a3dd-54d3-8fa0-3adfe0391d01" # vocabulary - assert gen_id("vocabulary", {"name": "facebook", "category": "account_type_ov"}) == "vocabulary--85ae7185-ff6f-509b-a011-3069921614aa" + assert gen_id({"type": "vocabulary", "name": "facebook", "category": "account_type_ov"}) == "vocabulary--85ae7185-ff6f-509b-a011-3069921614aa" # fmt: on diff --git a/tests/02-integration/utils/test_stix_crud.py b/tests/02-integration/utils/test_stix_crud.py index edadba800..7533ad76f 100644 --- a/tests/02-integration/utils/test_stix_crud.py +++ b/tests/02-integration/utils/test_stix_crud.py @@ -24,7 +24,7 @@ def test_entity_create(entity_class, api_stix, opencti_splitter): bundles_sent = api_stix.import_bundle_from_json(split_bundle, False, None, None) assert len(bundles_sent) == 1 - assert bundles_sent[0]["id"] == stix_object["id"] + assert bundles_sent[0]["id"] == api_stix.generate_standard_id_from_stix(stix_object) assert bundles_sent[0]["type"] == stix_object["type"] entity_class.base_class().delete(id=stix_object["id"]) From 37474cb03b6f33e9bbfec66442b9752728b2e783 Mon Sep 17 00:00:00 2001 From: Julien Richard Date: Tue, 28 May 2024 22:59:47 +0200 Subject: [PATCH 04/17] [client] Improve testing (#659) --- tests/02-integration/utils/test_stix_crud.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/02-integration/utils/test_stix_crud.py b/tests/02-integration/utils/test_stix_crud.py index 7533ad76f..2f6cf3cea 100644 --- a/tests/02-integration/utils/test_stix_crud.py +++ b/tests/02-integration/utils/test_stix_crud.py @@ -24,7 +24,8 @@ def test_entity_create(entity_class, api_stix, opencti_splitter): bundles_sent = api_stix.import_bundle_from_json(split_bundle, False, None, None) assert len(bundles_sent) == 1 - assert bundles_sent[0]["id"] == api_stix.generate_standard_id_from_stix(stix_object) + element_new_id = api_stix.generate_standard_id_from_stix(stix_object) + assert bundles_sent[0]["id"] == element_new_id assert bundles_sent[0]["type"] == stix_object["type"] - entity_class.base_class().delete(id=stix_object["id"]) + entity_class.base_class().delete(id=element_new_id) From 56ac00b5aabfb9a678017b7a422130782e1563eb Mon Sep 17 00:00:00 2001 From: Julien Richard Date: Thu, 30 May 2024 19:17:20 +0200 Subject: [PATCH 05/17] [client] Improve relationship standard id generation (#659) --- .../opencti_stix_core_relationship.py | 11 ++++++ .../opencti_stix_sighting_relationship.py | 35 ++++++++++++++++--- pycti/utils/opencti_stix2.py | 28 ++++++++------- tests/01-unit/stix/test_bundle_ids_rewrite.py | 11 ++++++ tests/data/bundle_ids_sample.json | 15 +++++++- 5 files changed, 81 insertions(+), 19 deletions(-) diff --git a/pycti/entities/opencti_stix_core_relationship.py b/pycti/entities/opencti_stix_core_relationship.py index b3f729216..f4edbc55d 100644 --- a/pycti/entities/opencti_stix_core_relationship.py +++ b/pycti/entities/opencti_stix_core_relationship.py @@ -336,6 +336,7 @@ def generate_id( start_time = start_time.isoformat() if isinstance(stop_time, datetime.datetime): stop_time = stop_time.isoformat() + if start_time is not None and stop_time is not None: data = { "relationship_type": relationship_type, @@ -376,6 +377,16 @@ def generate_id( :return List of stix_core_relationship objects """ + @staticmethod + def generate_id_from_data(data): + return StixCoreRelationship.generate_id( + data["relationship_type"], + data["source_ref"], + data["target_ref"], + data.get("start_time"), + data.get("stop_time"), + ) + def list(self, **kwargs): from_or_to_id = kwargs.get("fromOrToId", None) element_with_target_types = kwargs.get("elementWithTargetTypes", None) diff --git a/pycti/entities/opencti_stix_sighting_relationship.py b/pycti/entities/opencti_stix_sighting_relationship.py index babbf6d97..b106a8bb0 100644 --- a/pycti/entities/opencti_stix_sighting_relationship.py +++ b/pycti/entities/opencti_stix_sighting_relationship.py @@ -262,7 +262,13 @@ def __init__(self, opencti): """ @staticmethod - def generate_id(source_ref, target_ref, first_seen=None, last_seen=None): + def generate_id( + relationship_type, + sighting_of_ref, + where_sighted_refs, + first_seen=None, + last_seen=None, + ): if isinstance(first_seen, datetime.datetime): first_seen = first_seen.isoformat() if isinstance(last_seen, datetime.datetime): @@ -270,20 +276,39 @@ def generate_id(source_ref, target_ref, first_seen=None, last_seen=None): if first_seen is not None and last_seen is not None: data = { - "source_ref": source_ref, - "target_ref": target_ref, + "type": relationship_type, + "sighting_of_ref": sighting_of_ref, + "where_sighted_refs": where_sighted_refs, "first_seen": first_seen, "last_seen": last_seen, } + elif first_seen is not None: + data = { + "type": relationship_type, + "sighting_of_ref": sighting_of_ref, + "where_sighted_refs": where_sighted_refs, + "first_seen": first_seen, + } else: data = { - "source_ref": source_ref, - "target_ref": target_ref, + "type": relationship_type, + "sighting_of_ref": sighting_of_ref, + "where_sighted_refs": where_sighted_refs, } data = canonicalize(data, utf8=False) id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data)) return "sighting--" + id + @staticmethod + def generate_id_from_data(data): + return StixSightingRelationship.generate_id( + data["type"], + data["sighting_of_ref"], + data["where_sighted_refs"], + data.get("first_seen"), + data.get("last_seen"), + ) + """ List stix_sightings objects diff --git a/pycti/utils/opencti_stix2.py b/pycti/utils/opencti_stix2.py index a83014fc2..120601551 100644 --- a/pycti/utils/opencti_stix2.py +++ b/pycti/utils/opencti_stix2.py @@ -855,6 +855,7 @@ def get_reader(self, entity_type: str): def get_stix_helper(self): # Import return { + # entities "attack-pattern": self.opencti.attack_pattern, "campaign": self.opencti.campaign, "note": self.opencti.note, @@ -894,6 +895,9 @@ def get_stix_helper(self): "task": self.opencti.task, "x-opencti-task": self.opencti.task, "vocabulary": self.opencti.vocabulary, + # relationships + "relationship": self.opencti.stix_core_relationship, + "sighting": self.opencti.stix_sighting_relationship, } def generate_standard_id_from_stix(self, data): @@ -2399,23 +2403,21 @@ def prepare_bundle_ids(self, bundle, use_json=True, keep_original_id=False): # First iteration to cache all entity ids stix_helpers = self.get_stix_helper() for item in bundle_data["objects"]: - if item["type"] != "relationship" and item["type"] != "sighting": - helper = stix_helpers.get(item["type"]) - if hasattr(helper, "generate_id_from_data"): - standard_id = helper.generate_id_from_data(item) - cache_ids[item["id"]] = standard_id + helper = stix_helpers.get(item["type"]) + if hasattr(helper, "generate_id_from_data"): + standard_id = helper.generate_id_from_data(item) + cache_ids[item["id"]] = standard_id # Second iteration to replace and remap for item in bundle_data["objects"]: # For entities, try to replace the main id # Keep the current one if needed - if item["type"] != "relationship" and item["type"] != "sighting": - if cache_ids.get(item["id"]): - original_id = item["id"] - item["id"] = cache_ids[original_id] - if keep_original_id: - item["x_opencti_stix_ids"] = item.get( - "x_opencti_stix_ids", [] - ) + [original_id] + if cache_ids.get(item["id"]): + original_id = item["id"] + item["id"] = cache_ids[original_id] + if keep_original_id: + item["x_opencti_stix_ids"] = item.get("x_opencti_stix_ids", []) + [ + original_id + ] # For all elements, replace all refs (source_ref, object_refs, ...) ref_keys = list( filter(lambda i: i.endswith("_ref") or i.endswith("_refs"), item.keys()) diff --git a/tests/01-unit/stix/test_bundle_ids_rewrite.py b/tests/01-unit/stix/test_bundle_ids_rewrite.py index eef882143..2c49385f4 100644 --- a/tests/01-unit/stix/test_bundle_ids_rewrite.py +++ b/tests/01-unit/stix/test_bundle_ids_rewrite.py @@ -17,6 +17,7 @@ def load_test_file(): return bundle_data +# !! WARNING !!, this need to be changed along with 01-unit/domain/identifier-test.js # fmt: off def test_ids_generation(): gen_id = get_cti_helper().generate_standard_id_from_stix @@ -95,6 +96,16 @@ def test_ids_generation(): assert gen_id({"type": "threat-actor", "name": "CARD04", "x_opencti_type": "Threat-Actor-Individual"}) == "threat-actor--af15b6ae-a3dd-54d3-8fa0-3adfe0391d01" # vocabulary assert gen_id({"type": "vocabulary", "name": "facebook", "category": "account_type_ov"}) == "vocabulary--85ae7185-ff6f-509b-a011-3069921614aa" + # relationship + base_relationship = {"type": "relationship", "relationship_type": "based-on", "source_ref": "from_id", "target_ref": "to_id"} + assert gen_id(base_relationship) == "relationship--0b11fa67-da01-5d34-9864-67d4d71c3740" + assert gen_id({**base_relationship, "start_time": "2022-11-25T19:00:05.000Z"}) == "relationship--c5e1e2ce-14d6-535b-911d-267e92119e01" + assert gen_id({**base_relationship, "start_time": "2022-11-25T19:00:05.000Z", "stop_time": "2022-11-26T19:00:05.000Z"}) == "relationship--a7778a7d-a743-5193-9912-89f88f9ed0b4" + # sighting + base_sighting = {"type": "sighting", "sighting_of_ref": "from_id", "where_sighted_refs": ["to_id"]} + assert gen_id(base_sighting) == 'sighting--161901df-21bb-527a-b96b-354119279fe2' + assert gen_id({**base_sighting, "first_seen": "2022-11-25T19:00:05.000Z"}) == "sighting--3c59ceea-8e41-5adb-a257-d070d19e6d2b" + assert gen_id({**base_sighting, "first_seen": "2022-11-25T19:00:05.000Z", "last_seen": "2022-11-26T19:00:05.000Z"}) == "sighting--b4d307b6-d22c-5f22-b530-876c298493da" # fmt: on diff --git a/tests/data/bundle_ids_sample.json b/tests/data/bundle_ids_sample.json index 7f26d27d2..2e4af49e2 100644 --- a/tests/data/bundle_ids_sample.json +++ b/tests/data/bundle_ids_sample.json @@ -59,7 +59,7 @@ ] }, { - "id": "relationship--ba52fced-422a-4bee-816a-85aa21c9eaca", + "id": "relationship--ba52fced-422a-4bee-816a-85aa21c9eacc", "type": "relationship", "spec_version": "2.1", "relationship_type": "related-to", @@ -71,6 +71,19 @@ "stop_time": "1900-01-01T00:00:00.000Z", "created_by_ref": "identity--7b82b010-b1c0-4dae-981f-7756374a17da", "object_marking_refs": ["marking-definition--78ca4366-f5b8-4764-83f7-34ce38198e27"] + }, + { + "type": "sighting", + "spec_version": "2.1", + "id": "sighting--ee20065d-2555-424f-ad9e-0f8428623c75", + "created": "2016-08-06T20:08:31.000Z", + "modified": "2016-09-06T20:08:31.000Z", + "sighting_of_ref": "malware--d650c5b9-4b43-5781-8576-ea52bd6c7ce5", + "where_sighted_refs": ["identity--7b82b010-b1c0-4dae-981f-7756374a17da"], + "first_seen": "2016-08-06T20:08:31.000Z", + "last_seen": "2016-08-07T20:08:31.000Z", + "count": 12, + "x_opencti_negative": true } ] } From 82b8e3d0cc555a1b3e39bd01f70fa02d1126a9a2 Mon Sep 17 00:00:00 2001 From: Julien Richard Date: Mon, 7 Oct 2024 23:21:24 +0200 Subject: [PATCH 06/17] [client] Adapt grouping (#7158) --- pycti/entities/opencti_grouping.py | 2 +- tests/01-unit/stix/test_bundle_ids_rewrite.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pycti/entities/opencti_grouping.py b/pycti/entities/opencti_grouping.py index 8284277a3..d71fa5856 100644 --- a/pycti/entities/opencti_grouping.py +++ b/pycti/entities/opencti_grouping.py @@ -408,7 +408,7 @@ def generate_id(name, context, created): @staticmethod def generate_id_from_data(data): - return Grouping.generate_id(data["name"], data["context"]) + return Grouping.generate_id(data["name"], data["context"], data["created"]) """ List Grouping objects diff --git a/tests/01-unit/stix/test_bundle_ids_rewrite.py b/tests/01-unit/stix/test_bundle_ids_rewrite.py index 2c49385f4..557c53aef 100644 --- a/tests/01-unit/stix/test_bundle_ids_rewrite.py +++ b/tests/01-unit/stix/test_bundle_ids_rewrite.py @@ -82,7 +82,7 @@ def test_ids_generation(): # data-source assert gen_id({"type": "data-source", "name": "data-source"}) == "data-source--f0925972-35e1-5172-9161-4d7180908339" # grouping - assert gen_id({"type": "grouping", "name": "grouping", "context": "context"}) == "grouping--8462bd42-4cad-54ae-a261-efc1a762d83d" + assert gen_id({"type": "grouping", "name": "grouping", "context": "context", "created": "2022-11-25T19:00:05.000Z"}) == "grouping--7c3e3534-9c09-568a-9485-377054b4c588" # language assert gen_id({"type": "language", "name": "fr"}) == "language--0ef28873-9d49-5cdb-a53a-eb7613391ee9" # malware-analysis From 756512f843db600fd543762b97690d1d70e8031e Mon Sep 17 00:00:00 2001 From: Julien Richard Date: Thu, 10 Oct 2024 22:56:01 +0200 Subject: [PATCH 07/17] [client] Add keep_original_id option --- pycti/utils/opencti_stix2.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pycti/utils/opencti_stix2.py b/pycti/utils/opencti_stix2.py index 120601551..4a3bb889c 100644 --- a/pycti/utils/opencti_stix2.py +++ b/pycti/utils/opencti_stix2.py @@ -176,7 +176,11 @@ def pick_aliases(self, stix_object: Dict) -> Optional[List]: return None def import_bundle_from_file( - self, file_path: str, update: bool = False, types: List = None + self, + file_path: str, + update: bool = False, + types: List = None, + keep_original_id: bool = False, ) -> Optional[List]: """import a stix2 bundle from a file @@ -186,6 +190,8 @@ def import_bundle_from_file( :type update: bool, optional :param types: list of stix2 types, defaults to None :type types: list, optional + :param keep_original_id: import need to keep original id + :type keep_original_id: bool, optional :return: list of imported stix2 objects :rtype: List """ @@ -194,7 +200,7 @@ def import_bundle_from_file( return None with open(os.path.join(file_path), encoding="utf-8") as file: data = json.load(file) - return self.import_bundle(data, update, types) + return self.import_bundle(data, update, types, None, keep_original_id) def import_bundle_from_json( self, @@ -2669,6 +2675,7 @@ def import_bundle( update: bool = False, types: List = None, work_id: str = None, + keep_original_id: bool = False, ) -> List: # Check if the bundle is correctly formatted if "type" not in stix_bundle or stix_bundle["type"] != "bundle": @@ -2683,7 +2690,7 @@ def import_bundle( # Bundle ids must be rewritten stix_bundle = self.prepare_bundle_ids( - bundle=stix_bundle, use_json=False, keep_original_id=False + bundle=stix_bundle, use_json=False, keep_original_id=keep_original_id ) stix2_splitter = OpenCTIStix2Splitter() From 39bcaa1a1596b1576cc5d48a05fb1bd5a385c799 Mon Sep 17 00:00:00 2001 From: Julien Richard Date: Fri, 11 Oct 2024 02:30:38 +0200 Subject: [PATCH 08/17] [client] Adapt attack pattern and relationship stix ids --- pycti/entities/opencti_attack_pattern.py | 3 ++- .../opencti_stix_core_relationship.py | 11 ++++++++ pycti/utils/opencti_stix2.py | 27 ++++++++++++++----- tests/01-unit/stix/test_bundle_ids_rewrite.py | 21 ++++++++++----- 4 files changed, 49 insertions(+), 13 deletions(-) diff --git a/pycti/entities/opencti_attack_pattern.py b/pycti/entities/opencti_attack_pattern.py index cce8dcfd1..beada5a16 100644 --- a/pycti/entities/opencti_attack_pattern.py +++ b/pycti/entities/opencti_attack_pattern.py @@ -232,7 +232,8 @@ def generate_id(name, x_mitre_id=None): @staticmethod def generate_id_from_data(data): - return AttackPattern.generate_id(data.get("name"), data.get("x_mitre_id")) + external_id = data.get("x_mitre_id") or data.get("x_opencti_external_id") + return AttackPattern.generate_id(data.get("name"), external_id) """ List Attack-Pattern objects diff --git a/pycti/entities/opencti_stix_core_relationship.py b/pycti/entities/opencti_stix_core_relationship.py index f4edbc55d..2d56eb21a 100644 --- a/pycti/entities/opencti_stix_core_relationship.py +++ b/pycti/entities/opencti_stix_core_relationship.py @@ -610,6 +610,7 @@ def create(self, **kwargs): kill_chain_phases = kwargs.get("killChainPhases", None) granted_refs = kwargs.get("objectOrganization", None) x_opencti_workflow_id = kwargs.get("x_opencti_workflow_id", None) + x_opencti_stix_ids = kwargs.get("x_opencti_stix_ids", None) update = kwargs.get("update", False) self.opencti.app_logger.info( @@ -653,6 +654,7 @@ def create(self, **kwargs): "externalReferences": external_references, "killChainPhases": kill_chain_phases, "x_opencti_workflow_id": x_opencti_workflow_id, + "x_opencti_stix_ids": x_opencti_stix_ids, "update": update, } }, @@ -1143,6 +1145,10 @@ def import_from_stix2(self, **kwargs): default_date = kwargs.get("defaultDate", False) if stix_relation is not None: # Search in extensions + if "x_opencti_stix_ids" not in stix_relation: + stix_relation["x_opencti_stix_ids"] = ( + self.opencti.get_attribute_in_extension("stix_ids", stix_relation) + ) if "x_opencti_granted_refs" not in stix_relation: stix_relation["x_opencti_granted_refs"] = ( self.opencti.get_attribute_in_extension( @@ -1224,6 +1230,11 @@ def import_from_stix2(self, **kwargs): if "x_opencti_workflow_id" in stix_relation else None ), + x_opencti_stix_ids=( + stix_relation["x_opencti_stix_ids"] + if "x_opencti_stix_ids" in stix_relation + else None + ), update=update, ) else: diff --git a/pycti/utils/opencti_stix2.py b/pycti/utils/opencti_stix2.py index 4a3bb889c..26614c523 100644 --- a/pycti/utils/opencti_stix2.py +++ b/pycti/utils/opencti_stix2.py @@ -2412,7 +2412,22 @@ def prepare_bundle_ids(self, bundle, use_json=True, keep_original_id=False): helper = stix_helpers.get(item["type"]) if hasattr(helper, "generate_id_from_data"): standard_id = helper.generate_id_from_data(item) - cache_ids[item["id"]] = standard_id + if item["id"] != standard_id: + cache_ids[item["id"]] = standard_id + + # If no id to rewrite, return the processed bundle + if not bool(cache_ids): + # As rewrite can have multiple roundtrip + # we kept the first original id in a meta attribute + # final behavior is to rewrite this in the x_opencti_stix_ids + for item in bundle_data["objects"]: + if "x_keep_original_id" in item: + item["x_opencti_stix_ids"] = item.get("x_opencti_stix_ids", []) + [ + item["x_keep_original_id"] + ] + del item["x_keep_original_id"] + return json.dumps(bundle_data) if use_json else bundle_data + # Second iteration to replace and remap for item in bundle_data["objects"]: # For entities, try to replace the main id @@ -2420,10 +2435,8 @@ def prepare_bundle_ids(self, bundle, use_json=True, keep_original_id=False): if cache_ids.get(item["id"]): original_id = item["id"] item["id"] = cache_ids[original_id] - if keep_original_id: - item["x_opencti_stix_ids"] = item.get("x_opencti_stix_ids", []) + [ - original_id - ] + if keep_original_id and "x_keep_original_id" not in item: + item["x_keep_original_id"] = original_id # For all elements, replace all refs (source_ref, object_refs, ...) ref_keys = list( filter(lambda i: i.endswith("_ref") or i.endswith("_refs"), item.keys()) @@ -2436,7 +2449,9 @@ def prepare_bundle_ids(self, bundle, use_json=True, keep_original_id=False): else: item[ref_key] = cache_ids.get(item[ref_key], item[ref_key]) - return json.dumps(bundle_data) if use_json else bundle_data + return self.prepare_bundle_ids( + bundle_data, False, keep_original_id=keep_original_id + ) def import_item( self, diff --git a/tests/01-unit/stix/test_bundle_ids_rewrite.py b/tests/01-unit/stix/test_bundle_ids_rewrite.py index 557c53aef..720b8a193 100644 --- a/tests/01-unit/stix/test_bundle_ids_rewrite.py +++ b/tests/01-unit/stix/test_bundle_ids_rewrite.py @@ -23,7 +23,8 @@ def test_ids_generation(): gen_id = get_cti_helper().generate_standard_id_from_stix # attack-pattern assert gen_id({"type": "attack-pattern", "name": "attack"}) =='attack-pattern--25f21617-8de8-5d5e-8cd4-b7e88547ba76' - assert gen_id({"type": "attack-pattern", "name": "attack", "x_mitre_id": 'MITREID'}) == 'attack-pattern--b74cfee2-7b14-585e-862f-fea45e802da9' + assert gen_id({"type": "attack-pattern", "name": "attack", "x_opencti_external_id": 'MITREID'}) == 'attack-pattern--b74cfee2-7b14-585e-862f-fea45e802da9' + assert gen_id({"type": "attack-pattern", "name": "Spear phishing messages with malicious links", "x_mitre_id": 'T1368'}) == 'attack-pattern--a01046cc-192f-5d52-8e75-6e447fae3890' assert gen_id({"type": "attack-pattern", "x_mitre_id": "MITREID"}) == 'attack-pattern--b74cfee2-7b14-585e-862f-fea45e802da9' # campaign assert gen_id({"type": "campaign", "name": "attack"}) == 'campaign--25f21617-8de8-5d5e-8cd4-b7e88547ba76' @@ -101,6 +102,7 @@ def test_ids_generation(): assert gen_id(base_relationship) == "relationship--0b11fa67-da01-5d34-9864-67d4d71c3740" assert gen_id({**base_relationship, "start_time": "2022-11-25T19:00:05.000Z"}) == "relationship--c5e1e2ce-14d6-535b-911d-267e92119e01" assert gen_id({**base_relationship, "start_time": "2022-11-25T19:00:05.000Z", "stop_time": "2022-11-26T19:00:05.000Z"}) == "relationship--a7778a7d-a743-5193-9912-89f88f9ed0b4" + assert gen_id({"type": "relationship", 'relationship_type': 'uses', 'source_ref': 'malware--21c45dbe-54ec-5bb7-b8cd-9f27cc518714', 'start_time': '2020-02-29T22:30:00.000Z', 'stop_time': '2020-02-29T22:30:00.000Z', 'target_ref': 'attack-pattern--fd8179dd-1632-5ec8-8b93-d2ae121e05a4'}) == 'relationship--67f5f01f-6b15-5154-ae31-019a75fedcff' # sighting base_sighting = {"type": "sighting", "sighting_of_ref": "from_id", "where_sighted_refs": ["to_id"]} assert gen_id(base_sighting) == 'sighting--161901df-21bb-527a-b96b-354119279fe2' @@ -112,18 +114,23 @@ def test_ids_generation(): def test_prepare_bundle_ids_keep_original(): helper = get_cti_helper() bundle_data = load_test_file() - malware_source = bundle_data["objects"][0] - assert malware_source["id"] == "malware--d650c5b9-4b43-5781-8576-ea52bd6c7ce5" - assert malware_source.get("x_opencti_stix_ids") is None + # malware_source = bundle_data["objects"][0] + # assert malware_source["id"] == "malware--d650c5b9-4b43-5781-8576-ea52bd6c7ce5" + # assert malware_source.get("x_opencti_stix_ids") is None prepared_bundle = helper.prepare_bundle_ids( bundle=bundle_data, use_json=False, keep_original_id=True ) - print(json.dumps(prepared_bundle)) malware_target = prepared_bundle["objects"][0] assert malware_target["id"] == "malware--d650c5b9-4b43-5781-8576-ea52bd6c7ce0" assert malware_target.get("x_opencti_stix_ids") == [ "malware--d650c5b9-4b43-5781-8576-ea52bd6c7ce5" ] + sighting = prepared_bundle["objects"][5] + assert sighting["id"] == "sighting--287d622a-9ffd-5e7f-bb0b-67f1e320f752" + assert ( + sighting["x_opencti_stix_ids"][0] + == "sighting--ee20065d-2555-424f-ad9e-0f8428623c75" + ) def test_prepare_bundle_ids(): @@ -135,7 +142,9 @@ def test_prepare_bundle_ids(): prepared_bundle = helper.prepare_bundle_ids( bundle=bundle_data, use_json=False, keep_original_id=False ) - print(json.dumps(prepared_bundle)) malware_target = prepared_bundle["objects"][0] assert malware_target["id"] == "malware--d650c5b9-4b43-5781-8576-ea52bd6c7ce0" assert malware_target.get("x_opencti_stix_ids") is None + sighting = prepared_bundle["objects"][5] + assert sighting["id"] == "sighting--287d622a-9ffd-5e7f-bb0b-67f1e320f752" + assert ("x_opencti_stix_ids" in sighting) == False From cb68e2fb97f1d2ff6c67fe0ecc9e49812b8e1c93 Mon Sep 17 00:00:00 2001 From: Julien Richard Date: Fri, 11 Oct 2024 09:59:08 +0200 Subject: [PATCH 09/17] [client] Add x_opencti_stix_ids in sightings --- pycti/utils/opencti_stix2.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pycti/utils/opencti_stix2.py b/pycti/utils/opencti_stix2.py index 26614c523..a19c93ad9 100644 --- a/pycti/utils/opencti_stix2.py +++ b/pycti/utils/opencti_stix2.py @@ -1395,6 +1395,11 @@ def import_sighting( if "x_opencti_workflow_id" in stix_sighting else None ), + x_opencti_stix_ids=( + stix_sighting["x_opencti_stix_ids"] + if "x_opencti_stix_ids" in stix_sighting + else None + ), update=update, ignore_dates=( stix_sighting["x_opencti_ignore_dates"] From 8cbe2ada566b56bccc56fb762ea8b7ca3f5c744f Mon Sep 17 00:00:00 2001 From: Julien Richard Date: Fri, 11 Oct 2024 12:36:59 +0200 Subject: [PATCH 10/17] [client] As rewrite is not possible, remove this approach --- pycti/connector/opencti_connector_helper.py | 5 - pycti/utils/opencti_stix2.py | 61 ------- tests/01-unit/stix/__init__.py | 0 tests/01-unit/stix/test_bundle_ids_rewrite.py | 150 ------------------ 4 files changed, 216 deletions(-) delete mode 100644 tests/01-unit/stix/__init__.py delete mode 100644 tests/01-unit/stix/test_bundle_ids_rewrite.py diff --git a/pycti/connector/opencti_connector_helper.py b/pycti/connector/opencti_connector_helper.py index b3657cf2b..2b40154b3 100644 --- a/pycti/connector/opencti_connector_helper.py +++ b/pycti/connector/opencti_connector_helper.py @@ -1589,11 +1589,6 @@ def send_stix2_bundle(self, bundle: str, **kwargs) -> list: "send_to_directory_retention", self.bundle_send_to_directory_retention ) - # Bundle ids must be rewritten - bundle = self.api.stix2.prepare_bundle_ids( - bundle=bundle, use_json=True, keep_original_id=keep_original_id - ) - # In case of enrichment ingestion, ensure the sharing if needed if self.enrichment_shared_organizations is not None: # Every element of the bundle must be enriched with the same organizations diff --git a/pycti/utils/opencti_stix2.py b/pycti/utils/opencti_stix2.py index a19c93ad9..e5b85c46d 100644 --- a/pycti/utils/opencti_stix2.py +++ b/pycti/utils/opencti_stix2.py @@ -2402,62 +2402,6 @@ def export_selected( return bundle - def prepare_bundle_ids(self, bundle, use_json=True, keep_original_id=False): - if use_json: - try: - bundle_data = json.loads(bundle) - except: - raise Exception("File data is not a valid JSON") - else: - bundle_data = bundle - cache_ids = {} - # First iteration to cache all entity ids - stix_helpers = self.get_stix_helper() - for item in bundle_data["objects"]: - helper = stix_helpers.get(item["type"]) - if hasattr(helper, "generate_id_from_data"): - standard_id = helper.generate_id_from_data(item) - if item["id"] != standard_id: - cache_ids[item["id"]] = standard_id - - # If no id to rewrite, return the processed bundle - if not bool(cache_ids): - # As rewrite can have multiple roundtrip - # we kept the first original id in a meta attribute - # final behavior is to rewrite this in the x_opencti_stix_ids - for item in bundle_data["objects"]: - if "x_keep_original_id" in item: - item["x_opencti_stix_ids"] = item.get("x_opencti_stix_ids", []) + [ - item["x_keep_original_id"] - ] - del item["x_keep_original_id"] - return json.dumps(bundle_data) if use_json else bundle_data - - # Second iteration to replace and remap - for item in bundle_data["objects"]: - # For entities, try to replace the main id - # Keep the current one if needed - if cache_ids.get(item["id"]): - original_id = item["id"] - item["id"] = cache_ids[original_id] - if keep_original_id and "x_keep_original_id" not in item: - item["x_keep_original_id"] = original_id - # For all elements, replace all refs (source_ref, object_refs, ...) - ref_keys = list( - filter(lambda i: i.endswith("_ref") or i.endswith("_refs"), item.keys()) - ) - for ref_key in ref_keys: - if ref_key.endswith("_refs"): - item[ref_key] = list( - map(lambda id_ref: cache_ids.get(id_ref, id_ref), item[ref_key]) - ) - else: - item[ref_key] = cache_ids.get(item[ref_key], item[ref_key]) - - return self.prepare_bundle_ids( - bundle_data, False, keep_original_id=keep_original_id - ) - def import_item( self, item, @@ -2708,11 +2652,6 @@ def import_bundle( else None ) - # Bundle ids must be rewritten - stix_bundle = self.prepare_bundle_ids( - bundle=stix_bundle, use_json=False, keep_original_id=keep_original_id - ) - stix2_splitter = OpenCTIStix2Splitter() _, bundles = stix2_splitter.split_bundle_with_expectations( stix_bundle, False, event_version diff --git a/tests/01-unit/stix/__init__.py b/tests/01-unit/stix/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/01-unit/stix/test_bundle_ids_rewrite.py b/tests/01-unit/stix/test_bundle_ids_rewrite.py deleted file mode 100644 index 720b8a193..000000000 --- a/tests/01-unit/stix/test_bundle_ids_rewrite.py +++ /dev/null @@ -1,150 +0,0 @@ -import json - -from pycti import OpenCTIApiClient, OpenCTIStix2 - - -def get_cti_helper(): - client = OpenCTIApiClient( - "http://fake:4000", "fake", ssl_verify=False, perform_health_check=False - ) - return OpenCTIStix2(client) - - -def load_test_file(): - with open("tests/data/bundle_ids_sample.json", "r") as content_file: - content = content_file.read() - bundle_data = json.loads(content) - return bundle_data - - -# !! WARNING !!, this need to be changed along with 01-unit/domain/identifier-test.js -# fmt: off -def test_ids_generation(): - gen_id = get_cti_helper().generate_standard_id_from_stix - # attack-pattern - assert gen_id({"type": "attack-pattern", "name": "attack"}) =='attack-pattern--25f21617-8de8-5d5e-8cd4-b7e88547ba76' - assert gen_id({"type": "attack-pattern", "name": "attack", "x_opencti_external_id": 'MITREID'}) == 'attack-pattern--b74cfee2-7b14-585e-862f-fea45e802da9' - assert gen_id({"type": "attack-pattern", "name": "Spear phishing messages with malicious links", "x_mitre_id": 'T1368'}) == 'attack-pattern--a01046cc-192f-5d52-8e75-6e447fae3890' - assert gen_id({"type": "attack-pattern", "x_mitre_id": "MITREID"}) == 'attack-pattern--b74cfee2-7b14-585e-862f-fea45e802da9' - # campaign - assert gen_id({"type": "campaign", "name": "attack"}) == 'campaign--25f21617-8de8-5d5e-8cd4-b7e88547ba76' - # note - assert gen_id({"type": "note", "content": "My note content!"}) == "note--2b4ab5af-2307-58e1-8862-a6a269aae798" - assert gen_id({"type": "note", "content": "My note content!", "created": "2022-11-25T19:00:05.000Z"}) == "note--10861e5c-049e-54f6-9736-81c106e39a0b" - # observed-data - assert gen_id({"type": "observed-data", "object_refs": ["id"]}) == "observed-data--4765c523-81bc-54c8-b1af-ee81d961dad1" - # opinion - assert gen_id({"type": "opinion", "opinion": "Good"}) == "opinion--0aef8829-207e-508b-b1f1-9da07f3379cb" - assert gen_id({"type": "opinion", "opinion": "Good", "created": "2022-11-25T19:00:05.000Z"}) == "opinion--941dbd61-c6b1-5290-b63f-19a38983d7f7" - # report - assert gen_id({"type": "report", "name": "Report", "published": "2022-11-25T19:00:05.000Z"}) == "report--761c6602-975f-5e5e-b220-7a2d41f33ce4" - # course-of-action - assert gen_id({"type": "course-of-action", "x_mitre_id": "MITREID"}) == "course-of-action--b74cfee2-7b14-585e-862f-fea45e802da9" - assert gen_id({"type": "course-of-action", "x_mitre_id": "MITREID", "name": "Name"}) == "course-of-action--b74cfee2-7b14-585e-862f-fea45e802da9" - assert gen_id({"type": "course-of-action", "name": "Name"}) == "course-of-action--e6e2ee8d-e54d-50cd-b77c-df8c8eea7726" - # identity - assert gen_id({"type": "identity", "name": "julien", "identity_class": "Individual"}) == "identity--d969b177-497f-598d-8428-b128c8f5f819" - assert gen_id({"type": "identity", "name": "julien", "identity_class": "Sector"}) == "identity--14ffa2a4-e16a-522a-937a-784c0ac1fab0" - assert gen_id({"type": "identity", "name": "julien", "identity_class": "System"}) == "identity--8af97482-121d-53f7-a533-9c48f06b5a38" - assert gen_id({"type": "identity", "name": "organization", "identity_class": "individual"}) == "identity--00f7eb8c-6af2-5ed5-9ede-ede4c623de3b" - # infrastructure - assert gen_id({"type": "infrastructure", "name": "infra"}) == "infrastructure--8a20116f-5a41-5508-ae4b-c293ac67c527" - # intrusion-set - assert gen_id({"type": "intrusion-set", "name": "intrusion"}) == "intrusion-set--30757026-c4bd-574d-ae52-8d8503b4818e" - # location - assert gen_id({"type": "location", "name": "Lyon", "x_opencti_location_type": "City"}) == "location--da430873-42c8-57ca-b08b-a797558c6cbd" - assert gen_id({"type": "location", "latitude": 5.12, "name": "Position1", "x_opencti_location_type": "Position"}) == "location--56b3fc50-5091-5f2e-bd19-7b40ee3881e4" - assert gen_id({"type": "location", "longitude": 5.12, "name": 'Position2', "x_opencti_location_type": "Position"}) == "location--dd2cf94c-1d58-58a1-b21f-0ede4059aaf0" - assert gen_id({"type": "location", "latitude": 5.12, "longitude": 5.12, "x_opencti_location_type": "Position"}) == "location--57acef55-747a-55ef-9c49-06ca85f8d749" - assert gen_id({"type": "location", "name": 'Position3', "x_opencti_location_type": "Position"}) == "location--a4152781-8721-5d44-ae2d-e492665bc35b" - # malware - assert gen_id({"type": "malware", "name": "malware"}) == "malware--92ddf766-b27c-5159-8f46-27002bba2f04" - # threat-actor-group - assert gen_id({"type": "threat-actor", "name": "CARD04"}) == "threat-actor--6d458783-df3b-5398-8e30-282655ad7b94" - assert gen_id({"type": "threat-actor", "name": "CARD04", "x_opencti_type": "Threat-Actor-Group"}) == "threat-actor--6d458783-df3b-5398-8e30-282655ad7b94" - # tool - assert gen_id({"type": "tool", "name": "my-tool"}) == "tool--41cd21d0-f50e-5e3d-83fc-447e0def97b7" - # vulnerability - assert gen_id({"type": "vulnerability", "name": "vulnerability"}) == "vulnerability--2c690168-aec3-57f1-8295-adf53f4dc3da" - # incident - assert gen_id({"type": "incident", "name": "incident", "created": "2022-11-25T19:00:05.000Z"}) == "incident--0e117c15-0a94-5ad3-b090-0395613f5b29" - # case-incident - assert gen_id({"type": "case-incident", "name": "case", "created": "2022-11-25T19:00:05.000Z"}) == "case-incident--4838a141-bd19-542c-85d9-cce0382645b5" - # case-rfi - assert gen_id({"type": "case-rfi", "name": "case", "created": "2022-11-25T19:00:05.000Z"}) == "case-rfi--4838a141-bd19-542c-85d9-cce0382645b5" - # case-rft - assert gen_id({"type": "case-rft", "name": "case", "created": "2022-11-25T19:00:05.000Z"}) == "case-rft--4838a141-bd19-542c-85d9-cce0382645b5" - # feedback, not supported yet - # assert gen_id("case-feedback", {"name": "case", "created": "2022-11-25T19:00:05.000Z"}) == "feedback--4838a141-bd19-542c-85d9-cce0382645b5" - # channel - assert gen_id({"type": "channel", "name": "channel"}) == "channel--4936cdd5-6b6a-5c92-a756-cae1f09dcd80" - # data-component - assert gen_id({"type": "data-component", "name": "data-component"}) == "data-component--32fdc52a-b4c5-5268-af2f-cdf820271f0b" - # data-source - assert gen_id({"type": "data-source", "name": "data-source"}) == "data-source--f0925972-35e1-5172-9161-4d7180908339" - # grouping - assert gen_id({"type": "grouping", "name": "grouping", "context": "context", "created": "2022-11-25T19:00:05.000Z"}) == "grouping--7c3e3534-9c09-568a-9485-377054b4c588" - # language - assert gen_id({"type": "language", "name": "fr"}) == "language--0ef28873-9d49-5cdb-a53a-eb7613391ee9" - # malware-analysis - assert gen_id({"type": "malware-analysis", "product": "linux", "result_name": "result"}) == "malware-analysis--3d501241-a4a5-574d-a503-301a6426f8c1" - assert gen_id({"type": "malware-analysis", "product": "linux", "result_name": "result", "submitted": "2022-11-25T19:00:05.000Z"}) == "malware-analysis--d7ffe68a-0d5f-5fea-a375-3338ba4ea13c" - # narrative - assert gen_id({"type": "narrative", "name": "narrative"}) == "narrative--804a7e40-d39c-59b6-9e3f-1ba1bc92b739" - # task - assert gen_id({"type": "task", "name": "case", "created": "2022-11-25T19:00:05.000Z"}) == "task--4838a141-bd19-542c-85d9-cce0382645b5" - # Threat-actor-individual - assert gen_id({"type": "threat-actor", "name": "CARD04", "x_opencti_type": "Threat-Actor-Individual"}) == "threat-actor--af15b6ae-a3dd-54d3-8fa0-3adfe0391d01" - # vocabulary - assert gen_id({"type": "vocabulary", "name": "facebook", "category": "account_type_ov"}) == "vocabulary--85ae7185-ff6f-509b-a011-3069921614aa" - # relationship - base_relationship = {"type": "relationship", "relationship_type": "based-on", "source_ref": "from_id", "target_ref": "to_id"} - assert gen_id(base_relationship) == "relationship--0b11fa67-da01-5d34-9864-67d4d71c3740" - assert gen_id({**base_relationship, "start_time": "2022-11-25T19:00:05.000Z"}) == "relationship--c5e1e2ce-14d6-535b-911d-267e92119e01" - assert gen_id({**base_relationship, "start_time": "2022-11-25T19:00:05.000Z", "stop_time": "2022-11-26T19:00:05.000Z"}) == "relationship--a7778a7d-a743-5193-9912-89f88f9ed0b4" - assert gen_id({"type": "relationship", 'relationship_type': 'uses', 'source_ref': 'malware--21c45dbe-54ec-5bb7-b8cd-9f27cc518714', 'start_time': '2020-02-29T22:30:00.000Z', 'stop_time': '2020-02-29T22:30:00.000Z', 'target_ref': 'attack-pattern--fd8179dd-1632-5ec8-8b93-d2ae121e05a4'}) == 'relationship--67f5f01f-6b15-5154-ae31-019a75fedcff' - # sighting - base_sighting = {"type": "sighting", "sighting_of_ref": "from_id", "where_sighted_refs": ["to_id"]} - assert gen_id(base_sighting) == 'sighting--161901df-21bb-527a-b96b-354119279fe2' - assert gen_id({**base_sighting, "first_seen": "2022-11-25T19:00:05.000Z"}) == "sighting--3c59ceea-8e41-5adb-a257-d070d19e6d2b" - assert gen_id({**base_sighting, "first_seen": "2022-11-25T19:00:05.000Z", "last_seen": "2022-11-26T19:00:05.000Z"}) == "sighting--b4d307b6-d22c-5f22-b530-876c298493da" -# fmt: on - - -def test_prepare_bundle_ids_keep_original(): - helper = get_cti_helper() - bundle_data = load_test_file() - # malware_source = bundle_data["objects"][0] - # assert malware_source["id"] == "malware--d650c5b9-4b43-5781-8576-ea52bd6c7ce5" - # assert malware_source.get("x_opencti_stix_ids") is None - prepared_bundle = helper.prepare_bundle_ids( - bundle=bundle_data, use_json=False, keep_original_id=True - ) - malware_target = prepared_bundle["objects"][0] - assert malware_target["id"] == "malware--d650c5b9-4b43-5781-8576-ea52bd6c7ce0" - assert malware_target.get("x_opencti_stix_ids") == [ - "malware--d650c5b9-4b43-5781-8576-ea52bd6c7ce5" - ] - sighting = prepared_bundle["objects"][5] - assert sighting["id"] == "sighting--287d622a-9ffd-5e7f-bb0b-67f1e320f752" - assert ( - sighting["x_opencti_stix_ids"][0] - == "sighting--ee20065d-2555-424f-ad9e-0f8428623c75" - ) - - -def test_prepare_bundle_ids(): - helper = get_cti_helper() - bundle_data = load_test_file() - malware_source = bundle_data["objects"][0] - assert malware_source["id"] == "malware--d650c5b9-4b43-5781-8576-ea52bd6c7ce5" - assert malware_source.get("x_opencti_stix_ids") is None - prepared_bundle = helper.prepare_bundle_ids( - bundle=bundle_data, use_json=False, keep_original_id=False - ) - malware_target = prepared_bundle["objects"][0] - assert malware_target["id"] == "malware--d650c5b9-4b43-5781-8576-ea52bd6c7ce0" - assert malware_target.get("x_opencti_stix_ids") is None - sighting = prepared_bundle["objects"][5] - assert sighting["id"] == "sighting--287d622a-9ffd-5e7f-bb0b-67f1e320f752" - assert ("x_opencti_stix_ids" in sighting) == False From bdefe0a4bc49ce2167f7356c3fdd8372f19d17c0 Mon Sep 17 00:00:00 2001 From: Julien Richard Date: Fri, 11 Oct 2024 13:31:12 +0200 Subject: [PATCH 11/17] [client] As rewrite is not possible, remove this approach --- pycti/connector/opencti_connector_helper.py | 8 -------- pycti/utils/opencti_stix2.py | 6 +----- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/pycti/connector/opencti_connector_helper.py b/pycti/connector/opencti_connector_helper.py index 2b40154b3..4ebdb36a0 100644 --- a/pycti/connector/opencti_connector_helper.py +++ b/pycti/connector/opencti_connector_helper.py @@ -849,13 +849,6 @@ def __init__(self, config: Dict, playbook_compatible=False) -> None: False, True, ) - self.keep_original_id = get_config_variable( - "CONNECTOR_KEEP_ORIGINAL_ID", - ["connector", "keep_original_id"], - config, - False, - False, - ) self.bundle_send_to_directory = get_config_variable( "CONNECTOR_SEND_TO_DIRECTORY", ["connector", "send_to_directory"], @@ -1578,7 +1571,6 @@ def send_stix2_bundle(self, bundle: str, **kwargs) -> list: file_name = kwargs.get("file_name", None) bundle_send_to_queue = kwargs.get("send_to_queue", self.bundle_send_to_queue) cleanup_inconsistent_bundle = kwargs.get("cleanup_inconsistent_bundle", False) - keep_original_id = kwargs.get("keep_original_id", self.keep_original_id) bundle_send_to_directory = kwargs.get( "send_to_directory", self.bundle_send_to_directory ) diff --git a/pycti/utils/opencti_stix2.py b/pycti/utils/opencti_stix2.py index e5b85c46d..73ed63a3f 100644 --- a/pycti/utils/opencti_stix2.py +++ b/pycti/utils/opencti_stix2.py @@ -180,7 +180,6 @@ def import_bundle_from_file( file_path: str, update: bool = False, types: List = None, - keep_original_id: bool = False, ) -> Optional[List]: """import a stix2 bundle from a file @@ -190,8 +189,6 @@ def import_bundle_from_file( :type update: bool, optional :param types: list of stix2 types, defaults to None :type types: list, optional - :param keep_original_id: import need to keep original id - :type keep_original_id: bool, optional :return: list of imported stix2 objects :rtype: List """ @@ -200,7 +197,7 @@ def import_bundle_from_file( return None with open(os.path.join(file_path), encoding="utf-8") as file: data = json.load(file) - return self.import_bundle(data, update, types, None, keep_original_id) + return self.import_bundle(data, update, types, None) def import_bundle_from_json( self, @@ -2639,7 +2636,6 @@ def import_bundle( update: bool = False, types: List = None, work_id: str = None, - keep_original_id: bool = False, ) -> List: # Check if the bundle is correctly formatted if "type" not in stix_bundle or stix_bundle["type"] != "bundle": From 204bed0235a3e35e9fd4d5e1622aa57af855edd0 Mon Sep 17 00:00:00 2001 From: Julien Richard Date: Fri, 11 Oct 2024 21:15:24 +0200 Subject: [PATCH 12/17] [client] Reintroduce gen id testing --- tests/01-unit/stix/__init__.py | 0 tests/01-unit/stix/test_bundle_ids_rewrite.py | 112 ++++++++++++++++++ 2 files changed, 112 insertions(+) create mode 100644 tests/01-unit/stix/__init__.py create mode 100644 tests/01-unit/stix/test_bundle_ids_rewrite.py diff --git a/tests/01-unit/stix/__init__.py b/tests/01-unit/stix/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/01-unit/stix/test_bundle_ids_rewrite.py b/tests/01-unit/stix/test_bundle_ids_rewrite.py new file mode 100644 index 000000000..16143cec1 --- /dev/null +++ b/tests/01-unit/stix/test_bundle_ids_rewrite.py @@ -0,0 +1,112 @@ +import json + +from pycti import OpenCTIApiClient, OpenCTIStix2 + + +def get_cti_helper(): + client = OpenCTIApiClient( + "http://fake:4000", "fake", ssl_verify=False, perform_health_check=False + ) + return OpenCTIStix2(client) + + +def load_test_file(): + with open("tests/data/bundle_ids_sample.json", "r") as content_file: + content = content_file.read() + bundle_data = json.loads(content) + return bundle_data + + +# !! WARNING !!, this need to be changed along with 01-unit/domain/identifier-test.js +# fmt: off +def test_ids_generation(): + gen_id = get_cti_helper().generate_standard_id_from_stix + # attack-pattern + assert gen_id({"type": "attack-pattern", "name": "attack"}) =='attack-pattern--25f21617-8de8-5d5e-8cd4-b7e88547ba76' + assert gen_id({"type": "attack-pattern", "name": "attack", "x_opencti_external_id": 'MITREID'}) == 'attack-pattern--b74cfee2-7b14-585e-862f-fea45e802da9' + assert gen_id({"type": "attack-pattern", "name": "Spear phishing messages with malicious links", "x_mitre_id": 'T1368'}) == 'attack-pattern--a01046cc-192f-5d52-8e75-6e447fae3890' + assert gen_id({"type": "attack-pattern", "x_mitre_id": "MITREID"}) == 'attack-pattern--b74cfee2-7b14-585e-862f-fea45e802da9' + assert gen_id({"type": "attack-pattern", "name": "Evil Pattern!", "description": "Test Attack Pattern!"}) == 'attack-pattern--23a5b210-f675-5936-ae14-21327e9798e2' + # campaign + assert gen_id({"type": "campaign", "name": "attack"}) == 'campaign--25f21617-8de8-5d5e-8cd4-b7e88547ba76' + # note + assert gen_id({"type": "note", "content": "My note content!"}) == "note--2b4ab5af-2307-58e1-8862-a6a269aae798" + assert gen_id({"type": "note", "content": "My note content!", "created": "2022-11-25T19:00:05.000Z"}) == "note--10861e5c-049e-54f6-9736-81c106e39a0b" + # observed-data + assert gen_id({"type": "observed-data", "object_refs": ["id"]}) == "observed-data--4765c523-81bc-54c8-b1af-ee81d961dad1" + # opinion + assert gen_id({"type": "opinion", "opinion": "Good"}) == "opinion--0aef8829-207e-508b-b1f1-9da07f3379cb" + assert gen_id({"type": "opinion", "opinion": "Good", "created": "2022-11-25T19:00:05.000Z"}) == "opinion--941dbd61-c6b1-5290-b63f-19a38983d7f7" + # report + assert gen_id({"type": "report", "name": "Report", "published": "2022-11-25T19:00:05.000Z"}) == "report--761c6602-975f-5e5e-b220-7a2d41f33ce4" + # course-of-action + assert gen_id({"type": "course-of-action", "x_mitre_id": "MITREID"}) == "course-of-action--b74cfee2-7b14-585e-862f-fea45e802da9" + assert gen_id({"type": "course-of-action", "x_mitre_id": "MITREID", "name": "Name"}) == "course-of-action--b74cfee2-7b14-585e-862f-fea45e802da9" + assert gen_id({"type": "course-of-action", "name": "Name"}) == "course-of-action--e6e2ee8d-e54d-50cd-b77c-df8c8eea7726" + # identity + assert gen_id({"type": "identity", "name": "julien", "identity_class": "Individual"}) == "identity--d969b177-497f-598d-8428-b128c8f5f819" + assert gen_id({"type": "identity", "name": "julien", "identity_class": "Sector"}) == "identity--14ffa2a4-e16a-522a-937a-784c0ac1fab0" + assert gen_id({"type": "identity", "name": "julien", "identity_class": "System"}) == "identity--8af97482-121d-53f7-a533-9c48f06b5a38" + assert gen_id({"type": "identity", "name": "organization", "identity_class": "individual"}) == "identity--00f7eb8c-6af2-5ed5-9ede-ede4c623de3b" + # infrastructure + assert gen_id({"type": "infrastructure", "name": "infra"}) == "infrastructure--8a20116f-5a41-5508-ae4b-c293ac67c527" + # intrusion-set + assert gen_id({"type": "intrusion-set", "name": "intrusion"}) == "intrusion-set--30757026-c4bd-574d-ae52-8d8503b4818e" + # location + assert gen_id({"type": "location", "name": "Lyon", "x_opencti_location_type": "City"}) == "location--da430873-42c8-57ca-b08b-a797558c6cbd" + assert gen_id({"type": "location", "latitude": 5.12, "name": "Position1", "x_opencti_location_type": "Position"}) == "location--56b3fc50-5091-5f2e-bd19-7b40ee3881e4" + assert gen_id({"type": "location", "longitude": 5.12, "name": 'Position2', "x_opencti_location_type": "Position"}) == "location--dd2cf94c-1d58-58a1-b21f-0ede4059aaf0" + assert gen_id({"type": "location", "latitude": 5.12, "longitude": 5.12, "x_opencti_location_type": "Position"}) == "location--57acef55-747a-55ef-9c49-06ca85f8d749" + assert gen_id({"type": "location", "name": 'Position3', "x_opencti_location_type": "Position"}) == "location--a4152781-8721-5d44-ae2d-e492665bc35b" + # malware + assert gen_id({"type": "malware", "name": "malware"}) == "malware--92ddf766-b27c-5159-8f46-27002bba2f04" + # threat-actor-group + assert gen_id({"type": "threat-actor", "name": "CARD04"}) == "threat-actor--6d458783-df3b-5398-8e30-282655ad7b94" + assert gen_id({"type": "threat-actor", "name": "CARD04", "x_opencti_type": "Threat-Actor-Group"}) == "threat-actor--6d458783-df3b-5398-8e30-282655ad7b94" + # tool + assert gen_id({"type": "tool", "name": "my-tool"}) == "tool--41cd21d0-f50e-5e3d-83fc-447e0def97b7" + # vulnerability + assert gen_id({"type": "vulnerability", "name": "vulnerability"}) == "vulnerability--2c690168-aec3-57f1-8295-adf53f4dc3da" + # incident + assert gen_id({"type": "incident", "name": "incident", "created": "2022-11-25T19:00:05.000Z"}) == "incident--0e117c15-0a94-5ad3-b090-0395613f5b29" + # case-incident + assert gen_id({"type": "case-incident", "name": "case", "created": "2022-11-25T19:00:05.000Z"}) == "case-incident--4838a141-bd19-542c-85d9-cce0382645b5" + # case-rfi + assert gen_id({"type": "case-rfi", "name": "case", "created": "2022-11-25T19:00:05.000Z"}) == "case-rfi--4838a141-bd19-542c-85d9-cce0382645b5" + # case-rft + assert gen_id({"type": "case-rft", "name": "case", "created": "2022-11-25T19:00:05.000Z"}) == "case-rft--4838a141-bd19-542c-85d9-cce0382645b5" + # feedback, not supported yet + # assert gen_id("case-feedback", {"name": "case", "created": "2022-11-25T19:00:05.000Z"}) == "feedback--4838a141-bd19-542c-85d9-cce0382645b5" + # channel + assert gen_id({"type": "channel", "name": "channel"}) == "channel--4936cdd5-6b6a-5c92-a756-cae1f09dcd80" + # data-component + assert gen_id({"type": "data-component", "name": "data-component"}) == "data-component--32fdc52a-b4c5-5268-af2f-cdf820271f0b" + # data-source + assert gen_id({"type": "data-source", "name": "data-source"}) == "data-source--f0925972-35e1-5172-9161-4d7180908339" + # grouping + assert gen_id({"type": "grouping", "name": "grouping", "context": "context", "created": "2022-11-25T19:00:05.000Z"}) == "grouping--7c3e3534-9c09-568a-9485-377054b4c588" + # language + assert gen_id({"type": "language", "name": "fr"}) == "language--0ef28873-9d49-5cdb-a53a-eb7613391ee9" + # malware-analysis + assert gen_id({"type": "malware-analysis", "product": "linux", "result_name": "result"}) == "malware-analysis--3d501241-a4a5-574d-a503-301a6426f8c1" + assert gen_id({"type": "malware-analysis", "product": "linux", "result_name": "result", "submitted": "2022-11-25T19:00:05.000Z"}) == "malware-analysis--d7ffe68a-0d5f-5fea-a375-3338ba4ea13c" + # narrative + assert gen_id({"type": "narrative", "name": "narrative"}) == "narrative--804a7e40-d39c-59b6-9e3f-1ba1bc92b739" + # task + assert gen_id({"type": "task", "name": "case", "created": "2022-11-25T19:00:05.000Z"}) == "task--4838a141-bd19-542c-85d9-cce0382645b5" + # Threat-actor-individual + assert gen_id({"type": "threat-actor", "name": "CARD04", "x_opencti_type": "Threat-Actor-Individual"}) == "threat-actor--af15b6ae-a3dd-54d3-8fa0-3adfe0391d01" + # vocabulary + assert gen_id({"type": "vocabulary", "name": "facebook", "category": "account_type_ov"}) == "vocabulary--85ae7185-ff6f-509b-a011-3069921614aa" + # relationship + base_relationship = {"type": "relationship", "relationship_type": "based-on", "source_ref": "from_id", "target_ref": "to_id"} + assert gen_id(base_relationship) == "relationship--0b11fa67-da01-5d34-9864-67d4d71c3740" + assert gen_id({**base_relationship, "start_time": "2022-11-25T19:00:05.000Z"}) == "relationship--c5e1e2ce-14d6-535b-911d-267e92119e01" + assert gen_id({**base_relationship, "start_time": "2022-11-25T19:00:05.000Z", "stop_time": "2022-11-26T19:00:05.000Z"}) == "relationship--a7778a7d-a743-5193-9912-89f88f9ed0b4" + assert gen_id({"type": "relationship", 'relationship_type': 'uses', 'source_ref': 'malware--21c45dbe-54ec-5bb7-b8cd-9f27cc518714', 'start_time': '2020-02-29T22:30:00.000Z', 'stop_time': '2020-02-29T22:30:00.000Z', 'target_ref': 'attack-pattern--fd8179dd-1632-5ec8-8b93-d2ae121e05a4'}) == 'relationship--67f5f01f-6b15-5154-ae31-019a75fedcff' + # sighting + base_sighting = {"type": "sighting", "sighting_of_ref": "from_id", "where_sighted_refs": ["to_id"]} + assert gen_id(base_sighting) == 'sighting--161901df-21bb-527a-b96b-354119279fe2' + assert gen_id({**base_sighting, "first_seen": "2022-11-25T19:00:05.000Z"}) == "sighting--3c59ceea-8e41-5adb-a257-d070d19e6d2b" + assert gen_id({**base_sighting, "first_seen": "2022-11-25T19:00:05.000Z", "last_seen": "2022-11-26T19:00:05.000Z"}) == "sighting--b4d307b6-d22c-5f22-b530-876c298493da" +# fmt: on From a1cb55f23af3ed42f546359e410bf5ea2bc258cf Mon Sep 17 00:00:00 2001 From: Julien Richard Date: Fri, 11 Oct 2024 22:44:16 +0200 Subject: [PATCH 13/17] [client] Rollback test --- tests/02-integration/utils/test_stix_crud.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/02-integration/utils/test_stix_crud.py b/tests/02-integration/utils/test_stix_crud.py index 2f6cf3cea..edadba800 100644 --- a/tests/02-integration/utils/test_stix_crud.py +++ b/tests/02-integration/utils/test_stix_crud.py @@ -24,8 +24,7 @@ def test_entity_create(entity_class, api_stix, opencti_splitter): bundles_sent = api_stix.import_bundle_from_json(split_bundle, False, None, None) assert len(bundles_sent) == 1 - element_new_id = api_stix.generate_standard_id_from_stix(stix_object) - assert bundles_sent[0]["id"] == element_new_id + assert bundles_sent[0]["id"] == stix_object["id"] assert bundles_sent[0]["type"] == stix_object["type"] - entity_class.base_class().delete(id=element_new_id) + entity_class.base_class().delete(id=stix_object["id"]) From 08b045c5dbe17aedbc81676882452ae13ecba37f Mon Sep 17 00:00:00 2001 From: Julien Richard Date: Sun, 13 Oct 2024 12:11:31 +0200 Subject: [PATCH 14/17] [connectors] Adapt pylint, adding global rule no-value-for-parameter --- pycti/entities/opencti_stix_sighting_relationship.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pycti/entities/opencti_stix_sighting_relationship.py b/pycti/entities/opencti_stix_sighting_relationship.py index b106a8bb0..3a44e4b2c 100644 --- a/pycti/entities/opencti_stix_sighting_relationship.py +++ b/pycti/entities/opencti_stix_sighting_relationship.py @@ -263,7 +263,6 @@ def __init__(self, opencti): @staticmethod def generate_id( - relationship_type, sighting_of_ref, where_sighted_refs, first_seen=None, @@ -276,7 +275,7 @@ def generate_id( if first_seen is not None and last_seen is not None: data = { - "type": relationship_type, + "type": "sighting", "sighting_of_ref": sighting_of_ref, "where_sighted_refs": where_sighted_refs, "first_seen": first_seen, @@ -284,14 +283,14 @@ def generate_id( } elif first_seen is not None: data = { - "type": relationship_type, + "type": "sighting", "sighting_of_ref": sighting_of_ref, "where_sighted_refs": where_sighted_refs, "first_seen": first_seen, } else: data = { - "type": relationship_type, + "type": "sighting", "sighting_of_ref": sighting_of_ref, "where_sighted_refs": where_sighted_refs, } @@ -302,7 +301,6 @@ def generate_id( @staticmethod def generate_id_from_data(data): return StixSightingRelationship.generate_id( - data["type"], data["sighting_of_ref"], data["where_sighted_refs"], data.get("first_seen"), From e6aca56ac61a1710a615f5ef860ca42070cff450 Mon Sep 17 00:00:00 2001 From: Julien Richard Date: Mon, 14 Oct 2024 09:55:05 +0200 Subject: [PATCH 15/17] [connectors] Adapt linter + more fixes and take care of sightings --- pycti/entities/opencti_grouping.py | 7 +++++-- pycti/utils/opencti_stix2.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/pycti/entities/opencti_grouping.py b/pycti/entities/opencti_grouping.py index d71fa5856..a49c9dda4 100644 --- a/pycti/entities/opencti_grouping.py +++ b/pycti/entities/opencti_grouping.py @@ -396,12 +396,15 @@ def __init__(self, opencti): """ @staticmethod - def generate_id(name, context, created): + def generate_id(name, context, created = None): name = name.lower().strip() context = context.lower().strip() if isinstance(created, datetime.datetime): created = created.isoformat() - data = {"name": name, "context": context, "created": created} + if created is None: + data = {"name": name, "context": context} + else: + data = {"name": name, "context": context, "created": created} data = canonicalize(data, utf8=False) id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data)) return "grouping--" + id diff --git a/pycti/utils/opencti_stix2.py b/pycti/utils/opencti_stix2.py index 73ed63a3f..d8bea790a 100644 --- a/pycti/utils/opencti_stix2.py +++ b/pycti/utils/opencti_stix2.py @@ -2427,7 +2427,7 @@ def import_item( # region Resolve the to to_ids = [] if "x_opencti_where_sighted_refs" in item: - for where_sighted_ref in item["_opencti_where_sighted_refs"]: + for where_sighted_ref in item["x_opencti_where_sighted_refs"]: to_ids.append(where_sighted_ref) elif "where_sighted_refs" in item: for where_sighted_ref in item["where_sighted_refs"]: From 7fb126a0205d92dd3f0f2b5d5a51083f58145e86 Mon Sep 17 00:00:00 2001 From: Julien Richard Date: Tue, 15 Oct 2024 11:52:49 +0100 Subject: [PATCH 16/17] [client] Fix black --- pycti/entities/opencti_grouping.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pycti/entities/opencti_grouping.py b/pycti/entities/opencti_grouping.py index a49c9dda4..f3d0be6ea 100644 --- a/pycti/entities/opencti_grouping.py +++ b/pycti/entities/opencti_grouping.py @@ -396,7 +396,7 @@ def __init__(self, opencti): """ @staticmethod - def generate_id(name, context, created = None): + def generate_id(name, context, created=None): name = name.lower().strip() context = context.lower().strip() if isinstance(created, datetime.datetime): From 2f3a792bcb940f532390057897ca18620b11ef61 Mon Sep 17 00:00:00 2001 From: Helene Nguyen Date: Wed, 6 Nov 2024 14:17:27 +0100 Subject: [PATCH 17/17] [client] Add None for generate_id malware analysis for retro compat --- pycti/entities/opencti_malware_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pycti/entities/opencti_malware_analysis.py b/pycti/entities/opencti_malware_analysis.py index 5740f9f81..2c44e7abc 100644 --- a/pycti/entities/opencti_malware_analysis.py +++ b/pycti/entities/opencti_malware_analysis.py @@ -219,7 +219,7 @@ def __init__(self, opencti): """ @staticmethod - def generate_id(result_name, product, submitted): + def generate_id(result_name, product=None, submitted=None): result_name = result_name.lower().strip() data = {"result_name": result_name, "product": product} if submitted is not None: