diff --git a/tests/test_schemas.py b/tests/test_schemas.py index 9dcbff3670e..fe9c0e9b770 100644 --- a/tests/test_schemas.py +++ b/tests/test_schemas.py @@ -1,15 +1,21 @@ """Test format specifications""" -from utils import BaseTestCase, interfaces +from utils import BaseTestCase, interfaces, context, skipif class Test_Library(BaseTestCase): + @skipif(context.library <= "java@0.87.0", reason="missing feature") + @skipif(context.library > "java@0.87.0", reason="known bug in the http context") + @skipif(context.library == "dotnet", reason="known bug in the http context") def test_library_format(self): """Libraries's payload are valid regarding schemas""" interfaces.library.assert_schemas() class Test_Agent(BaseTestCase): + @skipif(context.library <= "java@0.87.0", reason="missing feature") + @skipif(context.library > "java@0.87.0", reason="known bug in the http context") + @skipif(context.library == "dotnet", reason="known bug in the http context") def test_agent_format(self): """Agents's payload are valid regarding schemas""" interfaces.agent.assert_schemas() diff --git a/utils/interfaces/_schemas_validators.py b/utils/interfaces/_schemas_validators.py index 1c3e55c091b..cb96bae5726 100644 --- a/utils/interfaces/_schemas_validators.py +++ b/utils/interfaces/_schemas_validators.py @@ -42,6 +42,7 @@ def _get_schemas_store(): for filename in _get_schemas_filenames(): schema = json.load(open(filename)) + assert "$id" in schema, filename assert schema["$id"] == filename[len("utils/interfaces/schemas") :], filename Draft7Validator.check_schema(schema) diff --git a/utils/interfaces/schemas/Dockerfile b/utils/interfaces/schemas/Dockerfile index 8a51fa2cb95..936bef94662 100644 --- a/utils/interfaces/schemas/Dockerfile +++ b/utils/interfaces/schemas/Dockerfile @@ -1,11 +1,13 @@ -FROM python:3.7 +FROM python:3.9 -COPY . /app +COPY ./requirements.txt /app/requirements.txt WORKDIR /app RUN pip install -r requirements.txt RUN pip install json-schema-for-humans==0.33.2 RUN pip install gunicorn +COPY . /app + EXPOSE 80 -CMD PYTHONPATH=. exec gunicorn -w 4 -b 0.0.0.0:80 utils.interfaces.schemas.serve_doc:app \ No newline at end of file +CMD PYTHONPATH=. exec gunicorn -w 4 -b 0.0.0.0:80 utils.interfaces.schemas.serve_doc:app diff --git a/utils/interfaces/schemas/agent/api/v2/appsecevts-request.json b/utils/interfaces/schemas/agent/api/v2/appsecevts-request.json index a07b4f92342..99c59f5bf4d 100644 --- a/utils/interfaces/schemas/agent/api/v2/appsecevts-request.json +++ b/utils/interfaces/schemas/agent/api/v2/appsecevts-request.json @@ -1,4 +1,24 @@ { - "$id": "/agent/api/v2/appsecevts-request.json", - "type": "object" - } \ No newline at end of file + "$id": "/agent/api/v2/appsecevts-request.json", + + "properties": { + "protocol_version": { + "type": "integer" + }, + "idempotency_key": { + "type": "string" + }, + "events": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "/miscs/appsec/events/_definitions/all_events.json" + } + } + }, + "required": [ + "protocol_version", + "idempotency_key", + "events" + ] +} diff --git a/utils/interfaces/schemas/library/appsec/miscs/attack-report-payload.json b/utils/interfaces/schemas/library/appsec/miscs/attack-report-payload.json deleted file mode 100644 index b3e055e29e8..00000000000 --- a/utils/interfaces/schemas/library/appsec/miscs/attack-report-payload.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "$id": "/library/appsec/miscs/attack-report-payload.json", - "properties": { - "event_id": { "type": "string" }, - "event_name": { "type": "string" }, - "event_version": { "type": "string" }, - "detected_at": { "type": "string", "format": "date-time" }, - "type": { "type": "string" }, - "blocked": { "type": "boolean" }, - "rule": { "$ref": "/library/appsec/miscs/rule.json" }, - "rule_match": { "$ref": "/library/appsec/miscs/rule_match.json" }, - "context": { "$ref": "/library/appsec/miscs/context.json" } - }, - "additionalProperties": false, - "required": [ - "event_id", - "event_name", - "event_version", - "detected_at", - "type", - "blocked", - "rule", - "rule_match", - "context" - ] -} diff --git a/utils/interfaces/schemas/library/appsec/miscs/context.json b/utils/interfaces/schemas/library/appsec/miscs/context.json deleted file mode 100644 index ae13027d020..00000000000 --- a/utils/interfaces/schemas/library/appsec/miscs/context.json +++ /dev/null @@ -1,169 +0,0 @@ -{ - "$id": "/library/appsec/miscs/context.json", - - "definitions": { - "host": { - "properties": { - "context_version": { "type": "string" }, - "hostname": { "type": "string" }, - "os_type": { "type": "string" } - }, - "additionalProperties": false, - "required": ["context_version", "hostname", "os_type"] - }, - "http": { - "properties": { - "context_version": { "type": "string" }, - "request": { "$ref": "#/definitions/http_request" }, - "response": { "$ref": "#/definitions/http_response" } - }, - "additionalProperties": false, - "required": ["context_version", "request", "response"] - }, - - "http_request": { - "properties": { - "host": { "type": "string" }, - "method": { "type": "string" }, - "path": { "type": "string" }, - "port": { "type": "number" }, - "remote_ip": { "$ref": "/miscs/ip.json" }, - "remote_port": { "type": "number" }, - "resource": { "type": "string" }, - "scheme": { "type": "string" } - }, - "additionalProperties": false, - "required": [ - "host", - "method", - "path", - "port", - "remote_ip", - "remote_port", - "resource", - "scheme" - ] - }, - - "http_response": { - "properties": { - "blocked": { "type": "boolean" }, - "status": { "type": "number" } - }, - "additionalProperties": false, - "required": ["blocked", "status"] - }, - - "service": { - "properties": { - "context_version": { "type": "string" }, - "environment": { "type": "string" }, - "name": { "type": "string" }, - "version": { "type": "string" } - }, - "additionalProperties": false, - "required": ["context_version", "environment", "name", "version"] - }, - - "service_stack": { - "properties": { - "context_version": { "type": "string" }, - "services": { - "type": "array", - "items": { - "properties": { - "environment": { "type": "string" }, - "name": { "type": "string" }, - "version": { "type": "string" }, - "when": { "type": "string", "format": "date-time" } - }, - "additionalProperties": false, - "required": ["environment", "name", "version", "when"] - } - } - }, - "additionalProperties": false, - "required": ["context_version", "services"] - }, - - "span": { - "properties": { - "context_version": { "type": "string" }, - "id": { "type": "string" } - }, - "additionalProperties": false, - "required": ["context_version", "id"] - }, - - "trace": { - "properties": { - "context_version": { "type": "string" }, - "id": { "type": "string" } - }, - "additionalProperties": false, - "required": ["context_version", "id"] - }, - - "tracer": { - "properties": { - "context_version": { "type": "string" }, - "lib_version": { "type": "string" }, - "runtime_type": { "type": "string" }, - "runtime_version": { "type": "string" } - }, - "additionalProperties": false, - "required": [ - "context_version", - "lib_version", - "runtime_type", - "runtime_version" - ] - }, - - "actor": { - "properties": { - "context_version": { "type": "string" }, - "_id": { "type": "string" }, - "identifiers": { - "properties": { - "email": { "type": "string" }, - "sqreen_id": { "type": "string" } - }, - "additionalProperties": false, - "required": ["email", "sqreen_id"] - }, - "ip": { - "properties": { - "address": { "$ref": "/miscs/ip.json" } - }, - "additionalProperties": false, - "required": ["address"] - } - }, - "additionalProperties": false, - "required": ["context_version", "_id", "identifiers", "ip"] - } - }, - - "properties": { - "actor": { "$ref": "#/definitions/actor" }, - "host": { "$ref": "#/definitions/host" }, - "http": { "$ref": "#/definitions/http" }, - "service": { "$ref": "#/definitions/service" }, - "service_stack": { "$ref": "#/definitions/service_stack" }, - "span": { "$ref": "#/definitions/span" }, - "trace": { "$ref": "#/definitions/trace" }, - "tracer": { "$ref": "#/definitions/tracer" } - }, - "additionalProperties": false, - "required": [ - "actor", - "host", - "http", - "service", - "service_stack", - "span", - "trace", - "tracer" - ] -} diff --git a/utils/interfaces/schemas/library/appsec/miscs/rule.json b/utils/interfaces/schemas/library/appsec/miscs/rule.json deleted file mode 100644 index 243b457ec8c..00000000000 --- a/utils/interfaces/schemas/library/appsec/miscs/rule.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "$id": "/library/appsec/miscs/rule.json", - "properties": { - "id": {"type": "string"}, - "name": {"type": "string"}, - "set": {"type": "string"} - }, - "additionalProperties": false, - "required": ["id", "name", "set"] -} diff --git a/utils/interfaces/schemas/library/appsec/miscs/rule_match.json b/utils/interfaces/schemas/library/appsec/miscs/rule_match.json deleted file mode 100644 index 7cfadf5269a..00000000000 --- a/utils/interfaces/schemas/library/appsec/miscs/rule_match.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "$id": "/library/appsec/miscs/rule_match.json", - "properties": { - "operator": { "enum": ["@rx"] }, - "operator_value": { "type": "string" }, - "parameters": { - "type": "array", - "items": { - "properties": { - "name": { "type": "string" }, - "value": { "type": "string" } - }, - "additionalProperties": false, - "required": ["name", "value"] - } - }, - "highlight": { "type": "array", "items": { "type": "string" } }, - "has_server_side_match": { "type": "boolean" } - }, - "additionalProperties": false, - "required": [ - "operator", - "operator_value", - "parameters", - "highlight", - "has_server_side_match" - ] -} diff --git a/utils/interfaces/schemas/library/appsec/proxy/api/v2/appsecevts-request.json b/utils/interfaces/schemas/library/appsec/proxy/api/v2/appsecevts-request.json index 92c0de89689..6f4b64d70bb 100644 --- a/utils/interfaces/schemas/library/appsec/proxy/api/v2/appsecevts-request.json +++ b/utils/interfaces/schemas/library/appsec/proxy/api/v2/appsecevts-request.json @@ -2,21 +2,23 @@ "$id": "/library/appsec/proxy/api/v2/appsecevts-request.json", "properties": { + "protocol_version": { + "type": "integer" + }, + "idempotency_key": { + "type": "string" + }, "events": { "type": "array", + "minItems": 1, "items": { - "properties": { - "context": { - "properties": { - "trace": {}, - "span": {} - }, - "required": ["trace", "span"] - } - }, - "required": ["context"] + "$ref": "/miscs/appsec/events/_definitions/all_events.json" } } }, - "required": ["events"] + "required": [ + "protocol_version", + "idempotency_key", + "events" + ] } diff --git a/utils/interfaces/schemas/library/appsec/proxy/v1/input-request.json b/utils/interfaces/schemas/library/appsec/proxy/v1/input-request.json index cfb61ea7c31..30d70701839 100644 --- a/utils/interfaces/schemas/library/appsec/proxy/v1/input-request.json +++ b/utils/interfaces/schemas/library/appsec/proxy/v1/input-request.json @@ -1,4 +1,25 @@ { - "$id": "/library/appsec/proxy/v1/input-request.json", - "$comments": "obsolete" -} \ No newline at end of file + "$id": "/library/appsec/proxy/v1/input-request.json", + "$comment": "v1 is deprecated", + + "properties": { + "protocol_version": { + "type": "integer" + }, + "idempotency_key": { + "type": "string" + }, + "events": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "/miscs/appsec/events/_definitions/all_events.json" + } + } + }, + "required": [ + "protocol_version", + "idempotency_key", + "events" + ] +} diff --git a/utils/interfaces/schemas/miscs/appsec/contexts/_definitions/all_context.json b/utils/interfaces/schemas/miscs/appsec/contexts/_definitions/all_context.json new file mode 100644 index 00000000000..4edd8e4c34c --- /dev/null +++ b/utils/interfaces/schemas/miscs/appsec/contexts/_definitions/all_context.json @@ -0,0 +1,96 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "/miscs/appsec/contexts/_definitions/all_context.json", + "type": "object", + "properties": { + "actor": { + "oneOf": [ + { + "$ref": "../actor/0.1.0.json" + } + ] + }, + "agent": { + "oneOf": [ + { + "$ref": "../agent/0.1.0.json" + } + ] + }, + "code_location": { + "oneOf": [ + { + "$ref": "../code_location/0.1.0.json" + } + ] + }, + "host": { + "oneOf": [ + { + "$ref": "../host/0.1.0.json" + } + ] + }, + "http": { + "allOf": [ + { + "if": { "properties": { "context_version": { "const": "0.1.0" } } }, + "then": { "$ref": "../http/0.1.0.json" } + }, + { + "if": { "properties": { "context_version": { "const": "1.0.0" } } }, + "then": { "$ref": "../http/1.0.0.json" } + } + ] + }, + "service": { + "oneOf": [ + { + "$ref": "../service/0.1.0.json" + } + ] + }, + "span": { + "oneOf": [ + { + "$ref": "../span/0.1.0.json" + } + ] + }, + "sqreen": { + "oneOf": [ + { + "$ref": "../sqreen/0.1.0.json" + } + ] + }, + "tags": { + "oneOf": [ + { + "$ref": "../tags/0.1.0.json" + } + ] + }, + "service_entry_span": { + "oneOf": [ + { + "$ref": "../service_entry_span/0.1.0.json" + } + ] + }, + "trace": { + "oneOf": [ + { + "$ref": "../trace/0.1.0.json" + } + ] + }, + "library": { + "oneOf": [ + { + "$ref": "../library/0.1.0.json" + } + ] + } + } +} diff --git a/utils/interfaces/schemas/miscs/appsec/contexts/actor/0.1.0.json b/utils/interfaces/schemas/miscs/appsec/contexts/actor/0.1.0.json new file mode 100644 index 00000000000..dd251f5ce03 --- /dev/null +++ b/utils/interfaces/schemas/miscs/appsec/contexts/actor/0.1.0.json @@ -0,0 +1,50 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "/miscs/appsec/contexts/actor/0.1.0.json", + "definitions": { + }, + "type": "object", + "properties": { + "context_version": { + "type": "string", + "const": "0.1.0" + }, + "ip": { + "type": "object", + "properties": { + "address": { + "type": "string", + "anyOf": [ + { + "format": "ipv4" + }, + { + "format": "ipv6" + } + ] + } + }, + "required": [ + "address" + ] + }, + "identifiers": { + "type": ["object", "null"], + "patternProperties": { + "^.+$": { + "type": "string" + } + } + }, + "_id": { + "type": ["string", "null"], + "description": "An internal identifier for the actor.", + "$comment": "Previously `sqreen_identifier`: b64(json(identifiers)) or sha512(b64(json(identifiers))) if too long." + } + }, + "required": [ + "context_version", + "ip" + ], + "additionalProperties": false +} diff --git a/utils/interfaces/schemas/miscs/appsec/contexts/agent/0.1.0.json b/utils/interfaces/schemas/miscs/appsec/contexts/agent/0.1.0.json new file mode 100644 index 00000000000..69863d243ad --- /dev/null +++ b/utils/interfaces/schemas/miscs/appsec/contexts/agent/0.1.0.json @@ -0,0 +1,19 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "/miscs/appsec/contexts/agent/0.1.0.json", + "type": "object", + "properties": { + "context_version": { + "type": "string", + "const": "0.1.0" + }, + "agent_version": { + "type": "string" + } + }, + "required": [ + "context_version", + "agent_version" + ], + "additionalProperties": false +} diff --git a/utils/interfaces/schemas/miscs/appsec/contexts/code_location/0.1.0.json b/utils/interfaces/schemas/miscs/appsec/contexts/code_location/0.1.0.json new file mode 100644 index 00000000000..35fb42179eb --- /dev/null +++ b/utils/interfaces/schemas/miscs/appsec/contexts/code_location/0.1.0.json @@ -0,0 +1,49 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "/miscs/appsec/contexts/code_location/0.1.0.json", + "definitions": { + "StackTraceItem": { + "in_app": { + "type": ["boolean", "null"] + }, + "module": { + "type": ["string", "null"] + }, + "function": { + "type": [ + "string", + "null" + ] + }, + "abs_path": { + "type": [ + "string", + "null" + ] + }, + "line_no": { + "type": [ + "number", + "null" + ] + } + } + }, + "type": "object", + "properties": { + "context_version": { + "type": "string", + "const": "0.1.0" + }, + "stacktrace": { + "type": "array", + "items": { + "$ref": "#/definitions/StackTraceItem" + } + } + }, + "required": [ + "context_version" + ], + "additionalProperties": false +} diff --git a/utils/interfaces/schemas/miscs/appsec/contexts/host/0.1.0.json b/utils/interfaces/schemas/miscs/appsec/contexts/host/0.1.0.json new file mode 100644 index 00000000000..3ae0fcd9ccc --- /dev/null +++ b/utils/interfaces/schemas/miscs/appsec/contexts/host/0.1.0.json @@ -0,0 +1,21 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "/miscs/appsec/contexts/host/0.1.0.json", + "type": "object", + "properties": { + "context_version": { + "type": "string", + "const": "0.1.0" + }, + "os_type": { + "type": "string" + }, + "hostname": { + "type": "string" + } + }, + "required": [ + "context_version" + ], + "additionalProperties": false +} diff --git a/utils/interfaces/schemas/miscs/appsec/contexts/http/0.1.0.json b/utils/interfaces/schemas/miscs/appsec/contexts/http/0.1.0.json new file mode 100644 index 00000000000..30bb7fbb6a6 --- /dev/null +++ b/utils/interfaces/schemas/miscs/appsec/contexts/http/0.1.0.json @@ -0,0 +1,187 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "/miscs/appsec/contexts/http/0.1.0.json", + "definitions": { + "HttpHeaders": { + "type": ["object", "null"], + "patternProperties": { + "^.*$": { + "type": "string" + } + } + }, + "HttpRequest": { + "type": "object", + "properties": { + "scheme": { + "type": "string", + "description": "The scheme part of the URL that generated the security event." + }, + "method": { + "$comment": "Sqreen name: verb", + "type": "string", + "description": "The verb part of the http header that generated the security event." + }, + "url": { + "type": "string", + "description": "The URL that generated the security event. It should not include the query string for PII reasons." + }, + "host": { + "type": "string", + "description": "[INDEXED]The authority part of the URL that generated the security event." + }, + "port": { + "type": "integer", + "description": "The port in the authority part of the URL that generated the security event." + }, + "path": { + "type": "string", + "description": "The path of the URL that generated the security event." + }, + "resource": { + "type": ["string", "null"], + "description": "[INDEXED]", + "$comment": "Sqreen name: endpoint" + }, + "remote_ip": { + "type": "string", + "anyOf": [ + { + "format": "ipv4" + }, + { + "format": "ipv6" + } + ] + }, + "remote_port": { + "type": "integer" + }, + "parameters": { + "description": "TODO Formalize this when more context.", + "type": ["object", "null"], + "properties": { + "form": { + "type": ["object", "null"], + "patternProperties": { + "^.+$": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "other": { + "type": ["object", "null"], + "$comment": "This is a dictionary and seems hard to describe consistently at the moment.", + "patternProperties": { + "^.+$": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "query": { + "type": ["object", "null"], + "$comment": "This is a dictionary and seems hard to describe consistently at the moment.", + "patternProperties": { + "^.+$": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "json": { + "type": ["object", "null"], + "$comment": "This is a dictionary and seems hard to describe consistently at the moment.", + "patternProperties": { + "^.+$": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + }, + "headers": { + "$ref": "#/definitions/HttpHeaders" + }, + "useragent": { + "$comment": "Single word without underscore, to be consistent with how the related datadog facet is spelled.", + "type": ["string", "null"] + }, + "referer": { + "type": ["string", "null"] + }, + "id": { + "description": "Request id", + "type": ["string", "null"] + }, + "start_processing_time": { + "type": ["string", "null"], + "format": "date-time" + }, + "end_processing_time": { + "type": ["string", "null"], + "format": "date-time" + } + }, + "required": [ + "scheme", + "method", + "host", + "port", + "url", + "remote_ip", + "remote_port", + "path" + ] + }, + "HttpResponse": { + "type": "object", + "$comments": "FIXME May not be complete", + "properties": { + "status": { + "type": ["integer", "null"] + }, + "content_length": { + "type": ["number", "null"] + }, + "content_type": { + "type": ["string", "null"] + }, + "blocked": { + "type": "boolean" + }, + "headers": { + "$ref": "#/definitions/HttpHeaders" + } + } + } + }, + "type": "object", + "properties": { + "context_version": { + "type": "string", + "const": "0.1.0" + }, + "request": { + "$ref": "#/definitions/HttpRequest" + }, + "response": { + "$ref": "#/definitions/HttpResponse" + } + }, + "required": [ + "context_version", + "request" + ], + "additionalProperties": false +} diff --git a/utils/interfaces/schemas/miscs/appsec/contexts/http/1.0.0.json b/utils/interfaces/schemas/miscs/appsec/contexts/http/1.0.0.json new file mode 100644 index 00000000000..4346b663179 --- /dev/null +++ b/utils/interfaces/schemas/miscs/appsec/contexts/http/1.0.0.json @@ -0,0 +1,153 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "/miscs/appsec/contexts/http/1.0.0.json", + "definitions": { + "HttpHeaders": { + "type": ["object", "null"], + "patternProperties": { + "^.*$": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "HttpRequest": { + "type": "object", + "properties": { + "method": { + "$comment": "Sqreen name: verb", + "type": "string", + "description": "The verb part of the http header that generated the security event." + }, + "url": { + "type": "string", + "description": "The URL that generated the security event. It should not include the query string for PII reasons." + }, + "resource": { + "type": ["string", "null"], + "description": "[INDEXED]", + "$comment": "Sqreen name: endpoint" + }, + "remote_ip": { + "type": "string", + "anyOf": [ + { + "format": "ipv4" + }, + { + "format": "ipv6" + } + ] + }, + "remote_port": { + "type": "integer" + }, + "headers": { + "$ref": "#/definitions/HttpHeaders" + }, + "id": { + "description": "Request id", + "type": ["string", "null"] + }, + "parameters": { + "description": "TODO Formalize this when more context.", + "type": ["object", "null"], + "properties": { + "form": { + "type": ["object", "null"], + "patternProperties": { + "^.+$": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "other": { + "type": ["object", "null"], + "$comment": "This is a dictionary and seems hard to describe consistently at the moment.", + "patternProperties": { + "^.+$": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "query": { + "type": ["object", "null"], + "$comment": "This is a dictionary and seems hard to describe consistently at the moment.", + "patternProperties": { + "^.+$": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "json": { + "type": ["object", "null"], + "$comment": "This is a dictionary and seems hard to describe consistently at the moment.", + "patternProperties": { + "^.+$": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + } + }, + "required": [ + "method", + "url", + "headers", + "remote_ip", + "remote_port" + ] + }, + "HttpResponse": { + "type": "object", + "properties": { + "status": { + "type": ["integer", "null"] + }, + "headers": { + "$ref": "#/definitions/HttpHeaders" + }, + "blocked": { + "type": "boolean" + } + }, + "required": [ + "status", + "headers" + ] + } + }, + "type": "object", + "properties": { + "context_version": { + "type": "string", + "const": "1.0.0" + }, + "request": { + "$ref": "#/definitions/HttpRequest" + }, + "response": { + "$ref": "#/definitions/HttpResponse" + } + }, + "required": [ + "context_version", + "request" + ], + "additionalProperties": false +} diff --git a/utils/interfaces/schemas/miscs/appsec/contexts/library/0.1.0.json b/utils/interfaces/schemas/miscs/appsec/contexts/library/0.1.0.json new file mode 100644 index 00000000000..7053783d941 --- /dev/null +++ b/utils/interfaces/schemas/miscs/appsec/contexts/library/0.1.0.json @@ -0,0 +1,27 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "/miscs/appsec/contexts/library/0.1.0.json", + "type": "object", + "properties": { + "context_version": { + "type": "string", + "const": "0.1.0" + }, + "runtime_type": { + "type": "string" + }, + "runtime_version": { + "type": "string" + }, + "lib_version": { + "type": "string" + } + }, + "required": [ + "context_version", + "runtime_type", + "runtime_version", + "lib_version" + ], + "additionalProperties": false +} diff --git a/utils/interfaces/schemas/miscs/appsec/contexts/service/0.1.0.json b/utils/interfaces/schemas/miscs/appsec/contexts/service/0.1.0.json new file mode 100644 index 00000000000..3e0790c6d67 --- /dev/null +++ b/utils/interfaces/schemas/miscs/appsec/contexts/service/0.1.0.json @@ -0,0 +1,24 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "/miscs/appsec/contexts/service/0.1.0.json", + "type": "object", + "properties": { + "context_version": { + "type": "string", + "const": "0.1.0" + }, + "name": { + "type": "string" + }, + "environment": { + "type": ["string", "null"] + }, + "version": { + "type": ["string", "null"] + } + }, + "required": [ + "context_version", + "name" + ] +} diff --git a/utils/interfaces/schemas/miscs/appsec/contexts/service_entry_span/0.1.0.json b/utils/interfaces/schemas/miscs/appsec/contexts/service_entry_span/0.1.0.json new file mode 100644 index 00000000000..a31ef5bda38 --- /dev/null +++ b/utils/interfaces/schemas/miscs/appsec/contexts/service_entry_span/0.1.0.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "/miscs/appsec/contexts/service_entry_span/0.1.0.json", + "type": "object", + "properties": { + "context_version": { + "type": "string", + "const": "0.1.0" + }, + "id": { + "description": "[INDEXED]", + "type": [ + "string", + "number" + ] + } + }, + "required": [ + "context_version", + "id" + ], + "additionalProperties": false +} diff --git a/utils/interfaces/schemas/miscs/appsec/contexts/span/0.1.0.json b/utils/interfaces/schemas/miscs/appsec/contexts/span/0.1.0.json new file mode 100644 index 00000000000..5cd3fe47569 --- /dev/null +++ b/utils/interfaces/schemas/miscs/appsec/contexts/span/0.1.0.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "/miscs/appsec/contexts/span/0.1.0.json", + "type": "object", + "properties": { + "context_version": { + "type": "string", + "const": "0.1.0" + }, + "id": { + "description": "[INDEXED]", + "type": [ + "string", + "number" + ] + } + }, + "required": [ + "context_version", + "id" + ], + "additionalProperties": false +} diff --git a/utils/interfaces/schemas/miscs/appsec/contexts/sqreen/0.1.0.json b/utils/interfaces/schemas/miscs/appsec/contexts/sqreen/0.1.0.json new file mode 100644 index 00000000000..fc5c93b4251 --- /dev/null +++ b/utils/interfaces/schemas/miscs/appsec/contexts/sqreen/0.1.0.json @@ -0,0 +1,38 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "/miscs/appsec/contexts/sqreen/0.1.0.json", + "type": "object", + "properties": { + "context_version": { + "type": "string", + "const": "0.1.0" + }, + "application_id": { + "type": "string" + }, + "organization_id": { + "type": "string" + }, + "instance_id": { + "type": "string" + }, + "trace_id": { + "type": "string" + }, + "signal_id": { + "type": "string" + }, + "is_from_backend": { + "type": "boolean" + } + }, + "required": [ + "context_version", + "application_id", + "organization_id", + "instance_id", + "signal_id", + "is_from_backend" + ], + "additionalProperties": false +} diff --git a/utils/interfaces/schemas/miscs/appsec/contexts/tags/0.1.0.json b/utils/interfaces/schemas/miscs/appsec/contexts/tags/0.1.0.json new file mode 100644 index 00000000000..386b2dcf0e6 --- /dev/null +++ b/utils/interfaces/schemas/miscs/appsec/contexts/tags/0.1.0.json @@ -0,0 +1,26 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "/miscs/appsec/contexts/tags/0.1.0.json", + "type": "object", + "properties": { + "context_version": { + "type": "string", + "const": "0.1.0" + }, + "values": { + "type": "array", + "description": "A list of tags.", + "uniqueItems": true, + "items": { + "type": "string", + "description": "A tag as described in https://docs.datadoghq.com/getting_started/tagging/#defining-tags.", + "$comment": "See https://docs.datadoghq.com/getting_started/tagging/#defining-tags" + } + } + }, + "required": [ + "context_version", + "values" + ], + "additionalProperties": false +} diff --git a/utils/interfaces/schemas/miscs/appsec/contexts/trace/0.1.0.json b/utils/interfaces/schemas/miscs/appsec/contexts/trace/0.1.0.json new file mode 100644 index 00000000000..c391442f7ac --- /dev/null +++ b/utils/interfaces/schemas/miscs/appsec/contexts/trace/0.1.0.json @@ -0,0 +1,22 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "/miscs/appsec/contexts/trace/0.1.0.json", + "type": "object", + "properties": { + "context_version": { + "type": "string", + "const": "0.1.0" + }, + "id": { + "type": [ + "string", + "number" + ] + } + }, + "required": [ + "context_version", + "id" + ], + "additionalProperties": false +} diff --git a/utils/interfaces/schemas/miscs/appsec/events/0.1.0.json b/utils/interfaces/schemas/miscs/appsec/events/0.1.0.json new file mode 100644 index 00000000000..632b0e7237d --- /dev/null +++ b/utils/interfaces/schemas/miscs/appsec/events/0.1.0.json @@ -0,0 +1,68 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "/miscs/appsec/events/0.1.0.json", + "title": "AppSec Event 0.1.0", + "deprecated": true, + "type": "object", + "properties": { + "event_id": { + "type": "string" + }, + "event_type": { + "type": "string", + "const": "appsec.threat.attack", + "description": "[INDEXED]" + }, + "event_version": { + "type": "string", + "const": "0.1.0" + }, + "detected_at": { + "type": "string", + "format": "date-time", + "description": "[INDEXED]" + }, + "type": { + "type": "string", + "description": "[INDEXED] The type of the triggered rule." + }, + "blocked": { + "type": "boolean", + "description": "[INDEXED]" + }, + "rule": { + "$ref": "_definitions/rule/0.1.0.json" + }, + "rule_match": { + "$ref": "_definitions/rule_match/0.1.0.json" + }, + "context": { + "allOf": [ + { + "$ref": "../contexts/_definitions/all_context.json" + }, + { + "title": "Required context", + "required": [ + "actor", + "service", + "span", + "trace", + "tracer" + ] + } + ] + } + }, + "required": [ + "event_id", + "event_type", + "event_version", + "detected_at", + "type", + "blocked", + "rule", + "rule_match", + "context" + ] +} diff --git a/utils/interfaces/schemas/miscs/appsec/events/1.0.0.json b/utils/interfaces/schemas/miscs/appsec/events/1.0.0.json new file mode 100644 index 00000000000..ec6201ff7e3 --- /dev/null +++ b/utils/interfaces/schemas/miscs/appsec/events/1.0.0.json @@ -0,0 +1,56 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "/miscs/appsec/events/1.0.0.json", + "title": "AppSec Event 1.0.0", + "type": "object", + "properties": { + "event_id": { + "type": "string", + "description": "Random UUID uniquely identifying this event." + }, + "event_type": { + "type": "string", + "const": "appsec" + }, + "event_version": { + "type": "string", + "const": "1.0.0" + }, + "detected_at": { + "type": "string", + "format": "date-time", + "description": "Time of the event detection using the format defined in RFC 3339, section 5.6. For example, ``2018-11-13T20:20:39+00:00``." + }, + "rule": { + "$ref": "_definitions/rule/1.0.0.json" + }, + "rule_match": { + "$ref": "_definitions/rule_match/1.0.0.json" + }, + "context": { + "allOf": [ + { + "$ref": "../contexts/_definitions/all_context.json" + }, + { + "title": "Required context", + "required": [ + "http", + "service", + "span", + "trace" + ] + } + ] + } + }, + "required": [ + "event_id", + "event_type", + "event_version", + "detected_at", + "rule", + "rule_match", + "context" + ] +} diff --git a/utils/interfaces/schemas/miscs/appsec/events/_definitions/all_events.json b/utils/interfaces/schemas/miscs/appsec/events/_definitions/all_events.json new file mode 100644 index 00000000000..f692cd97d86 --- /dev/null +++ b/utils/interfaces/schemas/miscs/appsec/events/_definitions/all_events.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "/miscs/appsec/events/_definitions/all_events.json", + "allOf": [ + { + "if": { "properties": { "event_version": { "const": "1.0.0" } } }, + "then": { "$ref": "../1.0.0.json" } + }, + { + "if": { "properties": { "event_version": { "const": "0.1.0" } } }, + "then": { "$ref": "../0.1.0.json" } + } + ] +} diff --git a/utils/interfaces/schemas/miscs/appsec/events/_definitions/rule/0.1.0.json b/utils/interfaces/schemas/miscs/appsec/events/_definitions/rule/0.1.0.json new file mode 100644 index 00000000000..8fa743c4600 --- /dev/null +++ b/utils/interfaces/schemas/miscs/appsec/events/_definitions/rule/0.1.0.json @@ -0,0 +1,24 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "/miscs/appsec/events/_definitions/rule/0.1.0.json", + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "[INDEXED] The unique identifier of the rule that triggered the event" + }, + "name": { + "type": "string", + "description": "The friendly name of the rule that triggered the event" + }, + "set": { + "type": ["string", "null"], + "description": "This field should hold the name of the rule set. This concept is not currently in the new rule format" + } + }, + "required": [ + "id", + "name" + ], + "additionalProperties": false +} diff --git a/utils/interfaces/schemas/miscs/appsec/events/_definitions/rule/1.0.0.json b/utils/interfaces/schemas/miscs/appsec/events/_definitions/rule/1.0.0.json new file mode 100644 index 00000000000..7838a528dd4 --- /dev/null +++ b/utils/interfaces/schemas/miscs/appsec/events/_definitions/rule/1.0.0.json @@ -0,0 +1,28 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "/miscs/appsec/events/_definitions/rule/1.0.0.json", + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "The unique identifier of the rule that triggered the event. For example, ``ua-910-xax``." + }, + "name": { + "type": "string", + "description": "The friendly name of the rule that triggered the event." + }, + "tags": { + "type": "object", + "description": "The tags associated to the rule in the event rules file.", + "patternProperties": { + "^.+$": { "type": "string" } + } + } + }, + "required": [ + "id", + "name", + "tags" + ], + "additionalProperties": false +} diff --git a/utils/interfaces/schemas/miscs/appsec/events/_definitions/rule_match/0.1.0.json b/utils/interfaces/schemas/miscs/appsec/events/_definitions/rule_match/0.1.0.json new file mode 100644 index 00000000000..41de4bdaf39 --- /dev/null +++ b/utils/interfaces/schemas/miscs/appsec/events/_definitions/rule_match/0.1.0.json @@ -0,0 +1,38 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "/miscs/appsec/events/_definitions/rule_match/0.1.0.json", + "type": "object", + "properties": { + "operator": { + "type": "string" + }, + "operator_value": { + "type": "string", + "description": "This field should hold the operator of the rule match. It should use the new rule format" + }, + "parameters": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "value": { + "type": "string" + } + } + } + }, + "highlight": { + "type": "array", + "items": { + "type": "string" + } + }, + "has_server_side_match": { + "type": "boolean", + "description": "This field is not currently used, it was added for backward compatibility" + } + } +} diff --git a/utils/interfaces/schemas/miscs/appsec/events/_definitions/rule_match/1.0.0.json b/utils/interfaces/schemas/miscs/appsec/events/_definitions/rule_match/1.0.0.json new file mode 100644 index 00000000000..eff5e5bd57e --- /dev/null +++ b/utils/interfaces/schemas/miscs/appsec/events/_definitions/rule_match/1.0.0.json @@ -0,0 +1,56 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "/miscs/appsec/events/_definitions/rule_match/1.0.0.json", + "type": "object", + "properties": { + "operator": { + "type": "string", + "description": "The rule operator that triggered this event. For example, ``match_regex`` or ``phrase_match``." + }, + "operator_value": { + "type": "string", + "description": "The rule operator operand that triggered this event. For example, the word that triggered using the ``phrase_match`` operator." + }, + "parameters": { + "type": "array", + "items": { + "type": "object", + "properties": { + "address": { + "type": "string", + "description": "The address containing the value that triggered the rule. For example ``http.server.query``." + }, + "key_path": { + "type": "array", + "description": "The path of the value that triggered the rule. For example ``[\"query\", 0]`` to refer to the value in ``{\"query\": [\"triggering value\"]}``.", + "items": { + "anyOf": [ + { "type": "string" }, + { "type": "number" } + ] + } + }, + "value": { + "type": "string", + "description": "The value that triggered the rule. It is currently optional to provide this value until we have a PII strategy." + } + }, + "required": [ + "address" + ] + } + }, + "highlight": { + "type": "array", + "description": "The part of the value that triggered the rule. It is currently optional to provide this value until we have a PII strategy.", + "items": { + "type": "string" + } + } + }, + "required": [ + "operator", + "operator_value", + "parameters" + ] +} diff --git a/utils/interfaces/schemas/serve_doc.py b/utils/interfaces/schemas/serve_doc.py index f8cdc48172b..197c3761952 100644 --- a/utils/interfaces/schemas/serve_doc.py +++ b/utils/interfaces/schemas/serve_doc.py @@ -4,6 +4,7 @@ from flask import Flask, send_from_directory, request, render_template from utils.interfaces._schemas_validators import _get_schemas_store, _get_schemas_filenames from json_schema_for_humans.generate import generate_from_schema, generate_from_filename +from json_schema_for_humans.generation_configuration import GenerationConfiguration import json_schema_for_humans @@ -13,15 +14,7 @@ app = Flask(__name__, static_url_path="/static", static_folder=static_folder, template_folder=template_folder) store = _get_schemas_store() - -documentations = {} - -for filename in _get_schemas_filenames(): - if filename.endswith("request.json"): - doc = generate_from_schema(filename, store) - doc = doc.replace("schema_doc.css", "/static/schema_doc.css") - doc = doc.replace("schema_doc.min.js", "/static/schema_doc.min.js") - documentations[filename[len("utils/interfaces/schemas") :]] = doc +store_config = GenerationConfiguration() @app.route("/", methods=["GET"]) @@ -29,12 +22,16 @@ def default(): data = {"schemas": []} - for filename in documentations: - doc_path = filename.replace(".json", ".html") + for id, schema in store.items(): + # skip some schemas + if not id.endswith("request.json") and not "title" in schema: + continue + + doc_path = id.replace(".json", ".html") # doc_path = doc_path[len("utils/interfaces/schemas"):] # filename = filename[len("utils/interfaces/schemas"):] - data["schemas"].append({"href": f"{doc_path}", "caption": filename}) + data["schemas"].append({"href": f"{doc_path}", "caption": id}) return render_template("index.html", data=data) @@ -43,13 +40,11 @@ def default(): def documentation(path): path = f"/{path}.json" - print(path) - print(documentations.keys()) - - if path not in documentations: - return "File not found", 404 + doc = generate_from_schema(path, store, config=store_config) + doc = doc.replace("schema_doc.css", "/static/schema_doc.css") + doc = doc.replace("schema_doc.min.js", "/static/schema_doc.min.js") - return documentations[path] + return doc if __name__ == "__main__":