Release v0.3.0

.github/workflows/changelog-ci.yml

@@ -20,7 +20,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Run Changelog CI
-        uses: saadmk11/changelog-ci@v1.1.2
+        uses: saadmk11/changelog-ci@v1.2.0
         with:
           # Optional, you can provide any name for your changelog file,
           # changelog_filename: CHANGELOG.md

.github/workflows/installation-test.yml

@@ -31,12 +31,19 @@ jobs:
                 cache-dependency-path: backend-agent/requirements.txt
             - run: pip install -r backend-agent/requirements.txt
 
-            - name: Start server
+            - name: Start server and check health
               run: |
                 cd backend-agent
-                DISABLE_AGENT=1 python main.py &
-                sleep 10
-
-            - name: Check server health
-              run: |
-                curl -s -o /dev/null -w "%{http_code}" http://localhost:8080/health
+                DISABLE_AGENT=1 DB_PATH=${RUNNER_TEMP}/data.db python main.py > server.log 2>&1 &
+                for i in {1..20}; do
+                  sleep 1
+                  status=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8080/health || true)
+                  if [ "$status" -eq 200 ]; then
+                    echo "Health check succeeded"
+                    cat server.log
+                    exit 0
+                  fi
+                done
+                echo "Health check failed after waiting"
+                cat server.log
+                exit 1

.gitignore

@@ -89,6 +89,11 @@ venv/
 ENV/
 env.bak/
 venv.bak/
+venv310
+cache
+
+# Frontend Environments
+frontend/src/environments/environment.ts
 
 # Spyder project settings
 .spyderproject
@@ -138,6 +143,3 @@ prompt_success.txt
 result_gptfuzz.txt
 codeattack_success.txt
 artprompt_success.json
-
-# Frontend Environments
-frontend/src/environments

CHANGELOG.md

@@ -1,3 +1,17 @@
+# Version: v0.3.0
+
+* [#46](https://github.com/SAP/STARS/pull/46): Risk dashboard UI
+* [#51](https://github.com/SAP/STARS/pull/51): Bump requests from 2.32.3 to 2.32.4 in /backend-agent
+* [#52](https://github.com/SAP/STARS/pull/52): Update pyrit.py implementation to ensure comatibility with pyrit 0.9.0
+* [#54](https://github.com/SAP/STARS/pull/54): Align langchain and pyrit dependencies
+* [#55](https://github.com/SAP/STARS/pull/55): Fix garak, langchain, and pyrit dependency conflicts
+* [#56](https://github.com/SAP/STARS/pull/56): Update models with June 2025 availabilities
+* [#59](https://github.com/SAP/STARS/pull/59): Fix db usage with attacks
+* [#60](https://github.com/SAP/STARS/pull/60): Merge develop into docker
+* [#61](https://github.com/SAP/STARS/pull/61): Dockerize services
+* [#63](https://github.com/SAP/STARS/pull/63): aligned frontend with db
+
+
 # Version: v0.2.1
 
 * [#34](https://github.com/SAP/STARS/pull/34): Support aicore-mistralai models

backend-agent/.dockerignore

@@ -3,7 +3,8 @@
 cache
 
 # Libraries
-venv
+venv*
+.venv*
 
 # Logs
 traces

backend-agent/.env.example

@@ -12,3 +12,18 @@ API_KEY=super-secret-change-me
 DEBUG=True
 
 RESULT_SUMMARIZE_MODEL=gpt-4
+
+# Models for agent.py
+AGENT_MODEL=gpt-4
+EMBEDDING_MODEL=text-embedding-ada-002
+
+# Database path
+DB_PATH=/path_to/database.db
+
+# AICORE configuration for backend (in case there is no configuration in
+# ~/.aicore/config.json). When using docker, these variables need to be set
+# AICORE_AUTH_URL=
+# AICORE_CLIENT_ID=
+# AICORE_CLIENT_SECRET=
+# AICORE_BASE_URL=
+# AICORE_RESOURCE_GROUP=

backend-agent/Dockerfile

@@ -3,10 +3,9 @@ FROM python:3.11
 WORKDIR /app
 
 COPY requirements.txt .
-RUN --mount=type=ssh pip install -r requirements.txt --no-cache-dir
+RUN pip install -r requirements.txt --no-cache-dir
 
 COPY . .
 
 EXPOSE 8080
 CMD [ "python", "main.py" ]
-

backend-agent/README.md

@@ -17,14 +17,13 @@ Before running the tool, make sure to have an account configured and fully
 working on SAP AI Core (requires a SAP BTP subaccount with a running AI Core service instance).
 
 Please note that the agent requires `gpt-4` LLM and `text-embedding-ada-002`
-embedding function. For the default attack suite, additional the model
-`mistralai--mixtral-8x7b-instruct-v01` is used.
+embedding function.
 They must be already **deployed and running in SAP AI Core** before running this
 tool.
-Refer [to the official documentation](https://help.sap.com/docs/sap-ai-core/sap-ai-core-service-guide/models-and-scenarios-in-generative-ai-hub) for what other models it is possible to deploy.
+Refer [to the official documentation](https://help.sap.com/docs/sap-ai-core/sap-ai-core-service-guide/models-and-scenarios-in-generative-ai-hub) for what other models it is possible to deploy and to the [official SAP note](https://me.sap.com/notes/3437766) for models and regions availability.
 
 ### Support for non-SAP AI Core models
-In general, the pentest tools integrated in the agent can be run on LLMs deployed in SAP AI Core, but also custom inference servers (e.g., vllm or a local ollama) are supported.
+In general, the pentest tools integrated in the agent can be run on LLMs deployed in SAP AI Core, but also custom inference servers (e.g., vllm and ollama) are supported.
 
 
 ## Installation

backend-agent/agent.py

@@ -1,3 +1,6 @@
+import os
+
+from dotenv import load_dotenv
 from gen_ai_hub.proxy.core.proxy_clients import set_proxy_version
 from gen_ai_hub.proxy.langchain.init_models import (
     init_llm, init_embedding_model)
@@ -10,6 +13,11 @@
 from langchain_community.document_loaders import DirectoryLoader
 from langchain_community.vectorstores import FAISS
 
+
+# load env variables
+load_dotenv()
+AGENT_MODEL = os.environ.get('AGENT_MODEL', 'gpt-4')
+EMBEDDING_MODEL = os.environ.get('EMBEDDING_MODEL', 'text-embedding-ada-002')
 # Use models deployed in SAP AI Core
 set_proxy_version('gen-ai-hub')
 
@@ -29,7 +37,7 @@
 ###############################################################################
 # SAP-compliant embedding models
 # https://github.tools.sap/AI-Playground-Projects/llm-commons#embedding-models
-underlying_embeddings = init_embedding_model('text-embedding-ada-002')
+underlying_embeddings = init_embedding_model(EMBEDDING_MODEL)
 # Initialize local cache for faster loading of subsequent executions
 fs = LocalFileStore('./cache')
 # Link the embedding and the local cache system, and define a namespace
@@ -131,7 +139,7 @@ def get_retriever(document_path: str,
 
 # Initialize the LLM model to use, among the ones provided by SAP
 # The max token count needs to be increased so that responses are not cut off.
-llm = init_llm(model_name='gpt-4', max_tokens=1024)
+llm = init_llm(model_name=AGENT_MODEL, max_tokens=4096)
 
 # Chain
 # https://python.langchain.com/docs/modules/chains

backend-agent/app/__init__.py

@@ -0,0 +1,31 @@
+import os
+
+from dotenv import load_dotenv
+from flask import Flask
+
+from .db.models import db
+
+
+load_dotenv()
+
+db_path = os.getenv('DB_PATH')
+
+if not db_path:
+    raise EnvironmentError(
+        'Missing DB_PATH environment variable. Please set DB_PATH in your '
+        '.env file to a valid SQLite file path.'
+    )
+
+
+def create_app():
+    app = Flask(__name__)
+    # Database URI configuration
+    app.config['SQLALCHEMY_DATABASE_URI'] = f'sqlite:///{db_path}'
+    app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
+
+    # Create every SQLAlchemy tables defined in models.py
+    with app.app_context():
+        db.init_app(app)
+        db.create_all()
+
+    return app

backend-agent/app/db/models.py

@@ -0,0 +1,57 @@
+from flask_sqlalchemy import SQLAlchemy
+
+db = SQLAlchemy()
+
+
+# Represents a target model that can be attacked by various attacks.
+class TargetModel(db.Model):
+    __tablename__ = 'target_models'
+    id = db.Column(db.Integer, primary_key=True)
+    name = db.Column(db.String, unique=True, nullable=False)
+    description = db.Column(db.String)
+
+
+# Represents an attack that can be performed on a target model.
+class Attack(db.Model):
+    __tablename__ = 'attacks'
+    id = db.Column(db.Integer, primary_key=True)
+    name = db.Column(db.String, nullable=False, unique=True)
+    weight = db.Column(db.Integer, nullable=False, default=1, server_default="1")  # noqa: E501
+
+
+# Represents a sub-attack that is part of a larger attack.
+class SubAttack(db.Model):
+    __tablename__ = 'sub_attacks'
+    id = db.Column(db.Integer, primary_key=True)
+    name = db.Column(db.String, nullable=False)
+    description = db.Column(db.String)
+    attack_id = db.Column(db.Integer, db.ForeignKey('attacks.id'), nullable=False)  # noqa: E501
+
+
+# Represents the results of each sigle attack on a target model.
+class AttackResult(db.Model):
+    __tablename__ = 'attack_results'
+    id = db.Column(db.Integer, primary_key=True)
+    target_model_id = db.Column(db.Integer, db.ForeignKey('target_models.id'), nullable=False)  # noqa: E501
+    attack_id = db.Column(db.Integer, db.ForeignKey('attacks.id'), nullable=False)  # noqa: E501
+    success = db.Column(db.Boolean, nullable=False)
+    vulnerability_type = db.Column(db.String, nullable=True)
+    details = db.Column(db.JSON, nullable=True)  # JSON field
+
+
+# Represents the global attack success rate of an attack on a target model,
+# including the total number of attacks and successful attacks.
+class ModelAttackScore(db.Model):
+    __tablename__ = 'model_attack_scores'
+    id = db.Column(db.Integer, primary_key=True)
+    target_model_id = db.Column(db.Integer, db.ForeignKey('target_models.id'), nullable=False)  # noqa: E501
+    attack_id = db.Column(db.Integer, db.ForeignKey('attacks.id'), nullable=False)  # noqa: E501
+    total_number_of_attack = db.Column(db.Integer, nullable=False)
+    total_success = db.Column(db.Integer, nullable=False)
+
+    __table_args__ = (
+        db.UniqueConstraint('target_model_id', 'attack_id', name='uix_model_attack'),  # noqa: E501
+    )
+
+
+db.configure_mappers()

backend-agent/app/db/utils.py

@@ -0,0 +1,91 @@
+import logging
+
+from .models import (
+    Attack as AttackDB,
+    db,
+    TargetModel as TargetModelDB,
+    AttackResult as AttackResultDB,
+    ModelAttackScore as ModelAttackScoreDB,
+)
+
+from status import status
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+logger.addHandler(status.trace_logging)
+
+
+# Persist the attack result into the database for each attack.
+def save_to_db(attack_results: AttackResultDB) -> list[AttackResultDB]:
+    """
+    Persist the attack result into the database.
+    Returns a list of AttackResults that were added.
+    """
+    inserted_records = []
+
+    # Retrieve what to save to db
+    attack_name = attack_results.attack.lower()
+    success = attack_results.success
+    vulnerability_type = attack_results.vulnerability_type.lower()
+    details = attack_results.details  # JSON column
+    target_name = details.get('target_model', '').lower()
+
+    # If target model name is not provided, skip saving
+    if not target_name:
+        logger.info("Skipping result: missing target model name.")
+        return
+
+    # If target model does not exist, create it
+    target_model = TargetModelDB.query.filter_by(name=target_name).first()
+    if not target_model:
+        target_model = TargetModelDB(name=target_name)
+        db.session.add(target_model)
+        db.session.flush()
+
+    # If attack does not exist, create it with default weight to 1
+    attack = AttackDB.query.filter_by(name=attack_name).first()
+    if not attack:
+        attack = AttackDB(name=attack_name, weight=1)
+        db.session.add(attack)
+        db.session.flush()
+
+    # Add the attack result to inserted_records
+    db_record = AttackResultDB(
+        target_model_id=target_model.id,
+        attack_id=attack.id,
+        success=success,
+        vulnerability_type=vulnerability_type,
+        details=details,
+    )
+    db.session.add(db_record)
+    inserted_records.append(db_record)
+
+    # If model_attack_score does not exist, create it
+    # otherwise, update the existing record
+    model_attack_score = ModelAttackScoreDB.query.filter_by(
+        target_model_id=target_model.id,
+        attack_id=attack.id
+    ).first()
+    if not model_attack_score:
+        model_attack_score = ModelAttackScoreDB(
+            target_model_id=target_model.id,
+            attack_id=attack.id,
+            total_number_of_attack=details.get('total_attacks', 0),
+            total_success=details.get('number_successful_attacks', 0)
+        )
+    else:
+        model_attack_score.total_number_of_attack += details.get('total_attacks', 0)  # noqa: E501
+        model_attack_score.total_success += details.get('number_successful_attacks', 0)  # noqa: E501
+    db.session.add(model_attack_score)
+    inserted_records.append(model_attack_score)
+
+    # Commit the session to save all changes to the database
+    # or rollback if an error occurs
+    try:
+        db.session.commit()
+        logger.info("Results successfully saved to the database.")
+        return inserted_records
+    except Exception as e:
+        db.session.rollback()
+        logger.error("Error while saving to the database: %s", e)
+        return []

backend-agent/attack.py

@@ -1,18 +1,27 @@
-from argparse import Namespace
-from dataclasses import asdict
 import json
-import os
 import logging
+import os
+from argparse import Namespace
+from dataclasses import asdict
 
+from app.db.utils import save_to_db
 from attack_result import AttackResult, SuiteResult
-from libs.artprompt import start_artprompt, \
-    OUTPUT_FILE as artprompt_out_file
-from libs.codeattack import start_codeattack, \
-    OUTPUT_FILE as codeattack_out_file
-from libs.gptfuzz import perform_gptfuzz_attack, \
-    OUTPUT_FILE as gptfuzz_out_file
-from libs.promptmap import start_prompt_map, \
-    OUTPUT_FILE as prompt_map_out_file
+from libs.artprompt import (
+    OUTPUT_FILE as artprompt_out_file,
+    start_artprompt,
+)
+from libs.codeattack import (
+    OUTPUT_FILE as codeattack_out_file,
+    start_codeattack,
+)
+from libs.gptfuzz import (
+    OUTPUT_FILE as gptfuzz_out_file,
+    perform_gptfuzz_attack,
+)
+from libs.promptmap import (
+    OUTPUT_FILE as prompt_map_out_file,
+    start_prompt_map,
+)
 from libs.pyrit import start_pyrit_attack
 from llm import LLM
 from status import Trace
@@ -247,6 +256,7 @@ def run(self, summarize_by_llm: bool = False) -> SuiteResult:
                 summary = self.summarize_attack_result(result)
                 result.details['summary'] = summary
             full_result.append(result)
+            save_to_db(result)
         return SuiteResult(full_result)
 
     def summarize_attack_result(self, attack_result: AttackResult) -> str: