Playground

server/app/__init__.py

@@ -1,12 +1,11 @@
 from flask import Flask
 from flask_cors import CORS
 from .config import Config
-from .api.auth_routes import auth_bp
 
 def create_app(config_class=Config):
     app = Flask(__name__, 
-                static_folder='../../src/static',  # Points to frontend files
-                template_folder='../../src/templates')  # Points to frontend files
+                static_folder='../../src/static',
+                template_folder='../../src/templates')
 
     # Load config
     app.config.from_object(config_class)
@@ -15,15 +14,18 @@ def create_app(config_class=Config):
     CORS(app)
 
     # Register blueprints
-    from .api.character_routes import character_bp
+    from .api.aiagent_routes import ai_agent_bp
     from .api.dataset_routes import dataset_bp
     from .api.evaluation_routes import evaluation_bp
+    from .api.config_routes import config_bp
     from .api.main import main_bp
+    from .api.auth_routes import auth_bp
 
     app.register_blueprint(main_bp)
-    app.register_blueprint(character_bp, url_prefix='/api')
+    app.register_blueprint(ai_agent_bp, url_prefix='/api')
     app.register_blueprint(dataset_bp, url_prefix='/api')
     app.register_blueprint(evaluation_bp, url_prefix='/api')
+    app.register_blueprint(config_bp, url_prefix='/api/config')
     app.register_blueprint(auth_bp, url_prefix='/api')
 
     return app

server/app/api/aiagent_routes.py

@@ -0,0 +1,66 @@
+from flask import Blueprint, request, jsonify, session
+from ..core.aiagents import (
+    get_all_agents, save_agent, delete_agent, get_agent,
+    add_variant, delete_variant
+)
+from ..core.models import AiAgent
+from ..auth.decorators import login_required
+
+ai_agent_bp = Blueprint('ai_agents', __name__)
+
+@ai_agent_bp.route('/ai-agents', methods=['GET'])
+@login_required
+def get_agents():
+    user_id = session['user']['uid']
+    return jsonify(get_all_agents(user_id))
+
+@ai_agent_bp.route('/ai-agents', methods=['POST'])
+@login_required
+def create_agent():
+    data = request.json
+    user_id = session['user']['uid']
+
+    agent = AiAgent(
+        name=data['name'],
+        prompt=data['prompt'],
+        config=data['config'],
+        included_variables=data.get('included_variables', [])
+    )
+
+    agent_id = save_agent(agent, user_id)
+    return jsonify({
+        "message": f"AI Agent '{data['name']}' created successfully",
+        "id": agent_id
+    }), 201
+
+@ai_agent_bp.route('/ai-agents/<agent_id>', methods=['DELETE'])
+@login_required
+def remove_agent(agent_id):
+    user_id = session['user']['uid']
+    if delete_agent(agent_id, user_id):
+        return jsonify({"message": "AI Agent deleted successfully"})
+    return jsonify({"message": "AI Agent not found"}), 404
+
+@ai_agent_bp.route('/ai-agents/<agent_id>/variants', methods=['POST'])
+@login_required
+def create_variant(agent_id):
+    data = request.json
+    user_id = session['user']['uid']
+    variant_name = data['variant_name']
+    variables = data['variables']
+
+    variant_id = add_variant(agent_id, user_id, variant_name, variables)
+    if variant_id:
+        return jsonify({
+            "message": f"Variant '{variant_name}' added successfully",
+            "variant_id": variant_id
+        })
+    return jsonify({"message": "Failed to add variant"}), 404
+
+@ai_agent_bp.route('/ai-agents/<agent_id>/variants/<variant_name>', methods=['DELETE'])
+@login_required
+def remove_variant(agent_id, variant_name):
+    user_id = session['user']['uid']
+    if delete_variant(agent_id, user_id, variant_name):
+        return jsonify({"message": "Variant deleted successfully"})
+    return jsonify({"message": "Variant not found"}), 404

server/app/api/config_routes.py

@@ -0,0 +1,8 @@
+from flask import Blueprint, jsonify
+from ..utils.model_map import MODELS
+
+config_bp = Blueprint('config', __name__)
+
+@config_bp.route('/models', methods=['GET'])
+def get_models():
+    return jsonify(MODELS)

server/app/api/dataset_routes.py

@@ -1,7 +1,7 @@
 from flask import Blueprint, request, jsonify, session
 from ..core.datasets import (
     get_all_datasets, save_dataset, delete_dataset,
-    get_dataset, update_dataset
+    get_dataset, update_dataset, SubjectiveDataset, ObjectiveDataset
 )
 from ..auth.decorators import login_required
 
@@ -49,4 +49,26 @@ def remove_dataset(dataset_id):
     user_id = session['user']['uid']
     if delete_dataset(dataset_id, user_id):
         return jsonify({"message": "Dataset deleted successfully"})
-    return jsonify({"message": "Dataset not found"}), 404
+    return jsonify({"message": "Dataset not found"}), 404
+
+@dataset_bp.route('/datasets/generate_questions', methods=['POST'])
+@login_required
+def generate_questions():
+    try:
+        data = request.json
+        dataset_class = SubjectiveDataset if data['datasetType'] == 'subjective' else ObjectiveDataset
+        dataset = dataset_class(name="temp", id="temp")
+
+        questions = dataset.generate_questions(
+            agent_data=data['agent'],
+            guidelines=data['guidelines'],
+            num_questions=int(data['numQuestions']),
+            inputs=data.get('inputs', {})
+        )
+        return jsonify({"questions": questions})
+
+    except ValueError as e:
+        return jsonify({"message": str(e)}), 400
+    except Exception as e:
+        print(f"Error in generate_questions endpoint: {str(e)}")
+        return jsonify({"message": "Failed to generate questions"}), 500

server/app/api/evaluation_routes.py

@@ -1,7 +1,7 @@
 from flask import Blueprint, request, jsonify, session
-from ..core.characters import get_character
+from ..core.aiagents import get_agent
 from ..core.datasets import get_dataset
-from ..core.models import Character, Evaluator
+from ..core.models import Evaluator
 from ..auth.decorators import login_required
 
 evaluation_bp = Blueprint('evaluation', __name__)

server/app/core/aiagents.py

@@ -0,0 +1,55 @@
+from typing import Dict, Optional, Tuple
+from ..firebase import aiagent_dao
+from .models import AiAgent
+
+def get_all_agents(user_id: str) -> Dict:
+    """Get all AI agents for a user"""
+    return aiagent_dao.get_all(user_id)
+
+def save_agent(agent: AiAgent, user_id: str) -> str:
+    """Create new AI agent"""
+    return aiagent_dao.save(agent, user_id)
+
+def get_agent(agent_id: str, user_id: str, variant_name: str = None) -> Optional[AiAgent]:
+    """Get a specific AI agent and optionally a specific variant"""
+    data = aiagent_dao.get_by_id(agent_id, user_id, variant_name)
+    if data:
+        agent = AiAgent(
+            name=data['name'],
+            prompt=data['prompt'],
+            config=data['config']
+        )
+
+        # Add variants if they exist
+        if 'variants' in data:
+            for var_name, var_data in data['variants'].items():
+                agent.add_variant(var_name, var_data['variables'])
+
+        return agent
+    return None
+
+def add_variant(agent_id: str, user_id: str, variant_name: str, variables: Dict[str, str]) -> bool:
+    """Add a new variant to an existing agent"""
+    # Get the base agent first
+    agent_data = aiagent_dao.get_by_id(agent_id, user_id)
+    if not agent_data:
+        return False
+
+    # Validate that all variables are in included_variables
+    included_vars = set(agent_data.get('included_variables', []))
+    if not included_vars:
+        return False
+
+    variant_vars = set(variables.keys())
+    if not variant_vars.issubset(included_vars):
+        return False
+
+    return aiagent_dao.add_variant(agent_id, user_id, variant_name, variables)
+
+def delete_variant(agent_id: str, user_id: str, variant_name: str) -> bool:
+    """Delete a specific variant"""
+    return aiagent_dao.delete_variant(agent_id, user_id, variant_name)
+
+def delete_agent(agent_id: str, user_id: str) -> bool:
+    """Delete an AI agent"""
+    return aiagent_dao.delete(agent_id, user_id)

server/app/core/datasets.py

@@ -33,10 +33,37 @@ def load(self, inputs):
 
         self.inputs = inputs 
 
-    def generate_samples(self):
-
-        '''Implement a way to generate samples in the dataset'''
-        return
+    def format_prompt(self, prompt: str, inputs: dict) -> str:
+        """Format prompt with input variables"""
+        try:
+            return prompt.format(**inputs)
+        except KeyError as e:
+            raise ValueError(f"Missing required input variable: {str(e)}")
+
+    def generate_questions(self, agent_data: dict, guidelines: str, num_questions: int, inputs: dict) -> list:
+        """Base method for generating questions"""
+        try:
+            formatted_prompt = self.format_prompt(agent_data['prompt'], inputs)
+
+            questions = self.generate_samples(
+                model=agent_data['config']['model'],
+                agent_prompt=formatted_prompt,
+                user_request=guidelines,
+                num_q=num_questions
+            )
+
+            if not questions:
+                raise ValueError("No questions were generated")
+
+            return questions
+
+        except Exception as e:
+            print(f"Error generating questions: {str(e)}")
+            raise
+
+    def generate_samples(self) -> list:
+        """Abstract method to be implemented by subclasses"""
+        raise NotImplementedError("Subclasses must implement generate_samples")
 
     def generate_input_testcases(self,model,prompt_template,num_test_cases,input_descriptions) -> list:
 
@@ -62,7 +89,7 @@ def generate_input_testcases(self,model,prompt_template,num_test_cases,input_des
         )
 
         service = LLMResponseService(
-            model_name = model,
+            model = model,
             system_prompt = ""
         )
 
@@ -75,7 +102,6 @@ def generate_input_testcases(self,model,prompt_template,num_test_cases,input_des
         except Exception as e:
             print(e)
             print("The LLM is probably not returning a valid JSON object, retry it probably")
-            print("The LLM response was: ", response)
 
 class SubjectiveDataset(Dataset):
 
@@ -87,7 +113,7 @@ def generate_samples(self,model: str,agent_prompt: str ,user_request: str, num_q
 
 
         service = LLMResponseService(
-            model_name=model,
+            model=model,
             system_prompt="",
             persist_history=False
         )
@@ -120,7 +146,7 @@ def generate_samples(self,model: str,agent_prompt: str ,user_request: str, num_q
 
 
         service = LLMResponseService(
-            model_name=model,
+            model=model,
             system_prompt="",
             persist_history=False
         )
@@ -140,12 +166,6 @@ def generate_samples(self,model: str,agent_prompt: str ,user_request: str, num_q
 
             return questions_list
         except Exception as e:
+            print(response)
             print(e)
             print("the LLM is probably not returning a valid list object, need to retry")
-
-
-
-
-
-
-