From c4a9fb800944eb767383d7f5c19f3faa184f8c78 Mon Sep 17 00:00:00 2001 From: rmdevv Date: Mon, 11 Mar 2024 18:49:40 +0100 Subject: [PATCH 1/2] Fix: Aggiunta postgres chat orm --- .../out/persistence/postgres/chat_models.py | 34 +++++++++++++++++++ .../postgres/configuration_models.py | 4 +-- .../out/persistence/postgres/database.py | 28 +++++++++++++++ .../persistence/postgres/postgres_chat_orm.py | 7 +++- .../postgres/postgres_configuration_orm.py | 26 ++------------ 3 - PB/MVP/src/backend/api_exceptions.py | 5 +++ 3 - PB/MVP/src/backend/app.py | 8 +++-- 7 files changed, 84 insertions(+), 28 deletions(-) create mode 100644 3 - PB/MVP/src/backend/adapter/out/persistence/postgres/database.py diff --git a/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/chat_models.py b/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/chat_models.py index e69de29b..f312be2e 100644 --- a/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/chat_models.py +++ b/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/chat_models.py @@ -0,0 +1,34 @@ +from sqlalchemy import Column, Integer, String, Enum as SQLEnum, ForeignKey, Text, JSON +from enum import Enum +from sqlalchemy.orm import relationship + +from adapter.out.persistence.postgres.database import Base + +class Chat(Base): + __tablename__ = 'chat' + id = Column('id', Integer, primary_key=True, autoincrement=True) + title = Column('title', Text) + + def __init__(self, title: str) -> None: + self.title = title + + def __repr__(self): + return f'({self.id}, {self.title})' + +class MessageStore(Base): + __tablename__ = 'message_store' + id = Column('id', Integer, primary_key=True, autoincrement=True) + sessionId = Column('session_id', Text, ForeignKey('chat.id')) + message = Column('message', JSON) + + def __init__(self, sessionId: str, message: str) -> None: + self.sessionId = sessionId + self.message = message + + def __repr__(self): + return f'({self.id}, {self.sessionId}, {self.message})' + +class MessageRelevantDocuments(Base): + id = Column('id', Integer, ForeignKey('message_store.id'), primary_key=True) + documentId = Column('document_id', Text, primary_key=True) + \ No newline at end of file diff --git a/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/configuration_models.py b/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/configuration_models.py index 84ec9b07..de5f19af 100644 --- a/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/configuration_models.py +++ b/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/configuration_models.py @@ -1,13 +1,13 @@ from sqlalchemy import Column, Integer, String, Enum as SQLEnum, ForeignKey from enum import Enum -from sqlalchemy.orm import declarative_base, relationship +from sqlalchemy.orm import relationship from domain.configuration.document_store_configuration import DocumentStoreConfiguration from domain.configuration.embedding_model_configuration import EmbeddingModelConfiguration from domain.configuration.llm_model_configuration import LLMModelConfiguration from domain.configuration.vector_store_configuration import VectorStoreConfiguration -Base = declarative_base() +from adapter.out.persistence.postgres.database import Base class PostgresDocumentStoreType(Enum): AWS = 1 diff --git a/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/database.py b/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/database.py new file mode 100644 index 00000000..316be136 --- /dev/null +++ b/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/database.py @@ -0,0 +1,28 @@ +import os + +from sqlalchemy import create_engine +from sqlalchemy.orm import declarative_base, scoped_session, sessionmaker, relationship + +from adapter.out.persistence.postgres.configuration_models import PostgresConfigurationChoice, PostgresVectorStoreConfiguration, PostgresEmbeddingModelConfiguration, PostgresLLMModelConfiguration, PostgresDocumentStoreConfiguration, PostgresLLMModelType, PostgresVectorStoreType, PostgresEmbeddingModelType, PostgresDocumentStoreType + +Base = declarative_base() + +engine = create_engine(os.environ.get('DATABASE_URL')) +db_session = scoped_session(sessionmaker(bind=engine)) + +def init_db(): + import adapter.out.persistence.postgres.configuration_models + import adapter.out.persistence.postgres.chat_models + Base.metadata.create_all(bind=engine) + + if db_session.query(PostgresConfigurationChoice).filter(PostgresConfigurationChoice.userId == 1).first() is None: + db_session.add(PostgresVectorStoreConfiguration(name=PostgresVectorStoreType.CHROMA_DB, organization='Chroma', description='Chroma DB is an open-source vector store.', type='Open-source', costIndicator='Free')) + db_session.add(PostgresVectorStoreConfiguration(name=PostgresVectorStoreType.PINECONE, organization='Pinecone', description='Pinecone is a vector database for building real-time applications.', type='On cloud', costIndicator='Paid')) + db_session.add(PostgresEmbeddingModelConfiguration(name=PostgresEmbeddingModelType.HUGGINGFACE, organization='Hugging Face', description='Hugging Face is a company that provides a large number of pre-trained models for natural language processing.', type='Local', costIndicator='Free')) + db_session.add(PostgresEmbeddingModelConfiguration(name=PostgresEmbeddingModelType.OPENAI, organization='OpenAI', description='OpenAI is an artificial intelligence research laboratory.', type='Commercial', costIndicator='Paid')) + db_session.add(PostgresLLMModelConfiguration(name=PostgresLLMModelType.HUGGINGFACE, organization='Hugging Face', description='Hugging Face is a company that provides a large number of pre-trained models for natural language processing.', type='Local', costIndicator='Free')) + db_session.add(PostgresLLMModelConfiguration(name=PostgresLLMModelType.OPENAI, organization='OpenAI', description='OpenAI is an artificial intelligence research laboratory.', type='Commercial', costIndicator='Paid')) + db_session.add(PostgresDocumentStoreConfiguration(name=PostgresDocumentStoreType.AWS, organization='Amazon', description='Amazon Web Services is a subsidiary of Amazon providing on-demand cloud computing platforms and APIs to individuals.', type='On cloud', costIndicator='Paid')) + db_session.commit() + db_session.add(PostgresConfigurationChoice(userId=1, vectorStore=PostgresVectorStoreType.CHROMA_DB, embeddingModel=PostgresEmbeddingModelType.HUGGINGFACE, LLMModel=PostgresLLMModelType.HUGGINGFACE, documentStore=PostgresDocumentStoreType.AWS)) + db_session.commit() \ No newline at end of file diff --git a/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/postgres_chat_orm.py b/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/postgres_chat_orm.py index 0427d8a1..99e3a00c 100644 --- a/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/postgres_chat_orm.py +++ b/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/postgres_chat_orm.py @@ -1,6 +1,11 @@ from typing import List -from adapter.out.persistence.postgres.postgres_message import PostgresMessage +from adapter.out.persistence.postgres.chat_models import Chat, ChatMessage + +from adapter.out.persistence.postgres.database import db_session + from adapter.out.persistence.postgres.postgres_chat_operation_response import PostgresChatOperationResponse +from adapter.out.persistence.postgres.postgres_message import PostgresMessage + class PostgresChatORM: def __init__(self) -> None: diff --git a/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/postgres_configuration_orm.py b/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/postgres_configuration_orm.py index 84e7f289..a6bc9de6 100644 --- a/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/postgres_configuration_orm.py +++ b/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/postgres_configuration_orm.py @@ -1,31 +1,11 @@ -import os from typing import List -from sqlalchemy import create_engine -from sqlalchemy.orm import scoped_session, sessionmaker -from adapter.out.persistence.postgres.configuration_models import Base, PostgresConfigurationChoice, PostgresVectorStoreConfiguration, PostgresEmbeddingModelConfiguration, PostgresLLMModelConfiguration, PostgresDocumentStoreConfiguration, PostgresLLMModelType, PostgresVectorStoreType, PostgresEmbeddingModelType, PostgresDocumentStoreType +from adapter.out.persistence.postgres.configuration_models import PostgresConfigurationChoice, PostgresVectorStoreConfiguration, PostgresEmbeddingModelConfiguration, PostgresLLMModelConfiguration, PostgresDocumentStoreConfiguration, PostgresLLMModelType, PostgresVectorStoreType, PostgresEmbeddingModelType, PostgresDocumentStoreType + +from adapter.out.persistence.postgres.database import db_session from adapter.out.persistence.postgres.postgres_configuration_operation_response import PostgresConfigurationOperationResponse from adapter.out.persistence.postgres.postgres_configuration import PostgresConfiguration -engine = create_engine(os.environ.get('DATABASE_URL')) -db_session = scoped_session(sessionmaker(bind=engine)) - -def init_db(): - import adapter.out.persistence.postgres.configuration_models - Base.metadata.create_all(bind=engine) - - if db_session.query(PostgresConfigurationChoice).filter(PostgresConfigurationChoice.userId == 1).first() is None: - db_session.add(PostgresVectorStoreConfiguration(name=PostgresVectorStoreType.CHROMA_DB, organization='Chroma', description='Chroma DB is an open-source vector store.', type='Open-source', costIndicator='Free')) - db_session.add(PostgresVectorStoreConfiguration(name=PostgresVectorStoreType.PINECONE, organization='Pinecone', description='Pinecone is a vector database for building real-time applications.', type='On cloud', costIndicator='Paid')) - db_session.add(PostgresEmbeddingModelConfiguration(name=PostgresEmbeddingModelType.HUGGINGFACE, organization='Hugging Face', description='Hugging Face is a company that provides a large number of pre-trained models for natural language processing.', type='Local', costIndicator='Free')) - db_session.add(PostgresEmbeddingModelConfiguration(name=PostgresEmbeddingModelType.OPENAI, organization='OpenAI', description='OpenAI is an artificial intelligence research laboratory.', type='Commercial', costIndicator='Paid')) - db_session.add(PostgresLLMModelConfiguration(name=PostgresLLMModelType.HUGGINGFACE, organization='Hugging Face', description='Hugging Face is a company that provides a large number of pre-trained models for natural language processing.', type='Local', costIndicator='Free')) - db_session.add(PostgresLLMModelConfiguration(name=PostgresLLMModelType.OPENAI, organization='OpenAI', description='OpenAI is an artificial intelligence research laboratory.', type='Commercial', costIndicator='Paid')) - db_session.add(PostgresDocumentStoreConfiguration(name=PostgresDocumentStoreType.AWS, organization='Amazon', description='Amazon Web Services is a subsidiary of Amazon providing on-demand cloud computing platforms and APIs to individuals.', type='On cloud', costIndicator='Paid')) - db_session.commit() - db_session.add(PostgresConfigurationChoice(userId=1, vectorStore=PostgresVectorStoreType.CHROMA_DB, embeddingModel=PostgresEmbeddingModelType.HUGGINGFACE, LLMModel=PostgresLLMModelType.HUGGINGFACE, documentStore=PostgresDocumentStoreType.AWS)) - db_session.commit() - class PostgresConfigurationORM: def getConfiguration(self, userId: int) -> PostgresConfiguration: diff --git a/3 - PB/MVP/src/backend/api_exceptions.py b/3 - PB/MVP/src/backend/api_exceptions.py index 6f6bd84b..e45df9e9 100644 --- a/3 - PB/MVP/src/backend/api_exceptions.py +++ b/3 - PB/MVP/src/backend/api_exceptions.py @@ -10,3 +10,8 @@ def __init__(self, message="Documento non supportato."): class InsufficientParameters(APIBadRequest): def __init__(self, message="Parametri insufficienti o errati."): super().__init__(message, status_code=400) + +class APIElaborationException(Exception): + def __init__(self, message): + self.message = message + self.status_code = 500 \ No newline at end of file diff --git a/3 - PB/MVP/src/backend/app.py b/3 - PB/MVP/src/backend/app.py index 04952d21..34ca80f4 100644 --- a/3 - PB/MVP/src/backend/app.py +++ b/3 - PB/MVP/src/backend/app.py @@ -2,12 +2,12 @@ from flask_cors import CORS from api_exceptions import APIBadRequest +from api_exceptions import APIElaborationException from blueprints.get_document_content import getDocumentContentBlueprint from blueprints.upload_documents import uploadDocumentsBlueprint from blueprints.delete_documents import deleteDocumentsBlueprint -from adapter.out.persistence.postgres.postgres_configuration_orm import db_session -from adapter.out.persistence.postgres.postgres_configuration_orm import init_db +from adapter.out.persistence.postgres.database import init_db, db_session from blueprints.change_configuration import changeConfigurationBlueprint from blueprints.conceal_documents import concealDocumentsBlueprint @@ -41,4 +41,8 @@ def shutdown_session(exception=None): @app.errorhandler(APIBadRequest) def handle_api_error(error): + return jsonify(error.message), error.status_code + +@app.errorhandler(APIElaborationException) +def handle_api_elaboration_error(error): return jsonify(error.message), error.status_code \ No newline at end of file From e913fc6f6f9224e138b82f6daaa7e30f37eac9c4 Mon Sep 17 00:00:00 2001 From: rmdevv Date: Tue, 12 Mar 2024 01:20:05 +0100 Subject: [PATCH 2/2] Fix: aggiunte chat orm operations --- .../out/ask_chatbot/ask_chatbot_langchain.py | 17 +-- .../out/ask_chatbot/postgres_persist_chat.py | 12 +- .../adapter/out/configuration_manager.py | 13 +- .../out/delete_chats/delete_chats_postgres.py | 4 +- .../out/persistence/postgres/chat_models.py | 13 +- .../postgres/configuration_models.py | 19 ++- .../out/persistence/postgres/database.py | 23 +--- .../out/persistence/postgres/postgres_chat.py | 13 +- .../persistence/postgres/postgres_chat_orm.py | 117 +++++++++++------- .../postgres/postgres_chat_preview.py | 2 - .../persistence/postgres/postgres_message.py | 2 +- .../vector_store_chromaDB_manager.py | 2 +- .../vector_store_pinecone_manager.py | 2 +- .../huggingface_embedding_model.py | 4 +- .../langchain_embedding_model.py | 3 +- .../openai_embedding_model.py | 4 +- .../service/ask_chatbot_service.py | 2 +- .../backend/blueprints/get_chat_messages.py | 6 +- .../backend/domain/document/document_id.py | 8 +- 3 - PB/MVP/src/backend/requirements.txt | 7 +- 20 files changed, 159 insertions(+), 114 deletions(-) diff --git a/3 - PB/MVP/src/backend/adapter/out/ask_chatbot/ask_chatbot_langchain.py b/3 - PB/MVP/src/backend/adapter/out/ask_chatbot/ask_chatbot_langchain.py index e24868c0..00d1b7b6 100644 --- a/3 - PB/MVP/src/backend/adapter/out/ask_chatbot/ask_chatbot_langchain.py +++ b/3 - PB/MVP/src/backend/adapter/out/ask_chatbot/ask_chatbot_langchain.py @@ -19,17 +19,18 @@ class AskChatbotLangchain(AskChatbotPort): def __init__(self, chain: Chain, chatHistoryManager: ChatHistoryManager): self.chain = chain self.chatHistoryManager = chatHistoryManager + def askChatbot(self, message: Message, chatId: ChatId) -> MessageResponse: - embeddingModel = LangchainEmbeddingModel() if chatId is not None: self.chain.memory = self.chatHistoryManager.getChatHistory(chatId) - answer = self.chain.run(message.content) - print(answer, flush=True) + answer = self.chain.invoke({"question": message.content, "chat_history": ""}) + return MessageResponse( True, - Message(content=answer, - timestamp=datetime.now(timezone.utc), - relevantDocuments=[DocumentId("DocumentoRilevante.pdf")], - sender=MessageSender.CHATBOT), - chatId + Message( + answer["answer"], + datetime.now(timezone.utc), + list(set(DocumentId(relevantDocumentId.metadata.get("source")) for relevantDocumentId in answer["source_documents"])), + MessageSender.CHATBOT + ), chatId ) \ No newline at end of file diff --git a/3 - PB/MVP/src/backend/adapter/out/ask_chatbot/postgres_persist_chat.py b/3 - PB/MVP/src/backend/adapter/out/ask_chatbot/postgres_persist_chat.py index d7fc5245..2e22d06a 100644 --- a/3 - PB/MVP/src/backend/adapter/out/ask_chatbot/postgres_persist_chat.py +++ b/3 - PB/MVP/src/backend/adapter/out/ask_chatbot/postgres_persist_chat.py @@ -1,9 +1,9 @@ from typing import List -from domain.chat.message import Message +from domain.chat.message import Message, MessageSender from domain.chat.chat_id import ChatId from domain.chat.chat_operation_response import ChatOperationResponse -from adapter.out.persistence.postgres.postgres_message import PostgresMessage +from adapter.out.persistence.postgres.postgres_message import PostgresMessage, PostgresMessageSenderType from application.port.out.persist_chat_port import PersistChatPort from adapter.out.persistence.postgres.postgres_chat_orm import PostgresChatORM @@ -13,13 +13,15 @@ def __init__(self, postgresChatORM: PostgresChatORM): self.postgresChatORM = postgresChatORM def persistChat(self, messages: List[Message], chatId: ChatId) -> ChatOperationResponse: - postgresChatOpeartionResponse = self.postgresChatORM.persistChat([self.toPostgresMessageFrom(message) for message in messages], chatId) - return postgresChatOpeartionResponse.toChatOperationResponse() + for message in messages: + print(self.toPostgresMessageFrom(message).sender.name, flush=True) + postgresChatOperationResponse = self.postgresChatORM.persistChat([self.toPostgresMessageFrom(message) for message in messages], chatId) + return postgresChatOperationResponse.toChatOperationResponse() def toPostgresMessageFrom(self, message: Message) -> PostgresMessage: return PostgresMessage( content=message.content, timestamp=message.timestamp, relevantDocuments=[relevantDocumentId.id for relevantDocumentId in message.relevantDocuments] if message.relevantDocuments else None, - sender=message.sender.value + sender=PostgresMessageSenderType.USER if message.sender.value == MessageSender.USER.value else PostgresMessageSenderType.CHATBOT ) \ No newline at end of file diff --git a/3 - PB/MVP/src/backend/adapter/out/configuration_manager.py b/3 - PB/MVP/src/backend/adapter/out/configuration_manager.py index cae7c0cc..77f13154 100644 --- a/3 - PB/MVP/src/backend/adapter/out/configuration_manager.py +++ b/3 - PB/MVP/src/backend/adapter/out/configuration_manager.py @@ -165,24 +165,25 @@ def getAskChatbotPort(self) -> AskChatbotPort: configuredVectorStore = VectorStoreChromaDBManager() else: raise ConfigurationException('Vector store non configurato.') + if configuration.embeddingModel == PostgresEmbeddingModelType.HUGGINGFACE: configuredEmbeddingModel = HuggingFaceEmbeddingModel() elif configuration.embeddingModel == PostgresEmbeddingModelType.OPENAI: configuredEmbeddingModel = OpenAIEmbeddingModel() else: - raise ConfigurationException('Embeddings model non configurato.') - if configuration.LLMModel == PostgresLLMModelType.HUGGINGFACE: + raise ConfigurationException('Embedding model non configurato.') + + if configuration.LLMModel == PostgresLLMModelType.OPENAI: with open('/run/secrets/openai_key', 'r') as file: openai_key = file.read() - configuredLLMModel = OpenAI(openai_api_key= openai_key, model_name="gpt-3.5-turbo-instruct", temperature=0.3) - elif configuration.LLMModel == PostgresLLMModelType.OPENAI: + configuredLLMModel = OpenAI(openai_api_key=openai_key, model_name="gpt-3.5-turbo-instruct", temperature=0.01,) + elif configuration.LLMModel == PostgresLLMModelType.HUGGINGFACE: with open('/run/secrets/huggingface_key', 'r') as file: hugging_face = file.read() - configuredLLMModel = HuggingFaceEndpoint(repo_id="google/flan-5-large", temperature=0.3, token=hugging_face) + configuredLLMModel = HuggingFaceEndpoint(repo_id="mistralai/Mistral-7B-v0.1", temperature=0.01, huggingfacehub_api_token=hugging_face) else: raise ConfigurationException('LLM model non configurato.') - chain = ConversationalRetrievalChain.from_llm( llm=configuredLLMModel, retriever=configuredVectorStore.getRetriever(configuredEmbeddingModel), diff --git a/3 - PB/MVP/src/backend/adapter/out/delete_chats/delete_chats_postgres.py b/3 - PB/MVP/src/backend/adapter/out/delete_chats/delete_chats_postgres.py index e33143d8..9ac2f2d8 100644 --- a/3 - PB/MVP/src/backend/adapter/out/delete_chats/delete_chats_postgres.py +++ b/3 - PB/MVP/src/backend/adapter/out/delete_chats/delete_chats_postgres.py @@ -9,5 +9,5 @@ def __init__(self, postgresChatORM: PostgresChatORM): self.postgresORM = postgresChatORM def deleteChats(self, chatsIdsList: List[ChatId]) -> List[ChatOperationResponse]: - postgresOpearationResponseList = self.postgresORM.deleteChats([chatId.id for chatId in chatsIdsList]) - return [postgresChatOpearationResponse.toChatOperationResponse() for postgresChatOpearationResponse in postgresOpearationResponseList] \ No newline at end of file + postgresOperationResponseList = self.postgresORM.deleteChats([chatId.id for chatId in chatsIdsList]) + return [postgresChatOperationResponse.toChatOperationResponse() for postgresChatOperationResponse in postgresOperationResponseList] \ No newline at end of file diff --git a/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/chat_models.py b/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/chat_models.py index f312be2e..192fe94e 100644 --- a/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/chat_models.py +++ b/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/chat_models.py @@ -2,12 +2,12 @@ from enum import Enum from sqlalchemy.orm import relationship -from adapter.out.persistence.postgres.database import Base +from adapter.out.persistence.postgres.database import Base, db_session class Chat(Base): __tablename__ = 'chat' id = Column('id', Integer, primary_key=True, autoincrement=True) - title = Column('title', Text) + title = Column('title', Text, unique=True, nullable=False) def __init__(self, title: str) -> None: self.title = title @@ -18,9 +18,11 @@ def __repr__(self): class MessageStore(Base): __tablename__ = 'message_store' id = Column('id', Integer, primary_key=True, autoincrement=True) - sessionId = Column('session_id', Text, ForeignKey('chat.id')) + sessionId = Column('session_id', Integer, ForeignKey('chat.id')) message = Column('message', JSON) + chatIdConstraint = relationship(Chat, foreign_keys=[sessionId]) + def __init__(self, sessionId: str, message: str) -> None: self.sessionId = sessionId self.message = message @@ -29,6 +31,9 @@ def __repr__(self): return f'({self.id}, {self.sessionId}, {self.message})' class MessageRelevantDocuments(Base): + __tablename__ = 'message_relevant_documents' id = Column('id', Integer, ForeignKey('message_store.id'), primary_key=True) documentId = Column('document_id', Text, primary_key=True) - \ No newline at end of file + +def initChat(): + Base.metadata.create_all(bind=db_session.bind) \ No newline at end of file diff --git a/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/configuration_models.py b/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/configuration_models.py index de5f19af..873d3cde 100644 --- a/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/configuration_models.py +++ b/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/configuration_models.py @@ -7,7 +7,7 @@ from domain.configuration.llm_model_configuration import LLMModelConfiguration from domain.configuration.vector_store_configuration import VectorStoreConfiguration -from adapter.out.persistence.postgres.database import Base +from adapter.out.persistence.postgres.database import Base, db_session class PostgresDocumentStoreType(Enum): AWS = 1 @@ -154,4 +154,19 @@ def __init__(self, userId: int, vectorStore: PostgresVectorStoreType, embeddingM self.documentStore = documentStore def __repr__(self): - return f'({self.userId}, {self.vectorStore}, {self.embeddingModel}, {self.LLMModel}, {self.documentStore})' \ No newline at end of file + return f'({self.userId}, {self.vectorStore}, {self.embeddingModel}, {self.LLMModel}, {self.documentStore})' + +def initConfiguration(): + Base.metadata.create_all(bind=db_session.bind) + + if db_session.query(PostgresConfigurationChoice).filter(PostgresConfigurationChoice.userId == 1).first() is None: + db_session.add(PostgresVectorStoreConfiguration(name=PostgresVectorStoreType.CHROMA_DB, organization='Chroma', description='Chroma DB is an open-source vector store.', type='Open-source', costIndicator='Free')) + db_session.add(PostgresVectorStoreConfiguration(name=PostgresVectorStoreType.PINECONE, organization='Pinecone', description='Pinecone is a vector database for building real-time applications.', type='On cloud', costIndicator='Paid')) + db_session.add(PostgresEmbeddingModelConfiguration(name=PostgresEmbeddingModelType.HUGGINGFACE, organization='Hugging Face', description='Hugging Face is a company that provides a large number of pre-trained models for natural language processing.', type='Local', costIndicator='Free')) + db_session.add(PostgresEmbeddingModelConfiguration(name=PostgresEmbeddingModelType.OPENAI, organization='OpenAI', description='OpenAI is an artificial intelligence research laboratory.', type='Commercial', costIndicator='Paid')) + db_session.add(PostgresLLMModelConfiguration(name=PostgresLLMModelType.HUGGINGFACE, organization='Hugging Face', description='Hugging Face is a company that provides a large number of pre-trained models for natural language processing.', type='Local', costIndicator='Free')) + db_session.add(PostgresLLMModelConfiguration(name=PostgresLLMModelType.OPENAI, organization='OpenAI', description='OpenAI is an artificial intelligence research laboratory.', type='Commercial', costIndicator='Paid')) + db_session.add(PostgresDocumentStoreConfiguration(name=PostgresDocumentStoreType.AWS, organization='Amazon', description='Amazon Web Services is a subsidiary of Amazon providing on-demand cloud computing platforms and APIs to individuals.', type='On cloud', costIndicator='Paid')) + db_session.commit() + db_session.add(PostgresConfigurationChoice(userId=1, vectorStore=PostgresVectorStoreType.CHROMA_DB, embeddingModel=PostgresEmbeddingModelType.HUGGINGFACE, LLMModel=PostgresLLMModelType.HUGGINGFACE, documentStore=PostgresDocumentStoreType.AWS)) + db_session.commit() diff --git a/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/database.py b/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/database.py index 316be136..1e1597dd 100644 --- a/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/database.py +++ b/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/database.py @@ -1,9 +1,7 @@ import os from sqlalchemy import create_engine -from sqlalchemy.orm import declarative_base, scoped_session, sessionmaker, relationship - -from adapter.out.persistence.postgres.configuration_models import PostgresConfigurationChoice, PostgresVectorStoreConfiguration, PostgresEmbeddingModelConfiguration, PostgresLLMModelConfiguration, PostgresDocumentStoreConfiguration, PostgresLLMModelType, PostgresVectorStoreType, PostgresEmbeddingModelType, PostgresDocumentStoreType +from sqlalchemy.orm import declarative_base, scoped_session, sessionmaker Base = declarative_base() @@ -11,18 +9,7 @@ db_session = scoped_session(sessionmaker(bind=engine)) def init_db(): - import adapter.out.persistence.postgres.configuration_models - import adapter.out.persistence.postgres.chat_models - Base.metadata.create_all(bind=engine) - - if db_session.query(PostgresConfigurationChoice).filter(PostgresConfigurationChoice.userId == 1).first() is None: - db_session.add(PostgresVectorStoreConfiguration(name=PostgresVectorStoreType.CHROMA_DB, organization='Chroma', description='Chroma DB is an open-source vector store.', type='Open-source', costIndicator='Free')) - db_session.add(PostgresVectorStoreConfiguration(name=PostgresVectorStoreType.PINECONE, organization='Pinecone', description='Pinecone is a vector database for building real-time applications.', type='On cloud', costIndicator='Paid')) - db_session.add(PostgresEmbeddingModelConfiguration(name=PostgresEmbeddingModelType.HUGGINGFACE, organization='Hugging Face', description='Hugging Face is a company that provides a large number of pre-trained models for natural language processing.', type='Local', costIndicator='Free')) - db_session.add(PostgresEmbeddingModelConfiguration(name=PostgresEmbeddingModelType.OPENAI, organization='OpenAI', description='OpenAI is an artificial intelligence research laboratory.', type='Commercial', costIndicator='Paid')) - db_session.add(PostgresLLMModelConfiguration(name=PostgresLLMModelType.HUGGINGFACE, organization='Hugging Face', description='Hugging Face is a company that provides a large number of pre-trained models for natural language processing.', type='Local', costIndicator='Free')) - db_session.add(PostgresLLMModelConfiguration(name=PostgresLLMModelType.OPENAI, organization='OpenAI', description='OpenAI is an artificial intelligence research laboratory.', type='Commercial', costIndicator='Paid')) - db_session.add(PostgresDocumentStoreConfiguration(name=PostgresDocumentStoreType.AWS, organization='Amazon', description='Amazon Web Services is a subsidiary of Amazon providing on-demand cloud computing platforms and APIs to individuals.', type='On cloud', costIndicator='Paid')) - db_session.commit() - db_session.add(PostgresConfigurationChoice(userId=1, vectorStore=PostgresVectorStoreType.CHROMA_DB, embeddingModel=PostgresEmbeddingModelType.HUGGINGFACE, LLMModel=PostgresLLMModelType.HUGGINGFACE, documentStore=PostgresDocumentStoreType.AWS)) - db_session.commit() \ No newline at end of file + from adapter.out.persistence.postgres.configuration_models import initConfiguration + from adapter.out.persistence.postgres.chat_models import initChat + initConfiguration() + initChat() \ No newline at end of file diff --git a/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/postgres_chat.py b/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/postgres_chat.py index abfb568b..ae86ba8c 100644 --- a/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/postgres_chat.py +++ b/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/postgres_chat.py @@ -1,3 +1,4 @@ +from dataclasses import dataclass from datetime import datetime from typing import List @@ -5,14 +6,12 @@ from domain.chat.chat import Chat from domain.chat.chat_id import ChatId - - +@dataclass class PostgresChat: - def __init__(self, id:int, title:str, timestamp:datetime, messages: List[PostgresMessage]): - self.id = id - self.title = title - self.timestamp = timestamp - self.messages = messages + id: int + title: str + messages: List[PostgresMessage] + def toChat(self): listOfMessages = [] for message in self.messages: diff --git a/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/postgres_chat_orm.py b/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/postgres_chat_orm.py index f34ecb1a..5c8f7763 100644 --- a/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/postgres_chat_orm.py +++ b/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/postgres_chat_orm.py @@ -1,12 +1,15 @@ from datetime import datetime from typing import List -from adapter.out.persistence.postgres.chat_models import Chat, ChatMessage +from adapter.out.persistence.postgres.chat_models import Chat, MessageStore, MessageRelevantDocuments from adapter.out.persistence.postgres.database import db_session from adapter.out.persistence.postgres.postgres_chat_operation_response import PostgresChatOperationResponse -from adapter.out.persistence.postgres.postgres_message import PostgresMessage +from adapter.out.persistence.postgres.postgres_message import PostgresMessage, PostgresMessageSenderType +from adapter.out.persistence.postgres.postgres_chat_preview import PostgresChatPreview +from adapter.out.persistence.postgres.postgres_chat import PostgresChat +from datetime import datetime class PostgresChatORM: def __init__(self) -> None: @@ -15,6 +18,7 @@ def __init__(self) -> None: def persistChat(self, messages: List[PostgresMessage], chatId: int = None) -> PostgresChatOperationResponse: if len(messages) == 0: return PostgresChatOperationResponse(False, "Nessun messaggio da salvare.", None) + if chatId is None: newChatResponse = self.createChat() if not newChatResponse.status: @@ -24,50 +28,79 @@ def persistChat(self, messages: List[PostgresMessage], chatId: int = None) -> Po return self.saveMessages(messages, chatId) def createChat(self) -> PostgresChatOperationResponse: - # try: - # db_session.add(Chat()) - # db_session.commit() - # except Exception as e: - # return PostgresChatOperationResponse(False, f"Errore nella creazione della chat: {str(e)}", None) - return PostgresChatOperationResponse(True, "Chat creata correttamente.", 3) + try: + newChat = Chat(f"Nuova chat {datetime.now().isoformat()}") + db_session.add(newChat) + db_session.commit() + newChatId = newChat.id + return PostgresChatOperationResponse(True, "Chat creata correttamente.", newChatId) + except Exception as e: + return PostgresChatOperationResponse(False, f"Errore nella creazione della chat: {str(e)}", None) def saveMessages(self, messages: List[PostgresMessage], chatId: int) -> PostgresChatOperationResponse: - # try: - # db_session.add_all(messages) - # db_session.commit() - # except Exception as e: - # return PostgresChatOperationResponse(False, f"Errore nel salvataggio dei messaggi: {str(e)}", None) - return PostgresChatOperationResponse(True, "Messaggi salvati correttamente.", chatId) + try: + newMessages = [MessageStore(chatId, {"data": {"type": message.sender.name, "content": message.content, "timestamp": message.timestamp.isoformat()}}) for message in messages] + db_session.add_all(newMessages) + db_session.commit() + newMessageIds = [newMessage.id for newMessage in newMessages] + + messageRelevantDocuments = [] + for i, message in enumerate(messages): + if message.relevantDocuments is not None: + for document in message.relevantDocuments: + messageRelevantDocuments.append(MessageRelevantDocuments(id=newMessageIds[i], documentId=document)) + db_session.add_all(messageRelevantDocuments) + return PostgresChatOperationResponse(True, "Messaggi salvati correttamente.", chatId) + except Exception as e: + return PostgresChatOperationResponse(False, f"Errore nel salvataggio dei messaggi: {str(e)}", None) - def deleteChat(self, chatId: int) -> PostgresChatOperationResponse: - # try: - # db_session.query(Chat).filter(Chat.id == chatId).delete() - # db_session.commit() - # except Exception as e: - # return PostgresChatOperationResponse(False, f"Errore nell'eliminazione della chat: {str(e)}", None) - return PostgresChatOperationResponse(True, "Chat eliminata correttamente.", chatId) + def deleteChats(self, chatIds: List[int]) -> List[PostgresChatOperationResponse]: + try: + db_session.query(Chat).filter(Chat.id.in_(chatIds)).delete(synchronize_session=False) + db_session.commit() + #TODO: eliminare anche i messaggi e i documenti associati + #TODO: vedere se รจ stata eliminata effettivamente la chat + return [PostgresChatOperationResponse(True, "Chat eliminata correttamente.", chatId) for chatId in chatIds] + except Exception as e: + return [PostgresChatOperationResponse(False, f"Errore nella eliminazione della chat: {str(e)}", chatId) for chatId in chatIds] def renameChat(self, chatId: int, newName: str) -> PostgresChatOperationResponse: - # try: - # db_session.query(Chat).filter(Chat.id == chatId).update({Chat.name: newName}) - # db_session.commit() - # except Exception as e: - # return PostgresChatOperationResponse(False, f"Errore nella rinominazione della chat: {str(e)}", None) - return PostgresChatOperationResponse(True, "Chat rinominata correttamente.", chatId) + try: + db_session.query(Chat).filter(Chat.id == chatId).update({"title": newName}) + db_session.commit() + return PostgresChatOperationResponse(True, "Chat rinominata correttamente.", chatId) + except Exception as e: + return PostgresChatOperationResponse(False, f"Errore nella rinomina della chat: {str(e)}", chatId) + def getChats(self, chatFilter:str) -> List[PostgresChatPreview]: - #try: - # listOfPostgresChatPreview = db_session.query(chatFilter).filter(chatFilter).all() - #except Exception as e: - # return #todo da capire come fare - return [PostgresChatPreview(1, "titolo", - PostgresMessage("content",datetime(2020,2,12), ["relevant docs"], PostgresMessageSenderType.USER)), - PostgresChatPreview(1, "titolo2", - PostgresMessage("content2",datetime(2020,2,12), ["relevant docs"], PostgresMessageSenderType.USER))] + try: + chats = db_session.query(Chat).filter(Chat.title.like(f"%{chatFilter}%")).all() + chatPreviews = [] + for chat in chats: + lastMessage = db_session.query(MessageStore).filter(MessageStore.sessionId == chat.id).order_by(MessageStore.id.desc()).first() + if lastMessage is not None: + chatPreviews.append(PostgresChatPreview(chat.id, chat.title, PostgresMessage( + lastMessage.message["data"]["content"], + datetime.fromisoformat(lastMessage.message["data"]["timestamp"]), + [document.documentId for document in db_session.query(MessageRelevantDocuments).filter(MessageRelevantDocuments.id == lastMessage.id).all()], + PostgresMessageSenderType[lastMessage.message["data"]["type"]])) + ) + else: + chatPreviews.append(PostgresChatPreview(chat.id, chat.title, None)) + return chatPreviews + except Exception as e: + return [] + def getChatMessages(self, chatId: int) -> PostgresChat: - #try: - # listOfPostgresMessage = db_session.query(PostgresMessage).filter(PostgresMessage.chatId == chatId).all() - #except Exception as e: - # return #todo da capire come fare - return PostgresChat(1, "titolo", datetime(2020,12,3), - [PostgresMessage("content", datetime(2020,12,3), ["relevantDocs"], PostgresMessageSenderType.USER), - PostgresMessage("content2", datetime(2020,12,3), ["relevantDocs2"], PostgresMessageSenderType.CHATBOT)]) + try: + chat = db_session.query(Chat).filter(Chat.id == chatId).first() + messages = db_session.query(MessageStore).filter(MessageStore.sessionId == chatId).all() + postgresMessages = [PostgresMessage( + message.message["data"]["content"], + datetime.fromisoformat(message.message["data"]["timestamp"]), + [document.documentId for document in db_session.query(MessageRelevantDocuments).filter(MessageRelevantDocuments.id == message.id).all()], + PostgresMessageSenderType[message.message["data"]["type"]]) for message in messages] + + return PostgresChat(chat.id, chat.title, postgresMessages) + except Exception as e: + return None diff --git a/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/postgres_chat_preview.py b/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/postgres_chat_preview.py index 73bd4741..4569cd64 100644 --- a/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/postgres_chat_preview.py +++ b/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/postgres_chat_preview.py @@ -4,8 +4,6 @@ from domain.chat.chat_preview import ChatPreview from adapter.out.persistence.postgres.postgres_message import PostgresMessage - - class PostgresChatPreview: def __init__(self, id: int, title:str, postgresMessage: PostgresMessage): self.id = id diff --git a/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/postgres_message.py b/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/postgres_message.py index 1c3e0635..f8b6f45b 100644 --- a/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/postgres_message.py +++ b/3 - PB/MVP/src/backend/adapter/out/persistence/postgres/postgres_message.py @@ -23,5 +23,5 @@ def toMessage(self) -> Message: return Message(self.content, self.timestamp, [DocumentId(relevantDocument) for relevantDocument in self.relevantDocuments], - MessageSender.USER if self.sender == PostgresMessageSenderType.USER else MessageSender.CHATBOT + MessageSender.USER if self.sender.value == PostgresMessageSenderType.USER.value else MessageSender.CHATBOT ) \ No newline at end of file diff --git a/3 - PB/MVP/src/backend/adapter/out/persistence/vector_store/vector_store_chromaDB_manager.py b/3 - PB/MVP/src/backend/adapter/out/persistence/vector_store/vector_store_chromaDB_manager.py index d08b1da9..18deefc2 100644 --- a/3 - PB/MVP/src/backend/adapter/out/persistence/vector_store/vector_store_chromaDB_manager.py +++ b/3 - PB/MVP/src/backend/adapter/out/persistence/vector_store/vector_store_chromaDB_manager.py @@ -114,4 +114,4 @@ def uploadEmbeddings(self, documentsIds: List[str], documentsChunks: List[List[L def getRetriever(self, embeddingModel : LangchainEmbeddingModel) -> BaseRetriever: - return Chroma(client=self.chromadb, collection_name = self.collection.name, embedding_function=embeddingModel.getEmbedQueryFunction()).as_retriever() \ No newline at end of file + return Chroma(client=self.chromadb, collection_name = self.collection.name, embedding_function=embeddingModel.getEmbeddingFunction()).as_retriever(search_type="similarity_score_threshold", search_kwargs={'filter': {'status':'ENABLED'}, 'score_threshold': 0.5}) \ No newline at end of file diff --git a/3 - PB/MVP/src/backend/adapter/out/persistence/vector_store/vector_store_pinecone_manager.py b/3 - PB/MVP/src/backend/adapter/out/persistence/vector_store/vector_store_pinecone_manager.py index fb92290f..3cdc3f99 100644 --- a/3 - PB/MVP/src/backend/adapter/out/persistence/vector_store/vector_store_pinecone_manager.py +++ b/3 - PB/MVP/src/backend/adapter/out/persistence/vector_store/vector_store_pinecone_manager.py @@ -187,4 +187,4 @@ def uploadEmbeddings(self, documentsId: List[str], documentsChunks: List[List[La return vectorStoreDocumentOperationResponses def getRetriever(self, embeddingModel : LangchainEmbeddingModel) -> BaseRetriever: - return PineconeLangchain(self.index, embeddingModel.getEmbedQueryFunction(), "text").as_retriever() \ No newline at end of file + return PineconeLangchain(self.index, embeddingModel.getEmbeddingFunction(), "text").as_retriever(search_type="similarity_score_threshold", search_kwargs={'filter': {'status':'ENABLED'}, 'score_threshold': 0.5}) \ No newline at end of file diff --git a/3 - PB/MVP/src/backend/adapter/out/upload_documents/huggingface_embedding_model.py b/3 - PB/MVP/src/backend/adapter/out/upload_documents/huggingface_embedding_model.py index 1b0722c1..81eefc6a 100644 --- a/3 - PB/MVP/src/backend/adapter/out/upload_documents/huggingface_embedding_model.py +++ b/3 - PB/MVP/src/backend/adapter/out/upload_documents/huggingface_embedding_model.py @@ -17,5 +17,5 @@ def embedDocument(self, documentChunks: List[str]) -> List[List[float]]: return self.model.embed_documents(documentChunks) except Exception as e: return [] - def getEmbedQueryFunction(self): - return self.model.embed_query \ No newline at end of file + def getEmbeddingFunction(self): + return self.model \ No newline at end of file diff --git a/3 - PB/MVP/src/backend/adapter/out/upload_documents/langchain_embedding_model.py b/3 - PB/MVP/src/backend/adapter/out/upload_documents/langchain_embedding_model.py index 6ad21bea..87dfc167 100644 --- a/3 - PB/MVP/src/backend/adapter/out/upload_documents/langchain_embedding_model.py +++ b/3 - PB/MVP/src/backend/adapter/out/upload_documents/langchain_embedding_model.py @@ -3,5 +3,6 @@ class LangchainEmbeddingModel: def embedDocument(self, documentChunks: List[str]) -> List[List[float]]: pass - def getEmbedQueryFunction(self): + + def getEmbeddingFunction(self): pass \ No newline at end of file diff --git a/3 - PB/MVP/src/backend/adapter/out/upload_documents/openai_embedding_model.py b/3 - PB/MVP/src/backend/adapter/out/upload_documents/openai_embedding_model.py index 5a00d526..c7b0f24e 100644 --- a/3 - PB/MVP/src/backend/adapter/out/upload_documents/openai_embedding_model.py +++ b/3 - PB/MVP/src/backend/adapter/out/upload_documents/openai_embedding_model.py @@ -17,5 +17,5 @@ def embedDocument(self, documentChunks: List[str]) -> List[List[float]]: return self.model.embed_documents(documentChunks) except Exception as e: return [] - def getEmbedQueryFunction(self): - return self.model.embed_query \ No newline at end of file + def getEmbeddingFunction(self): + return self.model \ No newline at end of file diff --git a/3 - PB/MVP/src/backend/application/service/ask_chatbot_service.py b/3 - PB/MVP/src/backend/application/service/ask_chatbot_service.py index a78d4001..e2a5739b 100644 --- a/3 - PB/MVP/src/backend/application/service/ask_chatbot_service.py +++ b/3 - PB/MVP/src/backend/application/service/ask_chatbot_service.py @@ -13,7 +13,7 @@ def __init__(self, askChatbotOutPort: AskChatbotPort, persistChatOutPort: Persis def askChatbot(self, message: Message, chatId: ChatId) -> MessageResponse: messageResponse = self.askChatbotOutPort.askChatbot(message, chatId) - + if messageResponse and messageResponse.status: chatOperationResponse = self.persistChatOutPort.persistChat([message, messageResponse.messageResponse], chatId) diff --git a/3 - PB/MVP/src/backend/blueprints/get_chat_messages.py b/3 - PB/MVP/src/backend/blueprints/get_chat_messages.py index c1333379..73700cc2 100644 --- a/3 - PB/MVP/src/backend/blueprints/get_chat_messages.py +++ b/3 - PB/MVP/src/backend/blueprints/get_chat_messages.py @@ -10,13 +10,11 @@ getChatMessagesBlueprint = Blueprint("getChatMessages", __name__) -@getChatMessagesBlueprint.route('/getChatMessages', defaults={'chatId': ''}, methods=['GET']) +@getChatMessagesBlueprint.route('/getChatMessages/', methods=['GET']) def getChatMessages(chatId): if chatId is None: raise InsufficientParameters() - configurationManager = ConfigurationManager(postgresConfigurationORM=PostgresConfigurationORM()) - controller = GetChatMessagesController( GetChatMessagesService( GetChatMessagesPostgres( @@ -30,6 +28,6 @@ def getChatMessages(chatId): "title": chatMessages.title, "id": chatMessages.chatId.id, "messages": [{"content": chatMessage.content, - "time": chatMessage.timestamp, + "timestamp": chatMessage.timestamp, "sender": chatMessage.sender.name} for chatMessage in chatMessages.messages] }) \ No newline at end of file diff --git a/3 - PB/MVP/src/backend/domain/document/document_id.py b/3 - PB/MVP/src/backend/domain/document/document_id.py index 688ed736..a585edcd 100644 --- a/3 - PB/MVP/src/backend/domain/document/document_id.py +++ b/3 - PB/MVP/src/backend/domain/document/document_id.py @@ -3,4 +3,10 @@ """The unique identifier of a document.""" @dataclass class DocumentId: - id: str \ No newline at end of file + id: str + + def __hash__(self): + return hash(self.id) + + def __eq__(self, other): + return isinstance(other, DocumentId) and self.id == other.id \ No newline at end of file diff --git a/3 - PB/MVP/src/backend/requirements.txt b/3 - PB/MVP/src/backend/requirements.txt index e084f7af..77796645 100644 --- a/3 - PB/MVP/src/backend/requirements.txt +++ b/3 - PB/MVP/src/backend/requirements.txt @@ -14,12 +14,11 @@ openai pypdf PyPDF2 pytest +pytest-mock python-dotenv pinecone-client -#psycopg2-binary -#psycopg -#psycopg-c -#pytest-mock +psycopg +psycopg2-binary # sentence-transformers tiktoken # torch \ No newline at end of file