Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -20,4 +20,4 @@ def getDocumentContent(self, documentId: str) -> Document:
Document: the Document containg the relative content.
"""
document = self.useCase.getDocumentsContent([DocumentId(documentId)])
return document[0]
return document[0] if document is not None else None
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from langchain.chains import ConversationalRetrievalChain
from langchain.chains.base import Chain
from langchain.memory import ConversationBufferWindowMemory
from langchain_core.messages import get_buffer_string

from domain.chat.message_response import MessageResponse
from domain.chat.message import Message
Expand All @@ -22,18 +23,22 @@ def __init__(self, chain: Chain, chatHistoryManager: ChatHistoryManager):

def askChatbot(self, message: Message, chatId: ChatId) -> MessageResponse:
if chatId is not None:
self.chain.memory = self.chatHistoryManager.getChatHistory(chatId)
print(self.chatHistoryManager.getChatHistory(chatId.id).messages, flush=True)
answer = self.chain.invoke({"question": message.content})
chatHistory = self.chatHistoryManager.getChatHistory(chatId.id)
if len(chatHistory.messages) == 0:
return MessageResponse(status=False, messageResponse=None, chatId=chatId)
else:
#TODO: Controllare se 6 messaggi sono sufficienti
answer = self.chain.invoke({"question": message.content, "chat_history": get_buffer_string(chatHistory.messages[:-6])})
else:
answer = self.chain.invoke({"question": message.content, "chat_history": ""})
answer = self.chain.invoke({"question": message.content, "chat_history": []})

return MessageResponse(
True,
Message(
status=True,
messageResponse=Message(
answer["answer"],
datetime.now(timezone.utc),
list(set(DocumentId(relevantDocumentId.metadata.get("source")) for relevantDocumentId in answer["source_documents"])),
MessageSender.CHATBOT
), chatId
),
chatId=chatId
)
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,7 @@ def __init__(self, postgresChatORM: PostgresChatORM):
self.postgresChatORM = postgresChatORM

def persistChat(self, messages: List[Message], chatId: ChatId) -> ChatOperationResponse:
postgresChatOperationResponse = self.postgresChatORM.persistChat([self.toPostgresMessageFrom(message) for message in messages], chatId)
print(postgresChatOperationResponse, flush=True)
postgresChatOperationResponse = self.postgresChatORM.persistChat([self.toPostgresMessageFrom(message) for message in messages], chatId.id if chatId else None)
return postgresChatOperationResponse.toChatOperationResponse()

def toPostgresMessageFrom(self, message: Message) -> PostgresMessage:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ class GetChatMessagesPostgres(GetChatMessagesPort):
def __init__(self, postgresORM: PostgresChatORM):
self.postgresORM = postgresORM

def getChatMessages(self, chatId:ChatId)->Chat:
def getChatMessages(self, chatId: ChatId) -> Chat:
chatMessages = self.postgresORM.getChatMessages(chatId.id)
chat = chatMessages.toChat()
return chat
return chatMessages.toChat() if chatMessages is not None else None
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,8 @@ def __init__(self, postgresConfigurationORM: PostgresConfigurationORM):

def getConfiguration(self) -> Configuration:
userId = os.environ.get('USER_ID')

postgresConfiguration = self.postgresConfigurationORM.getConfiguration(userId=userId)

if postgresConfiguration is None:
return None
return postgresConfiguration.toConfiguration()
try:
postgresConfiguration = self.postgresConfigurationORM.getConfiguration(userId=userId)
return postgresConfiguration.toConfiguration()
except Exception as e:
return None
Original file line number Diff line number Diff line change
@@ -1,10 +1,7 @@
import os
from typing import List

from domain.document.document_content import DocumentContent
from domain.document.document_filter import DocumentFilter
from domain.document.document_id import DocumentId
from domain.document.document_metadata import DocumentMetadata, DocumentType
from domain.document.plain_document import PlainDocument
from application.port.out.get_documents_content_port import GetDocumentsContentPort
from adapter.out.persistence.aws.AWS_manager import AWSS3Manager
Expand All @@ -21,15 +18,6 @@ def getDocumentsContent(self, documentIds: List[DocumentId]) -> List[PlainDocume
retrievedDocument = self.awsS3Manager.getDocumentContent(documentId.id)
documents.append(retrievedDocument)

plainDocuments = [
PlainDocument(
DocumentMetadata(
id=DocumentId(document.id),
type=DocumentType.PDF if document.type.split('.')[1].upper() == "PDF" else DocumentType.DOCX,
size=document.size,
uploadTime=document.uploadTime
),
DocumentContent(document.content)
) if document is not None else None for document in documents]
plainDocuments = [document.toPlainDocument() if document is not None else None for document in documents]
return plainDocuments

Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,7 @@

from adapter.out.persistence.aws.AWS_manager import AWSS3Manager
from domain.document.document_filter import DocumentFilter
from domain.document.document_id import DocumentId
from domain.document.document_metadata import DocumentMetadata, DocumentType
from adapter.out.persistence.aws.AWS_document_metadata import AWSDocumentMetadata
from domain.document.document_metadata import DocumentMetadata
from application.port.out.get_documents_metadata_port import GetDocumentsMetadataPort


Expand All @@ -14,9 +12,8 @@ def __init__(self, awsS3Manager: AWSS3Manager):
self.awsS3Manager = awsS3Manager

def getDocumentsMetadata(self, documentFilter: DocumentFilter) -> List[DocumentMetadata]:
listOfDocumentsMetadata = []
documentsMetadatas = self.awsS3Manager.getDocumentsMetadata(documentFilter.searchFilter)
for documentMetadata in documentsMetadatas:
documentM = documentMetadata.toDocumentMetadataFrom()
listOfDocumentsMetadata.append(documentM)
return listOfDocumentsMetadata
documentsMetadatas = []
documentsMetadata = self.awsS3Manager.getDocumentsMetadata(documentFilter.searchFilter)
for documentMetadata in documentsMetadata:
documentsMetadata.append(documentMetadata.toDocumentMetadataFrom())
return documentsMetadatas
Original file line number Diff line number Diff line change
@@ -1,5 +1,11 @@
from datetime import datetime
from dataclasses import dataclass

from domain.document.document_id import DocumentId
from domain.document.document_metadata import DocumentMetadata, DocumentType
from domain.document.document_content import DocumentContent
from domain.document.plain_document import PlainDocument

"""
This class is used to represent a document that is stored in the AWS S3 bucket.
"""
Expand All @@ -10,5 +16,15 @@ class AWSDocument:
type: str
size: float
uploadTime: datetime


def toPlainDocument(self) -> PlainDocument:
return PlainDocument(
metadata=DocumentMetadata(
id=DocumentId(self.id),
type=DocumentType.PDF if self.type == "PDF" else DocumentType.DOCX,
size=self.size,
uploadTime=self.uploadTime
),
content=DocumentContent(self.content)
)

Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,11 @@
class AWSDocumentMetadata:
id: str
size: float
type: str
uploadTime: datetime

def toDocumentMetadataFrom(self) -> DocumentMetadata:
return DocumentMetadata(id=DocumentId(self.id),
type=DocumentType.PDF if os.path.splitext(self.id)[1].upper() == ".PDF" else DocumentType.DOCX,
type=DocumentType.PDF if self.type == "PDF" else DocumentType.DOCX,
size=self.size,
uploadTime=self.uploadTime)
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,4 @@ class AWSDocumentOperationResponse:
message: str

def toDocumentOperationResponse(self) -> DocumentOperationResponse:
return DocumentOperationResponse(DocumentId(self.documentId), self.status, self.message)
return DocumentOperationResponse(DocumentId(self.documentId), self.status, self.message)
85 changes: 50 additions & 35 deletions 3 - PB/MVP/src/backend/adapter/out/persistence/aws/AWS_manager.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@
import os
from typing import List

import boto3
from adapter.out.persistence.aws.AWS_document import AWSDocument
from adapter.out.persistence.aws.AWS_document_operation_response import AWSDocumentOperationResponse
from adapter.out.persistence.aws.AWS_document_metadata import AWSDocumentMetadata
from domain.document.document_id import DocumentId
from domain.document.document_metadata import DocumentMetadata, DocumentType

from botocore.exceptions import ClientError

"""
This class is responsible for managing the AWS S3 bucket.
Expand Down Expand Up @@ -47,31 +46,45 @@ def getDocumentById(self, documentId: str) -> AWSDocument:
type = aws.get('ContentType')
size = aws.get('ContentLength')
uploadTime = aws.get('LastModified')
except:
except Exception as e:
return None
return AWSDocument(
id,
content,
type,
size,
uploadTime
id=id,
content=content,
type=type,
size=size,
uploadTime=uploadTime
)

def uploadDocuments(self, awsDocuments: List[AWSDocument], forceUpload: bool) -> List[AWSDocumentOperationResponse]:
AWSDocumentOperationResponses = []

for document in awsDocuments:
try:
if not forceUpload:
self.s3.head_object(Bucket=self.awsBucketName, Key=document.id)
AWSDocumentOperationResponses.append(AWSDocumentOperationResponse(document.id, False, "Il documento e' gia' presente nel sistema."))
else:
self.s3.put_object(Bucket=self.awsBucketName, Key=document.id, Body=document.content, ContentType=document.type)
AWSDocumentOperationResponses.append(AWSDocumentOperationResponse(document.id, True, "Caricamento del documento avvenuto con successo."))
except Exception as e:
AWSDocumentOperationResponses.append(AWSDocumentOperationResponse(document.id, False, f"Errore durante il caricamento del documento: {e}"))
continue

if not forceUpload:
for awsDocument in awsDocuments:
try:
self.s3.head_object(Bucket=self.awsBucketName, Key=awsDocument.id)
# The document is already present in the system, so it cannot be uploaded.
AWSDocumentOperationResponses.append(AWSDocumentOperationResponse(awsDocument.id, False, "Il documento è già presente nel sistema."))
except Exception as e:
# The document is not present in the system, so it can be uploaded.
try:
self.s3.put_object(Bucket=self.awsBucketName, Key=awsDocument.id, Body=awsDocument.content, ContentType=awsDocument.type)
AWSDocumentOperationResponses.append(AWSDocumentOperationResponse(awsDocument.id, True, "Caricamento del documento avvenuto con successo."))
except Exception as e:
# An error occurred during the put_object operation.
AWSDocumentOperationResponses.append(AWSDocumentOperationResponse(awsDocument.id, False, f"Errore durante il caricamento del documento: {e}"))
continue
else:
# The forceUpload flag is set to True, so the documents can be uploaded without checking if they are already present in the system.
for awsDocument in awsDocuments:
try:
self.s3.put_object(Bucket=self.awsBucketName, Key=awsDocument.id, Body=awsDocument.content, ContentType=awsDocument.type)
AWSDocumentOperationResponses.append(AWSDocumentOperationResponse(awsDocument.id, True, "Caricamento del documento avvenuto con successo."))
except Exception as e:
# An error occurred during the put_object operation.
AWSDocumentOperationResponses.append(AWSDocumentOperationResponse(awsDocument.id, False, f"Errore durante il caricamento del documento: {e}"))
continue

return AWSDocumentOperationResponses

def deleteDocuments(self, documentsIds: List[str]) -> List[AWSDocumentOperationResponse]:
Expand All @@ -81,10 +94,11 @@ def deleteDocuments(self, documentsIds: List[str]) -> List[AWSDocumentOperationR
try:
self.s3.delete_object(Bucket=self.awsBucketName, Key=documentId)
AWSDocumentOperationResponses.append(AWSDocumentOperationResponse(documentId, True, "Eliminazione del documento avvenuta con successo."))
except self.s3.exceptions.NoSuchKey: #TODO: Non fa partire l'eccezione se il documento non esiste
AWSDocumentOperationResponses.append(AWSDocumentOperationResponse(documentId, False, "Il documento non è presente nel sistema."))
except Exception as e:
AWSDocumentOperationResponses.append(AWSDocumentOperationResponse(documentId, False, f"Errore durante l'eliminazione del documento: {e}"))
except ClientError as e:
if e.response['Error']['Code'] == 'NoSuchKey':
AWSDocumentOperationResponses.append(AWSDocumentOperationResponse(documentId, False, "Il documento non è presente nel sistema."))
else:
AWSDocumentOperationResponses.append(AWSDocumentOperationResponse(documentId, False, f"Errore durante l'eliminazione del documento: {e}"))
continue

return AWSDocumentOperationResponses
Expand All @@ -96,9 +110,10 @@ def getDocumentsMetadata(self, documentFilter: str) -> List[AWSDocumentMetadata]
for content in contents:
awsDocumentsMetadata.append(
AWSDocumentMetadata(
content.get('Key'),
content.get('Size'),
content.get('LastModified')
id=content.get('Key'),
size=content.get('Size'),
uploadTime=content.get('LastModified'),
type=content.get('ContentType')
)
)
return awsDocumentsMetadata
Expand All @@ -107,11 +122,11 @@ def getDocumentContent(self, documentId: str) -> AWSDocument:
try:
documentContentResponse = self.s3.get_object(Bucket=self.awsBucketName, Key=documentId)
return AWSDocument(
documentId,
documentContentResponse.get('Body').read(),
documentContentResponse.get('ContentType'),
documentContentResponse.get('ContentLength'),
documentContentResponse.get('LastModified')
id=documentId,
content=documentContentResponse.get('Body').read(),
type=documentContentResponse.get('ContentType'),
size=documentContentResponse.get('ContentLength'),
uploadTime=documentContentResponse.get('LastModified')
)
except self.s3.exceptions.NoSuchKey:
return None
except Exception as e:
return None
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from typing import List

from langchain_community.chat_message_histories import (PostgresChatMessageHistory)
from langchain_community.chat_message_histories import PostgresChatMessageHistory
from langchain_core.messages import BaseMessage
import os

Expand All @@ -9,8 +9,11 @@

class ChatHistoryManager:
def getChatHistory(self, chatId: int)-> PostgresChatMessageHistory:
history = PostgresChatMessageHistory(
connection_string=os.environ.get('DATABASE_URL'),
session_id=str(chatId),
)
return history
try:
history = PostgresChatMessageHistory(
connection_string=os.environ.get('DATABASE_URL'),
session_id=str(chatId),
)
return history
except Exception:
return None
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ class Chat(Base):
messages_cascade = relationship(
'MessageStore',
back_populates="messages_cascade_rel",
cascade="all, delete",
cascade="all, delete, delete-orphan",
passive_deletes=True
)

Expand All @@ -23,14 +23,14 @@ def __repr__(self):
class MessageStore(Base):
__tablename__ = 'message_store'
id = Column('id', Integer, primary_key=True, autoincrement=True)
sessionId = Column('session_id', Integer, ForeignKey('chat.id'))
sessionId = Column('session_id', Integer, ForeignKey('chat.id', ondelete='CASCADE'))
message = Column('message', JSON)

messages_cascade_rel = relationship("Chat", back_populates="messages_cascade")
relevant_documents_cascade = relationship(
'MessageRelevantDocuments',
back_populates="relevant_documents_cascade_rel",
cascade="all, delete",
cascade="all, delete, delete-orphan",
passive_deletes=True
)

Expand All @@ -43,7 +43,7 @@ def __repr__(self):

class MessageRelevantDocuments(Base):
__tablename__ = 'message_relevant_documents'
id = Column('id', Integer, ForeignKey('message_store.id'), primary_key=True)
id = Column('id', Integer, ForeignKey('message_store.id', ondelete='CASCADE'), primary_key=True)
documentId = Column('document_id', Text, primary_key=True)

relevant_documents_cascade_rel = relationship("MessageStore", back_populates="relevant_documents_cascade")
Expand Down
Loading