Translating Expert Radiology Reports into Layman Summaries using pre-trained Flan-T5 Model. Task #13

recognition/FLAN_s4885380/.gitignore

@@ -0,0 +1 @@
+/__pycache__/*

recognition/FLAN_s4885380/constants.py

@@ -0,0 +1,35 @@
+# File containing all constants used
+MODEL_NAME = "google/flan-t5-base"
+
+# Pandas dataframe link for BioLaySumm dataset
+TRAIN_FILE = "hf://datasets/BioLaySumm/BioLaySumm2025-LaymanRRG-opensource-track/data/train-00000-of-00001.parquet"
+VALIDATION_FILE = "hf://datasets/BioLaySumm/BioLaySumm2025-LaymanRRG-opensource-track/data/validation-00000-of-00001.parquet"
+TEST_FILE = "hf://datasets/BioLaySumm/BioLaySumm2025-LaymanRRG-opensource-track/data/test-00000-of-00001.parquet"
+
+INPUT_COLUMN = "radiology_report"
+TARGET_COLUMN = "layman_report"
+
+# Prompt used exclusively in predict.py
+MODEL_PROMPT = "translate this radiology report into a summary for a layperson: "
+
+# Held-out splits
+TRAIN_SPLIT = 0.7
+VALIDATION_SPLIT = 0.3
+
+# Training parameters
+EPOCHS = 3
+LEARNING_RATE = 3e-4
+TRAIN_BATCH_SIZE = 64
+VALID_BATCH_SIZE = 128
+MAX_INPUT_LENGTH = 256
+MAX_TARGET_LENGTH = 128
+
+# LoRA Parameters
+LORA_R = 32
+LORA_ALPHA = 64
+LORA_DROPOUT = 0.05
+LORA_TARGET_MODULES = ["q", "v"]
+
+# File paths for model saving and loss plotting
+OUTPUT_DIR = "t5-base-lora-tuned"
+LOSS_OUT = "loss.png"

recognition/FLAN_s4885380/dataset.py

@@ -0,0 +1,78 @@
+import random
+import pandas as pd
+import numpy as np
+
+from torch.utils.data import Dataset
+from constants import *
+
+"""
+Held-out data splitter for training and evaluation.
+Splits the data into 70/30 ratio
+"""
+class SplitData:
+    def __init__(self, file_path: str, sample_size: int | None = None) -> None:
+        self.dataframe = pd.read_parquet(file_path)
+        if sample_size != None:
+            self.dataframe = self.dataframe[0:sample_size]
+        # else:
+        #     self.dataframe = self.dataframe[100:300] # Testing split
+
+    """
+    Returns both splits at once from the original dataframe
+    """
+    def get_splits(self) -> tuple[pd.DataFrame, pd.DataFrame]:
+        split_index = np.random.random(len(self.dataframe)) < 0.7
+        train = self.dataframe[split_index]
+        validation = self.dataframe[~split_index]
+
+        return train, validation
+
+
+"""
+Custom dataset loader and preprocessor. Prepends 1 of 4 similar prompts for training and evaluation.
+"""
+class FlanDataset(Dataset):
+    def __init__(self, dataframe: pd.DataFrame, tokenizer) -> None:
+        self.tokenizer = tokenizer
+        # self.prefix = MODEL_PROMPT
+        self._prompts = [
+            "Translate this radiology report into a summary for a layperson: ",
+            "Summarise the following medical report in simple, easy-to-understand terms: ",
+            "Explain this radiology report to a patient with no medical background: ",
+            "Provide a layperson's summary for this report: "
+        ]
+
+        self.dataframe = dataframe
+
+        # Biolaysumm dataset is of .parquet file type
+        # Future addition: add support for basic files
+        # self.dataframe = pd.read_parquet(file_path)
+        # self.dataframe = self.dataframe[0:50] # Slice data for subset
+
+    def __len__(self) -> int:
+        return len(self.dataframe)
+
+    """
+    Tokenises the inputs using the tokenizer API. Converts strings into NLP suitable tensors
+    """
+    def __getitem__(self, index: int) -> list:
+        row = self.dataframe.iloc[index] # Selects slices using iloc index
+
+        rand_prefix = random.choice(self._prompts) # Selects random prefix
+        report = rand_prefix + str(row[INPUT_COLUMN])
+        summary = str(row[TARGET_COLUMN])
+
+        model_inputs = self.tokenizer( # Tokenises the radiology report
+            report,
+            max_length=MAX_INPUT_LENGTH,
+            truncation=True
+        )
+
+        with self.tokenizer.as_target_tokenizer(): # Tokenises the layman summary
+            labels = self.tokenizer(
+                summary,
+                max_length=MAX_TARGET_LENGTH,
+                truncation=True
+            )
+        model_inputs["labels"] = labels["input_ids"] # Join report and summary together
+        return model_inputs

recognition/FLAN_s4885380/modules.py

@@ -0,0 +1,57 @@
+from typing import Tuple
+
+from torch.optim import AdamW
+from transformers import (
+    AutoTokenizer,
+    AutoModelForSeq2SeqLM,
+    get_scheduler
+)
+from peft import LoraConfig, get_peft_model, TaskType
+
+from dataset import *
+from constants import *
+
+"""
+Build and loads the pre-trained model as well as LoRA and AdamW optimiser
+"""
+class FlanModel:
+    def __init__(self):
+        pass
+
+    """
+    Loads the tokeniser and Flan-T5 base model. Configures LoRA to parameters described in constant.py
+    """
+    def build(self) -> Tuple[AutoModelForSeq2SeqLM, AutoTokenizer]:
+        # Load actual Flan-T5 models
+        tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
+        model = AutoModelForSeq2SeqLM.from_pretrained(MODEL_NAME)
+
+        # Load LoRA
+        lora_config = LoraConfig(
+            r=LORA_R,
+            lora_alpha=LORA_ALPHA,
+            target_modules=LORA_TARGET_MODULES,
+            lora_dropout=LORA_DROPOUT,
+            bias="none",
+            task_type=TaskType.SEQ_2_SEQ_LM
+        )
+
+        model = get_peft_model(model, lora_config)
+        model.print_trainable_parameters() # Show trainable parameters
+        return model, tokenizer
+
+    """
+    Setup optimiser and scheduler
+    """
+    def setup_optimiser(self, model, train_dataloader) -> Tuple[AdamW, get_scheduler]:
+        optimizer = AdamW(model.parameters(), lr=LEARNING_RATE)
+        num_training_steps = EPOCHS * len(train_dataloader)
+        lr_scheduler = get_scheduler(
+            "linear",
+            optimizer=optimizer,
+            num_warmup_steps=0,
+            num_training_steps=num_training_steps
+        )
+
+        return optimizer, lr_scheduler
+

recognition/FLAN_s4885380/predict.py

@@ -0,0 +1,81 @@
+import torch
+import evaluate
+from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
+from peft import PeftModel
+from datasets import load_dataset
+
+from constants import *
+
+FINETUNED_MODEL = "t5-base-lora-tuned/epoch_3" # Take last epoch for best performance
+
+"""
+Computes the perplexity score using the model loss
+"""
+def perplexity_score(model: AutoModelForSeq2SeqLM,
+                     tokenizer: AutoTokenizer,
+                     prompt: str,
+                     target_text: str,
+                     device="cuda") -> dict:
+    inputs = tokenizer(prompt, return_tensors="pt").to(device)
+    labels = tokenizer(target_text, return_tensors="pt").input_ids.to(device)
+
+    # Gets the loss during benchmarking
+    with torch.no_grad():
+        outputs = model(**inputs, labels=labels)
+        loss = outputs.loss
+
+    perplexity = torch.exp(loss) # Calculate perplexity
+    return perplexity.item()
+
+# Get new base flan-t5 model, and load in saved trained model
+base_model = AutoModelForSeq2SeqLM.from_pretrained(MODEL_NAME, torch_dtype=torch.bfloat16, device_map="auto")
+base_model.eval()
+
+# Use completely fresh Flan-T5 model
+new_t5 = AutoModelForSeq2SeqLM.from_pretrained(MODEL_NAME, torch_dtype=torch.bfloat16, device_map="auto")
+fine_tuned_model = PeftModel.from_pretrained(new_t5, FINETUNED_MODEL)
+fine_tuned_model.eval()
+
+tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
+
+# Use API for loading in dataset
+predict_dataset = load_dataset("BioLaySumm/BioLaySumm2025-LaymanRRG-opensource-track")
+predict_dataset = predict_dataset.shuffle(seed=3710)
+random_predict = predict_dataset["validation"]
+
+predictions = []
+references = []
+
+for i in range(5): # Number of evaluations
+    radiology_report = random_predict[i]['radiology_report']
+    layman_report = random_predict[i]['layman_report']
+
+    prompt = f"translate this radiology report into a summary for a layperson: {radiology_report}"
+    inputs = tokenizer(prompt, return_tensors="pt").to("cuda")
+
+    # Get fine tuned model to generate a summary
+    with torch.no_grad():
+        outputs = fine_tuned_model.generate(**inputs, max_new_tokens=MAX_INPUT_LENGTH)
+
+    prediction = tokenizer.decode(outputs[0], skip_special_tokens=True)
+
+    # Compare perplexity
+    fine_tune_perplexity = perplexity_score(fine_tuned_model, tokenizer, prompt, layman_report)
+    base_model_perplexity = perplexity_score(base_model, tokenizer, prompt, layman_report)
+
+    print(f"\nExample {i + 1}")
+    print(f"Official Layman Report: {layman_report}")
+    print(f"Fine tuned Model's Layman Report: {prediction}")
+
+    print(f"\nFine Tuned Model Perplexity on Official Report: {fine_tune_perplexity:.4f}")
+    print(f"\nBase Model Perplexity on Official Report: {base_model_perplexity:.4f}")
+
+    predictions.append(prediction)
+    references.append(layman_report) # Official report from dataset
+
+rouge_scores = evaluate.load("rouge")
+scores = rouge_scores.compute(predictions=predictions, references=references, use_stemmer=True)
+
+print(f"Final ROUGE scores after predictions")
+for key, value in scores.items():
+    print(f"{key}: {value * 100: .4f}")

recognition/FLAN_s4885380/requirements.txt

@@ -0,0 +1,18 @@
+# Core Fine Tuning and NLP packages
+accelerate==1.10.1
+bitsandbytes==0.48.1
+datasets==4.2.0
+evaluate==0.4.6
+peft==0.17.1
+safetensors==0.6.2
+tokenizers==0.22.1
+torch==2.8.0
+transformers==4.57.0
+
+# Other Utilities
+matplotlib==3.10.7
+nltk==3.9.2
+numpy==2.3.3
+pandas=2.3.3
+rouge-score==0.1.2
+tqdm==4.67.1

recognition/FLAN_s4885380/runners/benchmark

@@ -0,0 +1,11 @@
+#!/bin/bash
+#SBATCH --nodes=1
+#SBATCH --cpus-per-task=4
+#SBATCH --gres=gpu:1
+#SBATCH --partition=a100
+#SBATCH --time=5:00:00
+#SBATCH --job-name=flanbenchmark
+#SBATCH -o benchmark.out
+
+conda activate flan
+python predict.py

recognition/FLAN_s4885380/runners/trainer

@@ -0,0 +1,11 @@
+#!/bin/bash
+#SBATCH --nodes=1
+#SBATCH --cpus-per-task=4
+#SBATCH --gres=gpu:1
+#SBATCH --partition=a100
+#SBATCH --time=5:00:00
+#SBATCH --job-name=flant5finetune
+#SBATCH -o flantune.out
+
+conda activate flan
+python train.py