[FIX] Import dataset mapping

.github/workflows/extralit.yml

@@ -82,7 +82,8 @@ jobs:
           uv cache prune --ci
       - name: Wait for extralit-server to start
         run: |
-          while ! curl -XGET http://localhost:6900/api/_status; do sleep 5; done
+          while ! curl -s -o /dev/null -XGET http://localhost:6900/api/_status; do sleep 10; done
+
           # Create a directory for local storage that the container can access
           mkdir -p /tmp/extralit-files
           chmod -R 777 /tmp/extralit-files

.gitignore

@@ -159,5 +159,6 @@ extralit/site
 
 # Development files
 **/*.db
-
-.claude/
+**/*.pdf
+.claude/
+output/

.kiro/specs/papers-library-importer/design.md

@@ -561,7 +561,7 @@ class ImportHistory(DatabaseModel):
 
     # Index on reference field within the JSONB data column for efficient querying
     __table_args__ = (
-        Index('ix_import_history_data_reference', text("(data->'data'->0->>'reference')")),
+        Index('ix_imports_data_reference', text("(data->'data'->0->>'reference')")),
     )
 ```
 

.pre-commit-config.yaml

@@ -75,7 +75,7 @@ repos:
     hooks:
       - id: frontend-lint
         name: "Lint and fix extralit-frontend files"
-        entry: bash -c 'cd extralit-frontend && npx eslint --fix "${@#extralit-frontend/}" || true'
+        entry: bash -c 'cd extralit-frontend && npx eslint --fix --cache "$@" || true'
         language: system
         files: '^extralit-frontend/.*\.(js|ts|vue)$'
         pass_filenames: true

extralit-frontend/CHANGELOG.md

@@ -15,9 +15,14 @@ These are the section headers that we use:
 -->
 
 
-## [Extralit] [Unreleased](https://github.com/extralit/extralit/compare/v0.5.0...v0.6.1)
+## [Extralit] [v0.6.1](https://github.com/extralit/extralit/compare/v0.5.0...v0.6.1)
+### Added
+- Incremental Dataset Import: new `DatasetUpdateDialog` and update workflow in `DatasetConfigurationForm` to update existing datasets with imported data
+
 ### Changed
 - Refactored the frontend to use a single fetchDocument method that queries documents by any identifier and workspace, replacing the previous fetchDocumentByID and fetchDocumentByPubmedID methods. The view model and use case now expect and handle the new API response format
+ - Renamed `DatasetConfigurationDialog` to `DatasetCreateDialog` and improved TypeScript typings and prop validations across configuration components
+ - Improved button area layout, dialog interactions, and hid questions section during update flow to avoid unintended edits
 
 ## [Extralit] [0.6.0](https://github.com/extralit/extralit/compare/v0.4.1...v0.6.0)
 

extralit-frontend/components/features/annotation/container/mode/useDocumentViewModel.ts

@@ -35,12 +35,6 @@ export const useDocumentViewModel = (props: { record: any }) => {
 
       await getDocument.setDocument(params);
     } catch (e) {
-      const identifier = metadata?.pmid || metadata?.doi || metadata?.doc_id || metadata?.reference || "unknown";
-      console.error(`Error fetching document with identifier "${identifier}":`, e);
-      notification.notify({
-        message: `Error fetching document with identifier "${identifier}"`,
-        type: "danger",
-      });
       clearDocument();
     }
   };

extralit-frontend/components/features/dataset-creation/configuration/DatasetConfiguration.vue

@@ -79,7 +79,11 @@
           </template>
           <template #right>
             <div class="dataset-config__configuration">
-              <DatasetConfigurationForm :dataset="dataset" @change-subset="$emit('change-subset', $event)" />
+              <DatasetConfigurationForm
+                :dataset="dataset"
+                :data-source="dataSource"
+                @change-subset="$emit('change-subset', $event)"
+              />
             </div>
           </template>
         </VerticalResizable>
@@ -88,7 +92,7 @@
   </div>
 </template>
 
-<script>
+<script lang="ts">
 import "assets/icons/document";
 import { useDatasetConfiguration } from "./useDatasetConfiguration";
 import { ImportHistoryDetails } from "~/v1/domain/entities/import/ImportHistoryDetails";
@@ -102,7 +106,7 @@ export default {
     dataSource: {
       type: String,
       default: "hub",
-      validator: (value) => ["hub", "import"].includes(value),
+      validator: (value: string) => ["hub", "import"].includes(value),
     },
     importData: {
       type: [ImportHistoryDetails, Object],

extralit-frontend/components/features/dataset-creation/configuration/DatasetConfigurationForm.vue

@@ -45,7 +45,7 @@
               :options="['text', 'label_selection', 'multi_label_selection', 'rating', 'ranking', 'span']"
               @add-question="addQuestion($event)" />
           </div>
-          <div class="config-form__col__content --questions">
+          <div v-if="!isUpdateWorkflow" class="config-form__col__content --questions">
             <draggable v-if="dataset.selectedSubset.questions.length" class="config-form__draggable-area"
               ghost-class="config-form__ghost" :list="dataset.selectedSubset.questions" :group="{ name: 'questions' }"
               :disabled="isFocused">
@@ -59,11 +59,33 @@
           </div>
         </div>
         <div class="config-form__button-area">
-          <BaseButton class="primary" @click.prevent="visibleDatasetCreationDialog = !visibleDatasetCreationDialog">{{
-            $t("datasetCreation.button")
-            }}</BaseButton>
-          <DatasetConfigurationDialog v-if="visibleDatasetCreationDialog" :dataset="dataset" :is-loading="isLoading"
-            @close-dialog="visibleDatasetCreationDialog = false" @create-dataset="createDataset" />
+          <BaseButton
+            class="primary"
+            @click.prevent="openCreateDatasetDialog"
+            v-text="$t('datasetCreation.createButton')"
+          />
+          <BaseButton
+            v-if="dataSource === 'import'"
+            class="secondary"
+            @click.prevent="openUpdateDatasetDialog"
+            v-text="$t('datasetCreation.updateButton')"
+          />
+
+          <!-- Create Dataset Dialog -->
+          <DatasetCreateDialog
+            v-if="visibleDatasetCreationDialog"
+            :dataset="dataset"
+            :is-loading="isLoading"
+            @close-dialog="visibleDatasetCreationDialog = false"
+            @create-dataset="createDataset" />
+
+          <!-- Update Dataset Dialog -->
+          <DatasetUpdateDialog
+            v-if="visibleDatasetUpdateDialog"
+            :dataset="dataset"
+            :is-loading="isLoading"
+            @close-dialog="closeUpdateDialog"
+            @update-dataset="updateDataset" />
         </div>
       </div>
     </div>
@@ -80,12 +102,19 @@ export default {
       type: Object,
       required: true,
     },
+    dataSource: {
+      type: String,
+      default: "hub",
+      validator: (value: string) => ["hub", "import"].includes(value),
+    },
   },
   data() {
     return {
       isFocused: false,
       visibleDatasetCreationDialog: false,
+      visibleDatasetUpdateDialog: false,
       selectedMetadataFields: [],
+      isUpdateWorkflow: false,
     };
   },
   computed: {
@@ -105,11 +134,40 @@ export default {
       // Fallback to field names from the dataset
       return this.dataset.selectedSubset.fields.map((f: any) => f.name);
     },
+    // Extract field schema information from the dataset for compatibility API
+    datasetFieldSchema() {
+      return this.dataset.selectedSubset.fields.map(field => ({
+        name: field.name,
+        type: field.originalType?.value || field.type?.value || 'string'
+      }));
+    },
+    // Get column names for compatibility checking
+    columnNames() {
+      return this.dataset.selectedSubset.fields.map(f => f.name);
+    },
   },
   methods: {
     createDataset() {
       this.create(this.dataset);
     },
+    openCreateDatasetDialog() {
+      this.visibleDatasetCreationDialog = true;
+    },
+    openUpdateDatasetDialog() {
+      this.isUpdateWorkflow = true;
+      this.visibleDatasetUpdateDialog = true;
+    },
+    closeUpdateDialog() {
+      this.isUpdateWorkflow = false;
+      this.visibleDatasetUpdateDialog = false;
+    },
+    updateDataset(updateData) {
+      this.closeUpdateDialog();
+
+      // Use the source dataset from the dialog event, or fall back to this.dataset
+      const sourceDataset = updateData.source || this.dataset;
+      this.update(sourceDataset, updateData.targetDataset.id);
+    },
     generateName(type: string, number: string | number): string {
       const typeName = this.$t(`config.questionId.${type}`);
       return `${typeName}_${parseInt(number as string) || 0}`;
@@ -268,6 +326,7 @@ export default {
 
   &__button-area {
     display: flex;
+    gap: 1rem;
 
     .button {
       width: 100%;

extralit-frontend/components/features/dataset-creation/configuration/DatasetConfigurationMetadataSelector.vue

@@ -17,7 +17,7 @@
   </div>
 </template>
 
-<script>
+<script lang="ts">
 import "assets/icons/check";
 
 export default {

extralit-frontend/components/features/dataset-creation/configuration/DatasetConfigurationDialog.vue → extralit-frontend/components/features/dataset-creation/configuration/DatasetCreateDialog.vue

@@ -1,6 +1,6 @@
 <template>
   <transition name="fade" appear>
-    <dialog class="dataset-config-dialog" v-click-outside="closeDialog">
+    <dialog v-click-outside="closeDialog" class="dataset-config-dialog">
       <form class="dataset-config-dialog__content" @submit.prevent="createDataset">
         <h1 class="dataset-config-dialog__title" v-text="$t('datasetCreation.createDataset')" />
         <div class="dataset-config-dialog__row">
@@ -22,15 +22,15 @@
             class="dataset-config-dialog__unique-workspace"
             v-text="dataset.workspace.name"
           />
-          <DatasetConfigurationSelector v-else :options="workspaces" v-model="dataset.workspace" />
+          <DatasetConfigurationSelector v-else v-model="dataset.workspace" :options="workspaces" />
         </div>
 
-        <div class="dataset-config-dialog__row" v-if="dataset.selectedSubset.splits?.length > 1">
+        <div v-if="dataset.selectedSubset.splits?.length > 1" class="dataset-config-dialog__row">
           <label class="dataset-config-dialog__label" v-text="$t('datasetCreation.selectSplit')" />
           <DatasetConfigurationSelector
+            v-model="dataset.selectedSubset.selectedSplit"
             class="config-form__selector"
             :options="dataset.selectedSubset.splits"
-            v-model="dataset.selectedSubset.selectedSplit"
           />
         </div>
         <p class="dataset-config-dialog__info" v-text="$t('datasetCreation.recordWarning')" />
@@ -39,14 +39,14 @@
           :loading="isLoading"
           type="submit"
           class="dataset-config-dialog__button primary full"
-          >{{ $t("datasetCreation.button") }}</BaseButton
+          >{{ $t("datasetCreation.createButton") }}</BaseButton
         >
         <Validation v-if="!dataset.isValid" :validations="firstTranslatedValidation" />
       </form>
     </dialog>
   </transition>
 </template>
-<script>
+<script lang="ts">
 import Validation from "../../annotation/settings/Validation.vue";
 import { useDatasetConfigurationNameAndWorkspace } from "./useDatasetConfigurationNameAndWorkspace";
 
@@ -61,6 +61,11 @@ export default {
       type: Boolean,
       default: false,
     },
+    dataSource: {
+      type: String,
+      default: "hub",
+      validator: (value: string) => ["hub", "import"].includes(value),
+    },
   },
   watch: {
     firstWorkspace: {