Desktop Settings, VIAME CSV Parser, Pipeline Runner

.github/workflows/release.yml

@@ -23,7 +23,7 @@ jobs:
         # Specifies the registry, this field is required!
         registry-url: https://registry.npmjs.org/
     - run: yarn install --frozen-lockfile
-    # set up git since we will later push to the repo
+    # set up git because yarn publish creates a commit
     - run: git config --global user.name "GitHub Bot"
     - run: git config --global user.email "viame-web@kitware.com"
     # Run some sanity checks
@@ -32,12 +32,9 @@ jobs:
     - run: yarn test
     # Build and publish
     - run: yarn build:lib
+    - run: yarn build:electron
+    - run: yarn build:cli
     - run: yarn publish --new-version ${{ github.event.release.tag_name }}
       env:
         # Use a token to publish to NPM.  Must configure this!
         NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
-    # push the version changes to GitHub
-    - run: git push
-      env:
-        # The secret is passed automatically. Nothing to configure.
-        github-token: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -22,6 +22,7 @@ __pycache__/
 build/
 develop-eggs/
 dist/
+dist_electron/
 devel/
 downloads/
 eggs/

client/README.md

@@ -29,6 +29,11 @@ yarn lint:templates
 
 # Local verification of all tests, linting, builds
 ./checkbuild.sh
+
+# Parser CLI tools
+yarn serialize viame /path/to/viame.csv
+# output to file, suppress yarn's stdout
+yarn --silent serialize viame /path/to/viame.csv > tracks.json
 ```
 
 See [this issue](https://github.com/vuejs/vue-cli/issues/3065) for details on why our `yarn serve` command is weird.

client/package.json

@@ -7,16 +7,22 @@
     "build:web": "vue-cli-service build platform/web-girder/main.ts",
     "build:electron": "vue-cli-service electron:build",
     "build:lib": "rollup -c",
+    "build:cli": "tsc -p tsconfig.cli.json",
     "lint": "vue-cli-service lint src/ viame-web-common/ platform/",
     "lint:templates": "vtc --workspace . --srcDir src/",
-    "test": "vue-cli-service test:unit src/ viame-web-common/"
+    "test": "vue-cli-service test:unit src/ viame-web-common/",
+    "serialize": "ts-node --project tsconfig.cli.json platform/desktop/backend/serializers/cli.ts"
   },
   "resolutions": {
     "@types/jest": "^25.2.3"
   },
   "files": [
+    "/bin/",
     "/lib/"
   ],
+  "bin": {
+    "viamecli": "./bin/platform/desktop/backend/serializers/cli.js"
+  },
   "main": "lib/index.js",
   "types": "lib/types/index.d.ts",
   "dependencies": {
@@ -43,6 +49,7 @@
   },
   "devDependencies": {
     "@types/axios": "^0.14.0",
+    "@types/csv-parse": "^1.2.2",
     "@types/d3": "^5.7.2",
     "@types/electron-devtools-installer": "^2.2.0",
     "@types/geojson": "^7946.0.7",
@@ -67,6 +74,7 @@
     "babel-eslint": "^10.1.0",
     "babel-jest": "^26.0.1",
     "babel-register": "^6.26.0",
+    "csv-parse": "^4.13.1",
     "electron": "^10.1.3",
     "electron-devtools-installer": "^3.1.1",
     "eslint": "^6.7.2",
@@ -89,6 +97,7 @@
     "sass": "^1.26.3",
     "sass-loader": "^8.0.2",
     "ts-jest": "^26.0.0",
+    "ts-node": "^9.0.0",
     "typescript": "~3.8.3",
     "vue-cli-plugin-electron-builder": "^2.0.0-beta.5",
     "vue-cli-plugin-vuetify": "^2.0.5",
@@ -97,7 +106,8 @@
     "vue-type-check": "^1.0.0",
     "vuetify-loader": "^1.4.3",
     "xml-js": "^1.6.11",
-    "xml2json": "^0.12.0"
+    "xml2json": "^0.12.0",
+    "yargs": "^16.1.0"
   },
   "jest": {
     "verbose": true,

client/platform/desktop/.eslintrc

@@ -0,0 +1,5 @@
+{
+  "rules": {
+    "import/no-extraneous-dependencies": ["error", {"devDependencies": true}]
+  }
+}

client/platform/desktop/App.vue

@@ -7,13 +7,13 @@
 <script lang="ts">
 import { defineComponent } from '@vue/composition-api';
 import { provideApi } from 'viame-web-common/apispec';
-import { observe } from './store/dataset';
+import statefulApi from './store';
 
 export default defineComponent({
   name: 'App',
   components: {},
   setup() {
-    provideApi(observe());
+    provideApi(statefulApi());
   },
 });
 </script>

client/platform/desktop/api/main.ts

@@ -1,51 +1,34 @@
 import { AddressInfo } from 'net';
 import path from 'path';
 
-// eslint-disable-next-line import/no-extraneous-dependencies
 import { ipcRenderer, remote, FileFilter } from 'electron';
-// eslint-disable-next-line import/no-extraneous-dependencies
 import fs from 'fs-extra';
-// eslint-disable-next-line import/no-extraneous-dependencies
 import mime from 'mime-types';
 
 import {
-  Attribute, DatasetMeta,
+  Attribute,
   DatasetMetaMutable,
   DatasetType, FrameImage,
   Pipelines, TrainingConfigs,
 } from 'viame-web-common/apispec';
 
-// TODO: disable node integration in renderer
-// https://nklayman.github.io/vue-cli-plugin-electron-builder/guide/security.html
-import { loadDetections, saveDetections } from './nativeServices';
-
-const websafeVideoTypes = [
-  'video/mp4',
-  'video/webm',
-];
-
-const websafeImageTypes = [
-  'image/apng',
-  'image/bmp',
-  'image/gif',
-  'image/jpeg',
-  'image/png',
-  'image/svg+xml',
-  'image/webp',
-];
-
-export interface DesktopDataset {
-  name: string;
-  basePath: string;
-  root: string;
-  videoPath?: string;
-  meta: DatasetMeta;
-}
+import common from '../backend/platforms/common';
+import {
+  DesktopJob, NvidiaSmiReply, RunPipeline,
+  websafeImageTypes, websafeVideoTypes,
+  DesktopDataset, Settings,
+} from '../constants';
+
+const { loadDetections, saveDetections } = common;
 
 function mediaServerInfo(): Promise<AddressInfo> {
   return ipcRenderer.invoke('info');
 }
 
+function nvidiaSmi(): Promise<NvidiaSmiReply> {
+  return ipcRenderer.invoke('nvidia-smi');
+}
+
 function openLink(url: string): Promise<void> {
   return ipcRenderer.invoke('open-link-in-browser', url);
 }
@@ -68,13 +51,8 @@ async function getAttributes() {
   return Promise.resolve([] as Attribute[]);
 }
 
-async function getPipelineList(): Promise<Pipelines> {
-  return Promise.resolve({});
-}
-
-// eslint-disable-next-line @typescript-eslint/no-unused-vars
-async function runPipeline(itemId: string, pipeline: string) {
-  return Promise.resolve();
+async function getPipelineList(settings: Settings): Promise<Pipelines> {
+  return ipcRenderer.invoke('get-pipeline-list', settings);
 }
 
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
@@ -93,6 +71,7 @@ async function loadMetadata(datasetId: string): Promise<DesktopDataset> {
   let datasetType = undefined as 'video' | 'image-sequence' | undefined;
   let videoUrl = '';
   let videoPath = '';
+  let basePath = datasetId; // default to image-sequence type basepath
   const imageData = [] as FrameImage[];
   const serverInfo = await mediaServerInfo();
 
@@ -102,6 +81,7 @@ async function loadMetadata(datasetId: string): Promise<DesktopDataset> {
     const mimetype = mime.lookup(abspath);
     if (mimetype && websafeVideoTypes.includes(mimetype)) {
       datasetType = 'video';
+      basePath = path.dirname(datasetId); // parent directory of video;
       videoPath = abspath;
       videoUrl = abspathuri;
     } else if (mimetype && websafeImageTypes.includes(mimetype)) {
@@ -130,8 +110,7 @@ async function loadMetadata(datasetId: string): Promise<DesktopDataset> {
 
   return Promise.resolve({
     name: path.basename(datasetId),
-    basePath: path.dirname(datasetId),
-    root: datasetId,
+    basePath,
     videoPath,
     meta: {
       type: datasetType,
@@ -147,6 +126,16 @@ async function saveMetadata(datasetId: string, metadata: DatasetMetaMutable) {
   return Promise.resolve();
 }
 
+async function runPipeline(itemId: string, pipeline: string, settings: Settings) {
+  const args: RunPipeline = {
+    pipelineName: pipeline,
+    datasetId: itemId,
+    settings,
+  };
+  const job: DesktopJob = await ipcRenderer.invoke('run-pipeline', args);
+  return job;
+}
+
 export {
   /* Standard common APIs */
   getAttributes,
@@ -161,4 +150,5 @@ export {
   /* Nonstandard APIs */
   openFromDisk,
   openLink,
+  nvidiaSmi,
 };