Electron

.github/workflows/blank.yml

@@ -22,6 +22,7 @@ jobs:
     - run: yarn test
     - run: yarn build:web
     - run: yarn build:lib
+    - run: yarn build:electron
 
   test-server:
     defaults:

client/.eslintignore

@@ -0,0 +1,2 @@
+lib/**/*
+dist/**/*

client/.gitignore

@@ -1,6 +1,7 @@
 .DS_Store
 node_modules
 /dist
+/dist_electron
 /lib
 
 # local env files

client/README.md

@@ -88,3 +88,32 @@ This style guarantees matching types are passed through provide and inject witho
 ## Tests
 
 Note that `tsconfig.spec.json` is an exact copy of `tsconfig.json` but the `target` and `module` are changed such that babel is not required for jest to execute tests.
+
+## Typescript vue-media-annotator library
+
+Parts of the annotator in `src/` can be included from an external annotator library.  Requires `@vue/composition-api`.
+
+``` bash
+npm install vue-media-annotator
+```
+
+Now include the parts you want.
+
+``` js
+import {
+  providers,
+  use,
+  Track,
+  components,
+} from 'vue-media-annotator/lib';
+
+const {
+  VideoAnnotator, LayerManager, Controls, TimelineWrapper, Timeline, LineChart,
+} = components;
+```
+
+> **Note** that you must abandon `vuetify-loader` in order to use this lib.  It relies on vuetify's components to be registered with the global context, which doesn't happen with an a-la-carte installation.
+
+> **Note** you can clone this repo, use `yarn link`, and `yarn link vue-media-annotator` in your own project to modify the source library as you go.  You'll have to `yarn build:lib` after changes, and you must `mv node_modeles/ node_modules.old/` in order to prevent your consumer app from using this project's `node_modules` libs instead of yours.  This could cause problems like multiple instances of vue or composition api.
+
+The above problems are known and we are working to solve them.

client/package.json

@@ -3,7 +3,9 @@
   "version": "1.0.3",
   "scripts": {
     "serve": "vue-cli-service serve platform/web-girder/main.ts",
+    "serve:electron": "vue-cli-service electron:serve",
     "build:web": "vue-cli-service build platform/web-girder/main.ts",
+    "build:electron": "vue-cli-service electron:build",
     "build:lib": "rollup -c",
     "lint": "vue-cli-service lint src/ viame-web-common/ platform/",
     "lint:templates": "vtc --workspace . --srcDir src/",
@@ -24,6 +26,7 @@
     "@sentry/browser": "^5.24.2",
     "@sentry/integrations": "^5.24.2",
     "@types/mousetrap": "^1.6.3",
+    "@types/source-map": "0.5.2",
     "@vue/composition-api": "^1.0.0-beta.14",
     "axios": "^0.19.2",
     "core-js": "^3.6.4",
@@ -40,10 +43,15 @@
   "devDependencies": {
     "@types/axios": "^0.14.0",
     "@types/d3": "^5.7.2",
+    "@types/electron-devtools-installer": "^2.2.0",
     "@types/geojson": "^7946.0.7",
     "@types/jest": "^25.2.3",
     "@types/lodash": "^4.14.151",
+    "@types/mime-types": "^2.1.0",
     "@types/node": "^14.0.5",
+    "@types/pump": "^1.1.0",
+    "@types/range-parser": "^1.2.3",
+    "@types/request": "^2.48.5",
     "@typescript-eslint/eslint-plugin": "^2.33.0",
     "@typescript-eslint/parser": "^2.33.0",
     "@vue/cli-plugin-babel": "~4.3.1",
@@ -57,13 +65,19 @@
     "babel-eslint": "^10.1.0",
     "babel-jest": "^26.0.1",
     "babel-register": "^6.26.0",
+    "electron": "^10.1.3",
+    "electron-devtools-installer": "^3.1.1",
     "eslint": "^6.7.2",
     "eslint-import-resolver-typescript": "^2.2.0",
     "eslint-plugin-import": "^2.20.2",
     "eslint-plugin-vue": "^6.2.2",
     "git-describe": "^4.0.4",
     "jest": "^26.0.1",
     "jest-transform-stub": "^2.0.0",
+    "mime-types": "^2.1.27",
+    "pump": "^3.0.0",
+    "range-parser": "^1.2.1",
+    "request": "^2.88.2",
     "rollup": "^2.29.0",
     "rollup-plugin-cleaner": "^1.0.0",
     "rollup-plugin-scss": "^2.6.1",
@@ -73,6 +87,7 @@
     "sass-loader": "^8.0.2",
     "ts-jest": "^26.0.0",
     "typescript": "~3.8.3",
+    "vue-cli-plugin-electron-builder": "^2.0.0-beta.5",
     "vue-cli-plugin-vuetify": "^2.0.5",
     "vue-jest": "^3.0.5",
     "vue-template-compiler": "^2.6.12",

client/platform/desktop/App.vue

@@ -0,0 +1,29 @@
+<template>
+  <v-app>
+    <router-view />
+  </v-app>
+</template>
+
+<script lang="ts">
+import { defineComponent } from '@vue/composition-api';
+import { provideApi } from 'viame-web-common/apispec';
+import { observe } from './store/dataset';
+
+export default defineComponent({
+  name: 'App',
+  components: {},
+  setup() {
+    provideApi(observe());
+  },
+});
+</script>
+
+<style lang="scss">
+html {
+  overflow-y: auto;
+}
+
+.text-xs-center {
+  text-align: center !important;
+}
+</style>

client/platform/desktop/README.md

@@ -0,0 +1,23 @@
+# VIAME Electron Desktop client
+
+> Codename Heavy
+
+
+## Why make a desktop version?
+
+* The desktop client is intended for annotation of datasets on your local filesystem.
+* It may not be ideal for you to copy large datasets onto a server for annotation.
+* Your personal workstation might be ideal for training and pipeline execution.
+* You may prefer not to rely on Docker.
+
+## General architecture
+
+Electron applications are comprised of two main threads
+
+* a node.js main thread with full access to the node environment
+* a renderer thread which is like a browser tab that runs under a stricter security policy
+
+Due to security concerns in the renderer thread, this app uses a small embedded node.js webserver to serve media content (images and videos) from disk.  This is actually the most reasonable way to stream bytes with range requests into a browser environment.
+
+* The common frontend api is implemented in `api/`
+* The backend services are implemented in `backend/`

client/platform/desktop/api/main.ts

@@ -0,0 +1,137 @@
+import { TrackData } from 'vue-media-annotator/track';
+import fs from 'fs';
+// eslint-disable-next-line import/no-extraneous-dependencies
+import mime from 'mime-types';
+import path from 'path';
+import {
+  Attribute, DatasetMeta, DatasetMetaMutable, FrameImage,
+  Pipelines, SaveDetectionsArgs, TrainingConfigs,
+} from 'viame-web-common/apispec';
+// eslint-disable-next-line
+import { ipcRenderer, remote } from 'electron';
+import { AddressInfo } from 'net';
+
+const websafeVideoTypes = [
+  'video/mp4',
+  'video/webm',
+];
+
+const websafeImageTypes = [
+  'image/apng',
+  'image/bmp',
+  'image/gif',
+  'image/jpeg',
+  'image/png',
+  'image/svg+xml',
+  'image/webp',
+];
+
+export interface DesktopDataset {
+  root: string;
+  videoPath?: string;
+  meta: DatasetMeta;
+}
+
+const mediaServerInfo: AddressInfo = ipcRenderer.sendSync('info');
+
+async function openFromDisk() {
+  const results = await remote.dialog.showOpenDialog({
+    properties: ['openFile', 'openDirectory'],
+  });
+  return results;
+}
+
+async function getAttributes() {
+  return Promise.resolve([] as Attribute[]);
+}
+async function getPipelineList() {
+  return Promise.resolve({} as Pipelines);
+}
+// eslint-disable-next-line
+async function runPipeline(itemId: string, pipeline: string) {
+  return Promise.resolve();
+}
+// eslint-disable-next-line
+async function loadDetections(datasetId: string) {
+  return Promise.resolve({} as { [key: string]: TrackData });
+}
+// eslint-disable-next-line
+async function saveDetections(datasetId: string, args: SaveDetectionsArgs) {
+  return Promise.resolve();
+}
+// eslint-disable-next-line
+async function getTrainingConfigurations(): Promise<TrainingConfigs> {
+  return Promise.resolve({ configs: [], default: '' });
+}
+// eslint-disable-next-line
+async function runTraining(folderId: string, pipelineName: string, config: string): Promise<unknown> {
+  return Promise.resolve();
+}
+
+
+async function loadMetadata(datasetId: string): Promise<DesktopDataset> {
+  let datasetType = undefined as 'video' | 'image-sequence' | undefined;
+  let videoUrl = '';
+  let videoPath = '';
+  const imageData = [] as FrameImage[];
+
+  function processFile(abspath: string) {
+    const basename = path.basename(abspath);
+    const abspathuri = `http://localhost:${mediaServerInfo.port}/api/media?path=${abspath}`;
+    const mimetype = mime.lookup(abspath);
+    if (mimetype && websafeVideoTypes.includes(mimetype)) {
+      datasetType = 'video';
+      videoPath = abspath;
+      videoUrl = abspathuri;
+    } else if (mimetype && websafeImageTypes.includes(mimetype)) {
+      datasetType = 'image-sequence';
+      imageData.push({
+        url: abspathuri,
+        filename: basename,
+      });
+    }
+  }
+
+  const info = fs.statSync(datasetId);
+
+  if (info.isDirectory()) {
+    const contents = fs.readdirSync(datasetId);
+    for (let i = 0; i < contents.length; i += 1) {
+      processFile(path.join(datasetId, contents[i]));
+    }
+  } else {
+    processFile(datasetId);
+  }
+
+  if (datasetType === undefined) {
+    throw new Error(`Cannot open dataset ${datasetId}: No images or video found`);
+  }
+
+  return Promise.resolve({
+    root: datasetId,
+    videoPath,
+    meta: {
+      type: datasetType,
+      fps: 10,
+      imageData: datasetType === 'image-sequence' ? imageData : [],
+      videoUrl: datasetType === 'video' ? videoUrl : undefined,
+    },
+  });
+}
+// eslint-disable-next-line
+async function saveMetadata(datasetId: string, metadata: DatasetMetaMutable) {
+  return Promise.resolve();
+}
+
+export {
+  getAttributes,
+  getPipelineList,
+  runPipeline,
+  getTrainingConfigurations,
+  runTraining,
+  loadDetections,
+  openFromDisk,
+  saveDetections,
+  loadMetadata,
+  saveMetadata,
+};

client/platform/desktop/backend/router.ts

@@ -0,0 +1,47 @@
+import { IncomingMessage, RequestListener, ServerResponse } from 'http';
+import parser from 'url';
+
+class Handler {
+    method: RequestListener;
+
+    constructor(method: RequestListener) {
+      this.method = method;
+    }
+
+    process(req: IncomingMessage, res: ServerResponse) {
+      this.method.apply(this, [req, res]);
+    }
+}
+
+const handlers: Record<string, Handler> = {};
+
+function register(url: string, method: RequestListener) {
+  handlers[url] = new Handler(method);
+}
+
+function missing() {
+  return new Handler((req, res) => {
+    res.writeHead(404, { 'Content-Type': 'text/plain' });
+    res.write(`No route registered for ${req.url}`);
+    res.end();
+  });
+}
+
+function route(req: IncomingMessage): Handler {
+  let handler;
+  if (req.url) {
+    const url = parser.parse(req.url, true);
+    if (url.pathname) {
+      handler = handlers[url.pathname];
+    }
+  }
+  if (handler === undefined) {
+    return missing();
+  }
+  return handler;
+}
+
+export default {
+  register,
+  route,
+};