From ffdb8e0a1d6a17f54f062a088ab4e188cb0e87cc Mon Sep 17 00:00:00 2001
From: Dylan Caponi <dylan.caponi+github@gmail.com>
Date: Tue, 1 Aug 2023 15:50:24 -0700
Subject: [PATCH 1/4] Add celebs

---
 index.html | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/index.html b/index.html
index 84f1e7e..398e19e 100644
--- a/index.html
+++ b/index.html
@@ -248,6 +248,11 @@ <h5>Try with your data</h5>
             v-on:shot-clicked="jump_video">
         </explicit-content-detection-viz>
 
+        <celebrity-recognition-viz v-if="current_view == 'Celebrity Recognition'"
+            id="celebrity_recognition" v-bind:json_data="json_data" v-bind:video_info="video_info"
+            v-on:segment-clicked="jump_video">
+        </celebrity-recognition-viz>
+
     </div>
 
 
@@ -264,6 +269,7 @@ <h5>Try with your data</h5>
     <script src="components/logo_recognition.js"></script>
     <script src="components/text_detection.js"></script>
     <script src="components/explicit_content_detection.js"></script>
+    <script src="components/celebrity_recognition.js"></script>
 
     <script>
 
@@ -334,7 +340,8 @@ <h5>Try with your data</h5>
                     'Logo Recognition': 'logo_recognition_annotations',
                     'Speech Transcription': 'speech_transcriptions',
                     'Text Detection': 'text_annotations',
-                    'Explicit Content Detection': 'explicit_annotation'
+                    'Explicit Content Detection': 'explicit_annotation',
+                    'Celebrity Recognition': 'celebrity_recognition_annotations'
                 }
             }, methods: {
                 jump_video: function (event_data) {

From dab11169956a9c527c56ed607ce2bc763e246e28 Mon Sep 17 00:00:00 2001
From: Dylan Caponi <dylan.caponi+github@gmail.com>
Date: Tue, 1 Aug 2023 15:55:07 -0700
Subject: [PATCH 2/4] Add celeb recognition component

---
 components/celebrity_recognition.js | 275 ++++++++++++++++++++++++++++
 1 file changed, 275 insertions(+)
 create mode 100644 components/celebrity_recognition.js

diff --git a/components/celebrity_recognition.js b/components/celebrity_recognition.js
new file mode 100644
index 0000000..298e473
--- /dev/null
+++ b/components/celebrity_recognition.js
@@ -0,0 +1,275 @@
+console.log('IMPORTING CELEBRITY RECOGNITION')
+
+// define style rules to be programtically loaded
+var style = document.createElement('style');
+style.innerHTML = `
+
+
+`;
+document.getElementsByTagName('head')[0].appendChild(style);
+
+
+// define component
+Vue.component('celebrity-recognition-viz', {
+    props: ['json_data', 'video_info'],
+    data: function () {
+        return {
+            confidence_threshold: 0.5,
+            interval_timer: null,
+            ctx: null
+        }
+    },
+    computed: {
+        celebrity_tracks: function () {
+            `
+            Extract just the celebrity tracking data from json
+            `
+
+            if (!this.json_data.annotation_results)
+                return []
+
+            for (let index = 0; index < this.json_data.annotation_results.length; index++) {
+                if ('celebrity_recognition_annotations' in this.json_data.annotation_results[index])
+                    return this.json_data.annotation_results[index].celebrity_recognition_annotations
+            }
+            return []
+        },
+
+        indexed_celebrity_tracks: function () {
+            `
+            Create a clean list of celebrity tracking data with realisied nullable fields 
+            and scaled bounding boxes ready to be drawn by the canvas
+            `
+
+            const indexed_tracks = []
+
+
+            this.celebrity_tracks.forEach(element => {
+                const detected_celebrity = new Celebrity_Detected(element, this.video_info.height, this.video_info.width, this.confidence_threshold)
+                if (detected_celebrity.segments.length > 0)
+                    indexed_tracks.push(detected_celebrity)
+            })
+
+            return indexed_tracks
+        },
+
+        object_track_segments: function () {
+            ` 
+            create the list of cronological time segments that represent just when objects are present on screen
+            `
+            const segments = {}
+
+            this.indexed_celebrity_tracks.forEach(object_tracks => {
+
+                if (!(object_tracks.name in segments))
+                    segments[object_tracks.name] = { 'segments': [], 'count': 0 }
+
+
+                object_tracks.segments.forEach(celebrity_segemnt => {
+
+
+                    segments[object_tracks.name].count++
+
+                    var added = false
+
+                    for (let index = 0; index < segments[object_tracks.name].length; index++) {
+
+                        const segment = segments[object_tracks.name].segments[index]
+                        if (celebrity_segemnt.start_time < segment[1]) {
+                            segments[object_tracks.name].segments[index][1] = Math.max(segments[object_tracks.name].segments[index][1], celebrity_segemnt.end_time)
+                            added = true
+                            break
+                        }
+                    }
+
+                    if (!added)
+                        segments[object_tracks.name].segments.push([celebrity_segemnt.start_time, celebrity_segemnt.end_time])
+
+
+
+                })
+
+            })
+
+            return segments
+        }
+    },
+    methods: {
+        segment_style: function (segment) {
+            return {
+                left: ((segment[0] / this.video_info.length) * 100).toString() + '%',
+                width: (((segment[1] - segment[0]) / this.video_info.length) * 100).toString() + '%'
+            }
+        },
+        segment_clicked: function (segment_data) {
+            this.$emit('segment-clicked', { seconds: segment_data[0] - 0.5 })
+        }
+    },
+    template: `
+    <div calss="object-tracking-container">
+
+        <div class="confidence">
+            <span>Confidence threshold</span>
+            <input type="range" min="0.0" max="1" value="0.5" step="0.01" v-model="confidence_threshold">
+            <span class="confidence-value">{{confidence_threshold}}</span>
+        </div>
+
+        <div class="data-warning" v-if="celebrity_tracks.length == 0"> No celebrity detection data in JSON</div>
+
+        <transition-group name="segments" tag="div">
+            
+            <div class="segment-container" v-for="segments, key in object_track_segments" v-bind:key="key + 'z'">
+                <div class="label">{{key}} ({{segments.count}})</div>
+                <div class="segment-timeline">
+                    <div class="segment" v-for="segment in segments.segments" 
+                                        v-bind:style="segment_style(segment)" 
+                                        v-on:click="segment_clicked(segment)"
+                    ></div>
+                </div>
+            </div>
+        </transition-group>
+    </div>
+    `,
+    mounted: function () {
+        console.log('mounted component')
+        var canvas = document.getElementById("my_canvas")
+        this.ctx = canvas.getContext("2d")
+        this.ctx.font = "20px Roboto"
+        const ctx = this.ctx
+
+        const component = this
+
+        this.interval_timer = setInterval(function () {
+            console.log('running')
+            const object_tracks = component.indexed_celebrity_tracks
+
+            draw_bounding_boxes(object_tracks, ctx)
+        }, 1000 / 30)
+    },
+    beforeDestroy: function () {
+        console.log('destroying component')
+        clearInterval(this.interval_timer)
+        this.ctx.clearRect(0, 0, 800, 500)
+    }
+})
+
+
+
+class Celebrity_Frame {
+
+    constructor(json_data, video_height, video_width) {
+
+        this.time_offset = nullable_time_offset_to_seconds(json_data.time_offset)
+
+        this.box = {
+            'x': (json_data.normalized_bounding_box.left || 0) * video_width,
+            'y': (json_data.normalized_bounding_box.top || 0) * video_height,
+            'width': ((json_data.normalized_bounding_box.right || 0) - (json_data.normalized_bounding_box.left || 0)) * video_width,
+            'height': ((json_data.normalized_bounding_box.bottom || 0) - (json_data.normalized_bounding_box.top || 0)) * video_height
+        }
+    }
+}
+
+
+
+class Celebrity_Track {
+    constructor(json_data, video_height, video_width) {
+
+        this.start_time = nullable_time_offset_to_seconds(json_data.segment.start_time_offset)
+        this.end_time = nullable_time_offset_to_seconds(json_data.segment.end_time_offset)
+        this.confidence = json_data.confidence
+
+        this.frames = []
+
+        json_data.timestamped_objects.forEach(frame => {
+            this.frames.push(new Celebrity_Frame(frame, video_height, video_width))
+        })
+    }
+
+    has_frames_for_time(seconds) {
+        return ((this.start_time <= seconds) && (this.end_time >= seconds))
+    }
+
+    most_recent_real_bounding_box(seconds) {
+
+        for (let index = 0; index < this.frames.length; index++) {
+            if (this.frames[index].time_offset > seconds) {
+                if (index > 0)
+                    return this.frames[index - 1].box
+                else
+                    return null
+            }
+        }
+        return null
+    }
+
+    most_recent_interpolated_bounding_box(seconds) {
+
+        for (let index = 0; index < this.frames.length; index++) {
+            if (this.frames[index].time_offset > seconds) {
+                if (index > 0) {
+                    if ((index == 1) || (index == this.frames.length - 1))
+                        return this.frames[index - 1].box
+
+                    // create a new interpolated box between the 
+                    const start_box = this.frames[index - 1]
+                    const end_box = this.frames[index]
+                    const time_delt_ratio = (seconds - start_box.time_offset) / (end_box.time_offset - start_box.time_offset)
+
+                    const interpolated_box = {
+                        'x': start_box.box.x + (end_box.box.x - start_box.box.x) * time_delt_ratio,
+                        'y': start_box.box.y + (end_box.box.y - start_box.box.y) * time_delt_ratio,
+                        'width': start_box.box.width + (end_box.box.width - start_box.box.width) * time_delt_ratio,
+                        'height': start_box.box.height + (end_box.box.height - start_box.box.height) * time_delt_ratio
+                    }
+                    return interpolated_box
+
+                } else
+                    return null
+            }
+        }
+        return null
+    }
+
+    current_bounding_box(seconds, interpolate = true) {
+
+        if (interpolate)
+            return this.most_recent_interpolated_bounding_box(seconds)
+        else
+            return this.most_recent_real_bounding_box(seconds)
+    }
+}
+
+
+class Celebrity_Detected {
+    constructor(json_data, video_height, video_width, confidence_threshold) {
+
+        this.name = json_data.entity.description
+        this.id = json_data.entity.entity_id
+
+        this.segments = []
+
+        json_data.tracks.forEach(track => {
+            if (track.confidence > confidence_threshold)
+                this.segments.push(new Celebrity_Track(track, video_height, video_width))
+        })
+    }
+
+    has_frames_for_time(seconds) {
+
+        for (let index = 0; index < this.segments.length; index++) {
+            if (this.segments[index].has_frames_for_time(seconds))
+                return true
+        }
+        return false
+    }
+
+    current_bounding_box(seconds, interpolate = true) {
+
+        for (let index = 0; index < this.segments.length; index++) {
+            if (this.segments[index].has_frames_for_time(seconds))
+                return this.segments[index].current_bounding_box(seconds, interpolate)
+        }
+        return null
+    }
+}
\ No newline at end of file

From bafb9a426c3776d0a82c04eba7cd216d6215076d Mon Sep 17 00:00:00 2001
From: Dylan Caponi <dylan.caponi+github@gmail.com>
Date: Tue, 1 Aug 2023 15:57:53 -0700
Subject: [PATCH 3/4] Add celeb feature

---
 run_video_intelligence.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/run_video_intelligence.py b/run_video_intelligence.py
index 4f88c2e..2775624 100644
--- a/run_video_intelligence.py
+++ b/run_video_intelligence.py
@@ -18,7 +18,8 @@
     videointelligence.Feature.EXPLICIT_CONTENT_DETECTION,
     videointelligence.Feature.TEXT_DETECTION,
     videointelligence.Feature.FACE_DETECTION,
-    videointelligence.Feature.PERSON_DETECTION
+    videointelligence.Feature.PERSON_DETECTION,
+    videointelligence.Feature.CELEBRITY_RECOGNITION
 ]
 
 transcript_config = videointelligence.SpeechTranscriptionConfig(

From 671c51eff23ea33b19ecf92e9f0f0c5acef32b35 Mon Sep 17 00:00:00 2001
From: Dylan Caponi <dylan.caponi+github@gmail.com>
Date: Wed, 2 Aug 2023 15:53:40 -0700
Subject: [PATCH 4/4] Celeb recognition w bounding boxes working

---
 components/celebrity_recognition.js | 44 ++++++++++++++++++-----------
 1 file changed, 27 insertions(+), 17 deletions(-)

diff --git a/components/celebrity_recognition.js b/components/celebrity_recognition.js
index 298e473..c2ab7fd 100644
--- a/components/celebrity_recognition.js
+++ b/components/celebrity_recognition.js
@@ -9,6 +9,7 @@ style.innerHTML = `
 document.getElementsByTagName('head')[0].appendChild(style);
 
 
+
 // define component
 Vue.component('celebrity-recognition-viz', {
     props: ['json_data', 'video_info'],
@@ -20,19 +21,28 @@ Vue.component('celebrity-recognition-viz', {
         }
     },
     computed: {
+
         celebrity_tracks: function () {
             `
-            Extract just the celebrity tracking data from json
+            Extract just the logo tracking data from json
             `
 
-            if (!this.json_data.annotation_results)
-                return []
+            if (!this.json_data.annotation_results) return []
 
+            const tracksWithCelebrities = []
             for (let index = 0; index < this.json_data.annotation_results.length; index++) {
-                if ('celebrity_recognition_annotations' in this.json_data.annotation_results[index])
-                    return this.json_data.annotation_results[index].celebrity_recognition_annotations
+                if ('celebrity_recognition_annotations' in this.json_data.annotation_results[index] && 'celebrity_tracks' in this.json_data.annotation_results[index].celebrity_recognition_annotations) {
+                    const celebrityTracks = this.json_data.annotation_results[index].celebrity_recognition_annotations.celebrity_tracks
+                    celebrityTracks.forEach(track => {
+                        if ('celebrities' in track) {
+                            track.celebrities.forEach(() => {
+                                tracksWithCelebrities.push(track)
+                            })
+                        }
+                    })
+                }
             }
-            return []
+            return tracksWithCelebrities
         },
 
         indexed_celebrity_tracks: function () {
@@ -43,7 +53,6 @@ Vue.component('celebrity-recognition-viz', {
 
             const indexed_tracks = []
 
-
             this.celebrity_tracks.forEach(element => {
                 const detected_celebrity = new Celebrity_Detected(element, this.video_info.height, this.video_info.width, this.confidence_threshold)
                 if (detected_celebrity.segments.length > 0)
@@ -117,7 +126,7 @@ Vue.component('celebrity-recognition-viz', {
         <div class="data-warning" v-if="celebrity_tracks.length == 0"> No celebrity detection data in JSON</div>
 
         <transition-group name="segments" tag="div">
-            
+
             <div class="segment-container" v-for="segments, key in object_track_segments" v-bind:key="key + 'z'">
                 <div class="label">{{key}} ({{segments.count}})</div>
                 <div class="segment-timeline">
@@ -175,13 +184,13 @@ class Celebrity_Frame {
 class Celebrity_Track {
     constructor(json_data, video_height, video_width) {
 
-        this.start_time = nullable_time_offset_to_seconds(json_data.segment.start_time_offset)
-        this.end_time = nullable_time_offset_to_seconds(json_data.segment.end_time_offset)
+        this.start_time = nullable_time_offset_to_seconds(json_data.face_track.segment.start_time_offset)
+        this.end_time = nullable_time_offset_to_seconds(json_data.face_track.segment.end_time_offset)
         this.confidence = json_data.confidence
 
         this.frames = []
 
-        json_data.timestamped_objects.forEach(frame => {
+        json_data.face_track.timestamped_objects.forEach(frame => {
             this.frames.push(new Celebrity_Frame(frame, video_height, video_width))
         })
     }
@@ -242,16 +251,17 @@ class Celebrity_Track {
 
 
 class Celebrity_Detected {
-    constructor(json_data, video_height, video_width, confidence_threshold) {
-
-        this.name = json_data.entity.description
-        this.id = json_data.entity.entity_id
+    constructor(track, video_height, video_width, confidence_threshold) {
 
         this.segments = []
 
-        json_data.tracks.forEach(track => {
-            if (track.confidence > confidence_threshold)
+        track.celebrities.forEach(celebrity => {
+            // Some outputs do not contain a confidence
+            if (!celebrity.confidence || celebrity.confidence > confidence_threshold) {
+                this.name = celebrity.celebrity.display_name
+                this.id = celebrity.celebrity.name
                 this.segments.push(new Celebrity_Track(track, video_height, video_width))
+            }
         })
     }