From 718d568c6650f7658a96d7271862bd11db04571c Mon Sep 17 00:00:00 2001
From: Mattt Zmuda <mattt@replicate.com>
Date: Thu, 9 Nov 2023 04:13:51 -0800
Subject: [PATCH 1/5] Use llama-tokenizer-js to count tokens

---
 app/page.js          | 11 ++++-------
 app/src/tokenizer.js |  5 +++++
 package-lock.json    | 11 +++++++++++
 package.json         |  1 +
 4 files changed, 21 insertions(+), 7 deletions(-)
 create mode 100644 app/src/tokenizer.js

diff --git a/app/page.js b/app/page.js
index 1d6db1c..6429449 100644
--- a/app/page.js
+++ b/app/page.js
@@ -8,10 +8,7 @@ import EmptyState from "./components/EmptyState";
 import { Cog6ToothIcon, CodeBracketIcon } from "@heroicons/react/20/solid";
 import { useCompletion } from "ai/react";
 import { Toaster, toast } from "react-hot-toast";
-
-function approximateTokenCount(text) {
-  return Math.ceil(text.length * 0.4);
-}
+import { countTokens } from "./src/tokenizer.js";
 
 const VERSIONS = [
   {
@@ -177,7 +174,7 @@ export default function HomePage() {
     // Generate initial prompt and calculate tokens
     let prompt = `${generatePrompt(messageHistory)}\n`;
     // Check if we exceed max tokens and truncate the message history if so.
-    while (approximateTokenCount(prompt) > MAX_TOKENS) {
+    while (countTokens(prompt) > MAX_TOKENS) {
       if (messageHistory.length < 3) {
         setError(
           "Your message is too long. Please try again with a shorter message."
@@ -224,8 +221,8 @@ export default function HomePage() {
           {size.shortened == "Llava"
             ? "🌋"
             : size.shortened == "Salmonn"
-            ? "🐟"
-            : "🦙"}{" "}
+              ? "🐟"
+              : "🦙"}{" "}
           <span className="hidden sm:inline-block">Chat with</span>{" "}
           <button
             className="py-2 font-semibold text-gray-500 hover:underline"
diff --git a/app/src/tokenizer.js b/app/src/tokenizer.js
new file mode 100644
index 0000000..3d46d1f
--- /dev/null
+++ b/app/src/tokenizer.js
@@ -0,0 +1,5 @@
+import llamaTokenizer from "llama-tokenizer-js";
+
+export const countTokens = (text) => {
+    return llamaTokenizer.encode(text).length;
+};
diff --git a/package-lock.json b/package-lock.json
index 6eed1cb..3e0651d 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -15,6 +15,7 @@
         "ai": "^2.2.12",
         "eslint": "8.49.0",
         "eslint-config-next": "13.4.19",
+        "llama-tokenizer-js": "^1.1.3",
         "next": "13.4.19",
         "react": "18.2.0",
         "react-dom": "18.2.0",
@@ -3131,6 +3132,11 @@
       "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==",
       "dev": true
     },
+    "node_modules/llama-tokenizer-js": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/llama-tokenizer-js/-/llama-tokenizer-js-1.1.3.tgz",
+      "integrity": "sha512-+BUgsLCXVQJkjiD/t7PdESLn+yXJIRX/BJfwzVVYfKZ9aN3gsP9xoadBZxKnCxGz2Slby+S7x41gUr2TKNaS4Q=="
+    },
     "node_modules/locate-character": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/locate-character/-/locate-character-3.0.0.tgz",
@@ -7326,6 +7332,11 @@
       "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==",
       "dev": true
     },
+    "llama-tokenizer-js": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/llama-tokenizer-js/-/llama-tokenizer-js-1.1.3.tgz",
+      "integrity": "sha512-+BUgsLCXVQJkjiD/t7PdESLn+yXJIRX/BJfwzVVYfKZ9aN3gsP9xoadBZxKnCxGz2Slby+S7x41gUr2TKNaS4Q=="
+    },
     "locate-character": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/locate-character/-/locate-character-3.0.0.tgz",
diff --git a/package.json b/package.json
index 2299a92..38b2009 100644
--- a/package.json
+++ b/package.json
@@ -17,6 +17,7 @@
     "ai": "^2.2.12",
     "eslint": "8.49.0",
     "eslint-config-next": "13.4.19",
+    "llama-tokenizer-js": "^1.1.3",
     "next": "13.4.19",
     "react": "18.2.0",
     "react-dom": "18.2.0",

From ffd58311e329df8e3cb84581cd7bdf8248ae8282 Mon Sep 17 00:00:00 2001
From: Mattt Zmuda <mattt@replicate.com>
Date: Thu, 9 Nov 2023 05:22:46 -0800
Subject: [PATCH 2/5] Add performance metrics to chat form

---
 app/components/ChatForm.js | 10 ++++++++--
 app/components/Metrics.js  | 34 ++++++++++++++++++++++++++++++++++
 app/page.js                | 36 +++++++++++++++++++++++++++++++++++-
 3 files changed, 77 insertions(+), 3 deletions(-)
 create mode 100644 app/components/Metrics.js

diff --git a/app/components/ChatForm.js b/app/components/ChatForm.js
index 3b13ea8..1658dbe 100644
--- a/app/components/ChatForm.js
+++ b/app/components/ChatForm.js
@@ -1,6 +1,6 @@
 import { Uploader } from "uploader";
 import { UploadButton } from "react-uploader";
-
+import Metrics from "./Metrics";
 const uploader = Uploader({
   apiKey: "public_kW15biSARCJN7FAz6rANdRg3pNkh",
 });
@@ -38,7 +38,7 @@ const options = {
   },
 };
 
-const ChatForm = ({ prompt, setPrompt, onSubmit, handleFileUpload }) => {
+const ChatForm = ({ prompt, setPrompt, onSubmit, handleFileUpload, metrics, completion }) => {
   const handleSubmit = async (event) => {
     event.preventDefault();
     onSubmit(prompt);
@@ -56,6 +56,12 @@ const ChatForm = ({ prompt, setPrompt, onSubmit, handleFileUpload }) => {
   return (
     <footer className="z-10 fixed bottom-0 left-0 right-0 bg-slate-100 border-t-2">
       <div className="container max-w-2xl mx-auto p-5 pb-8">
+        <Metrics
+          startedAt={metrics.startedAt}
+          firstMessageAt={metrics.firstMessageAt}
+          completedAt={metrics.completedAt}
+          completion={completion} />
+
         <form className="w-full flex" onSubmit={handleSubmit}>
           <UploadButton
             uploader={uploader}
diff --git a/app/components/Metrics.js b/app/components/Metrics.js
new file mode 100644
index 0000000..15f0e7b
--- /dev/null
+++ b/app/components/Metrics.js
@@ -0,0 +1,34 @@
+import { countTokens } from "../src/tokenizer";
+
+export default function Metrics({ startedAt, firstMessageAt, completedAt, completion }) {
+    const timeToFirstToken = firstMessageAt && startedAt ? (new Date(firstMessageAt) - new Date(startedAt)) / 1000.0 : null;
+    const tokenCount = completion && countTokens(completion);
+    const runningDuration = firstMessageAt ? ((completedAt ? new Date(completedAt) : new Date()) - new Date(firstMessageAt)) / 1000.0 : null;
+    const tokensPerSecond = tokenCount > 0 && runningDuration > 0 && tokenCount / runningDuration;
+
+    return (
+        <dl className="tabular-nums pb-6" style={{
+            display: 'grid',
+            gridTemplateColumns: 'repeat(8, auto)',
+            gridTemplateAreas:
+                '"v1 k1 v2 k2 v3 k3 v4 k4"'
+        }}>
+            {<>
+                <dt title="Time to first token" className="text-gray-500" style={{ gridArea: 'k1' }}>sec to first token</dt>
+                <dd className="text-right pr-4" style={{ gridArea: 'v1' }}>{timeToFirstToken ? timeToFirstToken.toFixed(2) : "—"}</dd>
+            </>}
+            {<>
+                <dt title="Throughput" className="text-gray-500" style={{ gridArea: 'k2' }}>tokens / sec</dt>
+                <dd className="text-right pr-4" style={{ gridArea: 'v2' }}>{tokensPerSecond ? tokensPerSecond.toFixed(2) : "—"}</dd>
+            </>}
+            {<>
+                <dt title="Token count" className="text-gray-500" style={{ gridArea: 'k3' }}>tokens</dt>
+                <dd className="text-right pr-4" style={{ gridArea: 'v3' }}>{tokenCount || "—"}</dd>
+            </>}
+            {<>
+                <dt title="Run time" className="text-gray-500" style={{ gridArea: 'k4' }}>sec</dt>
+                <dd className="text-right pr-4" style={{ gridArea: 'v4' }}>{Math.max(runningDuration, 0).toFixed(2)}</dd>
+            </>}
+        </dl>
+    );
+};
diff --git a/app/page.js b/app/page.js
index 6429449..00d108e 100644
--- a/app/page.js
+++ b/app/page.js
@@ -1,6 +1,6 @@
 "use client";
 
-import { useEffect, useRef, useState } from "react";
+import { useEffect, useReducer, useRef, useState } from "react";
 import ChatForm from "./components/ChatForm";
 import Message from "./components/Message";
 import SlideOver from "./components/SlideOver";
@@ -72,6 +72,19 @@ function CTA({ shortenedModelName }) {
   }
 }
 
+const metricsReducer = (state, action) => {
+  switch (action.type) {
+    case 'START':
+      return { startedAt: new Date() };
+    case 'FIRST_MESSAGE':
+      return { ...state, firstMessageAt: new Date() };
+    case 'COMPLETE':
+      return { ...state, completedAt: new Date() };
+    default:
+      throw new Error(`Unsupported action type: ${action.type}`);
+  }
+};
+
 export default function HomePage() {
   const MAX_TOKENS = 4096;
   const bottomRef = useRef(null);
@@ -94,6 +107,14 @@ export default function HomePage() {
   // Salmonn params
   const [audio, setAudio] = useState(null);
 
+
+  const [metrics, dispatch] = useReducer(metricsReducer, {
+    startedAt: null,
+    firstMessageAt: null,
+    completedAt: null,
+  });
+
+
   const { complete, completion, setInput, input } = useCompletion({
     api: "/api",
     body: {
@@ -105,9 +126,17 @@ export default function HomePage() {
       image: image,
       audio: audio,
     },
+
     onError: (error) => {
       setError(error);
     },
+    onResponse: (response) => {
+      setError(null);
+      dispatch({ type: 'FIRST_MESSAGE' });
+    },
+    onFinish: () => {
+      dispatch({ type: 'COMPLETE' });
+    }
   });
 
   const handleFileUpload = (file) => {
@@ -192,6 +221,8 @@ export default function HomePage() {
 
     setMessages(messageHistory);
 
+    dispatch({ type: 'START' });
+
     complete(prompt);
   };
 
@@ -300,6 +331,8 @@ export default function HomePage() {
           setPrompt={setInput}
           onSubmit={handleSubmit}
           handleFileUpload={handleFileUpload}
+          completion={completion}
+          metrics={metrics}
         />
 
         {error && <div>{error}</div>}
@@ -313,6 +346,7 @@ export default function HomePage() {
             />
           ))}
           <Message message={completion} isUser={false} />
+
           <div ref={bottomRef} />
         </article>
       </main>

From 0a983e3b3787bf9a18943a58409d266de1c6f2b5 Mon Sep 17 00:00:00 2001
From: Mattt Zmuda <mattt@replicate.com>
Date: Wed, 6 Dec 2023 11:07:16 -0800
Subject: [PATCH 3/5] Update replicate to 0.23.0

---
 package-lock.json | 14 +++++++-------
 package.json      |  2 +-
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index 3e0651d..4664d18 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -21,7 +21,7 @@
         "react-dom": "18.2.0",
         "react-hot-toast": "^2.4.1",
         "react-uploader": "^3.43.0",
-        "replicate": "^0.16.0"
+        "replicate": "^0.23.0"
       },
       "devDependencies": {
         "@tailwindcss/forms": "^0.5.4",
@@ -4092,9 +4092,9 @@
       }
     },
     "node_modules/replicate": {
-      "version": "0.16.0",
-      "resolved": "https://registry.npmjs.org/replicate/-/replicate-0.16.0.tgz",
-      "integrity": "sha512-VFrR2RXmul/sUGJAcW7SCyljGtik8oLjSD7m6Sus6EyEpa/uqmLlFGvG72jG/7MBRkg0l8+5SEWDsth+k6kssw==",
+      "version": "0.23.0",
+      "resolved": "https://registry.npmjs.org/replicate/-/replicate-0.23.0.tgz",
+      "integrity": "sha512-mR9cb4aDXWDAn7/GUMlOL3Jc7rnUys/AG6+R9Cum0xyU3e72a/o8NJ71pgQ6mCXZnEL5iWp/i6AD8Cgh8TA/bw==",
       "engines": {
         "git": ">=2.11.0",
         "node": ">=18.0.0",
@@ -7958,9 +7958,9 @@
       }
     },
     "replicate": {
-      "version": "0.16.0",
-      "resolved": "https://registry.npmjs.org/replicate/-/replicate-0.16.0.tgz",
-      "integrity": "sha512-VFrR2RXmul/sUGJAcW7SCyljGtik8oLjSD7m6Sus6EyEpa/uqmLlFGvG72jG/7MBRkg0l8+5SEWDsth+k6kssw=="
+      "version": "0.23.0",
+      "resolved": "https://registry.npmjs.org/replicate/-/replicate-0.23.0.tgz",
+      "integrity": "sha512-mR9cb4aDXWDAn7/GUMlOL3Jc7rnUys/AG6+R9Cum0xyU3e72a/o8NJ71pgQ6mCXZnEL5iWp/i6AD8Cgh8TA/bw=="
     },
     "resolve": {
       "version": "1.22.2",
diff --git a/package.json b/package.json
index 38b2009..146c671 100644
--- a/package.json
+++ b/package.json
@@ -23,7 +23,7 @@
     "react-dom": "18.2.0",
     "react-hot-toast": "^2.4.1",
     "react-uploader": "^3.43.0",
-    "replicate": "^0.16.0"
+    "replicate": "^0.23.0"
   },
   "devDependencies": {
     "@tailwindcss/forms": "^0.5.4",

From 0fba86a5e58664c720617a47a68408569ef75b2f Mon Sep 17 00:00:00 2001
From: Mattt Zmuda <mattt@replicate.com>
Date: Wed, 6 Dec 2023 10:45:39 -0800
Subject: [PATCH 4/5] Refactor selection of models and versions

---
 app/api/route.js            | 23 ++++++++----
 app/components/SlideOver.js | 70 ++++++++++++++++++-------------------
 app/page.js                 | 38 ++++++++++----------
 3 files changed, 69 insertions(+), 62 deletions(-)

diff --git a/app/api/route.js b/app/api/route.js
index 8f225dc..e06e2e4 100644
--- a/app/api/route.js
+++ b/app/api/route.js
@@ -1,6 +1,8 @@
 import Replicate from "replicate";
 import { ReplicateStream, StreamingTextResponse } from "ai";
 
+export const runtime = "edge";
+
 const replicate = new Replicate({
   auth: process.env.REPLICATE_API_TOKEN,
 });
@@ -11,7 +13,13 @@ if (!process.env.REPLICATE_API_TOKEN) {
   );
 }
 
-export const runtime = "edge";
+const VERSIONS = {
+  "meta/llama-2-7b-chat": "13c3cdee13ee059ab779f0291d29054dab00a47dad8261375654de5540165fb0",
+  "meta/llama-2-13b-chat": "f4e2de70d66816a838a89eeeb621910adffb0dd0baba3976c96980970978018d",
+  "meta/llama-2-70b-chat": "02e509c789964a7ea8736978a43525956ef40397be9033abf9fd2badfe68c9e3",
+  "yorickvp/llava-13b": "e272157381e2a3bf12df3a8edd1f38d1dbd736bbb7437277c8b34175f8fce358",
+  "nateraw/salmonn": "ad1d3f9d2bd683628242b68d890bef7f7bd97f738a7c2ccbf1743a594c723d83",
+};
 
 export async function POST(req) {
   const params = await req.json();
@@ -19,8 +27,8 @@ export async function POST(req) {
   const response = params.image
     ? await runLlava(params)
     : params.audio
-    ? await runSalmonn(params)
-    : await runLlama(params);
+      ? await runSalmonn(params)
+      : await runLlama(params);
 
   // Convert the response into a friendly text-stream
   const stream = await ReplicateStream(response);
@@ -29,14 +37,15 @@ export async function POST(req) {
 }
 
 async function runLlama({
+  model,
   prompt,
   systemPrompt,
   maxTokens,
   temperature,
   topP,
-  version,
 }) {
   console.log("running llama");
+
   return await replicate.predictions.create({
     // IMPORTANT! You must enable streaming.
     stream: true,
@@ -49,7 +58,7 @@ async function runLlama({
       top_p: topP,
     },
     // IMPORTANT! The model must support streaming. See https://replicate.com/docs/streaming
-    version: version,
+    version: VERSIONS[model],
   });
 }
 
@@ -67,7 +76,7 @@ async function runLlava({ prompt, maxTokens, temperature, topP, image }) {
       image: image,
     },
     // IMPORTANT! The model must support streaming. See https://replicate.com/docs/streaming
-    version: "6bc1c7bb0d2a34e413301fee8f7cc728d2d4e75bfab186aa995f63292bda92fc", // hardcoded https://replicate.com/yorickvp/llava-13b/versions
+    version: models["yorickvp/llava-13b"]
   });
 }
 
@@ -85,6 +94,6 @@ async function runSalmonn({ prompt, maxTokens, temperature, topP, audio }) {
       wav_path: audio,
     },
     // IMPORTANT! The model must support streaming. See https://replicate.com/docs/streaming
-    version: "ad1d3f9d2bd683628242b68d890bef7f7bd97f738a7c2ccbf1743a594c723d83", // hardcoded https://replicate.com/yorickvp/llava-13b/versions
+    version: models["nateraw/salmonn"]
   });
 }
diff --git a/app/components/SlideOver.js b/app/components/SlideOver.js
index bc1651d..60b7f34 100644
--- a/app/components/SlideOver.js
+++ b/app/components/SlideOver.js
@@ -17,7 +17,7 @@ export default function SlideOver({
   setTopP,
   maxTokens,
   setMaxTokens,
-  versions,
+  models,
   size,
   setSize,
   handleSubmit,
@@ -108,44 +108,42 @@ export default function SlideOver({
                                       leaveTo="opacity-0"
                                     >
                                       <Listbox.Options className="absolute mt-1 max-h-60 w-full shadow-md overflow-auto border-gray-700 rounded-md bg-white py-1 text-base ring-1 ring-black ring-opacity-5 focus:outline-none sm:text-sm">
-                                        {versions
-                                          ? versions.map(
-                                              (version, versionIdx) => (
-                                                <Listbox.Option
-                                                  key={versionIdx}
-                                                  className={({ active }) =>
-                                                    `relative cursor-default select-none py-2 pl-10 pr-4 ${
-                                                      active
-                                                        ? "bg-gray-100 text-gray-900"
-                                                        : "text-gray-900"
-                                                    }`
-                                                  }
-                                                  value={version}
-                                                >
-                                                  {({ selected }) => (
-                                                    <>
-                                                      <span
-                                                        className={`block truncate ${
-                                                          selected
-                                                            ? "font-medium"
-                                                            : "font-normal"
+                                        {models
+                                          ? models.map(
+                                            (model, modelIdx) => (
+                                              <Listbox.Option
+                                                key={modelIdx}
+                                                className={({ active }) =>
+                                                  `relative cursor-default select-none py-2 pl-10 pr-4 ${active
+                                                    ? "bg-gray-100 text-gray-900"
+                                                    : "text-gray-900"
+                                                  }`
+                                                }
+                                                value={model}
+                                              >
+                                                {({ selected }) => (
+                                                  <>
+                                                    <span
+                                                      className={`block truncate ${selected
+                                                          ? "font-medium"
+                                                          : "font-normal"
                                                         }`}
-                                                      >
-                                                        {version.name}
+                                                    >
+                                                      {model.name}
+                                                    </span>
+                                                    {selected ? (
+                                                      <span className="absolute inset-y-0 left-0 flex items-center pl-3 text-gray-600">
+                                                        <CheckIcon
+                                                          className="h-5 w-5"
+                                                          aria-hidden="true"
+                                                        />
                                                       </span>
-                                                      {selected ? (
-                                                        <span className="absolute inset-y-0 left-0 flex items-center pl-3 text-gray-600">
-                                                          <CheckIcon
-                                                            className="h-5 w-5"
-                                                            aria-hidden="true"
-                                                          />
-                                                        </span>
-                                                      ) : null}
-                                                    </>
-                                                  )}
-                                                </Listbox.Option>
-                                              )
+                                                    ) : null}
+                                                  </>
+                                                )}
+                                              </Listbox.Option>
                                             )
+                                          )
                                           : null}
                                       </Listbox.Options>
                                     </Transition>
diff --git a/app/page.js b/app/page.js
index 00d108e..01c9cf0 100644
--- a/app/page.js
+++ b/app/page.js
@@ -10,30 +10,30 @@ import { useCompletion } from "ai/react";
 import { Toaster, toast } from "react-hot-toast";
 import { countTokens } from "./src/tokenizer.js";
 
-const VERSIONS = [
+const MODELS = [
   {
+    id: "meta/llama-2-7b-chat",
     name: "Llama 2 7B",
-    version: "13c3cdee13ee059ab779f0291d29054dab00a47dad8261375654de5540165fb0",
     shortened: "7B",
   },
   {
+    id: "meta/llama-2-13b-chat",
     name: "Llama 2 13B",
-    version: "f4e2de70d66816a838a89eeeb621910adffb0dd0baba3976c96980970978018d",
     shortened: "13B",
   },
   {
+    id: "meta/llama-2-70b-chat",
     name: "Llama 2 70B",
-    version: "02e509c789964a7ea8736978a43525956ef40397be9033abf9fd2badfe68c9e3",
     shortened: "70B",
   },
   {
+    id: "yorickvp/llava-13b",
     name: "Llava 13B",
-    version: "2facb4a474a0462c15041b78b1ad70952ea46b5ec6ad29583c0b29dbd4249591",
     shortened: "Llava",
   },
   {
+    id: "nateraw/salmonn",
     name: "Salmonn",
-    version: "ad1d3f9d2bd683628242b68d890bef7f7bd97f738a7c2ccbf1743a594c723d83",
     shortened: "Salmonn",
   },
 ];
@@ -93,7 +93,7 @@ export default function HomePage() {
   const [error, setError] = useState(null);
 
   //   Llama params
-  const [size, setSize] = useState(VERSIONS[2]); // default to 70B
+  const [model, setModel] = useState(MODELS[2]); // default to 70B
   const [systemPrompt, setSystemPrompt] = useState(
     "You are a helpful assistant."
   );
@@ -118,7 +118,7 @@ export default function HomePage() {
   const { complete, completion, setInput, input } = useCompletion({
     api: "/api",
     body: {
-      version: size.version,
+      model: model.id,
       systemPrompt: systemPrompt,
       temperature: parseFloat(temp),
       topP: parseFloat(topP),
@@ -149,13 +149,13 @@ export default function HomePage() {
         )
       ) {
         setAudio(file.fileUrl);
-        setSize(VERSIONS[4]);
+        setModel(MODELS[4]);
         toast.success(
           "You uploaded an audio file, so you're now speaking with Salmonn."
         );
       } else if (["image/jpeg", "image/png"].includes(file.originalFile.mime)) {
         setImage(file.fileUrl);
-        setSize(VERSIONS[3]);
+        setModel(MODELS[3]);
         toast.success(
           "You uploaded an image, so you're now speaking with Llava."
         );
@@ -244,14 +244,14 @@ export default function HomePage() {
   return (
     <>
       <div className="bg-slate-100 border-b-2 text-center p-3">
-        Powered by Replicate. <CTA shortenedModelName={size.shortened} />
+        Powered by Replicate. <CTA shortenedModelName={model.shortened} />
       </div>
       <nav className="grid grid-cols-2 pt-3 pl-6 pr-3 sm:grid-cols-3 sm:pl-0">
         <div className="hidden sm:inline-block"></div>
         <div className="font-semibold text-gray-500 sm:text-center">
-          {size.shortened == "Llava"
+          {model.shortened == "Llava"
             ? "🌋"
-            : size.shortened == "Salmonn"
+            : model.shortened == "Salmonn"
               ? "🐟"
               : "🦙"}{" "}
           <span className="hidden sm:inline-block">Chat with</span>{" "}
@@ -259,9 +259,9 @@ export default function HomePage() {
             className="py-2 font-semibold text-gray-500 hover:underline"
             onClick={() => setOpen(true)}
           >
-            {size.shortened == "Llava" || size.shortened == "Salmonn"
-              ? size.shortened
-              : "Llama 2 " + size.shortened}
+            {model.shortened == "Llava" || model.shortened == "Salmonn"
+              ? model.shortened
+              : "Llama 2 " + model.shortened}
           </button>
         </div>
         <div className="flex justify-end">
@@ -309,9 +309,9 @@ export default function HomePage() {
           setMaxTokens={setMaxTokens}
           topP={topP}
           setTopP={setTopP}
-          versions={VERSIONS}
-          size={size}
-          setSize={setSize}
+          models={MODELS}
+          size={model}
+          setSize={setModel}
         />
 
         {image && (

From a516fc6d06e42965d6a74a00ba382b2af9d36adf Mon Sep 17 00:00:00 2001
From: Mattt Zmuda <mattt@replicate.com>
Date: Wed, 6 Dec 2023 10:52:55 -0800
Subject: [PATCH 5/5] Use official model endpoints for llamas

---
 app/api/route.js            | 14 +++-----------
 app/components/SlideOver.js |  4 ++--
 2 files changed, 5 insertions(+), 13 deletions(-)

diff --git a/app/api/route.js b/app/api/route.js
index e06e2e4..e8a7890 100644
--- a/app/api/route.js
+++ b/app/api/route.js
@@ -14,9 +14,6 @@ if (!process.env.REPLICATE_API_TOKEN) {
 }
 
 const VERSIONS = {
-  "meta/llama-2-7b-chat": "13c3cdee13ee059ab779f0291d29054dab00a47dad8261375654de5540165fb0",
-  "meta/llama-2-13b-chat": "f4e2de70d66816a838a89eeeb621910adffb0dd0baba3976c96980970978018d",
-  "meta/llama-2-70b-chat": "02e509c789964a7ea8736978a43525956ef40397be9033abf9fd2badfe68c9e3",
   "yorickvp/llava-13b": "e272157381e2a3bf12df3a8edd1f38d1dbd736bbb7437277c8b34175f8fce358",
   "nateraw/salmonn": "ad1d3f9d2bd683628242b68d890bef7f7bd97f738a7c2ccbf1743a594c723d83",
 };
@@ -46,8 +43,9 @@ async function runLlama({
 }) {
   console.log("running llama");
 
-  return await replicate.predictions.create({
-    // IMPORTANT! You must enable streaming.
+  const [owner, name] = model.split("/");
+
+  return await replicate.models.predictions.create(owner, name, {
     stream: true,
     input: {
       prompt: `${prompt}`,
@@ -57,8 +55,6 @@ async function runLlama({
       repetition_penalty: 1,
       top_p: topP,
     },
-    // IMPORTANT! The model must support streaming. See https://replicate.com/docs/streaming
-    version: VERSIONS[model],
   });
 }
 
@@ -66,7 +62,6 @@ async function runLlava({ prompt, maxTokens, temperature, topP, image }) {
   console.log("running llava");
 
   return await replicate.predictions.create({
-    // IMPORTANT! You must enable streaming.
     stream: true,
     input: {
       prompt: `${prompt}`,
@@ -75,7 +70,6 @@ async function runLlava({ prompt, maxTokens, temperature, topP, image }) {
       max_tokens: maxTokens,
       image: image,
     },
-    // IMPORTANT! The model must support streaming. See https://replicate.com/docs/streaming
     version: models["yorickvp/llava-13b"]
   });
 }
@@ -84,7 +78,6 @@ async function runSalmonn({ prompt, maxTokens, temperature, topP, audio }) {
   console.log("running salmonn");
 
   return await replicate.predictions.create({
-    // IMPORTANT! You must enable streaming.
     stream: true,
     input: {
       prompt: `${prompt}`,
@@ -93,7 +86,6 @@ async function runSalmonn({ prompt, maxTokens, temperature, topP, audio }) {
       max_length: maxTokens,
       wav_path: audio,
     },
-    // IMPORTANT! The model must support streaming. See https://replicate.com/docs/streaming
     version: models["nateraw/salmonn"]
   });
 }
diff --git a/app/components/SlideOver.js b/app/components/SlideOver.js
index 60b7f34..e46d778 100644
--- a/app/components/SlideOver.js
+++ b/app/components/SlideOver.js
@@ -125,8 +125,8 @@ export default function SlideOver({
                                                   <>
                                                     <span
                                                       className={`block truncate ${selected
-                                                          ? "font-medium"
-                                                          : "font-normal"
+                                                        ? "font-medium"
+                                                        : "font-normal"
                                                         }`}
                                                     >
                                                       {model.name}