From eb5190fdabb84c93e5b5b3858fe0b13910e8ddeb Mon Sep 17 00:00:00 2001 From: Robert Hambrock Date: Sun, 21 May 2023 13:10:38 +0200 Subject: [PATCH 1/4] don't include `role` key with every stream message --- api/openai.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/api/openai.go b/api/openai.go index 0a85349c18c3..37ed6f640ec1 100644 --- a/api/openai.go +++ b/api/openai.go @@ -259,10 +259,17 @@ func embeddingsEndpoint(cm *ConfigMerger, debug bool, loader *model.ModelLoader, func chatEndpoint(cm *ConfigMerger, debug bool, loader *model.ModelLoader, threads, ctx int, f16 bool) func(c *fiber.Ctx) error { process := func(s string, req *OpenAIRequest, config *Config, loader *model.ModelLoader, responses chan OpenAIResponse) { + initialMessage := OpenAIResponse{ + Model: req.Model, // we have to return what the user sent here, due to OpenAI spec. + Choices: []Choice{{Delta: &Message{Role: "assistant"}}}, + Object: "chat.completion.chunk", + } + responses <- initialMessage + ComputeChoices(s, req, config, loader, func(s string, c *[]Choice) {}, func(s string) bool { resp := OpenAIResponse{ Model: req.Model, // we have to return what the user sent here, due to OpenAI spec. - Choices: []Choice{{Delta: &Message{Role: "assistant", Content: s}}}, + Choices: []Choice{{Delta: &Message{Content: s}}}, Object: "chat.completion.chunk", } log.Debug().Msgf("Sending goroutine: %s", s) From e011328c4f0484e144f48e90c8540e5ce9802003 Mon Sep 17 00:00:00 2001 From: Robert Hambrock Date: Sun, 21 May 2023 13:15:36 +0200 Subject: [PATCH 2/4] terminate stream with "[DONE]" message see https://platform.openai.com/docs/api-reference/chat/create#chat/create-stream --- api/openai.go | 1 + 1 file changed, 1 insertion(+) diff --git a/api/openai.go b/api/openai.go index 37ed6f640ec1..e2cd493d8406 100644 --- a/api/openai.go +++ b/api/openai.go @@ -360,6 +360,7 @@ func chatEndpoint(cm *ConfigMerger, debug bool, loader *model.ModelLoader, threa respData, _ := json.Marshal(resp) w.WriteString(fmt.Sprintf("data: %s\n\n", respData)) + w.WriteString("data: [DONE]\n\n") w.Flush() })) return nil From 36b1f5335480005850c607d852e1989d4492d4c3 Mon Sep 17 00:00:00 2001 From: Robert Hambrock Date: Sun, 21 May 2023 13:21:03 +0200 Subject: [PATCH 3/4] don't use named events for data messages see https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#event_stream_format - named events are superfluous for sequences of strictly data messages. --- api/openai.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/api/openai.go b/api/openai.go index e2cd493d8406..7ea09ac48c71 100644 --- a/api/openai.go +++ b/api/openai.go @@ -346,13 +346,11 @@ func chatEndpoint(cm *ConfigMerger, debug bool, loader *model.ModelLoader, threa enc := json.NewEncoder(&buf) enc.Encode(ev) - fmt.Fprintf(w, "event: data\n\n") fmt.Fprintf(w, "data: %v\n\n", buf.String()) log.Debug().Msgf("Sending chunk: %s", buf.String()) w.Flush() } - w.WriteString("event: data\n\n") resp := &OpenAIResponse{ Model: input.Model, // we have to return what the user sent here, due to OpenAI spec. Choices: []Choice{{FinishReason: "stop"}}, From c47796d543772021fdc3a89dc6b64ba76730cc6d Mon Sep 17 00:00:00 2001 From: Robert Hambrock Date: Sun, 21 May 2023 13:42:56 +0200 Subject: [PATCH 4/4] nit: remove superfluous newline already get one extra from the preceding `enc.Encode(ev)` call. --- api/openai.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/openai.go b/api/openai.go index 7ea09ac48c71..1a9f15c83893 100644 --- a/api/openai.go +++ b/api/openai.go @@ -346,8 +346,8 @@ func chatEndpoint(cm *ConfigMerger, debug bool, loader *model.ModelLoader, threa enc := json.NewEncoder(&buf) enc.Encode(ev) - fmt.Fprintf(w, "data: %v\n\n", buf.String()) log.Debug().Msgf("Sending chunk: %s", buf.String()) + fmt.Fprintf(w, "data: %v\n", buf.String()) w.Flush() }