Semantic Memory Integration into Copilot Chat

.github/workflows/copilot-build-backend.yml

@@ -9,7 +9,7 @@ on:
   workflow_call:
     outputs:
       artifact:
-        description: "The name of the uploaded artifact."
+        description: "The name of the uploaded web api artifact."
         value: ${{jobs.webapi.outputs.artifact}}
 
 permissions:

.github/workflows/copilot-build-memorypipeline.yml

@@ -0,0 +1,68 @@
+name: copilot-build-memorypipeline
+
+on:
+  pull_request:
+    branches: ["main"]
+    paths:
+      - "memorypipeline/**"
+  workflow_call:
+    outputs:
+      artifact:
+        description: "The name of the uploaded memory pipeline artifact."
+        value: ${{jobs.memory-pipeline.outputs.artifact}}
+
+permissions:
+  contents: read
+
+jobs:
+  memory-pipeline:
+    runs-on: windows-latest
+
+    env:
+      NUGET_CERT_REVOCATION_MODE: offline
+
+    outputs:
+      artifact: ${{steps.artifactoutput.outputs.artifactname}}
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          clean: true
+          fetch-depth: 0
+
+      - name: Install GitVersion
+        uses: gittools/actions/gitversion/setup@v0
+        with:
+          versionSpec: "5.x"
+
+      - name: Determine version
+        id: gitversion
+        uses: gittools/actions/gitversion/execute@v0
+
+      - name: Set version tag
+        id: versiontag
+        run: |
+          $VERSION_TAG = "${{ steps.gitversion.outputs.Major }}."
+          $VERSION_TAG += "${{ steps.gitversion.outputs.Minor }}."
+          $VERSION_TAG += "${{ steps.gitversion.outputs.CommitsSinceVersionSource }}"
+          echo $VERSION_TAG
+          Write-Output "versiontag=$VERSION_TAG" >> $env:GITHUB_OUTPUT
+
+      - name: Set .Net Core version
+        uses: actions/setup-dotnet@v1
+        with:
+          dotnet-version: 6.0.x
+
+      - name: Package Copilot Chat Memory Pipeline
+        run: |
+          scripts\deploy\package-memorypipeline.ps1 -Configuration Release -DotnetFramework net6.0 -TargetRuntime win-x64 -OutputDirectory ${{ github.workspace }}\scripts\deploy -Version ${{ steps.versiontag.outputs.versiontag }} -InformationalVersion "Built from commit ${{ steps.gitversion.outputs.ShortSha }} on $(Get-Date -Format "yyyy-MM-dd")"
+
+      - name: Upload package to artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: copilotchat-memorypipeline-${{ steps.versiontag.outputs.versiontag }}
+          path: ${{ github.workspace }}\scripts\deploy\out\memorypipeline.zip
+
+      - name: "Set outputs"
+        id: artifactoutput
+        run: Write-Output "artifactname=copilotchat-memorypipeline-${{ steps.versiontag.outputs.versiontag }}" >> $env:GITHUB_OUTPUT

.github/workflows/copilot-deploy-environment.yml

@@ -6,7 +6,10 @@ on:
       ENVIRONMENT:
         required: true
         type: string
-      ARTIFACT_NAME:
+      WEBAPI_ARTIFACT_NAME:
+        required: true
+        type: string
+      MEMORYPIPELINE_ARTIFACT_NAME:
         required: true
         type: string
     secrets:
@@ -42,7 +45,19 @@ jobs:
     needs: [deploy-infra]
     uses: ./.github/workflows/copilot-deploy-backend.yml
     with:
-      ARTIFACT_NAME: ${{inputs.ARTIFACT_NAME}}
+      ARTIFACT_NAME: ${{inputs.WEBAPI_ARTIFACT_NAME}}
+      DEPLOYMENT_NAME: ${{needs.deploy-infra.outputs.deployment-id}}
+      ENVIRONMENT: ${{inputs.ENVIRONMENT}}
+    secrets:
+      AZURE_CLIENT_ID: ${{secrets.AZURE_CLIENT_ID}}
+      AZURE_TENANT_ID: ${{secrets.AZURE_TENANT_ID}}
+      AZURE_SUBSCRIPTION_ID: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+
+  deploy-memorypipeline:
+    needs: [deploy-infra]
+    uses: ./.github/workflows/copilot-deploy-memorypipeline.yml
+    with:
+      ARTIFACT_NAME: ${{inputs.MEMORYPIPELINE_ARTIFACT_NAME}}
       DEPLOYMENT_NAME: ${{needs.deploy-infra.outputs.deployment-id}}
       ENVIRONMENT: ${{inputs.ENVIRONMENT}}
     secrets:

.github/workflows/copilot-deploy-memorypipeline.yml

@@ -0,0 +1,76 @@
+name: copilot-deploy-memorypipeline
+
+on:
+  workflow_call:
+    inputs:
+      ARTIFACT_NAME:
+        required: true
+        type: string
+      ENVIRONMENT:
+        required: true
+        type: string
+      DEPLOYMENT_NAME:
+        required: true
+        type: string
+    secrets:
+      AZURE_CLIENT_ID:
+        required: true
+      AZURE_TENANT_ID:
+        required: true
+      AZURE_SUBSCRIPTION_ID:
+        required: true
+
+permissions:
+  contents: read
+
+jobs:
+  memorypipeline:
+    environment: ${{inputs.ENVIRONMENT}}
+    permissions:
+      id-token: write
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - { dotnet: "6.0", configuration: Release, os: ubuntu-latest }
+
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          clean: true
+
+      - uses: actions/download-artifact@v3
+        with:
+          name: ${{inputs.ARTIFACT_NAME}}
+          path: "${{ github.workspace }}/${{inputs.ARTIFACT_NAME}}"
+
+      - name: "Display downloaded content"
+        run: ls -R
+        working-directory: "${{ github.workspace }}/${{inputs.ARTIFACT_NAME}}"
+
+      - name: Azure login
+        uses: azure/login@v1
+        with:
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          enable-AzPSSession: false
+
+      - name: Get app name
+        run: |
+          WEB_APP_NAME=$(az deployment group show --name ${{inputs.DEPLOYMENT_NAME}} --resource-group ${{vars.CC_DEPLOYMENT_GROUP_NAME}} --output json | jq -r '.properties.outputs.memoryPipelineName.value')
+          echo "AZURE_WEBAPP_NAME=$WEB_APP_NAME" >> $GITHUB_ENV
+
+      - name: Enable Run From Package
+        uses: azure/CLI@v1
+        with:
+          azcliversion: 2.30.0
+          inlineScript: |
+            az webapp config appsettings set --resource-group ${{vars.CC_DEPLOYMENT_GROUP_NAME}} --name ${{ env.AZURE_WEBAPP_NAME }} --settings WEBSITE_RUN_FROM_PACKAGE="1" -o none
+
+      - name: "Deploy"
+        uses: azure/webapps-deploy@v2
+        with:
+          app-name: ${{ env.AZURE_WEBAPP_NAME }}
+          package: "${{ github.workspace }}/${{inputs.ARTIFACT_NAME}}/memorypipeline.zip"

.github/workflows/copilot-deploy-pipeline.yml

@@ -14,15 +14,19 @@ permissions:
   id-token: write
 
 jobs:
-  build:
+  build-webapi:
     uses: ./.github/workflows/copilot-build-backend.yml
 
+  build-memorypipeline:
+    uses: ./.github/workflows/copilot-build-memorypipeline.yml
+
   int:
-    needs: build
+    needs: [build-webapi, build-memorypipeline]
     uses: ./.github/workflows/copilot-deploy-environment.yml
     with:
       ENVIRONMENT: int
-      ARTIFACT_NAME: ${{needs.build.outputs.artifact}}
+      WEBAPI_ARTIFACT_NAME: ${{needs.build-webapi.outputs.artifact}}
+      MEMORYPIPELINE_ARTIFACT_NAME: ${{needs.build-memorypipeline.outputs.artifact}}
     secrets:
       AZURE_CLIENT_ID: ${{secrets.AZURE_CLIENT_ID}}
       AZURE_TENANT_ID: ${{secrets.AZURE_TENANT_ID}}
@@ -37,11 +41,12 @@ jobs:
       BACKEND_HOST: ${{needs.int.outputs.backend-host}}
 
   stable:
-    uses: ./.github/workflows/copilot-deploy-environment.yml
     needs: int-tests
+    uses: ./.github/workflows/copilot-deploy-environment.yml
     with:
       ENVIRONMENT: stable
-      ARTIFACT_NAME: ${{needs.build.outputs.artifact}}
+      WEBAPI_ARTIFACT_NAME: ${{needs.build-webapi.outputs.artifact}}
+      MEMORYPIPELINE_ARTIFACT_NAME: ${{needs.build-memorypipeline.outputs.artifact}}
     secrets:
       AZURE_CLIENT_ID: ${{secrets.AZURE_CLIENT_ID}}
       AZURE_TENANT_ID: ${{secrets.AZURE_TENANT_ID}}

.github/workflows/dotnet-format.yml

@@ -5,6 +5,7 @@ on:
     branches: ["main", "feature*"]
     paths:
       - "webapi/**"
+      - "memorypipeline/**"
       - "tools/**"
 
 concurrency:

.gitignore

@@ -490,3 +490,8 @@ webapi/CopilotChatWebApi.sln
 
 # Tesseract OCR language data files
 *.traineddata
+
+# Semantic Memory local storage
+tmp-cache
+tmp-database
+tmp-queues

CopilotChat.sln

@@ -4,8 +4,14 @@ VisualStudioVersion = 17.6.33706.43
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CopilotChatWebApi", "webapi\CopilotChatWebApi.csproj", "{5252E68F-B653-44CE-9A32-360A75C54E0E}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CopilotChatMemoryPipeline", "memorypipeline\CopilotChatMemoryPipeline.csproj", "{EE1C907E-E90A-4AB5-9094-6CCE5F0DEDF8}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ImportDocument", "tools\importdocument\ImportDocument.csproj", "{F67EF16F-9A8A-4DC6-A9A1-E64CDFE4F285}"
+EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ChatCopilotIntegrationTests", "integration-tests\ChatCopilotIntegrationTests.csproj", "{0CD2CD95-536B-455F-B74A-772A455FA607}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CopilotChatShared", "shared\CopilotChatShared.csproj", "{94F12185-FAF9-43E3-B153-28A1708AC918}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -16,10 +22,22 @@ Global
 		{5252E68F-B653-44CE-9A32-360A75C54E0E}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{5252E68F-B653-44CE-9A32-360A75C54E0E}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{5252E68F-B653-44CE-9A32-360A75C54E0E}.Release|Any CPU.Build.0 = Release|Any CPU
+		{EE1C907E-E90A-4AB5-9094-6CCE5F0DEDF8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{EE1C907E-E90A-4AB5-9094-6CCE5F0DEDF8}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{EE1C907E-E90A-4AB5-9094-6CCE5F0DEDF8}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{EE1C907E-E90A-4AB5-9094-6CCE5F0DEDF8}.Release|Any CPU.Build.0 = Release|Any CPU
+		{F67EF16F-9A8A-4DC6-A9A1-E64CDFE4F285}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{F67EF16F-9A8A-4DC6-A9A1-E64CDFE4F285}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{F67EF16F-9A8A-4DC6-A9A1-E64CDFE4F285}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{F67EF16F-9A8A-4DC6-A9A1-E64CDFE4F285}.Release|Any CPU.Build.0 = Release|Any CPU
 		{0CD2CD95-536B-455F-B74A-772A455FA607}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{0CD2CD95-536B-455F-B74A-772A455FA607}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{0CD2CD95-536B-455F-B74A-772A455FA607}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{0CD2CD95-536B-455F-B74A-772A455FA607}.Release|Any CPU.Build.0 = Release|Any CPU
+		{94F12185-FAF9-43E3-B153-28A1708AC918}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{94F12185-FAF9-43E3-B153-28A1708AC918}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{94F12185-FAF9-43E3-B153-28A1708AC918}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{94F12185-FAF9-43E3-B153-28A1708AC918}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

README.md

@@ -1,6 +1,10 @@
 # Chat Copilot Sample Application
 
-This sample allows you to build your own integrated large language model (LLM) chat copilot. The sample is built on Microsoft [Semantic Kernel](https://github.com/microsoft/semantic-kernel) and has two components: a frontend [React web app](./webapp/) and a backend [.NET web API service](./webapi/).
+This sample allows you to build your own integrated large language model (LLM) chat copilot. The sample is built on Microsoft [Semantic Kernel](https://github.com/microsoft/semantic-kernel) and has three components:
+
+1. A frontend application [React web app](./webapp/)
+2. A backend REST API [.NET web API service](./webapi/)
+3. A [.NET worker service](./memorypipeline/) for processing semantic memory.
 
 These quick-start instructions run the sample locally. They can also be found on the official Chat Copilot Microsoft Learn documentation page for [getting started](https://learn.microsoft.com/semantic-kernel/chat-copilot/getting-started).
 
@@ -16,9 +20,9 @@ To deploy the sample to Azure, please view [Deploying Chat Copilot](./scripts/de
 
 You will need the following items to run the sample:
 
-- [.NET 7.0 SDK](https://dotnet.microsoft.com/download/dotnet/7.0) _(via Setup install.* script)_
-- [Node.js](https://nodejs.org/en/download) _(via Setup install.*  script)_
-- [Yarn](https://classic.yarnpkg.com/docs/install) _(via Setup install.*  script)_
+- [.NET 7.0 SDK](https://dotnet.microsoft.com/download/dotnet/7.0) _(via Setup install.\* script)_
+- [Node.js](https://nodejs.org/en/download) _(via Setup install.\* script)_
+- [Yarn](https://classic.yarnpkg.com/docs/install) _(via Setup install.\* script)_
 - AI Service
 
 | AI Service   | Requirement                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
@@ -149,6 +153,10 @@ You will need the following items to run the sample:
      ./start-backend.sh
      ```
 
+## (Optional) Run the [memory pipeline](./memorypipeline/README.md)
+
+By default, the webapi is configured to work without the memory pipeline for synchronous processing documents. To enable asynchronous document processing, you need to configure the webapi and the memory pipeline. Please refer to the [webapi README](./webapi/README.md) and the [memory pipeline README](./memorypipeline/README.md) for more information.
+
 ## (Optional) Enable backend authentication via Azure AD
 
 By default, Chat Copilot runs locally without authentication, using a guest user profile. If you want to enable authentication with Azure Active Directory, follow the steps below.

memorypipeline/CopilotChatMemoryPipeline.csproj

@@ -0,0 +1,22 @@
+<Project Sdk="Microsoft.NET.Sdk.Web">
+
+  <PropertyGroup>
+    <RootNamespace>CopilotChat.MemoryPipeline</RootNamespace>
+    <TargetFramework>net6.0</TargetFramework>
+    <RollForward>LatestMajor</RollForward>
+    <ImplicitUsings>disable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <UserSecretsId>5ee045b0-aea3-4f08-8d31-32d1a6f8fed0</UserSecretsId>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\shared\CopilotChatShared.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.ApplicationInsights.AspNetCore" Version="2.21.0" />
+    <PackageReference Include="Microsoft.SemanticKernel" Version="0.24.230918.1-preview" />
+    <PackageReference Include="Microsoft.SemanticMemory.Core" Version="0.2.230919.2-preview" />
+  </ItemGroup>
+
+</Project>

memorypipeline/Program.cs

@@ -0,0 +1,55 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System;
+using CopilotChat.Shared;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Http;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using Microsoft.SemanticMemory;
+using Microsoft.SemanticMemory.Diagnostics;
+
+// ********************************************************
+// ************** SETUP ***********************************
+// ********************************************************
+
+var builder = WebApplication.CreateBuilder();
+
+ISemanticMemoryClient memory =
+    new MemoryClientBuilder(builder.Services)
+        .FromAppSettings()
+        .WithoutSummarizeHandlers()
+        .WithCustomOcr(builder.Configuration)
+        .Build();
+
+builder.Services.AddSingleton(memory);
+
+builder.Services.AddApplicationInsightsTelemetry();
+
+var app = builder.Build();
+
+DateTimeOffset start = DateTimeOffset.UtcNow;
+
+// Simple ping endpoint
+app.MapGet("/", () =>
+{
+    var uptime = DateTimeOffset.UtcNow.ToUnixTimeSeconds() - start.ToUnixTimeSeconds();
+    var environment = Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT");
+    var message = $"Memory pipeline is running. Uptime: {uptime} secs.";
+    if (!string.IsNullOrEmpty(environment))
+    {
+        message += $" Environment: {environment}";
+    }
+    return Results.Ok(message);
+});
+
+// ********************************************************
+// ************** START ***********************************
+// ********************************************************
+
+app.Logger.LogInformation(
+    "Starting Chat Copilot Memory pipeline service, .NET Env: {0}, Log Level: {1}",
+    Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT"),
+    app.Logger.GetLogLevelName());
+
+app.Run();