From 406d07cdd1c9bfecd5f9a8f0b5de4abe7da0e4f7 Mon Sep 17 00:00:00 2001 From: limez Date: Sun, 20 Oct 2024 08:09:05 +0200 Subject: [PATCH 1/5] update stable-diffusion.cpp --- CMakeLists.txt | 2 +- index.d.ts | 9 +++++- src/NodeModule.cpp | 80 ++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 83 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b79b696..dd67d99 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,7 +21,7 @@ endif() FetchContent_Declare( stable-diffusion-cpp GIT_REPOSITORY https://github.com/lmagder/stable-diffusion.cpp.git - GIT_TAG f974370e049da45866d70f45673c5453e089251b # fix-untyped-tensors + GIT_TAG 14206fd48832ab600d9db75f15acb5062ae2c296 # fix clip tokenizer ) FetchContent_Declare( diff --git a/index.d.ts b/index.d.ts index a479500..265e748 100644 --- a/index.d.ts +++ b/index.d.ts @@ -8,6 +8,8 @@ declare module "@lmagder/node-stable-diffusion-cpp" { DPMPP2M, DPMPP2Mv2, LCM, + IPNDM, + IPNDM_V } declare enum Schedule { @@ -66,6 +68,7 @@ declare module "@lmagder/node-stable-diffusion-cpp" { negativePrompt?: string; clipSkip?: number; cfgScale?: number; + guidance?: number; width?: number; height?: number; sampleMethod?: SampleMethod; @@ -84,6 +87,7 @@ declare module "@lmagder/node-stable-diffusion-cpp" { negativePrompt?: string; clipSkip?: number; cfgScale?: number; + guidance?: number; width?: number; height?: number; sampleMethod?: SampleMethod; @@ -116,7 +120,10 @@ declare module "@lmagder/node-stable-diffusion-cpp" { export const createContext: ( params: { - model: string; + model?: string; + clipL?: string; + t5xxl?: string; + diffusionModel?: string; vae?: string; taesd?: string; controlNet?: string; diff --git a/src/NodeModule.cpp b/src/NodeModule.cpp index 74f9f7a..06a32e8 100644 --- a/src/NodeModule.cpp +++ b/src/NodeModule.cpp @@ -336,7 +336,10 @@ namespace { Napi::Value tmp; const auto params = info[0].ToObject(); - const auto model = params.Get("model").ToString().Utf8Value(); + const auto model = (tmp = params.Get("model"), tmp.IsUndefined() ? "" : tmp.ToString().Utf8Value()); + const auto clipL = (tmp = params.Get("clipL"), tmp.IsUndefined() ? "" : tmp.ToString().Utf8Value()); + const auto t5xxl = (tmp = params.Get("t5xxl"), tmp.IsUndefined() ? "" : tmp.ToString().Utf8Value()); + const auto diffusionModel = (tmp = params.Get("diffusionModel"), tmp.IsUndefined() ? "" : tmp.ToString().Utf8Value()); const auto vae = (tmp = params.Get("vae"), tmp.IsUndefined() ? "" : tmp.ToString().Utf8Value()); const auto taesd = (tmp = params.Get("taesd"), tmp.IsUndefined() ? "" : tmp.ToString().Utf8Value()); const auto controlNet = (tmp = params.Get("controlNet"), tmp.IsUndefined() ? "" : tmp.ToString().Utf8Value()); @@ -375,9 +378,33 @@ namespace return queueStableDiffusionWorker(info.Env(), cppContextData, [=](CPPContextData& ctx) { - ctx.sdCtx = { new_sd_ctx(model.c_str(), vae.c_str(), taesd.c_str(), controlNet.c_str(), loraDir.c_str(), - embedDir.c_str(), stackedIdEmbedDir.c_str(), vaeDecodeOnly, vaeTiling, freeParamsImmediately, numThreads, weightType, - cudaRng ? CUDA_RNG : STD_DEFAULT_RNG, schedule, keepClipOnCpu, keepControlNetOnCpu, keepVaeOnCpu), [](sd_ctx_t* c) { if (c) free_sd_ctx(c); } }; + ctx.sdCtx = { + new_sd_ctx( + model.c_str(), + clipL.c_str(), + t5xxl.c_str(), + diffusionModel.c_str(), + vae.c_str(), + taesd.c_str(), + controlNet.c_str(), + loraDir.c_str(), + embedDir.c_str(), + stackedIdEmbedDir.c_str(), + vaeDecodeOnly, + vaeTiling, + freeParamsImmediately, + numThreads, + weightType, + cudaRng ? CUDA_RNG : STD_DEFAULT_RNG, + schedule, + keepClipOnCpu, + keepControlNetOnCpu, + keepVaeOnCpu + ), + [](sd_ctx_t* c) { + if (c) free_sd_ctx(c); + } + }; if (!ctx.sdCtx) throw std::runtime_error("Context creation failed"); @@ -427,12 +454,32 @@ namespace const auto styleRatio = (tmp = params.Get("styleRatio"), tmp.IsUndefined() ? 20.0f : tmp.ToNumber().FloatValue()); const auto normalizeInput = (tmp = params.Get("normalizeInput"), tmp.IsUndefined() ? false : tmp.ToBoolean().Value()); const auto inputIdImagesPath = (tmp = params.Get("inputIdImagesPath"), tmp.IsUndefined() ? "" : tmp.ToString().Utf8Value()); + const auto guidance = (tmp = params.Get("guidance"), tmp.IsUndefined() ? 0.0f : tmp.ToNumber().FloatValue()); + if (sampleMethod >= N_SAMPLE_METHODS) throw Napi::Error::New(info.Env(), "Invalid sampleMethod"); return queueStableDiffusionWorker(info.Env(), cppContextData, [=, sdCtx = cppContextData->sdCtx, controlCond = std::move(controlCond)](CPPContextData& ctx) { - return SdImageList(txt2img(sdCtx.get(), prompt.c_str(), negativePrompt.c_str(), clipSkip, cfgScale, width, height, sampleMethod, sampleSteps, seed, batchCount, controlCond.get(), controlStrength, styleRatio, normalizeInput, inputIdImagesPath.c_str()), batchCount); + return SdImageList(txt2img( + sdCtx.get(), + prompt.c_str(), + negativePrompt.c_str(), + clipSkip, + cfgScale, + guidance, + width, + height, + sampleMethod, + sampleSteps, + seed, + batchCount, + controlCond.get(), + controlStrength, + styleRatio, + normalizeInput, + inputIdImagesPath.c_str() + ), batchCount); }, [batchCount](Napi::Env env, SdImageList&& images) { @@ -468,12 +515,33 @@ namespace const auto styleRatio = (tmp = params.Get("styleRatio"), tmp.IsUndefined() ? 20.0f : tmp.ToNumber().FloatValue()); const auto normalizeInput = (tmp = params.Get("normalizeInput"), tmp.IsUndefined() ? false : tmp.ToBoolean().Value()); const auto inputIdImagesPath = (tmp = params.Get("inputIdImagesPath"), tmp.IsUndefined() ? "" : tmp.ToString().Utf8Value()); + const auto guidance = (tmp = params.Get("guidance"), tmp.IsUndefined() ? 0.0f : tmp.ToNumber().FloatValue()); if (sampleMethod >= N_SAMPLE_METHODS) throw Napi::Error::New(info.Env(), "Invalid sampleMethod"); return queueStableDiffusionWorker(info.Env(), cppContextData, [=, sdCtx = cppContextData->sdCtx, initImage = std::move(initImage), controlCond = std::move(controlCond)](CPPContextData& ctx) { - return SdImageList(img2img(sdCtx.get(), *initImage, prompt.c_str(), negativePrompt.c_str(), clipSkip, cfgScale, width, height, sampleMethod, sampleSteps, strength, seed, batchCount, controlCond.get(), controlStrength, styleRatio, normalizeInput, inputIdImagesPath.c_str()), batchCount); + return SdImageList(img2img( + sdCtx.get(), + *initImage, + prompt.c_str(), + negativePrompt.c_str(), + clipSkip, + cfgScale, + guidance, + width, + height, + sampleMethod, + sampleSteps, + strength, + seed, + batchCount, + controlCond.get(), + controlStrength, + styleRatio, + normalizeInput, + inputIdImagesPath.c_str() + ), batchCount); }, [batchCount](Napi::Env env, SdImageList&& images) { From 21169e9f182ef9e9588041a988f181d94d4310fc Mon Sep 17 00:00:00 2001 From: limez Date: Sat, 26 Oct 2024 17:36:19 +0200 Subject: [PATCH 2/5] build with vulkan backend if its available --- CMakeLists.txt | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index dd67d99..8398799 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -30,6 +30,19 @@ FetchContent_Declare( GIT_TAG f9ff166845a59327eda431af82ee85a9c7532c5d ) +if(NOT WIN32) + if(EXISTS ${CMAKE_SYSROOT}/usr/lib/x86_64-linux-gnu/libvulkan.so) + set(Vulkan_INCLUDE_DIR ${CMAKE_SYSROOT}/usr/include) + set(Vulkan_LIBRARY ${CMAKE_SYSROOT}/usr/lib/x86_64-linux-gnu/libvulkan.so) + endif() +endif() +find_package(Vulkan) + +if(Vulkan_FOUND) + set(SD_FLASH_ATTN ON) + set(SD_VULKAN ON) +endif() + set(IMPLIB_SOURCE_FILES "") if (CUDAToolkit_FOUND) set(SD_CUBLAS ON) @@ -98,5 +111,3 @@ if(MSVC AND CMAKE_JS_NODELIB_DEF AND CMAKE_JS_NODELIB_TARGET) execute_process(COMMAND ${CMAKE_AR} /def:${CMAKE_JS_NODELIB_DEF} /out:${CMAKE_JS_NODELIB_TARGET} ${CMAKE_STATIC_LINKER_FLAGS}) target_link_options(node-stable-diffusion-cpp PRIVATE "/DELAYLOAD:nvcuda.dll") endif() - - From 391bd0c2c97919ef1b49465ab1b498923327dd92 Mon Sep 17 00:00:00 2001 From: limez Date: Sat, 26 Oct 2024 17:42:00 +0200 Subject: [PATCH 3/5] use leejet repo to try sd 3.5, add GITS schedule and clipG param --- CMakeLists.txt | 4 ++-- index.d.ts | 2 ++ src/NodeModule.cpp | 21 ++++++++++++--------- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8398799..30f5147 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,8 +20,8 @@ endif() FetchContent_Declare( stable-diffusion-cpp - GIT_REPOSITORY https://github.com/lmagder/stable-diffusion.cpp.git - GIT_TAG 14206fd48832ab600d9db75f15acb5062ae2c296 # fix clip tokenizer + GIT_REPOSITORY https://github.com/leejet/stable-diffusion.cpp.git + GIT_TAG ac54e0076052a196b7df961eb1f792c9ff4d7f22 # sd 3.5 support ) FetchContent_Declare( diff --git a/index.d.ts b/index.d.ts index 265e748..1a414c4 100644 --- a/index.d.ts +++ b/index.d.ts @@ -17,6 +17,7 @@ declare module "@lmagder/node-stable-diffusion-cpp" { Discrete, Karras, AYS, + GITS, } declare enum Type { @@ -122,6 +123,7 @@ declare module "@lmagder/node-stable-diffusion-cpp" { params: { model?: string; clipL?: string; + clipG?: string; t5xxl?: string; diffusionModel?: string; vae?: string; diff --git a/src/NodeModule.cpp b/src/NodeModule.cpp index 06a32e8..0ef783c 100644 --- a/src/NodeModule.cpp +++ b/src/NodeModule.cpp @@ -248,7 +248,7 @@ namespace return ret; } - + class NodeStableDiffusionCpp : public Napi::Addon { public: @@ -277,6 +277,7 @@ namespace Napi::PropertyDescriptor::Value("Default", Napi::Number::New(env, DEFAULT)), Napi::PropertyDescriptor::Value("Discrete", Napi::Number::New(env, DISCRETE)), Napi::PropertyDescriptor::Value("Karras", Napi::Number::New(env, KARRAS)), + Napi::PropertyDescriptor::Value("GITS", Napi::Number::New(env, GITS)), Napi::PropertyDescriptor::Value("AYS", Napi::Number::New(env, AYS)), }); scheduleEnum.Freeze(); @@ -336,10 +337,11 @@ namespace { Napi::Value tmp; const auto params = info[0].ToObject(); - const auto model = (tmp = params.Get("model"), tmp.IsUndefined() ? "" : tmp.ToString().Utf8Value()); + const auto model = (tmp = params.Get("model"), tmp.IsUndefined() ? "" : tmp.ToString().Utf8Value()); const auto clipL = (tmp = params.Get("clipL"), tmp.IsUndefined() ? "" : tmp.ToString().Utf8Value()); + const auto clipG = (tmp = params.Get("clipG"), tmp.IsUndefined() ? "" : tmp.ToString().Utf8Value()); const auto t5xxl = (tmp = params.Get("t5xxl"), tmp.IsUndefined() ? "" : tmp.ToString().Utf8Value()); - const auto diffusionModel = (tmp = params.Get("diffusionModel"), tmp.IsUndefined() ? "" : tmp.ToString().Utf8Value()); + const auto diffusionModel = (tmp = params.Get("diffusionModel"), tmp.IsUndefined() ? "" : tmp.ToString().Utf8Value()); const auto vae = (tmp = params.Get("vae"), tmp.IsUndefined() ? "" : tmp.ToString().Utf8Value()); const auto taesd = (tmp = params.Get("taesd"), tmp.IsUndefined() ? "" : tmp.ToString().Utf8Value()); const auto controlNet = (tmp = params.Get("controlNet"), tmp.IsUndefined() ? "" : tmp.ToString().Utf8Value()); @@ -378,10 +380,11 @@ namespace return queueStableDiffusionWorker(info.Env(), cppContextData, [=](CPPContextData& ctx) { - ctx.sdCtx = { + ctx.sdCtx = { new_sd_ctx( model.c_str(), clipL.c_str(), + clipG.c_str(), t5xxl.c_str(), diffusionModel.c_str(), vae.c_str(), @@ -400,10 +403,10 @@ namespace keepClipOnCpu, keepControlNetOnCpu, keepVaeOnCpu - ), - [](sd_ctx_t* c) { - if (c) free_sd_ctx(c); - } + ), + [](sd_ctx_t* c) { + if (c) free_sd_ctx(c); + } }; if (!ctx.sdCtx) @@ -455,7 +458,7 @@ namespace const auto normalizeInput = (tmp = params.Get("normalizeInput"), tmp.IsUndefined() ? false : tmp.ToBoolean().Value()); const auto inputIdImagesPath = (tmp = params.Get("inputIdImagesPath"), tmp.IsUndefined() ? "" : tmp.ToString().Utf8Value()); const auto guidance = (tmp = params.Get("guidance"), tmp.IsUndefined() ? 0.0f : tmp.ToNumber().FloatValue()); - + if (sampleMethod >= N_SAMPLE_METHODS) throw Napi::Error::New(info.Env(), "Invalid sampleMethod"); From 2bb93be0bde5b126aef1bd2f02ec1adbf20a14dc Mon Sep 17 00:00:00 2001 From: limez Date: Sun, 27 Oct 2024 10:31:29 +0100 Subject: [PATCH 4/5] fix missing enum, consistent order --- src/NodeModule.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/NodeModule.cpp b/src/NodeModule.cpp index 0ef783c..12c40b5 100644 --- a/src/NodeModule.cpp +++ b/src/NodeModule.cpp @@ -268,6 +268,8 @@ namespace Napi::PropertyDescriptor::Value("DPMPP2M", Napi::Number::New(env, DPMPP2M)), Napi::PropertyDescriptor::Value("DPMPP2Mv2", Napi::Number::New(env, DPMPP2Mv2)), Napi::PropertyDescriptor::Value("LCM", Napi::Number::New(env, LCM)), + Napi::PropertyDescriptor::Value("IPNDM", Napi::Number::New(env, IPNDM)), + Napi::PropertyDescriptor::Value("IPNDM_V", Napi::Number::New(env, IPNDM_V)), }); sampleMethodEnum.Freeze(); @@ -277,8 +279,8 @@ namespace Napi::PropertyDescriptor::Value("Default", Napi::Number::New(env, DEFAULT)), Napi::PropertyDescriptor::Value("Discrete", Napi::Number::New(env, DISCRETE)), Napi::PropertyDescriptor::Value("Karras", Napi::Number::New(env, KARRAS)), - Napi::PropertyDescriptor::Value("GITS", Napi::Number::New(env, GITS)), Napi::PropertyDescriptor::Value("AYS", Napi::Number::New(env, AYS)), + Napi::PropertyDescriptor::Value("GITS", Napi::Number::New(env, GITS)), }); scheduleEnum.Freeze(); From 93f65d148153f3de48585e127f18f0f51e4a233b Mon Sep 17 00:00:00 2001 From: limez Date: Sun, 27 Oct 2024 17:45:17 +0100 Subject: [PATCH 5/5] build for vulkan backend if its installed, make sure install script doesnt error if cuda_version.json is an empty object --- CMakeLists.txt | 9 +++++---- cudadeps.ts | 4 +++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 30f5147..5fc769c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -30,13 +30,14 @@ FetchContent_Declare( GIT_TAG f9ff166845a59327eda431af82ee85a9c7532c5d ) -if(NOT WIN32) +find_package(Vulkan QUIET) +if(NOT WIN32 AND NOT Vulkan_FOUND) if(EXISTS ${CMAKE_SYSROOT}/usr/lib/x86_64-linux-gnu/libvulkan.so) set(Vulkan_INCLUDE_DIR ${CMAKE_SYSROOT}/usr/include) set(Vulkan_LIBRARY ${CMAKE_SYSROOT}/usr/lib/x86_64-linux-gnu/libvulkan.so) + find_package(Vulkan) endif() endif() -find_package(Vulkan) if(Vulkan_FOUND) set(SD_FLASH_ATTN ON) @@ -44,7 +45,7 @@ if(Vulkan_FOUND) endif() set(IMPLIB_SOURCE_FILES "") -if (CUDAToolkit_FOUND) +if (CUDAToolkit_FOUND AND NOT Vulkan_FOUND) set(SD_CUBLAS ON) set(SD_FLASH_ATTN ON) set(GGML_CUDA_F16 ON) @@ -100,7 +101,7 @@ target_compile_definitions(node-stable-diffusion-cpp PRIVATE NAPI_VERSION=9 NODE target_link_libraries(node-stable-diffusion-cpp ${CMAKE_JS_LIB} stable-diffusion) -if (CUDAToolkit_FOUND) +if (CUDAToolkit_FOUND AND NOT Vulkan_FOUND) file(GENERATE OUTPUT $/cuda_version.json INPUT ${CUDAToolkit_LIBRARY_ROOT}/version.json) else() file(GENERATE OUTPUT $/cuda_version.json CONTENT "{}") diff --git a/cudadeps.ts b/cudadeps.ts index 73e23ee..592d7b4 100644 --- a/cudadeps.ts +++ b/cudadeps.ts @@ -59,8 +59,10 @@ if (fs.existsSync(versionListPath) && process.env.npm_command !== "ci") { const versionList = JSON.parse(fs.readFileSync(versionListPath, { encoding: "utf8" })); const versionListHash = hasher({ sort: true }).hash(versionList) + "_" + cudaSubfolder; const downloadMarkerPath = path.join(resolvedPath, downloadMarkerFile); + const componentCount = Object.keys(versionList).length; + const needsDownload = !fs.existsSync(downloadMarkerPath) || fs.readFileSync(downloadMarkerPath).toString() !== versionListHash; - if (!fs.existsSync(downloadMarkerPath) || fs.readFileSync(downloadMarkerPath).toString() !== versionListHash) { + if (componentCount > 0 && needsDownload) { console.info(`Downloading components ${components} for ${arch} - ${platform}`); for (const componentId of components) {