mirror of
https://github.com/PaddlePaddle/FastDeploy.git
synced 2025-12-24 13:28:13 +08:00
[stable-diffusion] paddle swift+coreml demo (#1494)
* paddle sd * renew ml-stable-diffusion submodule * stable diffusion swift demo
This commit is contained in:
lishicheng1996
8
.gitmodules
vendored
8
.gitmodules
vendored
@@ -4,3 +4,11 @@
|
||||
[submodule "poros/third_party/gflags"]
|
||||
path = poros/third_party/gflags
|
||||
url = https://github.com/gflags/gflags.git
|
||||
[submodule "examples/multimodal/stable_diffusion/swift+coreml/ml-stable-diffusion"]
|
||||
path = examples/multimodal/stable_diffusion/swift+coreml/ml-stable-diffusion
|
||||
url = https://github.com/lishicheng1996/ml-stable-diffusion.git
|
||||
branch = paddle_SD
|
||||
[submodule "examples/multimodal/stable_diffusion/swift+coreml/swift_coreml_diffusers"]
|
||||
path = examples/multimodal/stable_diffusion/swift+coreml/swift_coreml_diffusers
|
||||
url = https://github.com/lishicheng1996/swift-coreml-diffusers.git
|
||||
branch = paddle_stable_diffusion
|
||||
|
||||
35
examples/multimodal/stable_diffusion/swift+coreml/README.md
Normal file
35
examples/multimodal/stable_diffusion/swift+coreml/README.md
Normal file
@@ -0,0 +1,35 @@
|
||||
# swift+coreml stable-diffusion pipeline
|
||||
这里演示如何在苹果芯片上部署从paddle转换的stable-diffusion demo。
|
||||
|
||||
包括swift stable-diffusion pipeline，在ml-stable-diffusion repo中。另包含两个示例。一个是最简单的示例，只支持Macbook，另一个是开源的示例，支持Macbook和iPhone。
|
||||
|
||||
## 1. 准备模型
|
||||
参考mll-stable-diffusion的README.md，将模型转换为coreml格式。也可以直接下载我们已经转换好的[模型](https://ecloud.baidu.com?t=e2808cf7d527f679334538439571f969)
|
||||
|
||||
|
||||
## 2. 运行示例
|
||||
|
||||
### 2.1 最简示例（仅Macbook）
|
||||
|
||||
**Step 1:**
|
||||
打开swift_simplest_demo中的xcodeproj工程。
|
||||
|
||||
**Step 2:**
|
||||
将ml-stable-diffusion拖入。
|
||||
|
||||
**Step 3:**
|
||||
在代码中指定模型路径即可运行。
|
||||
|
||||
### 2.2 开源示例（Macbook + iPhone）
|
||||
|
||||
**Step 1:**
|
||||
打开swift_coreml_diffusers中的xcodeproj工程。
|
||||
|
||||
**Step 2:**
|
||||
将ml-stable-diffusion拖入, 替换掉原有的stable-diffusion package。
|
||||
|
||||
**Step 3:**
|
||||
启动一个简单的服务，便于iPhone用于下载我们之前准备的模型。
|
||||
|
||||
**Step 4:**
|
||||
修改ModelInfo.swift中的URL为我们开启的服务。这样iPhone就可以下载模型了，并且加载运行。
|
||||
Submodule examples/multimodal/stable_diffusion/swift+coreml/ml-stable-diffusion added at 080ea81a63
@@ -0,0 +1,289 @@
|
||||
// !$*UTF8*$!
|
||||
{
|
||||
archiveVersion = 1;
|
||||
classes = {
|
||||
};
|
||||
objectVersion = 56;
|
||||
objects = {
|
||||
|
||||
/* Begin PBXBuildFile section */
|
||||
8F9D6F5129AF59470008BD01 /* main.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8F9D6F5029AF59470008BD01 /* main.swift */; };
|
||||
/* End PBXBuildFile section */
|
||||
|
||||
/* Begin PBXCopyFilesBuildPhase section */
|
||||
8F9D6F4B29AF59470008BD01 /* CopyFiles */ = {
|
||||
isa = PBXCopyFilesBuildPhase;
|
||||
buildActionMask = 2147483647;
|
||||
dstPath = /usr/share/man/man1/;
|
||||
dstSubfolderSpec = 0;
|
||||
files = (
|
||||
);
|
||||
runOnlyForDeploymentPostprocessing = 1;
|
||||
};
|
||||
/* End PBXCopyFilesBuildPhase section */
|
||||
|
||||
/* Begin PBXFileReference section */
|
||||
8F9D6F4D29AF59470008BD01 /* swift_stable_diffusion_demo */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = swift_stable_diffusion_demo; sourceTree = BUILT_PRODUCTS_DIR; };
|
||||
8F9D6F5029AF59470008BD01 /* main.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = main.swift; sourceTree = "<group>"; };
|
||||
/* End PBXFileReference section */
|
||||
|
||||
/* Begin PBXFrameworksBuildPhase section */
|
||||
8F9D6F4A29AF59470008BD01 /* Frameworks */ = {
|
||||
isa = PBXFrameworksBuildPhase;
|
||||
buildActionMask = 2147483647;
|
||||
files = (
|
||||
);
|
||||
runOnlyForDeploymentPostprocessing = 0;
|
||||
};
|
||||
/* End PBXFrameworksBuildPhase section */
|
||||
|
||||
/* Begin PBXGroup section */
|
||||
8F9D6F4429AF59470008BD01 = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
8F9D6F4F29AF59470008BD01 /* swift_stable_diffusion_demo */,
|
||||
8F9D6F4E29AF59470008BD01 /* Products */,
|
||||
);
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
8F9D6F4E29AF59470008BD01 /* Products */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
8F9D6F4D29AF59470008BD01 /* swift_stable_diffusion_demo */,
|
||||
);
|
||||
name = Products;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
8F9D6F4F29AF59470008BD01 /* swift_stable_diffusion_demo */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
8F9D6F5029AF59470008BD01 /* main.swift */,
|
||||
);
|
||||
path = swift_stable_diffusion_demo;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
/* End PBXGroup section */
|
||||
|
||||
/* Begin PBXNativeTarget section */
|
||||
8F9D6F4C29AF59470008BD01 /* swift_stable_diffusion_demo */ = {
|
||||
isa = PBXNativeTarget;
|
||||
buildConfigurationList = 8F9D6F5429AF59470008BD01 /* Build configuration list for PBXNativeTarget "swift_stable_diffusion_demo" */;
|
||||
buildPhases = (
|
||||
8F9D6F4929AF59470008BD01 /* Sources */,
|
||||
8F9D6F4A29AF59470008BD01 /* Frameworks */,
|
||||
8F9D6F4B29AF59470008BD01 /* CopyFiles */,
|
||||
);
|
||||
buildRules = (
|
||||
);
|
||||
dependencies = (
|
||||
);
|
||||
name = swift_stable_diffusion_demo;
|
||||
productName = swift_stable_diffusion_demo;
|
||||
productReference = 8F9D6F4D29AF59470008BD01 /* swift_stable_diffusion_demo */;
|
||||
productType = "com.apple.product-type.tool";
|
||||
};
|
||||
/* End PBXNativeTarget section */
|
||||
|
||||
/* Begin PBXProject section */
|
||||
8F9D6F4529AF59470008BD01 /* Project object */ = {
|
||||
isa = PBXProject;
|
||||
attributes = {
|
||||
BuildIndependentTargetsInParallel = 1;
|
||||
LastSwiftUpdateCheck = 1410;
|
||||
LastUpgradeCheck = 1410;
|
||||
TargetAttributes = {
|
||||
8F9D6F4C29AF59470008BD01 = {
|
||||
CreatedOnToolsVersion = 14.1;
|
||||
};
|
||||
};
|
||||
};
|
||||
buildConfigurationList = 8F9D6F4829AF59470008BD01 /* Build configuration list for PBXProject "swift_stable_diffusion_demo" */;
|
||||
compatibilityVersion = "Xcode 14.0";
|
||||
developmentRegion = en;
|
||||
hasScannedForEncodings = 0;
|
||||
knownRegions = (
|
||||
en,
|
||||
Base,
|
||||
);
|
||||
mainGroup = 8F9D6F4429AF59470008BD01;
|
||||
productRefGroup = 8F9D6F4E29AF59470008BD01 /* Products */;
|
||||
projectDirPath = "";
|
||||
projectRoot = "";
|
||||
targets = (
|
||||
8F9D6F4C29AF59470008BD01 /* swift_stable_diffusion_demo */,
|
||||
);
|
||||
};
|
||||
/* End PBXProject section */
|
||||
|
||||
/* Begin PBXSourcesBuildPhase section */
|
||||
8F9D6F4929AF59470008BD01 /* Sources */ = {
|
||||
isa = PBXSourcesBuildPhase;
|
||||
buildActionMask = 2147483647;
|
||||
files = (
|
||||
8F9D6F5129AF59470008BD01 /* main.swift in Sources */,
|
||||
);
|
||||
runOnlyForDeploymentPostprocessing = 0;
|
||||
};
|
||||
/* End PBXSourcesBuildPhase section */
|
||||
|
||||
/* Begin XCBuildConfiguration section */
|
||||
8F9D6F5229AF59470008BD01 /* Debug */ = {
|
||||
isa = XCBuildConfiguration;
|
||||
buildSettings = {
|
||||
ALWAYS_SEARCH_USER_PATHS = NO;
|
||||
CLANG_ANALYZER_NONNULL = YES;
|
||||
CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
|
||||
CLANG_CXX_LANGUAGE_STANDARD = "gnu++20";
|
||||
CLANG_ENABLE_MODULES = YES;
|
||||
CLANG_ENABLE_OBJC_ARC = YES;
|
||||
CLANG_ENABLE_OBJC_WEAK = YES;
|
||||
CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
|
||||
CLANG_WARN_BOOL_CONVERSION = YES;
|
||||
CLANG_WARN_COMMA = YES;
|
||||
CLANG_WARN_CONSTANT_CONVERSION = YES;
|
||||
CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
|
||||
CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
|
||||
CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
|
||||
CLANG_WARN_EMPTY_BODY = YES;
|
||||
CLANG_WARN_ENUM_CONVERSION = YES;
|
||||
CLANG_WARN_INFINITE_RECURSION = YES;
|
||||
CLANG_WARN_INT_CONVERSION = YES;
|
||||
CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
|
||||
CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
|
||||
CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
|
||||
CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
|
||||
CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;
|
||||
CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
|
||||
CLANG_WARN_STRICT_PROTOTYPES = YES;
|
||||
CLANG_WARN_SUSPICIOUS_MOVE = YES;
|
||||
CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
|
||||
CLANG_WARN_UNREACHABLE_CODE = YES;
|
||||
CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
|
||||
COPY_PHASE_STRIP = NO;
|
||||
DEBUG_INFORMATION_FORMAT = dwarf;
|
||||
ENABLE_STRICT_OBJC_MSGSEND = YES;
|
||||
ENABLE_TESTABILITY = YES;
|
||||
GCC_C_LANGUAGE_STANDARD = gnu11;
|
||||
GCC_DYNAMIC_NO_PIC = NO;
|
||||
GCC_NO_COMMON_BLOCKS = YES;
|
||||
GCC_OPTIMIZATION_LEVEL = 0;
|
||||
GCC_PREPROCESSOR_DEFINITIONS = (
|
||||
"DEBUG=1",
|
||||
"$(inherited)",
|
||||
);
|
||||
GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
|
||||
GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
|
||||
GCC_WARN_UNDECLARED_SELECTOR = YES;
|
||||
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
|
||||
GCC_WARN_UNUSED_FUNCTION = YES;
|
||||
GCC_WARN_UNUSED_VARIABLE = YES;
|
||||
MACOSX_DEPLOYMENT_TARGET = 13.0;
|
||||
MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;
|
||||
MTL_FAST_MATH = YES;
|
||||
ONLY_ACTIVE_ARCH = YES;
|
||||
SDKROOT = macosx;
|
||||
SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG;
|
||||
SWIFT_OPTIMIZATION_LEVEL = "-Onone";
|
||||
};
|
||||
name = Debug;
|
||||
};
|
||||
8F9D6F5329AF59470008BD01 /* Release */ = {
|
||||
isa = XCBuildConfiguration;
|
||||
buildSettings = {
|
||||
ALWAYS_SEARCH_USER_PATHS = NO;
|
||||
CLANG_ANALYZER_NONNULL = YES;
|
||||
CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
|
||||
CLANG_CXX_LANGUAGE_STANDARD = "gnu++20";
|
||||
CLANG_ENABLE_MODULES = YES;
|
||||
CLANG_ENABLE_OBJC_ARC = YES;
|
||||
CLANG_ENABLE_OBJC_WEAK = YES;
|
||||
CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
|
||||
CLANG_WARN_BOOL_CONVERSION = YES;
|
||||
CLANG_WARN_COMMA = YES;
|
||||
CLANG_WARN_CONSTANT_CONVERSION = YES;
|
||||
CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
|
||||
CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
|
||||
CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
|
||||
CLANG_WARN_EMPTY_BODY = YES;
|
||||
CLANG_WARN_ENUM_CONVERSION = YES;
|
||||
CLANG_WARN_INFINITE_RECURSION = YES;
|
||||
CLANG_WARN_INT_CONVERSION = YES;
|
||||
CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
|
||||
CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
|
||||
CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
|
||||
CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
|
||||
CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;
|
||||
CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
|
||||
CLANG_WARN_STRICT_PROTOTYPES = YES;
|
||||
CLANG_WARN_SUSPICIOUS_MOVE = YES;
|
||||
CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
|
||||
CLANG_WARN_UNREACHABLE_CODE = YES;
|
||||
CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
|
||||
COPY_PHASE_STRIP = NO;
|
||||
DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
|
||||
ENABLE_NS_ASSERTIONS = NO;
|
||||
ENABLE_STRICT_OBJC_MSGSEND = YES;
|
||||
GCC_C_LANGUAGE_STANDARD = gnu11;
|
||||
GCC_NO_COMMON_BLOCKS = YES;
|
||||
GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
|
||||
GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
|
||||
GCC_WARN_UNDECLARED_SELECTOR = YES;
|
||||
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
|
||||
GCC_WARN_UNUSED_FUNCTION = YES;
|
||||
GCC_WARN_UNUSED_VARIABLE = YES;
|
||||
MACOSX_DEPLOYMENT_TARGET = 13.0;
|
||||
MTL_ENABLE_DEBUG_INFO = NO;
|
||||
MTL_FAST_MATH = YES;
|
||||
SDKROOT = macosx;
|
||||
SWIFT_COMPILATION_MODE = wholemodule;
|
||||
SWIFT_OPTIMIZATION_LEVEL = "-O";
|
||||
};
|
||||
name = Release;
|
||||
};
|
||||
8F9D6F5529AF59470008BD01 /* Debug */ = {
|
||||
isa = XCBuildConfiguration;
|
||||
buildSettings = {
|
||||
CODE_SIGN_STYLE = Automatic;
|
||||
DEVELOPMENT_TEAM = 3KJ2V67P66;
|
||||
ENABLE_HARDENED_RUNTIME = YES;
|
||||
PRODUCT_NAME = "$(TARGET_NAME)";
|
||||
SWIFT_VERSION = 5.0;
|
||||
};
|
||||
name = Debug;
|
||||
};
|
||||
8F9D6F5629AF59470008BD01 /* Release */ = {
|
||||
isa = XCBuildConfiguration;
|
||||
buildSettings = {
|
||||
CODE_SIGN_STYLE = Automatic;
|
||||
DEVELOPMENT_TEAM = 3KJ2V67P66;
|
||||
ENABLE_HARDENED_RUNTIME = YES;
|
||||
PRODUCT_NAME = "$(TARGET_NAME)";
|
||||
SWIFT_VERSION = 5.0;
|
||||
};
|
||||
name = Release;
|
||||
};
|
||||
/* End XCBuildConfiguration section */
|
||||
|
||||
/* Begin XCConfigurationList section */
|
||||
8F9D6F4829AF59470008BD01 /* Build configuration list for PBXProject "swift_stable_diffusion_demo" */ = {
|
||||
isa = XCConfigurationList;
|
||||
buildConfigurations = (
|
||||
8F9D6F5229AF59470008BD01 /* Debug */,
|
||||
8F9D6F5329AF59470008BD01 /* Release */,
|
||||
);
|
||||
defaultConfigurationIsVisible = 0;
|
||||
defaultConfigurationName = Release;
|
||||
};
|
||||
8F9D6F5429AF59470008BD01 /* Build configuration list for PBXNativeTarget "swift_stable_diffusion_demo" */ = {
|
||||
isa = XCConfigurationList;
|
||||
buildConfigurations = (
|
||||
8F9D6F5529AF59470008BD01 /* Debug */,
|
||||
8F9D6F5629AF59470008BD01 /* Release */,
|
||||
);
|
||||
defaultConfigurationIsVisible = 0;
|
||||
defaultConfigurationName = Release;
|
||||
};
|
||||
/* End XCConfigurationList section */
|
||||
};
|
||||
rootObject = 8F9D6F4529AF59470008BD01 /* Project object */;
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<Workspace
|
||||
version = "1.0">
|
||||
<FileRef
|
||||
location = "self:">
|
||||
</FileRef>
|
||||
</Workspace>
|
||||
@@ -0,0 +1,8 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>IDEDidComputeMac32BitWarning</key>
|
||||
<true/>
|
||||
</dict>
|
||||
</plist>
|
||||
Binary file not shown.
@@ -0,0 +1,14 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>SchemeUserState</key>
|
||||
<dict>
|
||||
<key>swift_stable_diffusion_demo.xcscheme_^#shared#^_</key>
|
||||
<dict>
|
||||
<key>orderHint</key>
|
||||
<integer>0</integer>
|
||||
</dict>
|
||||
</dict>
|
||||
</dict>
|
||||
</plist>
|
||||
@@ -0,0 +1,54 @@
|
||||
//
|
||||
// main.swift
|
||||
// SD_pipeline
|
||||
//
|
||||
// Created by lishicheng03 on 2023/1/19.
|
||||
//
|
||||
|
||||
import Foundation
|
||||
import StableDiffusion
|
||||
import CoreGraphics
|
||||
import ImageIO
|
||||
|
||||
|
||||
import CoreGraphics
|
||||
import CoreML
|
||||
import Foundation
|
||||
import StableDiffusion
|
||||
import UniformTypeIdentifiers
|
||||
|
||||
@discardableResult func writeCGImage(_ image: CGImage, to destinationURL: URL) -> Bool {
|
||||
guard let destination = CGImageDestinationCreateWithURL(destinationURL as CFURL, kUTTypePNG, 1, nil) else { return false }
|
||||
CGImageDestinationAddImage(destination, image, nil)
|
||||
return CGImageDestinationFinalize(destination)
|
||||
}
|
||||
|
||||
|
||||
let seed: Int = 42
|
||||
let prompt: String = "a pig"
|
||||
|
||||
let resourceURL: URL = URL.init(filePath: "/Users/lishicheng03/ml-stable-diffusion/SD2-einsum/Resources")
|
||||
let outURL: URL = URL.init(filePath: "/Users/lishicheng03/ml-stable-diffusion/pics/out_pic.png")
|
||||
|
||||
if #available(macOS 13.1, *) {
|
||||
let config = MLModelConfiguration()
|
||||
let unit : MLComputeUnits = MLComputeUnits.all
|
||||
config.computeUnits = unit
|
||||
|
||||
let pipeline = try StableDiffusionPipeline(resourcesAt: resourceURL, configuration: config)
|
||||
try pipeline.loadResources()
|
||||
|
||||
let startTime = CFAbsoluteTimeGetCurrent()
|
||||
let image : CGImage = try pipeline.generateImages(prompt: prompt, seed: UInt32(seed)).first as! CGImage
|
||||
let endTime = CFAbsoluteTimeGetCurrent()
|
||||
print("代码执行时长：%f 秒", (endTime - startTime))
|
||||
|
||||
try writeCGImage(image, to: outURL)
|
||||
|
||||
print("Mission completed!")
|
||||
} else {
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user