diff --git a/.gitignore b/.gitignore index 8a30d25..2d7278a 100644 --- a/.gitignore +++ b/.gitignore @@ -396,3 +396,6 @@ FodyWeavers.xsd # JetBrains Rider *.sln.iml + +.DS_Store +*.DS_Store \ No newline at end of file diff --git a/README.md b/README.md index 5cd7cec..24d1a87 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,9 @@ -# Project +# ONNXRuntime Swift Package Manager -> This repo has been populated by an initial template to help get you started. Please -> make sure to update the content to build a great experience for community-building. +A light-weight [ONNXRuntime](https://github.com/microsoft/onnxruntime) repository for hosting [Swift Package Manager(SPM)](https://www.swift.org/package-manager/) support. -As the maintainer of this project, please make a few updates: -- Improving this README.MD file to provide a great experience -- Updating SUPPORT.MD with content about this project's support experience -- Understanding the security reporting process in SECURITY.MD -- Remove this section from the README +SPM is the alternative to CocoaPods when desired platform to consume is mobile iOS. ## Contributing diff --git a/tools/ci_build/github/default_full_ios_framework_build_settings.json b/tools/ci_build/github/default_full_ios_framework_build_settings.json new file mode 100644 index 0000000..fd07074 --- /dev/null +++ b/tools/ci_build/github/default_full_ios_framework_build_settings.json @@ -0,0 +1,22 @@ +{ + "build_osx_archs": { + "iphoneos": [ + "arm64" + ], + "iphonesimulator": [ + "arm64", + "x86_64" + ] + }, + "build_params": [ + "--ios", + "--parallel", + "--use_xcode", + "--build_apple_framework", + "--use_coreml", + "--skip_tests", + "--cmake_extra_defines=onnxruntime_BUILD_UNIT_TESTS=OFF", + "--apple_deploy_target=12.0", + "--use_xnnpack" + ] +} diff --git a/tools/ci_build/github/mac-ios-swift-packaging-pipeline.yml b/tools/ci_build/github/mac-ios-swift-packaging-pipeline.yml index 497a245..9921349 100644 --- a/tools/ci_build/github/mac-ios-swift-packaging-pipeline.yml +++ b/tools/ci_build/github/mac-ios-swift-packaging-pipeline.yml @@ -47,7 +47,7 @@ jobs: parameters: xcodeVersion: ${{ variables.xcodeVersion }} - # - template: templates/install-appcenter.yml + - template: templates/install-appcenter.yml - script: | pip install -r tools/ci_build/github/requirements.txt @@ -89,18 +89,18 @@ jobs: # create and test full pods - script: | - python tools/ci_build/github/apple/build_and_assemble_ios_pods.py \ + python tools/ci_build/github/scripts/build_and_assemble_ios_pods.py \ --build-dir "$(Build.BinariesDirectory)/ios_framework_full" \ --staging-dir "$(Build.BinariesDirectory)/staging" \ --pod-version "${ORT_POD_VERSION}" \ --test \ --variant Full \ - --build-settings-file tools/ci_build/github/apple/default_full_ios_framework_build_settings.json \ + --build-settings-file tools/ci_build/github/default_full_ios_framework_build_settings.json \ -b="--path_to_protoc_exe" -b "$(Build.BinariesDirectory)/protobuf_install/bin/protoc" displayName: "[Full] Build iOS framework and assemble pod package files" - script: | - python tools/ci_build/github/apple/test_ios_packages.py \ + python tools/ci_build/github/scripts/test_ios_packages.py \ --fail_if_cocoapods_missing \ --framework_info_file "$(Build.BinariesDirectory)/ios_framework_full/framework_info.json" \ --c_framework_dir "$(Build.BinariesDirectory)/ios_framework_full/framework_out" \ @@ -123,16 +123,16 @@ jobs: workingDirectory: $(Build.BinariesDirectory)/app_center_test_full/ios_package_test/ displayName: '[Full] Build iphone arm64 tests' - # - script: | - # set -e -x - # appcenter test run xcuitest \ - # --app "AI-Frameworks/ORT-Mobile-iOS" \ - # --devices $(app_center_test_devices) \ - # --test-series "master" \ - # --locale "en_US" \ - # --build-dir $(Build.BinariesDirectory)/app_center_test_full/ios_package_test/DerivedData/Build/Products/Debug-iphoneos \ - # --token $(app_center_api_token) - # displayName: "[Full] Run E2E tests on App Center" + - script: | + set -e -x + appcenter test run xcuitest \ + --app "AI-Frameworks/ORT-Mobile-iOS" \ + --devices $(app_center_test_devices) \ + --test-series "master" \ + --locale "en_US" \ + --build-dir $(Build.BinariesDirectory)/app_center_test_full/ios_package_test/DerivedData/Build/Products/Debug-iphoneos \ + --token $(app_center_api_token) + displayName: "[Full] Run E2E tests on App Center" - task: AzureCLI@2 inputs: diff --git a/tools/ci_build/github/scripts/assemble_c_pod_package.py b/tools/ci_build/github/scripts/assemble_c_pod_package.py new file mode 100644 index 0000000..14e7729 --- /dev/null +++ b/tools/ci_build/github/scripts/assemble_c_pod_package.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 + +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import argparse +import pathlib +import shutil +import sys + +_script_dir = pathlib.Path(__file__).parent.resolve(strict=True) +sys.path.append(str(_script_dir.parent)) + + +from package_assembly_utils import ( # noqa: E402 + PackageVariant, + copy_repo_relative_to_dir, + gen_file_from_template, + load_json_config, +) + + +def get_pod_config_file(package_variant: PackageVariant): + """ + Gets the pod configuration file path for the given package variant. + """ + if package_variant == PackageVariant.Full: + return _script_dir / "onnxruntime-c.config.json" + elif package_variant == PackageVariant.Mobile: + return _script_dir / "onnxruntime-mobile-c.config.json" + elif package_variant == PackageVariant.Test: + return _script_dir / "onnxruntime-test-c.config.json" + elif package_variant == PackageVariant.Training: + return _script_dir / "onnxruntime-training-c.config.json" + else: + raise ValueError(f"Unhandled package variant: {package_variant}") + + +def assemble_c_pod_package( + staging_dir: pathlib.Path, + pod_version: str, + framework_info_file: pathlib.Path, + public_headers_dir: pathlib.Path, + framework_dir: pathlib.Path, + package_variant: PackageVariant, +): + """ + Assembles the files for the C/C++ pod package in a staging directory. + + :param staging_dir Path to the staging directory for the C/C++ pod files. + :param pod_version C/C++ pod version. + :param framework_info_file Path to the framework_info.json file containing additional values for the podspec. + :param public_headers_dir Path to the public headers directory to include in the pod. + :param framework_dir Path to the onnxruntime framework directory to include in the pod. + :param package_variant The pod package variant. + :return Tuple of (package name, path to the podspec file). + """ + staging_dir = staging_dir.resolve() + framework_info_file = framework_info_file.resolve(strict=True) + public_headers_dir = public_headers_dir.resolve(strict=True) + framework_dir = framework_dir.resolve(strict=True) + + framework_info = load_json_config(framework_info_file) + pod_config = load_json_config(get_pod_config_file(package_variant)) + + pod_name = pod_config["name"] + + print(f"Assembling files in staging directory: {staging_dir}") + if staging_dir.exists(): + print("Warning: staging directory already exists", file=sys.stderr) + + # copy the necessary files to the staging directory + shutil.copytree(framework_dir, staging_dir / framework_dir.name, dirs_exist_ok=True) + shutil.copytree(public_headers_dir, staging_dir / public_headers_dir.name, dirs_exist_ok=True) + copy_repo_relative_to_dir(["LICENSE"], staging_dir) + + # generate the podspec file from the template + variable_substitutions = { + "DESCRIPTION": pod_config["description"], + "IOS_DEPLOYMENT_TARGET": framework_info["IOS_DEPLOYMENT_TARGET"], + "LICENSE_FILE": "LICENSE", + "NAME": pod_name, + "ORT_C_FRAMEWORK": framework_dir.name, + "ORT_C_HEADERS_DIR": public_headers_dir.name, + "SUMMARY": pod_config["summary"], + "VERSION": pod_version, + "WEAK_FRAMEWORK": framework_info["WEAK_FRAMEWORK"], + } + + podspec_template = _script_dir / "c.podspec.template" + podspec = staging_dir / f"{pod_name}.podspec" + + gen_file_from_template(podspec_template, podspec, variable_substitutions) + + return pod_name, podspec + + +def parse_args(): + parser = argparse.ArgumentParser( + description=""" + Assembles the files for the C/C++ pod package in a staging directory. + This directory can be validated (e.g., with `pod lib lint`) and then zipped to create a package for release. + """ + ) + + parser.add_argument( + "--staging-dir", + type=pathlib.Path, + default=pathlib.Path("./c-staging"), + help="Path to the staging directory for the C/C++ pod files.", + ) + parser.add_argument("--pod-version", required=True, help="C/C++ pod version.") + parser.add_argument( + "--framework-info-file", + type=pathlib.Path, + required=True, + help="Path to the framework_info.json file containing additional values for the podspec. " + "This file should be generated by CMake in the build directory.", + ) + parser.add_argument( + "--public-headers-dir", + type=pathlib.Path, + required=True, + help="Path to the public headers directory to include in the pod.", + ) + parser.add_argument( + "--framework-dir", + type=pathlib.Path, + required=True, + help="Path to the onnxruntime framework directory to include in the pod.", + ) + parser.add_argument( + "--variant", choices=PackageVariant.all_variant_names(), required=True, help="Pod package variant." + ) + + return parser.parse_args() + + +def main(): + args = parse_args() + + assemble_c_pod_package( + staging_dir=args.staging_dir, + pod_version=args.pod_version, + framework_info_file=args.framework_info_file, + public_headers_dir=args.public_headers_dir, + framework_dir=args.framework_dir, + package_variant=PackageVariant[args.variant], + ) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/ci_build/github/scripts/assemble_objc_pod_package.py b/tools/ci_build/github/scripts/assemble_objc_pod_package.py new file mode 100755 index 0000000..709c7ea --- /dev/null +++ b/tools/ci_build/github/scripts/assemble_objc_pod_package.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 + +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import argparse +import pathlib +import sys + +_script_dir = pathlib.Path(__file__).parent.resolve(strict=True) +sys.path.append(str(_script_dir.parent)) + + +from assemble_c_pod_package import get_pod_config_file as get_c_pod_config_file # noqa: E402 +from package_assembly_utils import ( # noqa: E402 + PackageVariant, + copy_repo_relative_to_dir, + filter_files, + gen_file_from_template, + load_json_config, +) + +# these variables contain paths or path patterns that are relative to the repo root + +# the license file +license_file = "LICENSE" + +# include directories for compiling the pod itself +include_dirs = [ + "objectivec", +] + +all_objc_files = { + "source_files": [ + "objectivec/include/*.h", + "objectivec/*.h", + "objectivec/*.m", + "objectivec/*.mm", + ], + "public_header_files": [ + "objectivec/include/*.h", + ], + "test_source_files": [ + "objectivec/test/*.h", + "objectivec/test/*.m", + "objectivec/test/*.mm", + ], + "test_resource_files": [ + "objectivec/test/testdata/*.ort", + "onnxruntime/test/testdata/training_api/*", + ], +} + +training_only_objc_files = { + "source_files": [ + "objectivec/include/onnxruntime_training.h", + "objectivec/include/ort_checkpoint.h", + "objectivec/include/ort_training_session.h", + "objectivec/ort_checkpoint.mm", + "objectivec/ort_checkpoint_internal.h", + "objectivec/ort_training_session_internal.h", + "objectivec/ort_training_session.mm", + ], + "public_header_files": [ + "objectivec/include/ort_checkpoint.h", + "objectivec/include/ort_training_session.h", + "objectivec/include/onnxruntime_training.h", + ], + "test_source_files": [ + "objectivec/test/ort_training_session_test.mm", + "objectivec/test/ort_checkpoint_test.mm", + "objectivec/test/ort_training_utils_test.mm", + ], + "test_resource_files": [ + "onnxruntime/test/testdata/training_api/*", + ], +} + + +def get_pod_files(package_variant: PackageVariant): + """ + Gets the source and header files for the given package variant. + """ + if package_variant == PackageVariant.Training: + return all_objc_files + else: + # return files that are in pod_files but not in training_only_objc_files + filtered_pod_files = {} + for key in all_objc_files: + filtered_pod_files[key] = filter_files(all_objc_files[key], training_only_objc_files[key]) + return filtered_pod_files + + +def get_pod_config_file(package_variant: PackageVariant): + """ + Gets the pod configuration file path for the given package variant. + """ + if package_variant == PackageVariant.Full: + return _script_dir / "onnxruntime-objc.config.json" + elif package_variant == PackageVariant.Mobile: + return _script_dir / "onnxruntime-mobile-objc.config.json" + elif package_variant == PackageVariant.Training: + return _script_dir / "onnxruntime-training-objc.config.json" + else: + raise ValueError(f"Unhandled package variant: {package_variant}") + + +def assemble_objc_pod_package( + staging_dir: pathlib.Path, pod_version: str, framework_info_file: pathlib.Path, package_variant: PackageVariant +): + """ + Assembles the files for the Objective-C pod package in a staging directory. + + :param staging_dir Path to the staging directory for the Objective-C pod files. + :param pod_version Objective-C pod version. + :param framework_info_file Path to the framework_info.json file containing additional values for the podspec. + :param package_variant The pod package variant. + :return Tuple of (package name, path to the podspec file). + """ + staging_dir = staging_dir.resolve() + framework_info_file = framework_info_file.resolve(strict=True) + + framework_info = load_json_config(framework_info_file) + pod_config = load_json_config(get_pod_config_file(package_variant)) + c_pod_config = load_json_config(get_c_pod_config_file(package_variant)) + + pod_name = pod_config["name"] + + print(f"Assembling files in staging directory: {staging_dir}") + if staging_dir.exists(): + print("Warning: staging directory already exists", file=sys.stderr) + + pod_files = get_pod_files(package_variant) + + # copy the necessary files to the staging directory + copy_repo_relative_to_dir( + [license_file, *pod_files["source_files"], *pod_files["test_source_files"], *pod_files["test_resource_files"]], + staging_dir, + ) + + # generate the podspec file from the template + + def path_patterns_as_variable_value(patterns: list[str]): + return ", ".join([f'"{pattern}"' for pattern in patterns]) + + variable_substitutions = { + "C_POD_NAME": c_pod_config["name"], + "DESCRIPTION": pod_config["description"], + "INCLUDE_DIR_LIST": path_patterns_as_variable_value(include_dirs), + "IOS_DEPLOYMENT_TARGET": framework_info["IOS_DEPLOYMENT_TARGET"], + "LICENSE_FILE": license_file, + "NAME": pod_name, + "PUBLIC_HEADER_FILE_LIST": path_patterns_as_variable_value(pod_files["public_header_files"]), + "SOURCE_FILE_LIST": path_patterns_as_variable_value(pod_files["source_files"]), + "SUMMARY": pod_config["summary"], + "TEST_RESOURCE_FILE_LIST": path_patterns_as_variable_value(pod_files["test_resource_files"]), + "TEST_SOURCE_FILE_LIST": path_patterns_as_variable_value(pod_files["test_source_files"]), + "VERSION": pod_version, + } + + podspec_template = _script_dir / "objc.podspec.template" + podspec = staging_dir / f"{pod_name}.podspec" + + gen_file_from_template(podspec_template, podspec, variable_substitutions) + + return pod_name, podspec + + +def parse_args(): + parser = argparse.ArgumentParser( + description=""" + Assembles the files for the Objective-C pod package in a staging directory. + This directory can be validated (e.g., with `pod lib lint`) and then zipped to create a package for release. + """ + ) + + parser.add_argument( + "--staging-dir", + type=pathlib.Path, + default=pathlib.Path("./onnxruntime-mobile-objc-staging"), + help="Path to the staging directory for the Objective-C pod files.", + ) + parser.add_argument("--pod-version", required=True, help="Objective-C pod version.") + parser.add_argument( + "--framework-info-file", + type=pathlib.Path, + required=True, + help="Path to the framework_info.json file containing additional values for the podspec. " + "This file should be generated by CMake in the build directory.", + ) + parser.add_argument( + "--variant", choices=PackageVariant.release_variant_names(), required=True, help="Pod package variant." + ) + + return parser.parse_args() + + +def main(): + args = parse_args() + + assemble_objc_pod_package( + staging_dir=args.staging_dir, + pod_version=args.pod_version, + framework_info_file=args.framework_info_file, + package_variant=PackageVariant[args.variant], + ) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/ci_build/github/scripts/build_and_assemble_ios_pods.py b/tools/ci_build/github/scripts/build_and_assemble_ios_pods.py new file mode 100644 index 0000000..9713283 --- /dev/null +++ b/tools/ci_build/github/scripts/build_and_assemble_ios_pods.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 + +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import argparse +import logging +import pathlib +import shutil +import sys +import tempfile + +from assemble_c_pod_package import assemble_c_pod_package +from assemble_objc_pod_package import assemble_objc_pod_package +from package_assembly_utils import PackageVariant, get_ort_version + +SCRIPT_PATH = pathlib.Path(__file__).resolve() +SCRIPT_DIR = SCRIPT_PATH.parent +REPO_DIR = SCRIPT_PATH.parents[4] + + +logging.basicConfig(format="%(asctime)s %(name)s [%(levelname)s] - %(message)s", level=logging.DEBUG) +log = logging.getLogger(SCRIPT_PATH.stem) + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Builds an iOS framework and uses it to assemble iOS pod package files.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + + parser.add_argument( + "--build-dir", + type=pathlib.Path, + default=REPO_DIR / "build" / "ios_framework", + help="The build directory. This will contain the iOS framework build output.", + ) + parser.add_argument( + "--staging-dir", + type=pathlib.Path, + default=REPO_DIR / "build" / "ios_pod_staging", + help="The staging directory. This will contain the iOS pod package files. " + "The pod package files do not have dependencies on files in the build directory.", + ) + + parser.add_argument( + "--pod-version", + default=f"{get_ort_version()}-local", + help="The version string of the pod. The same version is used for all pods.", + ) + + parser.add_argument( + "--variant", + choices=PackageVariant.release_variant_names(), + default=PackageVariant.Mobile.name, + help="Pod package variant.", + ) + + parser.add_argument("--test", action="store_true", help="Run tests on the framework and pod package files.") + + build_framework_group = parser.add_argument_group( + title="iOS framework build arguments", + description="See the corresponding arguments in build_ios_framework.py for details.", + ) + + build_framework_group.add_argument("--include-ops-by-config") + build_framework_group.add_argument( + "--build-settings-file", required=True, help="The positional argument of build_ios_framework.py." + ) + build_framework_group.add_argument( + "-b", + "--build-ios-framework-arg", + action="append", + dest="build_ios_framework_extra_args", + default=[], + help="Pass an argument through to build_ios_framework.py. This may be specified multiple times.", + ) + + args = parser.parse_args() + + return args + + +def run(arg_list, cwd=None): + import os + import shlex + import subprocess + + log.info( + "Running subprocess in '{}'\n {}".format(cwd or os.getcwd(), " ".join([shlex.quote(arg) for arg in arg_list])) + ) + + return subprocess.run(arg_list, check=True, cwd=cwd) + + +def main(): + args = parse_args() + + build_dir = args.build_dir.resolve() + staging_dir = args.staging_dir.resolve() + + # build framework + package_variant = PackageVariant[args.variant] + framework_info_file = build_dir / "framework_info.json" + + log.info("Building iOS framework.") + + build_ios_framework_args = [ + sys.executable, + str(SCRIPT_DIR / "build_ios_framework.py"), + *args.build_ios_framework_extra_args, + ] + + if args.include_ops_by_config is not None: + build_ios_framework_args += ["--include_ops_by_config", args.include_ops_by_config] + + build_ios_framework_args += ["--build_dir", str(build_dir), args.build_settings_file] + + run(build_ios_framework_args) + + if args.test: + test_ios_packages_args = [ + sys.executable, + str(SCRIPT_DIR / "test_ios_packages.py"), + "--fail_if_cocoapods_missing", + "--framework_info_file", + str(framework_info_file), + "--c_framework_dir", + str(build_dir / "framework_out"), + "--variant", + package_variant.name, + ] + + run(test_ios_packages_args) + + # assemble pods and then move them to their target locations (staging_dir/) + staging_dir.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(dir=staging_dir) as pod_assembly_dir_name: + pod_assembly_dir = pathlib.Path(pod_assembly_dir_name) + + log.info("Assembling C/C++ pod.") + + c_pod_staging_dir = pod_assembly_dir / "c_pod" + c_pod_name, c_pod_podspec = assemble_c_pod_package( + staging_dir=c_pod_staging_dir, + pod_version=args.pod_version, + framework_info_file=framework_info_file, + framework_dir=build_dir / "framework_out" / "onnxruntime.xcframework", + public_headers_dir=build_dir / "framework_out" / "Headers", + package_variant=package_variant, + ) + + if args.test: + test_c_pod_args = ["pod", "lib", "lint", "--verbose"] + + run(test_c_pod_args, cwd=c_pod_staging_dir) + + log.info("Assembling Objective-C pod.") + + objc_pod_staging_dir = pod_assembly_dir / "objc_pod" + objc_pod_name, objc_pod_podspec = assemble_objc_pod_package( + staging_dir=objc_pod_staging_dir, + pod_version=args.pod_version, + framework_info_file=framework_info_file, + package_variant=package_variant, + ) + + if args.test: + test_objc_pod_args = ["pod", "lib", "lint", "--verbose", f"--include-podspecs={c_pod_podspec}"] + + run(test_objc_pod_args, cwd=objc_pod_staging_dir) + + def move_dir(src, dst): + if dst.is_dir(): + shutil.rmtree(dst) + shutil.move(src, dst) + + move_dir(c_pod_staging_dir, staging_dir / c_pod_name) + move_dir(objc_pod_staging_dir, staging_dir / objc_pod_name) + + log.info(f"Successfully assembled iOS pods at '{staging_dir}'.") + + +if __name__ == "__main__": + main() diff --git a/tools/ci_build/github/scripts/package_assembly_utils.py b/tools/ci_build/github/scripts/package_assembly_utils.py new file mode 100644 index 0000000..e594077 --- /dev/null +++ b/tools/ci_build/github/scripts/package_assembly_utils.py @@ -0,0 +1,130 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import enum +import json +import os +import pathlib +import re +import shutil +from typing import Dict, List + +_script_dir = pathlib.Path(__file__).parent.resolve(strict=True) +repo_root = _script_dir.parents[3] + + +class PackageVariant(enum.Enum): + Full = 0 # full ORT build with all opsets, ops, and types + Mobile = 1 # minimal ORT build with reduced ops + Training = 2 # full ORT build with all opsets, ops, and types, plus training APIs + Test = -1 # for testing purposes only + + @classmethod + def release_variant_names(cls): + return [v.name for v in cls if v.value >= 0] + + @classmethod + def all_variant_names(cls): + return [v.name for v in cls] + + +_template_variable_pattern = re.compile(r"@(\w+)@") # match "@var@" + + +def gen_file_from_template( + template_file: pathlib.Path, output_file: pathlib.Path, variable_substitutions: Dict[str, str], strict: bool = True +): + """ + Generates a file from a template file. + The template file may contain template variables that will be substituted + with the provided values in the generated output file. + In the template file, template variable names are delimited by "@"'s, + e.g., "@var@". + + :param template_file The template file path. + :param output_file The generated output file path. + :param variable_substitutions The mapping from template variable name to value. + :param strict Whether to require the set of template variable names in the file and the keys of + `variable_substitutions` to be equal. + """ + with open(template_file) as template: + content = template.read() + + variables_in_file = set() + + def replace_template_variable(match): + variable_name = match.group(1) + variables_in_file.add(variable_name) + return variable_substitutions.get(variable_name, match.group(0)) + + content = _template_variable_pattern.sub(replace_template_variable, content) + + if strict and variables_in_file != variable_substitutions.keys(): + variables_in_substitutions = set(variable_substitutions.keys()) + raise ValueError( + f"Template file variables and substitution variables do not match. " + f"Only in template file: {sorted(variables_in_file - variables_in_substitutions)}. " + f"Only in substitutions: {sorted(variables_in_substitutions - variables_in_file)}." + ) + + with open(output_file, mode="w") as output: + output.write(content) + + +def filter_files(all_file_patterns: List[str], excluded_file_patterns: List[str]): + """ + Filters file paths based on inclusion and exclusion patterns + + :param all_file_patterns The list of file paths to filter. + :param excluded_file_patterns The list of exclusion patterns. + + :return The filtered list of file paths + """ + # get all files matching the patterns in all_file_patterns + all_files = [str(path.relative_to(repo_root)) for pattern in all_file_patterns for path in repo_root.glob(pattern)] + + # get all files matching the patterns in excluded_file_patterns + exclude_files = [ + str(path.relative_to(repo_root)) for pattern in excluded_file_patterns for path in repo_root.glob(pattern) + ] + + # return the difference + return list(set(all_files) - set(exclude_files)) + + +def copy_repo_relative_to_dir(patterns: List[str], dest_dir: pathlib.Path): + """ + Copies file paths relative to the repo root to a directory. + The given paths or path patterns are relative to the repo root, and the + repo root-relative intermediate directory structure is maintained. + + :param patterns The paths or path patterns relative to the repo root. + :param dest_dir The destination directory. + """ + paths = [path for pattern in patterns for path in repo_root.glob(pattern)] + for path in paths: + repo_relative_path = path.relative_to(repo_root) + dst_path = dest_dir / repo_relative_path + os.makedirs(dst_path.parent, exist_ok=True) + shutil.copy(path, dst_path) + + +def load_json_config(json_config_file: pathlib.Path): + """ + Loads configuration info from a JSON file. + + :param json_config_file The JSON configuration file path. + :return The configuration info values. + """ + with open(json_config_file) as config: + return json.load(config) + + +def get_ort_version(): + """ + Gets the ONNX Runtime version string from the repo. + + :return The ONNX Runtime version string. + """ + with open(repo_root / "VERSION_NUMBER") as version_file: + return version_file.read().strip() diff --git a/tools/ci_build/github/scripts/test_ios_packages.py b/tools/ci_build/github/scripts/test_ios_packages.py new file mode 100644 index 0000000..e53a884 --- /dev/null +++ b/tools/ci_build/github/scripts/test_ios_packages.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import argparse +import contextlib +import os +import pathlib +import shutil +import subprocess +import tempfile + +from assemble_c_pod_package import assemble_c_pod_package +from package_assembly_utils import PackageVariant, gen_file_from_template, get_ort_version + +SCRIPT_PATH = pathlib.Path(__file__).resolve(strict=True) +REPO_DIR = SCRIPT_PATH.parents[4] + + +def _test_ios_packages(args): + # check if CocoaPods is installed + if shutil.which("pod") is None: + if args.fail_if_cocoapods_missing: + raise ValueError("CocoaPods is required for this test") + else: + print("CocoaPods is not installed, ignore this test") + return + + # Now we need to create a zip file contains the framework and the podspec file, both of these 2 files + # should be under the c_framework_dir + c_framework_dir = args.c_framework_dir.resolve() + if not c_framework_dir.is_dir(): + raise FileNotFoundError(f"c_framework_dir {c_framework_dir} is not a folder.") + + has_framework = (c_framework_dir / "onnxruntime.framework").exists() + has_xcframework = (c_framework_dir / "onnxruntime.xcframework").exists() + + if not has_framework and not has_xcframework: + raise FileNotFoundError(f"{c_framework_dir} does not have onnxruntime.framework/xcframework") + + if has_framework and has_xcframework: + raise ValueError("Cannot proceed when both onnxruntime.framework and onnxruntime.xcframework exist") + + framework_name = "onnxruntime.framework" if has_framework else "onnxruntime.xcframework" + + # create a temp folder + + with contextlib.ExitStack() as context_stack: + if args.test_project_stage_dir is None: + stage_dir = pathlib.Path(context_stack.enter_context(tempfile.TemporaryDirectory())).resolve() + else: + # If we specify the stage dir, then use it to create test project + stage_dir = args.test_project_stage_dir.resolve() + if os.path.exists(stage_dir): + shutil.rmtree(stage_dir) + os.makedirs(stage_dir) + + # assemble the test project here + target_proj_path = stage_dir / "ios_package_test" + + # copy the test project source files to target_proj_path + test_proj_path = pathlib.Path(REPO_DIR, "onnxruntime/test/platform/ios/ios_package_test") + shutil.copytree(test_proj_path, target_proj_path) + + # assemble local pod files here + local_pods_dir = stage_dir / "local_pods" + + # We will only publish xcframework, however, assembly of the xcframework is a post process + # and it cannot be done by CMake for now. See, https://gitlab.kitware.com/cmake/cmake/-/issues/21752 + # For a single sysroot and arch built by build.py or cmake, we can only generate framework + # We still need a way to test it. framework_dir and public_headers_dir have different values when testing a + # framework and a xcframework. + framework_dir = args.c_framework_dir / framework_name + public_headers_dir = framework_dir / "Headers" if has_framework else args.c_framework_dir / "Headers" + + pod_name, podspec = assemble_c_pod_package( + staging_dir=local_pods_dir, + pod_version=get_ort_version(), + framework_info_file=args.framework_info_file, + public_headers_dir=public_headers_dir, + framework_dir=framework_dir, + package_variant=PackageVariant[args.variant], + ) + + # move podspec out to target_proj_path first + podspec = shutil.move(podspec, target_proj_path / podspec.name) + + # create a zip file contains the framework + zip_file_path = local_pods_dir / f"{pod_name}.zip" + # shutil.make_archive require target file as full path without extension + shutil.make_archive(zip_file_path.with_suffix(""), "zip", root_dir=local_pods_dir) + + # update the podspec to point to the local framework zip file + with open(podspec) as file: + file_data = file.read() + + file_data = file_data.replace("file:///http_source_placeholder", f"file:///{zip_file_path}") + + with open(podspec, "w") as file: + file.write(file_data) + + # generate Podfile to point to pod + gen_file_from_template( + target_proj_path / "Podfile.template", + target_proj_path / "Podfile", + {"C_POD_NAME": pod_name, "C_POD_PODSPEC": f"./{podspec.name}"}, + ) + + # clean the Cocoapods cache first, in case the same pod was cached in previous runs + subprocess.run(["pod", "cache", "clean", "--all"], shell=False, check=True, cwd=target_proj_path) + + # install pods + subprocess.run(["pod", "install"], shell=False, check=True, cwd=target_proj_path) + + # run the tests + if not args.prepare_test_project_only: + simulator_device_name = subprocess.check_output( + ["bash", str(REPO_DIR / "tools" / "ci_build" / "github" / "apple" / "get_simulator_device_name.sh")], + text=True, + ).strip() + + subprocess.run( + [ + "xcrun", + "xcodebuild", + "test", + "-workspace", + "./ios_package_test.xcworkspace", + "-scheme", + "ios_package_test", + "-destination", + f"platform=iOS Simulator,OS=latest,name={simulator_device_name}", + ], + shell=False, + check=True, + cwd=target_proj_path, + ) + + +def parse_args(): + parser = argparse.ArgumentParser( + os.path.basename(__file__), description="Test iOS framework using CocoaPods package." + ) + + parser.add_argument( + "--fail_if_cocoapods_missing", + action="store_true", + help="This script will fail if CocoaPods is not installed, " + "will not throw error unless fail_if_cocoapod_missing is set.", + ) + + parser.add_argument( + "--framework_info_file", + type=pathlib.Path, + required=True, + help="Path to the framework_info.json file containing additional values for the podspec. " + "This file should be generated by CMake in the build directory.", + ) + + parser.add_argument( + "--c_framework_dir", type=pathlib.Path, required=True, help="Provide the parent directory for C/C++ framework" + ) + + parser.add_argument( + "--variant", + choices=PackageVariant.all_variant_names(), + default=PackageVariant.Test.name, + help="Pod package variant.", + ) + + parser.add_argument( + "--test_project_stage_dir", + type=pathlib.Path, + help="The stage dir for the test project, if not specified, will use a temporary path", + ) + + parser.add_argument( + "--prepare_test_project_only", + action="store_true", + help="Prepare the test project only, without running the tests", + ) + + return parser.parse_args() + + +def main(): + args = parse_args() + _test_ios_packages(args) + + +if __name__ == "__main__": + main()