pull more scripts and update readme
This commit is contained in:
rachguo
@@ -0,0 +1,155 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
# Licensed under the MIT License.
|
||||
|
||||
import argparse
|
||||
import pathlib
|
||||
import shutil
|
||||
import sys
|
||||
|
||||
_script_dir = pathlib.Path(__file__).parent.resolve(strict=True)
|
||||
sys.path.append(str(_script_dir.parent))
|
||||
|
||||
|
||||
from package_assembly_utils import ( # noqa: E402
|
||||
PackageVariant,
|
||||
copy_repo_relative_to_dir,
|
||||
gen_file_from_template,
|
||||
load_json_config,
|
||||
)
|
||||
|
||||
|
||||
def get_pod_config_file(package_variant: PackageVariant):
|
||||
"""
|
||||
Gets the pod configuration file path for the given package variant.
|
||||
"""
|
||||
if package_variant == PackageVariant.Full:
|
||||
return _script_dir / "onnxruntime-c.config.json"
|
||||
elif package_variant == PackageVariant.Mobile:
|
||||
return _script_dir / "onnxruntime-mobile-c.config.json"
|
||||
elif package_variant == PackageVariant.Test:
|
||||
return _script_dir / "onnxruntime-test-c.config.json"
|
||||
elif package_variant == PackageVariant.Training:
|
||||
return _script_dir / "onnxruntime-training-c.config.json"
|
||||
else:
|
||||
raise ValueError(f"Unhandled package variant: {package_variant}")
|
||||
|
||||
|
||||
def assemble_c_pod_package(
|
||||
staging_dir: pathlib.Path,
|
||||
pod_version: str,
|
||||
framework_info_file: pathlib.Path,
|
||||
public_headers_dir: pathlib.Path,
|
||||
framework_dir: pathlib.Path,
|
||||
package_variant: PackageVariant,
|
||||
):
|
||||
"""
|
||||
Assembles the files for the C/C++ pod package in a staging directory.
|
||||
|
||||
:param staging_dir Path to the staging directory for the C/C++ pod files.
|
||||
:param pod_version C/C++ pod version.
|
||||
:param framework_info_file Path to the framework_info.json file containing additional values for the podspec.
|
||||
:param public_headers_dir Path to the public headers directory to include in the pod.
|
||||
:param framework_dir Path to the onnxruntime framework directory to include in the pod.
|
||||
:param package_variant The pod package variant.
|
||||
:return Tuple of (package name, path to the podspec file).
|
||||
"""
|
||||
staging_dir = staging_dir.resolve()
|
||||
framework_info_file = framework_info_file.resolve(strict=True)
|
||||
public_headers_dir = public_headers_dir.resolve(strict=True)
|
||||
framework_dir = framework_dir.resolve(strict=True)
|
||||
|
||||
framework_info = load_json_config(framework_info_file)
|
||||
pod_config = load_json_config(get_pod_config_file(package_variant))
|
||||
|
||||
pod_name = pod_config["name"]
|
||||
|
||||
print(f"Assembling files in staging directory: {staging_dir}")
|
||||
if staging_dir.exists():
|
||||
print("Warning: staging directory already exists", file=sys.stderr)
|
||||
|
||||
# copy the necessary files to the staging directory
|
||||
shutil.copytree(framework_dir, staging_dir / framework_dir.name, dirs_exist_ok=True)
|
||||
shutil.copytree(public_headers_dir, staging_dir / public_headers_dir.name, dirs_exist_ok=True)
|
||||
copy_repo_relative_to_dir(["LICENSE"], staging_dir)
|
||||
|
||||
# generate the podspec file from the template
|
||||
variable_substitutions = {
|
||||
"DESCRIPTION": pod_config["description"],
|
||||
"IOS_DEPLOYMENT_TARGET": framework_info["IOS_DEPLOYMENT_TARGET"],
|
||||
"LICENSE_FILE": "LICENSE",
|
||||
"NAME": pod_name,
|
||||
"ORT_C_FRAMEWORK": framework_dir.name,
|
||||
"ORT_C_HEADERS_DIR": public_headers_dir.name,
|
||||
"SUMMARY": pod_config["summary"],
|
||||
"VERSION": pod_version,
|
||||
"WEAK_FRAMEWORK": framework_info["WEAK_FRAMEWORK"],
|
||||
}
|
||||
|
||||
podspec_template = _script_dir / "c.podspec.template"
|
||||
podspec = staging_dir / f"{pod_name}.podspec"
|
||||
|
||||
gen_file_from_template(podspec_template, podspec, variable_substitutions)
|
||||
|
||||
return pod_name, podspec
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="""
|
||||
Assembles the files for the C/C++ pod package in a staging directory.
|
||||
This directory can be validated (e.g., with `pod lib lint`) and then zipped to create a package for release.
|
||||
"""
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--staging-dir",
|
||||
type=pathlib.Path,
|
||||
default=pathlib.Path("./c-staging"),
|
||||
help="Path to the staging directory for the C/C++ pod files.",
|
||||
)
|
||||
parser.add_argument("--pod-version", required=True, help="C/C++ pod version.")
|
||||
parser.add_argument(
|
||||
"--framework-info-file",
|
||||
type=pathlib.Path,
|
||||
required=True,
|
||||
help="Path to the framework_info.json file containing additional values for the podspec. "
|
||||
"This file should be generated by CMake in the build directory.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--public-headers-dir",
|
||||
type=pathlib.Path,
|
||||
required=True,
|
||||
help="Path to the public headers directory to include in the pod.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--framework-dir",
|
||||
type=pathlib.Path,
|
||||
required=True,
|
||||
help="Path to the onnxruntime framework directory to include in the pod.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--variant", choices=PackageVariant.all_variant_names(), required=True, help="Pod package variant."
|
||||
)
|
||||
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
|
||||
assemble_c_pod_package(
|
||||
staging_dir=args.staging_dir,
|
||||
pod_version=args.pod_version,
|
||||
framework_info_file=args.framework_info_file,
|
||||
public_headers_dir=args.public_headers_dir,
|
||||
framework_dir=args.framework_dir,
|
||||
package_variant=PackageVariant[args.variant],
|
||||
)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
+212
@@ -0,0 +1,212 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
# Licensed under the MIT License.
|
||||
|
||||
import argparse
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
_script_dir = pathlib.Path(__file__).parent.resolve(strict=True)
|
||||
sys.path.append(str(_script_dir.parent))
|
||||
|
||||
|
||||
from assemble_c_pod_package import get_pod_config_file as get_c_pod_config_file # noqa: E402
|
||||
from package_assembly_utils import ( # noqa: E402
|
||||
PackageVariant,
|
||||
copy_repo_relative_to_dir,
|
||||
filter_files,
|
||||
gen_file_from_template,
|
||||
load_json_config,
|
||||
)
|
||||
|
||||
# these variables contain paths or path patterns that are relative to the repo root
|
||||
|
||||
# the license file
|
||||
license_file = "LICENSE"
|
||||
|
||||
# include directories for compiling the pod itself
|
||||
include_dirs = [
|
||||
"objectivec",
|
||||
]
|
||||
|
||||
all_objc_files = {
|
||||
"source_files": [
|
||||
"objectivec/include/*.h",
|
||||
"objectivec/*.h",
|
||||
"objectivec/*.m",
|
||||
"objectivec/*.mm",
|
||||
],
|
||||
"public_header_files": [
|
||||
"objectivec/include/*.h",
|
||||
],
|
||||
"test_source_files": [
|
||||
"objectivec/test/*.h",
|
||||
"objectivec/test/*.m",
|
||||
"objectivec/test/*.mm",
|
||||
],
|
||||
"test_resource_files": [
|
||||
"objectivec/test/testdata/*.ort",
|
||||
"onnxruntime/test/testdata/training_api/*",
|
||||
],
|
||||
}
|
||||
|
||||
training_only_objc_files = {
|
||||
"source_files": [
|
||||
"objectivec/include/onnxruntime_training.h",
|
||||
"objectivec/include/ort_checkpoint.h",
|
||||
"objectivec/include/ort_training_session.h",
|
||||
"objectivec/ort_checkpoint.mm",
|
||||
"objectivec/ort_checkpoint_internal.h",
|
||||
"objectivec/ort_training_session_internal.h",
|
||||
"objectivec/ort_training_session.mm",
|
||||
],
|
||||
"public_header_files": [
|
||||
"objectivec/include/ort_checkpoint.h",
|
||||
"objectivec/include/ort_training_session.h",
|
||||
"objectivec/include/onnxruntime_training.h",
|
||||
],
|
||||
"test_source_files": [
|
||||
"objectivec/test/ort_training_session_test.mm",
|
||||
"objectivec/test/ort_checkpoint_test.mm",
|
||||
"objectivec/test/ort_training_utils_test.mm",
|
||||
],
|
||||
"test_resource_files": [
|
||||
"onnxruntime/test/testdata/training_api/*",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def get_pod_files(package_variant: PackageVariant):
|
||||
"""
|
||||
Gets the source and header files for the given package variant.
|
||||
"""
|
||||
if package_variant == PackageVariant.Training:
|
||||
return all_objc_files
|
||||
else:
|
||||
# return files that are in pod_files but not in training_only_objc_files
|
||||
filtered_pod_files = {}
|
||||
for key in all_objc_files:
|
||||
filtered_pod_files[key] = filter_files(all_objc_files[key], training_only_objc_files[key])
|
||||
return filtered_pod_files
|
||||
|
||||
|
||||
def get_pod_config_file(package_variant: PackageVariant):
|
||||
"""
|
||||
Gets the pod configuration file path for the given package variant.
|
||||
"""
|
||||
if package_variant == PackageVariant.Full:
|
||||
return _script_dir / "onnxruntime-objc.config.json"
|
||||
elif package_variant == PackageVariant.Mobile:
|
||||
return _script_dir / "onnxruntime-mobile-objc.config.json"
|
||||
elif package_variant == PackageVariant.Training:
|
||||
return _script_dir / "onnxruntime-training-objc.config.json"
|
||||
else:
|
||||
raise ValueError(f"Unhandled package variant: {package_variant}")
|
||||
|
||||
|
||||
def assemble_objc_pod_package(
|
||||
staging_dir: pathlib.Path, pod_version: str, framework_info_file: pathlib.Path, package_variant: PackageVariant
|
||||
):
|
||||
"""
|
||||
Assembles the files for the Objective-C pod package in a staging directory.
|
||||
|
||||
:param staging_dir Path to the staging directory for the Objective-C pod files.
|
||||
:param pod_version Objective-C pod version.
|
||||
:param framework_info_file Path to the framework_info.json file containing additional values for the podspec.
|
||||
:param package_variant The pod package variant.
|
||||
:return Tuple of (package name, path to the podspec file).
|
||||
"""
|
||||
staging_dir = staging_dir.resolve()
|
||||
framework_info_file = framework_info_file.resolve(strict=True)
|
||||
|
||||
framework_info = load_json_config(framework_info_file)
|
||||
pod_config = load_json_config(get_pod_config_file(package_variant))
|
||||
c_pod_config = load_json_config(get_c_pod_config_file(package_variant))
|
||||
|
||||
pod_name = pod_config["name"]
|
||||
|
||||
print(f"Assembling files in staging directory: {staging_dir}")
|
||||
if staging_dir.exists():
|
||||
print("Warning: staging directory already exists", file=sys.stderr)
|
||||
|
||||
pod_files = get_pod_files(package_variant)
|
||||
|
||||
# copy the necessary files to the staging directory
|
||||
copy_repo_relative_to_dir(
|
||||
[license_file, *pod_files["source_files"], *pod_files["test_source_files"], *pod_files["test_resource_files"]],
|
||||
staging_dir,
|
||||
)
|
||||
|
||||
# generate the podspec file from the template
|
||||
|
||||
def path_patterns_as_variable_value(patterns: list[str]):
|
||||
return ", ".join([f'"{pattern}"' for pattern in patterns])
|
||||
|
||||
variable_substitutions = {
|
||||
"C_POD_NAME": c_pod_config["name"],
|
||||
"DESCRIPTION": pod_config["description"],
|
||||
"INCLUDE_DIR_LIST": path_patterns_as_variable_value(include_dirs),
|
||||
"IOS_DEPLOYMENT_TARGET": framework_info["IOS_DEPLOYMENT_TARGET"],
|
||||
"LICENSE_FILE": license_file,
|
||||
"NAME": pod_name,
|
||||
"PUBLIC_HEADER_FILE_LIST": path_patterns_as_variable_value(pod_files["public_header_files"]),
|
||||
"SOURCE_FILE_LIST": path_patterns_as_variable_value(pod_files["source_files"]),
|
||||
"SUMMARY": pod_config["summary"],
|
||||
"TEST_RESOURCE_FILE_LIST": path_patterns_as_variable_value(pod_files["test_resource_files"]),
|
||||
"TEST_SOURCE_FILE_LIST": path_patterns_as_variable_value(pod_files["test_source_files"]),
|
||||
"VERSION": pod_version,
|
||||
}
|
||||
|
||||
podspec_template = _script_dir / "objc.podspec.template"
|
||||
podspec = staging_dir / f"{pod_name}.podspec"
|
||||
|
||||
gen_file_from_template(podspec_template, podspec, variable_substitutions)
|
||||
|
||||
return pod_name, podspec
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="""
|
||||
Assembles the files for the Objective-C pod package in a staging directory.
|
||||
This directory can be validated (e.g., with `pod lib lint`) and then zipped to create a package for release.
|
||||
"""
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--staging-dir",
|
||||
type=pathlib.Path,
|
||||
default=pathlib.Path("./onnxruntime-mobile-objc-staging"),
|
||||
help="Path to the staging directory for the Objective-C pod files.",
|
||||
)
|
||||
parser.add_argument("--pod-version", required=True, help="Objective-C pod version.")
|
||||
parser.add_argument(
|
||||
"--framework-info-file",
|
||||
type=pathlib.Path,
|
||||
required=True,
|
||||
help="Path to the framework_info.json file containing additional values for the podspec. "
|
||||
"This file should be generated by CMake in the build directory.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--variant", choices=PackageVariant.release_variant_names(), required=True, help="Pod package variant."
|
||||
)
|
||||
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
|
||||
assemble_objc_pod_package(
|
||||
staging_dir=args.staging_dir,
|
||||
pod_version=args.pod_version,
|
||||
framework_info_file=args.framework_info_file,
|
||||
package_variant=PackageVariant[args.variant],
|
||||
)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,185 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
# Licensed under the MIT License.
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import pathlib
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
from assemble_c_pod_package import assemble_c_pod_package
|
||||
from assemble_objc_pod_package import assemble_objc_pod_package
|
||||
from package_assembly_utils import PackageVariant, get_ort_version
|
||||
|
||||
SCRIPT_PATH = pathlib.Path(__file__).resolve()
|
||||
SCRIPT_DIR = SCRIPT_PATH.parent
|
||||
REPO_DIR = SCRIPT_PATH.parents[4]
|
||||
|
||||
|
||||
logging.basicConfig(format="%(asctime)s %(name)s [%(levelname)s] - %(message)s", level=logging.DEBUG)
|
||||
log = logging.getLogger(SCRIPT_PATH.stem)
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Builds an iOS framework and uses it to assemble iOS pod package files.",
|
||||
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--build-dir",
|
||||
type=pathlib.Path,
|
||||
default=REPO_DIR / "build" / "ios_framework",
|
||||
help="The build directory. This will contain the iOS framework build output.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--staging-dir",
|
||||
type=pathlib.Path,
|
||||
default=REPO_DIR / "build" / "ios_pod_staging",
|
||||
help="The staging directory. This will contain the iOS pod package files. "
|
||||
"The pod package files do not have dependencies on files in the build directory.",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--pod-version",
|
||||
default=f"{get_ort_version()}-local",
|
||||
help="The version string of the pod. The same version is used for all pods.",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--variant",
|
||||
choices=PackageVariant.release_variant_names(),
|
||||
default=PackageVariant.Mobile.name,
|
||||
help="Pod package variant.",
|
||||
)
|
||||
|
||||
parser.add_argument("--test", action="store_true", help="Run tests on the framework and pod package files.")
|
||||
|
||||
build_framework_group = parser.add_argument_group(
|
||||
title="iOS framework build arguments",
|
||||
description="See the corresponding arguments in build_ios_framework.py for details.",
|
||||
)
|
||||
|
||||
build_framework_group.add_argument("--include-ops-by-config")
|
||||
build_framework_group.add_argument(
|
||||
"--build-settings-file", required=True, help="The positional argument of build_ios_framework.py."
|
||||
)
|
||||
build_framework_group.add_argument(
|
||||
"-b",
|
||||
"--build-ios-framework-arg",
|
||||
action="append",
|
||||
dest="build_ios_framework_extra_args",
|
||||
default=[],
|
||||
help="Pass an argument through to build_ios_framework.py. This may be specified multiple times.",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
return args
|
||||
|
||||
|
||||
def run(arg_list, cwd=None):
|
||||
import os
|
||||
import shlex
|
||||
import subprocess
|
||||
|
||||
log.info(
|
||||
"Running subprocess in '{}'\n {}".format(cwd or os.getcwd(), " ".join([shlex.quote(arg) for arg in arg_list]))
|
||||
)
|
||||
|
||||
return subprocess.run(arg_list, check=True, cwd=cwd)
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
|
||||
build_dir = args.build_dir.resolve()
|
||||
staging_dir = args.staging_dir.resolve()
|
||||
|
||||
# build framework
|
||||
package_variant = PackageVariant[args.variant]
|
||||
framework_info_file = build_dir / "framework_info.json"
|
||||
|
||||
log.info("Building iOS framework.")
|
||||
|
||||
build_ios_framework_args = [
|
||||
sys.executable,
|
||||
str(SCRIPT_DIR / "build_ios_framework.py"),
|
||||
*args.build_ios_framework_extra_args,
|
||||
]
|
||||
|
||||
if args.include_ops_by_config is not None:
|
||||
build_ios_framework_args += ["--include_ops_by_config", args.include_ops_by_config]
|
||||
|
||||
build_ios_framework_args += ["--build_dir", str(build_dir), args.build_settings_file]
|
||||
|
||||
run(build_ios_framework_args)
|
||||
|
||||
if args.test:
|
||||
test_ios_packages_args = [
|
||||
sys.executable,
|
||||
str(SCRIPT_DIR / "test_ios_packages.py"),
|
||||
"--fail_if_cocoapods_missing",
|
||||
"--framework_info_file",
|
||||
str(framework_info_file),
|
||||
"--c_framework_dir",
|
||||
str(build_dir / "framework_out"),
|
||||
"--variant",
|
||||
package_variant.name,
|
||||
]
|
||||
|
||||
run(test_ios_packages_args)
|
||||
|
||||
# assemble pods and then move them to their target locations (staging_dir/<pod_name>)
|
||||
staging_dir.mkdir(parents=True, exist_ok=True)
|
||||
with tempfile.TemporaryDirectory(dir=staging_dir) as pod_assembly_dir_name:
|
||||
pod_assembly_dir = pathlib.Path(pod_assembly_dir_name)
|
||||
|
||||
log.info("Assembling C/C++ pod.")
|
||||
|
||||
c_pod_staging_dir = pod_assembly_dir / "c_pod"
|
||||
c_pod_name, c_pod_podspec = assemble_c_pod_package(
|
||||
staging_dir=c_pod_staging_dir,
|
||||
pod_version=args.pod_version,
|
||||
framework_info_file=framework_info_file,
|
||||
framework_dir=build_dir / "framework_out" / "onnxruntime.xcframework",
|
||||
public_headers_dir=build_dir / "framework_out" / "Headers",
|
||||
package_variant=package_variant,
|
||||
)
|
||||
|
||||
if args.test:
|
||||
test_c_pod_args = ["pod", "lib", "lint", "--verbose"]
|
||||
|
||||
run(test_c_pod_args, cwd=c_pod_staging_dir)
|
||||
|
||||
log.info("Assembling Objective-C pod.")
|
||||
|
||||
objc_pod_staging_dir = pod_assembly_dir / "objc_pod"
|
||||
objc_pod_name, objc_pod_podspec = assemble_objc_pod_package(
|
||||
staging_dir=objc_pod_staging_dir,
|
||||
pod_version=args.pod_version,
|
||||
framework_info_file=framework_info_file,
|
||||
package_variant=package_variant,
|
||||
)
|
||||
|
||||
if args.test:
|
||||
test_objc_pod_args = ["pod", "lib", "lint", "--verbose", f"--include-podspecs={c_pod_podspec}"]
|
||||
|
||||
run(test_objc_pod_args, cwd=objc_pod_staging_dir)
|
||||
|
||||
def move_dir(src, dst):
|
||||
if dst.is_dir():
|
||||
shutil.rmtree(dst)
|
||||
shutil.move(src, dst)
|
||||
|
||||
move_dir(c_pod_staging_dir, staging_dir / c_pod_name)
|
||||
move_dir(objc_pod_staging_dir, staging_dir / objc_pod_name)
|
||||
|
||||
log.info(f"Successfully assembled iOS pods at '{staging_dir}'.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,130 @@
|
||||
# Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
# Licensed under the MIT License.
|
||||
|
||||
import enum
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import shutil
|
||||
from typing import Dict, List
|
||||
|
||||
_script_dir = pathlib.Path(__file__).parent.resolve(strict=True)
|
||||
repo_root = _script_dir.parents[3]
|
||||
|
||||
|
||||
class PackageVariant(enum.Enum):
|
||||
Full = 0 # full ORT build with all opsets, ops, and types
|
||||
Mobile = 1 # minimal ORT build with reduced ops
|
||||
Training = 2 # full ORT build with all opsets, ops, and types, plus training APIs
|
||||
Test = -1 # for testing purposes only
|
||||
|
||||
@classmethod
|
||||
def release_variant_names(cls):
|
||||
return [v.name for v in cls if v.value >= 0]
|
||||
|
||||
@classmethod
|
||||
def all_variant_names(cls):
|
||||
return [v.name for v in cls]
|
||||
|
||||
|
||||
_template_variable_pattern = re.compile(r"@(\w+)@") # match "@var@"
|
||||
|
||||
|
||||
def gen_file_from_template(
|
||||
template_file: pathlib.Path, output_file: pathlib.Path, variable_substitutions: Dict[str, str], strict: bool = True
|
||||
):
|
||||
"""
|
||||
Generates a file from a template file.
|
||||
The template file may contain template variables that will be substituted
|
||||
with the provided values in the generated output file.
|
||||
In the template file, template variable names are delimited by "@"'s,
|
||||
e.g., "@var@".
|
||||
|
||||
:param template_file The template file path.
|
||||
:param output_file The generated output file path.
|
||||
:param variable_substitutions The mapping from template variable name to value.
|
||||
:param strict Whether to require the set of template variable names in the file and the keys of
|
||||
`variable_substitutions` to be equal.
|
||||
"""
|
||||
with open(template_file) as template:
|
||||
content = template.read()
|
||||
|
||||
variables_in_file = set()
|
||||
|
||||
def replace_template_variable(match):
|
||||
variable_name = match.group(1)
|
||||
variables_in_file.add(variable_name)
|
||||
return variable_substitutions.get(variable_name, match.group(0))
|
||||
|
||||
content = _template_variable_pattern.sub(replace_template_variable, content)
|
||||
|
||||
if strict and variables_in_file != variable_substitutions.keys():
|
||||
variables_in_substitutions = set(variable_substitutions.keys())
|
||||
raise ValueError(
|
||||
f"Template file variables and substitution variables do not match. "
|
||||
f"Only in template file: {sorted(variables_in_file - variables_in_substitutions)}. "
|
||||
f"Only in substitutions: {sorted(variables_in_substitutions - variables_in_file)}."
|
||||
)
|
||||
|
||||
with open(output_file, mode="w") as output:
|
||||
output.write(content)
|
||||
|
||||
|
||||
def filter_files(all_file_patterns: List[str], excluded_file_patterns: List[str]):
|
||||
"""
|
||||
Filters file paths based on inclusion and exclusion patterns
|
||||
|
||||
:param all_file_patterns The list of file paths to filter.
|
||||
:param excluded_file_patterns The list of exclusion patterns.
|
||||
|
||||
:return The filtered list of file paths
|
||||
"""
|
||||
# get all files matching the patterns in all_file_patterns
|
||||
all_files = [str(path.relative_to(repo_root)) for pattern in all_file_patterns for path in repo_root.glob(pattern)]
|
||||
|
||||
# get all files matching the patterns in excluded_file_patterns
|
||||
exclude_files = [
|
||||
str(path.relative_to(repo_root)) for pattern in excluded_file_patterns for path in repo_root.glob(pattern)
|
||||
]
|
||||
|
||||
# return the difference
|
||||
return list(set(all_files) - set(exclude_files))
|
||||
|
||||
|
||||
def copy_repo_relative_to_dir(patterns: List[str], dest_dir: pathlib.Path):
|
||||
"""
|
||||
Copies file paths relative to the repo root to a directory.
|
||||
The given paths or path patterns are relative to the repo root, and the
|
||||
repo root-relative intermediate directory structure is maintained.
|
||||
|
||||
:param patterns The paths or path patterns relative to the repo root.
|
||||
:param dest_dir The destination directory.
|
||||
"""
|
||||
paths = [path for pattern in patterns for path in repo_root.glob(pattern)]
|
||||
for path in paths:
|
||||
repo_relative_path = path.relative_to(repo_root)
|
||||
dst_path = dest_dir / repo_relative_path
|
||||
os.makedirs(dst_path.parent, exist_ok=True)
|
||||
shutil.copy(path, dst_path)
|
||||
|
||||
|
||||
def load_json_config(json_config_file: pathlib.Path):
|
||||
"""
|
||||
Loads configuration info from a JSON file.
|
||||
|
||||
:param json_config_file The JSON configuration file path.
|
||||
:return The configuration info values.
|
||||
"""
|
||||
with open(json_config_file) as config:
|
||||
return json.load(config)
|
||||
|
||||
|
||||
def get_ort_version():
|
||||
"""
|
||||
Gets the ONNX Runtime version string from the repo.
|
||||
|
||||
:return The ONNX Runtime version string.
|
||||
"""
|
||||
with open(repo_root / "VERSION_NUMBER") as version_file:
|
||||
return version_file.read().strip()
|
||||
@@ -0,0 +1,192 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
# Licensed under the MIT License.
|
||||
|
||||
import argparse
|
||||
import contextlib
|
||||
import os
|
||||
import pathlib
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
from assemble_c_pod_package import assemble_c_pod_package
|
||||
from package_assembly_utils import PackageVariant, gen_file_from_template, get_ort_version
|
||||
|
||||
SCRIPT_PATH = pathlib.Path(__file__).resolve(strict=True)
|
||||
REPO_DIR = SCRIPT_PATH.parents[4]
|
||||
|
||||
|
||||
def _test_ios_packages(args):
|
||||
# check if CocoaPods is installed
|
||||
if shutil.which("pod") is None:
|
||||
if args.fail_if_cocoapods_missing:
|
||||
raise ValueError("CocoaPods is required for this test")
|
||||
else:
|
||||
print("CocoaPods is not installed, ignore this test")
|
||||
return
|
||||
|
||||
# Now we need to create a zip file contains the framework and the podspec file, both of these 2 files
|
||||
# should be under the c_framework_dir
|
||||
c_framework_dir = args.c_framework_dir.resolve()
|
||||
if not c_framework_dir.is_dir():
|
||||
raise FileNotFoundError(f"c_framework_dir {c_framework_dir} is not a folder.")
|
||||
|
||||
has_framework = (c_framework_dir / "onnxruntime.framework").exists()
|
||||
has_xcframework = (c_framework_dir / "onnxruntime.xcframework").exists()
|
||||
|
||||
if not has_framework and not has_xcframework:
|
||||
raise FileNotFoundError(f"{c_framework_dir} does not have onnxruntime.framework/xcframework")
|
||||
|
||||
if has_framework and has_xcframework:
|
||||
raise ValueError("Cannot proceed when both onnxruntime.framework and onnxruntime.xcframework exist")
|
||||
|
||||
framework_name = "onnxruntime.framework" if has_framework else "onnxruntime.xcframework"
|
||||
|
||||
# create a temp folder
|
||||
|
||||
with contextlib.ExitStack() as context_stack:
|
||||
if args.test_project_stage_dir is None:
|
||||
stage_dir = pathlib.Path(context_stack.enter_context(tempfile.TemporaryDirectory())).resolve()
|
||||
else:
|
||||
# If we specify the stage dir, then use it to create test project
|
||||
stage_dir = args.test_project_stage_dir.resolve()
|
||||
if os.path.exists(stage_dir):
|
||||
shutil.rmtree(stage_dir)
|
||||
os.makedirs(stage_dir)
|
||||
|
||||
# assemble the test project here
|
||||
target_proj_path = stage_dir / "ios_package_test"
|
||||
|
||||
# copy the test project source files to target_proj_path
|
||||
test_proj_path = pathlib.Path(REPO_DIR, "onnxruntime/test/platform/ios/ios_package_test")
|
||||
shutil.copytree(test_proj_path, target_proj_path)
|
||||
|
||||
# assemble local pod files here
|
||||
local_pods_dir = stage_dir / "local_pods"
|
||||
|
||||
# We will only publish xcframework, however, assembly of the xcframework is a post process
|
||||
# and it cannot be done by CMake for now. See, https://gitlab.kitware.com/cmake/cmake/-/issues/21752
|
||||
# For a single sysroot and arch built by build.py or cmake, we can only generate framework
|
||||
# We still need a way to test it. framework_dir and public_headers_dir have different values when testing a
|
||||
# framework and a xcframework.
|
||||
framework_dir = args.c_framework_dir / framework_name
|
||||
public_headers_dir = framework_dir / "Headers" if has_framework else args.c_framework_dir / "Headers"
|
||||
|
||||
pod_name, podspec = assemble_c_pod_package(
|
||||
staging_dir=local_pods_dir,
|
||||
pod_version=get_ort_version(),
|
||||
framework_info_file=args.framework_info_file,
|
||||
public_headers_dir=public_headers_dir,
|
||||
framework_dir=framework_dir,
|
||||
package_variant=PackageVariant[args.variant],
|
||||
)
|
||||
|
||||
# move podspec out to target_proj_path first
|
||||
podspec = shutil.move(podspec, target_proj_path / podspec.name)
|
||||
|
||||
# create a zip file contains the framework
|
||||
zip_file_path = local_pods_dir / f"{pod_name}.zip"
|
||||
# shutil.make_archive require target file as full path without extension
|
||||
shutil.make_archive(zip_file_path.with_suffix(""), "zip", root_dir=local_pods_dir)
|
||||
|
||||
# update the podspec to point to the local framework zip file
|
||||
with open(podspec) as file:
|
||||
file_data = file.read()
|
||||
|
||||
file_data = file_data.replace("file:///http_source_placeholder", f"file:///{zip_file_path}")
|
||||
|
||||
with open(podspec, "w") as file:
|
||||
file.write(file_data)
|
||||
|
||||
# generate Podfile to point to pod
|
||||
gen_file_from_template(
|
||||
target_proj_path / "Podfile.template",
|
||||
target_proj_path / "Podfile",
|
||||
{"C_POD_NAME": pod_name, "C_POD_PODSPEC": f"./{podspec.name}"},
|
||||
)
|
||||
|
||||
# clean the Cocoapods cache first, in case the same pod was cached in previous runs
|
||||
subprocess.run(["pod", "cache", "clean", "--all"], shell=False, check=True, cwd=target_proj_path)
|
||||
|
||||
# install pods
|
||||
subprocess.run(["pod", "install"], shell=False, check=True, cwd=target_proj_path)
|
||||
|
||||
# run the tests
|
||||
if not args.prepare_test_project_only:
|
||||
simulator_device_name = subprocess.check_output(
|
||||
["bash", str(REPO_DIR / "tools" / "ci_build" / "github" / "apple" / "get_simulator_device_name.sh")],
|
||||
text=True,
|
||||
).strip()
|
||||
|
||||
subprocess.run(
|
||||
[
|
||||
"xcrun",
|
||||
"xcodebuild",
|
||||
"test",
|
||||
"-workspace",
|
||||
"./ios_package_test.xcworkspace",
|
||||
"-scheme",
|
||||
"ios_package_test",
|
||||
"-destination",
|
||||
f"platform=iOS Simulator,OS=latest,name={simulator_device_name}",
|
||||
],
|
||||
shell=False,
|
||||
check=True,
|
||||
cwd=target_proj_path,
|
||||
)
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(
|
||||
os.path.basename(__file__), description="Test iOS framework using CocoaPods package."
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--fail_if_cocoapods_missing",
|
||||
action="store_true",
|
||||
help="This script will fail if CocoaPods is not installed, "
|
||||
"will not throw error unless fail_if_cocoapod_missing is set.",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--framework_info_file",
|
||||
type=pathlib.Path,
|
||||
required=True,
|
||||
help="Path to the framework_info.json file containing additional values for the podspec. "
|
||||
"This file should be generated by CMake in the build directory.",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--c_framework_dir", type=pathlib.Path, required=True, help="Provide the parent directory for C/C++ framework"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--variant",
|
||||
choices=PackageVariant.all_variant_names(),
|
||||
default=PackageVariant.Test.name,
|
||||
help="Pod package variant.",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--test_project_stage_dir",
|
||||
type=pathlib.Path,
|
||||
help="The stage dir for the test project, if not specified, will use a temporary path",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--prepare_test_project_only",
|
||||
action="store_true",
|
||||
help="Prepare the test project only, without running the tests",
|
||||
)
|
||||
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
_test_ios_packages(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user