remove scripts files not needed any more

This commit is contained in:
rachguo
2023-07-14 12:01:13 -07:00
parent 25ef7acc60
commit 6c53f44d9e
12 changed files with 0 additions and 1012 deletions
-44
View File
@@ -1,44 +0,0 @@
#Name;Url;SHA1
#This is a CSV file.
#The columns are separated by ";" because a list in cmake is just a ";" separated group of strings.
#Names should be in lower case. They will be used as variable names in cmake.
#URLs can be either https URLs or local file paths in cmake-style(directory separator is a forward slash character).
#SHA1 hashes can be generated by running sha1sum command.
#If you need to change abseil's version to a different one, you may also want to update external\abseil-cpp.natvis
#since the file contains a version string: "lts_20220623". However, the file is for debugging purposes only and would
#not affect built binaries.
abseil_cpp;https://github.com/abseil/abseil-cpp/archive/refs/tags/20220623.1.zip;50c137c88965cba015dfcc8fd5d9b46d23146751
cxxopts;https://github.com/jarro2783/cxxopts/archive/3c73d91c0b04e2b59462f0a741be8c07024c1bc0.zip;6c6ca7f8480b26c8d00476e0e24b7184717fe4f0
date;https://github.com/HowardHinnant/date/archive/refs/tags/v2.4.1.zip;ea99f021262b1d804a872735c658860a6a13cc98
dlpack;https://github.com/dmlc/dlpack/archive/refs/tags/v0.6.zip;4d565dd2e5b31321e5549591d78aa7f377173445
flatbuffers;https://github.com/google/flatbuffers/archive/refs/tags/v1.12.0.zip;ba0a75fd12dbef8f6557a74e611b7a3d0c5fe7bf
fp16;https://github.com/Maratyszcza/FP16/archive/0a92994d729ff76a58f692d3028ca1b64b145d91.zip;b985f6985a05a1c03ff1bb71190f66d8f98a1494
fxdiv;https://github.com/Maratyszcza/FXdiv/archive/63058eff77e11aa15bf531df5dd34395ec3017c8.zip;a5658f4036402dbca7cebee32be57fb8149811e1
google_benchmark;https://github.com/google/benchmark/archive/refs/tags/v1.7.0.zip;e97c368b176e8614e3f1bf13dd9abcf6a7ad9908
google_nsync;https://github.com/google/nsync/archive/refs/tags/1.23.0.zip;f3233450cf7156fc0bedd1b0e884eddec264897c
googletest;https://github.com/google/googletest/archive/519beb0e52c842729b4b53731d27c0e0c32ab4a2.zip;4b3c37972e4c1bef1185d46f702082f8772ee73f
googlexnnpack;https://github.com/google/XNNPACK/archive/003c580e696a774afdc984996ee909b7c8d8128c.zip;9f192e3f15e1e37ae9c78d53eeea47e45c5eb31c
json;https://github.com/nlohmann/json/archive/refs/tags/v3.10.5.zip;f257f8dc27c5b8c085dc887b40cddd18ae1f725c
microsoft_gsl;https://github.com/microsoft/GSL/archive/refs/tags/v4.0.0.zip;cf368104cd22a87b4dd0c80228919bb2df3e2a14
microsoft_wil;https://github.com/microsoft/wil/archive/5f4caba4e7a9017816e47becdd918fcc872039ba.zip;fd119887d0d17c37adf1fc227b054befa28158ad
mimalloc;https://github.com/microsoft/mimalloc/archive/refs/tags/v2.1.1.zip;d5ee7d34223d0567892db5179849939c8769dc41
mp11;https://github.com/boostorg/mp11/archive/refs/tags/boost-1.79.0.zip;c8f04e378535ededbe5af52c8f969d2dedbe73d5
onnx;https://github.com/onnx/onnx/archive/refs/tags/v1.14.0.zip;3c6e43a36f94addc15afe939860127a1d74a9488
#use the last commit of 8.6-GA branch (https://github.com/onnx/onnx-tensorrt/commit/6ba67d3428e05f690145373ca87fb8d32f98df45)
onnx_tensorrt;https://github.com/onnx/onnx-tensorrt/archive/6ba67d3428e05f690145373ca87fb8d32f98df45.zip;805902b4f03f09f07151e03b5ccc49968c9cc896
protobuf;https://github.com/protocolbuffers/protobuf/archive/refs/tags/v21.12.zip;7cf2733949036c7d52fda017badcab093fe73bfa
protoc_win64;https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip;b4521f7ada5b260380f94c4bd7f1b7684c76969a
protoc_win32;https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win32.zip;3688010318192c46ce73213cdfb6b3e5656da874
protoc_linux_x64;https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-linux-x86_64.zip;338462004aa5be9fba45b35b5b4be43f69b47a90
protoc_linux_x86;https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-linux-x86_32.zip;61fdbe7d6360e065ec6fea23bca2cca673115fb8
protoc_linux_aarch64;https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-linux-aarch_64.zip;df9d45470b0b8cf939dd2f0ec6b88e9cafc4d617
psimd;https://github.com/Maratyszcza/psimd/archive/072586a71b55b7f8c584153d223e95687148a900.zip;1f5454b01f06f9656b77e4a5e2e31d7422487013
pthreadpool;https://github.com/Maratyszcza/pthreadpool/archive/1787867f6183f056420e532eec640cba25efafea.zip;e43e80781560c5ab404a4da20f34d846f5f5d101
pybind11;https://github.com/pybind/pybind11/archive/refs/tags/v2.10.1.zip;769b6aa67a77f17a770960f604b727645b6f6a13
pytorch_cpuinfo;https://github.com/pytorch/cpuinfo/archive/5916273f79a21551890fd3d56fc5375a78d1598d.zip;2be4d2ae321fada97cb39eaf4eeba5f8c85597cf
re2;https://github.com/google/re2/archive/refs/tags/2022-06-01.zip;aa77313b76e91b531ee7f3e45f004c6a502a5374
safeint;https://github.com/dcleblanc/SafeInt/archive/ff15c6ada150a5018c5ef2172401cb4529eac9c0.zip;913a4046e5274d329af2806cb53194f617d8c0ab
tensorboard;https://github.com/tensorflow/tensorboard/archive/373eb09e4c5d2b3cc2493f0949dc4be6b6a45e81.zip;67b833913605a4f3f499894ab11528a702c2b381
cutlass;https://github.com/NVIDIA/cutlass/archive/refs/tags/v3.0.0.zip;0f95b3c1fc1bd1175c4a90b2c9e39074d1bccefd
extensions;https://github.com/microsoft/onnxruntime-extensions/archive/94142d8391c9791ec71c38336436319a2d4ac7a0.zip;4365ac5140338b4cb75a39944a4be276e3829b3c
eigen;https://gitlab.com/libeigen/eigen/-/archive/3.4/eigen-3.4.zip;ee201b07085203ea7bd8eb97cbcb31b07cfa3efb
BIN
View File
Binary file not shown.
Binary file not shown.
@@ -1,22 +0,0 @@
{
"build_osx_archs": {
"iphoneos": [
"arm64"
],
"iphonesimulator": [
"arm64",
"x86_64"
]
},
"build_params": [
"--ios",
"--parallel",
"--use_xcode",
"--build_apple_framework",
"--use_coreml",
"--skip_tests",
"--cmake_extra_defines=onnxruntime_BUILD_UNIT_TESTS=OFF",
"--apple_deploy_target=12.0",
"--use_xnnpack"
]
}
-1
View File
@@ -1 +0,0 @@
flatbuffers
@@ -1,155 +0,0 @@
#!/usr/bin/env python3
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
import argparse
import pathlib
import shutil
import sys
_script_dir = pathlib.Path(__file__).parent.resolve(strict=True)
sys.path.append(str(_script_dir.parent))
from package_assembly_utils import ( # noqa: E402
PackageVariant,
copy_repo_relative_to_dir,
gen_file_from_template,
load_json_config,
)
def get_pod_config_file(package_variant: PackageVariant):
"""
Gets the pod configuration file path for the given package variant.
"""
if package_variant == PackageVariant.Full:
return _script_dir / "onnxruntime-c.config.json"
elif package_variant == PackageVariant.Mobile:
return _script_dir / "onnxruntime-mobile-c.config.json"
elif package_variant == PackageVariant.Test:
return _script_dir / "onnxruntime-test-c.config.json"
elif package_variant == PackageVariant.Training:
return _script_dir / "onnxruntime-training-c.config.json"
else:
raise ValueError(f"Unhandled package variant: {package_variant}")
def assemble_c_pod_package(
staging_dir: pathlib.Path,
pod_version: str,
framework_info_file: pathlib.Path,
public_headers_dir: pathlib.Path,
framework_dir: pathlib.Path,
package_variant: PackageVariant,
):
"""
Assembles the files for the C/C++ pod package in a staging directory.
:param staging_dir Path to the staging directory for the C/C++ pod files.
:param pod_version C/C++ pod version.
:param framework_info_file Path to the framework_info.json file containing additional values for the podspec.
:param public_headers_dir Path to the public headers directory to include in the pod.
:param framework_dir Path to the onnxruntime framework directory to include in the pod.
:param package_variant The pod package variant.
:return Tuple of (package name, path to the podspec file).
"""
staging_dir = staging_dir.resolve()
framework_info_file = framework_info_file.resolve(strict=True)
public_headers_dir = public_headers_dir.resolve(strict=True)
framework_dir = framework_dir.resolve(strict=True)
framework_info = load_json_config(framework_info_file)
pod_config = load_json_config(get_pod_config_file(package_variant))
pod_name = pod_config["name"]
print(f"Assembling files in staging directory: {staging_dir}")
if staging_dir.exists():
print("Warning: staging directory already exists", file=sys.stderr)
# copy the necessary files to the staging directory
shutil.copytree(framework_dir, staging_dir / framework_dir.name, dirs_exist_ok=True)
shutil.copytree(public_headers_dir, staging_dir / public_headers_dir.name, dirs_exist_ok=True)
copy_repo_relative_to_dir(["LICENSE"], staging_dir)
# generate the podspec file from the template
variable_substitutions = {
"DESCRIPTION": pod_config["description"],
"IOS_DEPLOYMENT_TARGET": framework_info["IOS_DEPLOYMENT_TARGET"],
"LICENSE_FILE": "LICENSE",
"NAME": pod_name,
"ORT_C_FRAMEWORK": framework_dir.name,
"ORT_C_HEADERS_DIR": public_headers_dir.name,
"SUMMARY": pod_config["summary"],
"VERSION": pod_version,
"WEAK_FRAMEWORK": framework_info["WEAK_FRAMEWORK"],
}
podspec_template = _script_dir / "c.podspec.template"
podspec = staging_dir / f"{pod_name}.podspec"
gen_file_from_template(podspec_template, podspec, variable_substitutions)
return pod_name, podspec
def parse_args():
parser = argparse.ArgumentParser(
description="""
Assembles the files for the C/C++ pod package in a staging directory.
This directory can be validated (e.g., with `pod lib lint`) and then zipped to create a package for release.
"""
)
parser.add_argument(
"--staging-dir",
type=pathlib.Path,
default=pathlib.Path("./c-staging"),
help="Path to the staging directory for the C/C++ pod files.",
)
parser.add_argument("--pod-version", required=True, help="C/C++ pod version.")
parser.add_argument(
"--framework-info-file",
type=pathlib.Path,
required=True,
help="Path to the framework_info.json file containing additional values for the podspec. "
"This file should be generated by CMake in the build directory.",
)
parser.add_argument(
"--public-headers-dir",
type=pathlib.Path,
required=True,
help="Path to the public headers directory to include in the pod.",
)
parser.add_argument(
"--framework-dir",
type=pathlib.Path,
required=True,
help="Path to the onnxruntime framework directory to include in the pod.",
)
parser.add_argument(
"--variant", choices=PackageVariant.all_variant_names(), required=True, help="Pod package variant."
)
return parser.parse_args()
def main():
args = parse_args()
assemble_c_pod_package(
staging_dir=args.staging_dir,
pod_version=args.pod_version,
framework_info_file=args.framework_info_file,
public_headers_dir=args.public_headers_dir,
framework_dir=args.framework_dir,
package_variant=PackageVariant[args.variant],
)
return 0
if __name__ == "__main__":
sys.exit(main())
@@ -1,212 +0,0 @@
#!/usr/bin/env python3
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
import argparse
import pathlib
import sys
_script_dir = pathlib.Path(__file__).parent.resolve(strict=True)
sys.path.append(str(_script_dir.parent))
from assemble_c_pod_package import get_pod_config_file as get_c_pod_config_file # noqa: E402
from package_assembly_utils import ( # noqa: E402
PackageVariant,
copy_repo_relative_to_dir,
filter_files,
gen_file_from_template,
load_json_config,
)
# these variables contain paths or path patterns that are relative to the repo root
# the license file
license_file = "LICENSE"
# include directories for compiling the pod itself
include_dirs = [
"objectivec",
]
all_objc_files = {
"source_files": [
"objectivec/include/*.h",
"objectivec/*.h",
"objectivec/*.m",
"objectivec/*.mm",
],
"public_header_files": [
"objectivec/include/*.h",
],
"test_source_files": [
"objectivec/test/*.h",
"objectivec/test/*.m",
"objectivec/test/*.mm",
],
"test_resource_files": [
"objectivec/test/testdata/*.ort",
"onnxruntime/test/testdata/training_api/*",
],
}
training_only_objc_files = {
"source_files": [
"objectivec/include/onnxruntime_training.h",
"objectivec/include/ort_checkpoint.h",
"objectivec/include/ort_training_session.h",
"objectivec/ort_checkpoint.mm",
"objectivec/ort_checkpoint_internal.h",
"objectivec/ort_training_session_internal.h",
"objectivec/ort_training_session.mm",
],
"public_header_files": [
"objectivec/include/ort_checkpoint.h",
"objectivec/include/ort_training_session.h",
"objectivec/include/onnxruntime_training.h",
],
"test_source_files": [
"objectivec/test/ort_training_session_test.mm",
"objectivec/test/ort_checkpoint_test.mm",
"objectivec/test/ort_training_utils_test.mm",
],
"test_resource_files": [
"onnxruntime/test/testdata/training_api/*",
],
}
def get_pod_files(package_variant: PackageVariant):
"""
Gets the source and header files for the given package variant.
"""
if package_variant == PackageVariant.Training:
return all_objc_files
else:
# return files that are in pod_files but not in training_only_objc_files
filtered_pod_files = {}
for key in all_objc_files:
filtered_pod_files[key] = filter_files(all_objc_files[key], training_only_objc_files[key])
return filtered_pod_files
def get_pod_config_file(package_variant: PackageVariant):
"""
Gets the pod configuration file path for the given package variant.
"""
if package_variant == PackageVariant.Full:
return _script_dir / "onnxruntime-objc.config.json"
elif package_variant == PackageVariant.Mobile:
return _script_dir / "onnxruntime-mobile-objc.config.json"
elif package_variant == PackageVariant.Training:
return _script_dir / "onnxruntime-training-objc.config.json"
else:
raise ValueError(f"Unhandled package variant: {package_variant}")
def assemble_objc_pod_package(
staging_dir: pathlib.Path, pod_version: str, framework_info_file: pathlib.Path, package_variant: PackageVariant
):
"""
Assembles the files for the Objective-C pod package in a staging directory.
:param staging_dir Path to the staging directory for the Objective-C pod files.
:param pod_version Objective-C pod version.
:param framework_info_file Path to the framework_info.json file containing additional values for the podspec.
:param package_variant The pod package variant.
:return Tuple of (package name, path to the podspec file).
"""
staging_dir = staging_dir.resolve()
framework_info_file = framework_info_file.resolve(strict=True)
framework_info = load_json_config(framework_info_file)
pod_config = load_json_config(get_pod_config_file(package_variant))
c_pod_config = load_json_config(get_c_pod_config_file(package_variant))
pod_name = pod_config["name"]
print(f"Assembling files in staging directory: {staging_dir}")
if staging_dir.exists():
print("Warning: staging directory already exists", file=sys.stderr)
pod_files = get_pod_files(package_variant)
# copy the necessary files to the staging directory
copy_repo_relative_to_dir(
[license_file, *pod_files["source_files"], *pod_files["test_source_files"], *pod_files["test_resource_files"]],
staging_dir,
)
# generate the podspec file from the template
def path_patterns_as_variable_value(patterns: list[str]):
return ", ".join([f'"{pattern}"' for pattern in patterns])
variable_substitutions = {
"C_POD_NAME": c_pod_config["name"],
"DESCRIPTION": pod_config["description"],
"INCLUDE_DIR_LIST": path_patterns_as_variable_value(include_dirs),
"IOS_DEPLOYMENT_TARGET": framework_info["IOS_DEPLOYMENT_TARGET"],
"LICENSE_FILE": license_file,
"NAME": pod_name,
"PUBLIC_HEADER_FILE_LIST": path_patterns_as_variable_value(pod_files["public_header_files"]),
"SOURCE_FILE_LIST": path_patterns_as_variable_value(pod_files["source_files"]),
"SUMMARY": pod_config["summary"],
"TEST_RESOURCE_FILE_LIST": path_patterns_as_variable_value(pod_files["test_resource_files"]),
"TEST_SOURCE_FILE_LIST": path_patterns_as_variable_value(pod_files["test_source_files"]),
"VERSION": pod_version,
}
podspec_template = _script_dir / "objc.podspec.template"
podspec = staging_dir / f"{pod_name}.podspec"
gen_file_from_template(podspec_template, podspec, variable_substitutions)
return pod_name, podspec
def parse_args():
parser = argparse.ArgumentParser(
description="""
Assembles the files for the Objective-C pod package in a staging directory.
This directory can be validated (e.g., with `pod lib lint`) and then zipped to create a package for release.
"""
)
parser.add_argument(
"--staging-dir",
type=pathlib.Path,
default=pathlib.Path("./onnxruntime-mobile-objc-staging"),
help="Path to the staging directory for the Objective-C pod files.",
)
parser.add_argument("--pod-version", required=True, help="Objective-C pod version.")
parser.add_argument(
"--framework-info-file",
type=pathlib.Path,
required=True,
help="Path to the framework_info.json file containing additional values for the podspec. "
"This file should be generated by CMake in the build directory.",
)
parser.add_argument(
"--variant", choices=PackageVariant.release_variant_names(), required=True, help="Pod package variant."
)
return parser.parse_args()
def main():
args = parse_args()
assemble_objc_pod_package(
staging_dir=args.staging_dir,
pod_version=args.pod_version,
framework_info_file=args.framework_info_file,
package_variant=PackageVariant[args.variant],
)
return 0
if __name__ == "__main__":
sys.exit(main())
@@ -1,185 +0,0 @@
#!/usr/bin/env python3
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
import argparse
import logging
import pathlib
import shutil
import sys
import tempfile
from assemble_c_pod_package import assemble_c_pod_package
from assemble_objc_pod_package import assemble_objc_pod_package
from package_assembly_utils import PackageVariant, get_ort_version
SCRIPT_PATH = pathlib.Path(__file__).resolve()
SCRIPT_DIR = SCRIPT_PATH.parent
REPO_DIR = SCRIPT_PATH.parents[4]
logging.basicConfig(format="%(asctime)s %(name)s [%(levelname)s] - %(message)s", level=logging.DEBUG)
log = logging.getLogger(SCRIPT_PATH.stem)
def parse_args():
parser = argparse.ArgumentParser(
description="Builds an iOS framework and uses it to assemble iOS pod package files.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument(
"--build-dir",
type=pathlib.Path,
default=REPO_DIR / "build" / "ios_framework",
help="The build directory. This will contain the iOS framework build output.",
)
parser.add_argument(
"--staging-dir",
type=pathlib.Path,
default=REPO_DIR / "build" / "ios_pod_staging",
help="The staging directory. This will contain the iOS pod package files. "
"The pod package files do not have dependencies on files in the build directory.",
)
parser.add_argument(
"--pod-version",
default=f"{get_ort_version()}-local",
help="The version string of the pod. The same version is used for all pods.",
)
parser.add_argument(
"--variant",
choices=PackageVariant.release_variant_names(),
default=PackageVariant.Mobile.name,
help="Pod package variant.",
)
parser.add_argument("--test", action="store_true", help="Run tests on the framework and pod package files.")
build_framework_group = parser.add_argument_group(
title="iOS framework build arguments",
description="See the corresponding arguments in build_ios_framework.py for details.",
)
build_framework_group.add_argument("--include-ops-by-config")
build_framework_group.add_argument(
"--build-settings-file", required=True, help="The positional argument of build_ios_framework.py."
)
build_framework_group.add_argument(
"-b",
"--build-ios-framework-arg",
action="append",
dest="build_ios_framework_extra_args",
default=[],
help="Pass an argument through to build_ios_framework.py. This may be specified multiple times.",
)
args = parser.parse_args()
return args
def run(arg_list, cwd=None):
import os
import shlex
import subprocess
log.info(
"Running subprocess in '{}'\n {}".format(cwd or os.getcwd(), " ".join([shlex.quote(arg) for arg in arg_list]))
)
return subprocess.run(arg_list, check=True, cwd=cwd)
def main():
args = parse_args()
build_dir = args.build_dir.resolve()
staging_dir = args.staging_dir.resolve()
# build framework
package_variant = PackageVariant[args.variant]
framework_info_file = build_dir / "framework_info.json"
log.info("Building iOS framework.")
build_ios_framework_args = [
sys.executable,
str(SCRIPT_DIR / "build_ios_framework.py"),
*args.build_ios_framework_extra_args,
]
if args.include_ops_by_config is not None:
build_ios_framework_args += ["--include_ops_by_config", args.include_ops_by_config]
build_ios_framework_args += ["--build_dir", str(build_dir), args.build_settings_file]
run(build_ios_framework_args)
if args.test:
test_ios_packages_args = [
sys.executable,
str(SCRIPT_DIR / "test_ios_packages.py"),
"--fail_if_cocoapods_missing",
"--framework_info_file",
str(framework_info_file),
"--c_framework_dir",
str(build_dir / "framework_out"),
"--variant",
package_variant.name,
]
run(test_ios_packages_args)
# assemble pods and then move them to their target locations (staging_dir/<pod_name>)
staging_dir.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(dir=staging_dir) as pod_assembly_dir_name:
pod_assembly_dir = pathlib.Path(pod_assembly_dir_name)
log.info("Assembling C/C++ pod.")
c_pod_staging_dir = pod_assembly_dir / "c_pod"
c_pod_name, c_pod_podspec = assemble_c_pod_package(
staging_dir=c_pod_staging_dir,
pod_version=args.pod_version,
framework_info_file=framework_info_file,
framework_dir=build_dir / "framework_out" / "onnxruntime.xcframework",
public_headers_dir=build_dir / "framework_out" / "Headers",
package_variant=package_variant,
)
if args.test:
test_c_pod_args = ["pod", "lib", "lint", "--verbose"]
run(test_c_pod_args, cwd=c_pod_staging_dir)
log.info("Assembling Objective-C pod.")
objc_pod_staging_dir = pod_assembly_dir / "objc_pod"
objc_pod_name, objc_pod_podspec = assemble_objc_pod_package(
staging_dir=objc_pod_staging_dir,
pod_version=args.pod_version,
framework_info_file=framework_info_file,
package_variant=package_variant,
)
if args.test:
test_objc_pod_args = ["pod", "lib", "lint", "--verbose", f"--include-podspecs={c_pod_podspec}"]
run(test_objc_pod_args, cwd=objc_pod_staging_dir)
def move_dir(src, dst):
if dst.is_dir():
shutil.rmtree(dst)
shutil.move(src, dst)
move_dir(c_pod_staging_dir, staging_dir / c_pod_name)
move_dir(objc_pod_staging_dir, staging_dir / objc_pod_name)
log.info(f"Successfully assembled iOS pods at '{staging_dir}'.")
if __name__ == "__main__":
main()
@@ -1,59 +0,0 @@
#!/bin/bash
set -e -x
INSTALL_PREFIX='/usr'
DEP_FILE_PATH='/tmp/scripts/deps.txt'
while getopts "p:d:" parameter_Option
do case "${parameter_Option}"
in
p) INSTALL_PREFIX=${OPTARG};;
d) DEP_FILE_PATH=${OPTARG};;
esac
done
EXTRA_CMAKE_ARGS=""
case "$(uname -s)" in
Darwin*)
echo 'Building ONNX Runtime on Mac OS X'
EXTRA_CMAKE_ARGS="-DCMAKE_OSX_ARCHITECTURES=x86_64;arm64"
;;
Linux*)
# Depending on how the compiler has been configured when it was built, sometimes "gcc -dumpversion" shows the full version.
GCC_VERSION=$(gcc -dumpversion | cut -d . -f 1)
#-fstack-clash-protection prevents attacks based on an overlapping heap and stack.
if [ "$GCC_VERSION" -ge 8 ]; then
CFLAGS="$CFLAGS -fstack-clash-protection"
CXXFLAGS="$CXXFLAGS -fstack-clash-protection"
fi
ARCH=$(uname -m)
if [ "$ARCH" == "x86_64" ] && [ "$GCC_VERSION" -ge 9 ]; then
CFLAGS="$CFLAGS -fcf-protection"
CXXFLAGS="$CXXFLAGS -fcf-protection"
fi
export CFLAGS
export CXXFLAGS
;;
*)
exit 1
esac
mkdir -p $INSTALL_PREFIX
echo "Installing protobuf ..."
protobuf_url=$(grep '^protobuf' $DEP_FILE_PATH | cut -d ';' -f 2 )
if [[ "$protobuf_url" = https* ]]; then
protobuf_url=$(echo $protobuf_url | sed 's/\.zip$/\.tar.gz/')
curl -sSL --retry 5 --retry-delay 10 --create-dirs --fail -L -o protobuf_src.tar.gz $protobuf_url
mkdir protobuf
cd protobuf
tar -zxf ../protobuf_src.tar.gz --strip=1
else
cp $protobuf_url protobuf_src.zip
unzip protobuf_src.zip
cd protobuf-*
fi
cmake . -DCMAKE_INSTALL_PREFIX=$INSTALL_PREFIX -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release -Dprotobuf_WITH_ZLIB_DEFAULT=OFF -Dprotobuf_BUILD_SHARED_LIBS=OFF $EXTRA_CMAKE_ARGS
make -j$(getconf _NPROCESSORS_ONLN)
make install
cd ..
@@ -1,130 +0,0 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
import enum
import json
import os
import pathlib
import re
import shutil
from typing import Dict, List
_script_dir = pathlib.Path(__file__).parent.resolve(strict=True)
repo_root = _script_dir.parents[3]
class PackageVariant(enum.Enum):
Full = 0 # full ORT build with all opsets, ops, and types
Mobile = 1 # minimal ORT build with reduced ops
Training = 2 # full ORT build with all opsets, ops, and types, plus training APIs
Test = -1 # for testing purposes only
@classmethod
def release_variant_names(cls):
return [v.name for v in cls if v.value >= 0]
@classmethod
def all_variant_names(cls):
return [v.name for v in cls]
_template_variable_pattern = re.compile(r"@(\w+)@") # match "@var@"
def gen_file_from_template(
template_file: pathlib.Path, output_file: pathlib.Path, variable_substitutions: Dict[str, str], strict: bool = True
):
"""
Generates a file from a template file.
The template file may contain template variables that will be substituted
with the provided values in the generated output file.
In the template file, template variable names are delimited by "@"'s,
e.g., "@var@".
:param template_file The template file path.
:param output_file The generated output file path.
:param variable_substitutions The mapping from template variable name to value.
:param strict Whether to require the set of template variable names in the file and the keys of
`variable_substitutions` to be equal.
"""
with open(template_file) as template:
content = template.read()
variables_in_file = set()
def replace_template_variable(match):
variable_name = match.group(1)
variables_in_file.add(variable_name)
return variable_substitutions.get(variable_name, match.group(0))
content = _template_variable_pattern.sub(replace_template_variable, content)
if strict and variables_in_file != variable_substitutions.keys():
variables_in_substitutions = set(variable_substitutions.keys())
raise ValueError(
f"Template file variables and substitution variables do not match. "
f"Only in template file: {sorted(variables_in_file - variables_in_substitutions)}. "
f"Only in substitutions: {sorted(variables_in_substitutions - variables_in_file)}."
)
with open(output_file, mode="w") as output:
output.write(content)
def filter_files(all_file_patterns: List[str], excluded_file_patterns: List[str]):
"""
Filters file paths based on inclusion and exclusion patterns
:param all_file_patterns The list of file paths to filter.
:param excluded_file_patterns The list of exclusion patterns.
:return The filtered list of file paths
"""
# get all files matching the patterns in all_file_patterns
all_files = [str(path.relative_to(repo_root)) for pattern in all_file_patterns for path in repo_root.glob(pattern)]
# get all files matching the patterns in excluded_file_patterns
exclude_files = [
str(path.relative_to(repo_root)) for pattern in excluded_file_patterns for path in repo_root.glob(pattern)
]
# return the difference
return list(set(all_files) - set(exclude_files))
def copy_repo_relative_to_dir(patterns: List[str], dest_dir: pathlib.Path):
"""
Copies file paths relative to the repo root to a directory.
The given paths or path patterns are relative to the repo root, and the
repo root-relative intermediate directory structure is maintained.
:param patterns The paths or path patterns relative to the repo root.
:param dest_dir The destination directory.
"""
paths = [path for pattern in patterns for path in repo_root.glob(pattern)]
for path in paths:
repo_relative_path = path.relative_to(repo_root)
dst_path = dest_dir / repo_relative_path
os.makedirs(dst_path.parent, exist_ok=True)
shutil.copy(path, dst_path)
def load_json_config(json_config_file: pathlib.Path):
"""
Loads configuration info from a JSON file.
:param json_config_file The JSON configuration file path.
:return The configuration info values.
"""
with open(json_config_file) as config:
return json.load(config)
def get_ort_version():
"""
Gets the ONNX Runtime version string from the repo.
:return The ONNX Runtime version string.
"""
with open(repo_root / "VERSION_NUMBER") as version_file:
return version_file.read().strip()
@@ -1,192 +0,0 @@
#!/usr/bin/env python3
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
import argparse
import contextlib
import os
import pathlib
import shutil
import subprocess
import tempfile
from assemble_c_pod_package import assemble_c_pod_package
from package_assembly_utils import PackageVariant, gen_file_from_template, get_ort_version
SCRIPT_PATH = pathlib.Path(__file__).resolve(strict=True)
REPO_DIR = SCRIPT_PATH.parents[4]
def _test_ios_packages(args):
# check if CocoaPods is installed
if shutil.which("pod") is None:
if args.fail_if_cocoapods_missing:
raise ValueError("CocoaPods is required for this test")
else:
print("CocoaPods is not installed, ignore this test")
return
# Now we need to create a zip file contains the framework and the podspec file, both of these 2 files
# should be under the c_framework_dir
c_framework_dir = args.c_framework_dir.resolve()
if not c_framework_dir.is_dir():
raise FileNotFoundError(f"c_framework_dir {c_framework_dir} is not a folder.")
has_framework = (c_framework_dir / "onnxruntime.framework").exists()
has_xcframework = (c_framework_dir / "onnxruntime.xcframework").exists()
if not has_framework and not has_xcframework:
raise FileNotFoundError(f"{c_framework_dir} does not have onnxruntime.framework/xcframework")
if has_framework and has_xcframework:
raise ValueError("Cannot proceed when both onnxruntime.framework and onnxruntime.xcframework exist")
framework_name = "onnxruntime.framework" if has_framework else "onnxruntime.xcframework"
# create a temp folder
with contextlib.ExitStack() as context_stack:
if args.test_project_stage_dir is None:
stage_dir = pathlib.Path(context_stack.enter_context(tempfile.TemporaryDirectory())).resolve()
else:
# If we specify the stage dir, then use it to create test project
stage_dir = args.test_project_stage_dir.resolve()
if os.path.exists(stage_dir):
shutil.rmtree(stage_dir)
os.makedirs(stage_dir)
# assemble the test project here
target_proj_path = stage_dir / "ios_package_test"
# copy the test project source files to target_proj_path
test_proj_path = pathlib.Path(REPO_DIR, "onnxruntime/test/platform/ios/ios_package_test")
shutil.copytree(test_proj_path, target_proj_path)
# assemble local pod files here
local_pods_dir = stage_dir / "local_pods"
# We will only publish xcframework, however, assembly of the xcframework is a post process
# and it cannot be done by CMake for now. See, https://gitlab.kitware.com/cmake/cmake/-/issues/21752
# For a single sysroot and arch built by build.py or cmake, we can only generate framework
# We still need a way to test it. framework_dir and public_headers_dir have different values when testing a
# framework and a xcframework.
framework_dir = args.c_framework_dir / framework_name
public_headers_dir = framework_dir / "Headers" if has_framework else args.c_framework_dir / "Headers"
pod_name, podspec = assemble_c_pod_package(
staging_dir=local_pods_dir,
pod_version=get_ort_version(),
framework_info_file=args.framework_info_file,
public_headers_dir=public_headers_dir,
framework_dir=framework_dir,
package_variant=PackageVariant[args.variant],
)
# move podspec out to target_proj_path first
podspec = shutil.move(podspec, target_proj_path / podspec.name)
# create a zip file contains the framework
zip_file_path = local_pods_dir / f"{pod_name}.zip"
# shutil.make_archive require target file as full path without extension
shutil.make_archive(zip_file_path.with_suffix(""), "zip", root_dir=local_pods_dir)
# update the podspec to point to the local framework zip file
with open(podspec) as file:
file_data = file.read()
file_data = file_data.replace("file:///http_source_placeholder", f"file:///{zip_file_path}")
with open(podspec, "w") as file:
file.write(file_data)
# generate Podfile to point to pod
gen_file_from_template(
target_proj_path / "Podfile.template",
target_proj_path / "Podfile",
{"C_POD_NAME": pod_name, "C_POD_PODSPEC": f"./{podspec.name}"},
)
# clean the Cocoapods cache first, in case the same pod was cached in previous runs
subprocess.run(["pod", "cache", "clean", "--all"], shell=False, check=True, cwd=target_proj_path)
# install pods
subprocess.run(["pod", "install"], shell=False, check=True, cwd=target_proj_path)
# run the tests
if not args.prepare_test_project_only:
simulator_device_name = subprocess.check_output(
["bash", str(REPO_DIR / "tools" / "ci_build" / "github" / "apple" / "get_simulator_device_name.sh")],
text=True,
).strip()
subprocess.run(
[
"xcrun",
"xcodebuild",
"test",
"-workspace",
"./ios_package_test.xcworkspace",
"-scheme",
"ios_package_test",
"-destination",
f"platform=iOS Simulator,OS=latest,name={simulator_device_name}",
],
shell=False,
check=True,
cwd=target_proj_path,
)
def parse_args():
parser = argparse.ArgumentParser(
os.path.basename(__file__), description="Test iOS framework using CocoaPods package."
)
parser.add_argument(
"--fail_if_cocoapods_missing",
action="store_true",
help="This script will fail if CocoaPods is not installed, "
"will not throw error unless fail_if_cocoapod_missing is set.",
)
parser.add_argument(
"--framework_info_file",
type=pathlib.Path,
required=True,
help="Path to the framework_info.json file containing additional values for the podspec. "
"This file should be generated by CMake in the build directory.",
)
parser.add_argument(
"--c_framework_dir", type=pathlib.Path, required=True, help="Provide the parent directory for C/C++ framework"
)
parser.add_argument(
"--variant",
choices=PackageVariant.all_variant_names(),
default=PackageVariant.Test.name,
help="Pod package variant.",
)
parser.add_argument(
"--test_project_stage_dir",
type=pathlib.Path,
help="The stage dir for the test project, if not specified, will use a temporary path",
)
parser.add_argument(
"--prepare_test_project_only",
action="store_true",
help="Prepare the test project only, without running the tests",
)
return parser.parse_args()
def main():
args = parse_args()
_test_ios_packages(args)
if __name__ == "__main__":
main()
@@ -1,12 +0,0 @@
# Install appcenter CLI
parameters:
- name: appcenterVersion
type: string
default: "2.13.7"
steps:
- bash: |
set -e -x
npm install -g appcenter-cli@${{ parameters.appcenterVersion }}
displayName: Install appcenter CLI ${{ parameters.appcenterVersion }}