From 83a311e5f7ee543cc5f15cd4db30ad4ea68c2e7a Mon Sep 17 00:00:00 2001 From: rachguo Date: Tue, 11 Jul 2023 14:14:50 -0700 Subject: [PATCH] add initial test pipeline --- Package.swift | 16 +- cmake/deps.txt | 44 +++++ .../mac-ios-swift-packaging-pipeline.yml | 178 ++++++++++++++++++ .../github/scripts/install_protobuf.sh | 59 ++++++ ...t-governance-component-detection-steps.yml | 18 ++ .../github/templates/install-appcenter.yml | 12 ++ .../github/templates/use-xcode-version.yml | 14 ++ version.txt | 1 + 8 files changed, 334 insertions(+), 8 deletions(-) create mode 100644 cmake/deps.txt create mode 100644 tools/ci_build/github/mac-ios-swift-packaging-pipeline.yml create mode 100644 tools/ci_build/github/scripts/install_protobuf.sh create mode 100644 tools/ci_build/github/templates/component-governance-component-detection-steps.yml create mode 100644 tools/ci_build/github/templates/install-appcenter.yml create mode 100644 tools/ci_build/github/templates/use-xcode-version.yml create mode 100644 version.txt diff --git a/Package.swift b/Package.swift index e053732..e7f008a 100644 --- a/Package.swift +++ b/Package.swift @@ -32,14 +32,14 @@ let package = Package( .target(name: "OnnxRuntimeBindings", dependencies: ["onnxruntime"], path: "objectivec", - exclude: ["test", "docs", "ReadMe.md", "format_objc.sh", - "ort_checkpoint.mm", - "ort_checkpoint_internal.h", - "ort_training_session_internal.h", - "ort_training_session.mm", - "include/ort_checkpoint.h", - "include/ort_training_session.h", - "include/onnxruntime_training.h"], + // exclude: ["test", "docs", "ReadMe.md", "format_objc.sh", + // "ort_checkpoint.mm", + // "ort_checkpoint_internal.h", + // "ort_training_session_internal.h", + // "ort_training_session.mm", + // "include/ort_checkpoint.h", + // "include/ort_training_session.h", + // "include/onnxruntime_training.h"], cxxSettings: [ .define("SPM_BUILD"), .unsafeFlags(["-std=c++17", diff --git a/cmake/deps.txt b/cmake/deps.txt new file mode 100644 index 0000000..63bdf21 --- /dev/null +++ b/cmake/deps.txt @@ -0,0 +1,44 @@ +#Name;Url;SHA1 +#This is a CSV file. +#The columns are separated by ";" because a list in cmake is just a ";" separated group of strings. +#Names should be in lower case. They will be used as variable names in cmake. +#URLs can be either https URLs or local file paths in cmake-style(directory separator is a forward slash character). +#SHA1 hashes can be generated by running sha1sum command. +#If you need to change abseil's version to a different one, you may also want to update external\abseil-cpp.natvis +#since the file contains a version string: "lts_20220623". However, the file is for debugging purposes only and would +#not affect built binaries. +abseil_cpp;https://github.com/abseil/abseil-cpp/archive/refs/tags/20220623.1.zip;50c137c88965cba015dfcc8fd5d9b46d23146751 +cxxopts;https://github.com/jarro2783/cxxopts/archive/3c73d91c0b04e2b59462f0a741be8c07024c1bc0.zip;6c6ca7f8480b26c8d00476e0e24b7184717fe4f0 +date;https://github.com/HowardHinnant/date/archive/refs/tags/v2.4.1.zip;ea99f021262b1d804a872735c658860a6a13cc98 +dlpack;https://github.com/dmlc/dlpack/archive/refs/tags/v0.6.zip;4d565dd2e5b31321e5549591d78aa7f377173445 +flatbuffers;https://github.com/google/flatbuffers/archive/refs/tags/v1.12.0.zip;ba0a75fd12dbef8f6557a74e611b7a3d0c5fe7bf +fp16;https://github.com/Maratyszcza/FP16/archive/0a92994d729ff76a58f692d3028ca1b64b145d91.zip;b985f6985a05a1c03ff1bb71190f66d8f98a1494 +fxdiv;https://github.com/Maratyszcza/FXdiv/archive/63058eff77e11aa15bf531df5dd34395ec3017c8.zip;a5658f4036402dbca7cebee32be57fb8149811e1 +google_benchmark;https://github.com/google/benchmark/archive/refs/tags/v1.7.0.zip;e97c368b176e8614e3f1bf13dd9abcf6a7ad9908 +google_nsync;https://github.com/google/nsync/archive/refs/tags/1.23.0.zip;f3233450cf7156fc0bedd1b0e884eddec264897c +googletest;https://github.com/google/googletest/archive/519beb0e52c842729b4b53731d27c0e0c32ab4a2.zip;4b3c37972e4c1bef1185d46f702082f8772ee73f +googlexnnpack;https://github.com/google/XNNPACK/archive/003c580e696a774afdc984996ee909b7c8d8128c.zip;9f192e3f15e1e37ae9c78d53eeea47e45c5eb31c +json;https://github.com/nlohmann/json/archive/refs/tags/v3.10.5.zip;f257f8dc27c5b8c085dc887b40cddd18ae1f725c +microsoft_gsl;https://github.com/microsoft/GSL/archive/refs/tags/v4.0.0.zip;cf368104cd22a87b4dd0c80228919bb2df3e2a14 +microsoft_wil;https://github.com/microsoft/wil/archive/5f4caba4e7a9017816e47becdd918fcc872039ba.zip;fd119887d0d17c37adf1fc227b054befa28158ad +mimalloc;https://github.com/microsoft/mimalloc/archive/refs/tags/v2.1.1.zip;d5ee7d34223d0567892db5179849939c8769dc41 +mp11;https://github.com/boostorg/mp11/archive/refs/tags/boost-1.79.0.zip;c8f04e378535ededbe5af52c8f969d2dedbe73d5 +onnx;https://github.com/onnx/onnx/archive/refs/tags/v1.14.0.zip;3c6e43a36f94addc15afe939860127a1d74a9488 +#use the last commit of 8.6-GA branch (https://github.com/onnx/onnx-tensorrt/commit/6ba67d3428e05f690145373ca87fb8d32f98df45) +onnx_tensorrt;https://github.com/onnx/onnx-tensorrt/archive/6ba67d3428e05f690145373ca87fb8d32f98df45.zip;805902b4f03f09f07151e03b5ccc49968c9cc896 +protobuf;https://github.com/protocolbuffers/protobuf/archive/refs/tags/v21.12.zip;7cf2733949036c7d52fda017badcab093fe73bfa +protoc_win64;https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip;b4521f7ada5b260380f94c4bd7f1b7684c76969a +protoc_win32;https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win32.zip;3688010318192c46ce73213cdfb6b3e5656da874 +protoc_linux_x64;https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-linux-x86_64.zip;338462004aa5be9fba45b35b5b4be43f69b47a90 +protoc_linux_x86;https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-linux-x86_32.zip;61fdbe7d6360e065ec6fea23bca2cca673115fb8 +protoc_linux_aarch64;https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-linux-aarch_64.zip;df9d45470b0b8cf939dd2f0ec6b88e9cafc4d617 +psimd;https://github.com/Maratyszcza/psimd/archive/072586a71b55b7f8c584153d223e95687148a900.zip;1f5454b01f06f9656b77e4a5e2e31d7422487013 +pthreadpool;https://github.com/Maratyszcza/pthreadpool/archive/1787867f6183f056420e532eec640cba25efafea.zip;e43e80781560c5ab404a4da20f34d846f5f5d101 +pybind11;https://github.com/pybind/pybind11/archive/refs/tags/v2.10.1.zip;769b6aa67a77f17a770960f604b727645b6f6a13 +pytorch_cpuinfo;https://github.com/pytorch/cpuinfo/archive/5916273f79a21551890fd3d56fc5375a78d1598d.zip;2be4d2ae321fada97cb39eaf4eeba5f8c85597cf +re2;https://github.com/google/re2/archive/refs/tags/2022-06-01.zip;aa77313b76e91b531ee7f3e45f004c6a502a5374 +safeint;https://github.com/dcleblanc/SafeInt/archive/ff15c6ada150a5018c5ef2172401cb4529eac9c0.zip;913a4046e5274d329af2806cb53194f617d8c0ab +tensorboard;https://github.com/tensorflow/tensorboard/archive/373eb09e4c5d2b3cc2493f0949dc4be6b6a45e81.zip;67b833913605a4f3f499894ab11528a702c2b381 +cutlass;https://github.com/NVIDIA/cutlass/archive/refs/tags/v3.0.0.zip;0f95b3c1fc1bd1175c4a90b2c9e39074d1bccefd +extensions;https://github.com/microsoft/onnxruntime-extensions/archive/94142d8391c9791ec71c38336436319a2d4ac7a0.zip;4365ac5140338b4cb75a39944a4be276e3829b3c +eigen;https://gitlab.com/libeigen/eigen/-/archive/3.4/eigen-3.4.zip;ee201b07085203ea7bd8eb97cbcb31b07cfa3efb \ No newline at end of file diff --git a/tools/ci_build/github/mac-ios-swift-packaging-pipeline.yml b/tools/ci_build/github/mac-ios-swift-packaging-pipeline.yml new file mode 100644 index 0000000..c020be5 --- /dev/null +++ b/tools/ci_build/github/mac-ios-swift-packaging-pipeline.yml @@ -0,0 +1,178 @@ +parameters: +- name: BuildType + displayName: |- + Type of build. + "normal": A normal build not for publication. + type: string #TODO: add other build types name here + values: + - normal + default: normal + +name: "$(Date:yyyyMMdd)$(Rev:rrr)" # build number format + +jobs: +- job: IosPackaging + displayName: "iOS Packaging" + + pool: + vmImage: "macOS-13" + + variables: + xcodeVersion: "14.3" + + timeoutInMinutes: 300 + + steps: + - task: InstallAppleCertificate@2 + inputs: + certSecureFile: '$(ios_signing_certificate_name)' + certPwd: '$(ios_signing_certificate_password)' + keychain: 'temp' + deleteCert: true + displayName: 'Install ORT Mobile Test Signing Certificate' + + - task: InstallAppleProvisioningProfile@1 + inputs: + provProfileSecureFile: '$(ios_provision_profile_name)' + removeProfile: true + displayName: 'Install ORT Mobile Test Provisioning Profile' + + - task: UsePythonVersion@0 + inputs: + versionSpec: "3.9" + addToPath: true + architecture: "x64" + + - template: templates/use-xcode-version.yml + parameters: + xcodeVersion: ${{ variables.xcodeVersion }} + + # - template: templates/install-appcenter.yml + + - script: | + pip install -r tools/ci_build/github/apple/ios_packaging.requirements.txt + displayName: "Install Python requirements" + + - bash: | + set -e + + BUILD_TYPE="${{ parameters.BuildType }}" + VERSION_FILE="$(Build.SourcesDirectory)/version.txt" + BASE_VERSION="$(cat "${VERSION_FILE}")" + SHORT_COMMIT_HASH="$(git rev-parse --short HEAD)" + DEV_VERSION="${BASE_VERSION}-dev+$(Build.BuildNumber).${SHORT_COMMIT_HASH}" + + case "${BUILD_TYPE}" in + ("normal") + VERSION="${DEV_VERSION}"; SHOULD_UPLOAD_ARCHIVES="false" ;; + (*) + echo "Invalid build type: ${BUILD_TYPE}"; exit 1 ;; + esac + + # Do not output ##vso[] commands with `set -x` or they may be parsed again and include a trailing quote. + set +x + + set_var() { + local VAR_NAME=${1:?} + local VAR_VALUE=${2:?} + echo "##vso[task.setvariable variable=${VAR_NAME}]${VAR_VALUE}" + echo "${VAR_NAME}: ${VAR_VALUE}" + } + + set_var "ORT_POD_VERSION" "${VERSION}" + set_var "ORT_SHOULD_UPLOAD_ARCHIVES" "${SHOULD_UPLOAD_ARCHIVES}" + displayName: "Set variables" + + - script: | + $(Build.SourcesDirectory)/tools/ci_build/github/scripts/install_protobuf.sh -p $(Build.BinariesDirectory)/protobuf_install -d $(Build.SourcesDirectory)/cmake/deps.txt + displayName: "Build Host Protoc" + + # create and test full pods + - script: | + python tools/ci_build/github/apple/build_and_assemble_ios_pods.py \ + --build-dir "$(Build.BinariesDirectory)/ios_framework_full" \ + --staging-dir "$(Build.BinariesDirectory)/staging" \ + --pod-version "${ORT_POD_VERSION}" \ + --test \ + --variant Full \ + --build-settings-file tools/ci_build/github/apple/default_full_ios_framework_build_settings.json \ + -b="--path_to_protoc_exe" -b "$(Build.BinariesDirectory)/protobuf_install/bin/protoc" + displayName: "[Full] Build iOS framework and assemble pod package files" + + - script: | + python tools/ci_build/github/apple/test_ios_packages.py \ + --fail_if_cocoapods_missing \ + --framework_info_file "$(Build.BinariesDirectory)/ios_framework_full/framework_info.json" \ + --c_framework_dir "$(Build.BinariesDirectory)/ios_framework_full/framework_out" \ + --variant Full \ + --test_project_stage_dir "$(Build.BinariesDirectory)/app_center_test_full" \ + --prepare_test_project_only + displayName: "[Full] Assemble test project for App Center" + + - task: Xcode@5 + inputs: + actions: 'build-for-testing' + configuration: 'Debug' + xcWorkspacePath: '$(Build.BinariesDirectory)/app_center_test_full/ios_package_test/ios_package_test.xcworkspace' + sdk: 'iphoneos' + scheme: 'ios_package_test' + signingOption: 'manual' + signingIdentity: '$(APPLE_CERTIFICATE_SIGNING_IDENTITY)' + provisioningProfileName: 'iOS Team Provisioning Profile' + args: '-derivedDataPath $(Build.BinariesDirectory)/app_center_test_full/ios_package_test/DerivedData' + workingDirectory: $(Build.BinariesDirectory)/app_center_test_full/ios_package_test/ + displayName: '[Full] Build iphone arm64 tests' + + # - script: | + # set -e -x + # appcenter test run xcuitest \ + # --app "AI-Frameworks/ORT-Mobile-iOS" \ + # --devices $(app_center_test_devices) \ + # --test-series "master" \ + # --locale "en_US" \ + # --build-dir $(Build.BinariesDirectory)/app_center_test_full/ios_package_test/DerivedData/Build/Products/Debug-iphoneos \ + # --token $(app_center_api_token) + # displayName: "[Full] Run E2E tests on App Center" + + - task: AzureCLI@2 + inputs: + azureSubscription: 'AIInfraBuildOnnxRuntimeOSS' + scriptType: 'bash' + scriptLocation: 'scriptPath' + scriptPath: 'tools/ci_build/github/apple/assemble_ios_packaging_artifacts.sh' + arguments: >- + "$(Build.BinariesDirectory)/staging" + "$(Build.ArtifactStagingDirectory)" + "$(ORT_POD_VERSION)" + "$(ORT_SHOULD_UPLOAD_ARCHIVES)" + displayName: "Assemble artifacts" + + - script: | + set -e -x + ls -R "$(Build.ArtifactStagingDirectory)" + displayName: "List staged artifacts" + + - script: | + set -e -x + shasum -a 256 "$(Build.ArtifactStagingDirectory)/pod-archive-onnxruntime-c-${ORT_POD_VERSION}.zip" + displayName: "Print ORT iOS Pod checksum" + + + # copy the pod archive to a path relative to Package.swift and set the env var required by Package.swift to use that. + # xcodebuild will implicitly use Package.swift and build/run the .testTarget (tests in swift/onnxTests). + # once that's done cleanup the copy of the pod zip file + - script: | + set -e -x + cp "$(Build.ArtifactStagingDirectory)/pod-archive-onnxruntime-c-${ORT_POD_VERSION}.zip" swift/ + export ORT_IOS_POD_LOCAL_PATH="swift/pod-archive-onnxruntime-c-${ORT_POD_VERSION}.zip" + xcodebuild test -scheme onnxruntime -destination 'platform=iOS Simulator,name=iPhone 14' + rm swift/pod-archive-onnxruntime-c-*.zip + displayName: "Test Package.swift usage" + + - publish: "$(Build.ArtifactStagingDirectory)" + artifact: ios_packaging_artifacts + displayName: "Publish artifacts" + + - template: templates/component-governance-component-detection-steps.yml + parameters : + condition : 'succeeded' diff --git a/tools/ci_build/github/scripts/install_protobuf.sh b/tools/ci_build/github/scripts/install_protobuf.sh new file mode 100644 index 0000000..b912b8e --- /dev/null +++ b/tools/ci_build/github/scripts/install_protobuf.sh @@ -0,0 +1,59 @@ +#!/bin/bash +set -e -x + +INSTALL_PREFIX='/usr' +DEP_FILE_PATH='/tmp/scripts/deps.txt' +while getopts "p:d:" parameter_Option +do case "${parameter_Option}" +in +p) INSTALL_PREFIX=${OPTARG};; +d) DEP_FILE_PATH=${OPTARG};; +esac +done + +EXTRA_CMAKE_ARGS="" + +case "$(uname -s)" in + Darwin*) + echo 'Building ONNX Runtime on Mac OS X' + EXTRA_CMAKE_ARGS="-DCMAKE_OSX_ARCHITECTURES=x86_64;arm64" + ;; + Linux*) + # Depending on how the compiler has been configured when it was built, sometimes "gcc -dumpversion" shows the full version. + GCC_VERSION=$(gcc -dumpversion | cut -d . -f 1) + #-fstack-clash-protection prevents attacks based on an overlapping heap and stack. + if [ "$GCC_VERSION" -ge 8 ]; then + CFLAGS="$CFLAGS -fstack-clash-protection" + CXXFLAGS="$CXXFLAGS -fstack-clash-protection" + fi + ARCH=$(uname -m) + + if [ "$ARCH" == "x86_64" ] && [ "$GCC_VERSION" -ge 9 ]; then + CFLAGS="$CFLAGS -fcf-protection" + CXXFLAGS="$CXXFLAGS -fcf-protection" + fi + export CFLAGS + export CXXFLAGS + ;; + *) + exit 1 +esac +mkdir -p $INSTALL_PREFIX +echo "Installing protobuf ..." +protobuf_url=$(grep '^protobuf' $DEP_FILE_PATH | cut -d ';' -f 2 ) +if [[ "$protobuf_url" = https* ]]; then + protobuf_url=$(echo $protobuf_url | sed 's/\.zip$/\.tar.gz/') + curl -sSL --retry 5 --retry-delay 10 --create-dirs --fail -L -o protobuf_src.tar.gz $protobuf_url + mkdir protobuf + cd protobuf + tar -zxf ../protobuf_src.tar.gz --strip=1 +else + cp $protobuf_url protobuf_src.zip + unzip protobuf_src.zip + cd protobuf-* +fi + +cmake . -DCMAKE_INSTALL_PREFIX=$INSTALL_PREFIX -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release -Dprotobuf_WITH_ZLIB_DEFAULT=OFF -Dprotobuf_BUILD_SHARED_LIBS=OFF $EXTRA_CMAKE_ARGS +make -j$(getconf _NPROCESSORS_ONLN) +make install +cd .. \ No newline at end of file diff --git a/tools/ci_build/github/templates/component-governance-component-detection-steps.yml b/tools/ci_build/github/templates/component-governance-component-detection-steps.yml new file mode 100644 index 0000000..daea499 --- /dev/null +++ b/tools/ci_build/github/templates/component-governance-component-detection-steps.yml @@ -0,0 +1,18 @@ +# component detection for component governance checks +parameters: +- name: condition + type: string + default: 'succeeded' # could be 'ci_only', 'always', 'succeeded' + +steps: +- ${{ if eq(variables['System.TeamProject'], 'Lotus') }}: + - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection' + condition: + or(or(and(eq('${{parameters.condition}}', 'ci_only'), and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Scheduled'))), + and(eq('${{parameters.condition}}', 'always'), always())), + and(eq('${{parameters.condition}}', 'succeeded'), succeeded())) + inputs: + # ignore dmlc-core tracker for its CI, which is not used in onnxruntime build + # ignore unit tests in emscripten. emscripten unit tests are not used in onnxruntime build + ignoreDirectories: '$(Build.SourcesDirectory)/cmake/external/emsdk/upstream/emscripten/tests' diff --git a/tools/ci_build/github/templates/install-appcenter.yml b/tools/ci_build/github/templates/install-appcenter.yml new file mode 100644 index 0000000..51be73d --- /dev/null +++ b/tools/ci_build/github/templates/install-appcenter.yml @@ -0,0 +1,12 @@ +# Install appcenter CLI + +parameters: +- name: appcenterVersion + type: string + default: "2.13.7" + +steps: +- bash: | + set -e -x + npm install -g appcenter-cli@${{ parameters.appcenterVersion }} + displayName: Install appcenter CLI ${{ parameters.appcenterVersion }} diff --git a/tools/ci_build/github/templates/use-xcode-version.yml b/tools/ci_build/github/templates/use-xcode-version.yml new file mode 100644 index 0000000..c267ecd --- /dev/null +++ b/tools/ci_build/github/templates/use-xcode-version.yml @@ -0,0 +1,14 @@ +# Specify use of a specific Xcode version. + +parameters: +- name: xcodeVersion + type: string + default: "14.3" + +steps: +- bash: | + set -e -x + XCODE_DEVELOPER_DIR="/Applications/Xcode_${{ parameters.xcodeVersion }}.app/Contents/Developer" + sudo xcode-select --switch "${XCODE_DEVELOPER_DIR}" + + displayName: Use Xcode ${{ parameters.xcodeVersion }} \ No newline at end of file diff --git a/version.txt b/version.txt new file mode 100644 index 0000000..71bd5d9 --- /dev/null +++ b/version.txt @@ -0,0 +1 @@ +1.16.0 \ No newline at end of file