Merge branch 'main' into add-default-params

Author: jonsimantov

.github/workflows/android.yml

@@ -13,7 +13,7 @@ on:
 env:
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
   GITHUB_TOKEN: ${{ github.token }}
-  xcodeVersion: "14.1"  # Only affects Mac runners, and only for prerequisites.
+  xcodeVersion: "16.2"  # Only affects Mac runners, and only for prerequisites.
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.head_ref || github.ref }}
@@ -50,6 +50,15 @@ jobs:
         os: ${{ fromJson(needs.prepare_matrix.outputs.matrix_os) }}
         architecture: ${{ fromJson(needs.prepare_matrix.outputs.matrix_architecture) }}
         python_version: ${{ fromJson(needs.prepare_matrix.outputs.matrix_python_version) }}
+        exclude:
+        # Do not attempt to use arm64 on Windows or Linux.
+        - os: windows-latest
+          architecture: arm64
+        - os: ubuntu-22.04
+          architecture: arm64
+        # Do not attempt to use x64 on Mac.
+        - os: macos-14
+          architecture: x64
     steps:
       - uses: lukka/get-cmake@latest
         with:
@@ -58,8 +67,13 @@ jobs:
         if: runner.os == 'macOS'
         run: sudo xcode-select -s /Applications/Xcode_${{ env.xcodeVersion }}.app/Contents/Developer
       - name: Force Java 11
+        if: runner.os != 'macOS'
         shell: bash
         run: echo "JAVA_HOME=${JAVA_HOME_11_X64}" >> $GITHUB_ENV
+      - name: Force Java 11 (mac)
+        if: runner.os == 'macOS'
+        shell: bash
+        run: echo "JAVA_HOME=${JAVA_HOME_11_arm64}" >> $GITHUB_ENV
       - name: Store git credentials for all git commands
         # Forces all git commands to use authenticated https, to prevent throttling.
         shell: bash

.github/workflows/cpp-packaging.yml

@@ -32,9 +32,9 @@ env:
   demumbleVer: "df938e45c2b0e064fb5323d88b692d03b451d271"
   # Use SHA256 for hashing files.
   hashCommand: "sha256sum"
-  # Xcode version 15.1 is the version we build the SDK with.
+  # Xcode version 16.2 is the version we build the SDK with.
   # Our MacOS runners will use the version in /Applications/Xcode_${xcodeVersion}.app
-  xcodeVersion: "15.1"
+  xcodeVersion: "16.2"
   # LLVM version with ARM MachO support has no version number yet.
   llvmVer: "5f187f0afaad33013ba03454c4749d99b1362534"
   GITHUB_TOKEN: ${{ github.token }}
@@ -79,13 +79,13 @@ jobs:
     if: ${{ github.event.inputs.downloadPublicVersion == '' && github.event.inputs.downloadPreviousRun == '' }}
     strategy:
       matrix:
-        os: [ubuntu-22.04, macos-13]
+        os: [ubuntu-22.04, macos-14]
         include:
         - os: ubuntu-22.04
           tools_platform: linux
           # Binutils 2.35.1 released Sep 19, 2020
           binutils_version: "2.35.1"
-        - os: macos-13
+        - os: macos-14
           tools_platform: darwin
           # Binutils 2.35.1 released Sep 19, 2020
           binutils_version: "2.35.1"
@@ -188,7 +188,7 @@ jobs:
 
   build_and_package_ios_tvos:
     name: build-and-package-ios-tvos
-    runs-on: macos-13
+    runs-on: macos-14
     if: ${{ github.event.inputs.downloadPublicVersion == '' && github.event.inputs.downloadPreviousRun == '' }}
     steps:
       - uses: lukka/get-cmake@latest
@@ -317,7 +317,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [windows-latest, ubuntu-22.04, macos-13]
+        os: [windows-latest, ubuntu-22.04, macos-14]
         build_type: ["Release", "Debug"]
         architecture: ["x64", "x86", "arm64"]
         msvc_runtime: ["static", "dynamic"]
@@ -335,21 +335,21 @@ jobs:
           vcpkg_triplet_suffix: "linux"
           additional_build_flags: ""
           sdk_platform: "linux"
-        - os: macos-13
+        - os: macos-14
           vcpkg_triplet_suffix: "osx"
           additional_build_flags: "--target_format libraries"
           sdk_platform: "darwin"
 
         exclude:
         - os: windows-latest
           linux_abi: "c++11"
-        - os: macos-13
+        - os: macos-14
           architecture: "x86"
-        - os: macos-13
+        - os: macos-14
           msvc_runtime: "dynamic"
-        - os: macos-13
+        - os: macos-14
           linux_abi: "c++11"
-        - os: macos-13
+        - os: macos-14
           build_type: "Debug"
         - os: ubuntu-22.04
           msvc_runtime: "dynamic"
@@ -504,7 +504,7 @@ jobs:
           suffix: '-x64-Debug-dynamic'
           runs_on_platform: ubuntu-22.04
         - sdk_platform: darwin
-          runs_on_platform: macos-13
+          runs_on_platform: macos-14
         exclude:
         - sdk_platform: windows
           suffix: ''

.github/workflows/desktop.yml

@@ -72,11 +72,13 @@ jobs:
           # msvc_runtime excludes
           - os: ubuntu-22.04
             msvc_runtime: "dynamic"
-          - os: macos-13
+          - os: macos-14
             msvc_runtime: "dynamic"
           # architecture excluees
-          - os: macos-13
+          - os: macos-14
             architecture: "x86"
+          - os: macos-14
+            architecture: "x64"
           # Xcode excludes -- allow only one on osx and linux
           - os: ubuntu-22.04
             xcode_version: "11.7"
@@ -168,9 +170,17 @@ jobs:
 
       - name: Setup python
         uses: actions/setup-python@v4
+        if: startsWith(matrix.os, 'ubuntu')
+        with:
+          python-version: ${{ matrix.python_version }}
+          architecture: x64
+
+      - name: Setup python (Mac)
+        uses: actions/setup-python@v4
+        if: startsWith(matrix.os, 'macos')
         with:
           python-version: ${{ matrix.python_version }}
-          architecture: 'x64'
+          architecture: ${{ matrix.architecture }}
 
       - name: Install Desktop SDK prerequisites
         uses: nick-invision/retry@v2

.github/workflows/integration_tests.yml

@@ -17,11 +17,11 @@ on:
         required: true
       apis:
         description: 'CSV of apis to build and test'
-        default: 'analytics,app_check,auth,database,dynamic_links,firestore,functions,gma,installations,messaging,remote_config,storage'
+        default: 'analytics,app_check,auth,database,dynamic_links,firestore,functions,gma,installations,messaging,remote_config,storage,ump'
         required: true
       operating_systems:
         description: 'CSV of VMs to run on'
-        default: 'ubuntu-22.04,windows-latest,macos-13'
+        default: 'ubuntu-22.04,windows-latest,macos-14'
         required: true
       desktop_ssl_variants:
         description: 'CSV of desktop SSL variants to use'
@@ -49,14 +49,9 @@ env:
   triggerLabelFull: "tests-requested: full"
   triggerLabelQuick: "tests-requested: quick"
   pythonVersion: '3.8'
-  xcodeVersion: '15.1'
+  xcodeVersion: '16.2'
   artifactRetentionDays: 2
   GITHUB_TOKEN: ${{ github.token }}
-  # All self-hosted ARM Mac runners should have this label. Due to how
-  # our custom reporting works, it must be exactly two words separated
-  # by a hyphen. The first word must be "macos". The second word will
-  # be omitted from the summary log.
-  runnerLabelMacArm64: "macos-m1custom"
 
 jobs:
   check_and_prepare:
@@ -77,8 +72,6 @@ jobs:
       xcode_version: ${{ steps.matrix_config.outputs.xcode_version }}
       ios_device: ${{ steps.matrix_config.outputs.ios_device }}
       tvos_device: ${{ steps.matrix_config.outputs.tvos_device }}
-      # Copy the runner label here because matrix specifiers cannot see env.
-      runner_label_macos_arm64: ${{ env.runnerLabelMacArm64 }}
     steps:
     ### Fail the workflow if the user does not have admin access to run the tests.
     - name: Check if user has permission to trigger tests
@@ -192,7 +185,7 @@ jobs:
         # list. Then we can use fromJson to define the field in the matrix for the tests job.
         if [[ "${{ github.event.schedule }}" == "0 9 * * *" ]]; then
           # at 1am PST/2am PDT. Running integration tests and generate test report for all testapps except firestore
-          apis="analytics,app_check,auth,database,dynamic_links,functions,gma,installations,messaging,remote_config,storage"
+          apis="analytics,app_check,auth,database,dynamic_links,functions,gma,installations,messaging,remote_config,storage,ump"
           echo "::warning ::Running main nightly tests"
         elif [[ "${{ github.event.schedule }}" == "0 10 * * *" || "${{ github.event.schedule }}" == "0 11 * * *" ]]; then
           # at 2am PST/3am PDT and 3am PST/4am PDT. Running integration tests for firestore and generate test report.
@@ -206,7 +199,7 @@ jobs:
           # at 3am PST/4am PDT. Running firestore desktop integration test aginst tip-of-tree ios repo
           echo "::warning ::Running against Firestore tip-of-tree"
           matrix_platform="Desktop"
-          matrix_os=$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k os -o "ubuntu-22.04,macos-13")
+          matrix_os=$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k os -o "ubuntu-22.04,macos-14")
         else
           matrix_platform=$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k platform -o "${{github.event.inputs.platforms}}" --apis ${apis} )
           matrix_os=$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k os -o "${{github.event.inputs.operating_systems}}")
@@ -273,18 +266,18 @@ jobs:
         - os: ubuntu-22.04
           arch: arm64
         # Do not attempt to use x86 on Mac.
-        - os: macos-13
+        - os: macos-14
           arch: x86
         # Until we support building openssl from source, we can't use the
-        # system's openssl when cross-compiling, except on Linux. Builds all
-        # happen on x64 machines, so arm64 and x86 are technically
-        # cross-compiling.
+        # system's openssl when cross-compiling, except on Linux. Builds on Linux
+        # happen on x64 machines, so x86 is technically cross-compiling. Builds on
+        # Mac happen on arm64 machines, so x64 is technically cross-compiling.
         - os: windows-latest
           ssl_variant: openssl
           arch: x86
-        - os: macos-13
+        - os: macos-14
           ssl_variant: openssl
-          arch: arm64
+          arch: x64
     steps:
       - uses: lukka/get-cmake@latest
         with:
@@ -353,7 +346,7 @@ jobs:
           elif [[ "${{ github.event.inputs.firestore_dep_source }}" ]]; then
             additional_flags+=(--cmake_flag=-DFIRESTORE_DEP_SOURCE=${{ github.event.inputs.firestore_dep_source }})
           fi
-          python scripts/gha/build_testapps.py --p Desktop \
+          VERBOSE=1 python scripts/gha/build_testapps.py --p Desktop \
             --t ${{ needs.check_and_prepare.outputs.apis }} \
             --output_directory "${{ github.workspace }}" \
             --artifact_name "desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}" \
@@ -435,8 +428,13 @@ jobs:
           ref: ${{needs.check_and_prepare.outputs.github_ref}}
           submodules: true
       - name: Force Java 11
+        if: ${{ !(runner.os == 'macOS') }}
         shell: bash
         run: echo "JAVA_HOME=${JAVA_HOME_11_X64}" >> $GITHUB_ENV
+      - name: Force Java 11 (mac)
+        if: ${{ runner.os == 'macOS' }}
+        shell: bash
+        run: echo "JAVA_HOME=${JAVA_HOME_11_arm64}" >> $GITHUB_ENV
       - name: Add msbuild to PATH (Windows)
         if: startsWith(matrix.os, 'windows')
         uses: microsoft/[email protected]
@@ -539,7 +537,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [macos-13]
+        os: [macos-14]
     steps:
       - uses: lukka/get-cmake@latest
         with:
@@ -647,7 +645,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [macos-13]
+        os: [macos-14]
     steps:
       - uses: lukka/get-cmake@latest
         with:
@@ -767,20 +765,18 @@ jobs:
         - os: ubuntu-22.04
           arch: arm64
         # Do not attempt to use x86 on Mac.
-        - os: macos-13
+        - os: macos-14
           arch: x86
         # Until we support building openssl from source, we can't use the
-        # system's openssl when cross-compiling, except on Linux. Builds all
-        # happen on x64 machines, so arm64 and x86 are technically
-        # cross-compiling.
+        # system's openssl when cross-compiling, except on Linux. Builds on Linux
+        # happen on x64 machines, so x86 is technically cross-compiling. Builds on
+        # Mac happen on arm64 machines, so x64 is technically cross-compiling.
         - os: windows-latest
           ssl_variant: openssl
           arch: x86
-        # Custom for this matrix: MacOS GitHub-hosted runner cannot test arm64
-        # code. Exclude that scenario from running here; it will run in
-        # test_desktop_custom_runners.
-        - os: macos-13
-          arch: arm64
+        - os: macos-14
+          ssl_variant: openssl
+          arch: x64
     steps:
       - uses: actions/checkout@v3
         with:
@@ -925,7 +921,7 @@ jobs:
           distribution: 'temurin'
           java-version: '11'
       - name: Run Android integration tests on Emulator locally
-        timeout-minutes: 180
+        timeout-minutes: 240
         if: steps.device-info.outputs.device_type == 'virtual'
         run: |
           python scripts/gha/test_simulator.py --testapp_dir testapps \
@@ -1010,7 +1006,7 @@ jobs:
   test_ios:
     name: test-ios-${{ matrix.build_os }}-${{ matrix.ios_device }}-${{ matrix.test_type }}
     needs: [check_and_prepare, build_ios]
-    runs-on: macos-13
+    runs-on: macos-14
     if: contains(needs.check_and_prepare.outputs.matrix_platform, 'iOS') && needs.check_and_prepare.outputs.apis != '' && !cancelled()
     strategy:
       fail-fast: false
@@ -1024,7 +1020,7 @@ jobs:
              test_type: "uitest"
           -  ios_device: "ios_latest"
              test_type: "uitest"
-        build_os: [macos-13]
+        build_os: [macos-14]
     steps:
       - uses: actions/checkout@v3
         with:
@@ -1102,7 +1098,7 @@ jobs:
           security list-keychains -d user -s tmp-keychain
           security default-keychain -s tmp-keychain
       - name: Run iOS integration tests on Simulator locally
-        timeout-minutes: 180
+        timeout-minutes: 240
         if: steps.device-info.outputs.device_type == 'virtual'
         run: |
           python scripts/gha/test_simulator.py --testapp_dir testapps \
@@ -1113,7 +1109,7 @@ jobs:
       - id: ftl_test
         if: steps.device-info.outputs.device_type == 'ftl'
         uses: FirebaseExtended/github-actions/[email protected]
-        timeout-minutes: 120
+        timeout-minutes: 180
         with:
           credentials_json: ${{ secrets.FIREBASE_SERVICE_ACCOUNT_CREDENTIALS }}
           testapp_dir: testapps
@@ -1189,13 +1185,13 @@ jobs:
   test_tvos:
     name: test-tvos-${{ matrix.build_os }}-${{ matrix.tvos_device }}
     needs: [check_and_prepare, build_tvos]
-    runs-on: macos-13
+    runs-on: macos-14
     if: contains(needs.check_and_prepare.outputs.matrix_platform, 'tvOS') && needs.check_and_prepare.outputs.apis != '' && !cancelled()
     strategy:
       fail-fast: false
       matrix:
         tvos_device: ${{ fromJson(needs.check_and_prepare.outputs.tvos_device) }}
-        build_os: [macos-13]
+        build_os: [macos-14]
     steps:
       - uses: actions/checkout@v3
         with:

.github/workflows/ios.yml

@@ -41,7 +41,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ 'macos-13' ]
+        os: [ 'macos-14' ]
         xcode_version: ${{ fromJson(needs.prepare_matrix.outputs.matrix_xcode_version) }}
     steps:
       - uses: lukka/get-cmake@latest
@@ -66,7 +66,7 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: 3.8
-          architecture: "x64"
+          architecture: "arm64"
 
       - name: Install prerequisites
         run: |

.github/workflows/update-dependencies.yml

@@ -30,7 +30,7 @@ env:
 jobs:
   update_dependencies:
     name: update-deps
-    runs-on: macos-13
+    runs-on: macos-14
     steps:
       - name: Get token for firebase-workflow-trigger
         uses: tibdex/github-app-token@v1