From 5f07cf3b14c0b0c87c05400daedb2baef76c9310 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gunnar=20Farneb=C3=A4ck?=
 <gunnar.farneback@contextvision.se>
Date: Wed, 11 Oct 2023 23:45:42 +0200
Subject: [PATCH 1/9] Support CUDA on Julia 1.9+ via a package extension.

---
 Project.toml                |  11 +++
 ext/CUDAExt.jl              |  11 +++
 src/ONNXRunTime.jl          |  12 ++-
 src/highlevel.jl            |  29 +++++-
 test/test_cuda_extension.jl | 178 ++++++++++++++++++++++++++++++++++++
 5 files changed, 233 insertions(+), 8 deletions(-)
 create mode 100644 ext/CUDAExt.jl
 create mode 100644 test/test_cuda_extension.jl

diff --git a/Project.toml b/Project.toml
index 95fa55b..98a5a89 100644
--- a/Project.toml
+++ b/Project.toml
@@ -16,13 +16,24 @@ Requires = "ae029012-a4dd-5104-9daa-d747884805df"
 [compat]
 ArgCheck = "2"
 CEnum = "0.4"
+CUDA = "4, 5"
 DataStructures = "0.18"
 DocStringExtensions = "0.8, 0.9"
 Requires = "1"
+cuDNN = "1.1"
 julia = "1.6"
 
+[extensions]
+CUDAExt = ["CUDA", "cuDNN"]
+
 [extras]
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
+cuDNN = "02a925ec-e4fe-4b08-9a7e-0d78e3d38ccd"
 
 [targets]
 test = ["Test"]
+
+[weakdeps]
+CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
+cuDNN = "02a925ec-e4fe-4b08-9a7e-0d78e3d38ccd"
diff --git a/ext/CUDAExt.jl b/ext/CUDAExt.jl
new file mode 100644
index 0000000..725111a
--- /dev/null
+++ b/ext/CUDAExt.jl
@@ -0,0 +1,11 @@
+module CUDAExt
+
+# These functions are only defined for diagnostic purposes. Otherwise
+# the CUDA extension only relies on the CUDA and cuDNN dependencies to
+# have loaded the libraries needed by ONNXRunTime's CUDA execution
+# provider.
+import CUDA
+cuda_functional() = CUDA.functional()
+cuda_runtime_version() = CUDA.runtime_version()
+
+end
diff --git a/src/ONNXRunTime.jl b/src/ONNXRunTime.jl
index 752c101..7f81d53 100644
--- a/src/ONNXRunTime.jl
+++ b/src/ONNXRunTime.jl
@@ -1,5 +1,7 @@
 module ONNXRunTime
-using Requires:@require
+if !isdefined(Base, :get_extension)
+    using Requires: @require
+end
 
 function _perm(arr::AbstractArray{T,N}) where {T,N}
     ntuple(i->N+1-i, N)
@@ -14,9 +16,11 @@ end
 include("capi.jl")
 include("highlevel.jl")
 
-function __init__()
-    @require CUDA="052768ef-5323-5732-b1bb-66c8b64840ba" begin
-        CUDA.functional() && include("cuda.jl")
+@static if !isdefined(Base, :get_extension)
+    function __init__()
+        @require CUDA="052768ef-5323-5732-b1bb-66c8b64840ba" begin
+            CUDA.functional() && include("cuda.jl")
+        end
     end
 end
 
diff --git a/src/highlevel.jl b/src/highlevel.jl
index 9952097..452861b 100644
--- a/src/highlevel.jl
+++ b/src/highlevel.jl
@@ -65,10 +65,31 @@ function load_inference(path::AbstractString; execution_provider::Symbol=:cpu,
     if execution_provider === :cpu
         session_options = CreateSessionOptions(api)
     elseif execution_provider === :cuda
-        if !(isdefined(@__MODULE__, :CUDA))
-            @warn """
-            The $(repr(execution_provider)) requires the CUDA.jl package to be available. Try adding `import CUDA` to your code.
-            """
+        if isdefined(Base, :get_extension)
+            CUDAExt = Base.get_extension(@__MODULE__, :CUDAExt)
+            if isnothing(CUDAExt)
+                @warn """
+                The $(repr(execution_provider)) execution provider requires the CUDA.jl and cuDNN.jl packages to be available. Try adding `import CUDA, cuDNN` to your code.
+                """
+            elseif !getfield(CUDAExt, :cuda_functional)()
+                @warn """
+                The $(repr(execution_provider)) execution provider requires CUDA to be functional. See `CUDA.functional`.
+                """
+            elseif !(v"11.8" <= getfield(CUDAExt, :cuda_runtime_version)() < v"12")
+                # Note: The supported version range is a property
+                # inherited from the CUDA runtime library and needs to
+                # be updated when the library is updated. It may be a
+                # good idea to centralize this information somewhere.
+                @warn """
+                The $(repr(execution_provider)) execution provider requires a CUDA runtime version of at least 11.8 but less than 12. See `CUDA.set_runtime_version!`.
+                """
+            end
+        else
+            if !isdefined(@__MODULE__, :CUDA)
+                @warn """
+                The $(repr(execution_provider)) execution provider requires the CUDA.jl package to be available. Try adding `import CUDA` to your code.
+                """
+            end
         end
         session_options = CreateSessionOptions(api)
         cuda_options = OrtCUDAProviderOptions()
diff --git a/test/test_cuda_extension.jl b/test/test_cuda_extension.jl
new file mode 100644
index 0000000..020f9b5
--- /dev/null
+++ b/test/test_cuda_extension.jl
@@ -0,0 +1,178 @@
+# This file is not included from `runtests.jl` nor run in CI.
+#
+# Run it with `julia tests/test_cuda_extension.jl`. This requires that
+# Julia is installed with juliaup and will involve downloading of a
+# lot of big artifacts. The output will contain lots of error messages
+# from caught errors; what matters is that all testsets pass.
+
+using Test
+
+juliaup_found = false
+try run(pipeline(`juliaup --version`, stdout = devnull, stderr = devnull))
+    global juliaup_found = true
+catch e
+end
+
+if !juliaup_found
+    error("`juliaup` needs to be installed for the CUDA extension tests")
+end
+
+wait(run(`juliaup add 1.6`, wait = false))
+wait(run(`juliaup add 1.9`, wait = false))
+
+package_path = dirname(@__DIR__)
+onnx_path = joinpath(@__DIR__, "data", "copy2d.onnx")
+
+function with_environment(f::Function; cuda_runtime_version)
+    mktempdir() do env
+        write(joinpath(env, "LocalPreferences.toml"),
+              """
+              [CUDA_Runtime_jll]
+              version = "$(cuda_runtime_version)"
+              """)
+        write(joinpath(env, "Project.toml"),
+              """
+              [extras]
+              CUDA_Runtime_jll = "76a88914-d11a-5bdc-97e0-2f5a05c973a2"
+              """)
+        f(env)
+    end
+end
+
+@testset "Julia 1.6 CUDA 3" begin
+    with_environment(cuda_runtime_version = "11.8") do env
+        install_script = """
+                         using Pkg
+                         Pkg.develop(path = "$(package_path)")
+                         Pkg.add(name = "CUDA", version = "3")
+                         """
+        @test success(run(`julia +1.6 --project=$(env) -e "$(install_script)"`))
+        # Correct dependency for :cuda.
+        test_script = """
+                      using ONNXRunTime, CUDA
+                      load_inference("$(onnx_path)", execution_provider = :cuda)
+                      """
+        @test success(run(`julia +1.6 --project=$(env) -e "$(test_script)"`))
+        # CUDA not loaded.
+        test_script = """
+                      using ONNXRunTime
+                      load_inference("$(onnx_path)", execution_provider = :cuda)
+                      """
+        @test_throws ProcessFailedException run(`julia +1.6 --project=$(env) -e "$(test_script)"`)
+        # CUDA not loaded but running on CPU, so it's fine.
+        test_script = """
+                      using ONNXRunTime
+                      load_inference("$(onnx_path)", execution_provider = :cpu)
+                      """
+        @test success(run(`julia +1.6 --project=$(env) -e "$(test_script)"`))
+    end
+end
+
+@testset "Julia 1.9 CUDA 3" begin
+    with_environment(cuda_runtime_version = "11.8") do env
+        install_script = """
+                         using Pkg
+                         Pkg.develop(path = "$(package_path)")
+                         Pkg.add(name = "CUDA", version = "3")
+                         """
+        # CUDA 3 is not possible to install together with ONNXRunTime
+        # on Julia 1.9 due to Compat requirements.
+        @test_throws ProcessFailedException run(`julia +1.9 --project=$(env) -e "$(install_script)"`)
+    end
+end
+
+@testset "Julia 1.9 CUDA.jl $(cuda_version) CUDA runtime 11.8" for cuda_version in (4, 5)
+    with_environment(cuda_runtime_version = "11.8") do env
+        install_script = """
+                         using Pkg
+                         Pkg.develop(path = "$(package_path)")
+                         Pkg.add(name = "CUDA", version = "$(cuda_version)")
+                         Pkg.add(name = "cuDNN")
+                         """
+        @test success(run(`julia +1.9 --project=$(env) -e "$(install_script)"`))
+        # Correct dependencies for :cuda.
+        test_script = """
+                      using ONNXRunTime, CUDA, cuDNN
+                      load_inference("$(onnx_path)", execution_provider = :cuda)
+                      """
+        @test success(run(`julia +1.9 --project=$(env) -e "$(test_script)"`))
+        # Neither CUDA nor cuDNN loaded.
+        test_script = """
+                      using ONNXRunTime
+                      load_inference("$(onnx_path)", execution_provider = :cuda)
+                      """
+        @test_throws ProcessFailedException run(`julia +1.9 --project=$(env) -e "$(test_script)"`)
+        # Neither CUDA nor cuDNN loaded but running on CPU, so it's fine.
+        test_script = """
+                      using ONNXRunTime
+                      load_inference("$(onnx_path)", execution_provider = :cpu)
+                      """
+        # CUDA not loaded. Well, cuDNN pulls in CUDA so this passes anyway.
+        test_script = """
+                      using ONNXRunTime
+                      using cuDNN
+                      load_inference("$(onnx_path)", execution_provider = :cuda)
+                      """
+        @test success(run(`julia +1.9 --project=$(env) -e "$(test_script)"`))
+        # CUDA not loaded but running on CPU, so it's fine.
+        test_script = """
+                      using ONNXRunTime
+                      using cuDNN
+                      load_inference("$(onnx_path)", execution_provider = :cpu)
+                      """
+        @test success(run(`julia +1.9 --project=$(env) -e "$(test_script)"`))
+        # cuDNN not loaded.
+        test_script = """
+                      using ONNXRunTime
+                      using CUDA
+                      load_inference("$(onnx_path)", execution_provider = :cuda)
+                      """
+        @test_throws ProcessFailedException run(`julia +1.9 --project=$(env) -e "$(test_script)"`)
+        # cuDNN not loaded but running on CPU, so it's fine.
+        test_script = """
+                      using ONNXRunTime
+                      using CUDA
+                      load_inference("$(onnx_path)", execution_provider = :cpu)
+                      """
+        @test success(run(`julia +1.9 --project=$(env) -e "$(test_script)"`))
+    end
+end
+
+@testset "Julia 1.9 CUDA.jl $(cuda_version) CUDA runtime 11.6" for cuda_version in (4, 5)
+    with_environment(cuda_runtime_version = "11.6") do env
+        install_script = """
+                         using Pkg
+                         Pkg.develop(path = "$(package_path)")
+                         Pkg.add(name = "CUDA", version = "$(cuda_version)")
+                         Pkg.add(name = "cuDNN")
+                         """
+        @test success(run(`julia +1.9 --project=$(env) -e "$(install_script)"`))
+        # Correct dependencies for :cuda. CUDA runtime version is
+        # lower than officially supported but close enough to at least
+        # load so there will be a warning but no error.
+        test_script = """
+                      using ONNXRunTime, CUDA, cuDNN
+                      load_inference("$(onnx_path)", execution_provider = :cuda)
+                      """
+        @test success(run(`julia +1.9 --project=$(env) -e "$(test_script)"`))
+    end
+end
+
+@testset "Julia 1.9 CUDA.jl $(cuda_version) CUDA runtime 12.1" for cuda_version in (4, 5)
+    with_environment(cuda_runtime_version = "12.1") do env
+        install_script = """
+                         using Pkg
+                         Pkg.develop(path = "$(package_path)")
+                         Pkg.add(name = "CUDA", version = "$(cuda_version)")
+                         Pkg.add(name = "cuDNN")
+                         """
+        @test success(run(`julia +1.9 --project=$(env) -e "$(install_script)"`))
+        # Correct dependencies for :cuda but fails due to bad version
+        # of CUDA runtime.
+        test_script = """
+                      using ONNXRunTime, CUDA, cuDNN
+                      load_inference("$(onnx_path)", execution_provider = :cuda)
+                      """
+        @test_throws ProcessFailedException run(`julia +1.9 --project=$(env) -e "$(test_script)"`)
+    end
+end

From ab407a32bd0b30dbb67bce359af5151dfda29c80 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gunnar=20Farneb=C3=A4ck?=
 <gunnar.farneback@contextvision.se>
Date: Thu, 12 Oct 2023 10:25:06 +0200
Subject: [PATCH 2/9] Drop support for Julia < 1.9 and CUDA < 4. Remove
 Requires machinery.

---
 .github/workflows/CI.yml    |  2 +-
 Project.toml                |  6 ++----
 src/ONNXRunTime.jl          | 11 ----------
 src/highlevel.jl            | 42 +++++++++++++++----------------------
 test/LocalPreferences.toml  |  2 ++
 test/Project.toml           |  7 ++++++-
 test/test_cuda.jl           |  1 +
 test/test_cuda_extension.jl | 29 -------------------------
 8 files changed, 29 insertions(+), 71 deletions(-)
 create mode 100644 test/LocalPreferences.toml

diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
index f8de212..cdefe92 100644
--- a/.github/workflows/CI.yml
+++ b/.github/workflows/CI.yml
@@ -10,7 +10,7 @@ jobs:
       fail-fast: false
       matrix:
         version:
-          - '1.6'
+          - '1.9'
           - '1'
           - 'nightly'
         os:
diff --git a/Project.toml b/Project.toml
index 98a5a89..383886e 100644
--- a/Project.toml
+++ b/Project.toml
@@ -11,7 +11,6 @@ DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
 LazyArtifacts = "4af54fe1-eca0-43a8-85a7-787d91b784e3"
 Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
-Requires = "ae029012-a4dd-5104-9daa-d747884805df"
 
 [compat]
 ArgCheck = "2"
@@ -19,9 +18,8 @@ CEnum = "0.4"
 CUDA = "4, 5"
 DataStructures = "0.18"
 DocStringExtensions = "0.8, 0.9"
-Requires = "1"
 cuDNN = "1.1"
-julia = "1.6"
+julia = "1.9"
 
 [extensions]
 CUDAExt = ["CUDA", "cuDNN"]
@@ -32,7 +30,7 @@ CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
 cuDNN = "02a925ec-e4fe-4b08-9a7e-0d78e3d38ccd"
 
 [targets]
-test = ["Test"]
+test = ["Test", "CUDA", "cuDNN"]
 
 [weakdeps]
 CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
diff --git a/src/ONNXRunTime.jl b/src/ONNXRunTime.jl
index 7f81d53..9e921ed 100644
--- a/src/ONNXRunTime.jl
+++ b/src/ONNXRunTime.jl
@@ -1,7 +1,4 @@
 module ONNXRunTime
-if !isdefined(Base, :get_extension)
-    using Requires: @require
-end
 
 function _perm(arr::AbstractArray{T,N}) where {T,N}
     ntuple(i->N+1-i, N)
@@ -16,12 +13,4 @@ end
 include("capi.jl")
 include("highlevel.jl")
 
-@static if !isdefined(Base, :get_extension)
-    function __init__()
-        @require CUDA="052768ef-5323-5732-b1bb-66c8b64840ba" begin
-            CUDA.functional() && include("cuda.jl")
-        end
-    end
-end
-
 end #module
diff --git a/src/highlevel.jl b/src/highlevel.jl
index 452861b..2ecd988 100644
--- a/src/highlevel.jl
+++ b/src/highlevel.jl
@@ -65,31 +65,23 @@ function load_inference(path::AbstractString; execution_provider::Symbol=:cpu,
     if execution_provider === :cpu
         session_options = CreateSessionOptions(api)
     elseif execution_provider === :cuda
-        if isdefined(Base, :get_extension)
-            CUDAExt = Base.get_extension(@__MODULE__, :CUDAExt)
-            if isnothing(CUDAExt)
-                @warn """
-                The $(repr(execution_provider)) execution provider requires the CUDA.jl and cuDNN.jl packages to be available. Try adding `import CUDA, cuDNN` to your code.
-                """
-            elseif !getfield(CUDAExt, :cuda_functional)()
-                @warn """
-                The $(repr(execution_provider)) execution provider requires CUDA to be functional. See `CUDA.functional`.
-                """
-            elseif !(v"11.8" <= getfield(CUDAExt, :cuda_runtime_version)() < v"12")
-                # Note: The supported version range is a property
-                # inherited from the CUDA runtime library and needs to
-                # be updated when the library is updated. It may be a
-                # good idea to centralize this information somewhere.
-                @warn """
-                The $(repr(execution_provider)) execution provider requires a CUDA runtime version of at least 11.8 but less than 12. See `CUDA.set_runtime_version!`.
-                """
-            end
-        else
-            if !isdefined(@__MODULE__, :CUDA)
-                @warn """
-                The $(repr(execution_provider)) execution provider requires the CUDA.jl package to be available. Try adding `import CUDA` to your code.
-                """
-            end
+        CUDAExt = Base.get_extension(@__MODULE__, :CUDAExt)
+        if isnothing(CUDAExt)
+            @warn """
+            The $(repr(execution_provider)) execution provider requires the CUDA.jl and cuDNN.jl packages to be available. Try adding `import CUDA, cuDNN` to your code.
+            """
+        elseif !getfield(CUDAExt, :cuda_functional)()
+            @warn """
+            The $(repr(execution_provider)) execution provider requires CUDA to be functional. See `CUDA.functional`.
+            """
+        elseif !(v"11.8" <= getfield(CUDAExt, :cuda_runtime_version)() < v"12")
+            # Note: The supported version range is a property
+            # inherited from the CUDA runtime library and needs to
+            # be updated when the library is updated. It may be a
+            # good idea to centralize this information somewhere.
+            @warn """
+            The $(repr(execution_provider)) execution provider requires a CUDA runtime version of at least 11.8 but less than 12. See `CUDA.set_runtime_version!`.
+            """
         end
         session_options = CreateSessionOptions(api)
         cuda_options = OrtCUDAProviderOptions()
diff --git a/test/LocalPreferences.toml b/test/LocalPreferences.toml
new file mode 100644
index 0000000..5da06c7
--- /dev/null
+++ b/test/LocalPreferences.toml
@@ -0,0 +1,2 @@
+[CUDA_Runtime_jll]
+version = "11.8"
diff --git a/test/Project.toml b/test/Project.toml
index 03a1a32..2fe8444 100644
--- a/test/Project.toml
+++ b/test/Project.toml
@@ -1,6 +1,11 @@
 [deps]
 CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+cuDNN = "02a925ec-e4fe-4b08-9a7e-0d78e3d38ccd"
 
 [compat]
-CUDA = "3"
+CUDA = "5"
+cuDNN = "1.2"
+
+[extras]
+CUDA_Runtime_jll = "76a88914-d11a-5bdc-97e0-2f5a05c973a2"
diff --git a/test/test_cuda.jl b/test/test_cuda.jl
index 7ef1f18..8db6c3f 100644
--- a/test/test_cuda.jl
+++ b/test/test_cuda.jl
@@ -1,5 +1,6 @@
 module TestCUDA
 import CUDA
+import cuDNN
 using Test
 using ONNXRunTime
 const ORT = ONNXRunTime
diff --git a/test/test_cuda_extension.jl b/test/test_cuda_extension.jl
index 020f9b5..a82e512 100644
--- a/test/test_cuda_extension.jl
+++ b/test/test_cuda_extension.jl
@@ -39,35 +39,6 @@ function with_environment(f::Function; cuda_runtime_version)
     end
 end
 
-@testset "Julia 1.6 CUDA 3" begin
-    with_environment(cuda_runtime_version = "11.8") do env
-        install_script = """
-                         using Pkg
-                         Pkg.develop(path = "$(package_path)")
-                         Pkg.add(name = "CUDA", version = "3")
-                         """
-        @test success(run(`julia +1.6 --project=$(env) -e "$(install_script)"`))
-        # Correct dependency for :cuda.
-        test_script = """
-                      using ONNXRunTime, CUDA
-                      load_inference("$(onnx_path)", execution_provider = :cuda)
-                      """
-        @test success(run(`julia +1.6 --project=$(env) -e "$(test_script)"`))
-        # CUDA not loaded.
-        test_script = """
-                      using ONNXRunTime
-                      load_inference("$(onnx_path)", execution_provider = :cuda)
-                      """
-        @test_throws ProcessFailedException run(`julia +1.6 --project=$(env) -e "$(test_script)"`)
-        # CUDA not loaded but running on CPU, so it's fine.
-        test_script = """
-                      using ONNXRunTime
-                      load_inference("$(onnx_path)", execution_provider = :cpu)
-                      """
-        @test success(run(`julia +1.6 --project=$(env) -e "$(test_script)"`))
-    end
-end
-
 @testset "Julia 1.9 CUDA 3" begin
     with_environment(cuda_runtime_version = "11.8") do env
         install_script = """

From 34d7e7e63dc7ca223a514daccf39b871f61fec0a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gunnar=20Farneb=C3=A4ck?=
 <gunnar.farneback@contextvision.se>
Date: Thu, 12 Oct 2023 16:00:06 +0200
Subject: [PATCH 3/9] Error instead of warn in cases where a later error is
 unavoidable.

---
 src/highlevel.jl | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/highlevel.jl b/src/highlevel.jl
index 2ecd988..2f9fe39 100644
--- a/src/highlevel.jl
+++ b/src/highlevel.jl
@@ -67,18 +67,22 @@ function load_inference(path::AbstractString; execution_provider::Symbol=:cpu,
     elseif execution_provider === :cuda
         CUDAExt = Base.get_extension(@__MODULE__, :CUDAExt)
         if isnothing(CUDAExt)
-            @warn """
+            error("""
             The $(repr(execution_provider)) execution provider requires the CUDA.jl and cuDNN.jl packages to be available. Try adding `import CUDA, cuDNN` to your code.
-            """
+            """)
         elseif !getfield(CUDAExt, :cuda_functional)()
-            @warn """
+            error("""
             The $(repr(execution_provider)) execution provider requires CUDA to be functional. See `CUDA.functional`.
-            """
+            """)
         elseif !(v"11.8" <= getfield(CUDAExt, :cuda_runtime_version)() < v"12")
             # Note: The supported version range is a property
             # inherited from the CUDA runtime library and needs to
             # be updated when the library is updated. It may be a
             # good idea to centralize this information somewhere.
+            #
+            # Only warning here since it's plausible that it might
+            # work with some lower 11.x versions than officially
+            # supported.
             @warn """
             The $(repr(execution_provider)) execution provider requires a CUDA runtime version of at least 11.8 but less than 12. See `CUDA.set_runtime_version!`.
             """

From 2d699db80eaf3a43f7eccfe68dddf801019deef0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gunnar=20Farneb=C3=A4ck?=
 <gunnar.farneback@contextvision.se>
Date: Thu, 12 Oct 2023 16:01:59 +0200
Subject: [PATCH 4/9] Remove stray Julia 1.6 installation step.

---
 test/test_cuda_extension.jl | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/test/test_cuda_extension.jl b/test/test_cuda_extension.jl
index a82e512..6a5aca7 100644
--- a/test/test_cuda_extension.jl
+++ b/test/test_cuda_extension.jl
@@ -1,4 +1,4 @@
-# This file is not included from `runtests.jl` nor run in CI.
+# This file is neither included from `runtests.jl` nor run in CI.
 #
 # Run it with `julia tests/test_cuda_extension.jl`. This requires that
 # Julia is installed with juliaup and will involve downloading of a
@@ -17,7 +17,6 @@ if !juliaup_found
     error("`juliaup` needs to be installed for the CUDA extension tests")
 end
 
-wait(run(`juliaup add 1.6`, wait = false))
 wait(run(`juliaup add 1.9`, wait = false))
 
 package_path = dirname(@__DIR__)

From 7cc081601e08890162b58227e25948fe50a28ef6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gunnar=20Farneb=C3=A4ck?=
 <gunnar.farneback@contextvision.se>
Date: Thu, 12 Oct 2023 17:00:42 +0200
Subject: [PATCH 5/9] Update README.md.

---
 README.md | 67 +++++++++++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 65 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 9a616df..e191b64 100644
--- a/README.md
+++ b/README.md
@@ -28,12 +28,24 @@ julia> model(input)
 Dict{String, Matrix{Float32}} with 1 entry:
   "output" => [2.68127 2.18192 0.525979; -0.135185 2.02199 3.75168]
 ```
-For GPU usage simply do:
+
+For GPU usage the CUDA and cuDNN packages are required and the CUDA
+runtime needs to be set to 11.8 or a later 11.x version. To set this
+up, do
+
 ```julia
-pkg> add CUDA
+pkg> add CUDA cuDNN
 
 julia> import CUDA
 
+julia> CUDA.set_runtime_version!(v"11.8")
+```
+
+Then GPU inference is simply
+
+```julia
+julia> import CUDA, cuDNN
+
 julia> ORT.load_inference(path, execution_provider=:cuda)
 ```
 
@@ -63,3 +75,54 @@ output_array = GetTensorMutableData(api, output_tensor);
 * Use the onnxruntime python bindings via [PyCall.jl](https://github.com/JuliaPy/PyCall.jl).
 * [ONNX.jl](https://github.com/FluxML/ONNX.jl)
 * [ONNXNaiveNASflux.jl](https://github.com/DrChainsaw/ONNXNaiveNASflux.jl)
+
+# Breaking Changes in version 1.0.
+
+* Support for CUDA.jl is changed from version 3 to versions 4 and 5.
+
+* Support for Julia versions less than 1.9 is dropped. The reason for
+  this is to switch the conditional support of GPUs from being based
+  on the Requires package to being a package extension. As a
+  consequence the ONNXRunTime GPU support can now be precompiled and
+  the CUDA.jl versions can be properly controlled via Compat.
+
+# Setting the CUDA Runtime Version in Tests
+
+For GPU tests using ONNXRunTime, naturally the tests must depend on
+and import CUDA and cuDNN. Additionally a supported CUDA runtime
+version needs to be used, which can be somewhat tricky to set up for
+the tests.
+
+First some background. What `CUDA.set_runtime_version!(v"11.8")`
+effectively does is to
+
+1. Add a `LocalPreferences.toml` file containing
+
+```
+[CUDA_Runtime_jll]
+version = "11.8"
+```
+
+2. In `Project.toml`, add
+```
+[extras]
+CUDA_Runtime_jll = "76a88914-d11a-5bdc-97e0-2f5a05c973a2"
+```
+
+If your test environment is defined by a `test` target in the top
+`Project.toml` you need to
+
+1. Add a `LocalPreferences.toml` in your top directory with the same
+contents as above.
+
+2. Add `CUDA_Runtime_jll` to the `extras` section of `Project.toml`.
+
+3. Add `CUDA_Runtime_jll` to the `test` target of `Project.toml`.
+
+If your test environment is defined by a `Project.toml` in the `test`
+directory, you instead need to
+
+1. Add a `test/LocalPreferences.toml` file with the same contents as
+above.
+
+2. Add `CUDA_Runtime_jll` to the `extras` section of `test/Project.toml`.

From dbe64982bf217e4d7136a8586bd201122215e449 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gunnar=20Farneb=C3=A4ck?=
 <gunnar.farneback@contextvision.se>
Date: Fri, 13 Oct 2023 10:27:55 +0200
Subject: [PATCH 6/9] Stricter CUDA runtime version checks and version
 consistency tests.

---
 src/ONNXRunTime.jl          |  1 +
 src/highlevel.jl            | 19 +++++++------------
 src/versions.jl             | 21 +++++++++++++++++++++
 test/runtests.jl            |  1 +
 test/test_cuda_extension.jl | 24 ++----------------------
 test/test_versions.jl       | 32 ++++++++++++++++++++++++++++++++
 6 files changed, 64 insertions(+), 34 deletions(-)
 create mode 100644 src/versions.jl
 create mode 100644 test/test_versions.jl

diff --git a/src/ONNXRunTime.jl b/src/ONNXRunTime.jl
index 9e921ed..067dfef 100644
--- a/src/ONNXRunTime.jl
+++ b/src/ONNXRunTime.jl
@@ -10,6 +10,7 @@ function reversedims_lazy(arr)
     PermutedDimsArray(arr, _perm(arr))
 end
 
+include("versions.jl")
 include("capi.jl")
 include("highlevel.jl")
 
diff --git a/src/highlevel.jl b/src/highlevel.jl
index 2f9fe39..5c6ccd4 100644
--- a/src/highlevel.jl
+++ b/src/highlevel.jl
@@ -74,18 +74,13 @@ function load_inference(path::AbstractString; execution_provider::Symbol=:cpu,
             error("""
             The $(repr(execution_provider)) execution provider requires CUDA to be functional. See `CUDA.functional`.
             """)
-        elseif !(v"11.8" <= getfield(CUDAExt, :cuda_runtime_version)() < v"12")
-            # Note: The supported version range is a property
-            # inherited from the CUDA runtime library and needs to
-            # be updated when the library is updated. It may be a
-            # good idea to centralize this information somewhere.
-            #
-            # Only warning here since it's plausible that it might
-            # work with some lower 11.x versions than officially
-            # supported.
-            @warn """
-            The $(repr(execution_provider)) execution provider requires a CUDA runtime version of at least 11.8 but less than 12. See `CUDA.set_runtime_version!`.
-            """
+        else
+            cuda_runtime_version = getfield(CUDAExt, :cuda_runtime_version)()
+            if !(cuda_runtime_supported_version <= cuda_runtime_version < cuda_runtime_upper_bound)
+                error("""
+                Found CUDA runtime version $(cuda_runtime_version). The $(repr(execution_provider)) execution provider requires a CUDA runtime version of at least $(cuda_runtime_supported_version) but less than $(cuda_runtime_upper_bound). See `CUDA.set_runtime_version!` and the package README.
+                """)
+            end
         end
         session_options = CreateSessionOptions(api)
         cuda_options = OrtCUDAProviderOptions()
diff --git a/src/versions.jl b/src/versions.jl
new file mode 100644
index 0000000..52c9a87
--- /dev/null
+++ b/src/versions.jl
@@ -0,0 +1,21 @@
+# Version number of the ONNXRunTime library and supported versions of
+# the CUDA runtime for GPU processing with the CUDA execution
+# provider.
+#
+# * `onnxruntime_version`: This number must match the version number
+#   reported by the ONNXRunTime library, which is verified in the
+#   tests. The only real purpose of this variable is to help keep the
+#   next one up to date when the library is updated.
+#
+# * `cuda_runtime_supported_version`: This is the lowest supported
+#   version of the ONNX runtime library, which should match the
+#   information from
+#   https://onnxruntime.ai/docs/execution-providers/CUDA-ExecutionProvider.html#requirements
+#
+# * `cuda_runtime_upper_bound`: The lowest CUDA runtime version which
+#   is *not* accepted. Presumably CUDA runtime follows semantic
+#   versioning so this can automatically be set to the next major
+#   version.
+const onnxruntime_version = v"1.15.1"
+const cuda_runtime_supported_version = v"11.8"
+const cuda_runtime_upper_bound = VersionNumber(cuda_runtime_supported_version.major + 1)
diff --git a/test/runtests.jl b/test/runtests.jl
index 601346f..4e7cb27 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -1,3 +1,4 @@
+include("test_versions.jl")
 include("test_highlevel.jl")
 include("test_capi.jl")
 
diff --git a/test/test_cuda_extension.jl b/test/test_cuda_extension.jl
index 6a5aca7..fe2a1f5 100644
--- a/test/test_cuda_extension.jl
+++ b/test/test_cuda_extension.jl
@@ -108,28 +108,8 @@ end
     end
 end
 
-@testset "Julia 1.9 CUDA.jl $(cuda_version) CUDA runtime 11.6" for cuda_version in (4, 5)
-    with_environment(cuda_runtime_version = "11.6") do env
-        install_script = """
-                         using Pkg
-                         Pkg.develop(path = "$(package_path)")
-                         Pkg.add(name = "CUDA", version = "$(cuda_version)")
-                         Pkg.add(name = "cuDNN")
-                         """
-        @test success(run(`julia +1.9 --project=$(env) -e "$(install_script)"`))
-        # Correct dependencies for :cuda. CUDA runtime version is
-        # lower than officially supported but close enough to at least
-        # load so there will be a warning but no error.
-        test_script = """
-                      using ONNXRunTime, CUDA, cuDNN
-                      load_inference("$(onnx_path)", execution_provider = :cuda)
-                      """
-        @test success(run(`julia +1.9 --project=$(env) -e "$(test_script)"`))
-    end
-end
-
-@testset "Julia 1.9 CUDA.jl $(cuda_version) CUDA runtime 12.1" for cuda_version in (4, 5)
-    with_environment(cuda_runtime_version = "12.1") do env
+@testset "Julia 1.9 CUDA.jl $(cuda_version) CUDA runtime $(cuda_runtime_version)" for cuda_version in (4, 5), cuda_runtime_version in ("11.6", "12.1")
+    with_environment(; cuda_runtime_version) do env
         install_script = """
                          using Pkg
                          Pkg.develop(path = "$(package_path)")
diff --git a/test/test_versions.jl b/test/test_versions.jl
new file mode 100644
index 0000000..2a5b1c7
--- /dev/null
+++ b/test/test_versions.jl
@@ -0,0 +1,32 @@
+module TestVersions
+
+using Test
+using ONNXRunTime.CAPI: OrtGetApiBase, GetVersionString
+using ONNXRunTime: onnxruntime_version, cuda_runtime_supported_version
+
+# Verify that the ONNXRunTime artifacts are synchronized with the
+# information in `src/versions.jl`.
+@testset "ONNXRunTime library version" begin
+    @test GetVersionString(OrtGetApiBase()) == string(onnxruntime_version)
+end
+
+# Verify that the README information about the required CUDA runtime
+# version matches `src/versions.jl`. This is difficult to fully
+# automate but check that at least all mentions of
+# `set_runtime_version!` use the right version.
+@testset "Minimum CUDA runtime version in README." begin
+    s = read(joinpath(dirname(@__DIR__), "README.md"), String)
+    matches = collect(eachmatch(r"set_runtime_version!\(v\"(.+)\"\)", s))
+    # If this test fails, the README has changed so much that this
+    # testset is outdated and should be updated or removed.
+    @test !isempty(matches)
+
+    # If any of these tests fail, update the README information about
+    # supported CUDA runtime versions.
+    for m in matches
+        v = only(m.captures)
+        @test VersionNumber(v) == cuda_runtime_supported_version
+    end
+end
+
+end

From 0a9975829b4d71b3fa8b06e5ca2e3dc0c1c09572 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gunnar=20Farneb=C3=A4ck?=
 <gunnar.farneback@contextvision.se>
Date: Thu, 12 Oct 2023 17:01:20 +0200
Subject: [PATCH 7/9] Bump version to 0.4.

---
 Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Project.toml b/Project.toml
index 383886e..42e7791 100644
--- a/Project.toml
+++ b/Project.toml
@@ -1,7 +1,7 @@
 name = "ONNXRunTime"
 uuid = "e034b28e-924e-41b2-b98f-d2bbeb830c6a"
 authors = ["Jan Weidner <jw3126@gmail.com> and contributors"]
-version = "0.3.3"
+version = "0.4.0"
 
 [deps]
 ArgCheck = "dce04be8-c92d-5529-be00-80e4d2c0e197"

From 73e0b514737d660407e2910b95c789f328c711bb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gunnar=20Farneb=C3=A4ck?= <gunnar@lysator.liu.se>
Date: Mon, 16 Oct 2023 09:23:41 +0200
Subject: [PATCH 8/9] Update README.md

Co-authored-by: Jan Weidner <jw3126@gmail.com>
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index e191b64..6031538 100644
--- a/README.md
+++ b/README.md
@@ -76,7 +76,7 @@ output_array = GetTensorMutableData(api, output_tensor);
 * [ONNX.jl](https://github.com/FluxML/ONNX.jl)
 * [ONNXNaiveNASflux.jl](https://github.com/DrChainsaw/ONNXNaiveNASflux.jl)
 
-# Breaking Changes in version 1.0.
+# Breaking Changes in version 0.4.
 
 * Support for CUDA.jl is changed from version 3 to versions 4 and 5.
 

From bd6a35b338e335b0d8cf07e56d755278ad402c49 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gunnar=20Farneb=C3=A4ck?=
 <gunnar.farneback@contextvision.se>
Date: Mon, 16 Oct 2023 11:49:48 +0200
Subject: [PATCH 9/9] Remove no longer used file.

---
 src/cuda.jl | 19 -------------------
 1 file changed, 19 deletions(-)
 delete mode 100644 src/cuda.jl

diff --git a/src/cuda.jl b/src/cuda.jl
deleted file mode 100644
index ebc7328..0000000
--- a/src/cuda.jl
+++ /dev/null
@@ -1,19 +0,0 @@
-import .CUDA
-CUDA.libcufft()
-CUDA.libcudnn()
-CUDA.libcurand()
-path_cublas = CUDA.libcublas()
-using Libdl
-if isdefined(CUDA, :libcudart)
-    CUDA.libcudart()
-else
-    @warn """
-    HACK
-    CUDA.jl version not expose `libcudart`. See
-    https://discourse.julialang.org/t/shared-c-dependencies-and-artifacts/68525/4?u=jw3126
-    But as an implementation detail libcudart does ship with the cublas artifact
-    """
-    dir = splitdir(path_cublas)[1]
-    path_cudart = joinpath(dir, "libcudart.so.11.0")
-    dlopen(path_cudart)
-end