ONNX Runtime
Loading...
Searching...
No Matches
onnxruntime_cxx_api.h
1// Copyright (c) Microsoft Corporation. All rights reserved.
2// Licensed under the MIT License.
3
4// Summary: The Ort C++ API is a header only wrapper around the Ort C API.
5//
6// The C++ API simplifies usage by returning values directly instead of error codes, throwing exceptions on errors
7// and automatically releasing resources in the destructors. The primary purpose of C++ API is exception safety so
8// all the resources follow RAII and do not leak memory.
9//
10// Each of the C++ wrapper classes holds only a pointer to the C internal object. Treat them like smart pointers.
11// To create an empty object, pass 'nullptr' to the constructor (for example, Env e{nullptr};). However, you can't use them
12// until you assign an instance that actually holds an underlying object.
13//
14// For Ort objects only move assignment between objects is allowed, there are no copy constructors.
15// Some objects have explicit 'Clone' methods for this purpose.
16//
17// ConstXXXX types are copyable since they do not own the underlying C object, so you can pass them to functions as arguments
18// by value or by reference. ConstXXXX types are restricted to const only interfaces.
19//
20// UnownedXXXX are similar to ConstXXXX but also allow non-const interfaces.
21//
22// The lifetime of the corresponding owning object must eclipse the lifetimes of the ConstXXXX/UnownedXXXX types. They exists so you do not
23// have to fallback to C types and the API with the usual pitfalls. In general, do not use C API from your C++ code.
24
25#pragma once
26#include "onnxruntime_c_api.h"
27#include "onnxruntime_float16.h"
28
29#include <array>
30#include <cstddef>
31#include <cstdio>
32#include <memory>
33#include <stdexcept>
34#include <string>
35#include <type_traits>
36#include <unordered_map>
37#include <utility>
38#include <variant>
39#include <vector>
40
41#ifdef ORT_NO_EXCEPTIONS
42#include <iostream>
43#endif
44
48namespace Ort {
49
54struct Exception : std::exception {
55 Exception(const std::string& string, OrtErrorCode code) : message_{string}, code_{code} {}
56 Exception(std::string&& string, OrtErrorCode code) : message_{std::move(string)}, code_{code} {}
57
58 OrtErrorCode GetOrtErrorCode() const { return code_; }
59 const char* what() const noexcept override { return message_.c_str(); }
60
61 private:
62 std::string message_;
63 OrtErrorCode code_;
64};
65
66#ifdef ORT_NO_EXCEPTIONS
67// The #ifndef is for the very special case where the user of this library wants to define their own way of handling errors.
68// NOTE: This header expects control flow to not continue after calling ORT_CXX_API_THROW
69#ifndef ORT_CXX_API_THROW
70#define ORT_CXX_API_THROW(string, code) \
71 do { \
72 std::cerr << Ort::Exception(string, code) \
73 .what() \
74 << std::endl; \
75 abort(); \
76 } while (false)
77#endif
78#else
79#define ORT_CXX_API_THROW(string, code) \
80 throw Ort::Exception(string, code)
81#endif
82
83#ifdef ORT_API_MANUAL_INIT
84// If the macro ORT_API_MANUAL_INIT is defined, no static initialization
85// will be performed. Instead, users must call InitApi() before using the
86// ORT C++ APIs..
87//
88// InitApi() sets the global API object using the default initialization
89// logic. Users call this to initialize the ORT C++ APIs at a time that
90// makes sense in their program.
91inline void InitApi() noexcept;
92
93// InitApi(const OrtApi*) is used by custom operator libraries that are not
94// linked to onnxruntime. It sets the global API object, which is required
95// by the ORT C++ APIs.
96//
97// Example mycustomop.cc:
98//
99// #define ORT_API_MANUAL_INIT
100// #include <onnxruntime_cxx_api.h>
101// #undef ORT_API_MANUAL_INIT
102//
103// OrtStatus* ORT_API_CALL RegisterCustomOps(OrtSessionOptions* options, const OrtApiBase* api_base) {
104// Ort::InitApi(api_base->GetApi(ORT_API_VERSION));
105// // ...
106// }
107//
108inline void InitApi(const OrtApi* api) noexcept;
109#endif
110
111namespace detail {
112// This is used internally by the C++ API. This class holds the global
113// variable that points to the OrtApi.
114struct Global {
115 static const OrtApi* Api(const OrtApi* newValue = nullptr) noexcept {
116 // This block-level static will be initialized once when this function is
117 // first executed, delaying the call to DefaultInit() until it is first needed.
118 //
119 // When ORT_API_MANUAL_INIT is not defined, DefaultInit() calls
120 // OrtGetApiBase()->GetApi(), which may result in a shared library being
121 // loaded.
122 //
123 // Using a block-level static instead of a class-level static helps
124 // avoid issues with static initialization order and dynamic libraries
125 // loading other dynamic libraries.
126 //
127 // This makes it safe to include the C++ API headers in a shared library
128 // that is delay loaded or delay loads its dependencies.
129 //
130 // This DOES NOT make it safe to _use_ arbitrary ORT C++ APIs when
131 // initializing static members, however.
132 static const OrtApi* api = DefaultInit();
133
134 if (newValue) {
135 api = newValue;
136 }
137
138 return api;
139 }
140
141 private:
142 // Has different definitions based on ORT_API_MANUAL_INIT
143 static const OrtApi* DefaultInit() noexcept;
144
145#ifdef ORT_API_MANUAL_INIT
146 // Public APIs to set the OrtApi* to use.
147 friend void ::Ort::InitApi() noexcept;
148 friend void ::Ort::InitApi(const OrtApi*) noexcept;
149#endif
150};
151} // namespace detail
152
153#ifdef ORT_API_MANUAL_INIT
154
155// See comments on declaration above for usage.
156inline void InitApi(const OrtApi* api) noexcept { detail::Global::Api(api); }
157inline void InitApi() noexcept { InitApi(OrtGetApiBase()->GetApi(ORT_API_VERSION)); }
158
159#ifdef _MSC_VER
160// If you get a linker error about a mismatch here, you are trying to
161// link two compilation units that have different definitions for
162// ORT_API_MANUAL_INIT together. All compilation units must agree on the
163// definition of ORT_API_MANUAL_INIT.
164#pragma detect_mismatch("ORT_API_MANUAL_INIT", "enabled")
165#endif
166
167inline const OrtApi* detail::Global::DefaultInit() noexcept {
168 // When ORT_API_MANUAL_INIT is defined, there's no default init that can
169 // be done.
170 return nullptr;
171}
172
173#else // ORT_API_MANUAL_INIT
174
175#ifdef _MSC_VER
176// If you get a linker error about a mismatch here, you are trying to link
177// two compilation units that have different definitions for
178// ORT_API_MANUAL_INIT together. All compilation units must agree on the
179// definition of ORT_API_MANUAL_INIT.
180#pragma detect_mismatch("ORT_API_MANUAL_INIT", "disabled")
181#endif
182
183inline const OrtApi* detail::Global::DefaultInit() noexcept {
185}
186#endif // ORT_API_MANUAL_INIT
187
189inline const OrtApi& GetApi() noexcept { return *detail::Global::Api(); }
190
195std::string GetVersionString();
196
202std::string GetBuildInfoString();
203
209std::vector<std::string> GetAvailableProviders();
210
216 auto* api = GetApi().GetModelEditorApi();
217 if (api == nullptr) {
218 // minimal build
219 ORT_CXX_API_THROW("Model Editor API is not available in this build", ORT_FAIL);
220 }
221
222 return *api;
223}
224
230 auto* api = GetApi().GetCompileApi();
231 if (api == nullptr) {
232 // minimal build
233 ORT_CXX_API_THROW("Compile API is not available in this build", ORT_FAIL);
234 }
235
236 return *api;
237}
238
243inline const OrtEpApi& GetEpApi() {
244 auto* api = GetApi().GetEpApi();
245 if (api == nullptr) {
246 // minimal build
247 ORT_CXX_API_THROW("EP API is not available in this build", ORT_FAIL);
248 }
249
250 return *api;
251}
252
271struct Float16_t : onnxruntime_float16::Float16Impl<Float16_t> {
272 private:
278 constexpr explicit Float16_t(uint16_t v) noexcept { val = v; }
279
280 public:
281 using Base = onnxruntime_float16::Float16Impl<Float16_t>;
282
286 Float16_t() = default;
287
293 constexpr static Float16_t FromBits(uint16_t v) noexcept { return Float16_t(v); }
294
299 explicit Float16_t(float v) noexcept { val = Base::ToUint16Impl(v); }
300
305 float ToFloat() const noexcept { return Base::ToFloatImpl(); }
306
311 using Base::IsNegative;
312
317 using Base::IsNaN;
318
323 using Base::IsFinite;
324
329 using Base::IsPositiveInfinity;
330
335 using Base::IsNegativeInfinity;
336
341 using Base::IsInfinity;
342
347 using Base::IsNaNOrZero;
348
353 using Base::IsNormal;
354
359 using Base::IsSubnormal;
360
365 using Base::Abs;
366
371 using Base::Negate;
372
381 using Base::AreZero;
382
386 explicit operator float() const noexcept { return ToFloat(); }
387
388 using Base::operator==;
389 using Base::operator!=;
390 using Base::operator<;
391};
392
393static_assert(sizeof(Float16_t) == sizeof(uint16_t), "Sizes must match");
394
413struct BFloat16_t : onnxruntime_float16::BFloat16Impl<BFloat16_t> {
414 private:
422 constexpr explicit BFloat16_t(uint16_t v) noexcept { val = v; }
423
424 public:
425 using Base = onnxruntime_float16::BFloat16Impl<BFloat16_t>;
426
427 BFloat16_t() = default;
428
434 static constexpr BFloat16_t FromBits(uint16_t v) noexcept { return BFloat16_t(v); }
435
440 explicit BFloat16_t(float v) noexcept { val = Base::ToUint16Impl(v); }
441
446 float ToFloat() const noexcept { return Base::ToFloatImpl(); }
447
452 using Base::IsNegative;
453
458 using Base::IsNaN;
459
464 using Base::IsFinite;
465
470 using Base::IsPositiveInfinity;
471
476 using Base::IsNegativeInfinity;
477
482 using Base::IsInfinity;
483
488 using Base::IsNaNOrZero;
489
494 using Base::IsNormal;
495
500 using Base::IsSubnormal;
501
506 using Base::Abs;
507
512 using Base::Negate;
513
522 using Base::AreZero;
523
527 explicit operator float() const noexcept { return ToFloat(); }
528
529 // We do not have an inherited impl for the below operators
530 // as the internal class implements them a little differently
531 bool operator==(const BFloat16_t& rhs) const noexcept;
532 bool operator!=(const BFloat16_t& rhs) const noexcept { return !(*this == rhs); }
533 bool operator<(const BFloat16_t& rhs) const noexcept;
534};
535
536static_assert(sizeof(BFloat16_t) == sizeof(uint16_t), "Sizes must match");
537
544 uint8_t value;
545 constexpr Float8E4M3FN_t() noexcept : value(0) {}
546 constexpr Float8E4M3FN_t(uint8_t v) noexcept : value(v) {}
547 constexpr operator uint8_t() const noexcept { return value; }
548 // nan values are treated like any other value for operator ==, !=
549 constexpr bool operator==(const Float8E4M3FN_t& rhs) const noexcept { return value == rhs.value; };
550 constexpr bool operator!=(const Float8E4M3FN_t& rhs) const noexcept { return value != rhs.value; };
551};
552
553static_assert(sizeof(Float8E4M3FN_t) == sizeof(uint8_t), "Sizes must match");
554
561 uint8_t value;
562 constexpr Float8E4M3FNUZ_t() noexcept : value(0) {}
563 constexpr Float8E4M3FNUZ_t(uint8_t v) noexcept : value(v) {}
564 constexpr operator uint8_t() const noexcept { return value; }
565 // nan values are treated like any other value for operator ==, !=
566 constexpr bool operator==(const Float8E4M3FNUZ_t& rhs) const noexcept { return value == rhs.value; };
567 constexpr bool operator!=(const Float8E4M3FNUZ_t& rhs) const noexcept { return value != rhs.value; };
568};
569
570static_assert(sizeof(Float8E4M3FNUZ_t) == sizeof(uint8_t), "Sizes must match");
571
578 uint8_t value;
579 constexpr Float8E5M2_t() noexcept : value(0) {}
580 constexpr Float8E5M2_t(uint8_t v) noexcept : value(v) {}
581 constexpr operator uint8_t() const noexcept { return value; }
582 // nan values are treated like any other value for operator ==, !=
583 constexpr bool operator==(const Float8E5M2_t& rhs) const noexcept { return value == rhs.value; };
584 constexpr bool operator!=(const Float8E5M2_t& rhs) const noexcept { return value != rhs.value; };
585};
586
587static_assert(sizeof(Float8E5M2_t) == sizeof(uint8_t), "Sizes must match");
588
595 uint8_t value;
596 constexpr Float8E5M2FNUZ_t() noexcept : value(0) {}
597 constexpr Float8E5M2FNUZ_t(uint8_t v) noexcept : value(v) {}
598 constexpr operator uint8_t() const noexcept { return value; }
599 // nan values are treated like any other value for operator ==, !=
600 constexpr bool operator==(const Float8E5M2FNUZ_t& rhs) const noexcept { return value == rhs.value; };
601 constexpr bool operator!=(const Float8E5M2FNUZ_t& rhs) const noexcept { return value != rhs.value; };
602};
603
604static_assert(sizeof(Float8E5M2FNUZ_t) == sizeof(uint8_t), "Sizes must match");
605
606namespace detail {
607// This is used internally by the C++ API. This macro is to make it easy to generate overloaded methods for all of the various OrtRelease* functions for every Ort* type
608// This can't be done in the C API since C doesn't have function overloading.
609#define ORT_DEFINE_RELEASE(NAME) \
610 inline void OrtRelease(Ort##NAME* ptr) { GetApi().Release##NAME(ptr); }
611
612#define ORT_DEFINE_RELEASE_FROM_API_STRUCT(NAME, API_GETTER) \
613 inline void OrtRelease(Ort##NAME* ptr) { API_GETTER().Release##NAME(ptr); }
614
615ORT_DEFINE_RELEASE(Allocator);
616ORT_DEFINE_RELEASE(ArenaCfg);
617ORT_DEFINE_RELEASE(CustomOpDomain);
618ORT_DEFINE_RELEASE(Env);
619ORT_DEFINE_RELEASE(ExternalInitializerInfo);
620ORT_DEFINE_RELEASE(Graph);
621ORT_DEFINE_RELEASE(IoBinding);
622ORT_DEFINE_RELEASE(KernelInfo);
623ORT_DEFINE_RELEASE(KeyValuePairs);
624ORT_DEFINE_RELEASE(LoraAdapter);
625ORT_DEFINE_RELEASE(MemoryInfo);
626ORT_DEFINE_RELEASE(MapTypeInfo);
627ORT_DEFINE_RELEASE(Model);
628ORT_DEFINE_RELEASE(ModelMetadata);
629ORT_DEFINE_RELEASE(Node);
630ORT_DEFINE_RELEASE(Op);
631ORT_DEFINE_RELEASE(OpAttr);
632ORT_DEFINE_RELEASE(PrepackedWeightsContainer);
633ORT_DEFINE_RELEASE(RunOptions);
634ORT_DEFINE_RELEASE(Session);
635ORT_DEFINE_RELEASE(SessionOptions);
636ORT_DEFINE_RELEASE(SequenceTypeInfo);
637ORT_DEFINE_RELEASE(Status);
638ORT_DEFINE_RELEASE(SyncStream);
639ORT_DEFINE_RELEASE(TensorTypeAndShapeInfo);
640ORT_DEFINE_RELEASE(ThreadingOptions);
641ORT_DEFINE_RELEASE(TypeInfo);
642ORT_DEFINE_RELEASE(Value);
643ORT_DEFINE_RELEASE(ValueInfo);
644
645ORT_DEFINE_RELEASE_FROM_API_STRUCT(ModelCompilationOptions, GetCompileApi);
646ORT_DEFINE_RELEASE_FROM_API_STRUCT(EpDevice, GetEpApi);
647
648// This is defined explicitly since OrtTensorRTProviderOptionsV2 is not a C API type,
649// but the struct has V2 in its name to indicate that it is the second version of the options.
652
653#undef ORT_DEFINE_RELEASE
654#undef ORT_DEFINE_RELEASE_FROM_API_STRUCT
655
659template <typename T>
660struct Unowned {
661 using Type = T;
662};
663
683template <typename T>
684struct Base {
685 using contained_type = T;
686
687 constexpr Base() = default;
688 constexpr explicit Base(contained_type* p) noexcept : p_{p} {}
690 OrtRelease(p_);
691 }
692
693 Base(const Base&) = delete;
694 Base& operator=(const Base&) = delete;
695
696 Base(Base&& v) noexcept : p_{v.p_} { v.p_ = nullptr; }
697 Base& operator=(Base&& v) noexcept {
698 OrtRelease(p_);
699 p_ = v.release();
700 return *this;
701 }
702
703 constexpr operator contained_type*() const noexcept { return p_; }
704 constexpr contained_type& operator*() const noexcept { return *p_; }
705
709 T* p = p_;
710 p_ = nullptr;
711 return p;
712 }
713
714 protected:
716};
717
718// Undefined. For const types use Base<Unowned<const T>>
719template <typename T>
720struct Base<const T>;
721
729template <typename T>
730struct Base<Unowned<T>> {
732
733 constexpr Base() = default;
734 constexpr explicit Base(contained_type* p) noexcept : p_{p} {}
735
736 ~Base() = default;
737
738 Base(const Base&) = default;
739 Base& operator=(const Base&) = default;
740
741 Base(Base&& v) noexcept : p_{v.p_} { v.p_ = nullptr; }
742 Base& operator=(Base&& v) noexcept {
743 p_ = nullptr;
744 std::swap(p_, v.p_);
745 return *this;
746 }
747
748 constexpr operator contained_type*() const noexcept { return p_; }
749 constexpr contained_type& operator*() const noexcept { return *p_; }
750
751 protected:
753};
754
755// Light functor to release memory with OrtAllocator
758 explicit AllocatedFree(OrtAllocator* allocator)
759 : allocator_(allocator) {}
760 void operator()(void* ptr) const {
761 if (ptr) allocator_->Free(allocator_, ptr);
762 }
763};
764
765} // namespace detail
766
767struct AllocatorWithDefaultOptions;
768struct Env;
769struct EpDevice;
770struct ExternalInitializerInfo;
771struct Graph;
772struct Model;
773struct Node;
774struct ModelMetadata;
775struct TypeInfo;
776struct PrepackedWeightsContainer;
777struct Session;
778struct SessionOptions;
779struct SyncStream;
780struct TensorRTProviderOptions;
781struct Value;
782struct ValueInfo;
783
788using AllocatedStringPtr = std::unique_ptr<char, detail::AllocatedFree>;
789
794struct Status : detail::Base<OrtStatus> {
795 Status() = default; // Same as with std::nullptr_t. But can be used in re-sizable containers and represent success.
796 explicit Status(std::nullptr_t) noexcept {}
797 explicit Status(OrtStatus* status) noexcept;
798 explicit Status(const Exception&);
799 explicit Status(const std::exception&);
800 Status(const char* message, OrtErrorCode code);
801 std::string GetErrorMessage() const;
803 bool IsOK() const noexcept;
804};
805
835
840struct TensorRTProviderOptions : detail::Base<OrtTensorRTProviderOptionsV2> {
841 TensorRTProviderOptions(std::nullptr_t) {}
845 void Update(const std::unordered_map<std::string, std::string>& options);
847 void UpdateWithValue(const char* key, void* value);
848
850 void* GetOptionByName(const char* name) const;
853};
854
859struct CUDAProviderOptions : detail::Base<OrtCUDAProviderOptionsV2> {
860 CUDAProviderOptions(std::nullptr_t) {}
864 void Update(const std::unordered_map<std::string, std::string>& options);
868 void UpdateWithValue(const char* key, void* value);
870 void* GetOptionByName(const char* name) const;
871};
872
887
888namespace detail {
889template <typename T>
891 using B = Base<T>;
892 using B::B;
893
894 // Wraps OrtApi::ExternalInitializerInfo_GetFilePath
895 const std::basic_string<ORTCHAR_T> GetFilePath() const;
896 // Wraps OrtApi::ExternalInitializerInfo_GetFileOffset
897 int64_t GetFileOffset() const;
898 // Wraps OrtApi::ExternalInitializerInfo_GetByteSize
899 size_t GetByteSize() const;
900};
901} // namespace detail
902
903// Const object holder that does not own the underlying object
906
912 using Base::Base;
913
914 explicit ExternalInitializerInfo(std::nullptr_t) {}
916 : detail::ConstExternalInitializerInfoImpl<OrtExternalInitializerInfo>{p} {}
917
919
921 ExternalInitializerInfo(const ORTCHAR_T* filepath, int64_t file_offset, size_t byte_size);
922
924 static Status Create(const ORTCHAR_T* filepath, int64_t file_offset, size_t byte_size,
925 /*out*/ ExternalInitializerInfo& out);
926};
927
928namespace detail {
929template <typename T>
932 using B::B;
933
934 const char* GetValue(const char* key) const;
935
936 // get the pairs in unordered_map. needs to copy to std::string so the hash works as expected
937 std::unordered_map<std::string, std::string> GetKeyValuePairs() const;
938 // get the pairs in two vectors. entries will be 1:1 between keys and values. avoids copying to std::string
939 void GetKeyValuePairs(std::vector<const char*>& keys, std::vector<const char*>& values) const;
940};
941} // namespace detail
942
943// Const object holder that does not own the underlying object
945
947struct KeyValuePairs : detail::KeyValuePairsImpl<OrtKeyValuePairs> {
948 explicit KeyValuePairs(std::nullptr_t) {}
950 explicit KeyValuePairs(OrtKeyValuePairs* p) : KeyValuePairsImpl<OrtKeyValuePairs>{p} {}
951
953 explicit KeyValuePairs();
954
956 explicit KeyValuePairs(const std::unordered_map<std::string, std::string>& kv_pairs);
957
959 void Add(const char* key, const char* value);
960
962 void Remove(const char* key);
963
964 ConstKeyValuePairs GetConst() const { return ConstKeyValuePairs{this->p_}; }
965};
966
967namespace detail {
968template <typename T>
969struct MemoryInfoImpl : Base<T> {
970 using B = Base<T>;
971 using B::B;
972
973 std::string GetAllocatorName() const;
975 int GetDeviceId() const;
979 uint32_t GetVendorId() const;
980
981 template <typename U>
982 bool operator==(const MemoryInfoImpl<U>& o) const;
983};
984} // namespace detail
985
986// Const object holder that does not own the underlying object
988
992struct MemoryInfo : detail::MemoryInfoImpl<OrtMemoryInfo> {
994 explicit MemoryInfo(std::nullptr_t) {}
995 explicit MemoryInfo(OrtMemoryInfo* p) : MemoryInfoImpl<OrtMemoryInfo>{p} {}
996 MemoryInfo(const char* name, OrtAllocatorType type, int id, OrtMemType mem_type);
997 MemoryInfo(const char* name, OrtMemoryInfoDeviceType device_type, uint32_t vendor_id, uint32_t device_id,
998 OrtDeviceMemoryType mem_type, size_t alignment, OrtAllocatorType allocator_type);
999 ConstMemoryInfo GetConst() const { return ConstMemoryInfo{this->p_}; }
1000};
1001
1009 MemoryAllocation(OrtAllocator* allocator, void* p, size_t size);
1014 MemoryAllocation& operator=(MemoryAllocation&&) noexcept;
1015
1016 void* get() { return p_; }
1017 size_t size() const { return size_; }
1018
1019 private:
1020 OrtAllocator* allocator_;
1021 void* p_;
1022 size_t size_;
1023};
1024
1025namespace detail {
1026template <typename T>
1027struct AllocatorImpl : Base<T> {
1028 using B = Base<T>;
1029 using B::B;
1030
1031 void* Alloc(size_t size);
1032 MemoryAllocation GetAllocation(size_t size);
1033 void Free(void* p);
1034 ConstMemoryInfo GetInfo() const;
1035
1040 KeyValuePairs GetStats() const;
1041};
1042} // namespace detail
1043
1047struct AllocatorWithDefaultOptions : detail::AllocatorImpl<detail::Unowned<OrtAllocator>> {
1048 explicit AllocatorWithDefaultOptions(std::nullptr_t) {}
1050};
1051
1056struct Allocator : detail::AllocatorImpl<OrtAllocator> {
1057 explicit Allocator(std::nullptr_t) {}
1058 Allocator(const Session& session, const OrtMemoryInfo*);
1059};
1060
1061using UnownedAllocator = detail::AllocatorImpl<detail::Unowned<OrtAllocator>>;
1062
1066struct SyncStream : detail::Base<OrtSyncStream> {
1067 explicit SyncStream(std::nullptr_t) {}
1068 explicit SyncStream(OrtSyncStream* p) : Base<OrtSyncStream>{p} {}
1069 void* GetHandle() const;
1070};
1071
1072namespace detail {
1073template <typename T>
1076 using B::B;
1077
1079 uint32_t VendorId() const;
1080 uint32_t DeviceId() const;
1081 const char* Vendor() const;
1083};
1084} // namespace detail
1085
1090
1091namespace detail {
1092template <typename T>
1095 using B::B;
1096
1097 const char* EpName() const;
1098 const char* EpVendor() const;
1104};
1105} // namespace detail
1106
1111
1114struct EpDevice : detail::EpDeviceImpl<OrtEpDevice> {
1115 explicit EpDevice(std::nullptr_t) {}
1116 explicit EpDevice(OrtEpDevice* p) : EpDeviceImpl<OrtEpDevice>{p} {}
1117
1119 EpDevice(OrtEpFactory& ep_factory, ConstHardwareDevice& hardware_device,
1120 ConstKeyValuePairs ep_metadata = {}, ConstKeyValuePairs ep_options = {});
1121};
1122
1130 const std::vector<ConstEpDevice>& ep_devices,
1131 const char* compatibility_info);
1132
1138struct Env : detail::Base<OrtEnv> {
1139 explicit Env(std::nullptr_t) {}
1140
1142 Env(OrtLoggingLevel logging_level = ORT_LOGGING_LEVEL_WARNING, _In_ const char* logid = "");
1143
1145 Env(OrtLoggingLevel logging_level, const char* logid, OrtLoggingFunction logging_function, void* logger_param);
1146
1148 Env(const OrtThreadingOptions* tp_options, OrtLoggingLevel logging_level = ORT_LOGGING_LEVEL_WARNING, _In_ const char* logid = "");
1149
1151 Env(const OrtThreadingOptions* tp_options, OrtLoggingFunction logging_function, void* logger_param,
1152 OrtLoggingLevel logging_level = ORT_LOGGING_LEVEL_WARNING, _In_ const char* logid = "");
1153
1155 explicit Env(OrtEnv* p) : Base<OrtEnv>{p} {}
1156
1159
1161
1162 Env& CreateAndRegisterAllocator(const OrtMemoryInfo* mem_info, const OrtArenaCfg* arena_cfg);
1163
1164 Env& CreateAndRegisterAllocatorV2(const std::string& provider_type, const OrtMemoryInfo* mem_info,
1165 const std::unordered_map<std::string, std::string>& options,
1166 const OrtArenaCfg* arena_cfg);
1167
1169
1171
1173 OrtAllocatorType allocator_type,
1174 const OrtKeyValuePairs* allocator_options);
1175
1176 // Result may be nullptr
1178
1180 OrtDeviceMemoryType mem_type);
1181
1182 Env& RegisterExecutionProviderLibrary(const char* registration_name, const std::basic_string<ORTCHAR_T>& path);
1183 Env& UnregisterExecutionProviderLibrary(const char* registration_name);
1184
1185 std::vector<ConstEpDevice> GetEpDevices() const;
1186
1187 Status CopyTensors(const std::vector<Value>& src_tensors,
1188 const std::vector<Value>& dst_tensors,
1189 OrtSyncStream* stream) const;
1190};
1191
1195struct CustomOpDomain : detail::Base<OrtCustomOpDomain> {
1197 using Base::Base;
1198
1199 explicit CustomOpDomain(std::nullptr_t) {}
1200
1202 explicit CustomOpDomain(const char* domain);
1203
1204 // This does not take ownership of the op, simply registers it.
1205 void Add(const OrtCustomOp* op);
1206};
1207
1209struct LoraAdapter : detail::Base<OrtLoraAdapter> {
1211 using Base::Base;
1212
1213 explicit LoraAdapter(std::nullptr_t) {}
1220 static LoraAdapter CreateLoraAdapter(const std::basic_string<ORTCHAR_T>& adapter_path,
1221 OrtAllocator* allocator);
1222
1230 static LoraAdapter CreateLoraAdapterFromArray(const void* bytes, size_t num_bytes,
1231 OrtAllocator* allocator);
1232};
1233
1237struct RunOptions : detail::Base<OrtRunOptions> {
1238 explicit RunOptions(std::nullptr_t) {}
1240
1243
1246
1247 RunOptions& SetRunTag(const char* run_tag);
1248 const char* GetRunTag() const;
1249
1250 RunOptions& AddConfigEntry(const char* config_key, const char* config_value);
1251 const char* GetConfigEntry(const char* config_key);
1252
1259
1265
1273};
1274
1275namespace detail {
1276// Utility function that returns a SessionOption config entry key for a specific custom operator.
1277// Ex: custom_op.[custom_op_name].[config]
1278std::string MakeCustomOpConfigEntryKey(const char* custom_op_name, const char* config);
1279} // namespace detail
1280
1291 CustomOpConfigs() = default;
1292 ~CustomOpConfigs() = default;
1297
1306 CustomOpConfigs& AddConfig(const char* custom_op_name, const char* config_key, const char* config_value);
1307
1316 const std::unordered_map<std::string, std::string>& GetFlattenedConfigs() const;
1317
1318 private:
1319 std::unordered_map<std::string, std::string> flat_configs_;
1320};
1321
1327namespace detail {
1328// we separate const-only methods because passing const ptr to non-const methods
1329// is only discovered when inline methods are compiled which is counter-intuitive
1330template <typename T>
1331struct ConstSessionOptionsImpl : Base<T> {
1332 using B = Base<T>;
1333 using B::B;
1334
1335 SessionOptions Clone() const;
1336
1337 std::string GetConfigEntry(const char* config_key) const;
1338 bool HasConfigEntry(const char* config_key) const;
1339 std::string GetConfigEntryOrDefault(const char* config_key, const std::string& def) const;
1340};
1341
1342template <typename T>
1343struct SessionOptionsImpl : ConstSessionOptionsImpl<T> {
1344 using B = ConstSessionOptionsImpl<T>;
1345 using B::B;
1346
1347 SessionOptionsImpl& SetIntraOpNumThreads(int intra_op_num_threads);
1348 SessionOptionsImpl& SetInterOpNumThreads(int inter_op_num_threads);
1349 SessionOptionsImpl& SetGraphOptimizationLevel(GraphOptimizationLevel graph_optimization_level);
1350 SessionOptionsImpl& SetDeterministicCompute(bool value);
1351
1352 SessionOptionsImpl& EnableCpuMemArena();
1353 SessionOptionsImpl& DisableCpuMemArena();
1354
1355 SessionOptionsImpl& SetOptimizedModelFilePath(const ORTCHAR_T* optimized_model_file);
1356
1357 SessionOptionsImpl& EnableProfiling(const ORTCHAR_T* profile_file_prefix);
1358 SessionOptionsImpl& DisableProfiling();
1359
1360 SessionOptionsImpl& EnableOrtCustomOps();
1361
1362 SessionOptionsImpl& EnableMemPattern();
1363 SessionOptionsImpl& DisableMemPattern();
1364
1365 SessionOptionsImpl& SetExecutionMode(ExecutionMode execution_mode);
1366
1367 SessionOptionsImpl& SetLoadCancellationFlag(bool value);
1368
1369 SessionOptionsImpl& SetLogId(const char* logid);
1370 SessionOptionsImpl& SetLogSeverityLevel(int level);
1371
1372 SessionOptionsImpl& Add(OrtCustomOpDomain* custom_op_domain);
1373
1374 SessionOptionsImpl& DisablePerSessionThreads();
1375
1376 SessionOptionsImpl& AddConfigEntry(const char* config_key, const char* config_value);
1377
1378 SessionOptionsImpl& AddInitializer(const char* name, const OrtValue* ort_val);
1379 SessionOptionsImpl& AddExternalInitializers(const std::vector<std::string>& names, const std::vector<Value>& ort_values);
1380 SessionOptionsImpl& AddExternalInitializersFromFilesInMemory(const std::vector<std::basic_string<ORTCHAR_T>>& external_initializer_file_names,
1381 const std::vector<char*>& external_initializer_file_buffer_array,
1382 const std::vector<size_t>& external_initializer_file_lengths);
1383
1384 SessionOptionsImpl& AppendExecutionProvider_CPU(int use_arena);
1385 SessionOptionsImpl& AppendExecutionProvider_CUDA(const OrtCUDAProviderOptions& provider_options);
1386 SessionOptionsImpl& AppendExecutionProvider_CUDA_V2(const OrtCUDAProviderOptionsV2& provider_options);
1387 SessionOptionsImpl& AppendExecutionProvider_ROCM(const OrtROCMProviderOptions& provider_options);
1388 SessionOptionsImpl& AppendExecutionProvider_OpenVINO(const OrtOpenVINOProviderOptions& provider_options);
1390 SessionOptionsImpl& AppendExecutionProvider_OpenVINO_V2(const std::unordered_map<std::string, std::string>& provider_options = {});
1391 SessionOptionsImpl& AppendExecutionProvider_TensorRT(const OrtTensorRTProviderOptions& provider_options);
1392 SessionOptionsImpl& AppendExecutionProvider_TensorRT_V2(const OrtTensorRTProviderOptionsV2& provider_options);
1393 SessionOptionsImpl& AppendExecutionProvider_MIGraphX(const OrtMIGraphXProviderOptions& provider_options);
1395 SessionOptionsImpl& AppendExecutionProvider_CANN(const OrtCANNProviderOptions& provider_options);
1397 SessionOptionsImpl& AppendExecutionProvider_Dnnl(const OrtDnnlProviderOptions& provider_options);
1399 SessionOptionsImpl& AppendExecutionProvider(const std::string& provider_name,
1400 const std::unordered_map<std::string, std::string>& provider_options = {});
1401
1404 SessionOptionsImpl& AppendExecutionProvider_V2(Env& env, const std::vector<ConstEpDevice>& ep_devices,
1405 const KeyValuePairs& ep_options);
1408 SessionOptionsImpl& AppendExecutionProvider_V2(Env& env, const std::vector<ConstEpDevice>& ep_devices,
1409 const std::unordered_map<std::string, std::string>& ep_options);
1410
1412 SessionOptionsImpl& SetEpSelectionPolicy(OrtExecutionProviderDevicePolicy policy);
1413
1415 SessionOptionsImpl& SetEpSelectionPolicy(EpSelectionDelegate delegate, void* state = nullptr);
1416
1417 SessionOptionsImpl& SetCustomCreateThreadFn(OrtCustomCreateThreadFn ort_custom_create_thread_fn);
1418 SessionOptionsImpl& SetCustomThreadCreationOptions(void* ort_custom_thread_creation_options);
1419 SessionOptionsImpl& SetCustomJoinThreadFn(OrtCustomJoinThreadFn ort_custom_join_thread_fn);
1420
1424 SessionOptionsImpl& RegisterCustomOpsLibrary(const ORTCHAR_T* library_name, const CustomOpConfigs& custom_op_configs = {});
1425
1426 SessionOptionsImpl& RegisterCustomOpsUsingFunction(const char* function_name);
1427
1429 SessionOptionsImpl& AppendExecutionProvider_VitisAI(const std::unordered_map<std::string, std::string>& provider_options = {});
1430};
1431} // namespace detail
1432
1433using UnownedSessionOptions = detail::SessionOptionsImpl<detail::Unowned<OrtSessionOptions>>;
1434using ConstSessionOptions = detail::ConstSessionOptionsImpl<detail::Unowned<const OrtSessionOptions>>;
1435
1439struct SessionOptions : detail::SessionOptionsImpl<OrtSessionOptions> {
1440 explicit SessionOptions(std::nullptr_t) {}
1442 explicit SessionOptions(OrtSessionOptions* p) : SessionOptionsImpl<OrtSessionOptions>{p} {}
1445};
1446
1451struct ModelCompilationOptions : detail::Base<OrtModelCompilationOptions> {
1453 using Base::Base;
1454
1455 explicit ModelCompilationOptions(std::nullptr_t) {}
1456
1457 ModelCompilationOptions(const Env& env, const SessionOptions& session_options);
1458 ModelCompilationOptions(const Env& env, ConstSessionOptions session_options);
1459
1460 ModelCompilationOptions& SetInputModelPath(const ORTCHAR_T* input_model_path);
1462 size_t input_model_data_size);
1463 ModelCompilationOptions& SetEpContextEmbedMode(bool embed_ep_context_in_model);
1464 ModelCompilationOptions& SetOutputModelPath(const ORTCHAR_T* output_model_path);
1466 size_t initializer_size_threshold);
1467
1470 OrtGetInitializerLocationFunc get_initializer_location_func,
1471 void* state);
1472
1473 ModelCompilationOptions& SetOutputModelBuffer(OrtAllocator* allocator, void** output_model_buffer_ptr,
1474 size_t* output_model_buffer_size_ptr);
1475
1478
1479 ModelCompilationOptions& SetEpContextBinaryInformation(const ORTCHAR_T* output_directory,
1480 const ORTCHAR_T* model_name);
1482
1484};
1485
1492Status CompileModel(const Env& env, const ModelCompilationOptions& model_compilation_options);
1493
1497struct ModelMetadata : detail::Base<OrtModelMetadata> {
1499 using Base::Base;
1500
1501 explicit ModelMetadata(std::nullptr_t) {}
1502
1510
1518
1526
1534
1542
1549 std::vector<AllocatedStringPtr> GetCustomMetadataMapKeysAllocated(OrtAllocator* allocator) const;
1550
1561
1562 int64_t GetVersion() const;
1563};
1564
1565struct IoBinding;
1566
1567namespace detail {
1568
1569// we separate const-only methods because passing const ptr to non-const methods
1570// is only discovered when inline methods are compiled which is counter-intuitive
1571template <typename T>
1573 using B = Base<T>;
1574 using B::B;
1575
1576 size_t GetInputCount() const;
1577 size_t GetOutputCount() const;
1579
1580 std::vector<std::string> GetInputNames() const;
1581 std::vector<std::string> GetOutputNames() const;
1582 std::vector<std::string> GetOverridableInitializerNames() const;
1583
1584 std::vector<ConstMemoryInfo> GetMemoryInfoForInputs() const;
1585 std::vector<ConstMemoryInfo> GetMemoryInfoForOutputs() const;
1586 std::vector<ConstEpDevice> GetEpDeviceForInputs() const;
1587
1596
1605
1614
1615 uint64_t GetProfilingStartTimeNs() const;
1617
1618 TypeInfo GetInputTypeInfo(size_t index) const;
1619 TypeInfo GetOutputTypeInfo(size_t index) const;
1621
1622 int GetOpset(const std::string& domain) const;
1623
1624 // Will move before checkin if that's the case.
1625 std::vector<ValueInfo> GetInputs() const;
1626 std::vector<ValueInfo> GetOutputs() const;
1627};
1628
1629template <typename T>
1632 using B::B;
1633
1651 std::vector<Value> Run(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count,
1652 const char* const* output_names, size_t output_count);
1653
1657 void Run(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count,
1658 const char* const* output_names, Value* output_values, size_t output_count);
1659
1660 void Run(const RunOptions& run_options, const IoBinding&);
1661
1681 void RunAsync(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count,
1682 const char* const* output_names, Value* output_values, size_t output_count, RunAsyncCallbackFn callback, void* user_data);
1683
1691
1703 void SetEpDynamicOptions(const char* const* keys, const char* const* values, size_t kv_len);
1704
1705 void FinalizeModelEditorSession(const Model& model, const SessionOptions& options,
1706 OrtPrepackedWeightsContainer* prepacked_weights_container = nullptr);
1707};
1708
1709} // namespace detail
1710
1713
1717struct Session : detail::SessionImpl<OrtSession> {
1719 explicit Session(std::nullptr_t) {}
1720 explicit Session(OrtSession* p) : SessionImpl{p} {}
1721
1722 Session(const Env& env, const ORTCHAR_T* model_path, const SessionOptions& options);
1723
1725 Session(const Env& env, const ORTCHAR_T* model_path, const SessionOptions& options,
1726 OrtPrepackedWeightsContainer* prepacked_weights_container);
1727
1729 Session(const Env& env, const void* model_data, size_t model_data_length, const SessionOptions& options);
1730
1732 Session(const Env& env, const void* model_data, size_t model_data_length, const SessionOptions& options,
1733 OrtPrepackedWeightsContainer* prepacked_weights_container);
1734
1735#if !defined(ORT_MINIMAL_BUILD)
1737 Session(const Env& env, const Model& model, const SessionOptions& options);
1738
1740 static Session CreateModelEditorSession(const Env& env, const ORTCHAR_T* model_path, const SessionOptions& options);
1741
1743 static Session CreateModelEditorSession(const Env& env, const void* model_data, size_t model_data_length,
1744 const SessionOptions& options);
1745#endif // !defined(ORT_MINIMAL_BUILD)
1746
1747 ConstSession GetConst() const { return ConstSession{this->p_}; }
1748 UnownedSession GetUnowned() const { return UnownedSession{this->p_}; }
1749};
1750
1751namespace detail {
1752template <typename T>
1754 using B = Base<T>;
1755 using B::B;
1756
1758 size_t GetElementCount() const;
1759
1760 size_t GetDimensionsCount() const;
1761
1766 [[deprecated("use GetShape()")]] void GetDimensions(int64_t* values, size_t values_count) const;
1767
1768 void GetSymbolicDimensions(const char** values, size_t values_count) const;
1769 std::vector<const char*> GetSymbolicDimensions() const;
1770
1771 std::vector<int64_t> GetShape() const;
1772};
1773
1774} // namespace detail
1775
1777
1783 using Base::Base;
1784
1786 explicit TensorTypeAndShapeInfo(std::nullptr_t) {}
1788 explicit TensorTypeAndShapeInfo(OrtTensorTypeAndShapeInfo* p) : TensorTypeAndShapeInfoImpl{p} {}
1789
1790 // Create a TensorTypeAndShapeInfo object with the specified element type and dimensions
1791 // symbolic_dims are optional, but should be 1:1 with dims.
1792 // The value in symbolic_dims will be used for all entries in dims that are -1.
1794 const std::vector<int64_t>& dims,
1795 const std::vector<std::string>* symbolic_dims = nullptr);
1796
1798};
1799
1800namespace detail {
1801template <typename T>
1803 using B = Base<T>;
1804 using B::B;
1806};
1807
1808} // namespace detail
1809
1811
1815struct SequenceTypeInfo : detail::SequenceTypeInfoImpl<OrtSequenceTypeInfo> {
1817 using Base::Base;
1818
1819 explicit SequenceTypeInfo(std::nullptr_t) {}
1820 explicit SequenceTypeInfo(OrtSequenceTypeInfo* p) : SequenceTypeInfoImpl<OrtSequenceTypeInfo>{p} {}
1822};
1823
1824namespace detail {
1825template <typename T>
1827 using B = Base<T>;
1828 using B::B;
1830};
1831
1832} // namespace detail
1833
1834// This is always owned by the TypeInfo and can only be obtained from it.
1836
1837namespace detail {
1838template <typename T>
1845
1846} // namespace detail
1847
1849
1853struct MapTypeInfo : detail::MapTypeInfoImpl<OrtMapTypeInfo> {
1855 using Base::Base;
1856
1857 explicit MapTypeInfo(std::nullptr_t) {}
1858 explicit MapTypeInfo(OrtMapTypeInfo* p) : MapTypeInfoImpl<OrtMapTypeInfo>{p} {}
1859 ConstMapTypeInfo GetConst() const { return ConstMapTypeInfo{this->p_}; }
1860};
1861
1862namespace detail {
1863template <typename T>
1875} // namespace detail
1876
1882
1887struct TypeInfo : detail::TypeInfoImpl<OrtTypeInfo> {
1889 using Base::Base;
1890
1892 explicit TypeInfo(std::nullptr_t) {}
1893 explicit TypeInfo(OrtTypeInfo* p) : TypeInfoImpl<OrtTypeInfo>{p} {}
1894
1895#if !defined(ORT_MINIMAL_BUILD)
1901#endif // !defined(ORT_MINIMAL_BUILD)
1902
1903 ConstTypeInfo GetConst() const { return ConstTypeInfo{this->p_}; }
1904};
1905
1906namespace detail {
1907// This structure is used to feed sparse tensor values
1908// information for use with FillSparseTensor<Format>() API
1909// if the data type for the sparse tensor values is numeric
1910// use data.p_data, otherwise, use data.str pointer to feed
1911// values. data.str is an array of const char* that are zero terminated.
1912// number of strings in the array must match shape size.
1913// For fully sparse tensors use shape {0} and set p_data/str
1914// to nullptr.
1916 const int64_t* values_shape;
1918 union {
1919 const void* p_data;
1920 const char** str;
1921 } data;
1922};
1923
1924// Provides a way to pass shape in a single
1925// argument
1926struct Shape {
1927 const int64_t* shape;
1929};
1930
1931template <typename T>
1933 using B = Base<T>;
1934 using B::B;
1935
1939 template <typename R>
1940 void GetOpaqueData(const char* domain, const char* type_name, R&) const;
1941
1942 bool IsTensor() const;
1943 bool HasValue() const;
1944
1945 size_t GetCount() const; // If a non tensor, returns 2 for map and N for sequence, where N is the number of elements
1946 Value GetValue(int index, OrtAllocator* allocator) const;
1947
1955
1970 void GetStringTensorContent(void* buffer, size_t buffer_length, size_t* offsets, size_t offsets_count) const;
1971
1978 template <typename R>
1979 const R* GetTensorData() const;
1980
1985 const void* GetTensorRawData() const;
1986
1994
2002
2008
2017 void GetStringTensorElement(size_t buffer_length, size_t element_index, void* buffer) const;
2018
2025 std::string GetStringTensorElement(size_t element_index) const;
2026
2033 size_t GetStringTensorElementLength(size_t element_index) const;
2034
2041 size_t GetTensorSizeInBytes() const;
2042
2043#if !defined(DISABLE_SPARSE_TENSORS)
2051
2058
2067
2077 template <typename R>
2078 const R* GetSparseTensorIndicesData(OrtSparseIndicesFormat indices_format, size_t& num_indices) const;
2079
2084 bool IsSparseTensor() const;
2085
2094 template <typename R>
2095 const R* GetSparseTensorValues() const;
2096
2097#endif
2098};
2099
2100template <typename T>
2103 using B::B;
2104
2110 template <typename R>
2112
2118
2120 // Obtain a reference to an element of data at the location specified
2126 template <typename R>
2127 R& At(const std::vector<int64_t>& location);
2128
2134 void FillStringTensor(const char* const* s, size_t s_len);
2135
2141 void FillStringTensorElement(const char* s, size_t index);
2142
2155 char* GetResizedStringTensorElementBuffer(size_t index, size_t buffer_length);
2156
2157#if !defined(DISABLE_SPARSE_TENSORS)
2166 void UseCooIndices(int64_t* indices_data, size_t indices_num);
2167
2178 void UseCsrIndices(int64_t* inner_data, size_t inner_num, int64_t* outer_data, size_t outer_num);
2179
2188 void UseBlockSparseIndices(const Shape& indices_shape, int32_t* indices_data);
2189
2199 void FillSparseTensorCoo(const OrtMemoryInfo* data_mem_info, const OrtSparseValuesParam& values_param,
2200 const int64_t* indices_data, size_t indices_num);
2201
2213 void FillSparseTensorCsr(const OrtMemoryInfo* data_mem_info,
2214 const OrtSparseValuesParam& values,
2215 const int64_t* inner_indices_data, size_t inner_indices_num,
2216 const int64_t* outer_indices_data, size_t outer_indices_num);
2217
2228 const OrtSparseValuesParam& values,
2229 const Shape& indices_shape,
2230 const int32_t* indices_data);
2231
2232#endif
2233};
2234
2235} // namespace detail
2236
2239
2243struct Value : detail::ValueImpl<OrtValue> {
2245 using Base::Base;
2248
2249 Value(std::nullptr_t) {}
2250 Value(Value&&) = default;
2251 Value& operator=(Value&&) = default;
2252
2253 ConstValue GetConst() const { return ConstValue{this->p_}; }
2254 UnownedValue GetUnowned() const { return UnownedValue{this->p_}; }
2255
2264 template <typename T>
2265 static Value CreateTensor(const OrtMemoryInfo* info, T* p_data, size_t p_data_element_count,
2266 const int64_t* shape, size_t shape_len);
2267
2277 static Value CreateTensor(const OrtMemoryInfo* info, void* p_data, size_t p_data_byte_count,
2278 const int64_t* shape, size_t shape_len,
2280
2290 static Value CreateTensor(OrtAllocator* deleter, void* p_data, size_t p_data_byte_count,
2291 const int64_t* shape, size_t shape_len,
2293
2305 template <typename T>
2306 static Value CreateTensor(OrtAllocator* allocator, const int64_t* shape, size_t shape_len);
2307
2319 static Value CreateTensor(OrtAllocator* allocator, const int64_t* shape, size_t shape_len,
2321
2330 static Value CreateMap(const Value& keys, const Value& values);
2331
2339 static Value CreateSequence(const std::vector<Value>& values);
2340
2349 template <typename T>
2350 static Value CreateOpaque(const char* domain, const char* type_name, const T& value);
2351
2352#if !defined(DISABLE_SPARSE_TENSORS)
2363 template <typename T>
2364 static Value CreateSparseTensor(const OrtMemoryInfo* info, T* p_data, const Shape& dense_shape,
2365 const Shape& values_shape);
2366
2383 static Value CreateSparseTensor(const OrtMemoryInfo* info, void* p_data, const Shape& dense_shape,
2384 const Shape& values_shape, ONNXTensorElementDataType type);
2385
2395 template <typename T>
2396 static Value CreateSparseTensor(OrtAllocator* allocator, const Shape& dense_shape);
2397
2409 static Value CreateSparseTensor(OrtAllocator* allocator, const Shape& dense_shape, ONNXTensorElementDataType type);
2410
2411#endif // !defined(DISABLE_SPARSE_TENSORS)
2412};
2413
2414namespace detail {
2415namespace binding_utils {
2416// Bring these out of template
2417std::vector<std::string> GetOutputNamesHelper(const OrtIoBinding* binding, OrtAllocator*);
2418std::vector<Value> GetOutputValuesHelper(const OrtIoBinding* binding, OrtAllocator*);
2419} // namespace binding_utils
2420
2421template <typename T>
2423 using B = Base<T>;
2424 using B::B;
2425
2426 std::vector<std::string> GetOutputNames() const;
2427 std::vector<std::string> GetOutputNames(OrtAllocator*) const;
2428 std::vector<Value> GetOutputValues() const;
2429 std::vector<Value> GetOutputValues(OrtAllocator*) const;
2430};
2431
2432template <typename T>
2435 using B::B;
2436
2437 void BindInput(const char* name, const Value&);
2438 void BindOutput(const char* name, const Value&);
2439 void BindOutput(const char* name, const OrtMemoryInfo*);
2444};
2445
2446} // namespace detail
2447
2450
2454struct IoBinding : detail::IoBindingImpl<OrtIoBinding> {
2455 explicit IoBinding(std::nullptr_t) {}
2456 explicit IoBinding(Session& session);
2457 ConstIoBinding GetConst() const { return ConstIoBinding{this->p_}; }
2458 UnownedIoBinding GetUnowned() const { return UnownedIoBinding{this->p_}; }
2459};
2460
2465struct ArenaCfg : detail::Base<OrtArenaCfg> {
2466 explicit ArenaCfg(std::nullptr_t) {}
2475 ArenaCfg(size_t max_mem, int arena_extend_strategy, int initial_chunk_size_bytes, int max_dead_bytes_per_chunk);
2476
2481 explicit ArenaCfg(const std::unordered_map<std::string, size_t>& arena_config);
2482};
2483
2484//
2485// Custom OPs (only needed to implement custom OPs)
2486//
2487
2488namespace detail {
2489// Need to define a templated ConstOpAttr with const members
2490template <typename T>
2493 using B::B;
2494
2495 // Wraps OrtApi::OpAttr_GetName
2496 std::string GetName() const;
2497 // Wraps OrtApi::OpAttr_GetType
2499
2500 // Wraps OrtApi::ReadAttr for a single value
2501 // This does not support Tensor Attribute
2502 // Call GetTensorAttributeAsOrtValue() instead.
2503 template <typename R>
2504 Status GetValue(R& out) const;
2505
2506 // Wraps OrtApi::ReadAttr for an array of values
2507 template <typename R>
2508 Status GetValueArray(std::vector<R>& out) const;
2509 // Wraps OrtApi::OpAttr_GetTensorAttributeAsOrtValue
2511};
2512} // namespace detail
2513
2515
2519struct OpAttr : detail::ConstOpAttrImpl<OrtOpAttr> {
2521 using Base::Base;
2522
2523 OpAttr() = default; // Enable storing it in the container for resize()
2524 explicit OpAttr(std::nullptr_t) {}
2525 OpAttr(const char* name, const void* data, int len, OrtOpAttrType type);
2526
2527 ConstOpAttr GetConst() const { return ConstOpAttr{this->p_}; }
2528};
2529
2538#define ORT_CXX_LOG(logger, message_severity, message) \
2539 do { \
2540 if (message_severity >= logger.GetLoggingSeverityLevel()) { \
2541 Ort::ThrowOnError(logger.LogMessage(message_severity, ORT_FILE, __LINE__, \
2542 static_cast<const char*>(__FUNCTION__), message)); \
2543 } \
2544 } while (false)
2545
2554#define ORT_CXX_LOG_NOEXCEPT(logger, message_severity, message) \
2555 do { \
2556 if (message_severity >= logger.GetLoggingSeverityLevel()) { \
2557 static_cast<void>(logger.LogMessage(message_severity, ORT_FILE, __LINE__, \
2558 static_cast<const char*>(__FUNCTION__), message)); \
2559 } \
2560 } while (false)
2561
2573#define ORT_CXX_LOGF(logger, message_severity, /*format,*/...) \
2574 do { \
2575 if (message_severity >= logger.GetLoggingSeverityLevel()) { \
2576 Ort::ThrowOnError(logger.LogFormattedMessage(message_severity, ORT_FILE, __LINE__, \
2577 static_cast<const char*>(__FUNCTION__), __VA_ARGS__)); \
2578 } \
2579 } while (false)
2580
2592#define ORT_CXX_LOGF_NOEXCEPT(logger, message_severity, /*format,*/...) \
2593 do { \
2594 if (message_severity >= logger.GetLoggingSeverityLevel()) { \
2595 static_cast<void>(logger.LogFormattedMessage(message_severity, ORT_FILE, __LINE__, \
2596 static_cast<const char*>(__FUNCTION__), __VA_ARGS__)); \
2597 } \
2598 } while (false)
2599
2610struct Logger {
2614 Logger() = default;
2615
2619 explicit Logger(std::nullptr_t) {}
2620
2627 explicit Logger(const OrtLogger* logger);
2628
2629 ~Logger() = default;
2630
2631 Logger(const Logger&) = default;
2632 Logger& operator=(const Logger&) = default;
2633
2634 Logger(Logger&& v) noexcept = default;
2635 Logger& operator=(Logger&& v) noexcept = default;
2636
2643
2656 Status LogMessage(OrtLoggingLevel log_severity_level, const ORTCHAR_T* file_path, int line_number,
2657 const char* func_name, const char* message) const noexcept;
2658
2673 template <typename... Args>
2674 Status LogFormattedMessage(OrtLoggingLevel log_severity_level, const ORTCHAR_T* file_path, int line_number,
2675 const char* func_name, const char* format, Args&&... args) const noexcept;
2676
2677 private:
2678 const OrtLogger* logger_{};
2679 OrtLoggingLevel cached_severity_level_{};
2680};
2681
2690 size_t GetInputCount() const;
2691 size_t GetOutputCount() const;
2692 // If input is optional and is not present, the method returns an empty ConstValue
2693 // which can be compared to nullptr.
2694 ConstValue GetInput(size_t index) const;
2695 // If output is optional and is not present, the method returns an empty UnownedValue
2696 // which can be compared to nullptr.
2697 UnownedValue GetOutput(size_t index, const int64_t* dim_values, size_t dim_count) const;
2698 UnownedValue GetOutput(size_t index, const std::vector<int64_t>& dims) const;
2699 void* GetGPUComputeStream() const;
2701 OrtAllocator* GetAllocator(const OrtMemoryInfo& memory_info) const;
2702 OrtKernelContext* GetOrtKernelContext() const { return ctx_; }
2703 void ParallelFor(void (*fn)(void*, size_t), size_t total, size_t num_batch, void* usr_data) const;
2704
2705 private:
2706 OrtKernelContext* ctx_;
2707};
2708
2709struct KernelInfo;
2710
2711namespace detail {
2712namespace attr_utils {
2713void GetAttr(const OrtKernelInfo* p, const char* name, float&);
2714void GetAttr(const OrtKernelInfo* p, const char* name, int64_t&);
2715void GetAttr(const OrtKernelInfo* p, const char* name, std::string&);
2716void GetAttrs(const OrtKernelInfo* p, const char* name, std::vector<float>&);
2717void GetAttrs(const OrtKernelInfo* p, const char* name, std::vector<int64_t>&);
2718} // namespace attr_utils
2719
2720template <typename T>
2721struct KernelInfoImpl : Base<T> {
2722 using B = Base<T>;
2723 using B::B;
2724
2725 KernelInfo Copy() const;
2726
2727 template <typename R> // R is only implemented for float, int64_t, and string
2728 R GetAttribute(const char* name) const {
2729 R val;
2730 attr_utils::GetAttr(this->p_, name, val);
2731 return val;
2732 }
2733
2734 template <typename R> // R is only implemented for std::vector<float>, std::vector<int64_t>
2735 std::vector<R> GetAttributes(const char* name) const {
2736 std::vector<R> result;
2737 attr_utils::GetAttrs(this->p_, name, result);
2738 return result;
2739 }
2740
2741 Value GetTensorAttribute(const char* name, OrtAllocator* allocator) const;
2742
2743 size_t GetInputCount() const;
2744 size_t GetOutputCount() const;
2745
2746 std::string GetInputName(size_t index) const;
2747 std::string GetOutputName(size_t index) const;
2748
2749 TypeInfo GetInputTypeInfo(size_t index) const;
2750 TypeInfo GetOutputTypeInfo(size_t index) const;
2751
2752 ConstValue GetTensorConstantInput(size_t index, int* is_constant) const;
2753
2754 std::string GetNodeName() const;
2755 Logger GetLogger() const;
2756};
2757
2758} // namespace detail
2759
2760using ConstKernelInfo = detail::KernelInfoImpl<detail::Unowned<const OrtKernelInfo>>;
2761
2768struct KernelInfo : detail::KernelInfoImpl<OrtKernelInfo> {
2769 using Base = detail::KernelInfoImpl<OrtKernelInfo>;
2770 using Base::Base;
2771 explicit KernelInfo(std::nullptr_t) {}
2772 explicit KernelInfo(OrtKernelInfo* info);
2773 ConstKernelInfo GetConst() const { return ConstKernelInfo{this->p_}; }
2774};
2775
2779struct Op : detail::Base<OrtOp> {
2781 using Base::Base;
2782
2783 explicit Op(std::nullptr_t) {}
2784
2785 explicit Op(OrtOp*);
2786
2787 static Op Create(const OrtKernelInfo* info, const char* op_name, const char* domain,
2788 int version, const char** type_constraint_names,
2789 const ONNXTensorElementDataType* type_constraint_values,
2790 size_t type_constraint_count,
2791 const OpAttr* attr_values,
2792 size_t attr_count,
2793 size_t input_count, size_t output_count);
2794
2795 void Invoke(const OrtKernelContext* context,
2796 const Value* input_values,
2797 size_t input_count,
2798 Value* output_values,
2799 size_t output_count);
2800
2801 // For easier refactoring
2802 void Invoke(const OrtKernelContext* context,
2803 const OrtValue* const* input_values,
2804 size_t input_count,
2805 OrtValue* const* output_values,
2806 size_t output_count);
2807};
2808
2814 SymbolicInteger(int64_t i) : i_(i), is_int_(true) {};
2815 SymbolicInteger(const char* s) : s_(s), is_int_(false) {};
2818
2821
2822 bool operator==(const SymbolicInteger& dim) const {
2823 if (is_int_ == dim.is_int_) {
2824 if (is_int_) {
2825 return i_ == dim.i_;
2826 } else {
2827 return std::string{s_} == std::string{dim.s_};
2828 }
2829 }
2830 return false;
2831 }
2832
2833 bool IsInt() const { return is_int_; }
2834 int64_t AsInt() const { return i_; }
2835 const char* AsSym() const { return s_; }
2836
2837 static constexpr int INVALID_INT_DIM = -2;
2838
2839 private:
2840 union {
2841 int64_t i_;
2842 const char* s_;
2843 };
2844 bool is_int_;
2845 };
2846
2847 using Shape = std::vector<SymbolicInteger>;
2848
2850
2851 const Shape& GetInputShape(size_t indice) const { return input_shapes_.at(indice); }
2852
2853 size_t GetInputCount() const { return input_shapes_.size(); }
2854
2856
2857 int64_t GetAttrInt(const char* attr_name);
2858
2859 using Ints = std::vector<int64_t>;
2860 Ints GetAttrInts(const char* attr_name);
2861
2862 float GetAttrFloat(const char* attr_name);
2863
2864 using Floats = std::vector<float>;
2865 Floats GetAttrFloats(const char* attr_name);
2866
2867 std::string GetAttrString(const char* attr_name);
2868
2869 using Strings = std::vector<std::string>;
2870 Strings GetAttrStrings(const char* attr_name);
2871
2872 private:
2873 ConstOpAttr GetAttrHdl(const char* attr_name) const;
2874 const OrtApi* ort_api_;
2876 std::vector<Shape> input_shapes_;
2877};
2878
2880
2881#define MAX_CUSTOM_OP_END_VER (1UL << 31) - 1
2882
2883template <typename TOp, typename TKernel, bool WithStatus = false>
2887 OrtCustomOp::GetName = [](const OrtCustomOp* this_) { return static_cast<const TOp*>(this_)->GetName(); };
2888
2889 OrtCustomOp::GetExecutionProviderType = [](const OrtCustomOp* this_) { return static_cast<const TOp*>(this_)->GetExecutionProviderType(); };
2890
2891 OrtCustomOp::GetInputTypeCount = [](const OrtCustomOp* this_) { return static_cast<const TOp*>(this_)->GetInputTypeCount(); };
2892 OrtCustomOp::GetInputType = [](const OrtCustomOp* this_, size_t index) { return static_cast<const TOp*>(this_)->GetInputType(index); };
2893 OrtCustomOp::GetInputMemoryType = [](const OrtCustomOp* this_, size_t index) { return static_cast<const TOp*>(this_)->GetInputMemoryType(index); };
2894
2895 OrtCustomOp::GetOutputTypeCount = [](const OrtCustomOp* this_) { return static_cast<const TOp*>(this_)->GetOutputTypeCount(); };
2896 OrtCustomOp::GetOutputType = [](const OrtCustomOp* this_, size_t index) { return static_cast<const TOp*>(this_)->GetOutputType(index); };
2897
2898#if defined(_MSC_VER) && !defined(__clang__)
2899#pragma warning(push)
2900#pragma warning(disable : 26409)
2901#endif
2902 OrtCustomOp::KernelDestroy = [](void* op_kernel) { delete static_cast<TKernel*>(op_kernel); };
2903#if defined(_MSC_VER) && !defined(__clang__)
2904#pragma warning(pop)
2905#endif
2906 OrtCustomOp::GetInputCharacteristic = [](const OrtCustomOp* this_, size_t index) { return static_cast<const TOp*>(this_)->GetInputCharacteristic(index); };
2907 OrtCustomOp::GetOutputCharacteristic = [](const OrtCustomOp* this_, size_t index) { return static_cast<const TOp*>(this_)->GetOutputCharacteristic(index); };
2908
2909 OrtCustomOp::GetVariadicInputMinArity = [](const OrtCustomOp* this_) { return static_cast<const TOp*>(this_)->GetVariadicInputMinArity(); };
2910 OrtCustomOp::GetVariadicInputHomogeneity = [](const OrtCustomOp* this_) { return static_cast<int>(static_cast<const TOp*>(this_)->GetVariadicInputHomogeneity()); };
2911 OrtCustomOp::GetVariadicOutputMinArity = [](const OrtCustomOp* this_) { return static_cast<const TOp*>(this_)->GetVariadicOutputMinArity(); };
2912 OrtCustomOp::GetVariadicOutputHomogeneity = [](const OrtCustomOp* this_) { return static_cast<int>(static_cast<const TOp*>(this_)->GetVariadicOutputHomogeneity()); };
2913#ifdef __cpp_if_constexpr
2914 if constexpr (WithStatus) {
2915#else
2916 if (WithStatus) {
2917#endif
2918 OrtCustomOp::CreateKernelV2 = [](const OrtCustomOp* this_, const OrtApi* api, const OrtKernelInfo* info, void** op_kernel) -> OrtStatusPtr {
2919 return static_cast<const TOp*>(this_)->CreateKernelV2(*api, info, op_kernel);
2920 };
2921 OrtCustomOp::KernelComputeV2 = [](void* op_kernel, OrtKernelContext* context) -> OrtStatusPtr {
2922 return static_cast<TKernel*>(op_kernel)->ComputeV2(context);
2923 };
2924 } else {
2927
2928 OrtCustomOp::CreateKernel = [](const OrtCustomOp* this_, const OrtApi* api, const OrtKernelInfo* info) { return static_cast<const TOp*>(this_)->CreateKernel(*api, info); };
2929 OrtCustomOp::KernelCompute = [](void* op_kernel, OrtKernelContext* context) {
2930 static_cast<TKernel*>(op_kernel)->Compute(context);
2931 };
2932 }
2933
2934 SetShapeInferFn<TOp>(0);
2935
2936 OrtCustomOp::GetStartVersion = [](const OrtCustomOp* this_) {
2937 return static_cast<const TOp*>(this_)->start_ver_;
2938 };
2939
2940 OrtCustomOp::GetEndVersion = [](const OrtCustomOp* this_) {
2941 return static_cast<const TOp*>(this_)->end_ver_;
2942 };
2943
2946 OrtCustomOp::GetAliasMap = nullptr;
2948 }
2949
2950 // Default implementation of GetExecutionProviderType that returns nullptr to default to the CPU provider
2951 const char* GetExecutionProviderType() const { return nullptr; }
2952
2953 // Default implementations of GetInputCharacteristic() and GetOutputCharacteristic() below
2954 // (inputs and outputs are required by default)
2956 return OrtCustomOpInputOutputCharacteristic::INPUT_OUTPUT_REQUIRED;
2957 }
2958
2960 return OrtCustomOpInputOutputCharacteristic::INPUT_OUTPUT_REQUIRED;
2961 }
2962
2963 // Default implementation of GetInputMemoryType() that returns OrtMemTypeDefault
2964 OrtMemType GetInputMemoryType(size_t /*index*/) const {
2965 return OrtMemTypeDefault;
2966 }
2967
2968 // Default implementation of GetVariadicInputMinArity() returns 1 to specify that a variadic input
2969 // should expect at least 1 argument.
2971 return 1;
2972 }
2973
2974 // Default implementation of GetVariadicInputHomegeneity() returns true to specify that all arguments
2975 // to a variadic input should be of the same type.
2977 return true;
2978 }
2979
2980 // Default implementation of GetVariadicOutputMinArity() returns 1 to specify that a variadic output
2981 // should produce at least 1 output value.
2983 return 1;
2984 }
2985
2986 // Default implementation of GetVariadicOutputHomegeneity() returns true to specify that all output values
2987 // produced by a variadic output should be of the same type.
2989 return true;
2990 }
2991
2992 // Declare list of session config entries used by this Custom Op.
2993 // Implement this function in order to get configs from CustomOpBase::GetSessionConfigs().
2994 // This default implementation returns an empty vector of config entries.
2995 std::vector<std::string> GetSessionConfigKeys() const {
2996 return std::vector<std::string>{};
2997 }
2998
2999 // Ort::CustomOpBase derived class should provide the following static method with the type/shape inferencing
3000 // implementation if needed:
3001 // static OrtStatusPtr InferOutputShape(Ort::ShapeInferContext& context)
3002 template <typename C>
3003 decltype(&C::InferOutputShape) SetShapeInferFn(decltype(&C::InferOutputShape)) {
3005 ShapeInferContext ctx(&GetApi(), ort_ctx);
3006 return C::InferOutputShape(ctx);
3007 };
3008 return {};
3009 }
3010
3011 template <typename C>
3015
3016 protected:
3017 // Helper function that returns a map of session config entries specified by CustomOpBase::GetSessionConfigKeys.
3018 void GetSessionConfigs(std::unordered_map<std::string, std::string>& out, ConstSessionOptions options) const;
3019
3020 int start_ver_ = 1;
3021 int end_ver_ = MAX_CUSTOM_OP_END_VER;
3022};
3023
3024// Forward declaration to resolve circular dependency
3025// on ConstNode
3027
3028namespace detail {
3029template <typename T>
3031 using B = Base<T>;
3032 using B::B;
3033
3035 std::string GetName() const;
3041 std::vector<ValueInfoConsumerProducerInfo> GetConsumers() const;
3051 bool IsGraphOutput() const;
3055 bool IsFromOuterScope() const;
3056};
3057} // namespace detail
3058
3059// Const object holder that does not own the underlying object
3061
3066 ValueInfo() = default; // Same thing as with nullptr
3067 explicit ValueInfo(std::nullptr_t) {}
3069 explicit ValueInfo(OrtValueInfo* p) : ConstValueInfoImpl<OrtValueInfo>{p} {}
3070
3071#if !defined(ORT_MINIMAL_BUILD)
3072 // Create ValueInfo for a tensor
3073 explicit ValueInfo(const std::string& name, const ConstTypeInfo& type_info);
3074#endif
3075 ConstValueInfo GetConst() const { return ConstValueInfo{this->p_}; }
3076};
3077
3078// Forward declaration
3079struct AttrNameSubgraph;
3080
3081namespace detail {
3082// Forward decl
3083template <typename T>
3084struct ConstGraphImpl;
3085
3086template <typename T>
3087struct ConstNodeImpl : Base<T> {
3088 using B = Base<T>;
3089 using B::B;
3090
3091 // <Wraps OrtApi::Node_GetId
3092 size_t GetId() const;
3093 // <Wraps OrtApi::Node_GetName
3094 std::string GetName() const;
3095 // <Wraps OrtApi::Node_GetOperatorType
3096 std::string GetOperatorType() const;
3097 // <Wraps OrtApi::Node_GetDomain
3098 std::string GetDomain() const;
3099 // <Wraps OrtApi::Node_GetSinceVersion
3100 int GetSinceVersion() const;
3101
3102 // <Wraps OrtApi::Node_Inputs
3103 std::vector<ConstValueInfo> GetInputs() const;
3104 // <Wraps OrtApi::Node_Outputs
3105 std::vector<ConstValueInfo> GetOutputs() const;
3106 // <Wraps OrtApi::Node_ImplicitInputs
3107 std::vector<ConstValueInfo> GetImplicitInputs() const;
3108 // <Wraps OrtApi::Node_GetAttributes
3109 std::vector<ConstOpAttr> GetAttributes() const;
3110 // <Wraps OrtApi::Node_GetAttributeByName
3111 // Please, read C API doc for details
3112 Status GetAttributeByName(const std::string& name, ConstOpAttr& attr) const;
3113 // <Wraps OrtApi::Node_GetSubgraphs
3114 std::vector<AttrNameSubgraph> GetSubgraphs() const;
3115 // <Wraps OrtApi::Node_GetGraph
3116 // ConstGraph is not available yet
3118 // <Wraps OrtApi::Node_GetEpName
3119 std::string GetEpName() const;
3120};
3121} // namespace detail
3122
3124
3128struct Node : detail::ConstNodeImpl<OrtNode> {
3129 Node() = default; // Same thing as with nullptr
3130 explicit Node(std::nullptr_t) {}
3131 explicit Node(OrtNode* p) : ConstNodeImpl<OrtNode>{p} {}
3132
3133#if !defined(ORT_MINIMAL_BUILD)
3134 Node(const std::string& operator_name, const std::string& operator_domain,
3135 const std::string& node_name,
3136 const std::vector<std::string>& input_names,
3137 const std::vector<std::string>& output_names);
3138
3142 Node(const std::string& operator_name, const std::string& operator_domain,
3143 const std::string& node_name,
3144 const std::vector<std::string>& input_names,
3145 const std::vector<std::string>& output_names,
3146 std::vector<OpAttr>& attributes);
3147
3148 private:
3149 static void Init(const std::string& operator_name, const std::string& operator_domain,
3150 const std::string& node_name,
3151 const std::vector<std::string>& input_names,
3152 const std::vector<std::string>& output_names,
3153 std::vector<OpAttr>& attributes,
3154 OrtNode*& node);
3155#endif // !defined(ORT_MINIMAL_BUILD)
3156};
3157
3158// Return struct for some of ValueInfo APIs.
3159// Must be declared after ConstNode is available.
3162 // either producer output or consumer output index
3163 // producer is unsigned only, output can be -1
3164 int64_t index;
3165};
3166
3167// Represents a return value for Graph::GetOperatorSets()
3169 std::string domain;
3170 int64_t version;
3171};
3172
3173namespace detail {
3174template <typename T>
3176 using B = Base<T>;
3177 using B::B;
3178
3179 // <Wraps OrtApi::Graph_GetName
3180 std::string GetName() const;
3181 // <Wraps OrtApi::Graph_GetModelPath
3182 std::basic_string<ORTCHAR_T> GetModelPath() const;
3183 // <Wraps OrtApi::Graph_GetOnnxIRVersion
3184 int64_t GetOnnxIRVersion() const;
3185 // <Wraps OrtApi::Graph_GetOperatorSets
3186 std::vector<OperatorSet> GetOperatorSets() const;
3187 // <Wraps OrtApi::Graph_Inputs
3188 std::vector<ConstValueInfo> GetInputs() const;
3189 // <Wraps OrtApi::Graph_Outputs
3190 std::vector<ConstValueInfo> GetOutputs() const;
3191 // <Wraps OrtApi::Graph_Initializers
3192 std::vector<ConstValueInfo> GetInitializers() const;
3193 // <Wraps OrtApi::Graph_GetNodes
3194 std::vector<ConstNode> GetNodes() const;
3195 // <Wraps OrtApi::Graph_GetParentGraph
3197 // <Wraps OrtApi::Graph_GetGraphView
3198 Graph GetGraphView(const std::vector<ConstNode>& nodes) const;
3199 // <Wraps OrtApi::Graph_GetModelMetadata
3201};
3202
3203template <typename T>
3206 using B::B;
3207
3208#if !defined(ORT_MINIMAL_BUILD)
3209 // <Wraps GetModelEditorApi().SetGraphInputs()
3210 void SetInputs(std::vector<ValueInfo>& inputs);
3211 // <Wraps GetModelEditorApi().SetGraphOutputs()
3212 void SetOutputs(std::vector<ValueInfo>& outputs);
3213 // <Wraps GetModelEditorApi().AddInitializerToGraph()
3214 void AddInitializer(const std::string& name, Value& initializer, bool data_is_external); // Graph takes ownership of Value
3215 // <Wraps GetModelEditorApi().AddNodeToGraph()
3216 void AddNode(Node& node); // Graph takes ownership of Node
3217#endif // !defined(ORT_MINIMAL_BUILD)
3218};
3219} // namespace detail
3220
3222
3223// Return value for Node API
3224// Must be declared after ConstGraph
3229
3233struct Graph : detail::GraphImpl<OrtGraph> {
3234 explicit Graph(std::nullptr_t) {}
3235 explicit Graph(OrtGraph* p) : GraphImpl<OrtGraph>{p} {}
3236#if !defined(ORT_MINIMAL_BUILD)
3237 // <Wraps GetModelEditorApi().CreateGraph()
3239#endif
3240};
3241
3242namespace detail {
3243template <typename T>
3246 using B::B;
3247
3248#if !defined(ORT_MINIMAL_BUILD)
3249 // <Wraps GetModelEditorApi().AddGraphToModel()
3250 void AddGraph(Graph& graph);
3251#endif
3252};
3253} // namespace detail
3254
3255// Const object holder that does not own the underlying object
3257
3261struct Model : detail::ModelImpl<OrtModel> {
3262 using DomainOpsetPair = std::pair<std::string, int>;
3263
3264 explicit Model(std::nullptr_t) {}
3265 explicit Model(OrtModel* p) : ModelImpl<OrtModel>{p} {}
3266
3267#if !defined(ORT_MINIMAL_BUILD)
3268 //< Wraps GetModelEditorApi().CreateModel()
3269 explicit Model(const std::vector<DomainOpsetPair>& opsets);
3270#endif
3271};
3272} // namespace Ort
3273#include "onnxruntime_cxx_inline.h"
struct OrtMemoryInfo OrtMemoryInfo
Definition onnxruntime_c_api.h:296
struct OrtKernelInfo OrtKernelInfo
Definition onnxruntime_c_api.h:450
struct OrtNode OrtNode
Definition onnxruntime_c_api.h:324
OrtLoggingLevel
Logging severity levels.
Definition onnxruntime_c_api.h:246
OrtMemoryInfoDeviceType
This mimics OrtDevice type constants so they can be returned in the API.
Definition onnxruntime_c_api.h:485
struct OrtShapeInferContext OrtShapeInferContext
Definition onnxruntime_c_api.h:321
void(* OrtLoggingFunction)(void *param, OrtLoggingLevel severity, const char *category, const char *logid, const char *code_location, const char *message)
Definition onnxruntime_c_api.h:414
void(* OrtCustomJoinThreadFn)(OrtCustomThreadHandle ort_custom_thread_handle)
Custom thread join function.
Definition onnxruntime_c_api.h:938
OrtCustomOpInputOutputCharacteristic
Definition onnxruntime_c_api.h:6600
struct OrtTensorRTProviderOptionsV2 OrtTensorRTProviderOptionsV2
Definition onnxruntime_c_api.h:313
struct OrtEpApi OrtEpApi
Definition onnxruntime_c_api.h:882
struct OrtThreadingOptions OrtThreadingOptions
Definition onnxruntime_c_api.h:310
struct OrtSequenceTypeInfo OrtSequenceTypeInfo
Definition onnxruntime_c_api.h:304
struct OrtValueInfo OrtValueInfo
Definition onnxruntime_c_api.h:323
struct OrtDnnlProviderOptions OrtDnnlProviderOptions
Definition onnxruntime_c_api.h:317
OrtSparseIndicesFormat
Definition onnxruntime_c_api.h:235
struct OrtPrepackedWeightsContainer OrtPrepackedWeightsContainer
Definition onnxruntime_c_api.h:312
struct OrtSession OrtSession
Definition onnxruntime_c_api.h:298
OrtCompiledModelCompatibility
The C API.
Definition onnxruntime_c_api.h:961
OrtStatus *(* EpSelectionDelegate)(const OrtEpDevice **ep_devices, size_t num_devices, const OrtKeyValuePairs *model_metadata, const OrtKeyValuePairs *runtime_metadata, const OrtEpDevice **selected, size_t max_selected, size_t *num_selected, void *state)
Delegate to allow providing custom OrtEpDevice selection logic.
Definition onnxruntime_c_api.h:529
struct OrtCustomOpDomain OrtCustomOpDomain
Definition onnxruntime_c_api.h:307
struct OrtIoBinding OrtIoBinding
Definition onnxruntime_c_api.h:297
struct OrtExternalInitializerInfo OrtExternalInitializerInfo
Definition onnxruntime_c_api.h:332
OrtAllocatorType
Definition onnxruntime_c_api.h:456
struct OrtOp OrtOp
Definition onnxruntime_c_api.h:318
struct OrtTypeInfo OrtTypeInfo
Definition onnxruntime_c_api.h:301
struct OrtTensorTypeAndShapeInfo OrtTensorTypeAndShapeInfo
Definition onnxruntime_c_api.h:302
struct OrtCUDAProviderOptionsV2 OrtCUDAProviderOptionsV2
Definition onnxruntime_c_api.h:315
struct OrtKernelContext OrtKernelContext
Definition onnxruntime_c_api.h:452
struct OrtCANNProviderOptions OrtCANNProviderOptions
Definition onnxruntime_c_api.h:316
struct OrtEpDevice OrtEpDevice
Definition onnxruntime_c_api.h:329
void(* RunAsyncCallbackFn)(void *user_data, OrtValue **outputs, size_t num_outputs, OrtStatusPtr status)
Callback function for RunAsync.
Definition onnxruntime_c_api.h:949
OrtHardwareDeviceType
Definition onnxruntime_c_api.h:492
struct OrtModel OrtModel
Definition onnxruntime_c_api.h:326
struct OrtGraph OrtGraph
Definition onnxruntime_c_api.h:325
struct OrtSyncStream OrtSyncStream
Definition onnxruntime_c_api.h:331
struct OrtSessionOptions OrtSessionOptions
Definition onnxruntime_c_api.h:306
OrtDeviceMemoryType
This matches OrtDevice::MemoryType values.
Definition onnxruntime_c_api.h:478
struct OrtValue OrtValue
Definition onnxruntime_c_api.h:299
OrtStatus *(* OrtWriteBufferFunc)(void *state, const void *buffer, size_t buffer_num_bytes)
Function called by ORT to write a buffer to a custom destination (e.g., file, stream,...
Definition onnxruntime_c_api.h:548
GraphOptimizationLevel
Graph optimization level.
Definition onnxruntime_c_api.h:423
struct OrtKeyValuePairs OrtKeyValuePairs
Definition onnxruntime_c_api.h:330
OrtStatus * OrtStatusPtr
Definition onnxruntime_c_api.h:337
OrtMemType
Memory types for allocated memory, execution provider specific types should be extended in each provi...
Definition onnxruntime_c_api.h:466
OrtSparseFormat
Definition onnxruntime_c_api.h:227
ONNXType
Definition onnxruntime_c_api.h:215
struct OrtEnv OrtEnv
Definition onnxruntime_c_api.h:294
OrtErrorCode
Definition onnxruntime_c_api.h:254
struct OrtStatus OrtStatus
Definition onnxruntime_c_api.h:295
OrtStatus *(* OrtGetInitializerLocationFunc)(void *state, const char *initializer_name, const OrtValue *initializer_value, const OrtExternalInitializerInfo *external_info, OrtExternalInitializerInfo **new_external_info)
Function called by ORT to allow user to specify how an initializer should be saved,...
Definition onnxruntime_c_api.h:582
#define ORT_API_VERSION
The API version defined in this header.
Definition onnxruntime_c_api.h:41
struct OrtLogger OrtLogger
Definition onnxruntime_c_api.h:320
struct OrtMapTypeInfo OrtMapTypeInfo
Definition onnxruntime_c_api.h:303
struct OrtArenaCfg OrtArenaCfg
Definition onnxruntime_c_api.h:311
ExecutionMode
Definition onnxruntime_c_api.h:431
OrtOpAttrType
Definition onnxruntime_c_api.h:272
OrtCustomThreadHandle(* OrtCustomCreateThreadFn)(void *ort_custom_thread_creation_options, OrtThreadWorkerFn ort_thread_worker_fn, void *ort_worker_fn_param)
Ort custom thread creation function.
Definition onnxruntime_c_api.h:931
ONNXTensorElementDataType
Definition onnxruntime_c_api.h:184
OrtExecutionProviderDevicePolicy
These are the default EP selection policies used by ORT when doing automatic EP selection.
Definition onnxruntime_c_api.h:500
const OrtApiBase * OrtGetApiBase(void)
The Onnxruntime library's entry point to access the C API.
@ ORT_LOGGING_LEVEL_WARNING
Warning messages.
Definition onnxruntime_c_api.h:249
@ OrtMemTypeDefault
The default allocator for execution provider.
Definition onnxruntime_c_api.h:474
@ ORT_FAIL
Definition onnxruntime_c_api.h:256
@ ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT
Definition onnxruntime_c_api.h:186
std::vector< Value > GetOutputValuesHelper(const OrtIoBinding *binding, OrtAllocator *)
std::vector< std::string > GetOutputNamesHelper(const OrtIoBinding *binding, OrtAllocator *)
void OrtRelease(OrtAllocator *ptr)
Definition onnxruntime_cxx_api.h:615
std::string MakeCustomOpConfigEntryKey(const char *custom_op_name, const char *config)
All C++ Onnxruntime APIs are defined inside this namespace.
Definition onnxruntime_cxx_api.h:48
const OrtModelEditorApi & GetModelEditorApi()
This returns a reference to the ORT C Model Editor API. Used if building or augmenting a model at run...
Definition onnxruntime_cxx_api.h:215
std::unique_ptr< char, detail::AllocatedFree > AllocatedStringPtr
unique_ptr typedef used to own strings allocated by OrtAllocators and release them at the end of the ...
Definition onnxruntime_cxx_api.h:788
detail::ConstSessionOptionsImpl< detail::Unowned< const OrtSessionOptions > > ConstSessionOptions
Definition onnxruntime_cxx_api.h:1434
detail::KernelInfoImpl< detail::Unowned< const OrtKernelInfo > > ConstKernelInfo
Definition onnxruntime_cxx_api.h:2760
const OrtApi & GetApi() noexcept
This returns a reference to the ORT C API.
Definition onnxruntime_cxx_api.h:189
const OrtCompileApi & GetCompileApi()
This returns a reference to the ORT C Compile API. Used if compiling a model at runtime.
Definition onnxruntime_cxx_api.h:229
detail::AllocatorImpl< detail::Unowned< OrtAllocator > > UnownedAllocator
Definition onnxruntime_cxx_api.h:1061
OrtCompiledModelCompatibility GetModelCompatibilityForEpDevices(const std::vector< ConstEpDevice > &ep_devices, const char *compatibility_info)
Validate a compiled model's compatibility for one or more EP devices.
detail::SessionOptionsImpl< detail::Unowned< OrtSessionOptions > > UnownedSessionOptions
Definition onnxruntime_cxx_api.h:1433
std::string GetBuildInfoString()
This function returns the onnxruntime build information: including git branch, git commit id,...
const OrtEpApi & GetEpApi()
This returns a reference to the ORT C EP API. Used if authoring a plugin execution provider.
Definition onnxruntime_cxx_api.h:243
std::string GetVersionString()
This function returns the onnxruntime version string.
std::vector< std::string > GetAvailableProviders()
This is a C++ wrapper for OrtApi::GetAvailableProviders() and returns a vector of strings representin...
Ort::Status(*)(Ort::ShapeInferContext &) ShapeInferFn
Definition onnxruntime_cxx_api.h:2879
Status CompileModel(const Env &env, const ModelCompilationOptions &model_compilation_options)
Compiles an input model to generate a model with EPContext nodes that execute EP-specific kernels....
Wrapper around OrtAllocator.
Definition onnxruntime_cxx_api.h:1056
Allocator(const Session &session, const OrtMemoryInfo *)
Allocator(std::nullptr_t)
Convenience to create a class member and then replace with an instance.
Definition onnxruntime_cxx_api.h:1057
Wrapper around OrtAllocator default instance that is owned by Onnxruntime.
Definition onnxruntime_cxx_api.h:1047
AllocatorWithDefaultOptions(std::nullptr_t)
Convenience to create a class member and then replace with an instance.
Definition onnxruntime_cxx_api.h:1048
it is a structure that represents the configuration of an arena based allocator
Definition onnxruntime_cxx_api.h:2465
ArenaCfg(std::nullptr_t)
Create an empty ArenaCfg object, must be assigned a valid one to be used.
Definition onnxruntime_cxx_api.h:2466
ArenaCfg(const std::unordered_map< std::string, size_t > &arena_config)
ArenaCfg(size_t max_mem, int arena_extend_strategy, int initial_chunk_size_bytes, int max_dead_bytes_per_chunk)
Definition onnxruntime_cxx_api.h:3225
ConstGraph sub_graph
Definition onnxruntime_cxx_api.h:3227
std::string attr_name
Definition onnxruntime_cxx_api.h:3226
bfloat16 (Brain Floating Point) data type
Definition onnxruntime_cxx_api.h:413
bool operator==(const BFloat16_t &rhs) const noexcept
onnxruntime_float16::BFloat16Impl< BFloat16_t > Base
Definition onnxruntime_cxx_api.h:425
BFloat16_t()=default
static constexpr BFloat16_t FromBits(uint16_t v) noexcept
Explicit conversion to uint16_t representation of bfloat16.
Definition onnxruntime_cxx_api.h:434
bool operator!=(const BFloat16_t &rhs) const noexcept
Definition onnxruntime_cxx_api.h:532
BFloat16_t(float v) noexcept
__ctor from float. Float is converted into bfloat16 16-bit representation.
Definition onnxruntime_cxx_api.h:440
float ToFloat() const noexcept
Converts bfloat16 to float.
Definition onnxruntime_cxx_api.h:446
bool operator<(const BFloat16_t &rhs) const noexcept
The CUDAProviderOptions (V2)
Definition onnxruntime_cxx_api.h:859
CUDAProviderOptions()
Wraps OrtApi::CreateCUDAProviderOptions.
CUDAProviderOptions(std::nullptr_t)
Definition onnxruntime_cxx_api.h:860
void UpdateWithValue(const char *key, void *value)
Wrapper around OrtApi::GetCUDAProviderOptionsByName.
std::string GetCUDAProviderOptionsAsString() const
Wrapper around OrtApi::UpdateCUDAProviderOptionsWithValue.
void Update(const std::unordered_map< std::string, std::string > &options)
Wrapper around OrtApi::GetCUDAProviderOptionsAsString.
void * GetOptionByName(const char *name) const
Definition onnxruntime_cxx_api.h:2884
OrtCustomOpInputOutputCharacteristic GetOutputCharacteristic(size_t) const
Definition onnxruntime_cxx_api.h:2959
OrtCustomOpInputOutputCharacteristic GetInputCharacteristic(size_t) const
Definition onnxruntime_cxx_api.h:2955
OrtMemType GetInputMemoryType(size_t) const
Definition onnxruntime_cxx_api.h:2964
std::vector< std::string > GetSessionConfigKeys() const
Definition onnxruntime_cxx_api.h:2995
bool GetVariadicInputHomogeneity() const
Definition onnxruntime_cxx_api.h:2976
int GetVariadicInputMinArity() const
Definition onnxruntime_cxx_api.h:2970
void SetShapeInferFn(...)
Definition onnxruntime_cxx_api.h:3012
CustomOpBase()
Definition onnxruntime_cxx_api.h:2885
bool GetVariadicOutputHomogeneity() const
Definition onnxruntime_cxx_api.h:2988
int GetVariadicOutputMinArity() const
Definition onnxruntime_cxx_api.h:2982
decltype(&C::InferOutputShape) SetShapeInferFn(decltype(&C::InferOutputShape))
Definition onnxruntime_cxx_api.h:3003
const char * GetExecutionProviderType() const
Definition onnxruntime_cxx_api.h:2951
void GetSessionConfigs(std::unordered_map< std::string, std::string > &out, ConstSessionOptions options) const
Class that represents session configuration entries for one or more custom operators.
Definition onnxruntime_cxx_api.h:1290
~CustomOpConfigs()=default
CustomOpConfigs & AddConfig(const char *custom_op_name, const char *config_key, const char *config_value)
Adds a session configuration entry/value for a specific custom operator.
CustomOpConfigs & operator=(CustomOpConfigs &&o)=default
CustomOpConfigs(CustomOpConfigs &&o)=default
CustomOpConfigs()=default
const std::unordered_map< std::string, std::string > & GetFlattenedConfigs() const
Returns a flattened map of custom operator configuration entries and their values.
CustomOpConfigs(const CustomOpConfigs &)=default
CustomOpConfigs & operator=(const CustomOpConfigs &)=default
Custom Op Domain.
Definition onnxruntime_cxx_api.h:1195
CustomOpDomain(std::nullptr_t)
Create an empty CustomOpDomain object, must be assigned a valid one to be used.
Definition onnxruntime_cxx_api.h:1199
CustomOpDomain(const char *domain)
Wraps OrtApi::CreateCustomOpDomain.
void Add(const OrtCustomOp *op)
Wraps CustomOpDomain_Add.
The Env (Environment)
Definition onnxruntime_cxx_api.h:1138
Env & EnableTelemetryEvents()
Wraps OrtApi::EnableTelemetryEvents.
Env(OrtEnv *p)
C Interop Helper.
Definition onnxruntime_cxx_api.h:1155
Env & CreateAndRegisterAllocatorV2(const std::string &provider_type, const OrtMemoryInfo *mem_info, const std::unordered_map< std::string, std::string > &options, const OrtArenaCfg *arena_cfg)
Wraps OrtApi::CreateAndRegisterAllocatorV2.
Env & UnregisterExecutionProviderLibrary(const char *registration_name)
Wraps OrtApi::UnregisterExecutionProviderLibrary.
std::vector< ConstEpDevice > GetEpDevices() const
Env & UnregisterAllocator(const OrtMemoryInfo *mem_info)
Wraps OrtApi::UnregisterAllocator.
Env(std::nullptr_t)
Create an empty Env object, must be assigned a valid one to be used.
Definition onnxruntime_cxx_api.h:1139
Env(OrtLoggingLevel logging_level=ORT_LOGGING_LEVEL_WARNING, const char *logid="")
Wraps OrtApi::CreateEnv.
Env(const OrtThreadingOptions *tp_options, OrtLoggingLevel logging_level=ORT_LOGGING_LEVEL_WARNING, const char *logid="")
Wraps OrtApi::CreateEnvWithGlobalThreadPools.
Env(const OrtThreadingOptions *tp_options, OrtLoggingFunction logging_function, void *logger_param, OrtLoggingLevel logging_level=ORT_LOGGING_LEVEL_WARNING, const char *logid="")
Wraps OrtApi::CreateEnvWithCustomLoggerAndGlobalThreadPools.
Env & RegisterAllocator(OrtAllocator *allocator)
Wraps OrtApi::RegisterAllocator.
UnownedAllocator CreateSharedAllocator(const OrtEpDevice *ep_device, OrtDeviceMemoryType mem_type, OrtAllocatorType allocator_type, const OrtKeyValuePairs *allocator_options)
Wraps OrtApi::CreateSharedAllocator.
Env(OrtLoggingLevel logging_level, const char *logid, OrtLoggingFunction logging_function, void *logger_param)
Wraps OrtApi::CreateEnvWithCustomLogger.
Env & CreateAndRegisterAllocator(const OrtMemoryInfo *mem_info, const OrtArenaCfg *arena_cfg)
Wraps OrtApi::CreateAndRegisterAllocator.
UnownedAllocator GetSharedAllocator(const OrtMemoryInfo *mem_info)
Wraps OrtApi::GetSharedAllocator.
Env & RegisterExecutionProviderLibrary(const char *registration_name, const std::basic_string< char > &path)
Wraps OrtApi::RegisterExecutionProviderLibrary.
Env & UpdateEnvWithCustomLogLevel(OrtLoggingLevel log_severity_level)
Wraps OrtApi::UpdateEnvWithCustomLogLevel.
Status CopyTensors(const std::vector< Value > &src_tensors, const std::vector< Value > &dst_tensors, OrtSyncStream *stream) const
Wraps OrtApi::CopyTensors.
void ReleaseSharedAllocator(const OrtEpDevice *ep_device, OrtDeviceMemoryType mem_type)
Wraps OrtApi::ReleaseSharedAllocator.
Env & DisableTelemetryEvents()
Wraps OrtApi::DisableTelemetryEvents.
Mutable EpDevice that is created by EpApi users.
Definition onnxruntime_cxx_api.h:1114
EpDevice(OrtEpDevice *p)
Take ownership of a pointer created by C API.
Definition onnxruntime_cxx_api.h:1116
EpDevice(OrtEpFactory &ep_factory, ConstHardwareDevice &hardware_device, ConstKeyValuePairs ep_metadata={}, ConstKeyValuePairs ep_options={})
Wraps OrtEpApi::CreateEpDevice.
EpDevice(std::nullptr_t)
No instance is created.
Definition onnxruntime_cxx_api.h:1115
All C++ methods that can fail will throw an exception of this type.
Definition onnxruntime_cxx_api.h:54
const char * what() const noexcept override
Definition onnxruntime_cxx_api.h:59
Exception(const std::string &string, OrtErrorCode code)
Definition onnxruntime_cxx_api.h:55
OrtErrorCode GetOrtErrorCode() const
Definition onnxruntime_cxx_api.h:58
Exception(std::string &&string, OrtErrorCode code)
Definition onnxruntime_cxx_api.h:56
Wrapper around OrtExternalInitializerInfo.
Definition onnxruntime_cxx_api.h:910
ConstExternalInitializerInfo GetConst() const
Wraps OrtApi::CreateExternalInitializerInfo.
Definition onnxruntime_cxx_api.h:918
ExternalInitializerInfo(const char *filepath, int64_t file_offset, size_t byte_size)
Wrapper around CreateExternalInitializerInfo that does not throw an exception.
ExternalInitializerInfo(std::nullptr_t)
Definition onnxruntime_cxx_api.h:914
ExternalInitializerInfo(OrtExternalInitializerInfo *p)
Definition onnxruntime_cxx_api.h:915
static Status Create(const char *filepath, int64_t file_offset, size_t byte_size, ExternalInitializerInfo &out)
IEEE 754 half-precision floating point data type.
Definition onnxruntime_cxx_api.h:271
Float16_t()=default
Default constructor.
Float16_t(float v) noexcept
__ctor from float. Float is converted into float16 16-bit representation.
Definition onnxruntime_cxx_api.h:299
onnxruntime_float16::Float16Impl< Float16_t > Base
Definition onnxruntime_cxx_api.h:281
float ToFloat() const noexcept
Converts float16 to float.
Definition onnxruntime_cxx_api.h:305
static constexpr Float16_t FromBits(uint16_t v) noexcept
Explicit conversion to uint16_t representation of float16.
Definition onnxruntime_cxx_api.h:293
float8e4m3fn (Float8 Floating Point) data type
Definition onnxruntime_cxx_api.h:543
uint8_t value
Definition onnxruntime_cxx_api.h:544
constexpr Float8E4M3FN_t(uint8_t v) noexcept
Definition onnxruntime_cxx_api.h:546
constexpr bool operator==(const Float8E4M3FN_t &rhs) const noexcept
Definition onnxruntime_cxx_api.h:549
constexpr Float8E4M3FN_t() noexcept
Definition onnxruntime_cxx_api.h:545
constexpr bool operator!=(const Float8E4M3FN_t &rhs) const noexcept
Definition onnxruntime_cxx_api.h:550
float8e4m3fnuz (Float8 Floating Point) data type
Definition onnxruntime_cxx_api.h:560
constexpr bool operator==(const Float8E4M3FNUZ_t &rhs) const noexcept
Definition onnxruntime_cxx_api.h:566
uint8_t value
Definition onnxruntime_cxx_api.h:561
constexpr Float8E4M3FNUZ_t() noexcept
Definition onnxruntime_cxx_api.h:562
constexpr bool operator!=(const Float8E4M3FNUZ_t &rhs) const noexcept
Definition onnxruntime_cxx_api.h:567
constexpr Float8E4M3FNUZ_t(uint8_t v) noexcept
Definition onnxruntime_cxx_api.h:563
float8e5m2 (Float8 Floating Point) data type
Definition onnxruntime_cxx_api.h:577
constexpr Float8E5M2_t(uint8_t v) noexcept
Definition onnxruntime_cxx_api.h:580
uint8_t value
Definition onnxruntime_cxx_api.h:578
constexpr bool operator!=(const Float8E5M2_t &rhs) const noexcept
Definition onnxruntime_cxx_api.h:584
constexpr Float8E5M2_t() noexcept
Definition onnxruntime_cxx_api.h:579
constexpr bool operator==(const Float8E5M2_t &rhs) const noexcept
Definition onnxruntime_cxx_api.h:583
float8e5m2fnuz (Float8 Floating Point) data type
Definition onnxruntime_cxx_api.h:594
constexpr Float8E5M2FNUZ_t() noexcept
Definition onnxruntime_cxx_api.h:596
constexpr Float8E5M2FNUZ_t(uint8_t v) noexcept
Definition onnxruntime_cxx_api.h:597
constexpr bool operator!=(const Float8E5M2FNUZ_t &rhs) const noexcept
Definition onnxruntime_cxx_api.h:601
constexpr bool operator==(const Float8E5M2FNUZ_t &rhs) const noexcept
Definition onnxruntime_cxx_api.h:600
uint8_t value
Definition onnxruntime_cxx_api.h:595
Wrapper around OrtGraph.
Definition onnxruntime_cxx_api.h:3233
Graph(OrtGraph *p)
Take ownership of a pointer created by C API.
Definition onnxruntime_cxx_api.h:3235
Graph(std::nullptr_t)
No instance is created.
Definition onnxruntime_cxx_api.h:3234
Wrapper around OrtIoBinding.
Definition onnxruntime_cxx_api.h:2454
UnownedIoBinding GetUnowned() const
Definition onnxruntime_cxx_api.h:2458
ConstIoBinding GetConst() const
Definition onnxruntime_cxx_api.h:2457
IoBinding(Session &session)
IoBinding(std::nullptr_t)
Create an empty object for convenience. Sometimes, we want to initialize members later.
Definition onnxruntime_cxx_api.h:2455
This class wraps a raw pointer OrtKernelContext* that is being passed to the custom kernel Compute() ...
Definition onnxruntime_cxx_api.h:2688
KernelContext(OrtKernelContext *context)
Logger GetLogger() const
ConstValue GetInput(size_t index) const
OrtKernelContext * GetOrtKernelContext() const
Definition onnxruntime_cxx_api.h:2702
void ParallelFor(void(*fn)(void *, size_t), size_t total, size_t num_batch, void *usr_data) const
OrtAllocator * GetAllocator(const OrtMemoryInfo &memory_info) const
void * GetGPUComputeStream() const
size_t GetInputCount() const
size_t GetOutputCount() const
UnownedValue GetOutput(size_t index, const std::vector< int64_t > &dims) const
UnownedValue GetOutput(size_t index, const int64_t *dim_values, size_t dim_count) const
This struct owns the OrtKernInfo* pointer when a copy is made. For convenient wrapping of OrtKernelIn...
Definition onnxruntime_cxx_api.h:2768
KernelInfo(OrtKernelInfo *info)
Take ownership of the instance.
ConstKernelInfo GetConst() const
Definition onnxruntime_cxx_api.h:2773
detail::KernelInfoImpl< OrtKernelInfo > Base
Definition onnxruntime_cxx_api.h:2769
KernelInfo(std::nullptr_t)
Create an empty instance to initialize later.
Definition onnxruntime_cxx_api.h:2771
Wrapper around OrtKeyValuePairs.
Definition onnxruntime_cxx_api.h:947
KeyValuePairs()
Wraps OrtApi::CreateKeyValuePairs.
void Add(const char *key, const char *value)
Wraps OrtApi::AddKeyValuePair.
KeyValuePairs(const std::unordered_map< std::string, std::string > &kv_pairs)
Wraps OrtApi::CreateKeyValuePairs and OrtApi::AddKeyValuePair.
void Remove(const char *key)
Wraps OrtApi::RemoveKeyValuePair.
KeyValuePairs(std::nullptr_t)
Definition onnxruntime_cxx_api.h:948
ConstKeyValuePairs GetConst() const
Definition onnxruntime_cxx_api.h:964
KeyValuePairs(OrtKeyValuePairs *p)
Take ownership of a pointer created by C API.
Definition onnxruntime_cxx_api.h:950
This class represents an ONNX Runtime logger that can be used to log information with an associated s...
Definition onnxruntime_cxx_api.h:2610
Logger(Logger &&v) noexcept=default
Logger & operator=(Logger &&v) noexcept=default
Logger & operator=(const Logger &)=default
~Logger()=default
Logger(const Logger &)=default
Logger()=default
Logger(std::nullptr_t)
Definition onnxruntime_cxx_api.h:2619
Logger(const OrtLogger *logger)
OrtLoggingLevel GetLoggingSeverityLevel() const noexcept
LoraAdapter holds a set of Lora Parameters loaded from a single file.
Definition onnxruntime_cxx_api.h:1209
static LoraAdapter CreateLoraAdapter(const std::basic_string< char > &adapter_path, OrtAllocator *allocator)
Wraps OrtApi::CreateLoraAdapter.
LoraAdapter(std::nullptr_t)
Definition onnxruntime_cxx_api.h:1213
static LoraAdapter CreateLoraAdapterFromArray(const void *bytes, size_t num_bytes, OrtAllocator *allocator)
Wraps OrtApi::CreateLoraAdapterFromArray.
Wrapper around OrtMapTypeInfo.
Definition onnxruntime_cxx_api.h:1853
ConstMapTypeInfo GetConst() const
Definition onnxruntime_cxx_api.h:1859
MapTypeInfo(OrtMapTypeInfo *p)
Used for interop with the C API.
Definition onnxruntime_cxx_api.h:1858
MapTypeInfo(std::nullptr_t)
Create an empty MapTypeInfo object, must be assigned a valid one to be used.
Definition onnxruntime_cxx_api.h:1857
Represents native memory allocation coming from one of the OrtAllocators registered with OnnxRuntime....
Definition onnxruntime_cxx_api.h:1008
MemoryAllocation(MemoryAllocation &&) noexcept
MemoryAllocation & operator=(const MemoryAllocation &)=delete
MemoryAllocation(const MemoryAllocation &)=delete
MemoryAllocation(OrtAllocator *allocator, void *p, size_t size)
size_t size() const
Definition onnxruntime_cxx_api.h:1017
Wrapper around OrtMemoryInfo.
Definition onnxruntime_cxx_api.h:992
MemoryInfo(const char *name, OrtAllocatorType type, int id, OrtMemType mem_type)
MemoryInfo(std::nullptr_t)
No instance is created.
Definition onnxruntime_cxx_api.h:994
MemoryInfo(OrtMemoryInfo *p)
Take ownership of a pointer created by C API.
Definition onnxruntime_cxx_api.h:995
static MemoryInfo CreateCpu(OrtAllocatorType type, OrtMemType mem_type1)
ConstMemoryInfo GetConst() const
Definition onnxruntime_cxx_api.h:999
MemoryInfo(const char *name, OrtMemoryInfoDeviceType device_type, uint32_t vendor_id, uint32_t device_id, OrtDeviceMemoryType mem_type, size_t alignment, OrtAllocatorType allocator_type)
Wrapper around CreateMemoryInfo_V2.
Options object used when compiling a model.
Definition onnxruntime_cxx_api.h:1451
ModelCompilationOptions & SetOutputModelWriteFunc(OrtWriteBufferFunc write_func, void *state)
ModelCompilationOptions & SetEpContextEmbedMode(bool embed_ep_context_in_model)
Wraps OrtApi::ModelCompilationOptions_SetEpContextEmbedMode.
ModelCompilationOptions & SetInputModelFromBuffer(const void *input_model_data, size_t input_model_data_size)
Wraps OrtApi::ModelCompilationOptions_SetInputModelFromBuffer.
ModelCompilationOptions & SetOutputModelBuffer(OrtAllocator *allocator, void **output_model_buffer_ptr, size_t *output_model_buffer_size_ptr)
Wraps OrtApi::ModelCompilationOptions_SetOutputModelBuffer.
ModelCompilationOptions & SetFlags(uint32_t flags)
Wraps OrtApi::ModelCompilationOptions_SetFlags.
ModelCompilationOptions & SetOutputModelExternalInitializersFile(const char *file_path, size_t initializer_size_threshold)
Wraps OrtApi::ModelCompilationOptions_SetOutputModelExternalInitializersFile.
ModelCompilationOptions(std::nullptr_t)
Create an empty ModelCompilationOptions object, must be assigned a valid one to be used.
Definition onnxruntime_cxx_api.h:1455
ModelCompilationOptions(const Env &env, ConstSessionOptions session_options)
Wraps OrtApi::CreateModelCompilationOptionsFromSessionOptions.
ModelCompilationOptions & SetOutputModelPath(const char *output_model_path)
Wraps OrtApi::ModelCompilationOptions_SetOutputModelPath.
ModelCompilationOptions & SetInputModelPath(const char *input_model_path)
Wraps OrtApi::ModelCompilationOptions_SetInputModelPath.
ModelCompilationOptions & SetOutputModelGetInitializerLocationFunc(OrtGetInitializerLocationFunc get_initializer_location_func, void *state)
ModelCompilationOptions & SetEpContextBinaryInformation(const char *output_directory, const char *model_name)
Wraps OrtApi::ModelCompilationOptions_SetEpContextBinaryInformation.
ModelCompilationOptions & SetGraphOptimizationLevel(GraphOptimizationLevel graph_optimization_level)
Wraps OrtApi::ModelCompilationOptions_SetGraphOptimizationLevel.
ModelCompilationOptions(const Env &env, const SessionOptions &session_options)
Wraps OrtApi::CreateModelCompilationOptionsFromSessionOptions.
Wrapper around OrtModel.
Definition onnxruntime_cxx_api.h:3261
Model(const std::vector< DomainOpsetPair > &opsets)
Model(OrtModel *p)
Take ownership of a pointer created by C API.
Definition onnxruntime_cxx_api.h:3265
std::pair< std::string, int > DomainOpsetPair
Definition onnxruntime_cxx_api.h:3262
Model(std::nullptr_t)
No instance is created.
Definition onnxruntime_cxx_api.h:3264
Wrapper around OrtModelMetadata.
Definition onnxruntime_cxx_api.h:1497
AllocatedStringPtr GetDescriptionAllocated(OrtAllocator *allocator) const
Returns a copy of the description.
std::vector< AllocatedStringPtr > GetCustomMetadataMapKeysAllocated(OrtAllocator *allocator) const
Returns a vector of copies of the custom metadata keys.
ModelMetadata(std::nullptr_t)
Create an empty ModelMetadata object, must be assigned a valid one to be used.
Definition onnxruntime_cxx_api.h:1501
AllocatedStringPtr GetGraphDescriptionAllocated(OrtAllocator *allocator) const
Returns a copy of the graph description.
AllocatedStringPtr GetProducerNameAllocated(OrtAllocator *allocator) const
Returns a copy of the producer name.
AllocatedStringPtr GetGraphNameAllocated(OrtAllocator *allocator) const
Returns a copy of the graph name.
AllocatedStringPtr LookupCustomMetadataMapAllocated(const char *key, OrtAllocator *allocator) const
Looks up a value by a key in the Custom Metadata map.
AllocatedStringPtr GetDomainAllocated(OrtAllocator *allocator) const
Returns a copy of the domain name.
int64_t GetVersion() const
Wraps OrtApi::ModelMetadataGetVersion.
Wrapper around OrtNode.
Definition onnxruntime_cxx_api.h:3128
Node(const std::string &operator_name, const std::string &operator_domain, const std::string &node_name, const std::vector< std::string > &input_names, const std::vector< std::string > &output_names)
Node()=default
Node(std::nullptr_t)
No instance is created.
Definition onnxruntime_cxx_api.h:3130
Node(const std::string &operator_name, const std::string &operator_domain, const std::string &node_name, const std::vector< std::string > &input_names, const std::vector< std::string > &output_names, std::vector< OpAttr > &attributes)
Wraps CreateNode. Node takes ownership of attributes on success and updates the OpAttr in attributes ...
Node(OrtNode *p)
Take ownership of a pointer created by C API.
Definition onnxruntime_cxx_api.h:3131
This struct provides life time management for custom op attribute.
Definition onnxruntime_cxx_api.h:2519
OpAttr(const char *name, const void *data, int len, OrtOpAttrType type)
OpAttr()=default
OpAttr(std::nullptr_t)
Definition onnxruntime_cxx_api.h:2524
ConstOpAttr GetConst() const
Definition onnxruntime_cxx_api.h:2527
Create and own custom defined operation.
Definition onnxruntime_cxx_api.h:2779
Op(OrtOp *)
Take ownership of the OrtOp.
static Op Create(const OrtKernelInfo *info, const char *op_name, const char *domain, int version, const char **type_constraint_names, const ONNXTensorElementDataType *type_constraint_values, size_t type_constraint_count, const OpAttr *attr_values, size_t attr_count, size_t input_count, size_t output_count)
Op(std::nullptr_t)
Create an empty Operator object, must be assigned a valid one to be used.
Definition onnxruntime_cxx_api.h:2783
void Invoke(const OrtKernelContext *context, const OrtValue *const *input_values, size_t input_count, OrtValue *const *output_values, size_t output_count)
void Invoke(const OrtKernelContext *context, const Value *input_values, size_t input_count, Value *output_values, size_t output_count)
Definition onnxruntime_cxx_api.h:3168
std::string domain
Definition onnxruntime_cxx_api.h:3169
int64_t version
Definition onnxruntime_cxx_api.h:3170
The PrepackedWeightsContainer.
Definition onnxruntime_cxx_api.h:878
PrepackedWeightsContainer()
Wraps OrtApi::CreatePrepackedWeightsContainer.
PrepackedWeightsContainer(OrtPrepackedWeightsContainer *p)
Definition onnxruntime_cxx_api.h:883
PrepackedWeightsContainer(std::nullptr_t)
Take ownership of a pointer created by C API.
Definition onnxruntime_cxx_api.h:881
RunOptions.
Definition onnxruntime_cxx_api.h:1237
int GetRunLogSeverityLevel() const
Wraps OrtApi::RunOptionsGetRunLogSeverityLevel.
RunOptions & SetTerminate()
Terminates all currently executing Session::Run calls that were made using this RunOptions instance.
RunOptions & SetRunTag(const char *run_tag)
wraps OrtApi::RunOptionsSetRunTag
RunOptions & AddActiveLoraAdapter(const LoraAdapter &adapter)
Add the LoraAdapter to the list of active adapters. The setting does not affect RunWithBinding() call...
RunOptions & UnsetTerminate()
Clears the terminate flag so this RunOptions instance can be used in a new Session::Run call without ...
int GetRunLogVerbosityLevel() const
Wraps OrtApi::RunOptionsGetRunLogVerbosityLevel.
RunOptions(std::nullptr_t)
Create an empty RunOptions object, must be assigned a valid one to be used.
Definition onnxruntime_cxx_api.h:1238
RunOptions & SetRunLogVerbosityLevel(int)
Wraps OrtApi::RunOptionsSetRunLogVerbosityLevel.
RunOptions & SetRunLogSeverityLevel(int)
Wraps OrtApi::RunOptionsSetRunLogSeverityLevel.
RunOptions & AddConfigEntry(const char *config_key, const char *config_value)
Wraps OrtApi::AddRunConfigEntry.
const char * GetRunTag() const
Wraps OrtApi::RunOptionsGetRunTag.
RunOptions()
Wraps OrtApi::CreateRunOptions.
const char * GetConfigEntry(const char *config_key)
Wraps OrtApi::GetRunConfigEntry.
Wrapper around OrtSequenceTypeInfo.
Definition onnxruntime_cxx_api.h:1815
SequenceTypeInfo(std::nullptr_t)
Create an empty SequenceTypeInfo object, must be assigned a valid one to be used.
Definition onnxruntime_cxx_api.h:1819
ConstSequenceTypeInfo GetConst() const
Definition onnxruntime_cxx_api.h:1821
SequenceTypeInfo(OrtSequenceTypeInfo *p)
Used for interop with the C API.
Definition onnxruntime_cxx_api.h:1820
Wrapper around OrtSession.
Definition onnxruntime_cxx_api.h:1717
Session(std::nullptr_t)
Create an empty Session object, must be assigned a valid one to be used. Wraps OrtApi::CreateSession.
Definition onnxruntime_cxx_api.h:1719
static Session CreateModelEditorSession(const Env &env, const void *model_data, size_t model_data_length, const SessionOptions &options)
Wraps OrtModelEditorApi::CreateModelEditorSession.
UnownedSession GetUnowned() const
Definition onnxruntime_cxx_api.h:1748
Session(const Env &env, const char *model_path, const SessionOptions &options, OrtPrepackedWeightsContainer *prepacked_weights_container)
Wraps OrtApi::CreateSessionWithPrepackedWeightsContainer.
Session(const Env &env, const void *model_data, size_t model_data_length, const SessionOptions &options, OrtPrepackedWeightsContainer *prepacked_weights_container)
Wraps OrtApi::CreateSessionFromArrayWithPrepackedWeightsContainer.
Session(const Env &env, const Model &model, const SessionOptions &options)
Wraps OrtModelEditorApi::CreateSessionFromModel.
Session(OrtSession *p)
C API Interop.
Definition onnxruntime_cxx_api.h:1720
static Session CreateModelEditorSession(const Env &env, const char *model_path, const SessionOptions &options)
Wraps OrtModelEditorApi::CreateModelEditorSession.
Session(const Env &env, const char *model_path, const SessionOptions &options)
ConstSession GetConst() const
Definition onnxruntime_cxx_api.h:1747
Session(const Env &env, const void *model_data, size_t model_data_length, const SessionOptions &options)
Wraps OrtApi::CreateSessionFromArray.
Wrapper around OrtSessionOptions.
Definition onnxruntime_cxx_api.h:1439
SessionOptions(std::nullptr_t)
Create an empty SessionOptions object, must be assigned a valid one to be used.
Definition onnxruntime_cxx_api.h:1440
UnownedSessionOptions GetUnowned() const
Definition onnxruntime_cxx_api.h:1443
SessionOptions()
Wraps OrtApi::CreateSessionOptions.
ConstSessionOptions GetConst() const
Definition onnxruntime_cxx_api.h:1444
SessionOptions(OrtSessionOptions *p)
Used for interop with the C API.
Definition onnxruntime_cxx_api.h:1442
Definition onnxruntime_cxx_api.h:2813
SymbolicInteger & operator=(const SymbolicInteger &)=default
SymbolicInteger(const SymbolicInteger &)=default
int64_t AsInt() const
Definition onnxruntime_cxx_api.h:2834
int64_t i_
Definition onnxruntime_cxx_api.h:2841
const char * s_
Definition onnxruntime_cxx_api.h:2842
bool operator==(const SymbolicInteger &dim) const
Definition onnxruntime_cxx_api.h:2822
SymbolicInteger & operator=(SymbolicInteger &&)=default
SymbolicInteger(SymbolicInteger &&)=default
const char * AsSym() const
Definition onnxruntime_cxx_api.h:2835
SymbolicInteger(int64_t i)
Definition onnxruntime_cxx_api.h:2814
SymbolicInteger(const char *s)
Definition onnxruntime_cxx_api.h:2815
bool IsInt() const
Definition onnxruntime_cxx_api.h:2833
Provide access to per-node attributes and input shapes, so one could compute and set output shapes.
Definition onnxruntime_cxx_api.h:2812
Ints GetAttrInts(const char *attr_name)
Strings GetAttrStrings(const char *attr_name)
Status SetOutputShape(size_t indice, const Shape &shape, ONNXTensorElementDataType type=ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT)
std::vector< SymbolicInteger > Shape
Definition onnxruntime_cxx_api.h:2847
std::vector< float > Floats
Definition onnxruntime_cxx_api.h:2864
std::string GetAttrString(const char *attr_name)
std::vector< int64_t > Ints
Definition onnxruntime_cxx_api.h:2859
ShapeInferContext(const OrtApi *ort_api, OrtShapeInferContext *ctx)
int64_t GetAttrInt(const char *attr_name)
size_t GetInputCount() const
Definition onnxruntime_cxx_api.h:2853
std::vector< std::string > Strings
Definition onnxruntime_cxx_api.h:2869
Floats GetAttrFloats(const char *attr_name)
const Shape & GetInputShape(size_t indice) const
Definition onnxruntime_cxx_api.h:2851
float GetAttrFloat(const char *attr_name)
The Status that holds ownership of OrtStatus received from C API Use it to safely destroy OrtStatus* ...
Definition onnxruntime_cxx_api.h:794
OrtErrorCode GetErrorCode() const
Status(const Exception &)
Creates status instance out of exception.
bool IsOK() const noexcept
Returns true if instance represents an OK (non-error) status.
Status(OrtStatus *status) noexcept
Takes ownership of OrtStatus instance returned from the C API.
std::string GetErrorMessage() const
Status()=default
Status(const std::exception &)
Creates status instance out of exception.
Status(const char *message, OrtErrorCode code)
Creates status instance out of null-terminated string message.
Status(std::nullptr_t) noexcept
Create an empty object, must be assigned a valid one to be used.
Definition onnxruntime_cxx_api.h:796
Wrapper around OrtSyncStream.
Definition onnxruntime_cxx_api.h:1066
SyncStream(OrtSyncStream *p)
Take ownership of a pointer created by C API.
Definition onnxruntime_cxx_api.h:1068
SyncStream(std::nullptr_t)
Create an empty SyncStream object, must be assigned a valid one to be used.
Definition onnxruntime_cxx_api.h:1067
void * GetHandle() const
Wraps SyncStream_GetHandle.
The TensorRTOptions (V2)
Definition onnxruntime_cxx_api.h:840
void Update(const std::unordered_map< std::string, std::string > &options)
Wrapper around OrtApi::UpdateTensorRTProviderOptions.
void UpdateWithValue(const char *key, void *value)
Wrapper around OrtApi::GetTensorRTProviderOptionsByName.
std::string GetTensorRTProviderOptionsAsString() const
void * GetOptionByName(const char *name) const
Wrapper around OrtApi::GetTensorRTProviderOptionsAsString.
TensorRTProviderOptions(std::nullptr_t)
Definition onnxruntime_cxx_api.h:841
TensorRTProviderOptions()
Wraps OrtApi::CreateTensorRTProviderOptionsV2.
Wrapper around OrtTensorTypeAndShapeInfo.
Definition onnxruntime_cxx_api.h:1781
TensorTypeAndShapeInfo(std::nullptr_t)
Create an empty TensorTypeAndShapeInfo object, must be assigned a valid one to be used.
Definition onnxruntime_cxx_api.h:1786
ConstTensorTypeAndShapeInfo GetConst() const
Definition onnxruntime_cxx_api.h:1797
TensorTypeAndShapeInfo(OrtTensorTypeAndShapeInfo *p)
Used for interop with the C API.
Definition onnxruntime_cxx_api.h:1788
TensorTypeAndShapeInfo(ONNXTensorElementDataType element_type, const std::vector< int64_t > &dims, const std::vector< std::string > *symbolic_dims=nullptr)
The ThreadingOptions.
Definition onnxruntime_cxx_api.h:810
ThreadingOptions & SetGlobalCustomThreadCreationOptions(void *ort_custom_thread_creation_options)
Wraps OrtApi::SetGlobalCustomThreadCreationOptions.
ThreadingOptions()
Wraps OrtApi::CreateThreadingOptions.
ThreadingOptions & SetGlobalInterOpNumThreads(int inter_op_num_threads)
Wraps OrtApi::SetGlobalInterOpNumThreads.
ThreadingOptions & SetGlobalCustomCreateThreadFn(OrtCustomCreateThreadFn ort_custom_create_thread_fn)
Wraps OrtApi::SetGlobalCustomCreateThreadFn.
ThreadingOptions & SetGlobalCustomJoinThreadFn(OrtCustomJoinThreadFn ort_custom_join_thread_fn)
Wraps OrtApi::SetGlobalCustomJoinThreadFn.
ThreadingOptions & SetGlobalSpinControl(int allow_spinning)
Wraps OrtApi::SetGlobalSpinControl.
ThreadingOptions & SetGlobalDenormalAsZero()
Wraps OrtApi::SetGlobalDenormalAsZero.
ThreadingOptions & SetGlobalIntraOpNumThreads(int intra_op_num_threads)
Wraps OrtApi::SetGlobalIntraOpNumThreads.
Type information that may contain either TensorTypeAndShapeInfo or the information about contained se...
Definition onnxruntime_cxx_api.h:1887
static TypeInfo CreateOptionalTypeInfo(ConstTypeInfo contained_type)
static TypeInfo CreateSequenceTypeInfo(ConstTypeInfo sequence_type)
static TypeInfo CreateTensorInfo(ConstTensorTypeAndShapeInfo tensor_info)
static TypeInfo CreateSparseTensorInfo(ConstTensorTypeAndShapeInfo sparse_tensor_info)
TypeInfo(std::nullptr_t)
Create an empty TypeInfo object, must be assigned a valid one to be used.
Definition onnxruntime_cxx_api.h:1892
static TypeInfo CreateMapTypeInfo(ONNXTensorElementDataType key_type, ConstTypeInfo value_type)
ConstTypeInfo GetConst() const
Definition onnxruntime_cxx_api.h:1903
TypeInfo(OrtTypeInfo *p)
C API Interop.
Definition onnxruntime_cxx_api.h:1893
Wrapper around OrtValue.
Definition onnxruntime_cxx_api.h:2243
static Value CreateSparseTensor(const OrtMemoryInfo *info, void *p_data, const Shape &dense_shape, const Shape &values_shape, ONNXTensorElementDataType type)
Creates an OrtValue instance containing SparseTensor. This constructs a sparse tensor that makes use ...
static Value CreateSparseTensor(const OrtMemoryInfo *info, T *p_data, const Shape &dense_shape, const Shape &values_shape)
This is a simple forwarding method to the other overload that helps deducing data type enum value fro...
Value & operator=(Value &&)=default
static Value CreateSparseTensor(OrtAllocator *allocator, const Shape &dense_shape, ONNXTensorElementDataType type)
Creates an instance of OrtValue containing sparse tensor. The created instance has no data....
Value(Value &&)=default
Value(std::nullptr_t)
Create an empty Value object, must be assigned a valid one to be used.
Definition onnxruntime_cxx_api.h:2249
static Value CreateTensor(const OrtMemoryInfo *info, T *p_data, size_t p_data_element_count, const int64_t *shape, size_t shape_len)
Creates a tensor with a user supplied buffer. Wraps OrtApi::CreateTensorWithDataAsOrtValue.
static Value CreateSparseTensor(OrtAllocator *allocator, const Shape &dense_shape)
This is a simple forwarding method to the below CreateSparseTensor. This helps to specify data type e...
static Value CreateTensor(OrtAllocator *allocator, const int64_t *shape, size_t shape_len, ONNXTensorElementDataType type)
Creates an OrtValue with a tensor using the supplied OrtAllocator. Wraps OrtApi::CreateTensorAsOrtVal...
UnownedValue GetUnowned() const
Definition onnxruntime_cxx_api.h:2254
static Value CreateSequence(const std::vector< Value > &values)
Creates an OrtValue with a Sequence Onnx type representation. The API would ref-count the supplied Or...
static Value CreateMap(const Value &keys, const Value &values)
Creates an OrtValue with a Map Onnx type representation. The API would ref-count the supplied OrtValu...
static Value CreateTensor(const OrtMemoryInfo *info, void *p_data, size_t p_data_byte_count, const int64_t *shape, size_t shape_len, ONNXTensorElementDataType type)
Creates a tensor with a user supplied buffer. Wraps OrtApi::CreateTensorWithDataAsOrtValue.
static Value CreateTensor(OrtAllocator *allocator, const int64_t *shape, size_t shape_len)
Creates an OrtValue with a tensor using a supplied OrtAllocator. Wraps OrtApi::CreateTensorAsOrtValue...
static Value CreateOpaque(const char *domain, const char *type_name, const T &value)
Creates an OrtValue wrapping an Opaque type. This is used for experimental support of non-tensor type...
static Value CreateTensor(OrtAllocator *deleter, void *p_data, size_t p_data_byte_count, const int64_t *shape, size_t shape_len, ONNXTensorElementDataType type)
Creates a tensor with a user supplied buffer. Wraps OrtApi::CreateTensorWithDataAndDeleterAsOrtValue.
ConstValue GetConst() const
Definition onnxruntime_cxx_api.h:2253
Definition onnxruntime_cxx_api.h:3160
int64_t index
Definition onnxruntime_cxx_api.h:3164
ConstNode node
Definition onnxruntime_cxx_api.h:3161
Wrapper around OrtValueInfo.
Definition onnxruntime_cxx_api.h:3065
ConstValueInfo GetConst() const
Definition onnxruntime_cxx_api.h:3075
ValueInfo(std::nullptr_t)
Definition onnxruntime_cxx_api.h:3067
ValueInfo(const std::string &name, const ConstTypeInfo &type_info)
ValueInfo(OrtValueInfo *p)
Take ownership of a pointer created by C API.
Definition onnxruntime_cxx_api.h:3069
ValueInfo()=default
Definition onnxruntime_cxx_api.h:756
AllocatedFree(OrtAllocator *allocator)
Definition onnxruntime_cxx_api.h:758
OrtAllocator * allocator_
Definition onnxruntime_cxx_api.h:757
void operator()(void *ptr) const
Definition onnxruntime_cxx_api.h:760
Base & operator=(Base &&v) noexcept
Definition onnxruntime_cxx_api.h:742
constexpr contained_type & operator*() const noexcept
Definition onnxruntime_cxx_api.h:749
typename Unowned< T >::Type contained_type
Definition onnxruntime_cxx_api.h:731
Base(Base &&v) noexcept
Definition onnxruntime_cxx_api.h:741
Base(const Base &)=default
constexpr Base(contained_type *p) noexcept
Definition onnxruntime_cxx_api.h:734
Base & operator=(const Base &)=default
Used internally by the C++ API. C++ wrapper types inherit from this. This is a zero cost abstraction ...
Definition onnxruntime_cxx_api.h:684
Base(Base &&v) noexcept
Definition onnxruntime_cxx_api.h:696
constexpr Base()=default
constexpr contained_type & operator*() const noexcept
Definition onnxruntime_cxx_api.h:704
contained_type * release()
Relinquishes ownership of the contained C object pointer The underlying object is not destroyed.
Definition onnxruntime_cxx_api.h:708
Base(const Base &)=delete
constexpr Base(contained_type *p) noexcept
Definition onnxruntime_cxx_api.h:688
Base & operator=(const Base &)=delete
Base & operator=(Base &&v) noexcept
Definition onnxruntime_cxx_api.h:697
contained_type * p_
Definition onnxruntime_cxx_api.h:715
~Base()
Definition onnxruntime_cxx_api.h:689
T contained_type
Definition onnxruntime_cxx_api.h:685
Definition onnxruntime_cxx_api.h:890
const std::basic_string< char > GetFilePath() const
Definition onnxruntime_cxx_api.h:3175
std::vector< ConstNode > GetNodes() const
std::vector< ConstValueInfo > GetInputs() const
ConstNode GetParentNode() const
int64_t GetOnnxIRVersion() const
std::basic_string< char > GetModelPath() const
Graph GetGraphView(const std::vector< ConstNode > &nodes) const
ModelMetadata GetModelMetadata() const
Wraps OrtApi::Graph_GetModelMetadata.
std::vector< ConstValueInfo > GetInitializers() const
std::string GetName() const
std::vector< ConstValueInfo > GetOutputs() const
std::vector< OperatorSet > GetOperatorSets() const
Definition onnxruntime_cxx_api.h:2422
std::vector< Value > GetOutputValues(OrtAllocator *) const
std::vector< std::string > GetOutputNames(OrtAllocator *) const
std::vector< Value > GetOutputValues() const
std::vector< std::string > GetOutputNames() const
Definition onnxruntime_cxx_api.h:3087
std::vector< ConstValueInfo > GetOutputs() const
std::vector< ConstValueInfo > GetImplicitInputs() const
std::string GetName() const
std::string GetDomain() const
std::vector< AttrNameSubgraph > GetSubgraphs() const
ConstGraphImpl< detail::Unowned< const OrtGraph > > GetGraph() const
std::string GetOperatorType() const
std::vector< ConstOpAttr > GetAttributes() const
std::vector< ConstValueInfo > GetInputs() const
Status GetAttributeByName(const std::string &name, ConstOpAttr &attr) const
std::string GetEpName() const
Definition onnxruntime_cxx_api.h:2491
std::string GetName() const
Status GetValue(R &out) const
Status GetTensorAttributeAsOrtValue(Value &) const
Status GetValueArray(std::vector< R > &out) const
OrtOpAttrType GetType() const
Definition onnxruntime_cxx_api.h:1572
std::vector< std::string > GetOutputNames() const
TypeInfo GetInputTypeInfo(size_t index) const
Wraps OrtApi::SessionGetInputTypeInfo.
size_t GetOutputCount() const
Returns the number of model outputs.
std::vector< ValueInfo > GetOutputs() const
int GetOpset(const std::string &domain) const
Wraps OrtApi::SessionGetOpsetForDomain.
uint64_t GetProfilingStartTimeNs() const
Wraps OrtApi::SessionGetProfilingStartTimeNs.
std::vector< std::string > GetOverridableInitializerNames() const
ModelMetadata GetModelMetadata() const
Wraps OrtApi::SessionGetModelMetadata.
size_t GetInputCount() const
Returns the number of model inputs.
TypeInfo GetOutputTypeInfo(size_t index) const
Wraps OrtApi::SessionGetOutputTypeInfo.
std::vector< std::string > GetInputNames() const
AllocatedStringPtr GetOverridableInitializerNameAllocated(size_t index, OrtAllocator *allocator) const
Returns a copy of the overridable initializer name at then specified index.
std::vector< ConstEpDevice > GetEpDeviceForInputs() const
Wrapper for OrtApi::SessionGetEpDeviceForInputs.
AllocatedStringPtr GetOutputNameAllocated(size_t index, OrtAllocator *allocator) const
Returns a copy of output name at then specified index.
size_t GetOverridableInitializerCount() const
Returns the number of inputs that have defaults that can be overridden.
std::vector< ConstMemoryInfo > GetMemoryInfoForOutputs() const
Wrapper for OrtApi::SessionGetMemoryInfoForOutputs.
AllocatedStringPtr GetInputNameAllocated(size_t index, OrtAllocator *allocator) const
Returns a copy of input name at the specified index.
std::vector< ConstMemoryInfo > GetMemoryInfoForInputs() const
Wrapper for OrtApi::SessionGetMemoryInfoForInputs.
std::vector< ValueInfo > GetInputs() const
TypeInfo GetOverridableInitializerTypeInfo(size_t index) const
Wraps OrtApi::SessionGetOverridableInitializerTypeInfo.
Definition onnxruntime_cxx_api.h:1932
void GetStringTensorContent(void *buffer, size_t buffer_length, size_t *offsets, size_t offsets_count) const
The API copies all of the UTF-8 encoded string data contained within a tensor or a sparse tensor into...
void GetStringTensorElement(size_t buffer_length, size_t element_index, void *buffer) const
The API copies UTF-8 encoded bytes for the requested string element contained within a tensor or a sp...
TensorTypeAndShapeInfo GetSparseTensorIndicesTypeShapeInfo(OrtSparseIndicesFormat format) const
The API returns type and shape information for the specified indices. Each supported indices have the...
const void * GetTensorRawData() const
Returns a non-typed pointer to a tensor contained data.
std::string GetStringTensorElement(size_t element_index) const
Returns string tensor UTF-8 encoded string element. Use of this API is recommended over GetStringTens...
size_t GetStringTensorElementLength(size_t element_index) const
The API returns a byte length of UTF-8 encoded string element contained in either a tensor or a spare...
size_t GetStringTensorDataLength() const
This API returns a full length of string data contained within either a tensor or a sparse Tensor....
bool IsSparseTensor() const
Returns true if the OrtValue contains a sparse tensor.
TypeInfo GetTypeInfo() const
The API returns type information for data contained in a tensor. For sparse tensors it returns type i...
const R * GetSparseTensorIndicesData(OrtSparseIndicesFormat indices_format, size_t &num_indices) const
The API retrieves a pointer to the internal indices buffer. The API merely performs a convenience dat...
bool IsTensor() const
Returns true if Value is a tensor, false for other types like map/sequence/etc.
ConstMemoryInfo GetTensorMemoryInfo() const
This API returns information about the memory allocation used to hold data.
size_t GetTensorSizeInBytes() const
Returns the total size of the tensor data in bytes. Throws an exception if the OrtValue does not cont...
const R * GetSparseTensorValues() const
The API returns a pointer to an internal buffer of the sparse tensor containing non-zero values....
TensorTypeAndShapeInfo GetTensorTypeAndShapeInfo() const
The API returns type information for data contained in a tensor. For sparse tensors it returns type i...
Value GetValue(int index, OrtAllocator *allocator) const
size_t GetCount() const
< Return true if OrtValue contains data and returns false if the OrtValue is a None
void GetOpaqueData(const char *domain, const char *type_name, R &) const
Obtains a pointer to a user defined data for experimental purposes.
TensorTypeAndShapeInfo GetSparseTensorValuesTypeAndShapeInfo() const
The API returns type and shape information for stored non-zero values of the sparse tensor....
const R * GetTensorData() const
Returns a const typed pointer to the tensor contained data. No type checking is performed,...
OrtSparseFormat GetSparseFormat() const
The API returns the sparse data format this OrtValue holds in a sparse tensor. If the sparse tensor w...
Definition onnxruntime_cxx_api.h:3030
Status GetInitializer(ConstValue &value) const
< A wrapper around OrtApi::ValueInfo_GetInitializerValue
std::string GetName() const
< A wrapper around OrtApi::GetValueInfoName
bool IsFromOuterScope() const
< A wrapper around OrtApi::ValueInfo_IsFromOuterScope
Status GetExternalInitializerInfo(ExternalInitializerInfo &info) const
< A wrapper around OrtApi::ValueInfo_GetExternalInitializerInfo
bool IsConstantInitializer() const
< A wrapper around OrtApi::ValueInfo_IsConstantInitializer
std::vector< ValueInfoConsumerProducerInfo > GetConsumers() const
< A wrapper around OrtApi::ValueInfo_GetValueConsumers
bool IsGraphOutput() const
< A wrapper around OrtApi::ValueInfo_IsGraphOutput
bool IsRequiredGraphInput() const
< A wrapper around OrtApi::ValueInfo_IsRequiredGraphInput
ConstTypeInfo TypeInfo() const
< A wrapper around OrtApi::GetValueInfoTypeInfo
ValueInfoConsumerProducerInfo GetProducerNode() const
bool IsOptionalGraphInput() const
< A wrapper around OrtApi::ValueInfo_IsOptionalGraphInput
Definition onnxruntime_cxx_api.h:1093
const char * EpName() const
const char * EpVendor() const
ConstKeyValuePairs EpOptions() const
ConstHardwareDevice Device() const
ConstMemoryInfo GetMemoryInfo(OrtDeviceMemoryType memory_type) const
Wraps EpDevice_MemoryInfo.
SyncStream CreateSyncStream(ConstKeyValuePairs stream_options={}) const
ConstKeyValuePairs EpMetadata() const
Definition onnxruntime_cxx_api.h:3204
void SetInputs(std::vector< ValueInfo > &inputs)
void SetOutputs(std::vector< ValueInfo > &outputs)
void AddNode(Node &node)
void AddInitializer(const std::string &name, Value &initializer, bool data_is_external)
Definition onnxruntime_cxx_api.h:1074
OrtHardwareDeviceType Type() const
const char * Vendor() const
ConstKeyValuePairs Metadata() const
Definition onnxruntime_cxx_api.h:2433
void BindOutput(const char *name, const Value &)
void BindInput(const char *name, const Value &)
void BindOutput(const char *name, const OrtMemoryInfo *)
Definition onnxruntime_cxx_api.h:930
void GetKeyValuePairs(std::vector< const char * > &keys, std::vector< const char * > &values) const
std::unordered_map< std::string, std::string > GetKeyValuePairs() const
const char * GetValue(const char *key) const
Definition onnxruntime_cxx_api.h:1839
ONNXTensorElementDataType GetMapKeyType() const
Wraps OrtApi::GetMapKeyType.
TypeInfo GetMapValueType() const
Wraps OrtApi::GetMapValueType.
Definition onnxruntime_cxx_api.h:969
std::string GetAllocatorName() const
Wrapper MemoryInfoGetName.
int GetDeviceId() const
Wrapper MemoryInfoGetId.
OrtMemType GetMemoryType() const
Wrapper MemoryInfoGetMemType.
OrtDeviceMemoryType GetDeviceMemoryType() const
Wrapper MemoryInfoGetDeviceMemType.
OrtMemoryInfoDeviceType GetDeviceType() const
Wrapper MemoryInfoGetDeviceType.
OrtAllocatorType GetAllocatorType() const
Wrapper MemoryInfoGetType.
uint32_t GetVendorId() const
Wrapper MemoryInfoGetVendorId.
bool operator==(const MemoryInfoImpl< U > &o) const
Definition onnxruntime_cxx_api.h:3244
void AddGraph(Graph &graph)
Definition onnxruntime_cxx_api.h:1826
TypeInfo GetOptionalElementType() const
Wraps OrtApi::CastOptionalTypeToContainedTypeInfo.
Definition onnxruntime_cxx_api.h:1915
const char ** str
Definition onnxruntime_cxx_api.h:1920
const int64_t * values_shape
Definition onnxruntime_cxx_api.h:1916
size_t values_shape_len
Definition onnxruntime_cxx_api.h:1917
const void * p_data
Definition onnxruntime_cxx_api.h:1919
Definition onnxruntime_cxx_api.h:1802
TypeInfo GetSequenceElementType() const
Wraps OrtApi::GetSequenceElementType.
Definition onnxruntime_cxx_api.h:1630
void SetEpDynamicOptions(const char *const *keys, const char *const *values, size_t kv_len)
Set DynamicOptions for EPs (Execution Providers)
AllocatedStringPtr EndProfilingAllocated(OrtAllocator *allocator)
End profiling and return a copy of the profiling file name.
void FinalizeModelEditorSession(const Model &model, const SessionOptions &options, OrtPrepackedWeightsContainer *prepacked_weights_container=nullptr)
void Run(const RunOptions &run_options, const IoBinding &)
Wraps OrtApi::RunWithBinding.
void RunAsync(const RunOptions &run_options, const char *const *input_names, const Value *input_values, size_t input_count, const char *const *output_names, Value *output_values, size_t output_count, RunAsyncCallbackFn callback, void *user_data)
Run the model asynchronously in a thread owned by intra op thread pool.
std::vector< Value > Run(const RunOptions &run_options, const char *const *input_names, const Value *input_values, size_t input_count, const char *const *output_names, size_t output_count)
Run the model returning results in an Ort allocated vector.
void Run(const RunOptions &run_options, const char *const *input_names, const Value *input_values, size_t input_count, const char *const *output_names, Value *output_values, size_t output_count)
Run the model returning results in user provided outputs Same as Run(const RunOptions&,...
Definition onnxruntime_cxx_api.h:1926
const int64_t * shape
Definition onnxruntime_cxx_api.h:1927
size_t shape_len
Definition onnxruntime_cxx_api.h:1928
Definition onnxruntime_cxx_api.h:1753
size_t GetElementCount() const
Wraps OrtApi::GetTensorShapeElementCount.
void GetDimensions(int64_t *values, size_t values_count) const
Wraps OrtApi::GetDimensions.
std::vector< int64_t > GetShape() const
Uses GetDimensionsCount & GetDimensions to return a std::vector of the shape.
std::vector< const char * > GetSymbolicDimensions() const
void GetSymbolicDimensions(const char **values, size_t values_count) const
Wraps OrtApi::GetSymbolicDimensions.
size_t GetDimensionsCount() const
Wraps OrtApi::GetDimensionsCount.
ONNXTensorElementDataType GetElementType() const
Wraps OrtApi::GetTensorElementType.
Definition onnxruntime_cxx_api.h:1864
ONNXType GetONNXType() const
ConstSequenceTypeInfo GetSequenceTypeInfo() const
Wraps OrtApi::CastTypeInfoToSequenceTypeInfo.
ConstMapTypeInfo GetMapTypeInfo() const
Wraps OrtApi::CastTypeInfoToMapTypeInfo.
ConstOptionalTypeInfo GetOptionalTypeInfo() const
wraps OrtApi::CastTypeInfoToOptionalTypeInfo
ConstTensorTypeAndShapeInfo GetTensorTypeAndShapeInfo() const
Wraps OrtApi::CastTypeInfoToTensorInfo.
This is a tagging template type. Use it with Base<T> to indicate that the C++ interface object has no...
Definition onnxruntime_cxx_api.h:660
T Type
Definition onnxruntime_cxx_api.h:661
Definition onnxruntime_cxx_api.h:2101
void FillStringTensorElement(const char *s, size_t index)
Set a single string in a string tensor.
R * GetTensorMutableData()
Returns a non-const typed pointer to an OrtValue/Tensor contained buffer No type checking is performe...
R & At(const std::vector< int64_t > &location)
void UseBlockSparseIndices(const Shape &indices_shape, int32_t *indices_data)
Supplies BlockSparse format specific indices and marks the contained sparse tensor as being a BlockSp...
void FillSparseTensorBlockSparse(const OrtMemoryInfo *data_mem_info, const OrtSparseValuesParam &values, const Shape &indices_shape, const int32_t *indices_data)
The API will allocate memory using the allocator instance supplied to the CreateSparseTensor() API an...
void * GetTensorMutableRawData()
Returns a non-typed non-const pointer to a tensor contained data.
void UseCooIndices(int64_t *indices_data, size_t indices_num)
Supplies COO format specific indices and marks the contained sparse tensor as being a COO format tens...
void FillSparseTensorCoo(const OrtMemoryInfo *data_mem_info, const OrtSparseValuesParam &values_param, const int64_t *indices_data, size_t indices_num)
The API will allocate memory using the allocator instance supplied to the CreateSparseTensor() API an...
void FillStringTensor(const char *const *s, size_t s_len)
Set all strings at once in a string tensor.
void UseCsrIndices(int64_t *inner_data, size_t inner_num, int64_t *outer_data, size_t outer_num)
Supplies CSR format specific indices and marks the contained sparse tensor as being a CSR format tens...
void FillSparseTensorCsr(const OrtMemoryInfo *data_mem_info, const OrtSparseValuesParam &values, const int64_t *inner_indices_data, size_t inner_indices_num, const int64_t *outer_indices_data, size_t outer_indices_num)
The API will allocate memory using the allocator instance supplied to the CreateSparseTensor() API an...
char * GetResizedStringTensorElementBuffer(size_t index, size_t buffer_length)
Allocate if necessary and obtain a pointer to a UTF-8 encoded string element buffer indexed by the fl...
Memory allocation interface.
Definition onnxruntime_c_api.h:346
void(* Free)(struct OrtAllocator *this_, void *p)
Free a block of memory previously allocated with OrtAllocator::Alloc.
Definition onnxruntime_c_api.h:353
const OrtApi *(* GetApi)(uint32_t version)
Get a pointer to the requested version of the OrtApi.
Definition onnxruntime_c_api.h:898
Definition onnxruntime_c_api.h:968
const OrtCompileApi *(* GetCompileApi)()
Get the Compile API instance.
Definition onnxruntime_c_api.h:5169
const OrtModelEditorApi *(* GetModelEditorApi)()
Get the Model Editor API instance.
Definition onnxruntime_c_api.h:5111
const OrtEpApi *(* GetEpApi)()
Get the OrtEpApi instance for implementing an execution provider.
Definition onnxruntime_c_api.h:5437
void(* ReleaseTensorRTProviderOptions)(OrtTensorRTProviderOptionsV2 *input)
Release an OrtTensorRTProviderOptionsV2.
Definition onnxruntime_c_api.h:3224
void(* ReleaseCUDAProviderOptions)(OrtCUDAProviderOptionsV2 *input)
Release an OrtCUDAProviderOptionsV2.
Definition onnxruntime_c_api.h:3727
CUDA Provider Options.
Definition onnxruntime_c_api.h:601
The OrtCompileApi struct provides functions to compile ONNX models.
Definition onnxruntime_c_api.h:7137
Definition onnxruntime_c_api.h:6610
int(* GetVariadicInputHomogeneity)(const struct OrtCustomOp *op)
Definition onnxruntime_c_api.h:6656
OrtCustomOpInputOutputCharacteristic(* GetOutputCharacteristic)(const struct OrtCustomOp *op, size_t index)
Definition onnxruntime_c_api.h:6640
size_t(* GetInputTypeCount)(const struct OrtCustomOp *op)
Definition onnxruntime_c_api.h:6628
int(* GetVariadicOutputMinArity)(const struct OrtCustomOp *op)
Definition onnxruntime_c_api.h:6660
size_t(* GetAliasMap)(int **input_index, int **output_index)
Definition onnxruntime_c_api.h:6693
int(* GetStartVersion)(const struct OrtCustomOp *op)
Definition onnxruntime_c_api.h:6678
void(* ReleaseMayInplace)(int *input_index, int *output_index)
Definition onnxruntime_c_api.h:6690
const char *(* GetName)(const struct OrtCustomOp *op)
Definition onnxruntime_c_api.h:6621
size_t(* GetOutputTypeCount)(const struct OrtCustomOp *op)
Definition onnxruntime_c_api.h:6630
void(* KernelDestroy)(void *op_kernel)
Definition onnxruntime_c_api.h:6636
int(* GetVariadicOutputHomogeneity)(const struct OrtCustomOp *op)
Definition onnxruntime_c_api.h:6665
OrtMemType(* GetInputMemoryType)(const struct OrtCustomOp *op, size_t index)
Definition onnxruntime_c_api.h:6647
void *(* CreateKernel)(const struct OrtCustomOp *op, const OrtApi *api, const OrtKernelInfo *info)
Definition onnxruntime_c_api.h:6617
uint32_t version
Definition onnxruntime_c_api.h:6611
ONNXTensorElementDataType(* GetInputType)(const struct OrtCustomOp *op, size_t index)
Definition onnxruntime_c_api.h:6627
void(* ReleaseAliasMap)(int *input_index, int *output_index)
Definition onnxruntime_c_api.h:6694
OrtCustomOpInputOutputCharacteristic(* GetInputCharacteristic)(const struct OrtCustomOp *op, size_t index)
Definition onnxruntime_c_api.h:6639
const char *(* GetExecutionProviderType)(const struct OrtCustomOp *op)
Definition onnxruntime_c_api.h:6624
ONNXTensorElementDataType(* GetOutputType)(const struct OrtCustomOp *op, size_t index)
Definition onnxruntime_c_api.h:6629
int(* GetVariadicInputMinArity)(const struct OrtCustomOp *op)
Definition onnxruntime_c_api.h:6651
OrtStatusPtr(* InferOutputShapeFn)(const struct OrtCustomOp *op, OrtShapeInferContext *)
Definition onnxruntime_c_api.h:6675
int(* GetEndVersion)(const struct OrtCustomOp *op)
Definition onnxruntime_c_api.h:6679
OrtStatusPtr(* CreateKernelV2)(const struct OrtCustomOp *op, const OrtApi *api, const OrtKernelInfo *info, void **kernel)
Definition onnxruntime_c_api.h:6668
size_t(* GetMayInplace)(int **input_index, int **output_index)
Definition onnxruntime_c_api.h:6686
OrtStatusPtr(* KernelComputeV2)(void *op_kernel, OrtKernelContext *context)
Definition onnxruntime_c_api.h:6673
void(* KernelCompute)(void *op_kernel, OrtKernelContext *context)
Definition onnxruntime_c_api.h:6635
MIGraphX Provider Options.
Definition onnxruntime_c_api.h:805
The OrtModelEditorApi struct provides functions to create or edit an ONNX model.
Definition onnxruntime_c_api.h:6708
OpenVINO Provider Options.
Definition onnxruntime_c_api.h:844
ROCM Provider Options.
Definition onnxruntime_c_api.h:688
TensorRT Provider Options.
Definition onnxruntime_c_api.h:777