ONNX Runtime
Loading...
Searching...
No Matches
onnxruntime_cxx_api.h
1// Copyright (c) Microsoft Corporation. All rights reserved.
2// Licensed under the MIT License.
3
4// Summary: The Ort C++ API is a header only wrapper around the Ort C API.
5//
6// The C++ API simplifies usage by returning values directly instead of error codes, throwing exceptions on errors
7// and automatically releasing resources in the destructors. The primary purpose of C++ API is exception safety so
8// all the resources follow RAII and do not leak memory.
9//
10// Each of the C++ wrapper classes holds only a pointer to the C internal object. Treat them like smart pointers.
11// To create an empty object, pass 'nullptr' to the constructor (for example, Env e{nullptr};). However, you can't use them
12// until you assign an instance that actually holds an underlying object.
13//
14// For Ort objects only move assignment between objects is allowed, there are no copy constructors.
15// Some objects have explicit 'Clone' methods for this purpose.
16//
17// ConstXXXX types are copyable since they do not own the underlying C object, so you can pass them to functions as arguments
18// by value or by reference. ConstXXXX types are restricted to const only interfaces.
19//
20// UnownedXXXX are similar to ConstXXXX but also allow non-const interfaces.
21//
22// The lifetime of the corresponding owning object must eclipse the lifetimes of the ConstXXXX/UnownedXXXX types. They exists so you do not
23// have to fallback to C types and the API with the usual pitfalls. In general, do not use C API from your C++ code.
24
25#pragma once
26#include "onnxruntime_c_api.h"
27#include "onnxruntime_float16.h"
28
29#include <array>
30#include <cstddef>
31#include <cstdio>
32#include <memory>
33#include <stdexcept>
34#include <string>
35#include <type_traits>
36#include <unordered_map>
37#include <utility>
38#include <variant>
39#include <vector>
40
41#ifdef ORT_NO_EXCEPTIONS
42#include <iostream>
43#endif
44
48namespace Ort {
49
54struct Exception : std::exception {
55 Exception(std::string&& string, OrtErrorCode code) : message_{std::move(string)}, code_{code} {}
56
57 OrtErrorCode GetOrtErrorCode() const { return code_; }
58 const char* what() const noexcept override { return message_.c_str(); }
59
60 private:
61 std::string message_;
62 OrtErrorCode code_;
63};
64
65#ifdef ORT_NO_EXCEPTIONS
66// The #ifndef is for the very special case where the user of this library wants to define their own way of handling errors.
67// NOTE: This header expects control flow to not continue after calling ORT_CXX_API_THROW
68#ifndef ORT_CXX_API_THROW
69#define ORT_CXX_API_THROW(string, code) \
70 do { \
71 std::cerr << Ort::Exception(string, code) \
72 .what() \
73 << std::endl; \
74 abort(); \
75 } while (false)
76#endif
77#else
78#define ORT_CXX_API_THROW(string, code) \
79 throw Ort::Exception(string, code)
80#endif
81
82// This is used internally by the C++ API. This class holds the global variable that points to the OrtApi,
83// it's in a template so that we can define a global variable in a header and make
84// it transparent to the users of the API.
85template <typename T>
86struct Global {
87 static const OrtApi* api_;
88};
89
90// If macro ORT_API_MANUAL_INIT is defined, no static initialization will be performed. Instead, user must call InitApi() before using it.
91template <typename T>
92#ifdef ORT_API_MANUAL_INIT
93const OrtApi* Global<T>::api_{};
94inline void InitApi() noexcept { Global<void>::api_ = OrtGetApiBase()->GetApi(ORT_API_VERSION); }
95
96// Used by custom operator libraries that are not linked to onnxruntime. Sets the global API object, which is
97// required by C++ APIs.
98//
99// Example mycustomop.cc:
100//
101// #define ORT_API_MANUAL_INIT
102// #include <onnxruntime_cxx_api.h>
103// #undef ORT_API_MANUAL_INIT
104//
105// OrtStatus* ORT_API_CALL RegisterCustomOps(OrtSessionOptions* options, const OrtApiBase* api_base) {
106// Ort::InitApi(api_base->GetApi(ORT_API_VERSION));
107// // ...
108// }
109//
110inline void InitApi(const OrtApi* api) noexcept { Global<void>::api_ = api; }
111#else
112#if defined(_MSC_VER) && !defined(__clang__)
113#pragma warning(push)
114// "Global initializer calls a non-constexpr function." Therefore you can't use ORT APIs in the other global initializers.
115// Please define ORT_API_MANUAL_INIT if it conerns you.
116#pragma warning(disable : 26426)
117#endif
119#if defined(_MSC_VER) && !defined(__clang__)
120#pragma warning(pop)
121#endif
122#endif
123
125inline const OrtApi& GetApi() noexcept { return *Global<void>::api_; }
126
131std::string GetVersionString();
132
138std::string GetBuildInfoString();
139
145std::vector<std::string> GetAvailableProviders();
146
152 auto* api = GetApi().GetModelEditorApi();
153 if (api == nullptr) {
154 // minimal build
155 ORT_CXX_API_THROW("Model Editor API is not available in this build", ORT_FAIL);
156 }
157
158 return *api;
159}
160
166 auto* api = GetApi().GetCompileApi();
167 if (api == nullptr) {
168 // minimal build
169 ORT_CXX_API_THROW("Compile API is not available in this build", ORT_FAIL);
170 }
171
172 return *api;
173}
174
179inline const OrtEpApi& GetEpApi() {
180 auto* api = GetApi().GetEpApi();
181 if (api == nullptr) {
182 // minimal build
183 ORT_CXX_API_THROW("EP API is not available in this build", ORT_FAIL);
184 }
185
186 return *api;
187}
188
207struct Float16_t : onnxruntime_float16::Float16Impl<Float16_t> {
208 private:
214 constexpr explicit Float16_t(uint16_t v) noexcept { val = v; }
215
216 public:
217 using Base = onnxruntime_float16::Float16Impl<Float16_t>;
218
222 Float16_t() = default;
223
229 constexpr static Float16_t FromBits(uint16_t v) noexcept { return Float16_t(v); }
230
235 explicit Float16_t(float v) noexcept { val = Base::ToUint16Impl(v); }
236
241 float ToFloat() const noexcept { return Base::ToFloatImpl(); }
242
247 using Base::IsNegative;
248
253 using Base::IsNaN;
254
259 using Base::IsFinite;
260
265 using Base::IsPositiveInfinity;
266
271 using Base::IsNegativeInfinity;
272
277 using Base::IsInfinity;
278
283 using Base::IsNaNOrZero;
284
289 using Base::IsNormal;
290
295 using Base::IsSubnormal;
296
301 using Base::Abs;
302
307 using Base::Negate;
308
317 using Base::AreZero;
318
322 explicit operator float() const noexcept { return ToFloat(); }
323
324 using Base::operator==;
325 using Base::operator!=;
326 using Base::operator<;
327};
328
329static_assert(sizeof(Float16_t) == sizeof(uint16_t), "Sizes must match");
330
349struct BFloat16_t : onnxruntime_float16::BFloat16Impl<BFloat16_t> {
350 private:
358 constexpr explicit BFloat16_t(uint16_t v) noexcept { val = v; }
359
360 public:
361 using Base = onnxruntime_float16::BFloat16Impl<BFloat16_t>;
362
363 BFloat16_t() = default;
364
370 static constexpr BFloat16_t FromBits(uint16_t v) noexcept { return BFloat16_t(v); }
371
376 explicit BFloat16_t(float v) noexcept { val = Base::ToUint16Impl(v); }
377
382 float ToFloat() const noexcept { return Base::ToFloatImpl(); }
383
388 using Base::IsNegative;
389
394 using Base::IsNaN;
395
400 using Base::IsFinite;
401
406 using Base::IsPositiveInfinity;
407
412 using Base::IsNegativeInfinity;
413
418 using Base::IsInfinity;
419
424 using Base::IsNaNOrZero;
425
430 using Base::IsNormal;
431
436 using Base::IsSubnormal;
437
442 using Base::Abs;
443
448 using Base::Negate;
449
458 using Base::AreZero;
459
463 explicit operator float() const noexcept { return ToFloat(); }
464
465 // We do not have an inherited impl for the below operators
466 // as the internal class implements them a little differently
467 bool operator==(const BFloat16_t& rhs) const noexcept;
468 bool operator!=(const BFloat16_t& rhs) const noexcept { return !(*this == rhs); }
469 bool operator<(const BFloat16_t& rhs) const noexcept;
470};
471
472static_assert(sizeof(BFloat16_t) == sizeof(uint16_t), "Sizes must match");
473
480 uint8_t value;
481 constexpr Float8E4M3FN_t() noexcept : value(0) {}
482 constexpr Float8E4M3FN_t(uint8_t v) noexcept : value(v) {}
483 constexpr operator uint8_t() const noexcept { return value; }
484 // nan values are treated like any other value for operator ==, !=
485 constexpr bool operator==(const Float8E4M3FN_t& rhs) const noexcept { return value == rhs.value; };
486 constexpr bool operator!=(const Float8E4M3FN_t& rhs) const noexcept { return value != rhs.value; };
487};
488
489static_assert(sizeof(Float8E4M3FN_t) == sizeof(uint8_t), "Sizes must match");
490
497 uint8_t value;
498 constexpr Float8E4M3FNUZ_t() noexcept : value(0) {}
499 constexpr Float8E4M3FNUZ_t(uint8_t v) noexcept : value(v) {}
500 constexpr operator uint8_t() const noexcept { return value; }
501 // nan values are treated like any other value for operator ==, !=
502 constexpr bool operator==(const Float8E4M3FNUZ_t& rhs) const noexcept { return value == rhs.value; };
503 constexpr bool operator!=(const Float8E4M3FNUZ_t& rhs) const noexcept { return value != rhs.value; };
504};
505
506static_assert(sizeof(Float8E4M3FNUZ_t) == sizeof(uint8_t), "Sizes must match");
507
514 uint8_t value;
515 constexpr Float8E5M2_t() noexcept : value(0) {}
516 constexpr Float8E5M2_t(uint8_t v) noexcept : value(v) {}
517 constexpr operator uint8_t() const noexcept { return value; }
518 // nan values are treated like any other value for operator ==, !=
519 constexpr bool operator==(const Float8E5M2_t& rhs) const noexcept { return value == rhs.value; };
520 constexpr bool operator!=(const Float8E5M2_t& rhs) const noexcept { return value != rhs.value; };
521};
522
523static_assert(sizeof(Float8E5M2_t) == sizeof(uint8_t), "Sizes must match");
524
531 uint8_t value;
532 constexpr Float8E5M2FNUZ_t() noexcept : value(0) {}
533 constexpr Float8E5M2FNUZ_t(uint8_t v) noexcept : value(v) {}
534 constexpr operator uint8_t() const noexcept { return value; }
535 // nan values are treated like any other value for operator ==, !=
536 constexpr bool operator==(const Float8E5M2FNUZ_t& rhs) const noexcept { return value == rhs.value; };
537 constexpr bool operator!=(const Float8E5M2FNUZ_t& rhs) const noexcept { return value != rhs.value; };
538};
539
540static_assert(sizeof(Float8E5M2FNUZ_t) == sizeof(uint8_t), "Sizes must match");
541
542namespace detail {
543// This is used internally by the C++ API. This macro is to make it easy to generate overloaded methods for all of the various OrtRelease* functions for every Ort* type
544// This can't be done in the C API since C doesn't have function overloading.
545#define ORT_DEFINE_RELEASE(NAME) \
546 inline void OrtRelease(Ort##NAME* ptr) { GetApi().Release##NAME(ptr); }
547
548#define ORT_DEFINE_RELEASE_FROM_API_STRUCT(NAME, API_GETTER) \
549 inline void OrtRelease(Ort##NAME* ptr) { API_GETTER().Release##NAME(ptr); }
550
551ORT_DEFINE_RELEASE(Allocator);
552ORT_DEFINE_RELEASE(MemoryInfo);
553ORT_DEFINE_RELEASE(CustomOpDomain);
554ORT_DEFINE_RELEASE(ThreadingOptions);
555ORT_DEFINE_RELEASE(Env);
556ORT_DEFINE_RELEASE(RunOptions);
557ORT_DEFINE_RELEASE(LoraAdapter);
558ORT_DEFINE_RELEASE(Session);
559ORT_DEFINE_RELEASE(SessionOptions);
560ORT_DEFINE_RELEASE(TensorTypeAndShapeInfo);
561ORT_DEFINE_RELEASE(SequenceTypeInfo);
562ORT_DEFINE_RELEASE(MapTypeInfo);
563ORT_DEFINE_RELEASE(TypeInfo);
564ORT_DEFINE_RELEASE(Value);
565ORT_DEFINE_RELEASE(ModelMetadata);
566ORT_DEFINE_RELEASE(IoBinding);
567ORT_DEFINE_RELEASE(ArenaCfg);
568ORT_DEFINE_RELEASE(Status);
569ORT_DEFINE_RELEASE(OpAttr);
570ORT_DEFINE_RELEASE(Op);
571ORT_DEFINE_RELEASE(KernelInfo);
572ORT_DEFINE_RELEASE(ValueInfo);
573ORT_DEFINE_RELEASE(Node);
574ORT_DEFINE_RELEASE(Graph);
575ORT_DEFINE_RELEASE(Model);
576ORT_DEFINE_RELEASE(KeyValuePairs)
577ORT_DEFINE_RELEASE_FROM_API_STRUCT(ModelCompilationOptions, GetCompileApi);
578ORT_DEFINE_RELEASE_FROM_API_STRUCT(EpDevice, GetEpApi);
579
580#undef ORT_DEFINE_RELEASE
581#undef ORT_DEFINE_RELEASE_FROM_API_STRUCT
582
586template <typename T>
587struct Unowned {
588 using Type = T;
589};
590
610template <typename T>
611struct Base {
612 using contained_type = T;
613
614 constexpr Base() = default;
615 constexpr explicit Base(contained_type* p) noexcept : p_{p} {}
617 OrtRelease(p_);
618 }
619
620 Base(const Base&) = delete;
621 Base& operator=(const Base&) = delete;
622
623 Base(Base&& v) noexcept : p_{v.p_} { v.p_ = nullptr; }
624 Base& operator=(Base&& v) noexcept {
625 OrtRelease(p_);
626 p_ = v.release();
627 return *this;
628 }
629
630 constexpr operator contained_type*() const noexcept { return p_; }
631
635 T* p = p_;
636 p_ = nullptr;
637 return p;
638 }
639
640 protected:
642};
643
644// Undefined. For const types use Base<Unowned<const T>>
645template <typename T>
646struct Base<const T>;
647
655template <typename T>
656struct Base<Unowned<T>> {
658
659 constexpr Base() = default;
660 constexpr explicit Base(contained_type* p) noexcept : p_{p} {}
661
662 ~Base() = default;
663
664 Base(const Base&) = default;
665 Base& operator=(const Base&) = default;
666
667 Base(Base&& v) noexcept : p_{v.p_} { v.p_ = nullptr; }
668 Base& operator=(Base&& v) noexcept {
669 p_ = nullptr;
670 std::swap(p_, v.p_);
671 return *this;
672 }
673
674 constexpr operator contained_type*() const noexcept { return p_; }
675
676 protected:
678};
679
680// Light functor to release memory with OrtAllocator
683 explicit AllocatedFree(OrtAllocator* allocator)
684 : allocator_(allocator) {}
685 void operator()(void* ptr) const {
686 if (ptr) allocator_->Free(allocator_, ptr);
687 }
688};
689
690} // namespace detail
691
692struct AllocatorWithDefaultOptions;
693struct Env;
694struct EpDevice;
695struct Graph;
696struct Model;
697struct Node;
698struct ModelMetadata;
699struct TypeInfo;
700struct Value;
701struct ValueInfo;
702
707using AllocatedStringPtr = std::unique_ptr<char, detail::AllocatedFree>;
708
713struct Status : detail::Base<OrtStatus> {
715 using Base::Base;
716
717 explicit Status(std::nullptr_t) noexcept {}
718 explicit Status(OrtStatus* status) noexcept;
719 explicit Status(const Exception&) noexcept;
720 explicit Status(const std::exception&) noexcept;
721 Status(const char* message, OrtErrorCode code) noexcept;
722 std::string GetErrorMessage() const;
724 bool IsOK() const noexcept;
725};
726
756
757namespace detail {
758template <typename T>
761 using B::B;
762
763 const char* GetValue(const char* key) const;
764
765 // get the pairs in unordered_map. needs to copy to std::string so the hash works as expected
766 std::unordered_map<std::string, std::string> GetKeyValuePairs() const;
767 // get the pairs in two vectors. entries will be 1:1 between keys and values. avoids copying to std::string
768 void GetKeyValuePairs(std::vector<const char*>& keys, std::vector<const char*>& values) const;
769};
770} // namespace detail
771
772// Const object holder that does not own the underlying object
774
776struct KeyValuePairs : detail::KeyValuePairsImpl<OrtKeyValuePairs> {
777 explicit KeyValuePairs(std::nullptr_t) {}
779 explicit KeyValuePairs(OrtKeyValuePairs* p) : KeyValuePairsImpl<OrtKeyValuePairs>{p} {}
780
782 explicit KeyValuePairs();
783
785 explicit KeyValuePairs(const std::unordered_map<std::string, std::string>& kv_pairs);
786
788 void Add(const char* key, const char* value);
789
791 void Remove(const char* key);
792
793 ConstKeyValuePairs GetConst() const { return ConstKeyValuePairs{this->p_}; }
794};
795
796namespace detail {
797template <typename T>
800 using B::B;
801
803 uint32_t VendorId() const;
804 uint32_t DeviceId() const;
805 const char* Vendor() const;
807};
808} // namespace detail
809
814
815namespace detail {
816template <typename T>
819 using B::B;
820
821 const char* EpName() const;
822 const char* EpVendor() const;
826};
827} // namespace detail
828
833
836struct EpDevice : detail::EpDeviceImpl<OrtEpDevice> {
837 explicit EpDevice(std::nullptr_t) {}
838 explicit EpDevice(OrtEpDevice* p) : EpDeviceImpl<OrtEpDevice>{p} {}
839
841 EpDevice(OrtEpFactory& ep_factory, ConstHardwareDevice& hardware_device,
842 ConstKeyValuePairs ep_metadata = {}, ConstKeyValuePairs ep_options = {});
843};
844
850struct Env : detail::Base<OrtEnv> {
851 explicit Env(std::nullptr_t) {}
852
854 Env(OrtLoggingLevel logging_level = ORT_LOGGING_LEVEL_WARNING, _In_ const char* logid = "");
855
857 Env(OrtLoggingLevel logging_level, const char* logid, OrtLoggingFunction logging_function, void* logger_param);
858
860 Env(const OrtThreadingOptions* tp_options, OrtLoggingLevel logging_level = ORT_LOGGING_LEVEL_WARNING, _In_ const char* logid = "");
861
863 Env(const OrtThreadingOptions* tp_options, OrtLoggingFunction logging_function, void* logger_param,
864 OrtLoggingLevel logging_level = ORT_LOGGING_LEVEL_WARNING, _In_ const char* logid = "");
865
867 explicit Env(OrtEnv* p) : Base<OrtEnv>{p} {}
868
871
873
874 Env& CreateAndRegisterAllocator(const OrtMemoryInfo* mem_info, const OrtArenaCfg* arena_cfg);
875
876 Env& CreateAndRegisterAllocatorV2(const std::string& provider_type, const OrtMemoryInfo* mem_info,
877 const std::unordered_map<std::string, std::string>& options,
878 const OrtArenaCfg* arena_cfg);
879
880 Env& RegisterExecutionProviderLibrary(const char* registration_name, const std::basic_string<ORTCHAR_T>& path);
881 Env& UnregisterExecutionProviderLibrary(const char* registration_name);
882
883 std::vector<ConstEpDevice> GetEpDevices() const;
884};
885
889struct CustomOpDomain : detail::Base<OrtCustomOpDomain> {
891 using Base::Base;
892
893 explicit CustomOpDomain(std::nullptr_t) {}
894
896 explicit CustomOpDomain(const char* domain);
897
898 // This does not take ownership of the op, simply registers it.
899 void Add(const OrtCustomOp* op);
900};
901
903struct LoraAdapter : detail::Base<OrtLoraAdapter> {
905 using Base::Base;
906
907 explicit LoraAdapter(std::nullptr_t) {}
914 static LoraAdapter CreateLoraAdapter(const std::basic_string<ORTCHAR_T>& adapter_path,
915 OrtAllocator* allocator);
916
924 static LoraAdapter CreateLoraAdapterFromArray(const void* bytes, size_t num_bytes,
925 OrtAllocator* allocator);
926};
927
931struct RunOptions : detail::Base<OrtRunOptions> {
932 explicit RunOptions(std::nullptr_t) {}
934
937
940
941 RunOptions& SetRunTag(const char* run_tag);
942 const char* GetRunTag() const;
943
944 RunOptions& AddConfigEntry(const char* config_key, const char* config_value);
945 const char* GetConfigEntry(const char* config_key);
946
953
959
967};
968
969namespace detail {
970// Utility function that returns a SessionOption config entry key for a specific custom operator.
971// Ex: custom_op.[custom_op_name].[config]
972std::string MakeCustomOpConfigEntryKey(const char* custom_op_name, const char* config);
973} // namespace detail
974
985 CustomOpConfigs() = default;
986 ~CustomOpConfigs() = default;
991
1000 CustomOpConfigs& AddConfig(const char* custom_op_name, const char* config_key, const char* config_value);
1001
1010 const std::unordered_map<std::string, std::string>& GetFlattenedConfigs() const;
1011
1012 private:
1013 std::unordered_map<std::string, std::string> flat_configs_;
1014};
1015
1021struct SessionOptions;
1022
1023namespace detail {
1024// we separate const-only methods because passing const ptr to non-const methods
1025// is only discovered when inline methods are compiled which is counter-intuitive
1026template <typename T>
1027struct ConstSessionOptionsImpl : Base<T> {
1028 using B = Base<T>;
1029 using B::B;
1030
1031 SessionOptions Clone() const;
1032
1033 std::string GetConfigEntry(const char* config_key) const;
1034 bool HasConfigEntry(const char* config_key) const;
1035 std::string GetConfigEntryOrDefault(const char* config_key, const std::string& def) const;
1036};
1037
1038template <typename T>
1039struct SessionOptionsImpl : ConstSessionOptionsImpl<T> {
1040 using B = ConstSessionOptionsImpl<T>;
1041 using B::B;
1042
1043 SessionOptionsImpl& SetIntraOpNumThreads(int intra_op_num_threads);
1044 SessionOptionsImpl& SetInterOpNumThreads(int inter_op_num_threads);
1045 SessionOptionsImpl& SetGraphOptimizationLevel(GraphOptimizationLevel graph_optimization_level);
1046 SessionOptionsImpl& SetDeterministicCompute(bool value);
1047
1048 SessionOptionsImpl& EnableCpuMemArena();
1049 SessionOptionsImpl& DisableCpuMemArena();
1050
1051 SessionOptionsImpl& SetOptimizedModelFilePath(const ORTCHAR_T* optimized_model_file);
1052
1053 SessionOptionsImpl& EnableProfiling(const ORTCHAR_T* profile_file_prefix);
1054 SessionOptionsImpl& DisableProfiling();
1055
1056 SessionOptionsImpl& EnableOrtCustomOps();
1057
1058 SessionOptionsImpl& EnableMemPattern();
1059 SessionOptionsImpl& DisableMemPattern();
1060
1061 SessionOptionsImpl& SetExecutionMode(ExecutionMode execution_mode);
1062
1063 SessionOptionsImpl& SetLoadCancellationFlag(bool value);
1064
1065 SessionOptionsImpl& SetLogId(const char* logid);
1066 SessionOptionsImpl& SetLogSeverityLevel(int level);
1067
1068 SessionOptionsImpl& Add(OrtCustomOpDomain* custom_op_domain);
1069
1070 SessionOptionsImpl& DisablePerSessionThreads();
1071
1072 SessionOptionsImpl& AddConfigEntry(const char* config_key, const char* config_value);
1073
1074 SessionOptionsImpl& AddInitializer(const char* name, const OrtValue* ort_val);
1075 SessionOptionsImpl& AddExternalInitializers(const std::vector<std::string>& names, const std::vector<Value>& ort_values);
1076 SessionOptionsImpl& AddExternalInitializersFromFilesInMemory(const std::vector<std::basic_string<ORTCHAR_T>>& external_initializer_file_names,
1077 const std::vector<char*>& external_initializer_file_buffer_array,
1078 const std::vector<size_t>& external_initializer_file_lengths);
1079
1080 SessionOptionsImpl& AppendExecutionProvider_CUDA(const OrtCUDAProviderOptions& provider_options);
1081 SessionOptionsImpl& AppendExecutionProvider_CUDA_V2(const OrtCUDAProviderOptionsV2& provider_options);
1082 SessionOptionsImpl& AppendExecutionProvider_ROCM(const OrtROCMProviderOptions& provider_options);
1083 SessionOptionsImpl& AppendExecutionProvider_OpenVINO(const OrtOpenVINOProviderOptions& provider_options);
1085 SessionOptionsImpl& AppendExecutionProvider_OpenVINO_V2(const std::unordered_map<std::string, std::string>& provider_options = {});
1086 SessionOptionsImpl& AppendExecutionProvider_TensorRT(const OrtTensorRTProviderOptions& provider_options);
1087 SessionOptionsImpl& AppendExecutionProvider_TensorRT_V2(const OrtTensorRTProviderOptionsV2& provider_options);
1088 SessionOptionsImpl& AppendExecutionProvider_MIGraphX(const OrtMIGraphXProviderOptions& provider_options);
1090 SessionOptionsImpl& AppendExecutionProvider_CANN(const OrtCANNProviderOptions& provider_options);
1092 SessionOptionsImpl& AppendExecutionProvider_Dnnl(const OrtDnnlProviderOptions& provider_options);
1094 SessionOptionsImpl& AppendExecutionProvider(const std::string& provider_name,
1095 const std::unordered_map<std::string, std::string>& provider_options = {});
1096
1099 SessionOptionsImpl& AppendExecutionProvider_V2(Env& env, const std::vector<ConstEpDevice>& ep_devices,
1100 const KeyValuePairs& ep_options);
1103 SessionOptionsImpl& AppendExecutionProvider_V2(Env& env, const std::vector<ConstEpDevice>& ep_devices,
1104 const std::unordered_map<std::string, std::string>& ep_options);
1105
1107 SessionOptionsImpl& SetEpSelectionPolicy(OrtExecutionProviderDevicePolicy policy);
1108
1110 SessionOptionsImpl& SetEpSelectionPolicy(EpSelectionDelegate delegate, void* state = nullptr);
1111
1112 SessionOptionsImpl& SetCustomCreateThreadFn(OrtCustomCreateThreadFn ort_custom_create_thread_fn);
1113 SessionOptionsImpl& SetCustomThreadCreationOptions(void* ort_custom_thread_creation_options);
1114 SessionOptionsImpl& SetCustomJoinThreadFn(OrtCustomJoinThreadFn ort_custom_join_thread_fn);
1115
1119 SessionOptionsImpl& RegisterCustomOpsLibrary(const ORTCHAR_T* library_name, const CustomOpConfigs& custom_op_configs = {});
1120
1121 SessionOptionsImpl& RegisterCustomOpsUsingFunction(const char* function_name);
1122
1124 SessionOptionsImpl& AppendExecutionProvider_VitisAI(const std::unordered_map<std::string, std::string>& provider_options = {});
1125};
1126} // namespace detail
1127
1128using UnownedSessionOptions = detail::SessionOptionsImpl<detail::Unowned<OrtSessionOptions>>;
1129using ConstSessionOptions = detail::ConstSessionOptionsImpl<detail::Unowned<const OrtSessionOptions>>;
1130
1134struct SessionOptions : detail::SessionOptionsImpl<OrtSessionOptions> {
1135 explicit SessionOptions(std::nullptr_t) {}
1137 explicit SessionOptions(OrtSessionOptions* p) : SessionOptionsImpl<OrtSessionOptions>{p} {}
1140};
1141
1146struct ModelCompilationOptions : detail::Base<OrtModelCompilationOptions> {
1148 using Base::Base;
1149
1150 explicit ModelCompilationOptions(std::nullptr_t) {}
1151
1152 ModelCompilationOptions(const Env& env, const SessionOptions& session_options);
1153 ModelCompilationOptions(const Env& env, ConstSessionOptions session_options);
1154
1155 ModelCompilationOptions& SetInputModelPath(const ORTCHAR_T* input_model_path);
1157 size_t input_model_data_size);
1158 ModelCompilationOptions& SetEpContextEmbedMode(bool embed_ep_context_in_model);
1159 ModelCompilationOptions& SetOutputModelPath(const ORTCHAR_T* output_model_path);
1161 size_t initializer_size_threshold);
1162 ModelCompilationOptions& SetOutputModelBuffer(OrtAllocator* allocator, void** output_model_buffer_ptr,
1163 size_t* output_model_buffer_size_ptr);
1164 ModelCompilationOptions& SetEpContextBinaryInformation(const ORTCHAR_T* output_directory,
1165 const ORTCHAR_T* model_name);
1167};
1168
1175Status CompileModel(const Env& env, const ModelCompilationOptions& model_compilation_options);
1176
1180struct ModelMetadata : detail::Base<OrtModelMetadata> {
1182 using Base::Base;
1183
1184 explicit ModelMetadata(std::nullptr_t) {}
1185
1193
1201
1209
1217
1225
1232 std::vector<AllocatedStringPtr> GetCustomMetadataMapKeysAllocated(OrtAllocator* allocator) const;
1233
1244
1245 int64_t GetVersion() const;
1246};
1247
1248struct IoBinding;
1249
1250namespace detail {
1251
1252// we separate const-only methods because passing const ptr to non-const methods
1253// is only discovered when inline methods are compiled which is counter-intuitive
1254template <typename T>
1256 using B = Base<T>;
1257 using B::B;
1258
1259 size_t GetInputCount() const;
1260 size_t GetOutputCount() const;
1262
1263 std::vector<std::string> GetInputNames() const;
1264 std::vector<std::string> GetOutputNames() const;
1265 std::vector<std::string> GetOverridableInitializerNames() const;
1266
1275
1284
1293
1294 uint64_t GetProfilingStartTimeNs() const;
1296
1297 TypeInfo GetInputTypeInfo(size_t index) const;
1298 TypeInfo GetOutputTypeInfo(size_t index) const;
1300
1301 int GetOpset(const std::string& domain) const;
1302
1303 // Will move before checkin if that's the case.
1304 std::vector<ValueInfo> GetInputs() const;
1305 std::vector<ValueInfo> GetOutputs() const;
1306};
1307
1308template <typename T>
1311 using B::B;
1312
1330 std::vector<Value> Run(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count,
1331 const char* const* output_names, size_t output_count);
1332
1336 void Run(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count,
1337 const char* const* output_names, Value* output_values, size_t output_count);
1338
1339 void Run(const RunOptions& run_options, const IoBinding&);
1340
1360 void RunAsync(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count,
1361 const char* const* output_names, Value* output_values, size_t output_count, RunAsyncCallbackFn callback, void* user_data);
1362
1370
1382 void SetEpDynamicOptions(const char* const* keys, const char* const* values, size_t kv_len);
1383
1384 void FinalizeModelEditorSession(const Model& model, const SessionOptions& options,
1385 OrtPrepackedWeightsContainer* prepacked_weights_container = nullptr);
1386};
1387
1388} // namespace detail
1389
1392
1396struct Session : detail::SessionImpl<OrtSession> {
1398 explicit Session(std::nullptr_t) {}
1399 explicit Session(OrtSession* p) : SessionImpl{p} {}
1400
1401 Session(const Env& env, const ORTCHAR_T* model_path, const SessionOptions& options);
1402
1404 Session(const Env& env, const ORTCHAR_T* model_path, const SessionOptions& options,
1405 OrtPrepackedWeightsContainer* prepacked_weights_container);
1406
1408 Session(const Env& env, const void* model_data, size_t model_data_length, const SessionOptions& options);
1409
1411 Session(const Env& env, const void* model_data, size_t model_data_length, const SessionOptions& options,
1412 OrtPrepackedWeightsContainer* prepacked_weights_container);
1413
1414#if !defined(ORT_MINIMAL_BUILD)
1416 Session(const Env& env, const Model& model, const SessionOptions& options);
1417
1419 static Session CreateModelEditorSession(const Env& env, const ORTCHAR_T* model_path, const SessionOptions& options);
1420
1422 static Session CreateModelEditorSession(const Env& env, const void* model_data, size_t model_data_length,
1423 const SessionOptions& options);
1424#endif // !defined(ORT_MINIMAL_BUILD)
1425
1426 ConstSession GetConst() const { return ConstSession{this->p_}; }
1427 UnownedSession GetUnowned() const { return UnownedSession{this->p_}; }
1428};
1429
1430namespace detail {
1431template <typename T>
1433 using B = Base<T>;
1434 using B::B;
1435
1436 std::string GetAllocatorName() const;
1438 int GetDeviceId() const;
1441
1442 template <typename U>
1443 bool operator==(const MemoryInfoImpl<U>& o) const;
1444};
1445} // namespace detail
1446
1447// Const object holder that does not own the underlying object
1449
1453struct MemoryInfo : detail::MemoryInfoImpl<OrtMemoryInfo> {
1455 explicit MemoryInfo(std::nullptr_t) {}
1456 explicit MemoryInfo(OrtMemoryInfo* p) : MemoryInfoImpl<OrtMemoryInfo>{p} {}
1457 MemoryInfo(const char* name, OrtAllocatorType type, int id, OrtMemType mem_type);
1458 ConstMemoryInfo GetConst() const { return ConstMemoryInfo{this->p_}; }
1459};
1460
1461namespace detail {
1462template <typename T>
1464 using B = Base<T>;
1465 using B::B;
1466
1468 size_t GetElementCount() const;
1469
1470 size_t GetDimensionsCount() const;
1471
1476 [[deprecated("use GetShape()")]] void GetDimensions(int64_t* values, size_t values_count) const;
1477
1478 void GetSymbolicDimensions(const char** values, size_t values_count) const;
1479 std::vector<const char*> GetSymbolicDimensions() const;
1480
1481 std::vector<int64_t> GetShape() const;
1482};
1483
1484} // namespace detail
1485
1487
1493 using Base::Base;
1494
1496 explicit TensorTypeAndShapeInfo(std::nullptr_t) {}
1498 explicit TensorTypeAndShapeInfo(OrtTensorTypeAndShapeInfo* p) : TensorTypeAndShapeInfoImpl{p} {}
1499
1500 // Create a TensorTypeAndShapeInfo object with the specified element type and dimensions
1501 // symbolic_dims are optional, but should be 1:1 with dims.
1502 // The value in symbolic_dims will be used for all entries in dims that are -1.
1504 const std::vector<int64_t>& dims,
1505 const std::vector<std::string>* symbolic_dims = nullptr);
1506
1508};
1509
1510namespace detail {
1511template <typename T>
1513 using B = Base<T>;
1514 using B::B;
1516};
1517
1518} // namespace detail
1519
1521
1525struct SequenceTypeInfo : detail::SequenceTypeInfoImpl<OrtSequenceTypeInfo> {
1527 using Base::Base;
1528
1529 explicit SequenceTypeInfo(std::nullptr_t) {}
1530 explicit SequenceTypeInfo(OrtSequenceTypeInfo* p) : SequenceTypeInfoImpl<OrtSequenceTypeInfo>{p} {}
1532};
1533
1534namespace detail {
1535template <typename T>
1537 using B = Base<T>;
1538 using B::B;
1540};
1541
1542} // namespace detail
1543
1544// This is always owned by the TypeInfo and can only be obtained from it.
1546
1547namespace detail {
1548template <typename T>
1555
1556} // namespace detail
1557
1559
1563struct MapTypeInfo : detail::MapTypeInfoImpl<OrtMapTypeInfo> {
1565 using Base::Base;
1566
1567 explicit MapTypeInfo(std::nullptr_t) {}
1568 explicit MapTypeInfo(OrtMapTypeInfo* p) : MapTypeInfoImpl<OrtMapTypeInfo>{p} {}
1569 ConstMapTypeInfo GetConst() const { return ConstMapTypeInfo{this->p_}; }
1570};
1571
1572namespace detail {
1573template <typename T>
1585} // namespace detail
1586
1592
1597struct TypeInfo : detail::TypeInfoImpl<OrtTypeInfo> {
1599 using Base::Base;
1600
1602 explicit TypeInfo(std::nullptr_t) {}
1603 explicit TypeInfo(OrtTypeInfo* p) : TypeInfoImpl<OrtTypeInfo>{p} {}
1604
1605#if !defined(ORT_MINIMAL_BUILD)
1611#endif // !defined(ORT_MINIMAL_BUILD)
1612
1613 ConstTypeInfo GetConst() const { return ConstTypeInfo{this->p_}; }
1614};
1615
1616namespace detail {
1617// This structure is used to feed sparse tensor values
1618// information for use with FillSparseTensor<Format>() API
1619// if the data type for the sparse tensor values is numeric
1620// use data.p_data, otherwise, use data.str pointer to feed
1621// values. data.str is an array of const char* that are zero terminated.
1622// number of strings in the array must match shape size.
1623// For fully sparse tensors use shape {0} and set p_data/str
1624// to nullptr.
1626 const int64_t* values_shape;
1628 union {
1629 const void* p_data;
1630 const char** str;
1631 } data;
1632};
1633
1634// Provides a way to pass shape in a single
1635// argument
1636struct Shape {
1637 const int64_t* shape;
1639};
1640
1641template <typename T>
1643 using B = Base<T>;
1644 using B::B;
1645
1649 template <typename R>
1650 void GetOpaqueData(const char* domain, const char* type_name, R&) const;
1651
1652 bool IsTensor() const;
1653 bool HasValue() const;
1654
1655 size_t GetCount() const; // If a non tensor, returns 2 for map and N for sequence, where N is the number of elements
1656 Value GetValue(int index, OrtAllocator* allocator) const;
1657
1665
1680 void GetStringTensorContent(void* buffer, size_t buffer_length, size_t* offsets, size_t offsets_count) const;
1681
1688 template <typename R>
1689 const R* GetTensorData() const;
1690
1695 const void* GetTensorRawData() const;
1696
1704
1712
1718
1727 void GetStringTensorElement(size_t buffer_length, size_t element_index, void* buffer) const;
1728
1735 std::string GetStringTensorElement(size_t element_index) const;
1736
1743 size_t GetStringTensorElementLength(size_t element_index) const;
1744
1751 size_t GetTensorSizeInBytes() const;
1752
1753#if !defined(DISABLE_SPARSE_TENSORS)
1761
1768
1777
1787 template <typename R>
1788 const R* GetSparseTensorIndicesData(OrtSparseIndicesFormat indices_format, size_t& num_indices) const;
1789
1794 bool IsSparseTensor() const;
1795
1804 template <typename R>
1805 const R* GetSparseTensorValues() const;
1806
1807#endif
1808};
1809
1810template <typename T>
1813 using B::B;
1814
1820 template <typename R>
1822
1828
1830 // Obtain a reference to an element of data at the location specified
1836 template <typename R>
1837 R& At(const std::vector<int64_t>& location);
1838
1844 void FillStringTensor(const char* const* s, size_t s_len);
1845
1851 void FillStringTensorElement(const char* s, size_t index);
1852
1865 char* GetResizedStringTensorElementBuffer(size_t index, size_t buffer_length);
1866
1867#if !defined(DISABLE_SPARSE_TENSORS)
1876 void UseCooIndices(int64_t* indices_data, size_t indices_num);
1877
1888 void UseCsrIndices(int64_t* inner_data, size_t inner_num, int64_t* outer_data, size_t outer_num);
1889
1898 void UseBlockSparseIndices(const Shape& indices_shape, int32_t* indices_data);
1899
1909 void FillSparseTensorCoo(const OrtMemoryInfo* data_mem_info, const OrtSparseValuesParam& values_param,
1910 const int64_t* indices_data, size_t indices_num);
1911
1923 void FillSparseTensorCsr(const OrtMemoryInfo* data_mem_info,
1924 const OrtSparseValuesParam& values,
1925 const int64_t* inner_indices_data, size_t inner_indices_num,
1926 const int64_t* outer_indices_data, size_t outer_indices_num);
1927
1938 const OrtSparseValuesParam& values,
1939 const Shape& indices_shape,
1940 const int32_t* indices_data);
1941
1942#endif
1943};
1944
1945} // namespace detail
1946
1949
1953struct Value : detail::ValueImpl<OrtValue> {
1955 using Base::Base;
1958
1959 explicit Value(std::nullptr_t) {}
1960 Value(Value&&) = default;
1961 Value& operator=(Value&&) = default;
1962
1963 ConstValue GetConst() const { return ConstValue{this->p_}; }
1964 UnownedValue GetUnowned() const { return UnownedValue{this->p_}; }
1965
1974 template <typename T>
1975 static Value CreateTensor(const OrtMemoryInfo* info, T* p_data, size_t p_data_element_count,
1976 const int64_t* shape, size_t shape_len);
1977
1987 static Value CreateTensor(const OrtMemoryInfo* info, void* p_data, size_t p_data_byte_count,
1988 const int64_t* shape, size_t shape_len,
1990
2000 static Value CreateTensor(OrtAllocator* deleter, void* p_data, size_t p_data_byte_count,
2001 const int64_t* shape, size_t shape_len,
2003
2015 template <typename T>
2016 static Value CreateTensor(OrtAllocator* allocator, const int64_t* shape, size_t shape_len);
2017
2029 static Value CreateTensor(OrtAllocator* allocator, const int64_t* shape, size_t shape_len,
2031
2040 static Value CreateMap(const Value& keys, const Value& values);
2041
2049 static Value CreateSequence(const std::vector<Value>& values);
2050
2059 template <typename T>
2060 static Value CreateOpaque(const char* domain, const char* type_name, const T& value);
2061
2062#if !defined(DISABLE_SPARSE_TENSORS)
2073 template <typename T>
2074 static Value CreateSparseTensor(const OrtMemoryInfo* info, T* p_data, const Shape& dense_shape,
2075 const Shape& values_shape);
2076
2093 static Value CreateSparseTensor(const OrtMemoryInfo* info, void* p_data, const Shape& dense_shape,
2094 const Shape& values_shape, ONNXTensorElementDataType type);
2095
2105 template <typename T>
2106 static Value CreateSparseTensor(OrtAllocator* allocator, const Shape& dense_shape);
2107
2119 static Value CreateSparseTensor(OrtAllocator* allocator, const Shape& dense_shape, ONNXTensorElementDataType type);
2120
2121#endif // !defined(DISABLE_SPARSE_TENSORS)
2122};
2123
2131 MemoryAllocation(OrtAllocator* allocator, void* p, size_t size);
2136 MemoryAllocation& operator=(MemoryAllocation&&) noexcept;
2137
2138 void* get() { return p_; }
2139 size_t size() const { return size_; }
2140
2141 private:
2142 OrtAllocator* allocator_;
2143 void* p_;
2144 size_t size_;
2145};
2146
2147namespace detail {
2148template <typename T>
2149struct AllocatorImpl : Base<T> {
2150 using B = Base<T>;
2151 using B::B;
2152
2153 void* Alloc(size_t size);
2154 MemoryAllocation GetAllocation(size_t size);
2155 void Free(void* p);
2156 ConstMemoryInfo GetInfo() const;
2157
2162 KeyValuePairs GetStats() const;
2163};
2164
2165} // namespace detail
2166
2170struct AllocatorWithDefaultOptions : detail::AllocatorImpl<detail::Unowned<OrtAllocator>> {
2171 explicit AllocatorWithDefaultOptions(std::nullptr_t) {}
2173};
2174
2178struct Allocator : detail::AllocatorImpl<OrtAllocator> {
2179 explicit Allocator(std::nullptr_t) {}
2180 Allocator(const Session& session, const OrtMemoryInfo*);
2181};
2182
2183using UnownedAllocator = detail::AllocatorImpl<detail::Unowned<OrtAllocator>>;
2184
2185namespace detail {
2186namespace binding_utils {
2187// Bring these out of template
2188std::vector<std::string> GetOutputNamesHelper(const OrtIoBinding* binding, OrtAllocator*);
2189std::vector<Value> GetOutputValuesHelper(const OrtIoBinding* binding, OrtAllocator*);
2190} // namespace binding_utils
2191
2192template <typename T>
2194 using B = Base<T>;
2195 using B::B;
2196
2197 std::vector<std::string> GetOutputNames() const;
2198 std::vector<std::string> GetOutputNames(OrtAllocator*) const;
2199 std::vector<Value> GetOutputValues() const;
2200 std::vector<Value> GetOutputValues(OrtAllocator*) const;
2201};
2202
2203template <typename T>
2206 using B::B;
2207
2208 void BindInput(const char* name, const Value&);
2209 void BindOutput(const char* name, const Value&);
2210 void BindOutput(const char* name, const OrtMemoryInfo*);
2215};
2216
2217} // namespace detail
2218
2221
2225struct IoBinding : detail::IoBindingImpl<OrtIoBinding> {
2226 explicit IoBinding(std::nullptr_t) {}
2227 explicit IoBinding(Session& session);
2228 ConstIoBinding GetConst() const { return ConstIoBinding{this->p_}; }
2229 UnownedIoBinding GetUnowned() const { return UnownedIoBinding{this->p_}; }
2230};
2231
2236struct ArenaCfg : detail::Base<OrtArenaCfg> {
2237 explicit ArenaCfg(std::nullptr_t) {}
2246 ArenaCfg(size_t max_mem, int arena_extend_strategy, int initial_chunk_size_bytes, int max_dead_bytes_per_chunk);
2247};
2248
2249//
2250// Custom OPs (only needed to implement custom OPs)
2251//
2252
2256struct OpAttr : detail::Base<OrtOpAttr> {
2258 using Base::Base;
2259
2260 explicit OpAttr(std::nullptr_t) {}
2261 OpAttr(const char* name, const void* data, int len, OrtOpAttrType type);
2262};
2263
2272#define ORT_CXX_LOG(logger, message_severity, message) \
2273 do { \
2274 if (message_severity >= logger.GetLoggingSeverityLevel()) { \
2275 Ort::ThrowOnError(logger.LogMessage(message_severity, ORT_FILE, __LINE__, \
2276 static_cast<const char*>(__FUNCTION__), message)); \
2277 } \
2278 } while (false)
2279
2288#define ORT_CXX_LOG_NOEXCEPT(logger, message_severity, message) \
2289 do { \
2290 if (message_severity >= logger.GetLoggingSeverityLevel()) { \
2291 static_cast<void>(logger.LogMessage(message_severity, ORT_FILE, __LINE__, \
2292 static_cast<const char*>(__FUNCTION__), message)); \
2293 } \
2294 } while (false)
2295
2307#define ORT_CXX_LOGF(logger, message_severity, /*format,*/...) \
2308 do { \
2309 if (message_severity >= logger.GetLoggingSeverityLevel()) { \
2310 Ort::ThrowOnError(logger.LogFormattedMessage(message_severity, ORT_FILE, __LINE__, \
2311 static_cast<const char*>(__FUNCTION__), __VA_ARGS__)); \
2312 } \
2313 } while (false)
2314
2326#define ORT_CXX_LOGF_NOEXCEPT(logger, message_severity, /*format,*/...) \
2327 do { \
2328 if (message_severity >= logger.GetLoggingSeverityLevel()) { \
2329 static_cast<void>(logger.LogFormattedMessage(message_severity, ORT_FILE, __LINE__, \
2330 static_cast<const char*>(__FUNCTION__), __VA_ARGS__)); \
2331 } \
2332 } while (false)
2333
2344struct Logger {
2348 Logger() = default;
2349
2353 explicit Logger(std::nullptr_t) {}
2354
2361 explicit Logger(const OrtLogger* logger);
2362
2363 ~Logger() = default;
2364
2365 Logger(const Logger&) = default;
2366 Logger& operator=(const Logger&) = default;
2367
2368 Logger(Logger&& v) noexcept = default;
2369 Logger& operator=(Logger&& v) noexcept = default;
2370
2377
2390 Status LogMessage(OrtLoggingLevel log_severity_level, const ORTCHAR_T* file_path, int line_number,
2391 const char* func_name, const char* message) const noexcept;
2392
2407 template <typename... Args>
2408 Status LogFormattedMessage(OrtLoggingLevel log_severity_level, const ORTCHAR_T* file_path, int line_number,
2409 const char* func_name, const char* format, Args&&... args) const noexcept;
2410
2411 private:
2412 const OrtLogger* logger_{};
2413 OrtLoggingLevel cached_severity_level_{};
2414};
2415
2424 size_t GetInputCount() const;
2425 size_t GetOutputCount() const;
2426 // If input is optional and is not present, the method returns an empty ConstValue
2427 // which can be compared to nullptr.
2428 ConstValue GetInput(size_t index) const;
2429 // If output is optional and is not present, the method returns an empty UnownedValue
2430 // which can be compared to nullptr.
2431 UnownedValue GetOutput(size_t index, const int64_t* dim_values, size_t dim_count) const;
2432 UnownedValue GetOutput(size_t index, const std::vector<int64_t>& dims) const;
2433 void* GetGPUComputeStream() const;
2435 OrtAllocator* GetAllocator(const OrtMemoryInfo& memory_info) const;
2436 OrtKernelContext* GetOrtKernelContext() const { return ctx_; }
2437 void ParallelFor(void (*fn)(void*, size_t), size_t total, size_t num_batch, void* usr_data) const;
2438
2439 private:
2440 OrtKernelContext* ctx_;
2441};
2442
2443struct KernelInfo;
2444
2445namespace detail {
2446namespace attr_utils {
2447void GetAttr(const OrtKernelInfo* p, const char* name, float&);
2448void GetAttr(const OrtKernelInfo* p, const char* name, int64_t&);
2449void GetAttr(const OrtKernelInfo* p, const char* name, std::string&);
2450void GetAttrs(const OrtKernelInfo* p, const char* name, std::vector<float>&);
2451void GetAttrs(const OrtKernelInfo* p, const char* name, std::vector<int64_t>&);
2452} // namespace attr_utils
2453
2454template <typename T>
2455struct KernelInfoImpl : Base<T> {
2456 using B = Base<T>;
2457 using B::B;
2458
2459 KernelInfo Copy() const;
2460
2461 template <typename R> // R is only implemented for float, int64_t, and string
2462 R GetAttribute(const char* name) const {
2463 R val;
2464 attr_utils::GetAttr(this->p_, name, val);
2465 return val;
2466 }
2467
2468 template <typename R> // R is only implemented for std::vector<float>, std::vector<int64_t>
2469 std::vector<R> GetAttributes(const char* name) const {
2470 std::vector<R> result;
2471 attr_utils::GetAttrs(this->p_, name, result);
2472 return result;
2473 }
2474
2475 Value GetTensorAttribute(const char* name, OrtAllocator* allocator) const;
2476
2477 size_t GetInputCount() const;
2478 size_t GetOutputCount() const;
2479
2480 std::string GetInputName(size_t index) const;
2481 std::string GetOutputName(size_t index) const;
2482
2483 TypeInfo GetInputTypeInfo(size_t index) const;
2484 TypeInfo GetOutputTypeInfo(size_t index) const;
2485
2486 ConstValue GetTensorConstantInput(size_t index, int* is_constant) const;
2487
2488 std::string GetNodeName() const;
2489 Logger GetLogger() const;
2490};
2491
2492} // namespace detail
2493
2494using ConstKernelInfo = detail::KernelInfoImpl<detail::Unowned<const OrtKernelInfo>>;
2495
2502struct KernelInfo : detail::KernelInfoImpl<OrtKernelInfo> {
2503 using Base = detail::KernelInfoImpl<OrtKernelInfo>;
2504 using Base::Base;
2505 explicit KernelInfo(std::nullptr_t) {}
2506 explicit KernelInfo(OrtKernelInfo* info);
2507 ConstKernelInfo GetConst() const { return ConstKernelInfo{this->p_}; }
2508};
2509
2513struct Op : detail::Base<OrtOp> {
2515 using Base::Base;
2516
2517 explicit Op(std::nullptr_t) {}
2518
2519 explicit Op(OrtOp*);
2520
2521 static Op Create(const OrtKernelInfo* info, const char* op_name, const char* domain,
2522 int version, const char** type_constraint_names,
2523 const ONNXTensorElementDataType* type_constraint_values,
2524 size_t type_constraint_count,
2525 const OpAttr* attr_values,
2526 size_t attr_count,
2527 size_t input_count, size_t output_count);
2528
2529 void Invoke(const OrtKernelContext* context,
2530 const Value* input_values,
2531 size_t input_count,
2532 Value* output_values,
2533 size_t output_count);
2534
2535 // For easier refactoring
2536 void Invoke(const OrtKernelContext* context,
2537 const OrtValue* const* input_values,
2538 size_t input_count,
2539 OrtValue* const* output_values,
2540 size_t output_count);
2541};
2542
2548 SymbolicInteger(int64_t i) : i_(i), is_int_(true) {};
2549 SymbolicInteger(const char* s) : s_(s), is_int_(false) {};
2552
2555
2556 bool operator==(const SymbolicInteger& dim) const {
2557 if (is_int_ == dim.is_int_) {
2558 if (is_int_) {
2559 return i_ == dim.i_;
2560 } else {
2561 return std::string{s_} == std::string{dim.s_};
2562 }
2563 }
2564 return false;
2565 }
2566
2567 bool IsInt() const { return is_int_; }
2568 int64_t AsInt() const { return i_; }
2569 const char* AsSym() const { return s_; }
2570
2571 static constexpr int INVALID_INT_DIM = -2;
2572
2573 private:
2574 union {
2575 int64_t i_;
2576 const char* s_;
2577 };
2578 bool is_int_;
2579 };
2580
2581 using Shape = std::vector<SymbolicInteger>;
2582
2584
2585 const Shape& GetInputShape(size_t indice) const { return input_shapes_.at(indice); }
2586
2587 size_t GetInputCount() const { return input_shapes_.size(); }
2588
2590
2591 int64_t GetAttrInt(const char* attr_name);
2592
2593 using Ints = std::vector<int64_t>;
2594 Ints GetAttrInts(const char* attr_name);
2595
2596 float GetAttrFloat(const char* attr_name);
2597
2598 using Floats = std::vector<float>;
2599 Floats GetAttrFloats(const char* attr_name);
2600
2601 std::string GetAttrString(const char* attr_name);
2602
2603 using Strings = std::vector<std::string>;
2604 Strings GetAttrStrings(const char* attr_name);
2605
2606 private:
2607 const OrtOpAttr* GetAttrHdl(const char* attr_name) const;
2608 const OrtApi* ort_api_;
2610 std::vector<Shape> input_shapes_;
2611};
2612
2614
2615#define MAX_CUSTOM_OP_END_VER (1UL << 31) - 1
2616
2617template <typename TOp, typename TKernel, bool WithStatus = false>
2621 OrtCustomOp::GetName = [](const OrtCustomOp* this_) { return static_cast<const TOp*>(this_)->GetName(); };
2622
2623 OrtCustomOp::GetExecutionProviderType = [](const OrtCustomOp* this_) { return static_cast<const TOp*>(this_)->GetExecutionProviderType(); };
2624
2625 OrtCustomOp::GetInputTypeCount = [](const OrtCustomOp* this_) { return static_cast<const TOp*>(this_)->GetInputTypeCount(); };
2626 OrtCustomOp::GetInputType = [](const OrtCustomOp* this_, size_t index) { return static_cast<const TOp*>(this_)->GetInputType(index); };
2627 OrtCustomOp::GetInputMemoryType = [](const OrtCustomOp* this_, size_t index) { return static_cast<const TOp*>(this_)->GetInputMemoryType(index); };
2628
2629 OrtCustomOp::GetOutputTypeCount = [](const OrtCustomOp* this_) { return static_cast<const TOp*>(this_)->GetOutputTypeCount(); };
2630 OrtCustomOp::GetOutputType = [](const OrtCustomOp* this_, size_t index) { return static_cast<const TOp*>(this_)->GetOutputType(index); };
2631
2632#if defined(_MSC_VER) && !defined(__clang__)
2633#pragma warning(push)
2634#pragma warning(disable : 26409)
2635#endif
2636 OrtCustomOp::KernelDestroy = [](void* op_kernel) { delete static_cast<TKernel*>(op_kernel); };
2637#if defined(_MSC_VER) && !defined(__clang__)
2638#pragma warning(pop)
2639#endif
2640 OrtCustomOp::GetInputCharacteristic = [](const OrtCustomOp* this_, size_t index) { return static_cast<const TOp*>(this_)->GetInputCharacteristic(index); };
2641 OrtCustomOp::GetOutputCharacteristic = [](const OrtCustomOp* this_, size_t index) { return static_cast<const TOp*>(this_)->GetOutputCharacteristic(index); };
2642
2643 OrtCustomOp::GetVariadicInputMinArity = [](const OrtCustomOp* this_) { return static_cast<const TOp*>(this_)->GetVariadicInputMinArity(); };
2644 OrtCustomOp::GetVariadicInputHomogeneity = [](const OrtCustomOp* this_) { return static_cast<int>(static_cast<const TOp*>(this_)->GetVariadicInputHomogeneity()); };
2645 OrtCustomOp::GetVariadicOutputMinArity = [](const OrtCustomOp* this_) { return static_cast<const TOp*>(this_)->GetVariadicOutputMinArity(); };
2646 OrtCustomOp::GetVariadicOutputHomogeneity = [](const OrtCustomOp* this_) { return static_cast<int>(static_cast<const TOp*>(this_)->GetVariadicOutputHomogeneity()); };
2647#ifdef __cpp_if_constexpr
2648 if constexpr (WithStatus) {
2649#else
2650 if (WithStatus) {
2651#endif
2652 OrtCustomOp::CreateKernelV2 = [](const OrtCustomOp* this_, const OrtApi* api, const OrtKernelInfo* info, void** op_kernel) -> OrtStatusPtr {
2653 return static_cast<const TOp*>(this_)->CreateKernelV2(*api, info, op_kernel);
2654 };
2655 OrtCustomOp::KernelComputeV2 = [](void* op_kernel, OrtKernelContext* context) -> OrtStatusPtr {
2656 return static_cast<TKernel*>(op_kernel)->ComputeV2(context);
2657 };
2658 } else {
2661
2662 OrtCustomOp::CreateKernel = [](const OrtCustomOp* this_, const OrtApi* api, const OrtKernelInfo* info) { return static_cast<const TOp*>(this_)->CreateKernel(*api, info); };
2663 OrtCustomOp::KernelCompute = [](void* op_kernel, OrtKernelContext* context) {
2664 static_cast<TKernel*>(op_kernel)->Compute(context);
2665 };
2666 }
2667
2668 SetShapeInferFn<TOp>(0);
2669
2670 OrtCustomOp::GetStartVersion = [](const OrtCustomOp* this_) {
2671 return static_cast<const TOp*>(this_)->start_ver_;
2672 };
2673
2674 OrtCustomOp::GetEndVersion = [](const OrtCustomOp* this_) {
2675 return static_cast<const TOp*>(this_)->end_ver_;
2676 };
2677
2680 OrtCustomOp::GetAliasMap = nullptr;
2682 }
2683
2684 // Default implementation of GetExecutionProviderType that returns nullptr to default to the CPU provider
2685 const char* GetExecutionProviderType() const { return nullptr; }
2686
2687 // Default implementations of GetInputCharacteristic() and GetOutputCharacteristic() below
2688 // (inputs and outputs are required by default)
2690 return OrtCustomOpInputOutputCharacteristic::INPUT_OUTPUT_REQUIRED;
2691 }
2692
2694 return OrtCustomOpInputOutputCharacteristic::INPUT_OUTPUT_REQUIRED;
2695 }
2696
2697 // Default implementation of GetInputMemoryType() that returns OrtMemTypeDefault
2698 OrtMemType GetInputMemoryType(size_t /*index*/) const {
2699 return OrtMemTypeDefault;
2700 }
2701
2702 // Default implementation of GetVariadicInputMinArity() returns 1 to specify that a variadic input
2703 // should expect at least 1 argument.
2705 return 1;
2706 }
2707
2708 // Default implementation of GetVariadicInputHomegeneity() returns true to specify that all arguments
2709 // to a variadic input should be of the same type.
2711 return true;
2712 }
2713
2714 // Default implementation of GetVariadicOutputMinArity() returns 1 to specify that a variadic output
2715 // should produce at least 1 output value.
2717 return 1;
2718 }
2719
2720 // Default implementation of GetVariadicOutputHomegeneity() returns true to specify that all output values
2721 // produced by a variadic output should be of the same type.
2723 return true;
2724 }
2725
2726 // Declare list of session config entries used by this Custom Op.
2727 // Implement this function in order to get configs from CustomOpBase::GetSessionConfigs().
2728 // This default implementation returns an empty vector of config entries.
2729 std::vector<std::string> GetSessionConfigKeys() const {
2730 return std::vector<std::string>{};
2731 }
2732
2733 // Ort::CustomOpBase derived class should provide the following static method with the type/shape inferencing
2734 // implementation if needed:
2735 // static OrtStatusPtr InferOutputShape(Ort::ShapeInferContext& context)
2736 template <typename C>
2737 decltype(&C::InferOutputShape) SetShapeInferFn(decltype(&C::InferOutputShape)) {
2739 ShapeInferContext ctx(&GetApi(), ort_ctx);
2740 return C::InferOutputShape(ctx);
2741 };
2742 return {};
2743 }
2744
2745 template <typename C>
2749
2750 protected:
2751 // Helper function that returns a map of session config entries specified by CustomOpBase::GetSessionConfigKeys.
2752 void GetSessionConfigs(std::unordered_map<std::string, std::string>& out, ConstSessionOptions options) const;
2753
2754 int start_ver_ = 1;
2755 int end_ver_ = MAX_CUSTOM_OP_END_VER;
2756};
2757
2758namespace detail {
2759template <typename T>
2762 using B::B;
2763
2764 std::string Name() const;
2766};
2767} // namespace detail
2768
2769// Const object holder that does not own the underlying object
2771
2775struct ValueInfo : detail::ValueInfoImpl<OrtValueInfo> {
2776 explicit ValueInfo(std::nullptr_t) {}
2778 explicit ValueInfo(OrtValueInfo* p) : ValueInfoImpl<OrtValueInfo>{p} {}
2779
2780 // Create ValueInfo for a tensor
2781 explicit ValueInfo(const std::string& name, const ConstTypeInfo& type_info);
2782
2783 ConstValueInfo GetConst() const { return ConstValueInfo{this->p_}; }
2784};
2785
2786namespace detail {
2787template <typename T>
2790 using B::B;
2791};
2792} // namespace detail
2793
2797struct Node : detail::NodeImpl<OrtNode> {
2798 explicit Node(std::nullptr_t) {}
2799 explicit Node(OrtNode* p) : NodeImpl<OrtNode>{p} {}
2800
2801#if !defined(ORT_MINIMAL_BUILD)
2802 Node(const std::string& operator_name, const std::string& operator_domain,
2803 const std::string& node_name,
2804 const std::vector<std::string>& input_names,
2805 const std::vector<std::string>& output_names);
2806
2810 Node(const std::string& operator_name, const std::string& operator_domain,
2811 const std::string& node_name,
2812 const std::vector<std::string>& input_names,
2813 const std::vector<std::string>& output_names,
2814 std::vector<OpAttr>& attributes);
2815
2816 private:
2817 static void Init(const std::string& operator_name, const std::string& operator_domain,
2818 const std::string& node_name,
2819 const std::vector<std::string>& input_names,
2820 const std::vector<std::string>& output_names,
2821 std::vector<OpAttr>& attributes,
2822 OrtNode*& node);
2823#endif // !defined(ORT_MINIMAL_BUILD)
2824};
2825
2826namespace detail {
2827template <typename T>
2828struct GraphImpl : Ort::detail::Base<T> {
2829 using B = Ort::detail::Base<T>;
2830 using B::B;
2831
2832#if !defined(ORT_MINIMAL_BUILD)
2833 void SetInputs(std::vector<ValueInfo>& inputs);
2834 void SetOutputs(std::vector<ValueInfo>& outputs);
2835 void AddInitializer(const std::string& name, Value& initializer, bool data_is_external); // Graph takes ownership of Value
2836 void AddNode(Node& node); // Graph takes ownership of Node
2837#endif // !defined(ORT_MINIMAL_BUILD)
2838};
2839} // namespace detail
2840
2844struct Graph : detail::GraphImpl<OrtGraph> {
2845 explicit Graph(std::nullptr_t) {}
2846 explicit Graph(OrtGraph* p) : GraphImpl<OrtGraph>{p} {}
2847#if !defined(ORT_MINIMAL_BUILD)
2849#endif
2850};
2851
2852namespace detail {
2853template <typename T>
2856 using B::B;
2857
2858#if !defined(ORT_MINIMAL_BUILD)
2859 void AddGraph(Graph& graph);
2860#endif
2861};
2862} // namespace detail
2863
2864// Const object holder that does not own the underlying object
2866
2870struct Model : detail::ModelImpl<OrtModel> {
2871 using DomainOpsetPair = std::pair<std::string, int>;
2872
2873 explicit Model(std::nullptr_t) {}
2874 explicit Model(OrtModel* p) : ModelImpl<OrtModel>{p} {}
2875
2876#if !defined(ORT_MINIMAL_BUILD)
2877 explicit Model(const std::vector<DomainOpsetPair>& opsets);
2878#endif
2879
2880 ConstModel GetConst() const { return ConstModel{this->p_}; }
2881};
2882} // namespace Ort
2883#include "onnxruntime_cxx_inline.h"
struct OrtMemoryInfo OrtMemoryInfo
Definition onnxruntime_c_api.h:293
struct OrtKernelInfo OrtKernelInfo
Definition onnxruntime_c_api.h:447
struct OrtNode OrtNode
Definition onnxruntime_c_api.h:321
OrtLoggingLevel
Logging severity levels.
Definition onnxruntime_c_api.h:244
OrtMemoryInfoDeviceType
This mimics OrtDevice type constants so they can be returned in the API.
Definition onnxruntime_c_api.h:482
struct OrtShapeInferContext OrtShapeInferContext
Definition onnxruntime_c_api.h:318
void(* OrtLoggingFunction)(void *param, OrtLoggingLevel severity, const char *category, const char *logid, const char *code_location, const char *message)
Definition onnxruntime_c_api.h:411
void(* OrtCustomJoinThreadFn)(OrtCustomThreadHandle ort_custom_thread_handle)
Custom thread join function.
Definition onnxruntime_c_api.h:883
OrtCustomOpInputOutputCharacteristic
Definition onnxruntime_c_api.h:6473
struct OrtTensorRTProviderOptionsV2 OrtTensorRTProviderOptionsV2
Definition onnxruntime_c_api.h:310
struct OrtEpApi OrtEpApi
Definition onnxruntime_c_api.h:827
struct OrtOpAttr OrtOpAttr
Definition onnxruntime_c_api.h:316
struct OrtThreadingOptions OrtThreadingOptions
Definition onnxruntime_c_api.h:307
struct OrtSequenceTypeInfo OrtSequenceTypeInfo
Definition onnxruntime_c_api.h:301
struct OrtValueInfo OrtValueInfo
Definition onnxruntime_c_api.h:320
struct OrtDnnlProviderOptions OrtDnnlProviderOptions
Definition onnxruntime_c_api.h:314
OrtSparseIndicesFormat
Definition onnxruntime_c_api.h:233
struct OrtPrepackedWeightsContainer OrtPrepackedWeightsContainer
Definition onnxruntime_c_api.h:309
struct OrtSession OrtSession
Definition onnxruntime_c_api.h:295
OrtStatus *(* EpSelectionDelegate)(const OrtEpDevice **ep_devices, size_t num_devices, const OrtKeyValuePairs *model_metadata, const OrtKeyValuePairs *runtime_metadata, const OrtEpDevice **selected, size_t max_selected, size_t *num_selected, void *state)
Delegate to allow providing custom OrtEpDevice selection logic.
Definition onnxruntime_c_api.h:526
struct OrtCustomOpDomain OrtCustomOpDomain
Definition onnxruntime_c_api.h:304
struct OrtIoBinding OrtIoBinding
Definition onnxruntime_c_api.h:294
OrtAllocatorType
Definition onnxruntime_c_api.h:453
struct OrtOp OrtOp
Definition onnxruntime_c_api.h:315
struct OrtTypeInfo OrtTypeInfo
Definition onnxruntime_c_api.h:298
struct OrtTensorTypeAndShapeInfo OrtTensorTypeAndShapeInfo
Definition onnxruntime_c_api.h:299
struct OrtCUDAProviderOptionsV2 OrtCUDAProviderOptionsV2
Definition onnxruntime_c_api.h:312
struct OrtKernelContext OrtKernelContext
Definition onnxruntime_c_api.h:449
struct OrtCANNProviderOptions OrtCANNProviderOptions
Definition onnxruntime_c_api.h:313
struct OrtEpDevice OrtEpDevice
Definition onnxruntime_c_api.h:326
void(* RunAsyncCallbackFn)(void *user_data, OrtValue **outputs, size_t num_outputs, OrtStatusPtr status)
Callback function for RunAsync.
Definition onnxruntime_c_api.h:894
OrtHardwareDeviceType
Definition onnxruntime_c_api.h:489
struct OrtModel OrtModel
Definition onnxruntime_c_api.h:323
struct OrtGraph OrtGraph
Definition onnxruntime_c_api.h:322
struct OrtSessionOptions OrtSessionOptions
Definition onnxruntime_c_api.h:303
struct OrtValue OrtValue
Definition onnxruntime_c_api.h:296
GraphOptimizationLevel
Graph optimization level.
Definition onnxruntime_c_api.h:420
struct OrtKeyValuePairs OrtKeyValuePairs
Definition onnxruntime_c_api.h:327
OrtStatus * OrtStatusPtr
Definition onnxruntime_c_api.h:334
OrtMemType
Memory types for allocated memory, execution provider specific types should be extended in each provi...
Definition onnxruntime_c_api.h:463
OrtSparseFormat
Definition onnxruntime_c_api.h:225
ONNXType
Definition onnxruntime_c_api.h:213
struct OrtEnv OrtEnv
Definition onnxruntime_c_api.h:291
OrtErrorCode
Definition onnxruntime_c_api.h:252
struct OrtStatus OrtStatus
Definition onnxruntime_c_api.h:292
#define ORT_API_VERSION
The API version defined in this header.
Definition onnxruntime_c_api.h:41
struct OrtLogger OrtLogger
Definition onnxruntime_c_api.h:317
struct OrtMapTypeInfo OrtMapTypeInfo
Definition onnxruntime_c_api.h:300
struct OrtArenaCfg OrtArenaCfg
Definition onnxruntime_c_api.h:308
ExecutionMode
Definition onnxruntime_c_api.h:428
OrtOpAttrType
Definition onnxruntime_c_api.h:270
OrtCustomThreadHandle(* OrtCustomCreateThreadFn)(void *ort_custom_thread_creation_options, OrtThreadWorkerFn ort_thread_worker_fn, void *ort_worker_fn_param)
Ort custom thread creation function.
Definition onnxruntime_c_api.h:876
ONNXTensorElementDataType
Definition onnxruntime_c_api.h:184
OrtExecutionProviderDevicePolicy
These are the default EP selection policies used by ORT when doing automatic EP selection.
Definition onnxruntime_c_api.h:497
const OrtApiBase * OrtGetApiBase(void)
The Onnxruntime library's entry point to access the C API.
@ ORT_LOGGING_LEVEL_WARNING
Warning messages.
Definition onnxruntime_c_api.h:247
@ OrtMemTypeDefault
The default allocator for execution provider.
Definition onnxruntime_c_api.h:471
@ ORT_FAIL
Definition onnxruntime_c_api.h:254
@ ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT
Definition onnxruntime_c_api.h:186
std::vector< Value > GetOutputValuesHelper(const OrtIoBinding *binding, OrtAllocator *)
std::vector< std::string > GetOutputNamesHelper(const OrtIoBinding *binding, OrtAllocator *)
void OrtRelease(OrtAllocator *ptr)
Definition onnxruntime_cxx_api.h:551
std::string MakeCustomOpConfigEntryKey(const char *custom_op_name, const char *config)
All C++ Onnxruntime APIs are defined inside this namespace.
Definition onnxruntime_cxx_api.h:48
const OrtModelEditorApi & GetModelEditorApi()
This returns a reference to the ORT C Model Editor API. Used if building or augmenting a model at run...
Definition onnxruntime_cxx_api.h:151
std::unique_ptr< char, detail::AllocatedFree > AllocatedStringPtr
unique_ptr typedef used to own strings allocated by OrtAllocators and release them at the end of the ...
Definition onnxruntime_cxx_api.h:707
detail::ConstSessionOptionsImpl< detail::Unowned< const OrtSessionOptions > > ConstSessionOptions
Definition onnxruntime_cxx_api.h:1129
detail::KernelInfoImpl< detail::Unowned< const OrtKernelInfo > > ConstKernelInfo
Definition onnxruntime_cxx_api.h:2494
const OrtApi & GetApi() noexcept
This returns a reference to the ORT C API.
Definition onnxruntime_cxx_api.h:125
const OrtCompileApi & GetCompileApi()
This returns a reference to the ORT C Compile API. Used if compiling a model at runtime.
Definition onnxruntime_cxx_api.h:165
detail::AllocatorImpl< detail::Unowned< OrtAllocator > > UnownedAllocator
Definition onnxruntime_cxx_api.h:2183
detail::SessionOptionsImpl< detail::Unowned< OrtSessionOptions > > UnownedSessionOptions
Definition onnxruntime_cxx_api.h:1128
std::string GetBuildInfoString()
This function returns the onnxruntime build information: including git branch, git commit id,...
const OrtEpApi & GetEpApi()
This returns a reference to the ORT C EP API. Used if authoring a plugin execution provider.
Definition onnxruntime_cxx_api.h:179
std::string GetVersionString()
This function returns the onnxruntime version string.
std::vector< std::string > GetAvailableProviders()
This is a C++ wrapper for OrtApi::GetAvailableProviders() and returns a vector of strings representin...
Ort::Status(*)(Ort::ShapeInferContext &) ShapeInferFn
Definition onnxruntime_cxx_api.h:2613
Status CompileModel(const Env &env, const ModelCompilationOptions &model_compilation_options)
Compiles an input model to generate a model with EPContext nodes that execute EP-specific kernels....
Wrapper around OrtAllocator.
Definition onnxruntime_cxx_api.h:2178
Allocator(const Session &session, const OrtMemoryInfo *)
Allocator(std::nullptr_t)
Convenience to create a class member and then replace with an instance.
Definition onnxruntime_cxx_api.h:2179
Wrapper around OrtAllocator default instance that is owned by Onnxruntime.
Definition onnxruntime_cxx_api.h:2170
AllocatorWithDefaultOptions(std::nullptr_t)
Convenience to create a class member and then replace with an instance.
Definition onnxruntime_cxx_api.h:2171
it is a structure that represents the configuration of an arena based allocator
Definition onnxruntime_cxx_api.h:2236
ArenaCfg(std::nullptr_t)
Create an empty ArenaCfg object, must be assigned a valid one to be used.
Definition onnxruntime_cxx_api.h:2237
ArenaCfg(size_t max_mem, int arena_extend_strategy, int initial_chunk_size_bytes, int max_dead_bytes_per_chunk)
bfloat16 (Brain Floating Point) data type
Definition onnxruntime_cxx_api.h:349
bool operator==(const BFloat16_t &rhs) const noexcept
onnxruntime_float16::BFloat16Impl< BFloat16_t > Base
Definition onnxruntime_cxx_api.h:361
BFloat16_t()=default
static constexpr BFloat16_t FromBits(uint16_t v) noexcept
Explicit conversion to uint16_t representation of bfloat16.
Definition onnxruntime_cxx_api.h:370
bool operator!=(const BFloat16_t &rhs) const noexcept
Definition onnxruntime_cxx_api.h:468
BFloat16_t(float v) noexcept
__ctor from float. Float is converted into bfloat16 16-bit representation.
Definition onnxruntime_cxx_api.h:376
float ToFloat() const noexcept
Converts bfloat16 to float.
Definition onnxruntime_cxx_api.h:382
bool operator<(const BFloat16_t &rhs) const noexcept
Definition onnxruntime_cxx_api.h:2618
OrtCustomOpInputOutputCharacteristic GetOutputCharacteristic(size_t) const
Definition onnxruntime_cxx_api.h:2693
OrtCustomOpInputOutputCharacteristic GetInputCharacteristic(size_t) const
Definition onnxruntime_cxx_api.h:2689
OrtMemType GetInputMemoryType(size_t) const
Definition onnxruntime_cxx_api.h:2698
std::vector< std::string > GetSessionConfigKeys() const
Definition onnxruntime_cxx_api.h:2729
bool GetVariadicInputHomogeneity() const
Definition onnxruntime_cxx_api.h:2710
int GetVariadicInputMinArity() const
Definition onnxruntime_cxx_api.h:2704
void SetShapeInferFn(...)
Definition onnxruntime_cxx_api.h:2746
CustomOpBase()
Definition onnxruntime_cxx_api.h:2619
bool GetVariadicOutputHomogeneity() const
Definition onnxruntime_cxx_api.h:2722
int GetVariadicOutputMinArity() const
Definition onnxruntime_cxx_api.h:2716
decltype(&C::InferOutputShape) SetShapeInferFn(decltype(&C::InferOutputShape))
Definition onnxruntime_cxx_api.h:2737
const char * GetExecutionProviderType() const
Definition onnxruntime_cxx_api.h:2685
void GetSessionConfigs(std::unordered_map< std::string, std::string > &out, ConstSessionOptions options) const
Class that represents session configuration entries for one or more custom operators.
Definition onnxruntime_cxx_api.h:984
~CustomOpConfigs()=default
CustomOpConfigs & AddConfig(const char *custom_op_name, const char *config_key, const char *config_value)
Adds a session configuration entry/value for a specific custom operator.
CustomOpConfigs & operator=(CustomOpConfigs &&o)=default
CustomOpConfigs(CustomOpConfigs &&o)=default
CustomOpConfigs()=default
const std::unordered_map< std::string, std::string > & GetFlattenedConfigs() const
Returns a flattened map of custom operator configuration entries and their values.
CustomOpConfigs(const CustomOpConfigs &)=default
CustomOpConfigs & operator=(const CustomOpConfigs &)=default
Custom Op Domain.
Definition onnxruntime_cxx_api.h:889
CustomOpDomain(std::nullptr_t)
Create an empty CustomOpDomain object, must be assigned a valid one to be used.
Definition onnxruntime_cxx_api.h:893
CustomOpDomain(const char *domain)
Wraps OrtApi::CreateCustomOpDomain.
void Add(const OrtCustomOp *op)
Wraps CustomOpDomain_Add.
The Env (Environment)
Definition onnxruntime_cxx_api.h:850
Env & EnableTelemetryEvents()
Wraps OrtApi::EnableTelemetryEvents.
Env(OrtEnv *p)
C Interop Helper.
Definition onnxruntime_cxx_api.h:867
Env & CreateAndRegisterAllocatorV2(const std::string &provider_type, const OrtMemoryInfo *mem_info, const std::unordered_map< std::string, std::string > &options, const OrtArenaCfg *arena_cfg)
Wraps OrtApi::CreateAndRegisterAllocatorV2.
Env & UnregisterExecutionProviderLibrary(const char *registration_name)
Wraps OrtApi::UnregisterExecutionProviderLibrary.
std::vector< ConstEpDevice > GetEpDevices() const
Env(std::nullptr_t)
Create an empty Env object, must be assigned a valid one to be used.
Definition onnxruntime_cxx_api.h:851
Env(OrtLoggingLevel logging_level=ORT_LOGGING_LEVEL_WARNING, const char *logid="")
Wraps OrtApi::CreateEnv.
Env(const OrtThreadingOptions *tp_options, OrtLoggingLevel logging_level=ORT_LOGGING_LEVEL_WARNING, const char *logid="")
Wraps OrtApi::CreateEnvWithGlobalThreadPools.
Env(const OrtThreadingOptions *tp_options, OrtLoggingFunction logging_function, void *logger_param, OrtLoggingLevel logging_level=ORT_LOGGING_LEVEL_WARNING, const char *logid="")
Wraps OrtApi::CreateEnvWithCustomLoggerAndGlobalThreadPools.
Env(OrtLoggingLevel logging_level, const char *logid, OrtLoggingFunction logging_function, void *logger_param)
Wraps OrtApi::CreateEnvWithCustomLogger.
Env & CreateAndRegisterAllocator(const OrtMemoryInfo *mem_info, const OrtArenaCfg *arena_cfg)
Wraps OrtApi::CreateAndRegisterAllocator.
Env & RegisterExecutionProviderLibrary(const char *registration_name, const std::basic_string< char > &path)
Wraps OrtApi::RegisterExecutionProviderLibrary.
Env & UpdateEnvWithCustomLogLevel(OrtLoggingLevel log_severity_level)
Wraps OrtApi::UpdateEnvWithCustomLogLevel.
Env & DisableTelemetryEvents()
Wraps OrtApi::DisableTelemetryEvents.
Mutable EpDevice that is created by EpApi users.
Definition onnxruntime_cxx_api.h:836
EpDevice(OrtEpDevice *p)
Take ownership of a pointer created by C API.
Definition onnxruntime_cxx_api.h:838
EpDevice(OrtEpFactory &ep_factory, ConstHardwareDevice &hardware_device, ConstKeyValuePairs ep_metadata={}, ConstKeyValuePairs ep_options={})
Wraps OrtEpApi::CreateEpDevice.
EpDevice(std::nullptr_t)
No instance is created.
Definition onnxruntime_cxx_api.h:837
All C++ methods that can fail will throw an exception of this type.
Definition onnxruntime_cxx_api.h:54
const char * what() const noexcept override
Definition onnxruntime_cxx_api.h:58
OrtErrorCode GetOrtErrorCode() const
Definition onnxruntime_cxx_api.h:57
Exception(std::string &&string, OrtErrorCode code)
Definition onnxruntime_cxx_api.h:55
IEEE 754 half-precision floating point data type.
Definition onnxruntime_cxx_api.h:207
Float16_t()=default
Default constructor.
Float16_t(float v) noexcept
__ctor from float. Float is converted into float16 16-bit representation.
Definition onnxruntime_cxx_api.h:235
onnxruntime_float16::Float16Impl< Float16_t > Base
Definition onnxruntime_cxx_api.h:217
float ToFloat() const noexcept
Converts float16 to float.
Definition onnxruntime_cxx_api.h:241
static constexpr Float16_t FromBits(uint16_t v) noexcept
Explicit conversion to uint16_t representation of float16.
Definition onnxruntime_cxx_api.h:229
float8e4m3fn (Float8 Floating Point) data type
Definition onnxruntime_cxx_api.h:479
uint8_t value
Definition onnxruntime_cxx_api.h:480
constexpr Float8E4M3FN_t(uint8_t v) noexcept
Definition onnxruntime_cxx_api.h:482
constexpr bool operator==(const Float8E4M3FN_t &rhs) const noexcept
Definition onnxruntime_cxx_api.h:485
constexpr Float8E4M3FN_t() noexcept
Definition onnxruntime_cxx_api.h:481
constexpr bool operator!=(const Float8E4M3FN_t &rhs) const noexcept
Definition onnxruntime_cxx_api.h:486
float8e4m3fnuz (Float8 Floating Point) data type
Definition onnxruntime_cxx_api.h:496
constexpr bool operator==(const Float8E4M3FNUZ_t &rhs) const noexcept
Definition onnxruntime_cxx_api.h:502
uint8_t value
Definition onnxruntime_cxx_api.h:497
constexpr Float8E4M3FNUZ_t() noexcept
Definition onnxruntime_cxx_api.h:498
constexpr bool operator!=(const Float8E4M3FNUZ_t &rhs) const noexcept
Definition onnxruntime_cxx_api.h:503
constexpr Float8E4M3FNUZ_t(uint8_t v) noexcept
Definition onnxruntime_cxx_api.h:499
float8e5m2 (Float8 Floating Point) data type
Definition onnxruntime_cxx_api.h:513
constexpr Float8E5M2_t(uint8_t v) noexcept
Definition onnxruntime_cxx_api.h:516
uint8_t value
Definition onnxruntime_cxx_api.h:514
constexpr bool operator!=(const Float8E5M2_t &rhs) const noexcept
Definition onnxruntime_cxx_api.h:520
constexpr Float8E5M2_t() noexcept
Definition onnxruntime_cxx_api.h:515
constexpr bool operator==(const Float8E5M2_t &rhs) const noexcept
Definition onnxruntime_cxx_api.h:519
float8e5m2fnuz (Float8 Floating Point) data type
Definition onnxruntime_cxx_api.h:530
constexpr Float8E5M2FNUZ_t() noexcept
Definition onnxruntime_cxx_api.h:532
constexpr Float8E5M2FNUZ_t(uint8_t v) noexcept
Definition onnxruntime_cxx_api.h:533
constexpr bool operator!=(const Float8E5M2FNUZ_t &rhs) const noexcept
Definition onnxruntime_cxx_api.h:537
constexpr bool operator==(const Float8E5M2FNUZ_t &rhs) const noexcept
Definition onnxruntime_cxx_api.h:536
uint8_t value
Definition onnxruntime_cxx_api.h:531
Definition onnxruntime_cxx_api.h:86
static const OrtApi * api_
Definition onnxruntime_cxx_api.h:87
Wrapper around OrtGraph.
Definition onnxruntime_cxx_api.h:2844
Graph(OrtGraph *p)
Take ownership of a pointer created by C API.
Definition onnxruntime_cxx_api.h:2846
Graph(std::nullptr_t)
No instance is created.
Definition onnxruntime_cxx_api.h:2845
Wrapper around OrtIoBinding.
Definition onnxruntime_cxx_api.h:2225
UnownedIoBinding GetUnowned() const
Definition onnxruntime_cxx_api.h:2229
ConstIoBinding GetConst() const
Definition onnxruntime_cxx_api.h:2228
IoBinding(Session &session)
IoBinding(std::nullptr_t)
Create an empty object for convenience. Sometimes, we want to initialize members later.
Definition onnxruntime_cxx_api.h:2226
This class wraps a raw pointer OrtKernelContext* that is being passed to the custom kernel Compute() ...
Definition onnxruntime_cxx_api.h:2422
KernelContext(OrtKernelContext *context)
Logger GetLogger() const
ConstValue GetInput(size_t index) const
OrtKernelContext * GetOrtKernelContext() const
Definition onnxruntime_cxx_api.h:2436
void ParallelFor(void(*fn)(void *, size_t), size_t total, size_t num_batch, void *usr_data) const
OrtAllocator * GetAllocator(const OrtMemoryInfo &memory_info) const
void * GetGPUComputeStream() const
size_t GetInputCount() const
size_t GetOutputCount() const
UnownedValue GetOutput(size_t index, const std::vector< int64_t > &dims) const
UnownedValue GetOutput(size_t index, const int64_t *dim_values, size_t dim_count) const
This struct owns the OrtKernInfo* pointer when a copy is made. For convenient wrapping of OrtKernelIn...
Definition onnxruntime_cxx_api.h:2502
KernelInfo(OrtKernelInfo *info)
Take ownership of the instance.
ConstKernelInfo GetConst() const
Definition onnxruntime_cxx_api.h:2507
detail::KernelInfoImpl< OrtKernelInfo > Base
Definition onnxruntime_cxx_api.h:2503
KernelInfo(std::nullptr_t)
Create an empty instance to initialize later.
Definition onnxruntime_cxx_api.h:2505
Wrapper around OrtKeyValuePairs.
Definition onnxruntime_cxx_api.h:776
KeyValuePairs()
Wraps OrtApi::CreateKeyValuePairs.
void Add(const char *key, const char *value)
Wraps OrtApi::AddKeyValuePair.
KeyValuePairs(const std::unordered_map< std::string, std::string > &kv_pairs)
Wraps OrtApi::CreateKeyValuePairs and OrtApi::AddKeyValuePair.
void Remove(const char *key)
Wraps OrtApi::RemoveKeyValuePair.
KeyValuePairs(std::nullptr_t)
Definition onnxruntime_cxx_api.h:777
ConstKeyValuePairs GetConst() const
Definition onnxruntime_cxx_api.h:793
KeyValuePairs(OrtKeyValuePairs *p)
Take ownership of a pointer created by C API.
Definition onnxruntime_cxx_api.h:779
This class represents an ONNX Runtime logger that can be used to log information with an associated s...
Definition onnxruntime_cxx_api.h:2344
Logger(Logger &&v) noexcept=default
Logger & operator=(Logger &&v) noexcept=default
Logger & operator=(const Logger &)=default
~Logger()=default
Logger(const Logger &)=default
Logger()=default
Logger(std::nullptr_t)
Definition onnxruntime_cxx_api.h:2353
Logger(const OrtLogger *logger)
OrtLoggingLevel GetLoggingSeverityLevel() const noexcept
LoraAdapter holds a set of Lora Parameters loaded from a single file.
Definition onnxruntime_cxx_api.h:903
static LoraAdapter CreateLoraAdapter(const std::basic_string< char > &adapter_path, OrtAllocator *allocator)
Wraps OrtApi::CreateLoraAdapter.
LoraAdapter(std::nullptr_t)
Definition onnxruntime_cxx_api.h:907
static LoraAdapter CreateLoraAdapterFromArray(const void *bytes, size_t num_bytes, OrtAllocator *allocator)
Wraps OrtApi::CreateLoraAdapterFromArray.
Wrapper around OrtMapTypeInfo.
Definition onnxruntime_cxx_api.h:1563
ConstMapTypeInfo GetConst() const
Definition onnxruntime_cxx_api.h:1569
MapTypeInfo(OrtMapTypeInfo *p)
Used for interop with the C API.
Definition onnxruntime_cxx_api.h:1568
MapTypeInfo(std::nullptr_t)
Create an empty MapTypeInfo object, must be assigned a valid one to be used.
Definition onnxruntime_cxx_api.h:1567
Represents native memory allocation coming from one of the OrtAllocators registered with OnnxRuntime....
Definition onnxruntime_cxx_api.h:2130
MemoryAllocation(MemoryAllocation &&) noexcept
MemoryAllocation & operator=(const MemoryAllocation &)=delete
MemoryAllocation(const MemoryAllocation &)=delete
MemoryAllocation(OrtAllocator *allocator, void *p, size_t size)
size_t size() const
Definition onnxruntime_cxx_api.h:2139
Wrapper around OrtMemoryInfo.
Definition onnxruntime_cxx_api.h:1453
MemoryInfo(const char *name, OrtAllocatorType type, int id, OrtMemType mem_type)
MemoryInfo(std::nullptr_t)
No instance is created.
Definition onnxruntime_cxx_api.h:1455
MemoryInfo(OrtMemoryInfo *p)
Take ownership of a pointer created by C API.
Definition onnxruntime_cxx_api.h:1456
static MemoryInfo CreateCpu(OrtAllocatorType type, OrtMemType mem_type1)
ConstMemoryInfo GetConst() const
Definition onnxruntime_cxx_api.h:1458
Options object used when compiling a model.
Definition onnxruntime_cxx_api.h:1146
ModelCompilationOptions & SetEpContextEmbedMode(bool embed_ep_context_in_model)
Wraps OrtApi::ModelCompilationOptions_SetEpContextEmbedMode.
ModelCompilationOptions & SetInputModelFromBuffer(const void *input_model_data, size_t input_model_data_size)
Wraps OrtApi::ModelCompilationOptions_SetInputModelFromBuffer.
ModelCompilationOptions & SetOutputModelBuffer(OrtAllocator *allocator, void **output_model_buffer_ptr, size_t *output_model_buffer_size_ptr)
Wraps OrtApi::ModelCompilationOptions_SetOutputModelBuffer.
ModelCompilationOptions & SetOutputModelExternalInitializersFile(const char *file_path, size_t initializer_size_threshold)
Wraps OrtApi::ModelCompilationOptions_SetOutputModelExternalInitializersFile.
ModelCompilationOptions(std::nullptr_t)
Create an empty ModelCompilationOptions object, must be assigned a valid one to be used.
Definition onnxruntime_cxx_api.h:1150
ModelCompilationOptions(const Env &env, ConstSessionOptions session_options)
Wraps OrtApi::CreateModelCompilationOptionsFromSessionOptions.
ModelCompilationOptions & SetOutputModelPath(const char *output_model_path)
Wraps OrtApi::ModelCompilationOptions_SetOutputModelPath.
ModelCompilationOptions & SetInputModelPath(const char *input_model_path)
Wraps OrtApi::ModelCompilationOptions_SetInputModelPath.
ModelCompilationOptions & SetEpContextBinaryInformation(const char *output_directory, const char *model_name)
Wraps OrtApi::ModelCompilationOptions_SetEpContextBinaryInformation.
ModelCompilationOptions(const Env &env, const SessionOptions &session_options)
Wraps OrtApi::CreateModelCompilationOptionsFromSessionOptions.
ModelCompilationOptions & SetFlags(size_t flags)
Wraps OrtApi::ModelCompilationOptions_SetFlags.
Wrapper around OrtModel.
Definition onnxruntime_cxx_api.h:2870
Model(const std::vector< DomainOpsetPair > &opsets)
Model(OrtModel *p)
Take ownership of a pointer created by C API.
Definition onnxruntime_cxx_api.h:2874
std::pair< std::string, int > DomainOpsetPair
Definition onnxruntime_cxx_api.h:2871
Model(std::nullptr_t)
No instance is created.
Definition onnxruntime_cxx_api.h:2873
ConstModel GetConst() const
Definition onnxruntime_cxx_api.h:2880
Wrapper around OrtModelMetadata.
Definition onnxruntime_cxx_api.h:1180
AllocatedStringPtr GetDescriptionAllocated(OrtAllocator *allocator) const
Returns a copy of the description.
std::vector< AllocatedStringPtr > GetCustomMetadataMapKeysAllocated(OrtAllocator *allocator) const
Returns a vector of copies of the custom metadata keys.
ModelMetadata(std::nullptr_t)
Create an empty ModelMetadata object, must be assigned a valid one to be used.
Definition onnxruntime_cxx_api.h:1184
AllocatedStringPtr GetGraphDescriptionAllocated(OrtAllocator *allocator) const
Returns a copy of the graph description.
AllocatedStringPtr GetProducerNameAllocated(OrtAllocator *allocator) const
Returns a copy of the producer name.
AllocatedStringPtr GetGraphNameAllocated(OrtAllocator *allocator) const
Returns a copy of the graph name.
AllocatedStringPtr LookupCustomMetadataMapAllocated(const char *key, OrtAllocator *allocator) const
Looks up a value by a key in the Custom Metadata map.
AllocatedStringPtr GetDomainAllocated(OrtAllocator *allocator) const
Returns a copy of the domain name.
int64_t GetVersion() const
Wraps OrtApi::ModelMetadataGetVersion.
Wrapper around OrtNode.
Definition onnxruntime_cxx_api.h:2797
Node(const std::string &operator_name, const std::string &operator_domain, const std::string &node_name, const std::vector< std::string > &input_names, const std::vector< std::string > &output_names)
Node(std::nullptr_t)
No instance is created.
Definition onnxruntime_cxx_api.h:2798
Node(const std::string &operator_name, const std::string &operator_domain, const std::string &node_name, const std::vector< std::string > &input_names, const std::vector< std::string > &output_names, std::vector< OpAttr > &attributes)
Wraps CreateNode. Node takes ownership of attributes on success and updates the OpAttr in attributes ...
Node(OrtNode *p)
Take ownership of a pointer created by C API.
Definition onnxruntime_cxx_api.h:2799
This struct provides life time management for custom op attribute.
Definition onnxruntime_cxx_api.h:2256
OpAttr(const char *name, const void *data, int len, OrtOpAttrType type)
OpAttr(std::nullptr_t)
Definition onnxruntime_cxx_api.h:2260
Create and own custom defined operation.
Definition onnxruntime_cxx_api.h:2513
Op(OrtOp *)
Take ownership of the OrtOp.
static Op Create(const OrtKernelInfo *info, const char *op_name, const char *domain, int version, const char **type_constraint_names, const ONNXTensorElementDataType *type_constraint_values, size_t type_constraint_count, const OpAttr *attr_values, size_t attr_count, size_t input_count, size_t output_count)
Op(std::nullptr_t)
Create an empty Operator object, must be assigned a valid one to be used.
Definition onnxruntime_cxx_api.h:2517
void Invoke(const OrtKernelContext *context, const OrtValue *const *input_values, size_t input_count, OrtValue *const *output_values, size_t output_count)
void Invoke(const OrtKernelContext *context, const Value *input_values, size_t input_count, Value *output_values, size_t output_count)
RunOptions.
Definition onnxruntime_cxx_api.h:931
int GetRunLogSeverityLevel() const
Wraps OrtApi::RunOptionsGetRunLogSeverityLevel.
RunOptions & SetTerminate()
Terminates all currently executing Session::Run calls that were made using this RunOptions instance.
RunOptions & SetRunTag(const char *run_tag)
wraps OrtApi::RunOptionsSetRunTag
RunOptions & AddActiveLoraAdapter(const LoraAdapter &adapter)
Add the LoraAdapter to the list of active adapters. The setting does not affect RunWithBinding() call...
RunOptions & UnsetTerminate()
Clears the terminate flag so this RunOptions instance can be used in a new Session::Run call without ...
int GetRunLogVerbosityLevel() const
Wraps OrtApi::RunOptionsGetRunLogVerbosityLevel.
RunOptions(std::nullptr_t)
Create an empty RunOptions object, must be assigned a valid one to be used.
Definition onnxruntime_cxx_api.h:932
RunOptions & SetRunLogVerbosityLevel(int)
Wraps OrtApi::RunOptionsSetRunLogVerbosityLevel.
RunOptions & SetRunLogSeverityLevel(int)
Wraps OrtApi::RunOptionsSetRunLogSeverityLevel.
RunOptions & AddConfigEntry(const char *config_key, const char *config_value)
Wraps OrtApi::AddRunConfigEntry.
const char * GetRunTag() const
Wraps OrtApi::RunOptionsGetRunTag.
RunOptions()
Wraps OrtApi::CreateRunOptions.
const char * GetConfigEntry(const char *config_key)
Wraps OrtApi::GetRunConfigEntry.
Wrapper around OrtSequenceTypeInfo.
Definition onnxruntime_cxx_api.h:1525
SequenceTypeInfo(std::nullptr_t)
Create an empty SequenceTypeInfo object, must be assigned a valid one to be used.
Definition onnxruntime_cxx_api.h:1529
ConstSequenceTypeInfo GetConst() const
Definition onnxruntime_cxx_api.h:1531
SequenceTypeInfo(OrtSequenceTypeInfo *p)
Used for interop with the C API.
Definition onnxruntime_cxx_api.h:1530
Wrapper around OrtSession.
Definition onnxruntime_cxx_api.h:1396
Session(std::nullptr_t)
Create an empty Session object, must be assigned a valid one to be used. Wraps OrtApi::CreateSession.
Definition onnxruntime_cxx_api.h:1398
static Session CreateModelEditorSession(const Env &env, const void *model_data, size_t model_data_length, const SessionOptions &options)
Wraps OrtModelEditorApi::CreateModelEditorSession.
UnownedSession GetUnowned() const
Definition onnxruntime_cxx_api.h:1427
Session(const Env &env, const char *model_path, const SessionOptions &options, OrtPrepackedWeightsContainer *prepacked_weights_container)
Wraps OrtApi::CreateSessionWithPrepackedWeightsContainer.
Session(const Env &env, const void *model_data, size_t model_data_length, const SessionOptions &options, OrtPrepackedWeightsContainer *prepacked_weights_container)
Wraps OrtApi::CreateSessionFromArrayWithPrepackedWeightsContainer.
Session(const Env &env, const Model &model, const SessionOptions &options)
Wraps OrtModelEditorApi::CreateSessionFromModel.
Session(OrtSession *p)
C API Interop.
Definition onnxruntime_cxx_api.h:1399
static Session CreateModelEditorSession(const Env &env, const char *model_path, const SessionOptions &options)
Wraps OrtModelEditorApi::CreateModelEditorSession.
Session(const Env &env, const char *model_path, const SessionOptions &options)
ConstSession GetConst() const
Definition onnxruntime_cxx_api.h:1426
Session(const Env &env, const void *model_data, size_t model_data_length, const SessionOptions &options)
Wraps OrtApi::CreateSessionFromArray.
Wrapper around OrtSessionOptions.
Definition onnxruntime_cxx_api.h:1134
SessionOptions(std::nullptr_t)
Create an empty SessionOptions object, must be assigned a valid one to be used.
Definition onnxruntime_cxx_api.h:1135
UnownedSessionOptions GetUnowned() const
Definition onnxruntime_cxx_api.h:1138
SessionOptions()
Wraps OrtApi::CreateSessionOptions.
ConstSessionOptions GetConst() const
Definition onnxruntime_cxx_api.h:1139
SessionOptions(OrtSessionOptions *p)
Used for interop with the C API.
Definition onnxruntime_cxx_api.h:1137
Definition onnxruntime_cxx_api.h:2547
SymbolicInteger & operator=(const SymbolicInteger &)=default
SymbolicInteger(const SymbolicInteger &)=default
int64_t AsInt() const
Definition onnxruntime_cxx_api.h:2568
int64_t i_
Definition onnxruntime_cxx_api.h:2575
const char * s_
Definition onnxruntime_cxx_api.h:2576
bool operator==(const SymbolicInteger &dim) const
Definition onnxruntime_cxx_api.h:2556
SymbolicInteger & operator=(SymbolicInteger &&)=default
SymbolicInteger(SymbolicInteger &&)=default
const char * AsSym() const
Definition onnxruntime_cxx_api.h:2569
SymbolicInteger(int64_t i)
Definition onnxruntime_cxx_api.h:2548
SymbolicInteger(const char *s)
Definition onnxruntime_cxx_api.h:2549
bool IsInt() const
Definition onnxruntime_cxx_api.h:2567
Provide access to per-node attributes and input shapes, so one could compute and set output shapes.
Definition onnxruntime_cxx_api.h:2546
Ints GetAttrInts(const char *attr_name)
Strings GetAttrStrings(const char *attr_name)
Status SetOutputShape(size_t indice, const Shape &shape, ONNXTensorElementDataType type=ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT)
std::vector< SymbolicInteger > Shape
Definition onnxruntime_cxx_api.h:2581
std::vector< float > Floats
Definition onnxruntime_cxx_api.h:2598
std::string GetAttrString(const char *attr_name)
std::vector< int64_t > Ints
Definition onnxruntime_cxx_api.h:2593
ShapeInferContext(const OrtApi *ort_api, OrtShapeInferContext *ctx)
int64_t GetAttrInt(const char *attr_name)
size_t GetInputCount() const
Definition onnxruntime_cxx_api.h:2587
std::vector< std::string > Strings
Definition onnxruntime_cxx_api.h:2603
Floats GetAttrFloats(const char *attr_name)
const Shape & GetInputShape(size_t indice) const
Definition onnxruntime_cxx_api.h:2585
float GetAttrFloat(const char *attr_name)
The Status that holds ownership of OrtStatus received from C API Use it to safely destroy OrtStatus* ...
Definition onnxruntime_cxx_api.h:713
OrtErrorCode GetErrorCode() const
Status(const char *message, OrtErrorCode code) noexcept
Creates status instance out of null-terminated string message.
bool IsOK() const noexcept
Returns true if instance represents an OK (non-error) status.
Status(OrtStatus *status) noexcept
Takes ownership of OrtStatus instance returned from the C API.
std::string GetErrorMessage() const
Status(const Exception &) noexcept
Creates status instance out of exception.
Status(const std::exception &) noexcept
Creates status instance out of exception.
detail::Base< OrtStatus > Base
Definition onnxruntime_cxx_api.h:714
Status(std::nullptr_t) noexcept
Create an empty object, must be assigned a valid one to be used.
Definition onnxruntime_cxx_api.h:717
Wrapper around OrtTensorTypeAndShapeInfo.
Definition onnxruntime_cxx_api.h:1491
TensorTypeAndShapeInfo(std::nullptr_t)
Create an empty TensorTypeAndShapeInfo object, must be assigned a valid one to be used.
Definition onnxruntime_cxx_api.h:1496
ConstTensorTypeAndShapeInfo GetConst() const
Definition onnxruntime_cxx_api.h:1507
TensorTypeAndShapeInfo(OrtTensorTypeAndShapeInfo *p)
Used for interop with the C API.
Definition onnxruntime_cxx_api.h:1498
TensorTypeAndShapeInfo(ONNXTensorElementDataType element_type, const std::vector< int64_t > &dims, const std::vector< std::string > *symbolic_dims=nullptr)
The ThreadingOptions.
Definition onnxruntime_cxx_api.h:731
ThreadingOptions & SetGlobalCustomThreadCreationOptions(void *ort_custom_thread_creation_options)
Wraps OrtApi::SetGlobalCustomThreadCreationOptions.
ThreadingOptions()
Wraps OrtApi::CreateThreadingOptions.
ThreadingOptions & SetGlobalInterOpNumThreads(int inter_op_num_threads)
Wraps OrtApi::SetGlobalInterOpNumThreads.
ThreadingOptions & SetGlobalCustomCreateThreadFn(OrtCustomCreateThreadFn ort_custom_create_thread_fn)
Wraps OrtApi::SetGlobalCustomCreateThreadFn.
ThreadingOptions & SetGlobalCustomJoinThreadFn(OrtCustomJoinThreadFn ort_custom_join_thread_fn)
Wraps OrtApi::SetGlobalCustomJoinThreadFn.
ThreadingOptions & SetGlobalSpinControl(int allow_spinning)
Wraps OrtApi::SetGlobalSpinControl.
ThreadingOptions & SetGlobalDenormalAsZero()
Wraps OrtApi::SetGlobalDenormalAsZero.
ThreadingOptions & SetGlobalIntraOpNumThreads(int intra_op_num_threads)
Wraps OrtApi::SetGlobalIntraOpNumThreads.
Type information that may contain either TensorTypeAndShapeInfo or the information about contained se...
Definition onnxruntime_cxx_api.h:1597
static TypeInfo CreateOptionalTypeInfo(ConstTypeInfo contained_type)
static TypeInfo CreateSequenceTypeInfo(ConstTypeInfo sequence_type)
static TypeInfo CreateTensorInfo(ConstTensorTypeAndShapeInfo tensor_info)
static TypeInfo CreateSparseTensorInfo(ConstTensorTypeAndShapeInfo sparse_tensor_info)
TypeInfo(std::nullptr_t)
Create an empty TypeInfo object, must be assigned a valid one to be used.
Definition onnxruntime_cxx_api.h:1602
static TypeInfo CreateMapTypeInfo(ONNXTensorElementDataType key_type, ConstTypeInfo value_type)
ConstTypeInfo GetConst() const
Definition onnxruntime_cxx_api.h:1613
TypeInfo(OrtTypeInfo *p)
C API Interop.
Definition onnxruntime_cxx_api.h:1603
Wrapper around OrtValue.
Definition onnxruntime_cxx_api.h:1953
static Value CreateSparseTensor(const OrtMemoryInfo *info, void *p_data, const Shape &dense_shape, const Shape &values_shape, ONNXTensorElementDataType type)
Creates an OrtValue instance containing SparseTensor. This constructs a sparse tensor that makes use ...
static Value CreateSparseTensor(const OrtMemoryInfo *info, T *p_data, const Shape &dense_shape, const Shape &values_shape)
This is a simple forwarding method to the other overload that helps deducing data type enum value fro...
Value & operator=(Value &&)=default
static Value CreateSparseTensor(OrtAllocator *allocator, const Shape &dense_shape, ONNXTensorElementDataType type)
Creates an instance of OrtValue containing sparse tensor. The created instance has no data....
Value(Value &&)=default
Value(std::nullptr_t)
Create an empty Value object, must be assigned a valid one to be used.
Definition onnxruntime_cxx_api.h:1959
static Value CreateTensor(const OrtMemoryInfo *info, T *p_data, size_t p_data_element_count, const int64_t *shape, size_t shape_len)
Creates a tensor with a user supplied buffer. Wraps OrtApi::CreateTensorWithDataAsOrtValue.
static Value CreateSparseTensor(OrtAllocator *allocator, const Shape &dense_shape)
This is a simple forwarding method to the below CreateSparseTensor. This helps to specify data type e...
static Value CreateTensor(OrtAllocator *allocator, const int64_t *shape, size_t shape_len, ONNXTensorElementDataType type)
Creates an OrtValue with a tensor using the supplied OrtAllocator. Wraps OrtApi::CreateTensorAsOrtVal...
UnownedValue GetUnowned() const
Definition onnxruntime_cxx_api.h:1964
static Value CreateSequence(const std::vector< Value > &values)
Creates an OrtValue with a Sequence Onnx type representation. The API would ref-count the supplied Or...
static Value CreateMap(const Value &keys, const Value &values)
Creates an OrtValue with a Map Onnx type representation. The API would ref-count the supplied OrtValu...
static Value CreateTensor(const OrtMemoryInfo *info, void *p_data, size_t p_data_byte_count, const int64_t *shape, size_t shape_len, ONNXTensorElementDataType type)
Creates a tensor with a user supplied buffer. Wraps OrtApi::CreateTensorWithDataAsOrtValue.
static Value CreateTensor(OrtAllocator *allocator, const int64_t *shape, size_t shape_len)
Creates an OrtValue with a tensor using a supplied OrtAllocator. Wraps OrtApi::CreateTensorAsOrtValue...
static Value CreateOpaque(const char *domain, const char *type_name, const T &value)
Creates an OrtValue wrapping an Opaque type. This is used for experimental support of non-tensor type...
static Value CreateTensor(OrtAllocator *deleter, void *p_data, size_t p_data_byte_count, const int64_t *shape, size_t shape_len, ONNXTensorElementDataType type)
Creates a tensor with a user supplied buffer. Wraps OrtApi::CreateTensorWithDataAndDeleterAsOrtValue.
ConstValue GetConst() const
Definition onnxruntime_cxx_api.h:1963
Wrapper around OrtValueInfo.
Definition onnxruntime_cxx_api.h:2775
ConstValueInfo GetConst() const
Definition onnxruntime_cxx_api.h:2783
ValueInfo(std::nullptr_t)
Definition onnxruntime_cxx_api.h:2776
ValueInfo(const std::string &name, const ConstTypeInfo &type_info)
ValueInfo(OrtValueInfo *p)
Take ownership of a pointer created by C API.
Definition onnxruntime_cxx_api.h:2778
Definition onnxruntime_cxx_api.h:681
AllocatedFree(OrtAllocator *allocator)
Definition onnxruntime_cxx_api.h:683
OrtAllocator * allocator_
Definition onnxruntime_cxx_api.h:682
void operator()(void *ptr) const
Definition onnxruntime_cxx_api.h:685
Base & operator=(Base &&v) noexcept
Definition onnxruntime_cxx_api.h:668
typename Unowned< T >::Type contained_type
Definition onnxruntime_cxx_api.h:657
Base(Base &&v) noexcept
Definition onnxruntime_cxx_api.h:667
Base(const Base &)=default
constexpr Base(contained_type *p) noexcept
Definition onnxruntime_cxx_api.h:660
Base & operator=(const Base &)=default
Used internally by the C++ API. C++ wrapper types inherit from this. This is a zero cost abstraction ...
Definition onnxruntime_cxx_api.h:611
Base(Base &&v) noexcept
Definition onnxruntime_cxx_api.h:623
constexpr Base()=default
contained_type * release()
Relinquishes ownership of the contained C object pointer The underlying object is not destroyed.
Definition onnxruntime_cxx_api.h:634
Base(const Base &)=delete
constexpr Base(contained_type *p) noexcept
Definition onnxruntime_cxx_api.h:615
Base & operator=(const Base &)=delete
Base & operator=(Base &&v) noexcept
Definition onnxruntime_cxx_api.h:624
contained_type * p_
Definition onnxruntime_cxx_api.h:641
~Base()
Definition onnxruntime_cxx_api.h:616
T contained_type
Definition onnxruntime_cxx_api.h:612
Definition onnxruntime_cxx_api.h:2193
std::vector< Value > GetOutputValues(OrtAllocator *) const
std::vector< std::string > GetOutputNames(OrtAllocator *) const
std::vector< Value > GetOutputValues() const
std::vector< std::string > GetOutputNames() const
Definition onnxruntime_cxx_api.h:1255
std::vector< std::string > GetOutputNames() const
TypeInfo GetInputTypeInfo(size_t index) const
Wraps OrtApi::SessionGetInputTypeInfo.
size_t GetOutputCount() const
Returns the number of model outputs.
std::vector< ValueInfo > GetOutputs() const
int GetOpset(const std::string &domain) const
Wraps OrtApi::SessionGetOpsetForDomain.
uint64_t GetProfilingStartTimeNs() const
Wraps OrtApi::SessionGetProfilingStartTimeNs.
std::vector< std::string > GetOverridableInitializerNames() const
ModelMetadata GetModelMetadata() const
Wraps OrtApi::SessionGetModelMetadata.
size_t GetInputCount() const
Returns the number of model inputs.
TypeInfo GetOutputTypeInfo(size_t index) const
Wraps OrtApi::SessionGetOutputTypeInfo.
std::vector< std::string > GetInputNames() const
AllocatedStringPtr GetOverridableInitializerNameAllocated(size_t index, OrtAllocator *allocator) const
Returns a copy of the overridable initializer name at then specified index.
AllocatedStringPtr GetOutputNameAllocated(size_t index, OrtAllocator *allocator) const
Returns a copy of output name at then specified index.
size_t GetOverridableInitializerCount() const
Returns the number of inputs that have defaults that can be overridden.
AllocatedStringPtr GetInputNameAllocated(size_t index, OrtAllocator *allocator) const
Returns a copy of input name at the specified index.
std::vector< ValueInfo > GetInputs() const
TypeInfo GetOverridableInitializerTypeInfo(size_t index) const
Wraps OrtApi::SessionGetOverridableInitializerTypeInfo.
Definition onnxruntime_cxx_api.h:1642
void GetStringTensorContent(void *buffer, size_t buffer_length, size_t *offsets, size_t offsets_count) const
The API copies all of the UTF-8 encoded string data contained within a tensor or a sparse tensor into...
void GetStringTensorElement(size_t buffer_length, size_t element_index, void *buffer) const
The API copies UTF-8 encoded bytes for the requested string element contained within a tensor or a sp...
TensorTypeAndShapeInfo GetSparseTensorIndicesTypeShapeInfo(OrtSparseIndicesFormat format) const
The API returns type and shape information for the specified indices. Each supported indices have the...
const void * GetTensorRawData() const
Returns a non-typed pointer to a tensor contained data.
std::string GetStringTensorElement(size_t element_index) const
Returns string tensor UTF-8 encoded string element. Use of this API is recommended over GetStringTens...
size_t GetStringTensorElementLength(size_t element_index) const
The API returns a byte length of UTF-8 encoded string element contained in either a tensor or a spare...
size_t GetStringTensorDataLength() const
This API returns a full length of string data contained within either a tensor or a sparse Tensor....
bool IsSparseTensor() const
Returns true if the OrtValue contains a sparse tensor.
TypeInfo GetTypeInfo() const
The API returns type information for data contained in a tensor. For sparse tensors it returns type i...
const R * GetSparseTensorIndicesData(OrtSparseIndicesFormat indices_format, size_t &num_indices) const
The API retrieves a pointer to the internal indices buffer. The API merely performs a convenience dat...
bool IsTensor() const
Returns true if Value is a tensor, false for other types like map/sequence/etc.
ConstMemoryInfo GetTensorMemoryInfo() const
This API returns information about the memory allocation used to hold data.
size_t GetTensorSizeInBytes() const
Returns the total size of the tensor data in bytes. Throws an exception if the OrtValue does not cont...
const R * GetSparseTensorValues() const
The API returns a pointer to an internal buffer of the sparse tensor containing non-zero values....
TensorTypeAndShapeInfo GetTensorTypeAndShapeInfo() const
The API returns type information for data contained in a tensor. For sparse tensors it returns type i...
Value GetValue(int index, OrtAllocator *allocator) const
size_t GetCount() const
< Return true if OrtValue contains data and returns false if the OrtValue is a None
void GetOpaqueData(const char *domain, const char *type_name, R &) const
Obtains a pointer to a user defined data for experimental purposes.
TensorTypeAndShapeInfo GetSparseTensorValuesTypeAndShapeInfo() const
The API returns type and shape information for stored non-zero values of the sparse tensor....
const R * GetTensorData() const
Returns a const typed pointer to the tensor contained data. No type checking is performed,...
OrtSparseFormat GetSparseFormat() const
The API returns the sparse data format this OrtValue holds in a sparse tensor. If the sparse tensor w...
Definition onnxruntime_cxx_api.h:817
const char * EpName() const
const char * EpVendor() const
ConstKeyValuePairs EpOptions() const
ConstHardwareDevice Device() const
ConstKeyValuePairs EpMetadata() const
Definition onnxruntime_cxx_api.h:798
OrtHardwareDeviceType Type() const
const char * Vendor() const
ConstKeyValuePairs Metadata() const
Definition onnxruntime_cxx_api.h:2204
void BindOutput(const char *name, const Value &)
void BindInput(const char *name, const Value &)
void BindOutput(const char *name, const OrtMemoryInfo *)
Definition onnxruntime_cxx_api.h:759
void GetKeyValuePairs(std::vector< const char * > &keys, std::vector< const char * > &values) const
std::unordered_map< std::string, std::string > GetKeyValuePairs() const
const char * GetValue(const char *key) const
Definition onnxruntime_cxx_api.h:1549
ONNXTensorElementDataType GetMapKeyType() const
Wraps OrtApi::GetMapKeyType.
TypeInfo GetMapValueType() const
Wraps OrtApi::GetMapValueType.
Definition onnxruntime_cxx_api.h:1432
std::string GetAllocatorName() const
OrtMemType GetMemoryType() const
OrtMemoryInfoDeviceType GetDeviceType() const
OrtAllocatorType GetAllocatorType() const
bool operator==(const MemoryInfoImpl< U > &o) const
Definition onnxruntime_cxx_api.h:2854
void AddGraph(Graph &graph)
Definition onnxruntime_cxx_api.h:2788
Definition onnxruntime_cxx_api.h:1536
TypeInfo GetOptionalElementType() const
Wraps OrtApi::CastOptionalTypeToContainedTypeInfo.
Definition onnxruntime_cxx_api.h:1625
const char ** str
Definition onnxruntime_cxx_api.h:1630
const int64_t * values_shape
Definition onnxruntime_cxx_api.h:1626
size_t values_shape_len
Definition onnxruntime_cxx_api.h:1627
const void * p_data
Definition onnxruntime_cxx_api.h:1629
Definition onnxruntime_cxx_api.h:1512
TypeInfo GetSequenceElementType() const
Wraps OrtApi::GetSequenceElementType.
Definition onnxruntime_cxx_api.h:1309
void SetEpDynamicOptions(const char *const *keys, const char *const *values, size_t kv_len)
Set DynamicOptions for EPs (Execution Providers)
AllocatedStringPtr EndProfilingAllocated(OrtAllocator *allocator)
End profiling and return a copy of the profiling file name.
void FinalizeModelEditorSession(const Model &model, const SessionOptions &options, OrtPrepackedWeightsContainer *prepacked_weights_container=nullptr)
void Run(const RunOptions &run_options, const IoBinding &)
Wraps OrtApi::RunWithBinding.
void RunAsync(const RunOptions &run_options, const char *const *input_names, const Value *input_values, size_t input_count, const char *const *output_names, Value *output_values, size_t output_count, RunAsyncCallbackFn callback, void *user_data)
Run the model asynchronously in a thread owned by intra op thread pool.
std::vector< Value > Run(const RunOptions &run_options, const char *const *input_names, const Value *input_values, size_t input_count, const char *const *output_names, size_t output_count)
Run the model returning results in an Ort allocated vector.
void Run(const RunOptions &run_options, const char *const *input_names, const Value *input_values, size_t input_count, const char *const *output_names, Value *output_values, size_t output_count)
Run the model returning results in user provided outputs Same as Run(const RunOptions&,...
Definition onnxruntime_cxx_api.h:1636
const int64_t * shape
Definition onnxruntime_cxx_api.h:1637
size_t shape_len
Definition onnxruntime_cxx_api.h:1638
Definition onnxruntime_cxx_api.h:1463
size_t GetElementCount() const
Wraps OrtApi::GetTensorShapeElementCount.
void GetDimensions(int64_t *values, size_t values_count) const
Wraps OrtApi::GetDimensions.
std::vector< int64_t > GetShape() const
Uses GetDimensionsCount & GetDimensions to return a std::vector of the shape.
std::vector< const char * > GetSymbolicDimensions() const
void GetSymbolicDimensions(const char **values, size_t values_count) const
Wraps OrtApi::GetSymbolicDimensions.
size_t GetDimensionsCount() const
Wraps OrtApi::GetDimensionsCount.
ONNXTensorElementDataType GetElementType() const
Wraps OrtApi::GetTensorElementType.
Definition onnxruntime_cxx_api.h:1574
ONNXType GetONNXType() const
ConstSequenceTypeInfo GetSequenceTypeInfo() const
Wraps OrtApi::CastTypeInfoToSequenceTypeInfo.
ConstMapTypeInfo GetMapTypeInfo() const
Wraps OrtApi::CastTypeInfoToMapTypeInfo.
ConstOptionalTypeInfo GetOptionalTypeInfo() const
wraps OrtApi::CastTypeInfoToOptionalTypeInfo
ConstTensorTypeAndShapeInfo GetTensorTypeAndShapeInfo() const
Wraps OrtApi::CastTypeInfoToTensorInfo.
This is a tagging template type. Use it with Base<T> to indicate that the C++ interface object has no...
Definition onnxruntime_cxx_api.h:587
T Type
Definition onnxruntime_cxx_api.h:588
Definition onnxruntime_cxx_api.h:1811
void FillStringTensorElement(const char *s, size_t index)
Set a single string in a string tensor.
R * GetTensorMutableData()
Returns a non-const typed pointer to an OrtValue/Tensor contained buffer No type checking is performe...
R & At(const std::vector< int64_t > &location)
void UseBlockSparseIndices(const Shape &indices_shape, int32_t *indices_data)
Supplies BlockSparse format specific indices and marks the contained sparse tensor as being a BlockSp...
void FillSparseTensorBlockSparse(const OrtMemoryInfo *data_mem_info, const OrtSparseValuesParam &values, const Shape &indices_shape, const int32_t *indices_data)
The API will allocate memory using the allocator instance supplied to the CreateSparseTensor() API an...
void * GetTensorMutableRawData()
Returns a non-typed non-const pointer to a tensor contained data.
void UseCooIndices(int64_t *indices_data, size_t indices_num)
Supplies COO format specific indices and marks the contained sparse tensor as being a COO format tens...
void FillSparseTensorCoo(const OrtMemoryInfo *data_mem_info, const OrtSparseValuesParam &values_param, const int64_t *indices_data, size_t indices_num)
The API will allocate memory using the allocator instance supplied to the CreateSparseTensor() API an...
void FillStringTensor(const char *const *s, size_t s_len)
Set all strings at once in a string tensor.
void UseCsrIndices(int64_t *inner_data, size_t inner_num, int64_t *outer_data, size_t outer_num)
Supplies CSR format specific indices and marks the contained sparse tensor as being a CSR format tens...
void FillSparseTensorCsr(const OrtMemoryInfo *data_mem_info, const OrtSparseValuesParam &values, const int64_t *inner_indices_data, size_t inner_indices_num, const int64_t *outer_indices_data, size_t outer_indices_num)
The API will allocate memory using the allocator instance supplied to the CreateSparseTensor() API an...
char * GetResizedStringTensorElementBuffer(size_t index, size_t buffer_length)
Allocate if necessary and obtain a pointer to a UTF-8 encoded string element buffer indexed by the fl...
Definition onnxruntime_cxx_api.h:2760
ConstTypeInfo TypeInfo() const
std::string Name() const
Memory allocation interface.
Definition onnxruntime_c_api.h:343
void(* Free)(struct OrtAllocator *this_, void *p)
Free a block of memory previously allocated with OrtAllocator::Alloc.
Definition onnxruntime_c_api.h:350
const OrtApi *(* GetApi)(uint32_t version)
Get a pointer to the requested version of the OrtApi.
Definition onnxruntime_c_api.h:843
The C API.
Definition onnxruntime_c_api.h:903
const OrtCompileApi *(* GetCompileApi)()
Get the Compile API instance.
Definition onnxruntime_c_api.h:5104
const OrtModelEditorApi *(* GetModelEditorApi)()
Get the Model Editor API instance.
Definition onnxruntime_c_api.h:5046
const OrtEpApi *(* GetEpApi)()
Get the OrtEpApi instance for implementing an execution provider.
Definition onnxruntime_c_api.h:5372
CUDA Provider Options.
Definition onnxruntime_c_api.h:547
The OrtCompileApi struct provides functions to compile ONNX models.
Definition onnxruntime_c_api.h:7010
Definition onnxruntime_c_api.h:6483
int(* GetVariadicInputHomogeneity)(const struct OrtCustomOp *op)
Definition onnxruntime_c_api.h:6529
OrtCustomOpInputOutputCharacteristic(* GetOutputCharacteristic)(const struct OrtCustomOp *op, size_t index)
Definition onnxruntime_c_api.h:6513
size_t(* GetInputTypeCount)(const struct OrtCustomOp *op)
Definition onnxruntime_c_api.h:6501
int(* GetVariadicOutputMinArity)(const struct OrtCustomOp *op)
Definition onnxruntime_c_api.h:6533
size_t(* GetAliasMap)(int **input_index, int **output_index)
Definition onnxruntime_c_api.h:6566
int(* GetStartVersion)(const struct OrtCustomOp *op)
Definition onnxruntime_c_api.h:6551
void(* ReleaseMayInplace)(int *input_index, int *output_index)
Definition onnxruntime_c_api.h:6563
const char *(* GetName)(const struct OrtCustomOp *op)
Definition onnxruntime_c_api.h:6494
size_t(* GetOutputTypeCount)(const struct OrtCustomOp *op)
Definition onnxruntime_c_api.h:6503
void(* KernelDestroy)(void *op_kernel)
Definition onnxruntime_c_api.h:6509
int(* GetVariadicOutputHomogeneity)(const struct OrtCustomOp *op)
Definition onnxruntime_c_api.h:6538
OrtMemType(* GetInputMemoryType)(const struct OrtCustomOp *op, size_t index)
Definition onnxruntime_c_api.h:6520
void *(* CreateKernel)(const struct OrtCustomOp *op, const OrtApi *api, const OrtKernelInfo *info)
Definition onnxruntime_c_api.h:6490
uint32_t version
Definition onnxruntime_c_api.h:6484
ONNXTensorElementDataType(* GetInputType)(const struct OrtCustomOp *op, size_t index)
Definition onnxruntime_c_api.h:6500
void(* ReleaseAliasMap)(int *input_index, int *output_index)
Definition onnxruntime_c_api.h:6567
OrtCustomOpInputOutputCharacteristic(* GetInputCharacteristic)(const struct OrtCustomOp *op, size_t index)
Definition onnxruntime_c_api.h:6512
const char *(* GetExecutionProviderType)(const struct OrtCustomOp *op)
Definition onnxruntime_c_api.h:6497
ONNXTensorElementDataType(* GetOutputType)(const struct OrtCustomOp *op, size_t index)
Definition onnxruntime_c_api.h:6502
int(* GetVariadicInputMinArity)(const struct OrtCustomOp *op)
Definition onnxruntime_c_api.h:6524
OrtStatusPtr(* InferOutputShapeFn)(const struct OrtCustomOp *op, OrtShapeInferContext *)
Definition onnxruntime_c_api.h:6548
int(* GetEndVersion)(const struct OrtCustomOp *op)
Definition onnxruntime_c_api.h:6552
OrtStatusPtr(* CreateKernelV2)(const struct OrtCustomOp *op, const OrtApi *api, const OrtKernelInfo *info, void **kernel)
Definition onnxruntime_c_api.h:6541
size_t(* GetMayInplace)(int **input_index, int **output_index)
Definition onnxruntime_c_api.h:6559
OrtStatusPtr(* KernelComputeV2)(void *op_kernel, OrtKernelContext *context)
Definition onnxruntime_c_api.h:6546
void(* KernelCompute)(void *op_kernel, OrtKernelContext *context)
Definition onnxruntime_c_api.h:6508
MIGraphX Provider Options.
Definition onnxruntime_c_api.h:751
The OrtModelEditorApi struct provides functions to create or edit an ONNX model.
Definition onnxruntime_c_api.h:6581
OpenVINO Provider Options.
Definition onnxruntime_c_api.h:789
ROCM Provider Options.
Definition onnxruntime_c_api.h:634
TensorRT Provider Options.
Definition onnxruntime_c_api.h:723