ONNX Runtime
Loading...
Searching...
No Matches
onnxruntime_cxx_api.h
1// Copyright (c) Microsoft Corporation. All rights reserved.
2// Licensed under the MIT License.
3
4// Summary: The Ort C++ API is a header only wrapper around the Ort C API.
5//
6// The C++ API simplifies usage by returning values directly instead of error codes, throwing exceptions on errors
7// and automatically releasing resources in the destructors. The primary purpose of C++ API is exception safety so
8// all the resources follow RAII and do not leak memory.
9//
10// Each of the C++ wrapper classes holds only a pointer to the C internal object. Treat them like smart pointers.
11// To create an empty object, pass 'nullptr' to the constructor (for example, Env e{nullptr};). However, you can't use them
12// until you assign an instance that actually holds an underlying object.
13//
14// For Ort objects only move assignment between objects is allowed, there are no copy constructors.
15// Some objects have explicit 'Clone' methods for this purpose.
16//
17// ConstXXXX types are copyable since they do not own the underlying C object, so you can pass them to functions as arguments
18// by value or by reference. ConstXXXX types are restricted to const only interfaces.
19//
20// UnownedXXXX are similar to ConstXXXX but also allow non-const interfaces.
21//
22// The lifetime of the corresponding owning object must eclipse the lifetimes of the ConstXXXX/UnownedXXXX types. They exists so you do not
23// have to fallback to C types and the API with the usual pitfalls. In general, do not use C API from your C++ code.
24
25#pragma once
26#include "onnxruntime_c_api.h"
27#include "onnxruntime_float16.h"
28
29#include <cstddef>
30#include <cstdio>
31#include <array>
32#include <memory>
33#include <stdexcept>
34#include <string>
35#include <vector>
36#include <unordered_map>
37#include <utility>
38#include <type_traits>
39
40#ifdef ORT_NO_EXCEPTIONS
41#include <iostream>
42#endif
43
47namespace Ort {
48
53struct Exception : std::exception {
54 Exception(std::string&& string, OrtErrorCode code) : message_{std::move(string)}, code_{code} {}
55
56 OrtErrorCode GetOrtErrorCode() const { return code_; }
57 const char* what() const noexcept override { return message_.c_str(); }
58
59 private:
60 std::string message_;
61 OrtErrorCode code_;
62};
63
64#ifdef ORT_NO_EXCEPTIONS
65// The #ifndef is for the very special case where the user of this library wants to define their own way of handling errors.
66// NOTE: This header expects control flow to not continue after calling ORT_CXX_API_THROW
67#ifndef ORT_CXX_API_THROW
68#define ORT_CXX_API_THROW(string, code) \
69 do { \
70 std::cerr << Ort::Exception(string, code) \
71 .what() \
72 << std::endl; \
73 abort(); \
74 } while (false)
75#endif
76#else
77#define ORT_CXX_API_THROW(string, code) \
78 throw Ort::Exception(string, code)
79#endif
80
81// This is used internally by the C++ API. This class holds the global variable that points to the OrtApi,
82// it's in a template so that we can define a global variable in a header and make
83// it transparent to the users of the API.
84template <typename T>
85struct Global {
86 static const OrtApi* api_;
87};
88
89// If macro ORT_API_MANUAL_INIT is defined, no static initialization will be performed. Instead, user must call InitApi() before using it.
90template <typename T>
91#ifdef ORT_API_MANUAL_INIT
92const OrtApi* Global<T>::api_{};
93inline void InitApi() noexcept { Global<void>::api_ = OrtGetApiBase()->GetApi(ORT_API_VERSION); }
94
95// Used by custom operator libraries that are not linked to onnxruntime. Sets the global API object, which is
96// required by C++ APIs.
97//
98// Example mycustomop.cc:
99//
100// #define ORT_API_MANUAL_INIT
101// #include <onnxruntime_cxx_api.h>
102// #undef ORT_API_MANUAL_INIT
103//
104// OrtStatus* ORT_API_CALL RegisterCustomOps(OrtSessionOptions* options, const OrtApiBase* api_base) {
105// Ort::InitApi(api_base->GetApi(ORT_API_VERSION));
106// // ...
107// }
108//
109inline void InitApi(const OrtApi* api) noexcept { Global<void>::api_ = api; }
110#else
111#if defined(_MSC_VER) && !defined(__clang__)
112#pragma warning(push)
113// "Global initializer calls a non-constexpr function." Therefore you can't use ORT APIs in the other global initializers.
114// Please define ORT_API_MANUAL_INIT if it conerns you.
115#pragma warning(disable : 26426)
116#endif
118#if defined(_MSC_VER) && !defined(__clang__)
119#pragma warning(pop)
120#endif
121#endif
122
124inline const OrtApi& GetApi() noexcept { return *Global<void>::api_; }
125
130std::string GetVersionString();
131
137std::string GetBuildInfoString();
138
144std::vector<std::string> GetAvailableProviders();
145
164struct Float16_t : onnxruntime_float16::Float16Impl<Float16_t> {
165 private:
171 constexpr explicit Float16_t(uint16_t v) noexcept { val = v; }
172
173 public:
174 using Base = onnxruntime_float16::Float16Impl<Float16_t>;
175
179 Float16_t() = default;
180
186 constexpr static Float16_t FromBits(uint16_t v) noexcept { return Float16_t(v); }
187
192 explicit Float16_t(float v) noexcept { val = Base::ToUint16Impl(v); }
193
198 float ToFloat() const noexcept { return Base::ToFloatImpl(); }
199
204 using Base::IsNegative;
205
210 using Base::IsNaN;
211
216 using Base::IsFinite;
217
222 using Base::IsPositiveInfinity;
223
228 using Base::IsNegativeInfinity;
229
234 using Base::IsInfinity;
235
240 using Base::IsNaNOrZero;
241
246 using Base::IsNormal;
247
252 using Base::IsSubnormal;
253
258 using Base::Abs;
259
264 using Base::Negate;
265
274 using Base::AreZero;
275
279 explicit operator float() const noexcept { return ToFloat(); }
280
281 using Base::operator==;
282 using Base::operator!=;
283 using Base::operator<;
284};
285
286static_assert(sizeof(Float16_t) == sizeof(uint16_t), "Sizes must match");
287
306struct BFloat16_t : onnxruntime_float16::BFloat16Impl<BFloat16_t> {
307 private:
315 constexpr explicit BFloat16_t(uint16_t v) noexcept { val = v; }
316
317 public:
318 using Base = onnxruntime_float16::BFloat16Impl<BFloat16_t>;
319
320 BFloat16_t() = default;
321
327 static constexpr BFloat16_t FromBits(uint16_t v) noexcept { return BFloat16_t(v); }
328
333 explicit BFloat16_t(float v) noexcept { val = Base::ToUint16Impl(v); }
334
339 float ToFloat() const noexcept { return Base::ToFloatImpl(); }
340
345 using Base::IsNegative;
346
351 using Base::IsNaN;
352
357 using Base::IsFinite;
358
363 using Base::IsPositiveInfinity;
364
369 using Base::IsNegativeInfinity;
370
375 using Base::IsInfinity;
376
381 using Base::IsNaNOrZero;
382
387 using Base::IsNormal;
388
393 using Base::IsSubnormal;
394
399 using Base::Abs;
400
405 using Base::Negate;
406
415 using Base::AreZero;
416
420 explicit operator float() const noexcept { return ToFloat(); }
421
422 // We do not have an inherited impl for the below operators
423 // as the internal class implements them a little differently
424 bool operator==(const BFloat16_t& rhs) const noexcept;
425 bool operator!=(const BFloat16_t& rhs) const noexcept { return !(*this == rhs); }
426 bool operator<(const BFloat16_t& rhs) const noexcept;
427};
428
429static_assert(sizeof(BFloat16_t) == sizeof(uint16_t), "Sizes must match");
430
437 uint8_t value;
438 constexpr Float8E4M3FN_t() noexcept : value(0) {}
439 constexpr Float8E4M3FN_t(uint8_t v) noexcept : value(v) {}
440 constexpr operator uint8_t() const noexcept { return value; }
441 // nan values are treated like any other value for operator ==, !=
442 constexpr bool operator==(const Float8E4M3FN_t& rhs) const noexcept { return value == rhs.value; };
443 constexpr bool operator!=(const Float8E4M3FN_t& rhs) const noexcept { return value != rhs.value; };
444};
445
446static_assert(sizeof(Float8E4M3FN_t) == sizeof(uint8_t), "Sizes must match");
447
454 uint8_t value;
455 constexpr Float8E4M3FNUZ_t() noexcept : value(0) {}
456 constexpr Float8E4M3FNUZ_t(uint8_t v) noexcept : value(v) {}
457 constexpr operator uint8_t() const noexcept { return value; }
458 // nan values are treated like any other value for operator ==, !=
459 constexpr bool operator==(const Float8E4M3FNUZ_t& rhs) const noexcept { return value == rhs.value; };
460 constexpr bool operator!=(const Float8E4M3FNUZ_t& rhs) const noexcept { return value != rhs.value; };
461};
462
463static_assert(sizeof(Float8E4M3FNUZ_t) == sizeof(uint8_t), "Sizes must match");
464
471 uint8_t value;
472 constexpr Float8E5M2_t() noexcept : value(0) {}
473 constexpr Float8E5M2_t(uint8_t v) noexcept : value(v) {}
474 constexpr operator uint8_t() const noexcept { return value; }
475 // nan values are treated like any other value for operator ==, !=
476 constexpr bool operator==(const Float8E5M2_t& rhs) const noexcept { return value == rhs.value; };
477 constexpr bool operator!=(const Float8E5M2_t& rhs) const noexcept { return value != rhs.value; };
478};
479
480static_assert(sizeof(Float8E5M2_t) == sizeof(uint8_t), "Sizes must match");
481
488 uint8_t value;
489 constexpr Float8E5M2FNUZ_t() noexcept : value(0) {}
490 constexpr Float8E5M2FNUZ_t(uint8_t v) noexcept : value(v) {}
491 constexpr operator uint8_t() const noexcept { return value; }
492 // nan values are treated like any other value for operator ==, !=
493 constexpr bool operator==(const Float8E5M2FNUZ_t& rhs) const noexcept { return value == rhs.value; };
494 constexpr bool operator!=(const Float8E5M2FNUZ_t& rhs) const noexcept { return value != rhs.value; };
495};
496
497static_assert(sizeof(Float8E5M2FNUZ_t) == sizeof(uint8_t), "Sizes must match");
498
499namespace detail {
500// This is used internally by the C++ API. This macro is to make it easy to generate overloaded methods for all of the various OrtRelease* functions for every Ort* type
501// This can't be done in the C API since C doesn't have function overloading.
502#define ORT_DEFINE_RELEASE(NAME) \
503 inline void OrtRelease(Ort##NAME* ptr) { GetApi().Release##NAME(ptr); }
504
505ORT_DEFINE_RELEASE(Allocator);
506ORT_DEFINE_RELEASE(MemoryInfo);
507ORT_DEFINE_RELEASE(CustomOpDomain);
508ORT_DEFINE_RELEASE(ThreadingOptions);
509ORT_DEFINE_RELEASE(Env);
510ORT_DEFINE_RELEASE(RunOptions);
511ORT_DEFINE_RELEASE(Session);
512ORT_DEFINE_RELEASE(SessionOptions);
513ORT_DEFINE_RELEASE(TensorTypeAndShapeInfo);
514ORT_DEFINE_RELEASE(SequenceTypeInfo);
515ORT_DEFINE_RELEASE(MapTypeInfo);
516ORT_DEFINE_RELEASE(TypeInfo);
517ORT_DEFINE_RELEASE(Value);
518ORT_DEFINE_RELEASE(ModelMetadata);
519ORT_DEFINE_RELEASE(IoBinding);
520ORT_DEFINE_RELEASE(ArenaCfg);
521ORT_DEFINE_RELEASE(Status);
522ORT_DEFINE_RELEASE(OpAttr);
523ORT_DEFINE_RELEASE(Op);
524ORT_DEFINE_RELEASE(KernelInfo);
525
526#undef ORT_DEFINE_RELEASE
527
531template <typename T>
532struct Unowned {
533 using Type = T;
534};
535
555template <typename T>
556struct Base {
557 using contained_type = T;
558
559 constexpr Base() = default;
560 constexpr explicit Base(contained_type* p) noexcept : p_{p} {}
562
563 Base(const Base&) = delete;
564 Base& operator=(const Base&) = delete;
565
566 Base(Base&& v) noexcept : p_{v.p_} { v.p_ = nullptr; }
567 Base& operator=(Base&& v) noexcept {
568 OrtRelease(p_);
569 p_ = v.release();
570 return *this;
571 }
572
573 constexpr operator contained_type*() const noexcept { return p_; }
574
578 T* p = p_;
579 p_ = nullptr;
580 return p;
581 }
582
583 protected:
585};
586
587// Undefined. For const types use Base<Unowned<const T>>
588template <typename T>
589struct Base<const T>;
590
598template <typename T>
599struct Base<Unowned<T>> {
601
602 constexpr Base() = default;
603 constexpr explicit Base(contained_type* p) noexcept : p_{p} {}
604
605 ~Base() = default;
606
607 Base(const Base&) = default;
608 Base& operator=(const Base&) = default;
609
610 Base(Base&& v) noexcept : p_{v.p_} { v.p_ = nullptr; }
611 Base& operator=(Base&& v) noexcept {
612 p_ = nullptr;
613 std::swap(p_, v.p_);
614 return *this;
615 }
616
617 constexpr operator contained_type*() const noexcept { return p_; }
618
619 protected:
621};
622
623// Light functor to release memory with OrtAllocator
626 explicit AllocatedFree(OrtAllocator* allocator)
627 : allocator_(allocator) {}
628 void operator()(void* ptr) const {
629 if (ptr) allocator_->Free(allocator_, ptr);
630 }
631};
632
633} // namespace detail
634
635struct AllocatorWithDefaultOptions;
636struct Env;
637struct TypeInfo;
638struct Value;
639struct ModelMetadata;
640
645using AllocatedStringPtr = std::unique_ptr<char, detail::AllocatedFree>;
646
651struct Status : detail::Base<OrtStatus> {
652 explicit Status(std::nullptr_t) noexcept {}
653 explicit Status(OrtStatus* status) noexcept;
654 explicit Status(const Exception&) noexcept;
655 explicit Status(const std::exception&) noexcept;
656 Status(const char* message, OrtErrorCode code) noexcept;
657 std::string GetErrorMessage() const;
659 bool IsOK() const noexcept;
660};
661
691
697struct Env : detail::Base<OrtEnv> {
698 explicit Env(std::nullptr_t) {}
699
701 Env(OrtLoggingLevel logging_level = ORT_LOGGING_LEVEL_WARNING, _In_ const char* logid = "");
702
704 Env(OrtLoggingLevel logging_level, const char* logid, OrtLoggingFunction logging_function, void* logger_param);
705
707 Env(const OrtThreadingOptions* tp_options, OrtLoggingLevel logging_level = ORT_LOGGING_LEVEL_WARNING, _In_ const char* logid = "");
708
710 Env(const OrtThreadingOptions* tp_options, OrtLoggingFunction logging_function, void* logger_param,
711 OrtLoggingLevel logging_level = ORT_LOGGING_LEVEL_WARNING, _In_ const char* logid = "");
712
714 explicit Env(OrtEnv* p) : Base<OrtEnv>{p} {}
715
718
720
721 Env& CreateAndRegisterAllocator(const OrtMemoryInfo* mem_info, const OrtArenaCfg* arena_cfg);
722
723 Env& CreateAndRegisterAllocatorV2(const std::string& provider_type, const OrtMemoryInfo* mem_info, const std::unordered_map<std::string, std::string>& options, const OrtArenaCfg* arena_cfg);
724};
725
729struct CustomOpDomain : detail::Base<OrtCustomOpDomain> {
730 explicit CustomOpDomain(std::nullptr_t) {}
731
733 explicit CustomOpDomain(const char* domain);
734
735 // This does not take ownership of the op, simply registers it.
736 void Add(const OrtCustomOp* op);
737};
738
742struct RunOptions : detail::Base<OrtRunOptions> {
743 explicit RunOptions(std::nullptr_t) {}
745
748
751
752 RunOptions& SetRunTag(const char* run_tag);
753 const char* GetRunTag() const;
754
755 RunOptions& AddConfigEntry(const char* config_key, const char* config_value);
756
763
769};
770
771namespace detail {
772// Utility function that returns a SessionOption config entry key for a specific custom operator.
773// Ex: custom_op.[custom_op_name].[config]
774std::string MakeCustomOpConfigEntryKey(const char* custom_op_name, const char* config);
775} // namespace detail
776
787 CustomOpConfigs() = default;
788 ~CustomOpConfigs() = default;
793
802 CustomOpConfigs& AddConfig(const char* custom_op_name, const char* config_key, const char* config_value);
803
812 const std::unordered_map<std::string, std::string>& GetFlattenedConfigs() const;
813
814 private:
815 std::unordered_map<std::string, std::string> flat_configs_;
816};
817
823struct SessionOptions;
824
825namespace detail {
826// we separate const-only methods because passing const ptr to non-const methods
827// is only discovered when inline methods are compiled which is counter-intuitive
828template <typename T>
829struct ConstSessionOptionsImpl : Base<T> {
830 using B = Base<T>;
831 using B::B;
832
833 SessionOptions Clone() const;
834
835 std::string GetConfigEntry(const char* config_key) const;
836 bool HasConfigEntry(const char* config_key) const;
837 std::string GetConfigEntryOrDefault(const char* config_key, const std::string& def);
838};
839
840template <typename T>
841struct SessionOptionsImpl : ConstSessionOptionsImpl<T> {
842 using B = ConstSessionOptionsImpl<T>;
843 using B::B;
844
845 SessionOptionsImpl& SetIntraOpNumThreads(int intra_op_num_threads);
846 SessionOptionsImpl& SetInterOpNumThreads(int inter_op_num_threads);
847 SessionOptionsImpl& SetGraphOptimizationLevel(GraphOptimizationLevel graph_optimization_level);
848 SessionOptionsImpl& SetDeterministicCompute(bool value);
849
850 SessionOptionsImpl& EnableCpuMemArena();
851 SessionOptionsImpl& DisableCpuMemArena();
852
853 SessionOptionsImpl& SetOptimizedModelFilePath(const ORTCHAR_T* optimized_model_file);
854
855 SessionOptionsImpl& EnableProfiling(const ORTCHAR_T* profile_file_prefix);
856 SessionOptionsImpl& DisableProfiling();
857
858 SessionOptionsImpl& EnableOrtCustomOps();
859
860 SessionOptionsImpl& EnableMemPattern();
861 SessionOptionsImpl& DisableMemPattern();
862
863 SessionOptionsImpl& SetExecutionMode(ExecutionMode execution_mode);
864
865 SessionOptionsImpl& SetLogId(const char* logid);
866 SessionOptionsImpl& SetLogSeverityLevel(int level);
867
868 SessionOptionsImpl& Add(OrtCustomOpDomain* custom_op_domain);
869
870 SessionOptionsImpl& DisablePerSessionThreads();
871
872 SessionOptionsImpl& AddConfigEntry(const char* config_key, const char* config_value);
873
874 SessionOptionsImpl& AddInitializer(const char* name, const OrtValue* ort_val);
875 SessionOptionsImpl& AddExternalInitializers(const std::vector<std::string>& names, const std::vector<Value>& ort_values);
876
877 SessionOptionsImpl& AppendExecutionProvider_CUDA(const OrtCUDAProviderOptions& provider_options);
878 SessionOptionsImpl& AppendExecutionProvider_CUDA_V2(const OrtCUDAProviderOptionsV2& provider_options);
879 SessionOptionsImpl& AppendExecutionProvider_ROCM(const OrtROCMProviderOptions& provider_options);
880 SessionOptionsImpl& AppendExecutionProvider_OpenVINO(const OrtOpenVINOProviderOptions& provider_options);
882 SessionOptionsImpl& AppendExecutionProvider_OpenVINO_V2(const std::unordered_map<std::string, std::string>& provider_options = {});
883 SessionOptionsImpl& AppendExecutionProvider_TensorRT(const OrtTensorRTProviderOptions& provider_options);
884 SessionOptionsImpl& AppendExecutionProvider_TensorRT_V2(const OrtTensorRTProviderOptionsV2& provider_options);
885 SessionOptionsImpl& AppendExecutionProvider_MIGraphX(const OrtMIGraphXProviderOptions& provider_options);
887 SessionOptionsImpl& AppendExecutionProvider_CANN(const OrtCANNProviderOptions& provider_options);
889 SessionOptionsImpl& AppendExecutionProvider_Dnnl(const OrtDnnlProviderOptions& provider_options);
891 SessionOptionsImpl& AppendExecutionProvider(const std::string& provider_name,
892 const std::unordered_map<std::string, std::string>& provider_options = {});
893
894 SessionOptionsImpl& SetCustomCreateThreadFn(OrtCustomCreateThreadFn ort_custom_create_thread_fn);
895 SessionOptionsImpl& SetCustomThreadCreationOptions(void* ort_custom_thread_creation_options);
896 SessionOptionsImpl& SetCustomJoinThreadFn(OrtCustomJoinThreadFn ort_custom_join_thread_fn);
897
901 SessionOptionsImpl& RegisterCustomOpsLibrary(const ORTCHAR_T* library_name, const CustomOpConfigs& custom_op_configs = {});
902
903 SessionOptionsImpl& RegisterCustomOpsUsingFunction(const char* function_name);
904
906 SessionOptionsImpl& AppendExecutionProvider_VitisAI(const std::unordered_map<std::string, std::string>& provider_options = {});
907};
908} // namespace detail
909
910using UnownedSessionOptions = detail::SessionOptionsImpl<detail::Unowned<OrtSessionOptions>>;
911using ConstSessionOptions = detail::ConstSessionOptionsImpl<detail::Unowned<const OrtSessionOptions>>;
912
916struct SessionOptions : detail::SessionOptionsImpl<OrtSessionOptions> {
917 explicit SessionOptions(std::nullptr_t) {}
919 explicit SessionOptions(OrtSessionOptions* p) : SessionOptionsImpl<OrtSessionOptions>{p} {}
922};
923
927struct ModelMetadata : detail::Base<OrtModelMetadata> {
928 explicit ModelMetadata(std::nullptr_t) {}
930
938
946
954
962
970
977 std::vector<AllocatedStringPtr> GetCustomMetadataMapKeysAllocated(OrtAllocator* allocator) const;
978
989
990 int64_t GetVersion() const;
991};
992
993struct IoBinding;
994
995namespace detail {
996
997// we separate const-only methods because passing const ptr to non-const methods
998// is only discovered when inline methods are compiled which is counter-intuitive
999template <typename T>
1001 using B = Base<T>;
1002 using B::B;
1003
1004 size_t GetInputCount() const;
1005 size_t GetOutputCount() const;
1007
1016
1025
1034
1035 uint64_t GetProfilingStartTimeNs() const;
1037
1038 TypeInfo GetInputTypeInfo(size_t index) const;
1039 TypeInfo GetOutputTypeInfo(size_t index) const;
1041};
1042
1043template <typename T>
1046 using B::B;
1047
1065 std::vector<Value> Run(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count,
1066 const char* const* output_names, size_t output_count);
1067
1071 void Run(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count,
1072 const char* const* output_names, Value* output_values, size_t output_count);
1073
1074 void Run(const RunOptions& run_options, const IoBinding&);
1075
1095 void RunAsync(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count,
1096 const char* const* output_names, Value* output_values, size_t output_count, RunAsyncCallbackFn callback, void* user_data);
1097
1105};
1106
1107} // namespace detail
1108
1111
1115struct Session : detail::SessionImpl<OrtSession> {
1116 explicit Session(std::nullptr_t) {}
1117 Session(const Env& env, const ORTCHAR_T* model_path, const SessionOptions& options);
1118 Session(const Env& env, const ORTCHAR_T* model_path, const SessionOptions& options,
1119 OrtPrepackedWeightsContainer* prepacked_weights_container);
1120 Session(const Env& env, const void* model_data, size_t model_data_length, const SessionOptions& options);
1121 Session(const Env& env, const void* model_data, size_t model_data_length, const SessionOptions& options,
1122 OrtPrepackedWeightsContainer* prepacked_weights_container);
1123
1124 ConstSession GetConst() const { return ConstSession{this->p_}; }
1125 UnownedSession GetUnowned() const { return UnownedSession{this->p_}; }
1126};
1127
1128namespace detail {
1129template <typename T>
1131 using B = Base<T>;
1132 using B::B;
1133
1134 std::string GetAllocatorName() const;
1136 int GetDeviceId() const;
1139
1140 template <typename U>
1141 bool operator==(const MemoryInfoImpl<U>& o) const;
1142};
1143} // namespace detail
1144
1145// Const object holder that does not own the underlying object
1147
1151struct MemoryInfo : detail::MemoryInfoImpl<OrtMemoryInfo> {
1153 explicit MemoryInfo(std::nullptr_t) {}
1154 explicit MemoryInfo(OrtMemoryInfo* p) : MemoryInfoImpl<OrtMemoryInfo>{p} {}
1155 MemoryInfo(const char* name, OrtAllocatorType type, int id, OrtMemType mem_type);
1156 ConstMemoryInfo GetConst() const { return ConstMemoryInfo{this->p_}; }
1157};
1158
1159namespace detail {
1160template <typename T>
1162 using B = Base<T>;
1163 using B::B;
1164
1166 size_t GetElementCount() const;
1167
1168 size_t GetDimensionsCount() const;
1169
1174 [[deprecated("use GetShape()")]] void GetDimensions(int64_t* values, size_t values_count) const;
1175
1176 void GetSymbolicDimensions(const char** values, size_t values_count) const;
1177
1178 std::vector<int64_t> GetShape() const;
1179};
1180
1181} // namespace detail
1182
1184
1189 explicit TensorTypeAndShapeInfo(std::nullptr_t) {}
1190 explicit TensorTypeAndShapeInfo(OrtTensorTypeAndShapeInfo* p) : TensorTypeAndShapeInfoImpl{p} {}
1192};
1193
1194namespace detail {
1195template <typename T>
1197 using B = Base<T>;
1198 using B::B;
1200};
1201
1202} // namespace detail
1203
1205
1209struct SequenceTypeInfo : detail::SequenceTypeInfoImpl<OrtSequenceTypeInfo> {
1210 explicit SequenceTypeInfo(std::nullptr_t) {}
1211 explicit SequenceTypeInfo(OrtSequenceTypeInfo* p) : SequenceTypeInfoImpl<OrtSequenceTypeInfo>{p} {}
1213};
1214
1215namespace detail {
1216template <typename T>
1218 using B = Base<T>;
1219 using B::B;
1221};
1222
1223} // namespace detail
1224
1225// This is always owned by the TypeInfo and can only be obtained from it.
1227
1228namespace detail {
1229template <typename T>
1236
1237} // namespace detail
1238
1240
1244struct MapTypeInfo : detail::MapTypeInfoImpl<OrtMapTypeInfo> {
1245 explicit MapTypeInfo(std::nullptr_t) {}
1246 explicit MapTypeInfo(OrtMapTypeInfo* p) : MapTypeInfoImpl<OrtMapTypeInfo>{p} {}
1247 ConstMapTypeInfo GetConst() const { return ConstMapTypeInfo{this->p_}; }
1248};
1249
1250namespace detail {
1251template <typename T>
1263} // namespace detail
1264
1270
1275struct TypeInfo : detail::TypeInfoImpl<OrtTypeInfo> {
1276 explicit TypeInfo(std::nullptr_t) {}
1277 explicit TypeInfo(OrtTypeInfo* p) : TypeInfoImpl<OrtTypeInfo>{p} {}
1278
1279 ConstTypeInfo GetConst() const { return ConstTypeInfo{this->p_}; }
1280};
1281
1282namespace detail {
1283// This structure is used to feed sparse tensor values
1284// information for use with FillSparseTensor<Format>() API
1285// if the data type for the sparse tensor values is numeric
1286// use data.p_data, otherwise, use data.str pointer to feed
1287// values. data.str is an array of const char* that are zero terminated.
1288// number of strings in the array must match shape size.
1289// For fully sparse tensors use shape {0} and set p_data/str
1290// to nullptr.
1292 const int64_t* values_shape;
1294 union {
1295 const void* p_data;
1296 const char** str;
1297 } data;
1298};
1299
1300// Provides a way to pass shape in a single
1301// argument
1302struct Shape {
1303 const int64_t* shape;
1305};
1306
1307template <typename T>
1309 using B = Base<T>;
1310 using B::B;
1311
1315 template <typename R>
1316 void GetOpaqueData(const char* domain, const char* type_name, R&) const;
1317
1318 bool IsTensor() const;
1319 bool HasValue() const;
1320
1321 size_t GetCount() const; // If a non tensor, returns 2 for map and N for sequence, where N is the number of elements
1322 Value GetValue(int index, OrtAllocator* allocator) const;
1323
1331
1346 void GetStringTensorContent(void* buffer, size_t buffer_length, size_t* offsets, size_t offsets_count) const;
1347
1354 template <typename R>
1355 const R* GetTensorData() const;
1356
1361 const void* GetTensorRawData() const;
1362
1370
1378
1384
1393 void GetStringTensorElement(size_t buffer_length, size_t element_index, void* buffer) const;
1394
1401 std::string GetStringTensorElement(size_t element_index) const;
1402
1409 size_t GetStringTensorElementLength(size_t element_index) const;
1410
1411#if !defined(DISABLE_SPARSE_TENSORS)
1419
1426
1435
1445 template <typename R>
1446 const R* GetSparseTensorIndicesData(OrtSparseIndicesFormat indices_format, size_t& num_indices) const;
1447
1452 bool IsSparseTensor() const;
1453
1462 template <typename R>
1463 const R* GetSparseTensorValues() const;
1464
1465#endif
1466};
1467
1468template <typename T>
1471 using B::B;
1472
1478 template <typename R>
1480
1486
1488 // Obtain a reference to an element of data at the location specified
1494 template <typename R>
1495 R& At(const std::vector<int64_t>& location);
1496
1502 void FillStringTensor(const char* const* s, size_t s_len);
1503
1509 void FillStringTensorElement(const char* s, size_t index);
1510
1523 char* GetResizedStringTensorElementBuffer(size_t index, size_t buffer_length);
1524
1525#if !defined(DISABLE_SPARSE_TENSORS)
1534 void UseCooIndices(int64_t* indices_data, size_t indices_num);
1535
1546 void UseCsrIndices(int64_t* inner_data, size_t inner_num, int64_t* outer_data, size_t outer_num);
1547
1556 void UseBlockSparseIndices(const Shape& indices_shape, int32_t* indices_data);
1557
1567 void FillSparseTensorCoo(const OrtMemoryInfo* data_mem_info, const OrtSparseValuesParam& values_param,
1568 const int64_t* indices_data, size_t indices_num);
1569
1581 void FillSparseTensorCsr(const OrtMemoryInfo* data_mem_info,
1582 const OrtSparseValuesParam& values,
1583 const int64_t* inner_indices_data, size_t inner_indices_num,
1584 const int64_t* outer_indices_data, size_t outer_indices_num);
1585
1596 const OrtSparseValuesParam& values,
1597 const Shape& indices_shape,
1598 const int32_t* indices_data);
1599
1600#endif
1601};
1602
1603} // namespace detail
1604
1607
1611struct Value : detail::ValueImpl<OrtValue> {
1615
1616 explicit Value(std::nullptr_t) {}
1617 explicit Value(OrtValue* p) : Base{p} {}
1618 Value(Value&&) = default;
1619 Value& operator=(Value&&) = default;
1620
1621 ConstValue GetConst() const { return ConstValue{this->p_}; }
1622 UnownedValue GetUnowned() const { return UnownedValue{this->p_}; }
1623
1632 template <typename T>
1633 static Value CreateTensor(const OrtMemoryInfo* info, T* p_data, size_t p_data_element_count, const int64_t* shape, size_t shape_len);
1634
1644 static Value CreateTensor(const OrtMemoryInfo* info, void* p_data, size_t p_data_byte_count, const int64_t* shape, size_t shape_len,
1646
1658 template <typename T>
1659 static Value CreateTensor(OrtAllocator* allocator, const int64_t* shape, size_t shape_len);
1660
1672 static Value CreateTensor(OrtAllocator* allocator, const int64_t* shape, size_t shape_len, ONNXTensorElementDataType type);
1673
1682 static Value CreateMap(const Value& keys, const Value& values);
1683
1691 static Value CreateSequence(const std::vector<Value>& values);
1692
1701 template <typename T>
1702 static Value CreateOpaque(const char* domain, const char* type_name, const T& value);
1703
1704#if !defined(DISABLE_SPARSE_TENSORS)
1715 template <typename T>
1716 static Value CreateSparseTensor(const OrtMemoryInfo* info, T* p_data, const Shape& dense_shape,
1717 const Shape& values_shape);
1718
1735 static Value CreateSparseTensor(const OrtMemoryInfo* info, void* p_data, const Shape& dense_shape,
1736 const Shape& values_shape, ONNXTensorElementDataType type);
1737
1747 template <typename T>
1748 static Value CreateSparseTensor(OrtAllocator* allocator, const Shape& dense_shape);
1749
1761 static Value CreateSparseTensor(OrtAllocator* allocator, const Shape& dense_shape, ONNXTensorElementDataType type);
1762
1763#endif // !defined(DISABLE_SPARSE_TENSORS)
1764};
1765
1773 MemoryAllocation(OrtAllocator* allocator, void* p, size_t size);
1778 MemoryAllocation& operator=(MemoryAllocation&&) noexcept;
1779
1780 void* get() { return p_; }
1781 size_t size() const { return size_; }
1782
1783 private:
1784 OrtAllocator* allocator_;
1785 void* p_;
1786 size_t size_;
1787};
1788
1789namespace detail {
1790template <typename T>
1791struct AllocatorImpl : Base<T> {
1792 using B = Base<T>;
1793 using B::B;
1794
1795 void* Alloc(size_t size);
1796 MemoryAllocation GetAllocation(size_t size);
1797 void Free(void* p);
1798 ConstMemoryInfo GetInfo() const;
1799};
1800
1801} // namespace detail
1802
1806struct AllocatorWithDefaultOptions : detail::AllocatorImpl<detail::Unowned<OrtAllocator>> {
1807 explicit AllocatorWithDefaultOptions(std::nullptr_t) {}
1809};
1810
1814struct Allocator : detail::AllocatorImpl<OrtAllocator> {
1815 explicit Allocator(std::nullptr_t) {}
1816 Allocator(const Session& session, const OrtMemoryInfo*);
1817};
1818
1819using UnownedAllocator = detail::AllocatorImpl<detail::Unowned<OrtAllocator>>;
1820
1821namespace detail {
1822namespace binding_utils {
1823// Bring these out of template
1824std::vector<std::string> GetOutputNamesHelper(const OrtIoBinding* binding, OrtAllocator*);
1825std::vector<Value> GetOutputValuesHelper(const OrtIoBinding* binding, OrtAllocator*);
1826} // namespace binding_utils
1827
1828template <typename T>
1830 using B = Base<T>;
1831 using B::B;
1832
1833 std::vector<std::string> GetOutputNames() const;
1834 std::vector<std::string> GetOutputNames(OrtAllocator*) const;
1835 std::vector<Value> GetOutputValues() const;
1836 std::vector<Value> GetOutputValues(OrtAllocator*) const;
1837};
1838
1839template <typename T>
1842 using B::B;
1843
1844 void BindInput(const char* name, const Value&);
1845 void BindOutput(const char* name, const Value&);
1846 void BindOutput(const char* name, const OrtMemoryInfo*);
1851};
1852
1853} // namespace detail
1854
1857
1861struct IoBinding : detail::IoBindingImpl<OrtIoBinding> {
1862 explicit IoBinding(std::nullptr_t) {}
1863 explicit IoBinding(Session& session);
1864 ConstIoBinding GetConst() const { return ConstIoBinding{this->p_}; }
1865 UnownedIoBinding GetUnowned() const { return UnownedIoBinding{this->p_}; }
1866};
1867
1872struct ArenaCfg : detail::Base<OrtArenaCfg> {
1873 explicit ArenaCfg(std::nullptr_t) {}
1882 ArenaCfg(size_t max_mem, int arena_extend_strategy, int initial_chunk_size_bytes, int max_dead_bytes_per_chunk);
1883};
1884
1885//
1886// Custom OPs (only needed to implement custom OPs)
1887//
1888
1892struct OpAttr : detail::Base<OrtOpAttr> {
1893 OpAttr(const char* name, const void* data, int len, OrtOpAttrType type);
1894};
1895
1904#define ORT_CXX_LOG(logger, message_severity, message) \
1905 do { \
1906 if (message_severity >= logger.GetLoggingSeverityLevel()) { \
1907 Ort::ThrowOnError(logger.LogMessage(message_severity, ORT_FILE, __LINE__, \
1908 static_cast<const char*>(__FUNCTION__), message)); \
1909 } \
1910 } while (false)
1911
1920#define ORT_CXX_LOG_NOEXCEPT(logger, message_severity, message) \
1921 do { \
1922 if (message_severity >= logger.GetLoggingSeverityLevel()) { \
1923 static_cast<void>(logger.LogMessage(message_severity, ORT_FILE, __LINE__, \
1924 static_cast<const char*>(__FUNCTION__), message)); \
1925 } \
1926 } while (false)
1927
1939#define ORT_CXX_LOGF(logger, message_severity, /*format,*/...) \
1940 do { \
1941 if (message_severity >= logger.GetLoggingSeverityLevel()) { \
1942 Ort::ThrowOnError(logger.LogFormattedMessage(message_severity, ORT_FILE, __LINE__, \
1943 static_cast<const char*>(__FUNCTION__), __VA_ARGS__)); \
1944 } \
1945 } while (false)
1946
1958#define ORT_CXX_LOGF_NOEXCEPT(logger, message_severity, /*format,*/...) \
1959 do { \
1960 if (message_severity >= logger.GetLoggingSeverityLevel()) { \
1961 static_cast<void>(logger.LogFormattedMessage(message_severity, ORT_FILE, __LINE__, \
1962 static_cast<const char*>(__FUNCTION__), __VA_ARGS__)); \
1963 } \
1964 } while (false)
1965
1976struct Logger {
1980 Logger() = default;
1981
1985 explicit Logger(std::nullptr_t) {}
1986
1993 explicit Logger(const OrtLogger* logger);
1994
1995 ~Logger() = default;
1996
1997 Logger(const Logger&) = default;
1998 Logger& operator=(const Logger&) = default;
1999
2000 Logger(Logger&& v) noexcept = default;
2001 Logger& operator=(Logger&& v) noexcept = default;
2002
2009
2022 Status LogMessage(OrtLoggingLevel log_severity_level, const ORTCHAR_T* file_path, int line_number,
2023 const char* func_name, const char* message) const noexcept;
2024
2039 template <typename... Args>
2040 Status LogFormattedMessage(OrtLoggingLevel log_severity_level, const ORTCHAR_T* file_path, int line_number,
2041 const char* func_name, const char* format, Args&&... args) const noexcept;
2042
2043 private:
2044 const OrtLogger* logger_{};
2045 OrtLoggingLevel cached_severity_level_{};
2046};
2047
2056 size_t GetInputCount() const;
2057 size_t GetOutputCount() const;
2058 ConstValue GetInput(size_t index) const;
2059 UnownedValue GetOutput(size_t index, const int64_t* dim_values, size_t dim_count) const;
2060 UnownedValue GetOutput(size_t index, const std::vector<int64_t>& dims) const;
2061 void* GetGPUComputeStream() const;
2063 OrtAllocator* GetAllocator(const OrtMemoryInfo& memory_info) const;
2064 OrtKernelContext* GetOrtKernelContext() const { return ctx_; }
2065 void ParallelFor(void (*fn)(void*, size_t), size_t total, size_t num_batch, void* usr_data) const;
2066
2067 private:
2068 OrtKernelContext* ctx_;
2069};
2070
2071struct KernelInfo;
2072
2073namespace detail {
2074namespace attr_utils {
2075void GetAttr(const OrtKernelInfo* p, const char* name, float&);
2076void GetAttr(const OrtKernelInfo* p, const char* name, int64_t&);
2077void GetAttr(const OrtKernelInfo* p, const char* name, std::string&);
2078void GetAttrs(const OrtKernelInfo* p, const char* name, std::vector<float>&);
2079void GetAttrs(const OrtKernelInfo* p, const char* name, std::vector<int64_t>&);
2080} // namespace attr_utils
2081
2082template <typename T>
2083struct KernelInfoImpl : Base<T> {
2084 using B = Base<T>;
2085 using B::B;
2086
2087 KernelInfo Copy() const;
2088
2089 template <typename R> // R is only implemented for float, int64_t, and string
2090 R GetAttribute(const char* name) const {
2091 R val;
2092 attr_utils::GetAttr(this->p_, name, val);
2093 return val;
2094 }
2095
2096 template <typename R> // R is only implemented for std::vector<float>, std::vector<int64_t>
2097 std::vector<R> GetAttributes(const char* name) const {
2098 std::vector<R> result;
2099 attr_utils::GetAttrs(this->p_, name, result);
2100 return result;
2101 }
2102
2103 Value GetTensorAttribute(const char* name, OrtAllocator* allocator) const;
2104
2105 size_t GetInputCount() const;
2106 size_t GetOutputCount() const;
2107
2108 std::string GetInputName(size_t index) const;
2109 std::string GetOutputName(size_t index) const;
2110
2111 TypeInfo GetInputTypeInfo(size_t index) const;
2112 TypeInfo GetOutputTypeInfo(size_t index) const;
2113
2114 ConstValue GetTensorConstantInput(size_t index, int* is_constant) const;
2115
2116 std::string GetNodeName() const;
2117 Logger GetLogger() const;
2118};
2119
2120} // namespace detail
2121
2122using ConstKernelInfo = detail::KernelInfoImpl<detail::Unowned<const OrtKernelInfo>>;
2123
2130struct KernelInfo : detail::KernelInfoImpl<OrtKernelInfo> {
2131 explicit KernelInfo(std::nullptr_t) {}
2132 explicit KernelInfo(OrtKernelInfo* info);
2133 ConstKernelInfo GetConst() const { return ConstKernelInfo{this->p_}; }
2134};
2135
2139struct Op : detail::Base<OrtOp> {
2140 explicit Op(std::nullptr_t) {}
2141
2142 explicit Op(OrtOp*);
2143
2144 static Op Create(const OrtKernelInfo* info, const char* op_name, const char* domain,
2145 int version, const char** type_constraint_names,
2146 const ONNXTensorElementDataType* type_constraint_values,
2147 size_t type_constraint_count,
2148 const OpAttr* attr_values,
2149 size_t attr_count,
2150 size_t input_count, size_t output_count);
2151
2152 void Invoke(const OrtKernelContext* context,
2153 const Value* input_values,
2154 size_t input_count,
2155 Value* output_values,
2156 size_t output_count);
2157
2158 // For easier refactoring
2159 void Invoke(const OrtKernelContext* context,
2160 const OrtValue* const* input_values,
2161 size_t input_count,
2162 OrtValue* const* output_values,
2163 size_t output_count);
2164};
2165
2171 SymbolicInteger(int64_t i) : i_(i), is_int_(true){};
2172 SymbolicInteger(const char* s) : s_(s), is_int_(false){};
2175
2178
2179 bool operator==(const SymbolicInteger& dim) const {
2180 if (is_int_ == dim.is_int_) {
2181 if (is_int_) {
2182 return i_ == dim.i_;
2183 } else {
2184 return std::string{s_} == std::string{dim.s_};
2185 }
2186 }
2187 return false;
2188 }
2189
2190 bool IsInt() const { return is_int_; }
2191 int64_t AsInt() const { return i_; }
2192 const char* AsSym() const { return s_; }
2193
2194 static constexpr int INVALID_INT_DIM = -2;
2195
2196 private:
2197 union {
2198 int64_t i_;
2199 const char* s_;
2200 };
2201 bool is_int_;
2202 };
2203
2204 using Shape = std::vector<SymbolicInteger>;
2205
2207
2208 const Shape& GetInputShape(size_t indice) const { return input_shapes_.at(indice); }
2209
2210 size_t GetInputCount() const { return input_shapes_.size(); }
2211
2212 Status SetOutputShape(size_t indice, const Shape& shape);
2213
2214 int64_t GetAttrInt(const char* attr_name);
2215
2216 using Ints = std::vector<int64_t>;
2217 Ints GetAttrInts(const char* attr_name);
2218
2219 float GetAttrFloat(const char* attr_name);
2220
2221 using Floats = std::vector<float>;
2222 Floats GetAttrFloats(const char* attr_name);
2223
2224 std::string GetAttrString(const char* attr_name);
2225
2226 using Strings = std::vector<std::string>;
2227 Strings GetAttrStrings(const char* attr_name);
2228
2229 private:
2230 const OrtOpAttr* GetAttrHdl(const char* attr_name) const;
2231 const OrtApi* ort_api_;
2233 std::vector<Shape> input_shapes_;
2234};
2235
2237
2238#define MAX_CUSTOM_OP_END_VER (1UL << 31) - 1
2239
2240template <typename TOp, typename TKernel, bool WithStatus = false>
2244 OrtCustomOp::GetName = [](const OrtCustomOp* this_) { return static_cast<const TOp*>(this_)->GetName(); };
2245
2246 OrtCustomOp::GetExecutionProviderType = [](const OrtCustomOp* this_) { return static_cast<const TOp*>(this_)->GetExecutionProviderType(); };
2247
2248 OrtCustomOp::GetInputTypeCount = [](const OrtCustomOp* this_) { return static_cast<const TOp*>(this_)->GetInputTypeCount(); };
2249 OrtCustomOp::GetInputType = [](const OrtCustomOp* this_, size_t index) { return static_cast<const TOp*>(this_)->GetInputType(index); };
2250 OrtCustomOp::GetInputMemoryType = [](const OrtCustomOp* this_, size_t index) { return static_cast<const TOp*>(this_)->GetInputMemoryType(index); };
2251
2252 OrtCustomOp::GetOutputTypeCount = [](const OrtCustomOp* this_) { return static_cast<const TOp*>(this_)->GetOutputTypeCount(); };
2253 OrtCustomOp::GetOutputType = [](const OrtCustomOp* this_, size_t index) { return static_cast<const TOp*>(this_)->GetOutputType(index); };
2254
2255#if defined(_MSC_VER) && !defined(__clang__)
2256#pragma warning(push)
2257#pragma warning(disable : 26409)
2258#endif
2259 OrtCustomOp::KernelDestroy = [](void* op_kernel) { delete static_cast<TKernel*>(op_kernel); };
2260#if defined(_MSC_VER) && !defined(__clang__)
2261#pragma warning(pop)
2262#endif
2263 OrtCustomOp::GetInputCharacteristic = [](const OrtCustomOp* this_, size_t index) { return static_cast<const TOp*>(this_)->GetInputCharacteristic(index); };
2264 OrtCustomOp::GetOutputCharacteristic = [](const OrtCustomOp* this_, size_t index) { return static_cast<const TOp*>(this_)->GetOutputCharacteristic(index); };
2265
2266 OrtCustomOp::GetVariadicInputMinArity = [](const OrtCustomOp* this_) { return static_cast<const TOp*>(this_)->GetVariadicInputMinArity(); };
2267 OrtCustomOp::GetVariadicInputHomogeneity = [](const OrtCustomOp* this_) { return static_cast<int>(static_cast<const TOp*>(this_)->GetVariadicInputHomogeneity()); };
2268 OrtCustomOp::GetVariadicOutputMinArity = [](const OrtCustomOp* this_) { return static_cast<const TOp*>(this_)->GetVariadicOutputMinArity(); };
2269 OrtCustomOp::GetVariadicOutputHomogeneity = [](const OrtCustomOp* this_) { return static_cast<int>(static_cast<const TOp*>(this_)->GetVariadicOutputHomogeneity()); };
2270#ifdef __cpp_if_constexpr
2271 if constexpr (WithStatus) {
2272#else
2273 if (WithStatus) {
2274#endif
2275 OrtCustomOp::CreateKernelV2 = [](const OrtCustomOp* this_, const OrtApi* api, const OrtKernelInfo* info, void** op_kernel) -> OrtStatusPtr {
2276 return static_cast<const TOp*>(this_)->CreateKernelV2(*api, info, op_kernel);
2277 };
2278 OrtCustomOp::KernelComputeV2 = [](void* op_kernel, OrtKernelContext* context) -> OrtStatusPtr {
2279 return static_cast<TKernel*>(op_kernel)->ComputeV2(context);
2280 };
2281 } else {
2284
2285 OrtCustomOp::CreateKernel = [](const OrtCustomOp* this_, const OrtApi* api, const OrtKernelInfo* info) { return static_cast<const TOp*>(this_)->CreateKernel(*api, info); };
2286 OrtCustomOp::KernelCompute = [](void* op_kernel, OrtKernelContext* context) {
2287 static_cast<TKernel*>(op_kernel)->Compute(context);
2288 };
2289 }
2290
2291 SetShapeInferFn<TOp>(0);
2292
2293 OrtCustomOp::GetStartVersion = [](const OrtCustomOp* this_) {
2294 return static_cast<const TOp*>(this_)->start_ver_;
2295 };
2296
2297 OrtCustomOp::GetEndVersion = [](const OrtCustomOp* this_) {
2298 return static_cast<const TOp*>(this_)->end_ver_;
2299 };
2300 }
2301
2302 // Default implementation of GetExecutionProviderType that returns nullptr to default to the CPU provider
2303 const char* GetExecutionProviderType() const { return nullptr; }
2304
2305 // Default implementations of GetInputCharacteristic() and GetOutputCharacteristic() below
2306 // (inputs and outputs are required by default)
2308 return OrtCustomOpInputOutputCharacteristic::INPUT_OUTPUT_REQUIRED;
2309 }
2310
2312 return OrtCustomOpInputOutputCharacteristic::INPUT_OUTPUT_REQUIRED;
2313 }
2314
2315 // Default implemention of GetInputMemoryType() that returns OrtMemTypeDefault
2316 OrtMemType GetInputMemoryType(size_t /*index*/) const {
2317 return OrtMemTypeDefault;
2318 }
2319
2320 // Default implementation of GetVariadicInputMinArity() returns 1 to specify that a variadic input
2321 // should expect at least 1 argument.
2323 return 1;
2324 }
2325
2326 // Default implementation of GetVariadicInputHomegeneity() returns true to specify that all arguments
2327 // to a variadic input should be of the same type.
2329 return true;
2330 }
2331
2332 // Default implementation of GetVariadicOutputMinArity() returns 1 to specify that a variadic output
2333 // should produce at least 1 output value.
2335 return 1;
2336 }
2337
2338 // Default implementation of GetVariadicOutputHomegeneity() returns true to specify that all output values
2339 // produced by a variadic output should be of the same type.
2341 return true;
2342 }
2343
2344 // Declare list of session config entries used by this Custom Op.
2345 // Implement this function in order to get configs from CustomOpBase::GetSessionConfigs().
2346 // This default implementation returns an empty vector of config entries.
2347 std::vector<std::string> GetSessionConfigKeys() const {
2348 return std::vector<std::string>{};
2349 }
2350
2351 template <typename C>
2352 decltype(&C::InferOutputShape) SetShapeInferFn(decltype(&C::InferOutputShape)) {
2354 ShapeInferContext ctx(&GetApi(), ort_ctx);
2355 return C::InferOutputShape(ctx);
2356 };
2357 return {};
2358 }
2359
2360 template <typename C>
2364
2365 protected:
2366 // Helper function that returns a map of session config entries specified by CustomOpBase::GetSessionConfigKeys.
2367 void GetSessionConfigs(std::unordered_map<std::string, std::string>& out, ConstSessionOptions options) const;
2368
2369 int start_ver_ = 1;
2370 int end_ver_ = MAX_CUSTOM_OP_END_VER;
2371};
2372
2373} // namespace Ort
2374
2375#include "onnxruntime_cxx_inline.h"
struct OrtMemoryInfo OrtMemoryInfo
Definition onnxruntime_c_api.h:279
struct OrtKernelInfo OrtKernelInfo
Definition onnxruntime_c_api.h:359
OrtLoggingLevel
Logging severity levels.
Definition onnxruntime_c_api.h:234
OrtMemoryInfoDeviceType
This mimics OrtDevice type constants so they can be returned in the API.
Definition onnxruntime_c_api.h:383
struct OrtShapeInferContext OrtShapeInferContext
Definition onnxruntime_c_api.h:303
void(* OrtLoggingFunction)(void *param, OrtLoggingLevel severity, const char *category, const char *logid, const char *code_location, const char *message)
Definition onnxruntime_c_api.h:324
void(* OrtCustomJoinThreadFn)(OrtCustomThreadHandle ort_custom_thread_handle)
Custom thread join function.
Definition onnxruntime_c_api.h:702
OrtCustomOpInputOutputCharacteristic
Definition onnxruntime_c_api.h:4630
struct OrtTensorRTProviderOptionsV2 OrtTensorRTProviderOptionsV2
Definition onnxruntime_c_api.h:296
struct OrtOpAttr OrtOpAttr
Definition onnxruntime_c_api.h:301
struct OrtThreadingOptions OrtThreadingOptions
Definition onnxruntime_c_api.h:293
struct OrtSequenceTypeInfo OrtSequenceTypeInfo
Definition onnxruntime_c_api.h:287
struct OrtDnnlProviderOptions OrtDnnlProviderOptions
Definition onnxruntime_c_api.h:299
OrtSparseIndicesFormat
Definition onnxruntime_c_api.h:223
struct OrtPrepackedWeightsContainer OrtPrepackedWeightsContainer
Definition onnxruntime_c_api.h:295
struct OrtCustomOpDomain OrtCustomOpDomain
Definition onnxruntime_c_api.h:290
struct OrtIoBinding OrtIoBinding
Definition onnxruntime_c_api.h:280
OrtAllocatorType
Definition onnxruntime_c_api.h:365
struct OrtOp OrtOp
Definition onnxruntime_c_api.h:300
struct OrtModelMetadata OrtModelMetadata
Definition onnxruntime_c_api.h:291
struct OrtTypeInfo OrtTypeInfo
Definition onnxruntime_c_api.h:284
struct OrtTensorTypeAndShapeInfo OrtTensorTypeAndShapeInfo
Definition onnxruntime_c_api.h:285
struct OrtCUDAProviderOptionsV2 OrtCUDAProviderOptionsV2
Definition onnxruntime_c_api.h:297
struct OrtKernelContext OrtKernelContext
Definition onnxruntime_c_api.h:361
struct OrtCANNProviderOptions OrtCANNProviderOptions
Definition onnxruntime_c_api.h:298
void(* RunAsyncCallbackFn)(void *user_data, OrtValue **outputs, size_t num_outputs, OrtStatusPtr status)
Callback function for RunAsync.
Definition onnxruntime_c_api.h:713
struct OrtSessionOptions OrtSessionOptions
Definition onnxruntime_c_api.h:289
struct OrtValue OrtValue
Definition onnxruntime_c_api.h:282
GraphOptimizationLevel
Graph optimization level.
Definition onnxruntime_c_api.h:333
OrtStatus * OrtStatusPtr
Definition onnxruntime_c_api.h:308
OrtMemType
Memory types for allocated memory, execution provider specific types should be extended in each provi...
Definition onnxruntime_c_api.h:374
OrtSparseFormat
Definition onnxruntime_c_api.h:215
ONNXType
Definition onnxruntime_c_api.h:203
struct OrtEnv OrtEnv
Definition onnxruntime_c_api.h:277
OrtErrorCode
Definition onnxruntime_c_api.h:242
struct OrtStatus OrtStatus
Definition onnxruntime_c_api.h:278
#define ORT_API_VERSION
The API version defined in this header.
Definition onnxruntime_c_api.h:41
struct OrtLogger OrtLogger
Definition onnxruntime_c_api.h:302
struct OrtMapTypeInfo OrtMapTypeInfo
Definition onnxruntime_c_api.h:286
struct OrtArenaCfg OrtArenaCfg
Definition onnxruntime_c_api.h:294
ExecutionMode
Definition onnxruntime_c_api.h:340
OrtOpAttrType
Definition onnxruntime_c_api.h:257
OrtCustomThreadHandle(* OrtCustomCreateThreadFn)(void *ort_custom_thread_creation_options, OrtThreadWorkerFn ort_thread_worker_fn, void *ort_worker_fn_param)
Ort custom thread creation function.
Definition onnxruntime_c_api.h:695
ONNXTensorElementDataType
Definition onnxruntime_c_api.h:177
const OrtApiBase * OrtGetApiBase(void)
The Onnxruntime library's entry point to access the C API.
@ ORT_LOGGING_LEVEL_WARNING
Warning messages.
Definition onnxruntime_c_api.h:237
@ OrtMemTypeDefault
The default allocator for execution provider.
Definition onnxruntime_c_api.h:378
std::vector< Value > GetOutputValuesHelper(const OrtIoBinding *binding, OrtAllocator *)
std::vector< std::string > GetOutputNamesHelper(const OrtIoBinding *binding, OrtAllocator *)
void OrtRelease(OrtAllocator *ptr)
Definition onnxruntime_cxx_api.h:505
std::string MakeCustomOpConfigEntryKey(const char *custom_op_name, const char *config)
All C++ Onnxruntime APIs are defined inside this namespace.
Definition onnxruntime_cxx_api.h:47
std::unique_ptr< char, detail::AllocatedFree > AllocatedStringPtr
unique_ptr typedef used to own strings allocated by OrtAllocators and release them at the end of the ...
Definition onnxruntime_cxx_api.h:645
detail::ConstSessionOptionsImpl< detail::Unowned< const OrtSessionOptions > > ConstSessionOptions
Definition onnxruntime_cxx_api.h:911
detail::KernelInfoImpl< detail::Unowned< const OrtKernelInfo > > ConstKernelInfo
Definition onnxruntime_cxx_api.h:2122
const OrtApi & GetApi() noexcept
This returns a reference to the OrtApi interface in use.
Definition onnxruntime_cxx_api.h:124
detail::AllocatorImpl< detail::Unowned< OrtAllocator > > UnownedAllocator
Definition onnxruntime_cxx_api.h:1819
detail::SessionOptionsImpl< detail::Unowned< OrtSessionOptions > > UnownedSessionOptions
Definition onnxruntime_cxx_api.h:910
std::string GetBuildInfoString()
This function returns the onnxruntime build information: including git branch, git commit id,...
std::string GetVersionString()
This function returns the onnxruntime version string.
std::vector< std::string > GetAvailableProviders()
This is a C++ wrapper for OrtApi::GetAvailableProviders() and returns a vector of strings representin...
Ort::Status(*)(Ort::ShapeInferContext &) ShapeInferFn
Definition onnxruntime_cxx_api.h:2236
Wrapper around OrtAllocator.
Definition onnxruntime_cxx_api.h:1814
Allocator(const Session &session, const OrtMemoryInfo *)
Allocator(std::nullptr_t)
Convenience to create a class member and then replace with an instance.
Definition onnxruntime_cxx_api.h:1815
Wrapper around OrtAllocator default instance that is owned by Onnxruntime.
Definition onnxruntime_cxx_api.h:1806
AllocatorWithDefaultOptions(std::nullptr_t)
Convenience to create a class member and then replace with an instance.
Definition onnxruntime_cxx_api.h:1807
it is a structure that represents the configuration of an arena based allocator
Definition onnxruntime_cxx_api.h:1872
ArenaCfg(std::nullptr_t)
Create an empty ArenaCfg object, must be assigned a valid one to be used.
Definition onnxruntime_cxx_api.h:1873
ArenaCfg(size_t max_mem, int arena_extend_strategy, int initial_chunk_size_bytes, int max_dead_bytes_per_chunk)
bfloat16 (Brain Floating Point) data type
Definition onnxruntime_cxx_api.h:306
bool operator==(const BFloat16_t &rhs) const noexcept
onnxruntime_float16::BFloat16Impl< BFloat16_t > Base
Definition onnxruntime_cxx_api.h:318
BFloat16_t()=default
static constexpr BFloat16_t FromBits(uint16_t v) noexcept
Explicit conversion to uint16_t representation of bfloat16.
Definition onnxruntime_cxx_api.h:327
bool operator!=(const BFloat16_t &rhs) const noexcept
Definition onnxruntime_cxx_api.h:425
BFloat16_t(float v) noexcept
__ctor from float. Float is converted into bfloat16 16-bit representation.
Definition onnxruntime_cxx_api.h:333
float ToFloat() const noexcept
Converts bfloat16 to float.
Definition onnxruntime_cxx_api.h:339
bool operator<(const BFloat16_t &rhs) const noexcept
Definition onnxruntime_cxx_api.h:2241
OrtCustomOpInputOutputCharacteristic GetOutputCharacteristic(size_t) const
Definition onnxruntime_cxx_api.h:2311
OrtCustomOpInputOutputCharacteristic GetInputCharacteristic(size_t) const
Definition onnxruntime_cxx_api.h:2307
OrtMemType GetInputMemoryType(size_t) const
Definition onnxruntime_cxx_api.h:2316
std::vector< std::string > GetSessionConfigKeys() const
Definition onnxruntime_cxx_api.h:2347
bool GetVariadicInputHomogeneity() const
Definition onnxruntime_cxx_api.h:2328
int GetVariadicInputMinArity() const
Definition onnxruntime_cxx_api.h:2322
void SetShapeInferFn(...)
Definition onnxruntime_cxx_api.h:2361
CustomOpBase()
Definition onnxruntime_cxx_api.h:2242
bool GetVariadicOutputHomogeneity() const
Definition onnxruntime_cxx_api.h:2340
int GetVariadicOutputMinArity() const
Definition onnxruntime_cxx_api.h:2334
decltype(&C::InferOutputShape) SetShapeInferFn(decltype(&C::InferOutputShape))
Definition onnxruntime_cxx_api.h:2352
const char * GetExecutionProviderType() const
Definition onnxruntime_cxx_api.h:2303
void GetSessionConfigs(std::unordered_map< std::string, std::string > &out, ConstSessionOptions options) const
Class that represents session configuration entries for one or more custom operators.
Definition onnxruntime_cxx_api.h:786
~CustomOpConfigs()=default
CustomOpConfigs & AddConfig(const char *custom_op_name, const char *config_key, const char *config_value)
Adds a session configuration entry/value for a specific custom operator.
CustomOpConfigs & operator=(CustomOpConfigs &&o)=default
CustomOpConfigs(CustomOpConfigs &&o)=default
CustomOpConfigs()=default
const std::unordered_map< std::string, std::string > & GetFlattenedConfigs() const
Returns a flattened map of custom operator configuration entries and their values.
CustomOpConfigs(const CustomOpConfigs &)=default
CustomOpConfigs & operator=(const CustomOpConfigs &)=default
Custom Op Domain.
Definition onnxruntime_cxx_api.h:729
CustomOpDomain(std::nullptr_t)
Create an empty CustomOpDomain object, must be assigned a valid one to be used.
Definition onnxruntime_cxx_api.h:730
CustomOpDomain(const char *domain)
Wraps OrtApi::CreateCustomOpDomain.
void Add(const OrtCustomOp *op)
Wraps CustomOpDomain_Add.
The Env (Environment)
Definition onnxruntime_cxx_api.h:697
Env & EnableTelemetryEvents()
Wraps OrtApi::EnableTelemetryEvents.
Env(OrtEnv *p)
C Interop Helper.
Definition onnxruntime_cxx_api.h:714
Env & CreateAndRegisterAllocatorV2(const std::string &provider_type, const OrtMemoryInfo *mem_info, const std::unordered_map< std::string, std::string > &options, const OrtArenaCfg *arena_cfg)
Wraps OrtApi::CreateAndRegisterAllocatorV2.
Env(std::nullptr_t)
Create an empty Env object, must be assigned a valid one to be used.
Definition onnxruntime_cxx_api.h:698
Env(OrtLoggingLevel logging_level=ORT_LOGGING_LEVEL_WARNING, const char *logid="")
Wraps OrtApi::CreateEnv.
Env(const OrtThreadingOptions *tp_options, OrtLoggingLevel logging_level=ORT_LOGGING_LEVEL_WARNING, const char *logid="")
Wraps OrtApi::CreateEnvWithGlobalThreadPools.
Env(const OrtThreadingOptions *tp_options, OrtLoggingFunction logging_function, void *logger_param, OrtLoggingLevel logging_level=ORT_LOGGING_LEVEL_WARNING, const char *logid="")
Wraps OrtApi::CreateEnvWithCustomLoggerAndGlobalThreadPools.
Env(OrtLoggingLevel logging_level, const char *logid, OrtLoggingFunction logging_function, void *logger_param)
Wraps OrtApi::CreateEnvWithCustomLogger.
Env & CreateAndRegisterAllocator(const OrtMemoryInfo *mem_info, const OrtArenaCfg *arena_cfg)
Wraps OrtApi::CreateAndRegisterAllocator.
Env & UpdateEnvWithCustomLogLevel(OrtLoggingLevel log_severity_level)
Wraps OrtApi::UpdateEnvWithCustomLogLevel.
Env & DisableTelemetryEvents()
Wraps OrtApi::DisableTelemetryEvents.
All C++ methods that can fail will throw an exception of this type.
Definition onnxruntime_cxx_api.h:53
const char * what() const noexcept override
Definition onnxruntime_cxx_api.h:57
OrtErrorCode GetOrtErrorCode() const
Definition onnxruntime_cxx_api.h:56
Exception(std::string &&string, OrtErrorCode code)
Definition onnxruntime_cxx_api.h:54
IEEE 754 half-precision floating point data type.
Definition onnxruntime_cxx_api.h:164
Float16_t()=default
Default constructor.
Float16_t(float v) noexcept
__ctor from float. Float is converted into float16 16-bit representation.
Definition onnxruntime_cxx_api.h:192
onnxruntime_float16::Float16Impl< Float16_t > Base
Definition onnxruntime_cxx_api.h:174
float ToFloat() const noexcept
Converts float16 to float.
Definition onnxruntime_cxx_api.h:198
static constexpr Float16_t FromBits(uint16_t v) noexcept
Explicit conversion to uint16_t representation of float16.
Definition onnxruntime_cxx_api.h:186
float8e4m3fn (Float8 Floating Point) data type
Definition onnxruntime_cxx_api.h:436
uint8_t value
Definition onnxruntime_cxx_api.h:437
constexpr Float8E4M3FN_t(uint8_t v) noexcept
Definition onnxruntime_cxx_api.h:439
constexpr bool operator==(const Float8E4M3FN_t &rhs) const noexcept
Definition onnxruntime_cxx_api.h:442
constexpr Float8E4M3FN_t() noexcept
Definition onnxruntime_cxx_api.h:438
constexpr bool operator!=(const Float8E4M3FN_t &rhs) const noexcept
Definition onnxruntime_cxx_api.h:443
float8e4m3fnuz (Float8 Floating Point) data type
Definition onnxruntime_cxx_api.h:453
constexpr bool operator==(const Float8E4M3FNUZ_t &rhs) const noexcept
Definition onnxruntime_cxx_api.h:459
uint8_t value
Definition onnxruntime_cxx_api.h:454
constexpr Float8E4M3FNUZ_t() noexcept
Definition onnxruntime_cxx_api.h:455
constexpr bool operator!=(const Float8E4M3FNUZ_t &rhs) const noexcept
Definition onnxruntime_cxx_api.h:460
constexpr Float8E4M3FNUZ_t(uint8_t v) noexcept
Definition onnxruntime_cxx_api.h:456
float8e5m2 (Float8 Floating Point) data type
Definition onnxruntime_cxx_api.h:470
constexpr Float8E5M2_t(uint8_t v) noexcept
Definition onnxruntime_cxx_api.h:473
uint8_t value
Definition onnxruntime_cxx_api.h:471
constexpr bool operator!=(const Float8E5M2_t &rhs) const noexcept
Definition onnxruntime_cxx_api.h:477
constexpr Float8E5M2_t() noexcept
Definition onnxruntime_cxx_api.h:472
constexpr bool operator==(const Float8E5M2_t &rhs) const noexcept
Definition onnxruntime_cxx_api.h:476
float8e5m2fnuz (Float8 Floating Point) data type
Definition onnxruntime_cxx_api.h:487
constexpr Float8E5M2FNUZ_t() noexcept
Definition onnxruntime_cxx_api.h:489
constexpr Float8E5M2FNUZ_t(uint8_t v) noexcept
Definition onnxruntime_cxx_api.h:490
constexpr bool operator!=(const Float8E5M2FNUZ_t &rhs) const noexcept
Definition onnxruntime_cxx_api.h:494
constexpr bool operator==(const Float8E5M2FNUZ_t &rhs) const noexcept
Definition onnxruntime_cxx_api.h:493
uint8_t value
Definition onnxruntime_cxx_api.h:488
Definition onnxruntime_cxx_api.h:85
static const OrtApi * api_
Definition onnxruntime_cxx_api.h:86
Wrapper around OrtIoBinding.
Definition onnxruntime_cxx_api.h:1861
UnownedIoBinding GetUnowned() const
Definition onnxruntime_cxx_api.h:1865
ConstIoBinding GetConst() const
Definition onnxruntime_cxx_api.h:1864
IoBinding(Session &session)
IoBinding(std::nullptr_t)
Create an empty object for convenience. Sometimes, we want to initialize members later.
Definition onnxruntime_cxx_api.h:1862
This class wraps a raw pointer OrtKernelContext* that is being passed to the custom kernel Compute() ...
Definition onnxruntime_cxx_api.h:2054
KernelContext(OrtKernelContext *context)
Logger GetLogger() const
ConstValue GetInput(size_t index) const
OrtKernelContext * GetOrtKernelContext() const
Definition onnxruntime_cxx_api.h:2064
void ParallelFor(void(*fn)(void *, size_t), size_t total, size_t num_batch, void *usr_data) const
OrtAllocator * GetAllocator(const OrtMemoryInfo &memory_info) const
void * GetGPUComputeStream() const
size_t GetInputCount() const
size_t GetOutputCount() const
UnownedValue GetOutput(size_t index, const std::vector< int64_t > &dims) const
UnownedValue GetOutput(size_t index, const int64_t *dim_values, size_t dim_count) const
This struct owns the OrtKernInfo* pointer when a copy is made. For convenient wrapping of OrtKernelIn...
Definition onnxruntime_cxx_api.h:2130
KernelInfo(OrtKernelInfo *info)
Take ownership of the instance.
ConstKernelInfo GetConst() const
Definition onnxruntime_cxx_api.h:2133
KernelInfo(std::nullptr_t)
Create an empty instance to initialize later.
Definition onnxruntime_cxx_api.h:2131
This class represents an ONNX Runtime logger that can be used to log information with an associated s...
Definition onnxruntime_cxx_api.h:1976
Logger(Logger &&v) noexcept=default
Logger & operator=(Logger &&v) noexcept=default
Logger & operator=(const Logger &)=default
~Logger()=default
Logger(const Logger &)=default
Logger()=default
Logger(std::nullptr_t)
Definition onnxruntime_cxx_api.h:1985
Logger(const OrtLogger *logger)
OrtLoggingLevel GetLoggingSeverityLevel() const noexcept
Wrapper around OrtMapTypeInfo.
Definition onnxruntime_cxx_api.h:1244
ConstMapTypeInfo GetConst() const
Definition onnxruntime_cxx_api.h:1247
MapTypeInfo(OrtMapTypeInfo *p)
Used for interop with the C API.
Definition onnxruntime_cxx_api.h:1246
MapTypeInfo(std::nullptr_t)
Create an empty MapTypeInfo object, must be assigned a valid one to be used.
Definition onnxruntime_cxx_api.h:1245
Represents native memory allocation coming from one of the OrtAllocators registered with OnnxRuntime....
Definition onnxruntime_cxx_api.h:1772
MemoryAllocation(MemoryAllocation &&) noexcept
MemoryAllocation & operator=(const MemoryAllocation &)=delete
MemoryAllocation(const MemoryAllocation &)=delete
MemoryAllocation(OrtAllocator *allocator, void *p, size_t size)
size_t size() const
Definition onnxruntime_cxx_api.h:1781
Wrapper around OrtMemoryInfo.
Definition onnxruntime_cxx_api.h:1151
MemoryInfo(const char *name, OrtAllocatorType type, int id, OrtMemType mem_type)
MemoryInfo(std::nullptr_t)
No instance is created.
Definition onnxruntime_cxx_api.h:1153
MemoryInfo(OrtMemoryInfo *p)
Take ownership of a pointer created by C Api.
Definition onnxruntime_cxx_api.h:1154
static MemoryInfo CreateCpu(OrtAllocatorType type, OrtMemType mem_type1)
ConstMemoryInfo GetConst() const
Definition onnxruntime_cxx_api.h:1156
Wrapper around OrtModelMetadata.
Definition onnxruntime_cxx_api.h:927
AllocatedStringPtr GetDescriptionAllocated(OrtAllocator *allocator) const
Returns a copy of the description.
std::vector< AllocatedStringPtr > GetCustomMetadataMapKeysAllocated(OrtAllocator *allocator) const
Returns a vector of copies of the custom metadata keys.
ModelMetadata(std::nullptr_t)
Create an empty ModelMetadata object, must be assigned a valid one to be used.
Definition onnxruntime_cxx_api.h:928
AllocatedStringPtr GetGraphDescriptionAllocated(OrtAllocator *allocator) const
Returns a copy of the graph description.
AllocatedStringPtr GetProducerNameAllocated(OrtAllocator *allocator) const
Returns a copy of the producer name.
AllocatedStringPtr GetGraphNameAllocated(OrtAllocator *allocator) const
Returns a copy of the graph name.
AllocatedStringPtr LookupCustomMetadataMapAllocated(const char *key, OrtAllocator *allocator) const
Looks up a value by a key in the Custom Metadata map.
ModelMetadata(OrtModelMetadata *p)
Used for interop with the C API.
Definition onnxruntime_cxx_api.h:929
AllocatedStringPtr GetDomainAllocated(OrtAllocator *allocator) const
Returns a copy of the domain name.
int64_t GetVersion() const
Wraps OrtApi::ModelMetadataGetVersion.
This struct provides life time management for custom op attribute.
Definition onnxruntime_cxx_api.h:1892
OpAttr(const char *name, const void *data, int len, OrtOpAttrType type)
Create and own custom defined operation.
Definition onnxruntime_cxx_api.h:2139
Op(OrtOp *)
Take ownership of the OrtOp.
static Op Create(const OrtKernelInfo *info, const char *op_name, const char *domain, int version, const char **type_constraint_names, const ONNXTensorElementDataType *type_constraint_values, size_t type_constraint_count, const OpAttr *attr_values, size_t attr_count, size_t input_count, size_t output_count)
Op(std::nullptr_t)
Create an empty Operator object, must be assigned a valid one to be used.
Definition onnxruntime_cxx_api.h:2140
void Invoke(const OrtKernelContext *context, const OrtValue *const *input_values, size_t input_count, OrtValue *const *output_values, size_t output_count)
void Invoke(const OrtKernelContext *context, const Value *input_values, size_t input_count, Value *output_values, size_t output_count)
RunOptions.
Definition onnxruntime_cxx_api.h:742
int GetRunLogSeverityLevel() const
Wraps OrtApi::RunOptionsGetRunLogSeverityLevel.
RunOptions & SetTerminate()
Terminates all currently executing Session::Run calls that were made using this RunOptions instance.
RunOptions & SetRunTag(const char *run_tag)
wraps OrtApi::RunOptionsSetRunTag
RunOptions & UnsetTerminate()
Clears the terminate flag so this RunOptions instance can be used in a new Session::Run call without ...
int GetRunLogVerbosityLevel() const
Wraps OrtApi::RunOptionsGetRunLogVerbosityLevel.
RunOptions(std::nullptr_t)
Create an empty RunOptions object, must be assigned a valid one to be used.
Definition onnxruntime_cxx_api.h:743
RunOptions & SetRunLogVerbosityLevel(int)
Wraps OrtApi::RunOptionsSetRunLogVerbosityLevel.
RunOptions & SetRunLogSeverityLevel(int)
Wraps OrtApi::RunOptionsSetRunLogSeverityLevel.
RunOptions & AddConfigEntry(const char *config_key, const char *config_value)
Wraps OrtApi::AddRunConfigEntry.
const char * GetRunTag() const
Wraps OrtApi::RunOptionsGetRunTag.
RunOptions()
Wraps OrtApi::CreateRunOptions.
Wrapper around OrtSequenceTypeInfo.
Definition onnxruntime_cxx_api.h:1209
SequenceTypeInfo(std::nullptr_t)
Create an empty SequenceTypeInfo object, must be assigned a valid one to be used.
Definition onnxruntime_cxx_api.h:1210
ConstSequenceTypeInfo GetConst() const
Definition onnxruntime_cxx_api.h:1212
SequenceTypeInfo(OrtSequenceTypeInfo *p)
Used for interop with the C API.
Definition onnxruntime_cxx_api.h:1211
Wrapper around OrtSession.
Definition onnxruntime_cxx_api.h:1115
Session(std::nullptr_t)
Create an empty Session object, must be assigned a valid one to be used.
Definition onnxruntime_cxx_api.h:1116
UnownedSession GetUnowned() const
Definition onnxruntime_cxx_api.h:1125
Session(const Env &env, const char *model_path, const SessionOptions &options, OrtPrepackedWeightsContainer *prepacked_weights_container)
Wraps OrtApi::CreateSessionWithPrepackedWeightsContainer.
Session(const Env &env, const void *model_data, size_t model_data_length, const SessionOptions &options, OrtPrepackedWeightsContainer *prepacked_weights_container)
Wraps OrtApi::CreateSessionFromArrayWithPrepackedWeightsContainer.
Session(const Env &env, const char *model_path, const SessionOptions &options)
Wraps OrtApi::CreateSession.
ConstSession GetConst() const
Definition onnxruntime_cxx_api.h:1124
Session(const Env &env, const void *model_data, size_t model_data_length, const SessionOptions &options)
Wraps OrtApi::CreateSessionFromArray.
Wrapper around OrtSessionOptions.
Definition onnxruntime_cxx_api.h:916
SessionOptions(std::nullptr_t)
Create an empty SessionOptions object, must be assigned a valid one to be used.
Definition onnxruntime_cxx_api.h:917
UnownedSessionOptions GetUnowned() const
Definition onnxruntime_cxx_api.h:920
SessionOptions()
Wraps OrtApi::CreateSessionOptions.
ConstSessionOptions GetConst() const
Definition onnxruntime_cxx_api.h:921
SessionOptions(OrtSessionOptions *p)
Used for interop with the C API.
Definition onnxruntime_cxx_api.h:919
Definition onnxruntime_cxx_api.h:2170
SymbolicInteger & operator=(const SymbolicInteger &)=default
SymbolicInteger(const SymbolicInteger &)=default
int64_t AsInt() const
Definition onnxruntime_cxx_api.h:2191
int64_t i_
Definition onnxruntime_cxx_api.h:2198
const char * s_
Definition onnxruntime_cxx_api.h:2199
bool operator==(const SymbolicInteger &dim) const
Definition onnxruntime_cxx_api.h:2179
SymbolicInteger & operator=(SymbolicInteger &&)=default
SymbolicInteger(SymbolicInteger &&)=default
const char * AsSym() const
Definition onnxruntime_cxx_api.h:2192
SymbolicInteger(int64_t i)
Definition onnxruntime_cxx_api.h:2171
SymbolicInteger(const char *s)
Definition onnxruntime_cxx_api.h:2172
bool IsInt() const
Definition onnxruntime_cxx_api.h:2190
Provide access to per-node attributes and input shapes, so one could compute and set output shapes.
Definition onnxruntime_cxx_api.h:2169
Ints GetAttrInts(const char *attr_name)
Strings GetAttrStrings(const char *attr_name)
std::vector< SymbolicInteger > Shape
Definition onnxruntime_cxx_api.h:2204
std::vector< float > Floats
Definition onnxruntime_cxx_api.h:2221
std::string GetAttrString(const char *attr_name)
std::vector< int64_t > Ints
Definition onnxruntime_cxx_api.h:2216
ShapeInferContext(const OrtApi *ort_api, OrtShapeInferContext *ctx)
Status SetOutputShape(size_t indice, const Shape &shape)
int64_t GetAttrInt(const char *attr_name)
size_t GetInputCount() const
Definition onnxruntime_cxx_api.h:2210
std::vector< std::string > Strings
Definition onnxruntime_cxx_api.h:2226
Floats GetAttrFloats(const char *attr_name)
const Shape & GetInputShape(size_t indice) const
Definition onnxruntime_cxx_api.h:2208
float GetAttrFloat(const char *attr_name)
The Status that holds ownership of OrtStatus received from C API Use it to safely destroy OrtStatus* ...
Definition onnxruntime_cxx_api.h:651
OrtErrorCode GetErrorCode() const
Status(const char *message, OrtErrorCode code) noexcept
Creates status instance out of null-terminated string message.
bool IsOK() const noexcept
Returns true if instance represents an OK (non-error) status.
Status(OrtStatus *status) noexcept
Takes ownership of OrtStatus instance returned from the C API.
std::string GetErrorMessage() const
Status(const Exception &) noexcept
Creates status instance out of exception.
Status(const std::exception &) noexcept
Creates status instance out of exception.
Status(std::nullptr_t) noexcept
Create an empty object, must be assigned a valid one to be used.
Definition onnxruntime_cxx_api.h:652
Wrapper around OrtTensorTypeAndShapeInfo.
Definition onnxruntime_cxx_api.h:1188
TensorTypeAndShapeInfo(std::nullptr_t)
Create an empty TensorTypeAndShapeInfo object, must be assigned a valid one to be used.
Definition onnxruntime_cxx_api.h:1189
ConstTensorTypeAndShapeInfo GetConst() const
Definition onnxruntime_cxx_api.h:1191
TensorTypeAndShapeInfo(OrtTensorTypeAndShapeInfo *p)
Used for interop with the C API.
Definition onnxruntime_cxx_api.h:1190
The ThreadingOptions.
Definition onnxruntime_cxx_api.h:666
ThreadingOptions & SetGlobalCustomThreadCreationOptions(void *ort_custom_thread_creation_options)
Wraps OrtApi::SetGlobalCustomThreadCreationOptions.
ThreadingOptions()
Wraps OrtApi::CreateThreadingOptions.
ThreadingOptions & SetGlobalInterOpNumThreads(int inter_op_num_threads)
Wraps OrtApi::SetGlobalInterOpNumThreads.
ThreadingOptions & SetGlobalCustomCreateThreadFn(OrtCustomCreateThreadFn ort_custom_create_thread_fn)
Wraps OrtApi::SetGlobalCustomCreateThreadFn.
ThreadingOptions & SetGlobalCustomJoinThreadFn(OrtCustomJoinThreadFn ort_custom_join_thread_fn)
Wraps OrtApi::SetGlobalCustomJoinThreadFn.
ThreadingOptions & SetGlobalSpinControl(int allow_spinning)
Wraps OrtApi::SetGlobalSpinControl.
ThreadingOptions & SetGlobalDenormalAsZero()
Wraps OrtApi::SetGlobalDenormalAsZero.
ThreadingOptions & SetGlobalIntraOpNumThreads(int intra_op_num_threads)
Wraps OrtApi::SetGlobalIntraOpNumThreads.
Type information that may contain either TensorTypeAndShapeInfo or the information about contained se...
Definition onnxruntime_cxx_api.h:1275
TypeInfo(std::nullptr_t)
Create an empty TypeInfo object, must be assigned a valid one to be used.
Definition onnxruntime_cxx_api.h:1276
ConstTypeInfo GetConst() const
Definition onnxruntime_cxx_api.h:1279
TypeInfo(OrtTypeInfo *p)
C API Interop.
Definition onnxruntime_cxx_api.h:1277
Wrapper around OrtValue.
Definition onnxruntime_cxx_api.h:1611
static Value CreateSparseTensor(const OrtMemoryInfo *info, void *p_data, const Shape &dense_shape, const Shape &values_shape, ONNXTensorElementDataType type)
Creates an OrtValue instance containing SparseTensor. This constructs a sparse tensor that makes use ...
static Value CreateSparseTensor(const OrtMemoryInfo *info, T *p_data, const Shape &dense_shape, const Shape &values_shape)
This is a simple forwarding method to the other overload that helps deducing data type enum value fro...
Value & operator=(Value &&)=default
static Value CreateSparseTensor(OrtAllocator *allocator, const Shape &dense_shape, ONNXTensorElementDataType type)
Creates an instance of OrtValue containing sparse tensor. The created instance has no data....
Value(Value &&)=default
Value(std::nullptr_t)
Create an empty Value object, must be assigned a valid one to be used.
Definition onnxruntime_cxx_api.h:1616
static Value CreateTensor(const OrtMemoryInfo *info, T *p_data, size_t p_data_element_count, const int64_t *shape, size_t shape_len)
Creates a tensor with a user supplied buffer. Wraps OrtApi::CreateTensorWithDataAsOrtValue.
Value(OrtValue *p)
Used for interop with the C API.
Definition onnxruntime_cxx_api.h:1617
static Value CreateSparseTensor(OrtAllocator *allocator, const Shape &dense_shape)
This is a simple forwarding method to the below CreateSparseTensor. This helps to specify data type e...
static Value CreateTensor(OrtAllocator *allocator, const int64_t *shape, size_t shape_len, ONNXTensorElementDataType type)
Creates an OrtValue with a tensor using the supplied OrtAllocator. Wraps OrtApi::CreateTensorAsOrtVal...
UnownedValue GetUnowned() const
Definition onnxruntime_cxx_api.h:1622
static Value CreateSequence(const std::vector< Value > &values)
Creates an OrtValue with a Sequence Onnx type representation. The API would ref-count the supplied Or...
static Value CreateMap(const Value &keys, const Value &values)
Creates an OrtValue with a Map Onnx type representation. The API would ref-count the supplied OrtValu...
static Value CreateTensor(const OrtMemoryInfo *info, void *p_data, size_t p_data_byte_count, const int64_t *shape, size_t shape_len, ONNXTensorElementDataType type)
Creates a tensor with a user supplied buffer. Wraps OrtApi::CreateTensorWithDataAsOrtValue.
static Value CreateTensor(OrtAllocator *allocator, const int64_t *shape, size_t shape_len)
Creates an OrtValue with a tensor using a supplied OrtAllocator. Wraps OrtApi::CreateTensorAsOrtValue...
static Value CreateOpaque(const char *domain, const char *type_name, const T &value)
Creates an OrtValue wrapping an Opaque type. This is used for experimental support of non-tensor type...
ConstValue GetConst() const
Definition onnxruntime_cxx_api.h:1621
Definition onnxruntime_cxx_api.h:624
AllocatedFree(OrtAllocator *allocator)
Definition onnxruntime_cxx_api.h:626
OrtAllocator * allocator_
Definition onnxruntime_cxx_api.h:625
void operator()(void *ptr) const
Definition onnxruntime_cxx_api.h:628
Base & operator=(Base &&v) noexcept
Definition onnxruntime_cxx_api.h:611
typename Unowned< T >::Type contained_type
Definition onnxruntime_cxx_api.h:600
Base(Base &&v) noexcept
Definition onnxruntime_cxx_api.h:610
Base(const Base &)=default
constexpr Base(contained_type *p) noexcept
Definition onnxruntime_cxx_api.h:603
Base & operator=(const Base &)=default
Used internally by the C++ API. C++ wrapper types inherit from this. This is a zero cost abstraction ...
Definition onnxruntime_cxx_api.h:556
Base(Base &&v) noexcept
Definition onnxruntime_cxx_api.h:566
constexpr Base()=default
contained_type * release()
Relinquishes ownership of the contained C object pointer The underlying object is not destroyed.
Definition onnxruntime_cxx_api.h:577
Base(const Base &)=delete
constexpr Base(contained_type *p) noexcept
Definition onnxruntime_cxx_api.h:560
Base & operator=(const Base &)=delete
Base & operator=(Base &&v) noexcept
Definition onnxruntime_cxx_api.h:567
contained_type * p_
Definition onnxruntime_cxx_api.h:584
~Base()
Definition onnxruntime_cxx_api.h:561
T contained_type
Definition onnxruntime_cxx_api.h:557
Definition onnxruntime_cxx_api.h:1829
std::vector< Value > GetOutputValues(OrtAllocator *) const
std::vector< std::string > GetOutputNames(OrtAllocator *) const
std::vector< Value > GetOutputValues() const
std::vector< std::string > GetOutputNames() const
Definition onnxruntime_cxx_api.h:1000
TypeInfo GetInputTypeInfo(size_t index) const
Wraps OrtApi::SessionGetInputTypeInfo.
size_t GetOutputCount() const
Returns the number of model outputs.
uint64_t GetProfilingStartTimeNs() const
Wraps OrtApi::SessionGetProfilingStartTimeNs.
ModelMetadata GetModelMetadata() const
Wraps OrtApi::SessionGetModelMetadata.
size_t GetInputCount() const
Returns the number of model inputs.
TypeInfo GetOutputTypeInfo(size_t index) const
Wraps OrtApi::SessionGetOutputTypeInfo.
AllocatedStringPtr GetOverridableInitializerNameAllocated(size_t index, OrtAllocator *allocator) const
Returns a copy of the overridable initializer name at then specified index.
AllocatedStringPtr GetOutputNameAllocated(size_t index, OrtAllocator *allocator) const
Returns a copy of output name at then specified index.
size_t GetOverridableInitializerCount() const
Returns the number of inputs that have defaults that can be overridden.
AllocatedStringPtr GetInputNameAllocated(size_t index, OrtAllocator *allocator) const
Returns a copy of input name at the specified index.
TypeInfo GetOverridableInitializerTypeInfo(size_t index) const
Wraps OrtApi::SessionGetOverridableInitializerTypeInfo.
Definition onnxruntime_cxx_api.h:1308
void GetStringTensorContent(void *buffer, size_t buffer_length, size_t *offsets, size_t offsets_count) const
The API copies all of the UTF-8 encoded string data contained within a tensor or a sparse tensor into...
void GetStringTensorElement(size_t buffer_length, size_t element_index, void *buffer) const
The API copies UTF-8 encoded bytes for the requested string element contained within a tensor or a sp...
TensorTypeAndShapeInfo GetSparseTensorIndicesTypeShapeInfo(OrtSparseIndicesFormat format) const
The API returns type and shape information for the specified indices. Each supported indices have the...
const void * GetTensorRawData() const
Returns a non-typed pointer to a tensor contained data.
std::string GetStringTensorElement(size_t element_index) const
Returns string tensor UTF-8 encoded string element. Use of this API is recommended over GetStringTens...
size_t GetStringTensorElementLength(size_t element_index) const
The API returns a byte length of UTF-8 encoded string element contained in either a tensor or a spare...
size_t GetStringTensorDataLength() const
This API returns a full length of string data contained within either a tensor or a sparse Tensor....
bool IsSparseTensor() const
Returns true if the OrtValue contains a sparse tensor.
TypeInfo GetTypeInfo() const
The API returns type information for data contained in a tensor. For sparse tensors it returns type i...
const R * GetSparseTensorIndicesData(OrtSparseIndicesFormat indices_format, size_t &num_indices) const
The API retrieves a pointer to the internal indices buffer. The API merely performs a convenience dat...
bool IsTensor() const
Returns true if Value is a tensor, false for other types like map/sequence/etc.
ConstMemoryInfo GetTensorMemoryInfo() const
This API returns information about the memory allocation used to hold data.
const R * GetSparseTensorValues() const
The API returns a pointer to an internal buffer of the sparse tensor containing non-zero values....
TensorTypeAndShapeInfo GetTensorTypeAndShapeInfo() const
The API returns type information for data contained in a tensor. For sparse tensors it returns type i...
Value GetValue(int index, OrtAllocator *allocator) const
size_t GetCount() const
< Return true if OrtValue contains data and returns false if the OrtValue is a None
void GetOpaqueData(const char *domain, const char *type_name, R &) const
Obtains a pointer to a user defined data for experimental purposes.
TensorTypeAndShapeInfo GetSparseTensorValuesTypeAndShapeInfo() const
The API returns type and shape information for stored non-zero values of the sparse tensor....
const R * GetTensorData() const
Returns a const typed pointer to the tensor contained data. No type checking is performed,...
OrtSparseFormat GetSparseFormat() const
The API returns the sparse data format this OrtValue holds in a sparse tensor. If the sparse tensor w...
Definition onnxruntime_cxx_api.h:1840
void BindOutput(const char *name, const Value &)
void BindInput(const char *name, const Value &)
void BindOutput(const char *name, const OrtMemoryInfo *)
Definition onnxruntime_cxx_api.h:1230
ONNXTensorElementDataType GetMapKeyType() const
Wraps OrtApi::GetMapKeyType.
TypeInfo GetMapValueType() const
Wraps OrtApi::GetMapValueType.
Definition onnxruntime_cxx_api.h:1130
std::string GetAllocatorName() const
OrtMemType GetMemoryType() const
OrtMemoryInfoDeviceType GetDeviceType() const
OrtAllocatorType GetAllocatorType() const
bool operator==(const MemoryInfoImpl< U > &o) const
Definition onnxruntime_cxx_api.h:1217
TypeInfo GetOptionalElementType() const
Wraps OrtApi::CastOptionalTypeToContainedTypeInfo.
Definition onnxruntime_cxx_api.h:1291
const char ** str
Definition onnxruntime_cxx_api.h:1296
const int64_t * values_shape
Definition onnxruntime_cxx_api.h:1292
size_t values_shape_len
Definition onnxruntime_cxx_api.h:1293
const void * p_data
Definition onnxruntime_cxx_api.h:1295
Definition onnxruntime_cxx_api.h:1196
TypeInfo GetSequenceElementType() const
Wraps OrtApi::GetSequenceElementType.
Definition onnxruntime_cxx_api.h:1044
AllocatedStringPtr EndProfilingAllocated(OrtAllocator *allocator)
End profiling and return a copy of the profiling file name.
void Run(const RunOptions &run_options, const IoBinding &)
Wraps OrtApi::RunWithBinding.
void RunAsync(const RunOptions &run_options, const char *const *input_names, const Value *input_values, size_t input_count, const char *const *output_names, Value *output_values, size_t output_count, RunAsyncCallbackFn callback, void *user_data)
Run the model asynchronously in a thread owned by intra op thread pool.
std::vector< Value > Run(const RunOptions &run_options, const char *const *input_names, const Value *input_values, size_t input_count, const char *const *output_names, size_t output_count)
Run the model returning results in an Ort allocated vector.
void Run(const RunOptions &run_options, const char *const *input_names, const Value *input_values, size_t input_count, const char *const *output_names, Value *output_values, size_t output_count)
Run the model returning results in user provided outputs Same as Run(const RunOptions&,...
Definition onnxruntime_cxx_api.h:1302
const int64_t * shape
Definition onnxruntime_cxx_api.h:1303
size_t shape_len
Definition onnxruntime_cxx_api.h:1304
Definition onnxruntime_cxx_api.h:1161
size_t GetElementCount() const
Wraps OrtApi::GetTensorShapeElementCount.
void GetDimensions(int64_t *values, size_t values_count) const
Wraps OrtApi::GetDimensions.
std::vector< int64_t > GetShape() const
Uses GetDimensionsCount & GetDimensions to return a std::vector of the shape.
void GetSymbolicDimensions(const char **values, size_t values_count) const
Wraps OrtApi::GetSymbolicDimensions.
size_t GetDimensionsCount() const
Wraps OrtApi::GetDimensionsCount.
ONNXTensorElementDataType GetElementType() const
Wraps OrtApi::GetTensorElementType.
Definition onnxruntime_cxx_api.h:1252
ONNXType GetONNXType() const
ConstSequenceTypeInfo GetSequenceTypeInfo() const
Wraps OrtApi::CastTypeInfoToSequenceTypeInfo.
ConstMapTypeInfo GetMapTypeInfo() const
Wraps OrtApi::CastTypeInfoToMapTypeInfo.
ConstOptionalTypeInfo GetOptionalTypeInfo() const
wraps OrtApi::CastTypeInfoToOptionalTypeInfo
ConstTensorTypeAndShapeInfo GetTensorTypeAndShapeInfo() const
Wraps OrtApi::CastTypeInfoToTensorInfo.
This is a tagging template type. Use it with Base<T> to indicate that the C++ interface object has no...
Definition onnxruntime_cxx_api.h:532
T Type
Definition onnxruntime_cxx_api.h:533
Definition onnxruntime_cxx_api.h:1469
void FillStringTensorElement(const char *s, size_t index)
Set a single string in a string tensor.
R * GetTensorMutableData()
Returns a non-const typed pointer to an OrtValue/Tensor contained buffer No type checking is performe...
R & At(const std::vector< int64_t > &location)
void UseBlockSparseIndices(const Shape &indices_shape, int32_t *indices_data)
Supplies BlockSparse format specific indices and marks the contained sparse tensor as being a BlockSp...
void FillSparseTensorBlockSparse(const OrtMemoryInfo *data_mem_info, const OrtSparseValuesParam &values, const Shape &indices_shape, const int32_t *indices_data)
The API will allocate memory using the allocator instance supplied to the CreateSparseTensor() API an...
void * GetTensorMutableRawData()
Returns a non-typed non-const pointer to a tensor contained data.
void UseCooIndices(int64_t *indices_data, size_t indices_num)
Supplies COO format specific indices and marks the contained sparse tensor as being a COO format tens...
void FillSparseTensorCoo(const OrtMemoryInfo *data_mem_info, const OrtSparseValuesParam &values_param, const int64_t *indices_data, size_t indices_num)
The API will allocate memory using the allocator instance supplied to the CreateSparseTensor() API an...
void FillStringTensor(const char *const *s, size_t s_len)
Set all strings at once in a string tensor.
void UseCsrIndices(int64_t *inner_data, size_t inner_num, int64_t *outer_data, size_t outer_num)
Supplies CSR format specific indices and marks the contained sparse tensor as being a CSR format tens...
void FillSparseTensorCsr(const OrtMemoryInfo *data_mem_info, const OrtSparseValuesParam &values, const int64_t *inner_indices_data, size_t inner_indices_num, const int64_t *outer_indices_data, size_t outer_indices_num)
The API will allocate memory using the allocator instance supplied to the CreateSparseTensor() API an...
char * GetResizedStringTensorElementBuffer(size_t index, size_t buffer_length)
Allocate if necessary and obtain a pointer to a UTF-8 encoded string element buffer indexed by the fl...
Memory allocation interface.
Definition onnxruntime_c_api.h:317
void(* Free)(struct OrtAllocator *this_, void *p)
Free a block of memory previously allocated with OrtAllocator::Alloc.
Definition onnxruntime_c_api.h:320
const OrtApi *(* GetApi)(uint32_t version)
Get a pointer to the requested version of the OrtApi.
Definition onnxruntime_c_api.h:662
The C API.
Definition onnxruntime_c_api.h:722
CUDA Provider Options.
Definition onnxruntime_c_api.h:401
Definition onnxruntime_c_api.h:4640
int(* GetVariadicInputHomogeneity)(const struct OrtCustomOp *op)
Definition onnxruntime_c_api.h:4686
OrtCustomOpInputOutputCharacteristic(* GetOutputCharacteristic)(const struct OrtCustomOp *op, size_t index)
Definition onnxruntime_c_api.h:4670
size_t(* GetInputTypeCount)(const struct OrtCustomOp *op)
Definition onnxruntime_c_api.h:4658
int(* GetVariadicOutputMinArity)(const struct OrtCustomOp *op)
Definition onnxruntime_c_api.h:4690
int(* GetStartVersion)(const struct OrtCustomOp *op)
Definition onnxruntime_c_api.h:4708
const char *(* GetName)(const struct OrtCustomOp *op)
Definition onnxruntime_c_api.h:4651
size_t(* GetOutputTypeCount)(const struct OrtCustomOp *op)
Definition onnxruntime_c_api.h:4660
void(* KernelDestroy)(void *op_kernel)
Definition onnxruntime_c_api.h:4666
int(* GetVariadicOutputHomogeneity)(const struct OrtCustomOp *op)
Definition onnxruntime_c_api.h:4695
OrtMemType(* GetInputMemoryType)(const struct OrtCustomOp *op, size_t index)
Definition onnxruntime_c_api.h:4677
void *(* CreateKernel)(const struct OrtCustomOp *op, const OrtApi *api, const OrtKernelInfo *info)
Definition onnxruntime_c_api.h:4647
uint32_t version
Definition onnxruntime_c_api.h:4641
ONNXTensorElementDataType(* GetInputType)(const struct OrtCustomOp *op, size_t index)
Definition onnxruntime_c_api.h:4657
OrtCustomOpInputOutputCharacteristic(* GetInputCharacteristic)(const struct OrtCustomOp *op, size_t index)
Definition onnxruntime_c_api.h:4669
const char *(* GetExecutionProviderType)(const struct OrtCustomOp *op)
Definition onnxruntime_c_api.h:4654
ONNXTensorElementDataType(* GetOutputType)(const struct OrtCustomOp *op, size_t index)
Definition onnxruntime_c_api.h:4659
int(* GetVariadicInputMinArity)(const struct OrtCustomOp *op)
Definition onnxruntime_c_api.h:4681
OrtStatusPtr(* InferOutputShapeFn)(const struct OrtCustomOp *op, OrtShapeInferContext *)
Definition onnxruntime_c_api.h:4705
int(* GetEndVersion)(const struct OrtCustomOp *op)
Definition onnxruntime_c_api.h:4709
OrtStatusPtr(* CreateKernelV2)(const struct OrtCustomOp *op, const OrtApi *api, const OrtKernelInfo *info, void **kernel)
Definition onnxruntime_c_api.h:4698
OrtStatusPtr(* KernelComputeV2)(void *op_kernel, OrtKernelContext *context)
Definition onnxruntime_c_api.h:4703
void(* KernelCompute)(void *op_kernel, OrtKernelContext *context)
Definition onnxruntime_c_api.h:4665
MIGraphX Provider Options.
Definition onnxruntime_c_api.h:605
OpenVINO Provider Options.
Definition onnxruntime_c_api.h:617
ROCM Provider Options.
Definition onnxruntime_c_api.h:488
TensorRT Provider Options.
Definition onnxruntime_c_api.h:577