codekingpro/portable-devtools
115k
1/*
2 * SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: Apache-2.0
4 *
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
8 *
9 * http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
16 */
17
18#ifndef NV_INFER_RUNTIME_H
19#define NV_INFER_RUNTIME_H
20
21//!
22//! \file NvInferRuntime.h
23//!
24//! This is the top-level API file for TensorRT extended runtime library.
25//!
26
27#include "NvInferImpl.h" // IWYU pragma: export
28#define NV_INFER_INTERNAL_INCLUDE 1
29#include "NvInferPluginBase.h" // IWYU pragma: export
30#undef NV_INFER_INTERNAL_INCLUDE
31#include "NvInferRuntimeCommon.h" // IWYU pragma: export
32
33namespace nvinfer1
34{
35
36class IExecutionContext; //!< Forward declaration of IExecutionContext for use by other interfaces.
37class ICudaEngine; //!< Forward declaration of ICudaEngine for use by other interfaces.
38class IPluginFactory; //!< Forward declaration of IPluginFactory for use by other interfaces.
39class IEngineInspector; //!< Forward declaration of IEngineInspector for use by other interfaces.
40
41//!
42//! \class INoCopy
43//!
44//! \brief Base class for all TensorRT interfaces that are implemented by the TensorRT libraries
45//!
46//! Objects of such classes are not movable or copyable, and should only be manipulated
47//! via pointers.
48//!
49
50class INoCopy
51{
52protected:
53 INoCopy() = default;
54 virtual ~INoCopy() = default;
55 INoCopy(INoCopy const& other) = delete;
56 INoCopy& operator=(INoCopy const& other) = delete;
57 INoCopy(INoCopy&& other) = delete;
58 INoCopy& operator=(INoCopy&& other) = delete;
59};
60
61//!
62//! \enum EngineCapability
63//!
64//! \brief List of supported engine capability flows.
65//!
66//! \details The EngineCapability determines the restrictions of a network during build time and what runtime
67//! it targets. EngineCapability::kSTANDARD does not provide any restrictions on functionality and the resulting
68//! serialized engine can be executed with TensorRT's standard runtime APIs in the nvinfer1 namespace.
69//! EngineCapability::kSAFETY provides a restricted subset of network operations that are safety certified and the
70//! resulting serialized engine can be executed with TensorRT's safe runtime APIs in the nvinfer2::safe namespace.
71//! EngineCapability::kDLA_STANDALONE provides a restricted subset of network operations that are DLA compatible and the
72//! resulting serialized engine can be executed using standalone DLA runtime APIs. See sampleCudla for an example of
73//! integrating cuDLA APIs with TensorRT APIs.
74//!
75enum class EngineCapability : int32_t
76{
77 //!
78 //! Standard: TensorRT flow without targeting the safety runtime.
79 //! This flow supports both DeviceType::kGPU and DeviceType::kDLA.
80 //!
81 kSTANDARD = 0,
82
83 //!
84 //! Safety: TensorRT flow with restrictions targeting the safety runtime.
85 //! See safety documentation for list of supported layers and formats.
86 //! This flow supports only DeviceType::kGPU.
87 //!
88 //! This flag is only supported in NVIDIA Drive(R) products.
89 kSAFETY = 1,
90
91 //!
92 //! DLA Standalone: TensorRT flow with restrictions targeting external, to TensorRT, DLA runtimes.
93 //! See DLA documentation for list of supported layers and formats.
94 //! This flow supports only DeviceType::kDLA.
95 //!
96 kDLA_STANDALONE = 2,
97};
98
99namespace impl
100{
101//! Maximum number of elements in EngineCapability enum. \see EngineCapability
102template <>
103struct EnumMaxImpl<EngineCapability>
104{
105 static constexpr int32_t kVALUE = 3;
106};
107} // namespace impl
108
109//!
110//! \class Weights
111//!
112//! \brief An array of weights used as a layer parameter.
113//!
114//! When using the DLA, the cumulative size of all Weights used in a network
115//! must be less than 512MB in size. If the build option kGPU_FALLBACK is specified,
116//! then multiple DLA sub-networks may be generated from the single original network.
117//!
118//! The weights are held by reference until the engine has been built. Therefore the data referenced
119//! by \p values field should be preserved until the build is complete.
120//!
121//! The term "empty weights" refers to Weights with weight coefficients ( \p count == 0 and \p values == nullptr).
122//!
123class Weights
124{
125public:
126 DataType type; //!< The type of the weights.
127 void const* values; //!< The weight values, in a contiguous array.
128 int64_t count; //!< The number of weights in the array.
129};
130
131//!
132//! \class IHostMemory
133//!
134//! \brief Class to handle library allocated memory that is accessible to the user.
135//!
136//! The memory allocated via the host memory object is owned by the library and will
137//! be de-allocated when the destroy method is called.
138//!
139//! \warning Do not inherit from this class, as doing so will break forward-compatibility of the API and ABI.
140//!
141class IHostMemory : public INoCopy
142{
143public:
144 virtual ~IHostMemory() noexcept = default;
145
146 //! A pointer to the raw data that is owned by the library.
147 void* data() const noexcept
148 {
149 return mImpl->data();
150 }
151
152 //! The size in bytes of the data that was allocated.
153 std::size_t size() const noexcept
154 {
155 return mImpl->size();
156 }
157
158 //! The type of the memory that was allocated.
159 DataType type() const noexcept
160 {
161 return mImpl->type();
162 }
163
164protected:
165 apiv::VHostMemory* mImpl;
166};
167
168//!
169//! \enum DimensionOperation
170//!
171//! \brief An operation on two IDimensionExpr, which represent integer expressions used in dimension computations.
172//!
173//! For example, given two IDimensionExpr x and y and an IExprBuilder& eb,
174//! eb.operation(DimensionOperation::kSUM, x, y) creates a representation of x+y.
175//!
176//! \see IDimensionExpr, IExprBuilder
177//!
178enum class DimensionOperation : int32_t
179{
180 kSUM = 0, //!< Sum of the two operands.
181 kPROD = 1, //!< Product of the two operands.
182 kMAX = 2, //!< Maximum of the two operands.
183 kMIN = 3, //!< Minimum of the two operands.
184 kSUB = 4, //!< Substract the second element from the first.
185 kEQUAL = 5, //!< 1 if operands are equal, 0 otherwise.
186 kLESS = 6, //!< 1 if first operand is less than second operand, 0 otherwise.
187 kFLOOR_DIV = 7, //!< Floor division of the first element by the second.
188 kCEIL_DIV = 8 //!< Division rounding up
189};
190
191//! Maximum number of elements in DimensionOperation enum. \see DimensionOperation
192template <>
193constexpr inline int32_t EnumMax<DimensionOperation>() noexcept
194{
195 return 9;
196}
197
198//!
199//! \enum TensorLocation
200//!
201//! \brief The location for tensor data storage, device or host.
202//!
203enum class TensorLocation : int32_t
204{
205 kDEVICE = 0, //!< Data stored on device.
206 kHOST = 1, //!< Data stored on host.
207};
208
209namespace impl
210{
211//! Maximum number of elements in TensorLocation enum. \see TensorLocation
212template <>
213struct EnumMaxImpl<TensorLocation>
214{
215 static constexpr int32_t kVALUE = 2;
216};
217} // namespace impl
218
219//!
220//! \class IDimensionExpr
221//!
222//! \brief An IDimensionExpr represents an integer expression constructed from constants,
223//! input dimensions, and binary operations. These expressions are can be used
224//! in overrides of IPluginV2DynamicExt::getOutputDimensions or IPluginV3OneBuild::getOutputShapes() to define output
225//! dimensions in terms of input dimensions.
226//!
227//! \warning Do not inherit from this class, as doing so will break forward-compatibility of the API and ABI.
228//!
229//! \see DimensionOperation, IPluginV2DynamicExt::getOutputDimensions, IPluginV3OneBuild::getOutputShapes()
230//!
231class IDimensionExpr : public INoCopy
232{
233public:
234 //!
235 //! \brief Return true if expression is a build-time constant.
236 //!
237 bool isConstant() const noexcept
238 {
239 return mImpl->isConstant();
240 }
241
242 //!
243 //! \brief Get the value of the constant.
244 //!
245 //! If isConstant(), returns value of the constant.
246 //! If !isConstant(), return std::numeric_limits<int64_t>::min().
247 //!
248 int64_t getConstantValue() const noexcept
249 {
250 return mImpl->getConstantValue();
251 }
252
253protected:
254 apiv::VDimensionExpr* mImpl;
255 virtual ~IDimensionExpr() noexcept = default;
256
257public:
258 //!
259 //! \brief Return true if this denotes the value of a size tensor.
260 //!
261 //! \return True if this was created with method IExprBuilder::declareSizeTensor, false otherwise
262 //!
263 bool isSizeTensor() const noexcept
264 {
265 return mImpl->isSizeTensor();
266 }
267};
268
269//!
270//! \class IExprBuilder
271//!
272//! \brief Object for constructing IDimensionExpr.
273//!
274//! There is no public way to construct an IExprBuilder. It appears as an argument to
275//! method IPluginV2DynamicExt::getOutputDimensions() and IPluginV3OneBuild::getOutputShapes(). Overrides of that
276//! method can use that IExprBuilder argument to construct expressions that define output dimensions in terms of input
277//! dimensions.
278//!
279//! Clients should assume that any values constructed by the IExprBuilder are destroyed
280//! after IPluginV2DynamicExt::getOutputDimensions() or IPluginV3OneBuild::getOutputShapes() returns.
281//!
282//! \warning Do not inherit from this class, as doing so will break forward-compatibility of the API and ABI.
283//!
284//! \see IDimensionExpr
285//!
286class IExprBuilder : public INoCopy
287{
288public:
289 //!
290 //! \brief Return pointer to IDimensionExpr for given value.
291 //!
292 IDimensionExpr const* constant(int64_t value) noexcept
293 {
294 return mImpl->constant(value);
295 }
296
297 //!
298 //! \brief Get the operation.
299 //!
300 //! Return pointer to IDimensionExpr that represents the given operation applied to first and second.
301 //! Returns nullptr if op is not a valid DimensionOperation.
302 //!
303 IDimensionExpr const* operation(
304 DimensionOperation op, IDimensionExpr const& first, IDimensionExpr const& second) noexcept
305 {
306 return mImpl->operation(op, first, second);
307 }
308
309protected:
310 apiv::VExprBuilder* mImpl;
311 virtual ~IExprBuilder() noexcept = default;
312
313public:
314 //!
315 //! \brief Declare a size tensor at the given output index, with the specified auto-tuning formula and upper bound.
316 //!
317 //! A size tensor allows a plugin to have output dimensions that cannot be computed solely from input dimensions.
318 //! For example, suppose a plugin implements the equivalent of INonZeroLayer for 2D input. The plugin can
319 //! have one output for the indices of non-zero elements, and a second output containing the number of non-zero
320 //! elements. Suppose the input has size [M,N] and has K non-zero elements. The plugin can write K to the second
321 //! output. When telling TensorRT that the first output has shape [2,K], plugin uses IExprBuilder::constant() and
322 //! IExprBuilder::declareSizeTensor(1,...) to create the IDimensionExpr that respectively denote 2 and K.
323 //!
324 //! TensorRT also needs to know the value of K to use for auto-tuning and an upper bound on K so that it can
325 //! allocate memory for the output tensor. In the example, supposed typically half of the plugin's input elements
326 //! are non-zero, and all the elements might be nonzero. then using M*N/2 might be a good expression for the opt
327 //! parameter, and M*N for the upper bound. IDimensionsExpr for these expressions can be constructed from
328 //! IDimensionsExpr for the input dimensions.
329 //!
330 //! \param outputIndex index of a plugin output that is a size tensor.
331 //! \param opt formula for computing auto-tuning value. Must not depend on a size tensor.
332 //! \param upper Upper bound on the size tensor.
333 //!
334 //! \return IDimensionExpr denoting the value of the size tensor.
335 //!
336 //! \see IPluginV3OneBuild::getOutputShapes()
337 //!
338 IDimensionExpr const* declareSizeTensor(int32_t outputIndex, IDimensionExpr const& opt, IDimensionExpr const& upper)
339 {
340 return mImpl->declareSizeTensor(outputIndex, opt, upper);
341 }
342};
343
344//!
345//! \class DimsExprs
346//!
347//! \brief Analog of class Dims with expressions instead of constants for the dimensions.
348//!
349class DimsExprs
350{
351public:
352 int32_t nbDims; //!< The number of dimensions.
353 IDimensionExpr const* d[Dims::MAX_DIMS]; //!< The extent of each dimension.
354};
355
356//!
357//! \struct DynamicPluginTensorDesc
358//!
359//! \brief Summarizes tensors that a plugin might see for an input or output.
360//!
361struct DynamicPluginTensorDesc
362{
363 //! Information required to interpret a pointer to tensor data, except that desc.dims has -1 in place of any runtime dimension.
364 PluginTensorDesc desc;
365
366 //! Lower bounds on tensor’s dimensions
367 Dims min;
368
369 //! Upper bounds on tensor’s dimensions
370 Dims max;
371
372 //! Optimum value of tensor’s dimensions specified for auto-tuning
373 Dims opt;
374};
375
376//!
377//! \class IPluginV2DynamicExt
378//!
379//! \brief Similar to IPluginV2Ext, but with support for dynamic shapes.
380//!
381//! Clients should override the public methods, including the following inherited methods:
382//!
383//! * virtual int32_t getNbOutputs() const noexcept = 0;
384//!
385//! * virtual DataType getOutputDataType(int32_t index, DataType const* inputTypes,
386//! int32_t nbInputs) const noexcept = 0;
387//!
388//! * virtual size_t getSerializationSize() const noexcept = 0;
389//!
390//! * virtual void serialize(void* buffer) const noexcept = 0;
391//!
392//! * virtual void destroy() noexcept = 0;
393//!
394//! * virtual void setPluginNamespace(char const* pluginNamespace) noexcept = 0;
395//!
396//! * virtual char const* getPluginNamespace() const noexcept = 0;
397//!
398//! For weakly typed networks, the inputTypes will always be DataType::kFLOAT or DataType::kINT32,
399//! and the returned type is canonicalized to DataType::kFLOAT if it is DataType::kHALF or DataType:kINT8.
400//! For strongly typed networks, inputTypes are inferred from previous operations, and getOutputDataType
401//! specifies the returned type based on the inputTypes.
402//! Details about the floating-point precision are elicited later by method supportsFormatCombination.
403//!
404//! \deprecated Deprecated in TensorRT 10.0. Please implement IPluginV3 instead.
405//!
406class TRT_DEPRECATED IPluginV2DynamicExt : public nvinfer1::IPluginV2Ext
407{
408public:
409 IPluginV2DynamicExt* clone() const noexcept override = 0;
410
411 //!
412 //! \brief Get expressions for computing dimensions of an output tensor from dimensions of the input tensors.
413 //!
414 //! \param outputIndex The index of the output tensor
415 //! \param inputs Expressions for dimensions of the input tensors
416 //! \param nbInputs The number of input tensors
417 //! \param exprBuilder Object for generating new expressions
418 //!
419 //! This function is called by the implementations of IBuilder during analysis of the network.
420 //!
421 //! Example #1: A plugin has a single output that transposes the last two dimensions of the plugin's single input.
422 //! The body of the override of getOutputDimensions can be:
423 //!
424 //! DimsExprs output(inputs[0]);
425 //! std::swap(output.d[output.nbDims-1], output.d[output.nbDims-2]);
426 //! return output;
427 //!
428 //! Example #2: A plugin concatenates its two inputs along the first dimension.
429 //! The body of the override of getOutputDimensions can be:
430 //!
431 //! DimsExprs output(inputs[0]);
432 //! output.d[0] = exprBuilder.operation(DimensionOperation::kSUM, *inputs[0].d[0], *inputs[1].d[0]);
433 //! return output;
434 //!
435 virtual DimsExprs getOutputDimensions(
436 int32_t outputIndex, DimsExprs const* inputs, int32_t nbInputs, IExprBuilder& exprBuilder) noexcept = 0;
437
438 //!
439 //! \brief Limit on number of format combinations accepted.
440 //!
441 static constexpr int32_t kFORMAT_COMBINATION_LIMIT = 100;
442
443 //!
444 //! \brief Return true if plugin supports the format and datatype for the input/output indexed by pos.
445 //!
446 //! For this method inputs are numbered 0..(nbInputs-1) and outputs are numbered nbInputs..(nbInputs+nbOutputs-1).
447 //! Using this numbering, pos is an index into InOut, where 0 <= pos < nbInputs+nbOutputs.
448 //!
449 //! TensorRT invokes this method to ask if the input/output indexed by pos supports the format/datatype specified
450 //! by inOut[pos].format and inOut[pos].type. The override should return true if that format/datatype at inOut[pos]
451 //! are supported by the plugin. If support is conditional on other input/output formats/datatypes, the plugin can
452 //! make its result conditional on the formats/datatypes in inOut[0..pos-1], which will be set to values
453 //! that the plugin supports. The override should not inspect inOut[pos+1..nbInputs+nbOutputs-1],
454 //! which will have invalid values. In other words, the decision for pos must be based on inOut[0..pos] only.
455 //!
456 //! Some examples:
457 //!
458 //! * A definition for a plugin that supports only FP16 NCHW:
459 //!
460 //! return inOut[pos].format == TensorFormat::kLINEAR && inOut[pos].type == DataType::kHALF;
461 //!
462 //! * A definition for a plugin that supports only FP16 NCHW for its two inputs,
463 //! and FP32 NCHW for its single output:
464 //!
465 //! return inOut[pos].format == TensorFormat::kLINEAR && (inOut[pos].type == (pos < 2 ? DataType::kHALF :
466 //! DataType::kFLOAT));
467 //!
468 //! * A definition for a "polymorphic" plugin with two inputs and one output that supports
469 //! any format or type, but the inputs and output must have the same format and type:
470 //!
471 //! return pos == 0 || (inOut[pos].format == inOut.format[0] && inOut[pos].type == inOut[0].type);
472 //!
473 //! Warning: TensorRT will stop asking for formats once it finds kFORMAT_COMBINATION_LIMIT on combinations.
474 //!
475 virtual bool supportsFormatCombination(
476 int32_t pos, PluginTensorDesc const* inOut, int32_t nbInputs, int32_t nbOutputs) noexcept = 0;
477
478 //!
479 //! \brief Configure the plugin.
480 //!
481 //! configurePlugin() can be called multiple times in both the build and execution phases. The build phase happens
482 //! before initialize() is called and only occurs during creation of an engine by IBuilder. The execution phase
483 //! happens after initialize() is called and occurs during both creation of an engine by IBuilder and execution
484 //! of an engine by IExecutionContext.
485 //!
486 //! Build phase:
487 //! IPluginV2DynamicExt->configurePlugin is called when a plugin is being prepared for profiling but not for any
488 //! specific input size. This provides an opportunity for the plugin to make algorithmic choices on the basis of
489 //! input and output formats, along with the bound of possible dimensions. The min and max value of the
490 //! DynamicPluginTensorDesc correspond to the kMIN and kMAX value of the current profile that the plugin is being
491 //! profiled for, with the desc.dims field corresponding to the dimensions of plugin specified at network creation.
492 //! Wildcard dimensions will exist during this phase in the desc.dims field.
493 //!
494 //! Execution phase:
495 //! IPluginV2DynamicExt->configurePlugin is called when a plugin is being prepared for executing the plugin for a
496 //! specific dimensions. This provides an opportunity for the plugin to change algorithmic choices based on the
497 //! explicit input dimensions stored in desc.dims field.
498 //! * IBuilder will call this function once per profile, with desc.dims resolved to the values specified by the
499 //! kOPT
500 //! field of the current profile. Wildcard dimensions will not exist during this phase.
501 //! * IExecutionContext will call this during the next subsequent instance enqueue[V2]() or execute[V2]() if:
502 //! - The batch size is changed from previous call of execute()/enqueue() if hasImplicitBatchDimension() returns
503 //! true.
504 //! - The optimization profile is changed via setOptimizationProfileAsync().
505 //! - An input execution binding is changed via setInputShape().
506 //! \warning The execution phase is timing critical during IExecutionContext but is not part of the timing loop when
507 //! called from IBuilder. Performance bottlenecks of configurePlugin won't show up during engine building but will
508 //! be visible during execution after calling functions that trigger layer resource updates.
509 //!
510 //! \param in The input tensors attributes that are used for configuration.
511 //! \param nbInputs Number of input tensors.
512 //! \param out The output tensors attributes that are used for configuration.
513 //! \param nbOutputs Number of output tensors.
514 //!
515 virtual void configurePlugin(DynamicPluginTensorDesc const* in, int32_t nbInputs,
516 DynamicPluginTensorDesc const* out, int32_t nbOutputs) noexcept = 0;
517
518 //!
519 //! \brief Find the workspace size required by the layer.
520 //!
521 //! This function is called after the plugin is configured, and possibly during execution.
522 //! The result should be a sufficient workspace size to deal with inputs and outputs of the given size
523 //! or any smaller problem.
524 //!
525 //! \return The workspace size.
526 //!
527 virtual size_t getWorkspaceSize(PluginTensorDesc const* inputs, int32_t nbInputs, PluginTensorDesc const* outputs,
528 int32_t nbOutputs) const noexcept = 0;
529
530 //!
531 //! \brief Execute the layer.
532 //!
533 //! \param inputDesc how to interpret the memory for the input tensors.
534 //! \param outputDesc how to interpret the memory for the output tensors.
535 //! \param inputs The memory for the input tensors.
536 //! \param outputs The memory for the output tensors.
537 //! \param workspace Workspace for execution.
538 //! \param stream The stream in which to execute the kernels.
539 //!
540 //! \return 0 for success, else non-zero (which will cause engine termination).
541 //!
542 virtual int32_t enqueue(PluginTensorDesc const* inputDesc, PluginTensorDesc const* outputDesc,
543 void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept = 0;
544
545protected:
546 //!
547 //! \brief Return the API version with which this plugin was built. The
548 //! upper byte reserved by TensorRT and is used to differentiate this from IPluginV2.
549 //!
550 //! Do not override this method as it is used by the TensorRT library to maintain backwards-compatibility with
551 //! plugins.
552 //!
553 int32_t getTensorRTVersion() const noexcept override
554 {
555 return (static_cast<int32_t>(PluginVersion::kV2_DYNAMICEXT) << 24 | (NV_TENSORRT_VERSION & 0xFFFFFF));
556 }
557
558 virtual ~IPluginV2DynamicExt() noexcept {}
559
560private:
561 // Following are obsolete base class methods, and must not be implemented or used.
562
563 //!
564 //! \brief Set plugin configuration
565 //!
566 void configurePlugin(Dims const*, int32_t, Dims const*, int32_t, DataType const*, DataType const*, bool const*,
567 bool const*, PluginFormat, int32_t) noexcept final
568 {
569 }
570
571 //!
572 //! \brief Check if provided data type is supported
573 //!
574 bool supportsFormat(DataType, PluginFormat) const noexcept final
575 {
576 return false;
577 }
578
579 //!
580 //! \brief Get output dimensions.
581 //!
582 Dims getOutputDimensions(int32_t, Dims const*, int32_t) noexcept final
583 {
584 return Dims{-1, {}};
585 }
586
587 //!
588 //! \brief Is output broadcasted across batch.
589 //!
590 //! \warning Expected to return false as implicit batch support was removed in TensorRT 10.0.
591 //!
592 //! \deprecated Deprecated in TensorRT 10.0. Implicit batch support is removed in TensorRT 10.0.
593 //!
594 TRT_DEPRECATED bool isOutputBroadcastAcrossBatch(int32_t, bool const*, int32_t) const noexcept final
595 {
596 return false;
597 }
598
599 //!
600 //! \brief Can output broadcasted across batch.
601 //!
602 //! \warning Expected to return false as implicit batch support was removed in TensorRT 10.0.
603 //!
604 //! \deprecated Deprecated in TensorRT 10.0. Implicit batch support is removed in TensorRT 10.0.
605 //!
606 TRT_DEPRECATED bool canBroadcastInputAcrossBatch(int32_t) const noexcept final
607 {
608 return true;
609 }
610
611 //!
612 //! \brief Get required workspace size in bytes.
613 //!
614 size_t getWorkspaceSize(int32_t) const noexcept final
615 {
616 return 0;
617 }
618
619 //!
620 //! \brief Run inference.
621 //!
622 int32_t enqueue(int32_t, void const* const*, void* const*, void*, cudaStream_t) noexcept final
623 {
624 return 1;
625 }
626};
627
628namespace v_1_0
629{
630class IStreamReader : public IVersionedInterface
631{
632public:
633 //!
634 //! TensorRT never calls the destructor for an IStreamReader defined by the
635 //! application.
636 //!
637 ~IStreamReader() override = default;
638 IStreamReader() = default;
639
640 //!
641 //! \brief Return version information associated with this interface. Applications must not override this method.
642 //!
643 InterfaceInfo getInterfaceInfo() const noexcept override
644 {
645 return InterfaceInfo{"IStreamReader", 1, 0};
646 }
647
648 //!
649 //! \brief Read the next number of bytes in the stream.
650 //!
651 //! \param destination The memory to write to
652 //! \param nbBytes The number of bytes to read
653 //!
654 //! \returns The number of bytes read. Negative values will be considered an automatic error.
655 //!
656 virtual int64_t read(void* destination, int64_t nbBytes) = 0;
657
658protected:
659 IStreamReader(IStreamReader const&) = default;
660 IStreamReader(IStreamReader&&) = default;
661 IStreamReader& operator=(IStreamReader const&) & = default;
662 IStreamReader& operator=(IStreamReader&&) & = default;
663};
664
665class IStreamWriter : public IVersionedInterface
666{
667public:
668 //!
669 //! TensorRT never calls the destructor for an IStreamWriter defined by the
670 //! application.
671 //!
672 ~IStreamWriter() override = default;
673 IStreamWriter() = default;
674
675 //!
676 //! \brief Return version information associated with this interface. Applications must not override this method.
677 //!
678 InterfaceInfo getInterfaceInfo() const noexcept final
679 {
680 return InterfaceInfo{"IStreamWriter", 1, 0};
681 }
682
683 //!
684 //! \brief write nbBytes of data into the stream.
685 //!
686 //! \param data The data to be written to stream
687 //! \param nbBytes The number of bytes to write
688 //!
689 //! \returns The number of bytes written. A value that is negative or less than nBytes indicates that an error
690 //! occurred and TensorRT will give up on writing to the stream.
691 //!
692 virtual int64_t write(void const* data, int64_t nbBytes) = 0;
693
694protected:
695 IStreamWriter(IStreamWriter const&) = default;
696 IStreamWriter(IStreamWriter&&) = default;
697 IStreamWriter& operator=(IStreamWriter const&) & = default;
698 IStreamWriter& operator=(IStreamWriter&&) & = default;
699};
700} // namespace v_1_0
701
702//!
703//! \class IStreamReader
704//!
705//! \brief Application-implemented class for reading data in a stream-based manner.
706//!
707//! \note To ensure compatibility of source code with future versions of TensorRT, use IStreamReader, not
708//! v_1_0::IStreamReader
709//!
710using IStreamReader = v_1_0::IStreamReader;
711
712//!
713//! \class IStreamWriter
714//!
715//! \brief Application-implemented class for writing data in a stream-based manner.
716//!
717//! \note To ensure compatibility of source code with future versions of TensorRT, use IStreamWriter, not
718//! v_1_0::IStreamWriter
719//!
720using IStreamWriter = v_1_0::IStreamWriter;
721
722//!
723//! \enum SeekPosition
724//! \brief Controls the seek mode of IStreamReaderV2.
725//!
726enum class SeekPosition : int32_t
727{
728 //! From the beginning of the file.
729 kSET = 0,
730
731 //! From the current position of the file.
732 kCUR = 1,
733
734 //! From the tail of the file.
735 kEND = 2,
736};
737
738namespace v_1_0
739{
740class IStreamReaderV2 : public IVersionedInterface
741{
742public:
743 //!
744 //! TensorRT never calls the destructor for an IStreamReaderV2 defined by the
745 //! application.
746 //!
747 ~IStreamReaderV2() override = default;
748 IStreamReaderV2() = default;
749
750 //!
751 //! \brief Return version information associated with this interface. Applications must not override this method.
752 //!
753 InterfaceInfo getInterfaceInfo() const noexcept override
754 {
755 return InterfaceInfo{"IStreamReaderV2", 1, 0};
756 }
757
758 //!
759 //! \brief Read the next number of bytes in the stream asynchronously.
760 //!
761 //! \param destination The memory to write to, call cudaPointerGetAttributes to get the memory location
762 //! \param nbBytes The number of bytes to read
763 //! \param stream The CUDA stream used to do the copy
764 //!
765 //! \returns The number of bytes read. Negative values indicate an unrecoverable error.
766 //! A zero indicates that the end of the stream has been reached.
767 //!
768 virtual int64_t read(void* destination, int64_t nbBytes, cudaStream_t stream) noexcept = 0;
769
770 //!
771 //! \brief Sets the position of the stream to the given offset.
772 //!
773 //! \param offset The number of bytes to offset from where.
774 //! \param where The position from where the offset is added. \see SeekPosition
775 //!
776 //! \returns True if the position is updated successfully.
777 //!
778 virtual bool seek(int64_t offset, SeekPosition where) noexcept = 0;
779
780protected:
781 IStreamReaderV2(IStreamReaderV2 const&) = default;
782 IStreamReaderV2(IStreamReaderV2&&) = default;
783 IStreamReaderV2& operator=(IStreamReaderV2 const&) & = default;
784 IStreamReaderV2& operator=(IStreamReaderV2&&) & = default;
785};
786} // namespace v_1_0
787
788//!
789//! \class IStreamReaderV2
790//!
791//! \brief Application-implemented class for reading data in a stream-based manner asynchronously. Intended for use with
792//! the GDS API for optimizing load times.
793//!
794//! \note To ensure compatibility of source code with future versions of TensorRT, use IStreamReaderV2, not
795//! v_1_0::IStreamReaderV2
796//!
797using IStreamReaderV2 = v_1_0::IStreamReaderV2;
798
799//!
800//! \class IPluginResourceContext
801//!
802//! \brief Interface for plugins to access per context resources provided by TensorRT
803//!
804//! There is no public way to construct an IPluginResourceContext. It appears as an argument to
805//! IPluginV3OneRuntime::attachToContext(). Overrides of that method can use the IPluginResourceContext object to access
806//! any available per context resources.
807//!
808//! \warning Do not inherit from this class, as doing so will break forward-compatibility of the API and ABI.
809//!
810//! \see IPluginV3OneRuntime::attachToContext()
811//!
812class IPluginResourceContext
813{
814public:
815 //! \brief Get the GPU allocator associated with the resource context
816 //!
817 //! \see IPluginV3OneRuntime::attachToContext()
818 //!
819 virtual IGpuAllocator* getGpuAllocator() const noexcept = 0;
820
821 //! \brief Get the error recorder associated with the resource context
822 //!
823 //! \see IPluginV3OneRuntime::attachToContext()
824 //!
825 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
826 virtual ~IPluginResourceContext() noexcept = default;
827
828protected:
829 IPluginResourceContext() = default;
830 IPluginResourceContext(IPluginResourceContext const&) = default;
831 IPluginResourceContext(IPluginResourceContext&&) = default;
832 IPluginResourceContext& operator=(IPluginResourceContext const&) & = default;
833 IPluginResourceContext& operator=(IPluginResourceContext&&) & = default;
834};
835
836namespace v_1_0
837{
838class IPluginV3OneCore : public IPluginCapability
839{
840public:
841 //!
842 //! \brief Return version information associated with this interface. Applications must not override this method.
843 //!
844 InterfaceInfo getInterfaceInfo() const noexcept override
845 {
846 return InterfaceInfo{"PLUGIN_V3ONE_CORE", 1, 0};
847 }
848
849 //!
850 //! \brief Return the plugin name. Should match the plugin name returned by the corresponding plugin creator.
851 //!
852 //! \see IPluginCreatorV3One::getPluginName()
853 //!
854 //! \warning The string returned must be NULL-terminated and have a length of 1024 bytes or less including the
855 //! NULL terminator.
856 //!
857 virtual AsciiChar const* getPluginName() const noexcept = 0;
858
859 //!
860 //! \brief Return the plugin version. Should match the plugin version returned by the corresponding plugin creator.
861 //!
862 //! \see IPluginCreatorV3One::getPluginVersion()
863 //!
864 //! \warning The string returned must be NULL-terminated and have a length of 1024 bytes or less including the
865 //! NULL terminator.
866 //!
867 virtual AsciiChar const* getPluginVersion() const noexcept = 0;
868
869 //!
870 //! \brief Return the namespace of the plugin object. Should match the plugin namespace returned by the
871 //! corresponding plugin creator.
872 //!
873 //! \see IPluginCreatorV3One::getPluginNamespace()
874 //!
875 //! \warning The string returned must be NULL-terminated and have a length of 1024 bytes or less including the
876 //! NULL terminator.
877 //!
878 virtual AsciiChar const* getPluginNamespace() const noexcept = 0;
879};
880
881class IPluginV3OneBuild : public IPluginCapability
882{
883public:
884 //!
885 //! \brief The default maximum number of format combinations that will be timed by TensorRT during the build phase
886 //!
887 //! \see getFormatCombinationLimit
888 //!
889 static constexpr int32_t kDEFAULT_FORMAT_COMBINATION_LIMIT = 100;
890
891 //!
892 //! \brief Return version information associated with this interface. Applications must not override this method.
893 //!
894 InterfaceInfo getInterfaceInfo() const noexcept override
895 {
896 return InterfaceInfo{"PLUGIN_V3ONE_BUILD", 1, 0};
897 }
898
899 //!
900 //! \brief Configure the plugin.
901 //!
902 //! configurePlugin() can be called multiple times in the build phase during creation of an engine by IBuilder.
903 //!
904 //! configurePlugin() is called when a plugin is being prepared for profiling but not for any
905 //! specific input size. This provides an opportunity for the plugin to make algorithmic choices on the basis of
906 //! input and output formats, along with the bound of possible dimensions. The min, opt and max value of the
907 //! DynamicPluginTensorDesc correspond to the kMIN, kOPT and kMAX value of the current profile that the plugin is
908 //! being profiled for, with the desc.dims field corresponding to the dimensions of plugin specified at network
909 //! creation. Wildcard dimensions may exist during this phase in the desc.dims field.
910 //!
911 //! \param in The input tensors attributes that are used for configuration.
912 //! \param nbInputs Number of input tensors.
913 //! \param out The output tensors attributes that are used for configuration.
914 //! \param nbOutputs Number of output tensors.
915 //!
916 //! \return 0 for success, else non-zero (which will cause engine termination, if invoked by TensorRT).
917 //!
918 virtual int32_t configurePlugin(DynamicPluginTensorDesc const* in, int32_t nbInputs,
919 DynamicPluginTensorDesc const* out, int32_t nbOutputs) noexcept = 0;
920
921 //!
922 //! \brief Provide the data types of the plugin outputs if the input tensors have the data types provided.
923 //!
924 //! \param outputTypes Pre-allocated array to which the output data types should be written.
925 //! \param nbOutputs The number of output tensors. This matches the value returned from getNbOutputs().
926 //! \param inputTypes The input data types.
927 //! \param nbInputs The number of input tensors.
928 //!
929 //! \return 0 for success, else non-zero (which will cause engine termination). The returned code will be reported
930 //! through the error recorder.
931 //!
932 //! \note Provide `DataType::kFLOAT`s if the layer has no inputs. The data type for any size tensor outputs must be
933 //! `DataType::kINT32`. The returned data types must each have a format that is supported by the plugin.
934 //!
935 //! \warning DataType:kBOOL and DataType::kUINT8 are not supported.
936 //!
937 virtual int32_t getOutputDataTypes(
938 DataType* outputTypes, int32_t nbOutputs, const DataType* inputTypes, int32_t nbInputs) const noexcept = 0;
939
940 //!
941 //! \brief Provide expressions for computing dimensions of the output tensors from dimensions of the input tensors.
942 //!
943 //! \param inputs Expressions for dimensions of the input tensors
944 //! \param nbInputs The number of input tensors
945 //! \param shapeInputs Expressions for values of the shape tensor inputs
946 //! \param nbShapeInputs The number of shape tensor inputs
947 //! \param outputs Pre-allocated array to which the output dimensions must be written
948 //! \param nbOutputs Number of outputs.
949 //! \param exprBuilder Object for generating new dimension expressions
950 //!
951 //! \note Any size tensor outputs must be declared to be 0D.
952 //!
953 //! \note The declaration of shapeInputs as DimsExprs is slightly abusive, because the "dimensions"
954 //! are actually the values of the shape tensor. For example, if the input shape tensor
955 //! is a 2x3 matrix, the DimsExprs will have six "dimensions": the three values from the first
956 //! row of the matrix followed by the three values from the second row of the matrix.
957 //!
958 //! \return 0 for success, else non-zero (which will cause engine termination). Returned code will be reported
959 //! through the error recorder.
960 //!
961 virtual int32_t getOutputShapes(DimsExprs const* inputs, int32_t nbInputs, DimsExprs const* shapeInputs,
962 int32_t nbShapeInputs, DimsExprs* outputs, int32_t nbOutputs, IExprBuilder& exprBuilder) noexcept = 0;
963
964 //!
965 //! \brief Return true if plugin supports the format and datatype for the input/output indexed by pos.
966 //!
967 //! For this method inputs are numbered 0.. (nbInputs - 1) and outputs are numbered nbInputs.. (nbInputs + nbOutputs
968 //! - 1). Using this numbering, pos is an index into InOut, where 0 <= pos < nbInputs + nbOutputs - 1.
969 //!
970 //! TensorRT invokes this method to ask if the input/output indexed by pos supports the format/datatype specified
971 //! by inOut[pos].format and inOut[pos].type. The override should return true if that format/datatype at inOut[pos]
972 //! are supported by the plugin. If support is conditional on other input/output formats/datatypes, the plugin can
973 //! make its result conditional on the formats/datatypes in inOut[0.. pos - 1], which will be set to values
974 //! that the plugin supports. The override should not inspect inOut[pos1.. nbInputs + nbOutputs - 1],
975 //! which will have invalid values. In other words, the decision for pos must be based on inOut[0..pos] only.
976 //!
977 //! Some examples:
978 //!
979 //! * A definition for a plugin that supports only FP16 NCHW:
980 //!
981 //! return inOut.format[pos] == TensorFormat::kLINEAR && inOut.type[pos] == DataType::kHALF;
982 //!
983 //! * A definition for a plugin that supports only FP16 NCHW for its two inputs,
984 //! and FP32 NCHW for its single output:
985 //!
986 //! return inOut.format[pos] == TensorFormat::kLINEAR && (inOut.type[pos] == pos < 2 ? DataType::kHALF :
987 //! DataType::kFLOAT);
988 //!
989 //! * A definition for a "polymorphic" plugin with two inputs and one output that supports
990 //! any format or type, but the inputs and output must have the same format and type:
991 //!
992 //! return pos == 0 || (inOut.format[pos] == inOut.format[0] && inOut.type[pos] == inOut.type[0]);
993 //!
994 //! \warning TensorRT will stop querying once it finds getFormatCombinationLimit() of combinations.
995 //!
996 //! \see getFormatCombinationLimit
997 //!
998 virtual bool supportsFormatCombination(
999 int32_t pos, DynamicPluginTensorDesc const* inOut, int32_t nbInputs, int32_t nbOutputs) noexcept = 0;
1000
1001 //!
1002 //! \brief Get the number of outputs from the plugin.
1003 //!
1004 //! \return The number of outputs, which must be a positive integer.
1005 //!
1006 virtual int32_t getNbOutputs() const noexcept = 0;
1007
1008 //!
1009 //! \brief Find the workspace size required by the layer.
1010 //!
1011 //! This function is called after the plugin is configured, and possibly during execution.
1012 //! The result should be a sufficient workspace size to deal with inputs and outputs of the given size
1013 //! or any smaller problem.
1014 //!
1015 //! \return The workspace size.
1016 //!
1017 virtual size_t getWorkspaceSize(DynamicPluginTensorDesc const* inputs, int32_t nbInputs,
1018 DynamicPluginTensorDesc const* outputs, int32_t nbOutputs) const noexcept
1019 {
1020 return 0;
1021 }
1022
1023 //!
1024 //! \brief Query for any custom tactics that the plugin intends to use
1025 //!
1026 //! This method queries for the set of tactics T(f) supported by the plugin for the format combination f indicated
1027 //! by the immediately preceding call to configurePlugin(). It is guaranteed to be called after configurePlugin().
1028 //!
1029 //! For each format combination provided through configurePlugin(), up to a maximum of getFormatCombinationLimit(),
1030 //! the plugin will be timed for each tactic advertised through this method for that format combination. i.e. The
1031 //! plugin will be timed \f$N = \sum_{i=0}^{i<getFormatCombinationLimit()} (T(f[i]))\f$ times. If \f$N = 1\f$, the
1032 //! plugin may not be timed. In pseudocode, the timing protocol appears as the following:
1033 //!
1034 //! counter = 0
1035 //! for each supported format combination
1036 //! ++counter
1037 //! if counter > getFormatCombinationLimit()
1038 //! goto done
1039 //! configurePlugin(...)
1040 //! for each tactic in getValidTactics(...)
1041 //! time tactic
1042 //! done:
1043 //!
1044 //!
1045 //! \param tactics Pre-allocated buffer to which the tactic values should be written
1046 //! \param nbTactics The number of tactics advertised through getNbTactics()
1047 //!
1048 //! \note The provided tactic values must be unique and non-zero. The tactic value 0 is reserved for the default
1049 //! tactic attached to each format combination.
1050 //!
1051 //! \return 0 for success, else non-zero (which will cause engine termination). The returned code will be reported
1052 //! through the error recorder.
1053 //!
1054 virtual int32_t getValidTactics(int32_t* tactics, int32_t nbTactics) noexcept
1055 {
1056 return 0;
1057 }
1058
1059 //!
1060 //! \brief Query for the number of custom tactics the plugin intends to use
1061 //!
1062 virtual int32_t getNbTactics() noexcept
1063 {
1064 return 0;
1065 }
1066
1067 //!
1068 //! \brief Called to query the suffix to use for the timing cache ID. May be called anytime after plugin creation.
1069 //!
1070 //! \return Suffix to use for timing cache ID, considering only the creation state of the plugin.
1071 //! Returning nullptr will disable timing caching for the plugin altogether.
1072 //!
1073 //! \note If timing caching is enabled for the plugin (by returning non-null), the I/O shape and format information
1074 //! will be automatically considered to form the prefix of the timing cache ID. Therefore, only other factors
1075 //! determining the creation state of the plugin, such as its attribute values, should be considered to compose the
1076 //! return value.
1077 //!
1078 virtual char const* getTimingCacheID() noexcept
1079 {
1080 return nullptr;
1081 }
1082
1083 //!
1084 //! \brief Return the maximum number of format combinations that will be timed by TensorRT during the build phase
1085 //!
1086 virtual int32_t getFormatCombinationLimit() noexcept
1087 {
1088 return kDEFAULT_FORMAT_COMBINATION_LIMIT;
1089 }
1090
1091 //!
1092 //! \brief Query for a string representing the configuration of the plugin. May be called anytime after
1093 //! plugin creation.
1094 //!
1095 //! \return A string representing the plugin's creation state, especially with regard to its attribute values.
1096 //!
1097 virtual char const* getMetadataString() noexcept
1098 {
1099 return nullptr;
1100 }
1101};
1102
1103class IPluginV3OneRuntime : public IPluginCapability
1104{
1105public:
1106 //!
1107 //! \brief Return version information associated with this interface. Applications must not override this method.
1108 //!
1109 InterfaceInfo getInterfaceInfo() const noexcept override
1110 {
1111 return InterfaceInfo{"PLUGIN_V3ONE_RUNTIME", 1, 0};
1112 }
1113
1114 //!
1115 //! \brief Set the tactic to be used in the subsequent call to enqueue(). If no custom tactics were advertised, this
1116 //! will have a value of 0, which is designated as the default tactic.
1117 //!
1118 //! \return 0 for success, else non-zero (which will cause engine termination). The returned code will be reported
1119 //! through the error recorder.
1120 //!
1121 virtual int32_t setTactic(int32_t tactic) noexcept
1122 {
1123 return 0;
1124 }
1125
1126 //!
1127 //! \brief Called when a plugin is being prepared for execution for specific dimensions. This could
1128 //! happen multiple times in the execution phase, both during creation of an engine by IBuilder and execution of an
1129 //! engine by IExecutionContext.
1130 //! * IBuilder will call this function once per profile, with `in` resolved to the values specified by the
1131 //! kOPT field of the current profile.
1132 //! * IExecutionContext will call this during the next subsequent instance of enqueueV3() or executeV2() if:
1133 //! - The optimization profile is changed via setOptimizationProfile() or setOptimizationProfileAsync().
1134 //! - An input binding is changed via setInputTensorAddress() or setTensorAddress() or setInputShape().
1135 //! \warning The execution phase is timing critical during IExecutionContext but is not part of the timing loop when
1136 //! called from IBuilder. Performance bottlenecks of onShapeChange() will not show up during engine building but
1137 //! will be visible during execution if any triggering functions are called.
1138 //!
1139 //! \param in The input tensors attributes that are used for configuration.
1140 //! \param nbInputs Number of input tensors.
1141 //! \param out The output tensors attributes that are used for configuration.
1142 //! \param nbOutputs Number of output tensors.
1143 //!
1144 virtual int32_t onShapeChange(
1145 PluginTensorDesc const* in, int32_t nbInputs, PluginTensorDesc const* out, int32_t nbOutputs) noexcept = 0;
1146
1147 //!
1148 //! \brief Execute the layer.
1149 //!
1150 //! \param inputDesc how to interpret the memory for the input tensors.
1151 //! \param outputDesc how to interpret the memory for the output tensors.
1152 //! \param inputs The memory for the input tensors.
1153 //! \param outputs The memory for the output tensors.
1154 //! \param workspace Workspace for execution.
1155 //! \param stream The stream in which to execute the kernels.
1156 //!
1157 //! \return 0 for success, else non-zero (which will cause engine termination). The returned code will be reported
1158 //! through the error recorder.
1159 //!
1160 virtual int32_t enqueue(PluginTensorDesc const* inputDesc, PluginTensorDesc const* outputDesc,
1161 void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept = 0;
1162
1163 //!
1164 //! \brief Clone the plugin, attach the cloned plugin object to a execution context and grant the cloned plugin
1165 //! access to some context resources.
1166 //!
1167 //! This function is called automatically for each plugin when a new execution context is created. The plugin may
1168 //! use resources provided by the IPluginResourceContext until the plugin is deleted by TensorRT.
1169 //!
1170 //! If the plugin needs per-context resources, it can be allocated here.
1171 //!
1172 //! \param context A resource context that exposes methods to get access to execution context specific resources.
1173 //! A different resource context is guaranteed for each different execution context to which the
1174 //! plugin is attached.
1175 //! \see IPluginResourceContext
1176 //!
1177 //! \note This method should clone the entire IPluginV3 object, not just the runtime interface
1178 //!
1179 //! \return A clone of the IPluginV3 object whose runtime interface on which this method is invoked, which has
1180 //! attached to the provided resource context.
1181 //!
1182 virtual IPluginV3* attachToContext(IPluginResourceContext* context) noexcept = 0;
1183
1184 //!
1185 //! \brief Get the plugin fields which should be serialized.
1186 //!
1187 //! \note The set of plugin fields returned does not necessarily need to match that advertised through
1188 //! getFieldNames() of the corresponding plugin creator.
1189
1190 //! \note To serialize arbitrary plugin data, use a PluginField of
1191 //! PluginFieldType::kUNKNOWN, with the length of the PluginField set to the correct number of bytes.
1192 //!
1193 virtual PluginFieldCollection const* getFieldsToSerialize() noexcept = 0;
1194};
1195} // namespace v_1_0
1196
1197namespace v_2_0
1198{
1199
1200class IPluginV3OneBuild : public v_1_0::IPluginV3OneBuild