codekingpro/portable-devtools
114k
1/*
2 * SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: Apache-2.0
4 *
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
8 *
9 * http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
16 */
17
18#ifndef NV_INFER_IMPL_H
19#define NV_INFER_IMPL_H
20
21#include "NvInferLegacyDims.h"
22#include "NvInferRuntimeCommon.h"
23
24// @cond SuppressDoxyWarnings
25
26namespace nvinfer1
27{
28
29class ILogger;
30
31namespace v_1_0
32{
33class IProgressMonitor;
34} // namespace v_1_0
35using IProgressMonitor = v_1_0::IProgressMonitor;
36
37namespace v_1_0
38{
39class IAlgorithmSelector;
40} // namespace v_1_0
41using IAlgorithmSelector = v_1_0::IAlgorithmSelector;
42
43namespace v_1_0
44{
45class IProfiler;
46} // namespace v_1_0
47using IProfiler = v_1_0::IProfiler;
48
49namespace v_1_0
50{
51class IOutputAllocator;
52} // namespace v_1_0
53using IOutputAllocator = v_1_0::IOutputAllocator;
54
55namespace v_1_0
56{
57class IDebugListener;
58} // namespace v_1_0
59using IDebugListener = v_1_0::IDebugListener;
60
61class IActivationLayer;
62class IAlgorithm;
63class IAlgorithmContext;
64class IAlgorithmIOInfo;
65class IAlgorithmVariant;
66class IAssertionLayer;
67class IAttention;
68class IBuilder;
69class IBuilderConfig;
70class IConcatenationLayer;
71class IConditionLayer;
72class IConstantLayer;
73class IConvolutionLayer;
74class ICudaEngine;
75class ICumulativeLayer;
76class IDeconvolutionLayer;
77class IDequantizeLayer;
78class IDimensionExpr;
79class IDynamicQuantizeLayer;
80class IEinsumLayer;
81class IElementWiseLayer;
82class IEngineInspector;
83class IExecutionContext;
84class IFillLayer;
85class IGatherLayer;
86class IGridSampleLayer;
87class IHostMemory;
88class IIdentityLayer;
89class ICastLayer;
90class IIfConditional;
91class IIfConditionalInputLayer;
92class IIfConditionalOutputLayer;
93class IInt8Calibrator;
94class IIteratorLayer;
95class IKVCacheUpdateLayer;
96class ILayer;
97class ILoop;
98class ILoopOutputLayer;
99class ILRNLayer;
100class IMatrixMultiplyLayer;
101class IMoELayer;
102class IDistCollectiveLayer;
103class INetworkDefinition;
104class INormalizationLayer;
105class INMSLayer;
106class INonZeroLayer;
107class IOneHotLayer;
108class IOptimizationProfile;
109class IPaddingLayer;
110class IParametricReLULayer;
111class IPlugin;
112class IPluginExt;
113class IPluginFactory;
114class IPluginLayer;
115class IPluginRegistry;
116class IPluginV2Layer;
117class IRotaryEmbeddingLayer;
118class IRuntimeConfig;
119
120namespace v_1_0
121{
122class IPluginV3;
123} // namespace v_1_0
124using IPluginV3 = v_1_0::IPluginV3;
125
126namespace v_1_0
127{
128class IStreamReader;
129class IStreamWriter;
130} // namespace v_1_0
131using IStreamReader = v_1_0::IStreamReader;
132using IStreamWriter = v_1_0::IStreamWriter;
133namespace v_1_0
134{
135class IStreamReaderV2;
136} // namespace v_1_0
137using IStreamReaderV2 = v_1_0::IStreamReaderV2;
138
139class IPluginV3Layer;
140class IPoolingLayer;
141class IQuantizeLayer;
142class IRaggedSoftMaxLayer;
143class IRecurrenceLayer;
144class IReduceLayer;
145class IRefitter;
146class IResizeLayer;
147class IReverseSequenceLayer;
148class IRuntime;
149class IScaleLayer;
150class IScatterLayer;
151class ISelectLayer;
152class ISerializationConfig;
153class IShapeLayer;
154class IShuffleLayer;
155class ISliceLayer;
156class ISoftMaxLayer;
157class ISqueezeLayer;
158class ITensor;
159
160namespace v_1_0
161{
162struct TimingCacheKey;
163struct TimingCacheValue;
164} // namespace v_1_0
165using TimingCacheKey = v_1_0::TimingCacheKey;
166using TimingCacheValue = v_1_0::TimingCacheValue;
167
168class ITimingCache;
169class ITopKLayer;
170class ITripLimitLayer;
171class IUnaryLayer;
172class IUnsqueezeLayer;
173struct Permutation;
174class Weights;
175
176enum class ActivationType : int32_t;
177enum class AttentionNormalizationOp : int32_t;
178enum class BoundingBoxFormat : int32_t;
179enum class BuilderFlag : int32_t;
180enum class CalibrationAlgoType : int32_t;
181enum class CumulativeOperation : int32_t;
182enum class DeviceType : int32_t;
183enum class DimensionOperation : int32_t;
184enum class ElementWiseOperation : int32_t;
185enum class EngineCapability : int32_t;
186enum class FillOperation : int32_t;
187enum class GatherMode : int32_t;
188enum class KVCacheMode : int32_t;
189enum class LayerInformationFormat : int32_t;
190enum class LayerType : int32_t;
191enum class LoopOutput : int32_t;
192enum class MatrixOperation : int32_t;
193enum class MemoryPoolType : int32_t;
194enum class MoEActType : int32_t;
195enum class NetworkDefinitionCreationFlag : int32_t;
196enum class OptProfileSelector : int32_t;
197enum class PaddingMode : int32_t;
198enum class PoolingType : int32_t;
199enum class ProfilingVerbosity : int32_t;
200enum class QuantizationFlag : int32_t;
201enum class ReduceOperation : int32_t;
202enum class CollectiveOperation : int32_t;
203enum class ResizeCoordinateTransformation : int32_t;
204enum class InterpolationMode : int32_t;
205enum class ResizeRoundMode : int32_t;
206enum class ResizeSelector : int32_t;
207enum class ScaleMode : int32_t;
208enum class ScatterMode : int32_t;
209enum class SampleMode : int32_t;
210enum class SerializationFlag : int32_t;
211enum class TensorIOMode : int32_t;
212enum class TensorLocation : int32_t;
213enum class TopKOperation : int32_t;
214enum class TripLimit : int32_t;
215enum class UnaryOperation : int32_t;
216enum class WeightsRole : int32_t;
217enum class PreviewFeature : int32_t;
218enum class HardwareCompatibilityLevel : int32_t;
219enum class ExecutionContextAllocationStrategy : int32_t;
220enum class RuntimePlatform : int32_t;
221enum class TilingOptimizationLevel : int32_t;
222enum class EngineStat : int32_t;
223
224
225using TacticSources = uint32_t;
226using TensorFormats = uint32_t;
227using BuilderFlags = uint32_t;
228using NetworkDefinitionCreationFlags = uint32_t;
229using QuantizationFlags = uint32_t;
230using TempfileControlFlags = uint32_t;
231using SerializationFlags = uint32_t;
232
233//!
234//! \file NvInferImpl.h
235//!
236//! This file contains definitions for API methods that cross the shared library boundary. These
237//! methods must not be called directly by applications; they should only be called through the
238//! API classes.
239//!
240
241namespace apiv
242{
243
244class VRoot
245{
246public:
247 virtual ~VRoot() noexcept = default;
248};
249
250class VHostMemory : public VRoot
251{
252public:
253 virtual void* data() const noexcept = 0;
254 virtual std::size_t size() const noexcept = 0;
255 virtual DataType type() const noexcept = 0;
256};
257
258class VDimensionExpr : public VRoot
259{
260public:
261 virtual bool isConstant() const = 0;
262 virtual int64_t getConstantValue() const = 0;
263 virtual bool isSizeTensor() const = 0;
264};
265
266class VExprBuilder : public VRoot
267{
268public:
269 virtual IDimensionExpr const* constant(int64_t value) = 0;
270 virtual IDimensionExpr const* operation(
271 DimensionOperation op, IDimensionExpr const& first, IDimensionExpr const& second)
272 = 0;
273 virtual IDimensionExpr const* declareSizeTensor(
274 int32_t outputIndex, IDimensionExpr const& opt, IDimensionExpr const& upper)
275 = 0;
276};
277
278class VRuntime : public VRoot
279{
280public:
281 virtual IRuntime* getPImpl() noexcept = 0;
282 virtual nvinfer1::ICudaEngine* deserializeCudaEngine(void const* blob, std::size_t size) noexcept = 0;
283 virtual nvinfer1::ICudaEngine* deserializeCudaEngine(IStreamReader& streamReader) noexcept = 0;
284 virtual void setDLACore(int32_t dlaCore) noexcept = 0;
285 virtual int32_t getDLACore() const noexcept = 0;
286 virtual int32_t getNbDLACores() const noexcept = 0;
287 virtual void setGpuAllocator(IGpuAllocator* allocator) noexcept = 0;
288 virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
289 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
290 virtual ILogger* getLogger() const noexcept = 0;
291 virtual bool setMaxThreads(int32_t maxThreads) noexcept = 0;
292 virtual int32_t getMaxThreads() const noexcept = 0;
293 virtual void setTemporaryDirectory(char const*) noexcept = 0;
294 virtual char const* getTemporaryDirectory() const noexcept = 0;
295 virtual void setTempfileControlFlags(TempfileControlFlags) noexcept = 0;
296 virtual TempfileControlFlags getTempfileControlFlags() const noexcept = 0;
297 virtual IPluginRegistry& getPluginRegistry() noexcept = 0;
298 virtual void setPluginRegistryParent(IPluginRegistry* parent) noexcept = 0;
299 virtual IRuntime* loadRuntime(char const* path) noexcept = 0;
300 virtual void setEngineHostCodeAllowed(bool allowed) noexcept = 0;
301 virtual bool getEngineHostCodeAllowed() const noexcept = 0;
302 // Added in TensorRT version 10.7
303 virtual nvinfer1::ICudaEngine* deserializeCudaEngineV2(IStreamReaderV2& streamReader) noexcept = 0;
304};
305
306class VRefitter : public VRoot
307{
308public:
309 virtual IRefitter* getPImpl() noexcept = 0;
310 virtual bool setWeights(char const* layerName, WeightsRole role, const Weights weights) noexcept = 0;
311 virtual bool refitCudaEngine() noexcept = 0;
312 virtual int32_t getMissing(int32_t size, char const** layerNames, WeightsRole* roles) noexcept = 0;
313 virtual int32_t getAll(int32_t size, char const** layerNames, WeightsRole* roles) noexcept = 0;
314 virtual bool setDynamicRange(char const* tensorName, float min, float max) noexcept = 0;
315 virtual float getDynamicRangeMin(char const* tensorName) const noexcept = 0;
316 virtual float getDynamicRangeMax(char const* tensorName) const noexcept = 0;
317 virtual int32_t getTensorsWithDynamicRange(int32_t size, char const** tensorNames) const noexcept = 0;
318 virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
319 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
320 virtual bool setNamedWeights(char const* name, Weights weights) noexcept = 0;
321 virtual int32_t getMissingWeights(int32_t size, char const** weightsNames) noexcept = 0;
322 virtual int32_t getAllWeights(int32_t size, char const** weightsNames) noexcept = 0;
323 virtual ILogger* getLogger() const noexcept = 0;
324 virtual bool setMaxThreads(int32_t maxThreads) noexcept = 0;
325 virtual int32_t getMaxThreads() const noexcept = 0;
326 virtual bool setNamedWeightsWithLocation(char const* name, Weights weights, TensorLocation location) noexcept = 0;
327 virtual Weights getNamedWeights(char const* weightsName) const noexcept = 0;
328 virtual TensorLocation getWeightsLocation(char const* weightsName) const noexcept = 0;
329 virtual bool unsetNamedWeights(char const* weightsName) noexcept = 0;
330 virtual void setWeightsValidation(bool weightsValidation) noexcept = 0;
331 virtual bool getWeightsValidation() const noexcept = 0;
332 virtual bool refitCudaEngineAsync(cudaStream_t stream) noexcept = 0;
333 virtual Weights getWeightsPrototype(char const* weightsName) const noexcept = 0;
334};
335
336class VOptimizationProfile : public VRoot
337{
338public:
339 virtual bool setDimensions(char const* inputName, OptProfileSelector select, Dims const& dims) noexcept = 0;
340 virtual Dims getDimensions(char const* inputName, OptProfileSelector select) const noexcept = 0;
341 virtual bool setShapeValues(
342 char const* inputName, OptProfileSelector select, int32_t const* values, int32_t nbValues) noexcept = 0;
343 virtual int32_t getNbShapeValues(char const* inputName) const noexcept = 0;
344 virtual int32_t const* getShapeValues(char const* inputName, OptProfileSelector select) const noexcept = 0;
345 virtual bool setExtraMemoryTarget(float target) noexcept = 0;
346 virtual float getExtraMemoryTarget() const noexcept = 0;
347 virtual bool isValid() const noexcept = 0;
348 // Added in TensorRT 10.11
349 TRT_NODISCARD virtual bool setShapeValuesV2(
350 char const* inputName, OptProfileSelector select, int64_t const* values, int32_t nbValues) noexcept = 0;
351 TRT_NODISCARD virtual int64_t const* getShapeValuesV2(
352 char const* inputName, OptProfileSelector select) const noexcept = 0;
353};
354
355class VCudaEngine : public VRoot
356{
357public:
358 virtual ICudaEngine* getPImpl() noexcept = 0;
359 virtual int32_t getNbLayers() const noexcept = 0;
360 virtual IHostMemory* serialize() const noexcept = 0;
361 virtual IExecutionContext* createExecutionContext(ExecutionContextAllocationStrategy strategy) noexcept = 0;
362 virtual IExecutionContext* createExecutionContextWithoutDeviceMemory() noexcept = 0;
363 virtual size_t getDeviceMemorySize() const noexcept = 0;
364 virtual bool isRefittable() const noexcept = 0;
365 virtual char const* getName() const noexcept = 0;
366 virtual int32_t getNbOptimizationProfiles() const noexcept = 0;
367 virtual int32_t const* getProfileTensorValues(
368 char const* tensorName, int32_t profileIndex, OptProfileSelector select) const noexcept = 0;
369 virtual EngineCapability getEngineCapability() const noexcept = 0;
370 virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
371 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
372 virtual bool hasImplicitBatchDimension() const noexcept = 0;
373 virtual TacticSources getTacticSources() const noexcept = 0;
374 virtual ProfilingVerbosity getProfilingVerbosity() const noexcept = 0;
375 virtual IEngineInspector* createEngineInspector() const noexcept = 0;
376 virtual Dims getTensorShape(char const* tensorName) const noexcept = 0;
377 virtual DataType getTensorDataType(char const* tensorName) const noexcept = 0;
378 virtual TensorLocation getTensorLocation(char const* tensorName) const noexcept = 0;
379 virtual bool isShapeInferenceIO(char const* tensorName) const noexcept = 0;
380 virtual TensorIOMode getTensorIOMode(char const* tensorName) const noexcept = 0;
381 virtual int32_t getTensorBytesPerComponent(char const* tensorName) const noexcept = 0;
382 virtual int32_t getTensorComponentsPerElement(char const* tensorName) const noexcept = 0;
383 virtual TensorFormat getTensorFormat(char const* tensorName) const noexcept = 0;
384 virtual char const* getTensorFormatDesc(char const* tensorName) const noexcept = 0;
385 virtual int32_t getTensorVectorizedDim(char const* tensorName) const noexcept = 0;
386 virtual Dims getProfileShape(
387 char const* tensorName, int32_t profileIndex, OptProfileSelector select) const noexcept = 0;
388 virtual int32_t getNbIOTensors() const noexcept = 0;
389 virtual char const* getIOTensorName(int32_t index) const noexcept = 0;
390 virtual HardwareCompatibilityLevel getHardwareCompatibilityLevel() const noexcept = 0;
391 virtual int32_t getNbAuxStreams() const noexcept = 0;
392
393 virtual int32_t getTensorBytesPerComponentV2(char const* tensorName, int32_t profileIndex) const noexcept = 0;
394 virtual int32_t getTensorComponentsPerElementV2(char const* tensorName, int32_t profileIndex) const noexcept = 0;
395 virtual TensorFormat getTensorFormatV2(char const* tensorName, int32_t profileIndex) const noexcept = 0;
396 virtual char const* getTensorFormatDescV2(char const* tensorName, int32_t profileIndex) const noexcept = 0;
397 virtual int32_t getTensorVectorizedDimV2(char const* tensorName, int32_t profileIndex) const noexcept = 0;
398
399 virtual ISerializationConfig* createSerializationConfig() noexcept = 0;
400 virtual IHostMemory* serializeWithConfig(ISerializationConfig& config) const noexcept = 0;
401
402 virtual size_t getDeviceMemorySizeForProfile(int32_t profileIndex) const noexcept = 0;
403 virtual IRefitter* createRefitter(ILogger& logger) noexcept = 0;
404
405 virtual bool setWeightStreamingBudget(int64_t gpuMemoryBudget) noexcept = 0;
406 virtual int64_t getWeightStreamingBudget() const noexcept = 0;
407 virtual int64_t getMinimumWeightStreamingBudget() const noexcept = 0;
408 virtual int64_t getStreamableWeightsSize() const noexcept = 0;
409
410 virtual bool isDebugTensor(char const* name) const noexcept = 0;
411
412 // Added in TensorRT 10.1
413 virtual bool setWeightStreamingBudgetV2(int64_t gpuMemoryBudget) noexcept = 0;
414 virtual int64_t getWeightStreamingBudgetV2() const noexcept = 0;
415 virtual int64_t getWeightStreamingAutomaticBudget() const noexcept = 0;
416 virtual int64_t getWeightStreamingScratchMemorySize() const noexcept = 0;
417 virtual int64_t getDeviceMemorySizeV2() const noexcept = 0;
418 virtual int64_t getDeviceMemorySizeForProfileV2(int32_t profileIndex) const noexcept = 0;
419 // Added in TensorRT 10.11
420 TRT_NODISCARD virtual int64_t const* getProfileTensorValuesV2(
421 char const* tensorName, int32_t profileIndex, OptProfileSelector select) const noexcept = 0;
422 TRT_NODISCARD virtual IExecutionContext* createExecutionContextWithRuntimeConfig(
423 IRuntimeConfig* runtimeConfig) noexcept = 0;
424 TRT_NODISCARD virtual IRuntimeConfig* createRuntimeConfig() noexcept = 0;
425 TRT_NODISCARD virtual int64_t getEngineStat(EngineStat stat) const noexcept = 0;
426 // Added in TensorRT 10.15
427 TRT_NODISCARD virtual char const* getAliasedInputTensor(char const* tensorName) const noexcept = 0;
428};
429
430class VExecutionContext : public VRoot
431{
432public:
433 virtual IExecutionContext* getPImpl() noexcept = 0;
434 virtual void setDebugSync(bool sync) noexcept = 0;
435 virtual bool getDebugSync() const noexcept = 0;
436 virtual void setProfiler(IProfiler*) noexcept = 0;
437 virtual IProfiler* getProfiler() const noexcept = 0;
438 virtual ICudaEngine const& getEngine() const noexcept = 0;
439 virtual void setName(char const* name) noexcept = 0;
440 virtual char const* getName() const noexcept = 0;
441 virtual void setDeviceMemory(void* memory) noexcept = 0;
442 virtual int32_t getOptimizationProfile() const noexcept = 0;
443 virtual bool allInputDimensionsSpecified() const noexcept = 0;
444 virtual bool allInputShapesSpecified() const noexcept = 0;
445 virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
446 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
447 virtual bool executeV2(void* const* bindings) noexcept = 0;
448 virtual bool setOptimizationProfileAsync(int32_t profileIndex, cudaStream_t stream) noexcept = 0;
449 virtual void setEnqueueEmitsProfile(bool enqueueEmitsProfile) noexcept = 0;
450 virtual bool getEnqueueEmitsProfile() const noexcept = 0;
451 virtual bool reportToProfiler() const noexcept = 0;
452 virtual bool setInputShape(char const* tensorName, Dims const& dims) noexcept = 0;
453 virtual Dims getTensorShape(char const* tensorName) const noexcept = 0;
454 virtual Dims getTensorStrides(char const* tensorName) const noexcept = 0;
455 virtual bool setTensorAddress(char const* tensorName, void* data) noexcept = 0;
456 virtual void const* getTensorAddress(char const* tensorName) const noexcept = 0;
457 virtual bool setInputTensorAddress(char const* tensorName, void const* data) noexcept = 0;
458 virtual bool setOutputTensorAddress(char const* tensorName, void* data) noexcept = 0;
459 virtual int32_t inferShapes(int32_t nbMaxNames, char const** tensorNames) noexcept = 0;
460 virtual bool setInputConsumedEvent(cudaEvent_t event) noexcept = 0;
461 virtual cudaEvent_t getInputConsumedEvent() const noexcept = 0;
462 virtual void* getOutputTensorAddress(char const* tensorName) const noexcept = 0;
463 virtual bool setOutputAllocator(char const* tensorName, IOutputAllocator* outputAllocator) noexcept = 0;
464 virtual IOutputAllocator* getOutputAllocator(char const* name) noexcept = 0;
465 virtual int64_t getMaxOutputSize(char const* tensorName) const noexcept = 0;
466 virtual bool setTemporaryStorageAllocator(IGpuAllocator* allocator) noexcept = 0;
467 virtual IGpuAllocator* getTemporaryStorageAllocator() const noexcept = 0;
468 virtual bool enqueueV3(cudaStream_t stream) noexcept = 0;
469 virtual void setPersistentCacheLimit(size_t size) noexcept = 0;
470 virtual size_t getPersistentCacheLimit() const noexcept = 0;
471 virtual bool setNvtxVerbosity(ProfilingVerbosity verbosity) noexcept = 0;
472 virtual ProfilingVerbosity getNvtxVerbosity() const noexcept = 0;
473 virtual void setAuxStreams(cudaStream_t* auxStreams, int32_t nbStreams) noexcept = 0;
474 virtual bool setDebugListener(IDebugListener* listener) noexcept = 0;
475 virtual IDebugListener* getDebugListener() noexcept = 0;
476 virtual bool setTensorDebugState(char const* name, bool flag) noexcept = 0;
477 virtual bool getDebugState(char const* name) const noexcept = 0;
478 virtual bool setAllTensorsDebugState(bool flag) noexcept = 0;
479 virtual size_t updateDeviceMemorySizeForShapes() noexcept = 0;
480 virtual void setDeviceMemoryV2(void* memory, int64_t size) noexcept = 0;
481 TRT_NODISCARD virtual IRuntimeConfig* getRuntimeConfig() const noexcept = 0;
482 virtual bool setUnfusedTensorsDebugState(bool flag) noexcept = 0;
483 virtual bool getUnfusedTensorsDebugState() const noexcept = 0;
484#if ENABLE_FEATURE_DISABLE_RUNTIME_ALLOCATION
485 virtual bool isStreamCapturable(cudaStream_t stream) const noexcept = 0;
486#endif // ENABLE_FEATURE_DISABLE_RUNTIME_ALLOCATION
487 virtual bool setCommunicator(void* communicator) noexcept = 0;
488};
489
490class VEngineInspector : public VRoot
491{
492public:
493 virtual IEngineInspector* getPImpl() noexcept = 0;
494 virtual bool setExecutionContext(IExecutionContext const* context) noexcept = 0;
495 virtual IExecutionContext const* getExecutionContext() const noexcept = 0;
496 virtual char const* getLayerInformation(int32_t layerIndex, LayerInformationFormat format) const noexcept = 0;
497 virtual char const* getEngineInformation(LayerInformationFormat format) const noexcept = 0;
498 virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
499 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
500};
501
502class VTensor : public VRoot
503{
504public:
505 virtual void setName(char const* name) noexcept = 0;
506 virtual char const* getName() const noexcept = 0;
507 virtual void setDimensions(Dims const& dimensions) noexcept = 0;
508 virtual Dims getDimensions() const noexcept = 0;
509 virtual void setType(DataType type) noexcept = 0;
510 virtual DataType getType() const noexcept = 0;
511 virtual bool setDynamicRange(float min, float max) noexcept = 0;
512 virtual bool isNetworkInput() const noexcept = 0;
513 virtual bool isNetworkOutput() const noexcept = 0;
514 virtual void setBroadcastAcrossBatch(bool broadcastAcrossBatch) noexcept = 0;
515 virtual bool getBroadcastAcrossBatch() const noexcept = 0;
516 virtual TensorLocation getLocation() const noexcept = 0;
517 virtual void setLocation(TensorLocation location) noexcept = 0;
518 virtual bool dynamicRangeIsSet() const noexcept = 0;
519 virtual void resetDynamicRange() noexcept = 0;
520 virtual float getDynamicRangeMin() const noexcept = 0;
521 virtual float getDynamicRangeMax() const noexcept = 0;
522 virtual void setAllowedFormats(TensorFormats formats) noexcept = 0;
523 virtual TensorFormats getAllowedFormats() const noexcept = 0;
524 virtual bool isShapeTensor() const noexcept = 0;
525 virtual bool isExecutionTensor() const noexcept = 0;
526 virtual void setDimensionName(int32_t index, char const* name) noexcept = 0;
527 virtual char const* getDimensionName(int32_t index) const noexcept = 0;
528};
529
530class VLayer : public VRoot
531{
532public:
533 virtual LayerType getType() const noexcept = 0;
534 virtual void setName(char const* name) noexcept = 0;
535 virtual char const* getName() const noexcept = 0;
536 virtual int32_t getNbInputs() const noexcept = 0;
537 virtual ITensor* getInput(int32_t index) const noexcept = 0;
538 virtual int32_t getNbOutputs() const noexcept = 0;
539 virtual ITensor* getOutput(int32_t index) const noexcept = 0;
540 virtual void setInput(int32_t index, ITensor& tensor) noexcept = 0;
541 virtual void setPrecision(DataType dataType) noexcept = 0;
542 virtual DataType getPrecision() const noexcept = 0;
543 virtual bool precisionIsSet() const noexcept = 0;
544 virtual void resetPrecision() noexcept = 0;
545 virtual void setOutputType(int32_t index, DataType dataType) noexcept = 0;
546 virtual DataType getOutputType(int32_t index) const noexcept = 0;
547 virtual bool outputTypeIsSet(int32_t index) const noexcept = 0;
548 virtual void resetOutputType(int32_t index) noexcept = 0;
549 virtual void setMetadata(char const* docString) noexcept = 0;
550 virtual char const* getMetadata() const noexcept = 0;
551 virtual bool setNbRanks(int32_t nbRanks) noexcept = 0;
552 virtual int32_t getNbRanks() const noexcept = 0;
553};
554
555class VConvolutionLayer : public VRoot
556{
557public:
558 virtual void setNbOutputMaps(int64_t nbOutputMaps) noexcept = 0;
559 virtual int64_t getNbOutputMaps() const noexcept = 0;
560 virtual void setNbGroups(int64_t nbGroups) noexcept = 0;
561 virtual int64_t getNbGroups() const noexcept = 0;
562 virtual void setKernelWeights(Weights weights) noexcept = 0;
563 virtual Weights getKernelWeights() const noexcept = 0;
564 virtual void setBiasWeights(Weights weights) noexcept = 0;
565 virtual Weights getBiasWeights() const noexcept = 0;
566 virtual void setPrePadding(Dims const& padding) noexcept = 0;
567 virtual Dims getPrePadding() const noexcept = 0;
568 virtual void setPostPadding(Dims const& padding) noexcept = 0;
569 virtual Dims getPostPadding() const noexcept = 0;
570 virtual void setPaddingMode(PaddingMode paddingMode) noexcept = 0;
571 virtual PaddingMode getPaddingMode() const noexcept = 0;
572 virtual void setKernelSizeNd(Dims const& kernelSize) noexcept = 0;
573 virtual Dims getKernelSizeNd() const noexcept = 0;
574 virtual void setStrideNd(Dims const& stride) noexcept = 0;
575 virtual Dims getStrideNd() const noexcept = 0;
576 virtual void setPaddingNd(Dims const& padding) noexcept = 0;
577 virtual Dims getPaddingNd() const noexcept = 0;
578 virtual void setDilationNd(Dims const& dilation) noexcept = 0;
579 virtual Dims getDilationNd() const noexcept = 0;
580};
581
582class VActivationLayer : public VRoot
583{
584public:
585 virtual void setActivationType(ActivationType type) noexcept = 0;
586 virtual ActivationType getActivationType() const noexcept = 0;
587 virtual void setAlpha(float alpha) noexcept = 0;
588 virtual void setBeta(float beta) noexcept = 0;
589 virtual float getAlpha() const noexcept = 0;
590 virtual float getBeta() const noexcept = 0;
591};
592
593class VPoolingLayer : public VRoot
594{
595public:
596 virtual void setPoolingType(PoolingType type) noexcept = 0;
597 virtual PoolingType getPoolingType() const noexcept = 0;
598 virtual void setBlendFactor(float blendFactor) noexcept = 0;
599 virtual float getBlendFactor() const noexcept = 0;
600 virtual void setAverageCountExcludesPadding(bool exclusive) noexcept = 0;
601 virtual bool getAverageCountExcludesPadding() const noexcept = 0;
602 virtual void setPrePadding(Dims const& padding) noexcept = 0;
603 virtual Dims getPrePadding() const noexcept = 0;
604 virtual void setPostPadding(Dims const& padding) noexcept = 0;
605 virtual Dims getPostPadding() const noexcept = 0;
606 virtual void setPaddingMode(PaddingMode paddingMode) noexcept = 0;
607 virtual PaddingMode getPaddingMode() const noexcept = 0;
608 virtual void setWindowSizeNd(Dims const& windowSize) noexcept = 0;
609 virtual Dims getWindowSizeNd() const noexcept = 0;
610 virtual void setStrideNd(Dims const& stride) noexcept = 0;
611 virtual Dims getStrideNd() const noexcept = 0;
612 virtual void setPaddingNd(Dims const& padding) noexcept = 0;
613 virtual Dims getPaddingNd() const noexcept = 0;
614};
615
616class VLRNLayer : public VRoot
617{
618public:
619 virtual void setWindowSize(int64_t windowSize) noexcept = 0;
620 virtual int64_t getWindowSize() const noexcept = 0;
621 virtual void setAlpha(float alpha) noexcept = 0;
622 virtual float getAlpha() const noexcept = 0;
623 virtual void setBeta(float beta) noexcept = 0;
624 virtual float getBeta() const noexcept = 0;
625 virtual void setK(float k) noexcept = 0;
626 virtual float getK() const noexcept = 0;
627};
628
629class VScaleLayer : public VRoot
630{
631public:
632 virtual void setMode(ScaleMode mode) noexcept = 0;
633 virtual ScaleMode getMode() const noexcept = 0;
634 virtual void setShift(Weights shift) noexcept = 0;
635 virtual Weights getShift() const noexcept = 0;
636 virtual void setScale(Weights scale) noexcept = 0;
637 virtual Weights getScale() const noexcept = 0;
638 virtual void setPower(Weights power) noexcept = 0;
639 virtual Weights getPower() const noexcept = 0;
640 virtual int32_t getChannelAxis() const noexcept = 0;
641 virtual void setChannelAxis(int32_t channelAxis) noexcept = 0;
642};
643
644class VSoftMaxLayer : public VRoot
645{
646public:
647 virtual void setAxes(uint32_t axes) noexcept = 0;
648 virtual uint32_t getAxes() const noexcept = 0;
649};
650
651class VConcatenationLayer : public VRoot
652{
653public:
654 virtual void setAxis(int32_t axis) noexcept = 0;
655 virtual int32_t getAxis() const noexcept = 0;
656};
657
658class VDeconvolutionLayer : public VRoot
659{
660public:
661 virtual void setNbOutputMaps(int64_t nbOutputMaps) noexcept = 0;
662 virtual int64_t getNbOutputMaps() const noexcept = 0;
663 virtual void setNbGroups(int64_t nbGroups) noexcept = 0;
664 virtual int64_t getNbGroups() const noexcept = 0;
665 virtual void setKernelWeights(Weights weights) noexcept = 0;
666 virtual Weights getKernelWeights() const noexcept = 0;
667 virtual void setBiasWeights(Weights weights) noexcept = 0;
668 virtual Weights getBiasWeights() const noexcept = 0;
669 virtual void setPrePadding(Dims const& padding) noexcept = 0;
670 virtual Dims getPrePadding() const noexcept = 0;
671 virtual void setPostPadding(Dims const& padding) noexcept = 0;
672 virtual Dims getPostPadding() const noexcept = 0;
673 virtual void setPaddingMode(PaddingMode paddingMode) noexcept = 0;
674 virtual PaddingMode getPaddingMode() const noexcept = 0;
675 virtual void setKernelSizeNd(Dims const& kernelSize) noexcept = 0;
676 virtual Dims getKernelSizeNd() const noexcept = 0;
677 virtual void setStrideNd(Dims const& stride) noexcept = 0;
678 virtual Dims getStrideNd() const noexcept = 0;
679 virtual void setPaddingNd(Dims const& padding) noexcept = 0;
680 virtual Dims getPaddingNd() const noexcept = 0;
681 virtual void setDilationNd(Dims const& dilation) noexcept = 0;
682 virtual Dims getDilationNd() const noexcept = 0;
683};
684
685class VElementWiseLayer : public VRoot
686{
687public:
688 virtual void setOperation(ElementWiseOperation op) noexcept = 0;
689 virtual ElementWiseOperation getOperation() const noexcept = 0;
690};
691
692class VGatherLayer : public VRoot
693{
694public:
695 virtual void setGatherAxis(int32_t axis) noexcept = 0;
696 virtual int32_t getGatherAxis() const noexcept = 0;
697 virtual void setNbElementWiseDims(int32_t k) noexcept = 0;
698 virtual int32_t getNbElementWiseDims() const noexcept = 0;
699 virtual void setMode(GatherMode mode) noexcept = 0;
700 virtual GatherMode getMode() const noexcept = 0;
701};
702
703class VPluginLayer : public VRoot
704{
705public:
706 virtual IPlugin& getPlugin() noexcept = 0;
707};
708
709class VPluginV2Layer : public VRoot
710{
711public:
712 virtual IPluginV2& getPlugin() noexcept = 0;
713};
714
715class VPluginV3Layer : public VRoot
716{
717public:
718 virtual IPluginV3& getPlugin() noexcept = 0;
719};
720
721class VUnaryLayer : public VRoot
722{
723public:
724 virtual void setOperation(UnaryOperation op) noexcept = 0;
725 virtual UnaryOperation getOperation() const noexcept = 0;
726};
727
728class VReduceLayer : public VRoot
729{
730public:
731 virtual void setOperation(ReduceOperation op) noexcept = 0;
732 virtual ReduceOperation getOperation() const noexcept = 0;
733 virtual void setReduceAxes(uint32_t reduceAxes) noexcept = 0;
734 virtual uint32_t getReduceAxes() const noexcept = 0;
735 virtual void setKeepDimensions(bool keepDimensions) noexcept = 0;
736 virtual bool getKeepDimensions() const noexcept = 0;
737};
738
739class VPaddingLayer : public VRoot
740{
741public:
742 virtual void setPrePaddingNd(Dims const& padding) noexcept = 0;
743 virtual Dims getPrePaddingNd() const noexcept = 0;
744 virtual void setPostPaddingNd(Dims const& padding) noexcept = 0;
745 virtual Dims getPostPaddingNd() const noexcept = 0;
746};
747
748class VShuffleLayer : public VRoot
749{
750public:
751 virtual void setFirstTranspose(Permutation const& permutation) noexcept = 0;
752 virtual Permutation const& getFirstTranspose() const noexcept = 0;
753 virtual void setReshapeDimensions(Dims const& dimensions) noexcept = 0;
754 virtual Dims getReshapeDimensions() const noexcept = 0;
755 virtual void setSecondTranspose(Permutation const& permutation) noexcept = 0;
756 virtual Permutation const& getSecondTranspose() const noexcept = 0;
757 virtual void setZeroIsPlaceholder(bool zeroIsPlaceholder) noexcept = 0;
758 virtual bool getZeroIsPlaceholder() const noexcept = 0;
759};
760
761class VSliceLayer : public VRoot
762{
763public:
764 virtual void setStart(Dims const& start) noexcept = 0;
765 virtual Dims getStart() const noexcept = 0;
766 virtual void setSize(Dims const& size) noexcept = 0;
767 virtual Dims getSize() const noexcept = 0;
768 virtual void setStride(Dims const& stride) noexcept = 0;
769 virtual Dims getStride() const noexcept = 0;
770 virtual void setMode(SampleMode mode) noexcept = 0;
771 virtual SampleMode getMode() const noexcept = 0;
772 virtual void setAxes(Dims const& axes) noexcept = 0;
773 virtual Dims getAxes() const noexcept = 0;
774};
775
776class VShapeLayer : public VRoot
777{
778public:
779};
780
781class VTopKLayer : public VRoot
782{
783public:
784 virtual void setOperation(TopKOperation op) noexcept = 0;
785 virtual TopKOperation getOperation() const noexcept = 0;
786 virtual void setK(int32_t k) noexcept = 0;
787 virtual int32_t getK() const noexcept = 0;
788 virtual void setReduceAxes(uint32_t reduceAxes) noexcept = 0;
789 virtual uint32_t getReduceAxes() const noexcept = 0;
790 virtual bool setIndicesType(DataType type) noexcept = 0;
791 virtual DataType getIndicesType() const noexcept = 0;
792};
793
794class VMatrixMultiplyLayer : public VRoot
795{
796public:
797 virtual void setOperation(int32_t index, MatrixOperation op) noexcept = 0;
798 virtual MatrixOperation getOperation(int32_t index) const noexcept = 0;
799};
800
801class VNonZeroLayer : public VRoot
802{
803public:
804 virtual bool setIndicesType(DataType type) noexcept = 0;
805 virtual DataType getIndicesType() const noexcept = 0;
806};
807
808class VRaggedSoftMaxLayer : public VRoot
809{
810public:
811};
812
813class VIdentityLayer : public VRoot
814{
815public:
816};
817
818class VCastLayer : public VRoot
819{
820public:
821 virtual void setToType(DataType toType) noexcept = 0;
822 virtual DataType getToType() const noexcept = 0;
823};
824
825class VConstantLayer : public VRoot
826{
827public:
828 virtual void setWeights(Weights weights) noexcept = 0;
829 virtual Weights getWeights() const noexcept = 0;
830 virtual void setDimensions(Dims const& dimensions) noexcept = 0;
831 virtual Dims getDimensions() const noexcept = 0;
832};
833
834class VParametricReLULayer : public VRoot
835{
836public:
837};
838
839class VResizeLayer : public VRoot
840{
841public:
842 virtual void setOutputDimensions(Dims const& dimensions) noexcept = 0;
843 virtual Dims getOutputDimensions() const noexcept = 0;
844 virtual void setScales(float const* scales, int32_t nbScales) noexcept = 0;
845 virtual int32_t getScales(int32_t size, float* scales) const noexcept = 0;
846 virtual void setResizeMode(InterpolationMode interpolationMode) noexcept = 0;
847 virtual InterpolationMode getResizeMode() const noexcept = 0;
848 virtual void setCoordinateTransformation(ResizeCoordinateTransformation coordTransform) noexcept = 0;
849 virtual ResizeCoordinateTransformation getCoordinateTransformation() const noexcept = 0;
850 virtual void setSelectorForSinglePixel(ResizeSelector selector) noexcept = 0;
851 virtual ResizeSelector getSelectorForSinglePixel() const noexcept = 0;
852 virtual void setNearestRounding(ResizeRoundMode value) noexcept = 0;
853 virtual ResizeRoundMode getNearestRounding() const noexcept = 0;
854 virtual void setCubicCoeff(float value) noexcept = 0;
855 virtual float getCubicCoeff() const noexcept = 0;
856 virtual void setExcludeOutside(bool value) noexcept = 0;
857 virtual bool getExcludeOutside() const noexcept = 0;
858};
859
860class VLoopBoundaryLayer : public VRoot
861{
862public:
863 virtual ILoop* getLoop() const noexcept = 0;
864};
865
866class VRecurrenceLayer : public VRoot
867{
868public:
869};
870
871class VLoopOutputLayer : public VRoot
872{
873public:
874 virtual LoopOutput getLoopOutput() const noexcept = 0;
875 virtual void setAxis(int32_t axis) noexcept = 0;
876 virtual int32_t getAxis() const noexcept = 0;
877};
878
879class VTripLimitLayer : public VRoot
880{
881public:
882 virtual TripLimit getTripLimit() const noexcept = 0;
883};
884
885class VIteratorLayer : public VRoot
886{
887public:
888 virtual void setAxis(int32_t axis) noexcept = 0;
889 virtual int32_t getAxis() const noexcept = 0;
890 virtual void setReverse(bool reverse) noexcept = 0;
891 virtual bool getReverse() const noexcept = 0;
892};
893class VLoop : public VRoot
894{
895public:
896 virtual IRecurrenceLayer* addRecurrence(ITensor& initialValue) noexcept = 0;
897 virtual ITripLimitLayer* addTripLimit(ITensor& tensor, TripLimit limit) noexcept = 0;
898 virtual IIteratorLayer* addIterator(ITensor& tensor, int32_t axis = 0, bool reverse = false) noexcept = 0;
899 virtual ILoopOutputLayer* addLoopOutput(ITensor& tensor, LoopOutput outputKind, int32_t axis = 0) noexcept = 0;
900 virtual void setName(char const* name) noexcept = 0;
901 virtual char const* getName() const noexcept = 0;
902};
903
904class VConditionalBoundaryLayer : public VRoot
905{
906public:
907 virtual IIfConditional* getConditional() const noexcept = 0;
908};
909
910class VConditionLayer : public VRoot
911{
912public:
913};
914
915class VConditionalInputLayer : public VRoot
916{
917public:
918};
919
920class VConditionalOutputLayer : public VRoot
921{
922public:
923};
924
925class VIfConditional : public VRoot
926{
927public:
928 virtual IConditionLayer* setCondition(ITensor& tensor) noexcept = 0;
929 virtual IIfConditionalInputLayer* addInput(ITensor& tensor) noexcept = 0;
930 virtual IIfConditionalOutputLayer* addOutput(ITensor& trueTensor, ITensor& falseTensor) noexcept = 0;
931 virtual void setName(char const* name) noexcept = 0;
932 virtual char const* getName() const noexcept = 0;
933};
934
935class VAttentionBoundaryLayer : public VRoot
936{
937public:
938 virtual IAttention* getAttention() const noexcept = 0;
939};
940
941class VAttentionInputLayer : public VRoot
942{
943public:
944};
945
946class VAttentionOutputLayer : public VRoot
947{
948public:
949};
950
951class VAttention : public VRoot
952{
953public:
954 TRT_NODISCARD virtual bool setInput(int32_t index, ITensor& input) noexcept = 0;
955 TRT_NODISCARD virtual int32_t getNbInputs() const noexcept = 0;
956 TRT_NODISCARD virtual ITensor* getInput(int32_t index) const noexcept = 0;
957 TRT_NODISCARD virtual int32_t getNbOutputs() const noexcept = 0;
958 TRT_NODISCARD virtual ITensor* getOutput(int32_t index) const noexcept = 0;
959 TRT_NODISCARD virtual bool setName(char const* name) noexcept = 0;
960 TRT_NODISCARD virtual char const* getName() const noexcept = 0;
961 TRT_NODISCARD virtual bool setNormalizationOperation(AttentionNormalizationOp op) noexcept = 0;
962 TRT_NODISCARD virtual AttentionNormalizationOp getNormalizationOperation() const noexcept = 0;
963 TRT_NODISCARD virtual bool setCausal(bool isCausal) noexcept = 0;
964 TRT_NODISCARD virtual bool getCausal() const noexcept = 0;
965 TRT_NODISCARD virtual bool setMask(ITensor& mask) noexcept = 0;
966 TRT_NODISCARD virtual ITensor* getMask() const noexcept = 0;
967 TRT_NODISCARD virtual bool setDecomposable(bool decomposable) noexcept = 0;
968 TRT_NODISCARD virtual bool getDecomposable() const noexcept = 0;
969 TRT_NODISCARD virtual bool setNormalizationQuantizeScale(ITensor& tensor) noexcept = 0;
970 TRT_NODISCARD virtual ITensor* getNormalizationQuantizeScale() const noexcept = 0;
971 TRT_NODISCARD virtual bool setNormalizationQuantizeToType(DataType type) noexcept = 0;
972 TRT_NODISCARD virtual DataType getNormalizationQuantizeToType() const noexcept = 0;
973 TRT_NODISCARD virtual bool setMetadata(char const* docString) noexcept = 0;
974 TRT_NODISCARD virtual char const* getMetadata() const noexcept = 0;
975 TRT_NODISCARD virtual bool setNbRanks(int32_t nbRanks) noexcept = 0;
976 TRT_NODISCARD virtual int32_t getNbRanks() const noexcept = 0;
977}; // class VAttention
978
979class VSelectLayer : public VRoot
980{
981};
982
983class VAssertionLayer : public VRoot
984{
985public:
986 virtual void setMessage(char const* message) noexcept = 0;
987 virtual char const* getMessage() const noexcept = 0;
988};
989
990class VFillLayer : public VRoot
991{
992public:
993 virtual void setDimensions(Dims const& dimensions) noexcept = 0;
994 virtual Dims getDimensions() const noexcept = 0;
995 virtual void setOperation(FillOperation op) noexcept = 0;
996 virtual FillOperation getOperation() const noexcept = 0;
997 virtual void setAlpha(double alpha) noexcept = 0;
998 virtual double getAlpha() const noexcept = 0;
999 virtual void setBeta(double beta) noexcept = 0;
1000 virtual double getBeta() const noexcept = 0;
1001 virtual void setAlphaInt64(int64_t alpha) noexcept = 0;
1002 virtual int64_t getAlphaInt64() const noexcept = 0;
1003 virtual void setBetaInt64(int64_t beta) noexcept = 0;
1004 virtual int64_t getBetaInt64() const noexcept = 0;
1005 virtual bool isAlphaBetaInt64() const noexcept = 0;
1006 virtual DataType getToType() const noexcept = 0;
1007 virtual void setToType(DataType toType) noexcept = 0;
1008};
1009
1010class VQuantizeLayer : public VRoot
1011{
1012public:
1013 virtual int32_t getAxis() const noexcept = 0;
1014 virtual void setAxis(int32_t axis) noexcept = 0;
1015 virtual DataType getToType() const noexcept = 0;
1016 virtual void setToType(DataType toType) noexcept = 0;
1017 virtual Dims getBlockShape() const noexcept = 0;
1018 virtual bool setBlockShape(Dims const& blockShape) noexcept = 0;
1019};
1020
1021class VDequantizeLayer : public VRoot
1022{
1023public:
1024 virtual int32_t getAxis() const noexcept = 0;
1025 virtual void setAxis(int32_t axis) noexcept = 0;
1026 virtual DataType getToType() const noexcept = 0;
1027 virtual void setToType(DataType toType) noexcept = 0;
1028 virtual Dims getBlockShape() const noexcept = 0;
1029 virtual bool setBlockShape(Dims const& blockShape) noexcept = 0;
1030};
1031
1032class VDynamicQuantizeLayer : public VRoot
1033{
1034public:
1035 TRT_DEPRECATED virtual int32_t getAxis() const noexcept = 0;
1036 TRT_DEPRECATED virtual void setAxis(int32_t axis) noexcept = 0;
1037 TRT_DEPRECATED virtual int32_t getBlockSize() const noexcept = 0;
1038 TRT_DEPRECATED virtual void setBlockSize(int32_t axis) noexcept = 0;
1039 virtual DataType getScaleType() const noexcept = 0;
1040 virtual void setScaleType(DataType axis) noexcept = 0;
1041 virtual DataType getToType() const noexcept = 0;
1042 virtual void setToType(DataType toType) noexcept = 0;
1043 virtual Dims getBlockShape() const noexcept = 0;
1044 virtual void setBlockShape(Dims const& blockShape) noexcept = 0;
1045};
1046
1047class VScatterLayer : public VRoot
1048{
1049public:
1050 virtual void setMode(ScatterMode mode) noexcept = 0;
1051 virtual ScatterMode getMode() const noexcept = 0;
1052 virtual void setAxis(int32_t axis) noexcept = 0;
1053 virtual int32_t getAxis() const noexcept = 0;
1054}; // class VScatterLayer
1055
1056class VEinsumLayer : public VRoot
1057{
1058public:
1059 virtual bool setEquation(char const* equation) noexcept = 0;
1060 virtual char const* getEquation() const noexcept = 0;
1061};
1062
1063class VOneHotLayer : public VRoot
1064{
1065public:
1066 virtual int32_t getAxis() const noexcept = 0;
1067 virtual void setAxis(int32_t axis) noexcept = 0;
1068}; // class VOneHotLayer
1069
1070class VGridSampleLayer : public VRoot
1071{
1072public:
1073 virtual void setInterpolationMode(InterpolationMode mode) noexcept = 0;
1074 virtual InterpolationMode getInterpolationMode() const noexcept = 0;
1075 virtual void setAlignCorners(bool alignCorners) noexcept = 0;
1076 virtual bool getAlignCorners() const noexcept = 0;
1077 virtual bool setSampleMode(SampleMode mode) noexcept = 0;
1078 virtual SampleMode getSampleMode() const noexcept = 0;
1079}; // class VGridSampleLayer
1080
1081class VNMSLayer : public VRoot
1082{
1083public:
1084 virtual void setBoundingBoxFormat(BoundingBoxFormat fmt) noexcept = 0;
1085 virtual BoundingBoxFormat getBoundingBoxFormat() const noexcept = 0;
1086 virtual void setTopKBoxLimit(int32_t limit) noexcept = 0;
1087 virtual int32_t getTopKBoxLimit() const noexcept = 0;
1088 virtual bool setIndicesType(DataType type) noexcept = 0;
1089 virtual DataType getIndicesType() const noexcept = 0;
1090}; // class VNMSLayer
1091
1092class VReverseSequenceLayer : public VRoot
1093{
1094public:
1095 virtual void setBatchAxis(int32_t batchAxis) noexcept = 0;
1096 virtual int32_t getBatchAxis() const noexcept = 0;
1097
1098 virtual void setSequenceAxis(int32_t sequenceAxis) noexcept = 0;
1099 virtual int32_t getSequenceAxis() const noexcept = 0;
1100}; // class VReverseSequenceLayer
1101
1102class VNormalizationLayer : public VRoot
1103{
1104public:
1105 virtual void setEpsilon(float eps) noexcept = 0;
1106 virtual float getEpsilon() const noexcept = 0;
1107 virtual void setAxes(uint32_t axesMask) noexcept = 0;
1108 virtual uint32_t getAxes() const noexcept = 0;
1109 virtual void setNbGroups(int64_t nbGroups) noexcept = 0;
1110 virtual int64_t getNbGroups() const noexcept = 0;
1111 virtual void setComputePrecision(DataType type) noexcept = 0;
1112 virtual DataType getComputePrecision() const noexcept = 0;
1113 virtual bool isV2() const noexcept = 0;
1114}; // class VNormalizationLayer
1115
1116class VSqueezeLayer : public VRoot
1117{
1118};
1119
1120class VUnsqueezeLayer : public VRoot
1121{
1122};
1123
1124class VCumulativeLayer : public VRoot
1125{
1126public:
1127 virtual bool setOperation(CumulativeOperation op) noexcept = 0;
1128 virtual CumulativeOperation getOperation() const noexcept = 0;
1129 virtual void setExclusive(bool exclusive) noexcept = 0;
1130 virtual bool getExclusive() const noexcept = 0;
1131 virtual void setReverse(bool reverse) noexcept = 0;
1132 virtual bool getReverse() const noexcept = 0;
1133}; // class VCumulativeLayer
1134
1135class VRotaryEmbeddingLayer : public VRoot
1136{
1137public:
1138 virtual void setInterleaved(bool interleaved) noexcept = 0;
1139 virtual bool getInterleaved() const noexcept = 0;
1140 virtual bool setRotaryEmbeddingDim(int32_t rotaryEmbeddingDim) noexcept = 0;
1141 virtual int32_t getRotaryEmbeddingDim() const noexcept = 0;
1142 virtual void setInput(int32_t index, ITensor& input) noexcept = 0;
1143}; // class VRotaryEmbeddingLayer
1144
1145class VKVCacheUpdateLayer : public VRoot
1146{
1147public:
1148 TRT_NODISCARD virtual bool setCacheMode(KVCacheMode cacheMode) noexcept = 0;
1149 TRT_NODISCARD virtual KVCacheMode getCacheMode() const noexcept = 0;
1150}; // class VKVCacheUpdateLayer
1151
1152class VMoELayer : public VRoot
1153{
1154public:
1155 virtual void setGatedWeights(ITensor& fcGateWeights, ITensor& fcUpWeights, ITensor& fcDownWeights, MoEActType activationType) noexcept = 0;
1156 virtual void setGatedBiases(ITensor& fcGateBiases, ITensor& fcUpBiases, ITensor& fcDownBiases) noexcept = 0;
1157 virtual void setActivationType(MoEActType activationType) noexcept = 0;
1158 virtual MoEActType getActivationType() const noexcept = 0;
1159 virtual void setQuantizationStatic(ITensor& fcDownActivationScale, DataType dataType) noexcept = 0;
1160 virtual void setQuantizationDynamicDblQ(ITensor& fcDownActivationDblQScale, DataType dataType, Dims const& blockShape, DataType dynQOutputScaleType) noexcept = 0;
1161 virtual void setQuantizationToType(DataType type) noexcept = 0;
1162 virtual DataType getQuantizationToType() const noexcept = 0;
1163 virtual void setQuantizationBlockShape(Dims const& blockShape) noexcept = 0;
1164 virtual Dims getQuantizationBlockShape() const noexcept = 0;
1165 virtual void setDynQOutputScaleType(DataType type) noexcept = 0;
1166 virtual DataType getDynQOutputScaleType() const noexcept = 0;
1167 virtual void setSwigluParams(float limit, float alpha, float beta) noexcept = 0;
1168 virtual void setSwigluParamLimit(float limit) noexcept = 0;
1169 virtual float getSwigluParamLimit() const noexcept = 0;
1170 virtual void setSwigluParamAlpha(float alpha) noexcept = 0;
1171 virtual float getSwigluParamAlpha() const noexcept = 0;
1172 virtual void setSwigluParamBeta(float beta) noexcept = 0;
1173 virtual float getSwigluParamBeta() const noexcept = 0;
1174 virtual void setInput(int32_t index, ITensor& tensor) noexcept = 0;
1175}; // class VMoELayer
1176
1177
1178class VNetworkDefinition : public VRoot
1179{
1180public:
1181 virtual ITensor* addInput(char const* name, DataType type, Dims const& dimensions) noexcept = 0;
1182 virtual void markOutput(ITensor& tensor) noexcept = 0;
1183 virtual IActivationLayer* addActivation(ITensor& input, ActivationType type) noexcept = 0;
1184 virtual ILRNLayer* addLRN(ITensor& input, int64_t window, float alpha, float beta, float k) noexcept = 0;
1185 virtual IScaleLayer* addScale(
1186 ITensor& input, ScaleMode mode, Weights shift, Weights scale, Weights power) noexcept = 0;
1187 virtual ISoftMaxLayer* addSoftMax(ITensor& input) noexcept = 0;
1188 virtual IConcatenationLayer* addConcatenation(ITensor* const* inputs, int32_t nbInputs) noexcept = 0;
1189 virtual IElementWiseLayer* addElementWise(ITensor& input1, ITensor& input2, ElementWiseOperation op) noexcept = 0;
1190 virtual IUnaryLayer* addUnary(ITensor& input, UnaryOperation operation) noexcept = 0;
1191 virtual IShuffleLayer* addShuffle(ITensor& input) noexcept = 0;
1192 virtual int32_t getNbLayers() const noexcept = 0;
1193 virtual ILayer* getLayer(int32_t index) const noexcept = 0;
1194 virtual int32_t getNbInputs() const noexcept = 0;
1195 virtual ITensor* getInput(int32_t index) const noexcept = 0;
1196 virtual int32_t getNbOutputs() const noexcept = 0;
1197 virtual ITensor* getOutput(int32_t index) const noexcept = 0;
1198 virtual IReduceLayer* addReduce(
1199 ITensor& input, ReduceOperation operation, uint32_t reduceAxes, bool keepDimensions) noexcept
1200 = 0;