Package inference

Class ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest

java.lang.Object

com.google.protobuf.AbstractMessageLite

com.google.protobuf.AbstractMessage

com.google.protobuf.GeneratedMessageV3

inference.ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest

All Implemented Interfaces:

com.google.protobuf.Message, com.google.protobuf.MessageLite, com.google.protobuf.MessageLiteOrBuilder, com.google.protobuf.MessageOrBuilder, ModelConfigOuterClass.ModelSequenceBatching.StrategyOldestOrBuilder, Serializable

Enclosing class:

ModelConfigOuterClass.ModelSequenceBatching

public static final class ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest
extends com.google.protobuf.GeneratedMessageV3
implements ModelConfigOuterClass.ModelSequenceBatching.StrategyOldestOrBuilder

@@  .. cpp:var:: message StrategyOldest
@@
@@     The sequence batcher maintains up to 'max_candidate_sequences'
@@     candidate sequences. 'max_candidate_sequences' can be greater
@@     than the model's 'max_batch_size'. For inferencing the batcher
@@     chooses from the candidate sequences up to 'max_batch_size'
@@     inference requests. Requests are chosen in an oldest-first
@@     manner across all candidate sequences. A given sequence is
@@     not guaranteed to be assigned to the same batch slot for
@@     all inference requests of that sequence.
@@
 

Protobuf type inference.ModelSequenceBatching.StrategyOldest

See Also:

• Serialized Form

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
static final class	ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest.Builder	@@ ..

Nested classes/interfaces inherited from class com.google.protobuf.GeneratedMessageV3

com.google.protobuf.GeneratedMessageV3.BuilderParent, com.google.protobuf.GeneratedMessageV3.ExtendableBuilder<MessageT extends com.google.protobuf.GeneratedMessageV3.ExtendableMessage<MessageT>,BuilderT extends com.google.protobuf.GeneratedMessageV3.ExtendableBuilder<MessageT,BuilderT>>, com.google.protobuf.GeneratedMessageV3.ExtendableMessage<MessageT extends com.google.protobuf.GeneratedMessageV3.ExtendableMessage<MessageT>>, com.google.protobuf.GeneratedMessageV3.ExtendableMessageOrBuilder<MessageT extends com.google.protobuf.GeneratedMessageV3.ExtendableMessage<MessageT>>, com.google.protobuf.GeneratedMessageV3.FieldAccessorTable, com.google.protobuf.GeneratedMessageV3.UnusedPrivateParameter

Nested classes/interfaces inherited from class com.google.protobuf.AbstractMessageLite

com.google.protobuf.AbstractMessageLite.InternalOneOfEnum

Field Summary

Fields

Modifier and Type	Field	Description
static final int	MAX_CANDIDATE_SEQUENCES_FIELD_NUMBER
static final int	MAX_QUEUE_DELAY_MICROSECONDS_FIELD_NUMBER
static final int	PREFERRED_BATCH_SIZE_FIELD_NUMBER
static final int	PRESERVE_ORDERING_FIELD_NUMBER

Fields inherited from class com.google.protobuf.GeneratedMessageV3

alwaysUseFieldBuilders, unknownFields

Fields inherited from class com.google.protobuf.AbstractMessage

memoizedSize

Fields inherited from class com.google.protobuf.AbstractMessageLite

memoizedHashCode

Method Summary

Modifier and Type	Method	Description
boolean	equals(Object obj)
static ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest	getDefaultInstance()
ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest	getDefaultInstanceForType()
static final com.google.protobuf.Descriptors.Descriptor	getDescriptor()
int	getMaxCandidateSequences()	@@ ..
long	getMaxQueueDelayMicroseconds()	@@ ..
com.google.protobuf.Parser<ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest>	getParserForType()
int	getPreferredBatchSize(int index)	@@ ..
int	getPreferredBatchSizeCount()	@@ ..
List<Integer>	getPreferredBatchSizeList()	@@ ..
boolean	getPreserveOrdering()	@@ ..
int	getSerializedSize()
int	hashCode()
protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable	internalGetFieldAccessorTable()
final boolean	isInitialized()
static ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest.Builder	newBuilder()
static ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest.Builder	newBuilder(ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest prototype)
ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest.Builder	newBuilderForType()
protected ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest.Builder	newBuilderForType(com.google.protobuf.GeneratedMessageV3.BuilderParent parent)
protected Object	newInstance(com.google.protobuf.GeneratedMessageV3.UnusedPrivateParameter unused)
static ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest	parseDelimitedFrom(InputStream input)
static ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest	parseDelimitedFrom(InputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
static ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest	parseFrom(byte[] data)
static ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest	parseFrom(byte[] data, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
static ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest	parseFrom(com.google.protobuf.ByteString data)
static ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest	parseFrom(com.google.protobuf.ByteString data, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
static ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest	parseFrom(com.google.protobuf.CodedInputStream input)
static ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest	parseFrom(com.google.protobuf.CodedInputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
static ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest	parseFrom(InputStream input)
static ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest	parseFrom(InputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
static ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest	parseFrom(ByteBuffer data)
static ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest	parseFrom(ByteBuffer data, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
static com.google.protobuf.Parser<ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest>	parser()
ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest.Builder	toBuilder()
void	writeTo(com.google.protobuf.CodedOutputStream output)

Methods inherited from class com.google.protobuf.GeneratedMessageV3

canUseUnsafe, computeStringSize, computeStringSizeNoTag, emptyBooleanList, emptyDoubleList, emptyFloatList, emptyIntList, emptyList, emptyLongList, getAllFields, getDescriptorForType, getField, getOneofFieldDescriptor, getRepeatedField, getRepeatedFieldCount, getUnknownFields, hasField, hasOneof, internalGetMapField, internalGetMapFieldReflection, isStringEmpty, makeExtensionsImmutable, makeMutableCopy, makeMutableCopy, mergeFromAndMakeImmutableInternal, mutableCopy, mutableCopy, mutableCopy, mutableCopy, mutableCopy, newBooleanList, newBuilderForType, newDoubleList, newFloatList, newIntList, newLongList, parseDelimitedWithIOException, parseDelimitedWithIOException, parseUnknownField, parseUnknownFieldProto3, parseWithIOException, parseWithIOException, parseWithIOException, parseWithIOException, serializeBooleanMapTo, serializeIntegerMapTo, serializeLongMapTo, serializeStringMapTo, writeReplace, writeString, writeStringNoTag

Methods inherited from class com.google.protobuf.AbstractMessage

findInitializationErrors, getInitializationErrorString, hashBoolean, hashEnum, hashEnumList, hashFields, hashLong, toString

Methods inherited from class com.google.protobuf.AbstractMessageLite

addAll, addAll, checkByteStringIsUtf8, toByteArray, toByteString, writeDelimitedTo, writeTo

Methods inherited from class java.lang.Object

clone, finalize, getClass, notify, notifyAll, wait, wait, wait

Methods inherited from interface com.google.protobuf.MessageLite

toByteArray, toByteString, writeDelimitedTo, writeTo

Methods inherited from interface com.google.protobuf.MessageOrBuilder

findInitializationErrors, getAllFields, getDescriptorForType, getField, getInitializationErrorString, getOneofFieldDescriptor, getRepeatedField, getRepeatedFieldCount, getUnknownFields, hasField, hasOneof

Field Details

MAX_CANDIDATE_SEQUENCES_FIELD_NUMBER

public static final int MAX_CANDIDATE_SEQUENCES_FIELD_NUMBER

See Also:

• Constant Field Values

PREFERRED_BATCH_SIZE_FIELD_NUMBER

public static final int PREFERRED_BATCH_SIZE_FIELD_NUMBER

See Also:

• Constant Field Values

MAX_QUEUE_DELAY_MICROSECONDS_FIELD_NUMBER

public static final int MAX_QUEUE_DELAY_MICROSECONDS_FIELD_NUMBER

See Also:

• Constant Field Values

PRESERVE_ORDERING_FIELD_NUMBER

public static final int PRESERVE_ORDERING_FIELD_NUMBER

See Also:

• Constant Field Values

Method Details

newInstance

protected Object newInstance(com.google.protobuf.GeneratedMessageV3.UnusedPrivateParameter unused)

Overrides:

newInstance in class com.google.protobuf.GeneratedMessageV3

getDescriptor

public static final com.google.protobuf.Descriptors.Descriptor getDescriptor()

internalGetFieldAccessorTable

protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable()

Specified by:

internalGetFieldAccessorTable in class com.google.protobuf.GeneratedMessageV3

getMaxCandidateSequences

public int getMaxCandidateSequences()

@@    .. cpp:var:: int32 max_candidate_sequences
@@
@@       Maximum number of candidate sequences that the batcher
@@       maintains. Excess sequences are kept in an ordered backlog
@@       and become candidates when existing candidate sequences
@@       complete.
@@
 

int32 max_candidate_sequences = 1;

Specified by:

getMaxCandidateSequences in interface ModelConfigOuterClass.ModelSequenceBatching.StrategyOldestOrBuilder

Returns:

The maxCandidateSequences.

getPreferredBatchSizeList

public List<Integer> getPreferredBatchSizeList()

@@    .. cpp:var:: int32 preferred_batch_size (repeated)
@@
@@       Preferred batch sizes for dynamic batching of candidate
@@       sequences. If a batch of one of these sizes can be formed
@@       it will be executed immediately. If not specified a
@@       preferred batch size will be chosen automatically
@@       based on model and GPU characteristics.
@@
 

repeated int32 preferred_batch_size = 2;

Specified by:

getPreferredBatchSizeList in interface ModelConfigOuterClass.ModelSequenceBatching.StrategyOldestOrBuilder

Returns:

A list containing the preferredBatchSize.

getPreferredBatchSizeCount

public int getPreferredBatchSizeCount()

@@    .. cpp:var:: int32 preferred_batch_size (repeated)
@@
@@       Preferred batch sizes for dynamic batching of candidate
@@       sequences. If a batch of one of these sizes can be formed
@@       it will be executed immediately. If not specified a
@@       preferred batch size will be chosen automatically
@@       based on model and GPU characteristics.
@@
 

repeated int32 preferred_batch_size = 2;

Specified by:

getPreferredBatchSizeCount in interface ModelConfigOuterClass.ModelSequenceBatching.StrategyOldestOrBuilder

Returns:

The count of preferredBatchSize.

getPreferredBatchSize

public int getPreferredBatchSize(int index)

@@    .. cpp:var:: int32 preferred_batch_size (repeated)
@@
@@       Preferred batch sizes for dynamic batching of candidate
@@       sequences. If a batch of one of these sizes can be formed
@@       it will be executed immediately. If not specified a
@@       preferred batch size will be chosen automatically
@@       based on model and GPU characteristics.
@@
 

repeated int32 preferred_batch_size = 2;

Specified by:

getPreferredBatchSize in interface ModelConfigOuterClass.ModelSequenceBatching.StrategyOldestOrBuilder

Parameters:

index - The index of the element to return.

Returns:

The preferredBatchSize at the given index.

getMaxQueueDelayMicroseconds

public long getMaxQueueDelayMicroseconds()

@@    .. cpp:var:: uint64 max_queue_delay_microseconds
@@
@@       The maximum time, in microseconds, a candidate request
@@       will be delayed in the dynamic batch scheduling queue to
@@       wait for additional requests for batching. Default is 0.
@@
 

uint64 max_queue_delay_microseconds = 3;

Specified by:

getMaxQueueDelayMicroseconds in interface ModelConfigOuterClass.ModelSequenceBatching.StrategyOldestOrBuilder

Returns:

The maxQueueDelayMicroseconds.

getPreserveOrdering

public boolean getPreserveOrdering()

@@    .. cpp:var:: bool preserve_ordering
@@
@@       Should the dynamic batcher preserve the ordering of responses to
@@       match the order of requests received by the scheduler. Default is
@@       false. If true, the responses will be returned in the same order
@@       as the order of requests sent to the scheduler. If false, the
@@       responses may be returned in arbitrary order. This option is
@@       specifically needed when a sequence of related inference requests
@@       (i.e. inference requests with the same correlation ID) are sent
@@       to the dynamic batcher to ensure that the sequence responses are
@@       in the correct order.
@@
@@       When using decoupled models, setting this to true may block the
@@       responses from independent sequences from being returned to the
@@       client until the previous request completes, hurting overall
@@       performance. If using GRPC streaming protocol, the stream
@@       ordering guarantee may be sufficient alone to ensure the
@@       responses for each sequence are returned in sequence-order
@@       without blocking based on independent requests, depending on the
@@       use case.
@@
 

bool preserve_ordering = 4;

Specified by:

getPreserveOrdering in interface ModelConfigOuterClass.ModelSequenceBatching.StrategyOldestOrBuilder

Returns:

The preserveOrdering.

isInitialized

public final boolean isInitialized()

Specified by:

isInitialized in interface com.google.protobuf.MessageLiteOrBuilder

Overrides:

isInitialized in class com.google.protobuf.GeneratedMessageV3

writeTo

public void writeTo(com.google.protobuf.CodedOutputStream output)
             throws IOException

Specified by:

writeTo in interface com.google.protobuf.MessageLite

Overrides:

writeTo in class com.google.protobuf.GeneratedMessageV3

Throws:

IOException

getSerializedSize

public int getSerializedSize()

Specified by:

getSerializedSize in interface com.google.protobuf.MessageLite

Overrides:

getSerializedSize in class com.google.protobuf.GeneratedMessageV3

equals

public boolean equals(Object obj)

Specified by:

equals in interface com.google.protobuf.Message

Overrides:

equals in class com.google.protobuf.AbstractMessage

hashCode

public int hashCode()

Specified by:

hashCode in interface com.google.protobuf.Message

Overrides:

hashCode in class com.google.protobuf.AbstractMessage

parseFrom

public static ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest parseFrom(ByteBuffer data)
                                                                            throws com.google.protobuf.InvalidProtocolBufferException

Throws:

com.google.protobuf.InvalidProtocolBufferException

parseFrom

public static ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest parseFrom(ByteBuffer data,
 com.google.protobuf.ExtensionRegistryLite extensionRegistry)
                                                                            throws com.google.protobuf.InvalidProtocolBufferException

Throws:

com.google.protobuf.InvalidProtocolBufferException

parseFrom

public static ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest parseFrom(com.google.protobuf.ByteString data)
                                                                            throws com.google.protobuf.InvalidProtocolBufferException

Throws:

com.google.protobuf.InvalidProtocolBufferException

parseFrom

public static ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest parseFrom(com.google.protobuf.ByteString data,
 com.google.protobuf.ExtensionRegistryLite extensionRegistry)
                                                                            throws com.google.protobuf.InvalidProtocolBufferException

Throws:

com.google.protobuf.InvalidProtocolBufferException

parseFrom

public static ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest parseFrom(byte[] data)
                                                                            throws com.google.protobuf.InvalidProtocolBufferException

Throws:

com.google.protobuf.InvalidProtocolBufferException

parseFrom

public static ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest parseFrom(byte[] data,
 com.google.protobuf.ExtensionRegistryLite extensionRegistry)
                                                                            throws com.google.protobuf.InvalidProtocolBufferException

Throws:

com.google.protobuf.InvalidProtocolBufferException

parseFrom

public static ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest parseFrom(InputStream input)
                                                                            throws IOException

Throws:

IOException

parseFrom

public static ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest parseFrom(InputStream input,
 com.google.protobuf.ExtensionRegistryLite extensionRegistry)
                                                                            throws IOException

Throws:

IOException

parseDelimitedFrom

public static ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest parseDelimitedFrom(InputStream input)
                                                                                     throws IOException

Throws:

IOException

parseDelimitedFrom

public static ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest parseDelimitedFrom(InputStream input,
 com.google.protobuf.ExtensionRegistryLite extensionRegistry)
                                                                                     throws IOException

Throws:

IOException

parseFrom

public static ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest parseFrom(com.google.protobuf.CodedInputStream input)
                                                                            throws IOException

Throws:

IOException

parseFrom

public static ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest parseFrom(com.google.protobuf.CodedInputStream input,
 com.google.protobuf.ExtensionRegistryLite extensionRegistry)
                                                                            throws IOException

Throws:

IOException

newBuilderForType

public ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest.Builder newBuilderForType()

Specified by:

newBuilderForType in interface com.google.protobuf.Message

Specified by:

newBuilderForType in interface com.google.protobuf.MessageLite

newBuilder

public static ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest.Builder newBuilder()

newBuilder

public static ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest.Builder newBuilder(ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest prototype)

toBuilder

public ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest.Builder toBuilder()

Specified by:

toBuilder in interface com.google.protobuf.Message

Specified by:

toBuilder in interface com.google.protobuf.MessageLite

newBuilderForType

protected ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest.Builder newBuilderForType(com.google.protobuf.GeneratedMessageV3.BuilderParent parent)

Specified by:

newBuilderForType in class com.google.protobuf.GeneratedMessageV3

getDefaultInstance

public static ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest getDefaultInstance()

parser

public static com.google.protobuf.Parser<ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest> parser()

getParserForType

public com.google.protobuf.Parser<ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest> getParserForType()

Specified by:

getParserForType in interface com.google.protobuf.Message

Specified by:

getParserForType in interface com.google.protobuf.MessageLite

Overrides:

getParserForType in class com.google.protobuf.GeneratedMessageV3

getDefaultInstanceForType

public ModelConfigOuterClass.ModelSequenceBatching.StrategyOldest getDefaultInstanceForType()

Specified by:

getDefaultInstanceForType in interface com.google.protobuf.MessageLiteOrBuilder

Specified by:

getDefaultInstanceForType in interface com.google.protobuf.MessageOrBuilder