Module org.apache.lucene.core

Package org.apache.lucene.index

Class IndexingChain

java.lang.Object

org.apache.lucene.index.IndexingChain

All Implemented Interfaces:

Accountable

final class IndexingChain
extends Object
implements Accountable

Default general purpose indexing chain, which handles indexing all types of fields.

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
private static final class	IndexingChain.FieldSchema	A schema of the field in the current document.
private static class	IndexingChain.IntBlockAllocator
private final class	IndexingChain.PerField	NOTE: not static: accesses at least docState, termsHash.
(package private) static final class	IndexingChain.ReservedField<T extends IndexableField>

Field Summary

Fields

Modifier and Type	Field	Description
private final Consumer<Throwable>	abortingExceptionConsumer
private final ByteBlockPool.Allocator	byteBlockAllocator
(package private) final Counter	bytesUsed
private IndexingChain.PerField[]	docFields
(package private) final ByteBlockPool	docValuesBytePool
private IndexingChain.PerField[]	fieldHash
(package private) final FieldInfos.Builder	fieldInfos
private IndexingChain.PerField[]	fields
private boolean	hasHitAbortingException
private int	hashMask
private final int	indexCreatedVersionMajor
private final LiveIndexWriterConfig	indexWriterConfig
private final InfoStream	infoStream
private long	nextFieldGen
(package private) final StoredFieldsConsumer	storedFieldsConsumer
(package private) final TermsHash	termsHash
(package private) final TermVectorsConsumer	termVectorsWriter
private int	totalFieldCount
(package private) final VectorValuesConsumer	vectorValuesConsumer

Fields inherited from interface org.apache.lucene.util.Accountable

NULL_ACCOUNTABLE

Constructor Summary

Constructors

Constructor	Description
IndexingChain(int indexCreatedVersionMajor, SegmentInfo segmentInfo, Directory directory, FieldInfos.Builder fieldInfos, LiveIndexWriterConfig indexWriterConfig, Consumer<Throwable> abortingExceptionConsumer)

Method Summary

Modifier and Type	Method	Description
(package private) void	abort()
private void	finishStoredFields()	Calls StoredFieldsWriter.finishDocument, aborting the segment if it hits any exception.
(package private) Sorter.DocMap	flush(SegmentWriteState state)
Collection<Accountable>	getChildResources()	Returns nested resources of this class.
private LeafReader	getDocValuesLeafReader()
(package private) DocIdSetIterator	getHasDocValues(String field)
private IndexingChain.PerField	getOrAddPerField(String fieldName, boolean reserved)	Returns a previously created IndexingChain.PerField, absorbing the type information from FieldType, and creates a new IndexingChain.PerField if this field name wasn't seen yet.
private IndexingChain.PerField	getPerField(String name)	Returns a previously created IndexingChain.PerField, or null if this field name wasn't seen yet.
private void	indexDocValue(int docID, IndexingChain.PerField fp, DocValuesType dvType, IndexableField field)	Called from processDocument to index one field's doc value
private void	indexVectorValue(int docID, IndexingChain.PerField pf, VectorEncoding vectorEncoding, IndexableField field)
private void	initializeFieldInfo(IndexingChain.PerField pf)
(package private) <T extends IndexableField> IndexingChain.ReservedField<T>	markAsReserved(T field)	Wraps the given field in a reserved field and registers it as reserved.
private Sorter.DocMap	maybeSortSegment(SegmentWriteState state)
private void	onAbortingException(Throwable th)
private void	oversizeDocFields()
(package private) void	processDocument(int docID, Iterable<? extends IndexableField> document)
private boolean	processField(int docID, IndexableField field, IndexingChain.PerField pf)	Index each field Returns true, if we are indexing a unique field with postings
long	ramBytesUsed()	Return the memory usage of this object in bytes.
private void	rehash()
private void	startStoredFields(int docID)	Calls StoredFieldsWriter.startDocument, aborting the segment if it hits any exception.
private static void	updateDocFieldSchema(String fieldName, IndexingChain.FieldSchema schema, IndexableFieldType fieldType)
private void	validateIndexSortDVType(Sort indexSort, String fieldToValidate, DocValuesType dvType)
private static void	validateMaxVectorDimension(String fieldName, int vectorDim, int maxVectorDim)
private static void	verifyUnIndexedFieldType(String name, IndexableFieldType ft)
private void	writeDocValues(SegmentWriteState state, Sorter.DocMap sortMap)	Writes all buffered doc values (called from flush(org.apache.lucene.index.SegmentWriteState)).
private void	writeNorms(SegmentWriteState state, Sorter.DocMap sortMap)
private void	writePoints(SegmentWriteState state, Sorter.DocMap sortMap)	Writes all buffered points.

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Details

bytesUsed

final Counter bytesUsed

fieldInfos

final FieldInfos.Builder fieldInfos

termsHash

final TermsHash termsHash

docValuesBytePool

final ByteBlockPool docValuesBytePool

storedFieldsConsumer

final StoredFieldsConsumer storedFieldsConsumer

vectorValuesConsumer

final VectorValuesConsumer vectorValuesConsumer

termVectorsWriter

final TermVectorsConsumer termVectorsWriter

fieldHash

private IndexingChain.PerField[] fieldHash

hashMask

private int hashMask

totalFieldCount

private int totalFieldCount

nextFieldGen

private long nextFieldGen

fields

private IndexingChain.PerField[] fields

docFields

private IndexingChain.PerField[] docFields

infoStream

private final InfoStream infoStream

byteBlockAllocator

private final ByteBlockPool.Allocator byteBlockAllocator

indexWriterConfig

private final LiveIndexWriterConfig indexWriterConfig

indexCreatedVersionMajor

private final int indexCreatedVersionMajor

abortingExceptionConsumer

private final Consumer<Throwable> abortingExceptionConsumer

hasHitAbortingException

private boolean hasHitAbortingException

Constructor Details

IndexingChain

IndexingChain(int indexCreatedVersionMajor,
 SegmentInfo segmentInfo,
 Directory directory,
 FieldInfos.Builder fieldInfos,
 LiveIndexWriterConfig indexWriterConfig,
 Consumer<Throwable> abortingExceptionConsumer)

Method Details

onAbortingException

private void onAbortingException(Throwable th)

getDocValuesLeafReader

private LeafReader getDocValuesLeafReader()

maybeSortSegment

private Sorter.DocMap maybeSortSegment(SegmentWriteState state)
                                throws IOException

Throws:

IOException

flush

Sorter.DocMap flush(SegmentWriteState state)
             throws IOException

Throws:

IOException

writePoints

private void writePoints(SegmentWriteState state,
 Sorter.DocMap sortMap)
                  throws IOException

Writes all buffered points.

Throws:

IOException

writeDocValues

private void writeDocValues(SegmentWriteState state,
 Sorter.DocMap sortMap)
                     throws IOException

Writes all buffered doc values (called from flush(org.apache.lucene.index.SegmentWriteState)).

Throws:

IOException

writeNorms

private void writeNorms(SegmentWriteState state,
 Sorter.DocMap sortMap)
                 throws IOException

Throws:

IOException

abort

void abort()
    throws IOException

Throws:

IOException

rehash

private void rehash()

startStoredFields

private void startStoredFields(int docID)
                        throws IOException

Calls StoredFieldsWriter.startDocument, aborting the segment if it hits any exception.

Throws:

IOException

finishStoredFields

private void finishStoredFields()
                         throws IOException

Calls StoredFieldsWriter.finishDocument, aborting the segment if it hits any exception.

Throws:

IOException

processDocument

void processDocument(int docID,
 Iterable<? extends IndexableField> document)
              throws IOException

Throws:

IOException

oversizeDocFields

private void oversizeDocFields()

initializeFieldInfo

private void initializeFieldInfo(IndexingChain.PerField pf)
                          throws IOException

Throws:

IOException

processField

private boolean processField(int docID,
 IndexableField field,
 IndexingChain.PerField pf)
                      throws IOException

Index each field Returns true, if we are indexing a unique field with postings

Throws:

IOException

getOrAddPerField

private IndexingChain.PerField getOrAddPerField(String fieldName,
 boolean reserved)

Returns a previously created IndexingChain.PerField, absorbing the type information from FieldType, and creates a new IndexingChain.PerField if this field name wasn't seen yet.

updateDocFieldSchema

private static void updateDocFieldSchema(String fieldName,
 IndexingChain.FieldSchema schema,
 IndexableFieldType fieldType)

verifyUnIndexedFieldType

private static void verifyUnIndexedFieldType(String name,
 IndexableFieldType ft)

validateMaxVectorDimension

private static void validateMaxVectorDimension(String fieldName,
 int vectorDim,
 int maxVectorDim)

validateIndexSortDVType

private void validateIndexSortDVType(Sort indexSort,
 String fieldToValidate,
 DocValuesType dvType)
                              throws IOException

Throws:

IOException

indexDocValue

private void indexDocValue(int docID,
 IndexingChain.PerField fp,
 DocValuesType dvType,
 IndexableField field)

Called from processDocument to index one field's doc value

indexVectorValue

private void indexVectorValue(int docID,
 IndexingChain.PerField pf,
 VectorEncoding vectorEncoding,
 IndexableField field)
                       throws IOException

Throws:

IOException

getPerField

private IndexingChain.PerField getPerField(String name)

Returns a previously created IndexingChain.PerField, or null if this field name wasn't seen yet.

ramBytesUsed

public long ramBytesUsed()

Description copied from interface: Accountable

Return the memory usage of this object in bytes. Negative values are illegal.

Specified by:

ramBytesUsed in interface Accountable

getChildResources

public Collection<Accountable> getChildResources()

Description copied from interface: Accountable

Returns nested resources of this class. The result should be a point-in-time snapshot (to avoid race conditions).

Specified by:

getChildResources in interface Accountable

See Also:

• Accountables

getHasDocValues

DocIdSetIterator getHasDocValues(String field)

markAsReserved

<T extends IndexableField>
IndexingChain.ReservedField<T> markAsReserved(T field)

Wraps the given field in a reserved field and registers it as reserved. Only DWPT should do this to mark fields as private / reserved to prevent this fieldname to be used from the outside of the IW / DWPT eco-system