store

package
v0.0.0-...-309f818 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: May 2, 2021 License: Apache-2.0 Imports: 19 Imported by: 0

Documentation

Index

Constants

View Source
const (
	CFD_DATA_CODEC       = "CompoundFileWriterData"
	CFD_VERSION_START    = 0
	CFD_VERSION_CHECKSUM = 1
	CFD_VERSION_CURRENT  = CFD_VERSION_CHECKSUM

	CFD_ENTRY_CODEC = "CompoundFileWriterEntries"

	COMPOUND_FILE_EXTENSION         = "cfs"
	COMPOUND_FILE_ENTRIES_EXTENSION = "cfe"
)
View Source
const (
	CODEC_MAGIC_BYTE1 = byte(uint32(codec.CODEC_MAGIC) >> 24 & 0xFF)
	CODEC_MAGIC_BYTE2 = byte(uint32(codec.CODEC_MAGIC) >> 16 & 0xFF)
	CODEC_MAGIC_BYTE3 = byte(uint32(codec.CODEC_MAGIC) >> 8 & 0xFF)
	CODEC_MAGIC_BYTE4 = byte(codec.CODEC_MAGIC & 0xFF)
)
View Source
const (
	IO_CONTEXT_TYPE_MERGE   = 1
	IO_CONTEXT_TYPE_READ    = 2
	IO_CONTEXT_TYPE_FLUSH   = 3
	IO_CONTEXT_TYPE_DEFAULT = 4
)
View Source
const (
	BUFFER_SIZE       = 1024
	MERGE_BUFFER_SIZE = 4096
)
View Source
const CHUNK_SIZE = 8192

The maximum chunk size is 8192 bytes, becaues Java RamdomAccessFile mallocs a native buffer outside of stack if the read buffer size is larger. GoLucene takes the same default value. TODO: test larger value here

View Source
const DEFAULT_BUFFER_SIZE = 16384

The default buffer size in bytes.

View Source
const LOCK_OBTAIN_WAIT_FOREVER = -1

Pass this value to obtain() to try forever to obtain the lock

View Source
const LOCK_POOL_INTERVAL = 1000

How long obtain() waits, in milliseconds, in between attempts to acquire the lock.

View Source
const MIN_BUFFER_SIZE = 8

Minimum buffer size allowed

View Source
const NRT_VERBOSE = false

Variables

View Source
var (
	IO_CONTEXT_DEFAULT  = NewIOContextFromType(IOContextType(IO_CONTEXT_TYPE_DEFAULT))
	IO_CONTEXT_READONCE = NewIOContextBool(true)
	IO_CONTEXT_READ     = NewIOContextBool(false)
)
View Source
var SENTINEL = make(map[string]FileSlice)

Functions

func ChecksumEntireFile

func ChecksumEntireFile(input IndexInput) (hash int64, err error)

Clones the provided input, reads all bytes from the file, and calls CheckFooter().

Note that this method may be slow, as it must process the entire file. If you just need to extract the checksum value, call retrieveChecksum().

func FSDirectoryListAll

func FSDirectoryListAll(path string) (paths []string, err error)

func WithLock

func WithLock(lock Lock, lockWaitTimeout int64, body func() interface{}) interface{}

Utility to execute code with exclusive access.

Types

type AtomicBool

type AtomicBool struct {
	*sync.RWMutex
	// contains filtered or unexported fields
}

func NewAtomicBool

func NewAtomicBool() *AtomicBool

func (*AtomicBool) CompareAndSet

func (b *AtomicBool) CompareAndSet(from, to bool) bool

func (*AtomicBool) Get

func (b *AtomicBool) Get() bool

type BaseDirectory

type BaseDirectory struct {
	IsOpen bool
	// contains filtered or unexported fields
}

Base implementation for a concrete Directory.

func NewBaseDirectory

func NewBaseDirectory(spi BaseDirectorySPI) *BaseDirectory

func (*BaseDirectory) ClearLock

func (d *BaseDirectory) ClearLock(name string) error

func (*BaseDirectory) EnsureOpen

func (d *BaseDirectory) EnsureOpen()

func (*BaseDirectory) LockFactory

func (d *BaseDirectory) LockFactory() LockFactory

func (*BaseDirectory) MakeLock

func (d *BaseDirectory) MakeLock(name string) Lock

func (*BaseDirectory) SetLockFactory

func (d *BaseDirectory) SetLockFactory(lockFactory LockFactory)

type BaseDirectorySPI

type BaseDirectorySPI interface {
	// DirectoryImplSPI
	LockID() string
}

type BufferedChecksum

type BufferedChecksum struct {
	// contains filtered or unexported fields
}

Wraps another Checksum with an internal buffer to speed up checksum calculations.

func (*BufferedChecksum) BlockSize

func (bc *BufferedChecksum) BlockSize() int

func (*BufferedChecksum) Reset

func (bc *BufferedChecksum) Reset()

func (*BufferedChecksum) Size

func (bc *BufferedChecksum) Size() int

func (*BufferedChecksum) Sum

func (bc *BufferedChecksum) Sum(p []byte) []byte

func (*BufferedChecksum) Sum32

func (bc *BufferedChecksum) Sum32() uint32

func (*BufferedChecksum) Write

func (bc *BufferedChecksum) Write(p []byte) (int, error)

type BufferedChecksumIndexInput

type BufferedChecksumIndexInput struct {
	*ChecksumIndexInputImpl
	// contains filtered or unexported fields
}

Simple implementation of ChecksumIndexInput that wraps another input and delegates calls.

func (*BufferedChecksumIndexInput) Checksum

func (in *BufferedChecksumIndexInput) Checksum() int64

func (*BufferedChecksumIndexInput) Clone

func (*BufferedChecksumIndexInput) Close

func (in *BufferedChecksumIndexInput) Close() error

func (*BufferedChecksumIndexInput) FilePointer

func (in *BufferedChecksumIndexInput) FilePointer() int64

func (*BufferedChecksumIndexInput) Length

func (in *BufferedChecksumIndexInput) Length() int64

func (*BufferedChecksumIndexInput) ReadByte

func (in *BufferedChecksumIndexInput) ReadByte() (b byte, err error)

func (*BufferedChecksumIndexInput) ReadBytes

func (in *BufferedChecksumIndexInput) ReadBytes(p []byte) (err error)

func (*BufferedChecksumIndexInput) Slice

func (in *BufferedChecksumIndexInput) Slice(desc string, offset, length int64) (IndexInput, error)

type BufferedIndexInput

type BufferedIndexInput struct {
	*IndexInputImpl
	// contains filtered or unexported fields
}

Base implementation class for buffered IndexInput.

func (*BufferedIndexInput) Clone

func (*BufferedIndexInput) FilePointer

func (in *BufferedIndexInput) FilePointer() int64

func (*BufferedIndexInput) ReadByte

func (in *BufferedIndexInput) ReadByte() (b byte, err error)

func (*BufferedIndexInput) ReadBytes

func (in *BufferedIndexInput) ReadBytes(buf []byte) error

func (*BufferedIndexInput) ReadBytesBuffered

func (in *BufferedIndexInput) ReadBytesBuffered(buf []byte, useBuffer bool) error

func (*BufferedIndexInput) ReadInt

func (in *BufferedIndexInput) ReadInt() (n int32, err error)

func (*BufferedIndexInput) ReadLong

func (in *BufferedIndexInput) ReadLong() (n int64, err error)

func (*BufferedIndexInput) ReadShort

func (in *BufferedIndexInput) ReadShort() (n int16, err error)

func (*BufferedIndexInput) ReadVInt

func (in *BufferedIndexInput) ReadVInt() (n int32, err error)

func (*BufferedIndexInput) ReadVLong

func (in *BufferedIndexInput) ReadVLong() (n int64, err error)

func (*BufferedIndexInput) Seek

func (in *BufferedIndexInput) Seek(pos int64) error

func (*BufferedIndexInput) Slice

func (in *BufferedIndexInput) Slice(desc string, offset, length int64) (IndexInput, error)

type ByteArrayDataInput

type ByteArrayDataInput struct {
	*util.DataInputImpl

	Pos int
	// contains filtered or unexported fields
}

DataInput backed by a byte array. Warning: this class omits all low-level checks.

func NewByteArrayDataInput

func NewByteArrayDataInput(bytes []byte) *ByteArrayDataInput

func NewEmptyByteArrayDataInput

func NewEmptyByteArrayDataInput() *ByteArrayDataInput

func (*ByteArrayDataInput) Length

func (in *ByteArrayDataInput) Length() int

func (*ByteArrayDataInput) Position

func (in *ByteArrayDataInput) Position() int

func (*ByteArrayDataInput) ReadByte

func (in *ByteArrayDataInput) ReadByte() (b byte, err error)

func (*ByteArrayDataInput) ReadBytes

func (in *ByteArrayDataInput) ReadBytes(buf []byte) error

func (*ByteArrayDataInput) ReadInt

func (in *ByteArrayDataInput) ReadInt() (n int32, err error)

func (*ByteArrayDataInput) ReadLong

func (in *ByteArrayDataInput) ReadLong() (n int64, err error)

func (*ByteArrayDataInput) ReadShort

func (in *ByteArrayDataInput) ReadShort() (n int16, err error)

func (*ByteArrayDataInput) ReadVInt

func (in *ByteArrayDataInput) ReadVInt() (n int32, err error)

func (*ByteArrayDataInput) ReadVLong

func (in *ByteArrayDataInput) ReadVLong() (n int64, err error)

func (*ByteArrayDataInput) Reset

func (in *ByteArrayDataInput) Reset(bytes []byte)

func (*ByteArrayDataInput) Rewind

func (in *ByteArrayDataInput) Rewind()

NOTE: sets pos to 0, which is not right if you had called reset w/ non-zero offset!!

func (*ByteArrayDataInput) SkipBytes

func (in *ByteArrayDataInput) SkipBytes(count int64)

type ByteArrayDataOutput

type ByteArrayDataOutput struct {
	*util.DataOutputImpl
	// contains filtered or unexported fields
}

func NewByteArrayDataOutput

func NewByteArrayDataOutput(data []byte) *ByteArrayDataOutput

func (*ByteArrayDataOutput) Position

func (o *ByteArrayDataOutput) Position() int

func (*ByteArrayDataOutput) WriteByte

func (o *ByteArrayDataOutput) WriteByte(b byte) error

func (*ByteArrayDataOutput) WriteBytes

func (o *ByteArrayDataOutput) WriteBytes(b []byte) error

type ChecksumIndexInput

type ChecksumIndexInput interface {
	IndexInput
	Checksum() int64
}

Extension of IndexInput, computing checksum as it goes. Callers can retrieve the checksum via Checksum().

type ChecksumIndexInputImpl

type ChecksumIndexInputImpl struct {
	*IndexInputImpl
	// contains filtered or unexported fields
}

func (*ChecksumIndexInputImpl) Seek

func (in *ChecksumIndexInputImpl) Seek(pos int64) error

type ChecksumIndexInputImplSPI

type ChecksumIndexInputImplSPI interface {
	util.DataReader
	FilePointer() int64
}

type CompoundFileDirectory

type CompoundFileDirectory struct {
	*DirectoryImpl
	*BaseDirectory
	sync.Locker
	// contains filtered or unexported fields
}

func NewCompoundFileDirectory

func NewCompoundFileDirectory(directory Directory, fileName string, context IOContext, openForWrite bool) (d *CompoundFileDirectory, err error)

func (*CompoundFileDirectory) Close

func (d *CompoundFileDirectory) Close() error

func (*CompoundFileDirectory) CreateOutput

func (d *CompoundFileDirectory) CreateOutput(name string, context IOContext) (out IndexOutput, err error)

func (*CompoundFileDirectory) DeleteFile

func (d *CompoundFileDirectory) DeleteFile(name string) error

func (*CompoundFileDirectory) FileExists

func (d *CompoundFileDirectory) FileExists(name string) bool

func (*CompoundFileDirectory) FileLength

func (d *CompoundFileDirectory) FileLength(name string) (n int64, err error)

Returns the length of a file in the directory.

func (*CompoundFileDirectory) ListAll

func (d *CompoundFileDirectory) ListAll() (paths []string, err error)

func (*CompoundFileDirectory) MakeLock

func (d *CompoundFileDirectory) MakeLock(name string) Lock

func (*CompoundFileDirectory) OpenInput

func (d *CompoundFileDirectory) OpenInput(name string, context IOContext) (in IndexInput, err error)

func (*CompoundFileDirectory) String

func (d *CompoundFileDirectory) String() string

func (*CompoundFileDirectory) Sync

func (d *CompoundFileDirectory) Sync(names []string) error

type CompoundFileWriter

type CompoundFileWriter struct {
	sync.Locker
	// contains filtered or unexported fields
}

Combines multiple files into a single compound file

func (*CompoundFileWriter) Close

func (w *CompoundFileWriter) Close() (err error)

Closes all resouces and writes the entry table

type DirectCFSIndexOutput

type DirectCFSIndexOutput struct {
	*IndexOutputImpl
	// contains filtered or unexported fields
}

func (*DirectCFSIndexOutput) Checksum

func (out *DirectCFSIndexOutput) Checksum() int64

func (*DirectCFSIndexOutput) Close

func (out *DirectCFSIndexOutput) Close() error

func (*DirectCFSIndexOutput) FilePointer

func (out *DirectCFSIndexOutput) FilePointer() int64

func (*DirectCFSIndexOutput) Flush

func (out *DirectCFSIndexOutput) Flush() error

func (*DirectCFSIndexOutput) WriteByte

func (out *DirectCFSIndexOutput) WriteByte(b byte) error

func (*DirectCFSIndexOutput) WriteBytes

func (out *DirectCFSIndexOutput) WriteBytes(b []byte) error

type Directory

type Directory interface {
	io.Closer
	// Files related methods
	ListAll() (paths []string, err error)
	// Returns true iff a file with the given name exists.
	// @deprecated This method will be removed in 5.0
	FileExists(name string) bool
	// Removes an existing file in the directory.
	DeleteFile(name string) error
	// Returns the length of a file in the directory. This method
	// follows the following contract:
	// 	- Must return error if the file doesn't exists.
	// 	- Returns a value >=0 if the file exists, which specifies its
	// length.
	FileLength(name string) (n int64, err error)
	// Creates a new, empty file in the directory with the given name.
	// Returns a stream writing this file.
	CreateOutput(name string, ctx IOContext) (out IndexOutput, err error)
	// Ensure that any writes to these files ar emoved to stable
	// storage. Lucene uses this to properly commit changes to the
	// index, to prevent a machine/OS crash from corrupting the index.
	//
	// NOTE: Clients may call this method for same files over and over
	// again, so some impls might optimize for that. For other impls
	// the operation can be a noop, for various reasons.
	Sync(names []string) error
	OpenInput(name string, context IOContext) (in IndexInput, err error)
	// Returns a stream reading an existing file, computing checksum as it reads
	OpenChecksumInput(name string, ctx IOContext) (ChecksumIndexInput, error)
	// Locks related methods
	MakeLock(name string) Lock
	ClearLock(name string) error
	SetLockFactory(lockFactory LockFactory)
	LockFactory() LockFactory
	LockID() string
	// Utilities
	Copy(to Directory, src, dest string, ctx IOContext) error

	EnsureOpen()
}

func OpenFSDirectory

func OpenFSDirectory(path string) (d Directory, err error)

type DirectoryImpl

type DirectoryImpl struct {
	// contains filtered or unexported fields
}

func NewDirectoryImpl

func NewDirectoryImpl(spi DirectoryImplSPI) *DirectoryImpl

func (*DirectoryImpl) Copy

func (d *DirectoryImpl) Copy(to Directory, src, dest string, ctx IOContext) (err error)

Copies the file src to 'to' under the new file name dest.

If you want to copy the entire source directory to the destination one, you can do so like this:

var to Directory // the directory to copy to
for _, file := range dir.ListAll() {
	dir.Copy(to, file, newFile, IO_CONTEXT_DEFAULT)
	// newFile can be either file, or a new name
}

NOTE: this method does not check whether dest exists and will overwrite it if it does.

func (*DirectoryImpl) LockID

func (d *DirectoryImpl) LockID() string

Return a string identifier that uniquely differentiates this Directory instance from other Directory instances. This ID should be the same if two Directory instances (even in different JVMs and/or on different machines) are considered "the same index". This is how locking "scopes" to the right index.

func (*DirectoryImpl) OpenChecksumInput

func (d *DirectoryImpl) OpenChecksumInput(name string, ctx IOContext) (ChecksumIndexInput, error)

func (*DirectoryImpl) String

func (d *DirectoryImpl) String() string

type DirectoryImplSPI

type DirectoryImplSPI interface {
	OpenInput(string, IOContext) (IndexInput, error)
	LockFactory() LockFactory
}

type FSDirectory

type FSDirectory struct {
	*DirectoryImpl
	*BaseDirectory
	FSDirectorySPI
	sync.Locker
	// contains filtered or unexported fields
}

func (*FSDirectory) Close

func (d *FSDirectory) Close() error

func (*FSDirectory) CreateOutput

func (d *FSDirectory) CreateOutput(name string, ctx IOContext) (out IndexOutput, err error)

Creates an IndexOutput for the file with the given name.

func (*FSDirectory) DeleteFile

func (d *FSDirectory) DeleteFile(name string) (err error)

Removes an existing file in the directory.

func (*FSDirectory) FileExists

func (d *FSDirectory) FileExists(name string) bool

func (*FSDirectory) FileLength

func (d *FSDirectory) FileLength(name string) (n int64, err error)

Returns the length in bytes of a file in the directory.

func (*FSDirectory) ListAll

func (d *FSDirectory) ListAll() (paths []string, err error)

func (*FSDirectory) LockID

func (d *FSDirectory) LockID() string

func (*FSDirectory) SetLockFactory

func (d *FSDirectory) SetLockFactory(lockFactory LockFactory)

func (*FSDirectory) String

func (d *FSDirectory) String() string

func (*FSDirectory) Sync

func (d *FSDirectory) Sync(names []string) (err error)

type FSDirectorySPI

type FSDirectorySPI interface {
	OpenInput(string, IOContext) (IndexInput, error)
}

type FSIndexOutput

type FSIndexOutput struct {
	*OutputStreamIndexOutput
	*FSDirectory
	// contains filtered or unexported fields
}

Writes output with File.Write([]byte) (int, error)

func (*FSIndexOutput) Close

func (out *FSIndexOutput) Close() (err error)

type FSLockFactory

type FSLockFactory struct {
	*LockFactoryImpl
	// contains filtered or unexported fields
}

func (*FSLockFactory) Clear

func (f *FSLockFactory) Clear(name string) error

func (*FSLockFactory) Make

func (f *FSLockFactory) Make(name string) Lock

func (*FSLockFactory) String

func (f *FSLockFactory) String() string

type FileEntry

type FileEntry struct {
	// contains filtered or unexported fields
}

type FileSlice

type FileSlice struct {
	// contains filtered or unexported fields
}

type FilteredWriteCloser

type FilteredWriteCloser struct {
	io.WriteCloser
	// contains filtered or unexported fields
}

func (*FilteredWriteCloser) Write

func (w *FilteredWriteCloser) Write(p []byte) (int, error)

type FlushInfo

type FlushInfo struct {
	NumDocs              int
	EstimatedSegmentSize int64
}

type IOContext

type IOContext struct {
	MergeInfo *MergeInfo
	FlushInfo *FlushInfo
	// contains filtered or unexported fields
}

IOContext holds additional details on the merge/search context. A IOContext object can never be initialized as nil as passed as a parameter to either OpenInput() or CreateOutput()

func NewIOContextBool

func NewIOContextBool(readOnce bool) IOContext

func NewIOContextForFlush

func NewIOContextForFlush(flushInfo *FlushInfo) IOContext

func NewIOContextForMerge

func NewIOContextForMerge(mergeInfo *MergeInfo) IOContext

func NewIOContextFromType

func NewIOContextFromType(context IOContextType) IOContext

func (IOContext) String

func (ctx IOContext) String() string

type IOContextType

type IOContextType int

type IndexInput

type IndexInput interface {
	util.DataInput
	IndexInputService
	ReadBytesBuffered(buf []byte, useBuffer bool) error
	// Clone
	Clone() IndexInput
	// Creates a slice of this index input, with the given description,
	// offset, and length. The slice is seeked to the beginning.
	Slice(desc string, offset, length int64) (IndexInput, error)
}

type IndexInputImpl

type IndexInputImpl struct {
	*util.DataInputImpl
	// contains filtered or unexported fields
}

func NewIndexInputImpl

func NewIndexInputImpl(desc string, r util.DataReader) *IndexInputImpl

func (*IndexInputImpl) String

func (in *IndexInputImpl) String() string

type IndexInputService

type IndexInputService interface {
	io.Closer
	// Returns the current position in this file, where the next read will occur.
	FilePointer() int64
	// Sets current position in this file, where the next read will occur.
	Seek(pos int64) error
	Length() int64
}

type IndexInputSub

type IndexInputSub interface {
	io.Closer
	util.DataReader
	FilePointer() int64
	Seek(pos int64) error
	Length() int64
}

type IndexOutput

type IndexOutput interface {
	io.Closer
	util.DataOutput
	// Forces any buffered output to be written.
	// Flush() error
	// Returns the current position in this file, where the next write will occur.
	FilePointer() int64
	// Returns the current checksum of bytes written so far
	Checksum() int64
}

type IndexOutputImpl

type IndexOutputImpl struct {
	*util.DataOutputImpl
}

func NewIndexOutput

func NewIndexOutput(part util.DataWriter) *IndexOutputImpl

type IndexOutputStream

type IndexOutputStream struct {
	// contains filtered or unexported fields
}

Wrap IndexOutput to allow coding in flow style without worrying about error check. If error happens in the middle, following calls are just ignored.

func Stream

func Stream(out IndexOutput) *IndexOutputStream

func (*IndexOutputStream) Close

func (ios *IndexOutputStream) Close() error

func (*IndexOutputStream) WriteByte

func (ios *IndexOutputStream) WriteByte(b byte) *IndexOutputStream

func (*IndexOutputStream) WriteBytes

func (ios *IndexOutputStream) WriteBytes(b []byte) *IndexOutputStream

func (*IndexOutputStream) WriteInt

func (ios *IndexOutputStream) WriteInt(i int32) *IndexOutputStream

func (*IndexOutputStream) WriteLong

func (ios *IndexOutputStream) WriteLong(l int64) *IndexOutputStream

func (*IndexOutputStream) WriteString

func (ios *IndexOutputStream) WriteString(s string) *IndexOutputStream

func (*IndexOutputStream) WriteStringSet

func (ios *IndexOutputStream) WriteStringSet(m map[string]bool) *IndexOutputStream

func (*IndexOutputStream) WriteStringStringMap

func (ios *IndexOutputStream) WriteStringStringMap(m map[string]string) *IndexOutputStream

func (*IndexOutputStream) WriteVInt

func (ios *IndexOutputStream) WriteVInt(i int32) *IndexOutputStream

type Lock

type Lock interface {
	// Releases exclusive access.
	io.Closer
	// Attempts to obtain exclusive access and immediately return
	// upon success or failure. Use Close() to release the lock.
	Obtain() (ok bool, err error)
	// Attempts to obtain an exclusive lock within amount of time
	// given. Pools once per LOCK_POLL_INTERVAL (currently 1000)
	// milliseconds until lockWaitTimeout is passed.
	ObtainWithin(lockWaitTimeout int64) (ok bool, err error)
	// Returns true if the resource is currently locked. Note that one
	// must still call obtain() before using the resource.
	IsLocked() bool
}

An interprocess mutex lock.

Typical use might look like:

WithLock(directory.MakeLock("my.lock"), func() interface{} {
	// code to execute while locked
})

type LockFactory

type LockFactory interface {
	Make(name string) Lock
	Clear(name string) error
	SetLockPrefix(prefix string)
	LockPrefix() string
}

type LockFactoryImpl

type LockFactoryImpl struct {
	// contains filtered or unexported fields
}

func (*LockFactoryImpl) LockPrefix

func (f *LockFactoryImpl) LockPrefix() string

func (*LockFactoryImpl) SetLockPrefix

func (f *LockFactoryImpl) SetLockPrefix(prefix string)

type LockImpl

type LockImpl struct {
	// contains filtered or unexported fields
}

func NewLockImpl

func NewLockImpl(self Lock) *LockImpl

func (*LockImpl) ObtainWithin

func (lock *LockImpl) ObtainWithin(lockWaitTimeout int64) (locked bool, err error)

type MergeInfo

type MergeInfo struct {
	TotalDocCount       int
	EstimatedMergeBytes int64
	IsExternal          bool
	MergeMaxNumSegments int
}

type MultiLevelSkipListWriter

type MultiLevelSkipListWriter struct {
	// contains filtered or unexported fields
}

TODO: migrate original comment.

Note: this class was moved from package codec to store since it caused cyclic dependency (store<->codec).

func NewMultiLevelSkipListWriter

func NewMultiLevelSkipListWriter(spi MultiLevelSkipListWriterSPI,
	skipInterval, skipMultiplier, maxSkipLevels, df int) *MultiLevelSkipListWriter

Creates a MultiLevelSkipListWriter.

func (*MultiLevelSkipListWriter) BufferSkip

func (w *MultiLevelSkipListWriter) BufferSkip(df int) error

Writes the current skip data to the buffers. The current document frequency determines the max level is skip ddata is to be written to.

func (*MultiLevelSkipListWriter) ResetSkip

func (w *MultiLevelSkipListWriter) ResetSkip()

Creates new buffers or empties the existing ones

func (*MultiLevelSkipListWriter) WriteSkip

func (w *MultiLevelSkipListWriter) WriteSkip(output IndexOutput) (int64, error)

Writes the buffered skip lists to the given output.

type MultiLevelSkipListWriterSPI

type MultiLevelSkipListWriterSPI interface {
	WriteSkipData(level int, skipBuffer IndexOutput) error
}

type NRTCachingDirectory

type NRTCachingDirectory struct {
	Directory
	sync.Locker
	// contains filtered or unexported fields
}

Wraps a RAMDirectory around any provided delegate directory, to be used during NRT search.

This class is likely only useful in a near-real-time context, where indexing rate is lowish but reopen rate is highish, resulting in many tiny files being written. This directory keeps such segments (as well as the segments produced by merging them, as long as they are small enough), in RAM.

This is safe to use: when you app calls IndexWriter.Commit(), all cached files will be flushed from the cached and sync'd.

Here's a simple example usage:

fsDir, _ := OpenFSDirectory("/path/to/index")
cachedFSDir := NewNRTCachingDirectory(fsDir, 5.0, 60.0)
conf := NewIndexWriterConfig(VERSION_49, analyzer)
writer := NewIndexWriter(cachedFSDir, conf)

This will cache all newly flushed segments, all merged whose expected segment size is <= 5 MB, unless the net cached bytes exceeds 60 MB at which point all writes will not be cached (until the net bytes falls below 60 MB).

func NewNRTCachingDirectory

func NewNRTCachingDirectory(delegate Directory, maxMergeSizeMB, maxCachedMB float64) (nrt *NRTCachingDirectory)

We will cache a newly created output if 1) it's a flush or a merge and the estimated size of the merged segment is <= maxMergedSizeMB, and 2) the total cached bytes is <= maxCachedMB.

func (*NRTCachingDirectory) Close

func (nrt *NRTCachingDirectory) Close() error

Close this directory, which flushes any cached files to the delegate and then closes the delegate.

func (*NRTCachingDirectory) CreateOutput

func (nrt *NRTCachingDirectory) CreateOutput(name string, context IOContext) (out IndexOutput, err error)

func (*NRTCachingDirectory) DeleteFile

func (nrt *NRTCachingDirectory) DeleteFile(name string) error

func (*NRTCachingDirectory) FileExists

func (nrt *NRTCachingDirectory) FileExists(name string) bool

func (*NRTCachingDirectory) FileLength

func (nrt *NRTCachingDirectory) FileLength(name string) (length int64, err error)

func (*NRTCachingDirectory) ListAll

func (nrt *NRTCachingDirectory) ListAll() (all []string, err error)

func (*NRTCachingDirectory) OpenInput

func (nrt *NRTCachingDirectory) OpenInput(name string, context IOContext) (in IndexInput, err error)

func (*NRTCachingDirectory) String

func (nrt *NRTCachingDirectory) String() string

func (*NRTCachingDirectory) Sync

func (nrt *NRTCachingDirectory) Sync(fileNames []string) (err error)

type NoSuchDirectoryError

type NoSuchDirectoryError struct {
	// contains filtered or unexported fields
}

func (*NoSuchDirectoryError) Error

func (err *NoSuchDirectoryError) Error() string

type OutputStreamIndexOutput

type OutputStreamIndexOutput struct {
	*IndexOutputImpl
	// contains filtered or unexported fields
}

Implementation class for buffered IndexOutput that writes to a WriterCloser.

func (*OutputStreamIndexOutput) Checksum

func (out *OutputStreamIndexOutput) Checksum() int64

func (*OutputStreamIndexOutput) Close

func (out *OutputStreamIndexOutput) Close() error

func (*OutputStreamIndexOutput) FilePointer

func (out *OutputStreamIndexOutput) FilePointer() int64

func (*OutputStreamIndexOutput) WriteByte

func (out *OutputStreamIndexOutput) WriteByte(b byte) error

func (*OutputStreamIndexOutput) WriteBytes

func (out *OutputStreamIndexOutput) WriteBytes(p []byte) error

type RAMDirectory

type RAMDirectory struct {
	*DirectoryImpl
	*BaseDirectory
	// contains filtered or unexported fields
}

A memory-resident Directory implementation. Locking implementation is by default the SingleInstanceLockFactory but can be changed with SetLockFactory().

Warning: This class is not intended to work with huge indexes. Everything beyond several hundred megabytes will waste resources (GC cycles), becaues it uses an internal buffer size of 1024 bytes, producing millions of byte[1024] arrays. This class is optimized for small memory-resident indexes. It also has bad concurrency on multithreaded environments.

It is recommended to materialze large indexes on disk and use MMapDirectory, which is a high-performance directory implementation working diretly on the file system cache of the operating system, so copying dat to Java heap space is not useful.

func NewRAMDirectory

func NewRAMDirectory() *RAMDirectory

func (*RAMDirectory) ChangeSize

func (rd *RAMDirectory) ChangeSize(diff int64)

test-only

func (*RAMDirectory) Close

func (rd *RAMDirectory) Close() error

Closes the store to future operations, releasing associated memroy.

func (*RAMDirectory) CreateOutput

func (rd *RAMDirectory) CreateOutput(name string, context IOContext) (out IndexOutput, err error)

Creates a new, empty file in the directory with the given name. Returns a stream writing this file:

func (*RAMDirectory) DeleteFile

func (rd *RAMDirectory) DeleteFile(name string) error

Removes an existing file in the directory

func (*RAMDirectory) FileExists

func (rd *RAMDirectory) FileExists(name string) bool

Returns true iff the named file exists in this directory

func (*RAMDirectory) FileLength

func (rd *RAMDirectory) FileLength(name string) (length int64, err error)

Returns the length in bytes of a file in the directory.

func (*RAMDirectory) GetRAMFile

func (rd *RAMDirectory) GetRAMFile(name string) *RAMFile

test-only

func (*RAMDirectory) ListAll

func (rd *RAMDirectory) ListAll() (names []string, err error)

func (*RAMDirectory) LockID

func (d *RAMDirectory) LockID() string

func (*RAMDirectory) OpenInput

func (rd *RAMDirectory) OpenInput(name string, context IOContext) (in IndexInput, err error)

Returns a stream reading an existing file.

func (*RAMDirectory) PutRAMFile

func (d *RAMDirectory) PutRAMFile(name string, file *RAMFile)

test-only

func (*RAMDirectory) RamBytesUsed

func (rd *RAMDirectory) RamBytesUsed() int64

Return total size in bytes of all files in this directory. This is currently quantized to BUFFER_SIZE.

func (*RAMDirectory) String

func (rd *RAMDirectory) String() string

func (*RAMDirectory) Sync

func (rd *RAMDirectory) Sync(names []string) error

type RAMFile

type RAMFile struct {
	sync.Locker
	// contains filtered or unexported fields
}

Represents a file in RAM as a list of []byte buffers.

func NewRAMFile

func NewRAMFile(directory *RAMDirectory) *RAMFile

func NewRAMFileBuffer

func NewRAMFileBuffer() *RAMFile

func (*RAMFile) Buffer

func (rf *RAMFile) Buffer(index int) []byte

func (*RAMFile) Length

func (rf *RAMFile) Length() int64

func (*RAMFile) RamBytesUsed

func (rf *RAMFile) RamBytesUsed() int64

func (*RAMFile) SetLength

func (rf *RAMFile) SetLength(length int64)

type RAMInputStream

type RAMInputStream struct {
	*IndexInputImpl
	// contains filtered or unexported fields
}

A memory-resident IndexInput implementation.

func (*RAMInputStream) Clone

func (in *RAMInputStream) Clone() IndexInput

func (*RAMInputStream) Close

func (in *RAMInputStream) Close() error

func (*RAMInputStream) FilePointer

func (in *RAMInputStream) FilePointer() int64

func (*RAMInputStream) Length

func (in *RAMInputStream) Length() int64

func (*RAMInputStream) ReadByte

func (in *RAMInputStream) ReadByte() (byte, error)

func (*RAMInputStream) ReadBytes

func (in *RAMInputStream) ReadBytes(buf []byte) error

func (*RAMInputStream) Seek

func (in *RAMInputStream) Seek(pos int64) error

func (*RAMInputStream) Slice

func (in *RAMInputStream) Slice(desc string, offset, length int64) (IndexInput, error)

func (*RAMInputStream) String

func (in *RAMInputStream) String() string

type RAMOutputStream

type RAMOutputStream struct {
	*IndexOutputImpl
	// contains filtered or unexported fields
}

A memory-resident IndexOutput implementation

func NewRAMOutputStream

func NewRAMOutputStream(f *RAMFile, checksum bool) *RAMOutputStream

func NewRAMOutputStreamBuffer

func NewRAMOutputStreamBuffer() *RAMOutputStream

Construct an empty output buffer.

func (*RAMOutputStream) Checksum

func (out *RAMOutputStream) Checksum() int64

func (*RAMOutputStream) Close

func (out *RAMOutputStream) Close() error

func (*RAMOutputStream) FilePointer

func (out *RAMOutputStream) FilePointer() int64

func (*RAMOutputStream) Flush

func (out *RAMOutputStream) Flush() error

func (*RAMOutputStream) Reset

func (out *RAMOutputStream) Reset()

Resets this to an empty file.

func (*RAMOutputStream) WriteByte

func (out *RAMOutputStream) WriteByte(b byte) error

func (*RAMOutputStream) WriteBytes

func (out *RAMOutputStream) WriteBytes(buf []byte) error

func (*RAMOutputStream) WriteTo

func (out *RAMOutputStream) WriteTo(output util.DataOutput) error

Copy the current contents of this buffer to the named output.

func (*RAMOutputStream) WriteToBytes

func (out *RAMOutputStream) WriteToBytes(bytes []byte) error

Copy the current contents of this buffer to output byte slice

type RateLimitedDirectoryWrapper

type RateLimitedDirectoryWrapper struct {
	Directory
	// contains filtered or unexported fields
}

A Directory wrapper that allows IndexOutput rate limiting using IO context specific rate limiters.

func NewRateLimitedDirectoryWrapper

func NewRateLimitedDirectoryWrapper(wrapped Directory) *RateLimitedDirectoryWrapper

func (*RateLimitedDirectoryWrapper) CreateOutput

func (w *RateLimitedDirectoryWrapper) CreateOutput(name string, ctx IOContext) (IndexOutput, error)

func (*RateLimitedDirectoryWrapper) MaxWriteMBPerSec

func (w *RateLimitedDirectoryWrapper) MaxWriteMBPerSec(context int)

func (*RateLimitedDirectoryWrapper) SetMaxWriteMBPerSec

func (w *RateLimitedDirectoryWrapper) SetMaxWriteMBPerSec(mbPerSec float64, context int)

Sets the maximum (approx) MB/sec allowed by all write IO performed by IndexOutput created with the given context. Pass non-positve value to have no limit.

NOTE: For already created IndexOutput instances there is no guarantee this new rate will apply to them; it will only be guaranteed to apply for new created IndexOutput instances.

NOTE: this is an optional operation and might not be respected by all Directory implementations. Currently only buffered Directory implementations use rate-limiting.

type RateLimitedIndexOutput

type RateLimitedIndexOutput struct {
	*IndexOutputImpl
	// contains filtered or unexported fields
}

A rate limiting IndexOutput

func (*RateLimitedIndexOutput) Checksum

func (out *RateLimitedIndexOutput) Checksum() int64

func (*RateLimitedIndexOutput) Close

func (out *RateLimitedIndexOutput) Close() error

func (*RateLimitedIndexOutput) FilePointer

func (out *RateLimitedIndexOutput) FilePointer() int64

func (*RateLimitedIndexOutput) WriteByte

func (out *RateLimitedIndexOutput) WriteByte(b byte) error

func (*RateLimitedIndexOutput) WriteBytes

func (out *RateLimitedIndexOutput) WriteBytes(p []byte) error

type RateLimiter

type RateLimiter interface {
	// Sets an updated mb per second rate limit.
	SetMbPerSec(mbPerSec float64)
	// The current mb per second rate limit.
	MbPerSec() float64
	/*
		Pause, if necessary, to keep the instantaneous IO rate at or below
		the target.

		Note: the implementation is thread-safe
	*/
	Pause(bytes int64) int64
}

Abstract base class to rate limit IO. Typically implementations are shared across multiple IndexInputs or IndexOutputs (for example those involved all merging). Those IndexInputs and IndexOutputs would call pause() whenever they have read or written more than minPauseCheckBytes() bytes.

type SeekReader

type SeekReader interface {
	Length() int64
	// contains filtered or unexported methods
}

type SimpleFSDirectory

type SimpleFSDirectory struct {
	*FSDirectory
}

func NewSimpleFSDirectory

func NewSimpleFSDirectory(path string) (d *SimpleFSDirectory, err error)

func (*SimpleFSDirectory) OpenInput

func (d *SimpleFSDirectory) OpenInput(name string, context IOContext) (IndexInput, error)

type SimpleFSIndexInput

type SimpleFSIndexInput struct {
	*BufferedIndexInput
	// contains filtered or unexported fields
}

func (*SimpleFSIndexInput) Clone

func (in *SimpleFSIndexInput) Clone() IndexInput

func (*SimpleFSIndexInput) Close

func (in *SimpleFSIndexInput) Close() error

func (*SimpleFSIndexInput) Length

func (in *SimpleFSIndexInput) Length() int64

func (*SimpleFSIndexInput) Slice

func (in *SimpleFSIndexInput) Slice(desc string, offset, length int64) (IndexInput, error)

type SimpleFSLock

type SimpleFSLock struct {
	*LockImpl
	// contains filtered or unexported fields
}

func (*SimpleFSLock) Close

func (lock *SimpleFSLock) Close() error

func (*SimpleFSLock) IsLocked

func (lock *SimpleFSLock) IsLocked() bool

func (*SimpleFSLock) Obtain

func (lock *SimpleFSLock) Obtain() (ok bool, err error)

func (*SimpleFSLock) String

func (lock *SimpleFSLock) String() string

type SimpleFSLockFactory

type SimpleFSLockFactory struct {
	*FSLockFactory
}

Implements LockFactory using os.Create().

NOTE: This API may has the same issue as the one in Lucene Java that the write lock may not be released when Go program exists abnormally.

When this happens, an error is returned when trying to create a writer, in which case you need to explicitly clear the lock file first. You can either manually remove the file, or use UnlockDirectory() API. But, first be certain that no writer is in fact writing to the index otherwise you can easily corrupt your index.

If you suspect that this or any other LockFactory is not working properly in your environment, you can easily test it by using VerifyingLockFactory, LockVerifyServer and LockStressTest.

func NewSimpleFSLockFactory

func NewSimpleFSLockFactory(path string) *SimpleFSLockFactory

func (*SimpleFSLockFactory) Clear

func (f *SimpleFSLockFactory) Clear(name string) error

func (*SimpleFSLockFactory) Make

func (f *SimpleFSLockFactory) Make(name string) Lock

type SimpleRateLimiter

type SimpleRateLimiter struct {
	// contains filtered or unexported fields
}

Simple class to rate limit IO Ian: volatile is not supported

func (*SimpleRateLimiter) MbPerSec

func (srl *SimpleRateLimiter) MbPerSec() float64

func (*SimpleRateLimiter) Pause

func (srl *SimpleRateLimiter) Pause(bytes int64) int64

Pause, if necessary, to keep the instantaneous IO rate at or below the target. Be sure to only call this method when bytes > minPauseCheckBytes(), otherwise it will pause way too long!

func (*SimpleRateLimiter) SetMbPerSec

func (srl *SimpleRateLimiter) SetMbPerSec(mbPerSec float64)

type SingleInstanceLock

type SingleInstanceLock struct {
	*LockImpl
	// contains filtered or unexported fields
}

func (*SingleInstanceLock) Close

func (lock *SingleInstanceLock) Close() error

func (*SingleInstanceLock) IsLocked

func (lock *SingleInstanceLock) IsLocked() bool

func (*SingleInstanceLock) Obtain

func (lock *SingleInstanceLock) Obtain() (ok bool, err error)

func (*SingleInstanceLock) String

func (lock *SingleInstanceLock) String() string

type SingleInstanceLockFactory

type SingleInstanceLockFactory struct {
	*LockFactoryImpl
	// contains filtered or unexported fields
}

Implements LockFactory for a single in-process instance, meaning all locking will take place through this one instance. Only use this LockFactory when you are certain all IndexReaders and IndexWriters for a given index are running against a single shared in-process Directory instance. This is currently the default locking for RAMDirectory.

func (*SingleInstanceLockFactory) Clear

func (fac *SingleInstanceLockFactory) Clear(name string) error

func (*SingleInstanceLockFactory) Make

func (fac *SingleInstanceLockFactory) Make(name string) Lock

func (*SingleInstanceLockFactory) String

func (fac *SingleInstanceLockFactory) String() string

type TrackingDirectoryWrapper

type TrackingDirectoryWrapper struct {
	Directory
	sync.Locker
	// contains filtered or unexported fields
}

A delegating Directory that records which files were written to and deleted.

func NewTrackingDirectoryWrapper

func NewTrackingDirectoryWrapper(other Directory) *TrackingDirectoryWrapper

func (*TrackingDirectoryWrapper) ContainsFile

func (w *TrackingDirectoryWrapper) ContainsFile(name string) bool

func (*TrackingDirectoryWrapper) Copy

func (w *TrackingDirectoryWrapper) Copy(to Directory, src, dest string, ctx IOContext) error

func (*TrackingDirectoryWrapper) CreateOutput

func (w *TrackingDirectoryWrapper) CreateOutput(name string, ctx IOContext) (IndexOutput, error)

func (*TrackingDirectoryWrapper) DeleteFile

func (w *TrackingDirectoryWrapper) DeleteFile(name string) error

func (*TrackingDirectoryWrapper) EachCreatedFiles

func (w *TrackingDirectoryWrapper) EachCreatedFiles(f func(name string))

func (*TrackingDirectoryWrapper) String

func (w *TrackingDirectoryWrapper) String() string

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL