haystack

package
v0.0.0-...-3e5f7a9 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Feb 3, 2017 License: MIT Imports: 17 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	// 一个卷（volume）数据文件大小最大32GB。
	DatafileMaxSize       = 4 * 1024 * 1024 * 1024 * NeedlePaddingSize
	DatafileMaxOffset     = 4*1024*1024*1024 - 1 // 4294967295
	DatafileMaxCacheWrite = 1
)
View Source 
const (

	// IndexfileMaxSize -
	IndexfileMaxSize = int64((unsafe.Sizeof(int64(0)) + unsafe.Sizeof(int64(0))) * 32 * 1024 * 1024)
	// IndexEntrySize -
	IndexEntrySize = 16
	// IndexfileMaxCacheWrite -
	IndexfileMaxCacheWrite = 1
)
View Source 
const (
	NeedlePaddingSize = 8

	NeedleMagicSize    = 4
	NeedleCookieSize   = 4
	NeedleKeySize      = 8
	NeedleFlagsSize    = 1
	NeedleSizeSize     = 4
	NeedleChecksumSize = 4

	// constant 21
	NeedleHeaderSize = NeedleMagicSize + NeedleCookieSize + NeedleKeySize + NeedleFlagsSize + NeedleSizeSize
	// constant 8
	NeedleFooterSize = NeedleMagicSize + NeedleChecksumSize

	NeedleMagicOffset  = 0
	NeedleCookieOffset = NeedleMagicOffset + NeedleMagicSize
	NeedleKeyOffset    = NeedleCookieOffset + NeedleCookieSize
	NeedleFlagsOffset  = NeedleKeyOffset + NeedleKeySize
	NeedleSizeOffset   = NeedleFlagsOffset + NeedleFlagsSize

	NeedleDataOffset = NeedleSizeOffset + NeedleSizeSize

	NeedleChecksumOffset = NeedleMagicOffset + NeedleMagicSize
	NeedlePaddingOffset  = NeedleChecksumOffset + NeedleChecksumSize
)
View Source 
const (
	SuperBlockSize          = NeedlePaddingSize
	SuperBlockMagicOffset   = 0
	SuperBlockMagicSize     = 4
	SuperBlockVersionOffset = SuperBlockMagicOffset + SuperBlockMagicSize
	SuperBlockVersionSize   = 1
	SuperBlockPaddingOffset = SuperBlockVersionOffset + SuperBlockVersionSize
	SuperBlockPaddingSize   = SuperBlockSize - SuperBlockPaddingOffset
)

Variables

This section is empty.

Functions

func BlockOffset 

func BlockOffset(offset uint32) int64

BlockOffset get super block file offset.

func NeedleOffset 

func NeedleOffset(offset int64) uint32

NeedleOffset convert offset to needle offset.

func Size 

func Size(n uint32) uint32

Size get a needle size with meta data.

Types

type Data 

type Data struct {
	Dir      string
	FileSize uint64

	AlignedOffset uint64 // FileSize / NeedlePaddingSize
	// contains filtered or unexported fields
}

Data - **************** Data ****************

func NewData 

func NewData(vid int32, store_dir string) (data *Data)

======== NewData() ========

func (*Data) AppendNeedle 

func (_this *Data) AppendNeedle(needle *Needle) (region NeedleRegion, err error)

AppendNeedle - ======== AppendNeedle() ======== Keep write needles to the end of data file.

func (*Data) Close 

func (_this *Data) Close()

Close - ======== Close() ========

func (*Data) DeleteNeedle 

func (_this *Data) DeleteNeedle(needle *Needle) (err error)

DeleteNeedle - ======== DeleteNeedle() ========

func (*Data) FindNeedle 

func (_this *Data) FindNeedle(key int64) (needle *Needle, err error)

FindNeedle - ======== FindNeedle() ========

func (*Data) GetNeedle 

func (_this *Data) GetNeedle(key int64, region NeedleRegion) (needle *Needle, err error)

GetNeedle -

func (*Data) Init 

func (_this *Data) Init() (err error)

Init - ======== Init() ========

func (*Data) String 

func (_this *Data) String() string

String - ======== String() ========

func (*Data) UpdateNeedle 

func (_this *Data) UpdateNeedle(needle *Needle) (region NeedleRegion, err error)

UpdateNeedle - ======== UpdateNeedle() ========

type Index 

type Index struct {
	Dir string

	FileSize uint64
	// contains filtered or unexported fields
}

Index - **************** Index ****************

func NewIndex 

func NewIndex(vid int32, storeDir string) (index *Index)

NewIndex () ======== NewIndex() ========

func (*Index) AppendIndexEntry 

func (_this *Index) AppendIndexEntry(entry IndexEntry) (err error)

AppendIndexEntry () 

======== Index::AppendIndexEntry() ========

func (*Index) Close 

func (_this *Index) Close()

Close () ======== Index::Close() ========

func (*Index) GetNeedleRegion 

func (_this *Index) GetNeedleRegion(key int64) (NeedleRegion, bool)

GetNeedleRegion () ======== Index::GetNeedleRegion() ========

func (*Index) Init 

func (_this *Index) Init() (err error)

Init () ======== Index::Init() ========

func (*Index) SetNeedleRegion 

func (_this *Index) SetNeedleRegion(key int64, region NeedleRegion) error

SetNeedleRegion () ======== Index::SetNeedleRegion() ========

func (*Index) String 

func (_this *Index) String() string

======== String() ========

type IndexEntry 

type IndexEntry struct {
	Key    int64
	Region NeedleRegion
}

IndexEntry -

type Metrics 

type Metrics struct {
	ReadCount   uint64
	ReadBytes   uint64
	ReadTime    uint64
	WriteCount  uint64
	WriteBytes  uint64
	WriteTime   uint64
	DeleteCount uint64
	DeleteBytes uint64
	DeleteTime  uint64
}

Metrics - **************** Metrics ****************

func (*Metrics) Reset 

func (_this *Metrics) Reset()

Reset () ======== Metrics::Reset() ========

func (*Metrics) String 

func (_this *Metrics) String() string

String () ======== String() ========

type Needle 

type Needle struct {
	HeaderMagic []byte
	Cookie      int32
	Key         int64
	MTime       int64
	Flags       byte
	Size        uint32
	Data        []byte
	//DataReader  *io.Reader
	FooterMagic []byte
	Checksum    uint32
	Padding     []byte

	WriteSize   uint32 // 对齐后实际写入的字节数
	PaddingSize uint32 // 对齐后增加的字节数
	FooterSize  uint32 // 对齐后底部区域的字节数
	AlignedSize uint32 // needle.WriteSize / NeedlePaddingSize
	// contains filtered or unexported fields
}

**************** Needle ****************

func NewNeedle 

func NewNeedle(key int64, cookie int32, size uint32) *Needle

NewNeedle () ======== NewNeedle() ========

func (*Needle) Buffer 

func (_this *Needle) Buffer() []byte

Buffer () ======== Buffer() ========

func (*Needle) BuildFrom 

func (_this *Needle) BuildFrom(buf []byte) (err error)

======== BuildFrom() ========

func (*Needle) Close 

func (_this *Needle) Close()

Close () ======== Close() ========

func (*Needle) DataBuffer 

func (_this *Needle) DataBuffer() []byte

DataBuffer () ======== DataBuffer() ========

func (*Needle) FillBuffer 

func (_this *Needle) FillBuffer() (err error)

FillBuffer () ======== FillBuffer() ========

func (*Needle) ReadFrom 

func (_this *Needle) ReadFrom(file io.Reader) (err error)

ReadFrom () ======== ReadFrom() ========

func (*Needle) Renew 

func (_this *Needle) Renew(size uint32)

Renew () ======== Renew() ========

func (*Needle) String 

func (_this *Needle) String() string

String () ======== Needle::String() ========

type NeedleRegion 

type NeedleRegion struct {
	// use late 40 bits.
	// Single data file max size is
	// 1TB (2^40=1,099,511,627,776) * NEEDLE_PADDINGSIZE (8 bytes aligned.)
	AlignedOffset uint64
	// use late 24 bits. single file size has to less 16MB (2^24).
	Size uint32
}

NeedleRegion - **************** NeedleRegion ****************

func (*NeedleRegion) GetOffset 

func (_this *NeedleRegion) GetOffset() uint64

GetOffset () 

======== NeedleRegion::GetOffset() ========

func (*NeedleRegion) GetSize 

func (_this *NeedleRegion) GetSize() uint32

GetSize () ======== NeedleRegion::GetSize() ========

type Store 

type Store struct {
	Volumes map[int32]*Volume
	Dir     string
}

**************** Store ****************

func NewStore 

func NewStore(store_dir string) (store *Store)

======== NewStore() ========

func (*Store) Close 

func (this *Store) Close()

======== Close() ========

func (*Store) CreateVolume 

func (this *Store) CreateVolume(vid int32) (volume *Volume, err error)

======== CreateVolume() ========

func (*Store) GetVolume 

func (store *Store) GetVolume(vid int32) (*Volume, bool)

======== GetVolume() ========

func (*Store) Init 

func (this *Store) Init() (err error)

======== Init() ========

type SuperBlock 

type SuperBlock struct {
	Magic   []byte
	Version byte
	Padding []byte
}

SuperBlock - **************** SuperBlock ****************

func NewSuperBlock 

func NewSuperBlock() *SuperBlock

NewSuperBlock - ======== NewSuperBlock() ========

func (*SuperBlock) ReadFromFile 

func (_this *SuperBlock) ReadFromFile(reader *os.File) (err error)

ReadFromFile - ======== ReadFromFile() ========

func (*SuperBlock) WriteToFile 

func (_this *SuperBlock) WriteToFile(writer *os.File) (writedSize uint64, err error)

WriteToFile - ======== WriteToFile() ========

type Volume 

type Volume struct {
	ID int32
	// contains filtered or unexported fields
}

Volume - **************** Volume ****************

func NewVolume 

func NewVolume(vid int32, storeDir string) (volume *Volume)

NewVolume () ======== NewVolume() ========

func (*Volume) Close 

func (_this *Volume) Close()

Close () ======== Close() ========

func (*Volume) Init 

func (_this *Volume) Init() (err error)

Init () ======== Init() ========

func (*Volume) ReadNeedle 

func (_this *Volume) ReadNeedle(key int64) (needle *Needle, err error)

ReadNeedle () ======== ReadNeedle() ========

func (*Volume) String 

func (_this *Volume) String() string

String () ======== String() ========

func (*Volume) WriteNeedle 

func (_this *Volume) WriteNeedle(needle *Needle) (err error)

WriteNeedle () ======== WriteNeedle() ========

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.