Documentation
¶
Index ¶
- Constants
- func Count(fd FileObjectModel) (int64, error)
- func Merge(src []FileObjectModel, dest FileObjectModel) error
- func RegisterChecksumer(key string, ch Checksumer)
- func RegisterFileType(ftype string, reader FileItemReader, writer FileItemWriter, ...)
- func Split(srcFd FileObjectModel, destFds []FileObjectModel, strategy FileSplitStrategy) error
- type ChecksumFlusher
- type ChecksumVerifier
- type Checksumer
- type FTPFileInStream
- type FTPFileOutStream
- type FTPFileSystem
- type FileItemReader
- type FileItemWriter
- type FileMerger
- type FileMove
- type FileObjectModel
- type FileSplitStrategy
- type FileSplitter
- type FileStorage
- type LocalFileSystem
- type MD5Checksumer
- type MergeSplitter
- type OKFlagChecksumer
- type SHA1Checksumer
- type SHA256Checksumer
- type SHA512Checksumer
Constants ¶
const ( //LocalFileStorage name of local file storage LocalFileStorage = "File" //FTPFileStorage name of ftp file storage FTPFileStorage = "FTP" )
const ( //TSV a file type that have tab-seperated values as its content TSV = "tsv" //CSV a file type that have comma-seperated values as its content CSV = "csv" //JSON a file type that have json string as its content JSON = "json" )
const ( OKFlag = "OK" MD5 = "MD5" SHA1 = "SHA1" SHA256 = "SHA256" SHA512 = "SHA512" )
Variables ¶
This section is empty.
Functions ¶
func Count ¶
func Count(fd FileObjectModel) (int64, error)
Count the default implementation to count number of items contained by the file
func Merge ¶
func Merge(src []FileObjectModel, dest FileObjectModel) error
Merge the default implementation to merge multiple files into a simple one
func RegisterChecksumer ¶
func RegisterChecksumer(key string, ch Checksumer)
func RegisterFileType ¶
func RegisterFileType(ftype string, reader FileItemReader, writer FileItemWriter, mergeSplitter MergeSplitter)
RegisterFileType register file type
func Split ¶
func Split(srcFd FileObjectModel, destFds []FileObjectModel, strategy FileSplitStrategy) error
Split the default implementation to split a file into multiple destinations
Types ¶
type ChecksumFlusher ¶
type ChecksumFlusher interface {
//Checksum generate checksum and write a check file for the corresponding data file
Checksum(fd FileObjectModel) error
}
ChecksumFlusher write a check file for the corresponding data file
type ChecksumVerifier ¶
type ChecksumVerifier interface {
//Verify verify a data file according to the corresponding check file
Verify(fd FileObjectModel) (bool, error)
}
ChecksumVerifier verify a data file according to the corresponding check file
type Checksumer ¶
type Checksumer interface {
ChecksumVerifier
ChecksumFlusher
}
Checksumer the combination of ChecksumVerifier and ChecksumFlusher
type FTPFileInStream ¶
type FTPFileInStream struct {
// contains filtered or unexported fields
}
FTPFileInStream represents a ftp file input stream
func (*FTPFileInStream) Close ¶
func (fis *FTPFileInStream) Close() (err error)
type FTPFileOutStream ¶
type FTPFileOutStream struct {
// contains filtered or unexported fields
}
FTPFileOutStream represents a ftp file output stream
func (*FTPFileOutStream) Close ¶
func (fis *FTPFileOutStream) Close() (err error)
type FTPFileSystem ¶
type FTPFileSystem struct {
Hort string
Port int
User string
Password string
ConnTimeout time.Duration
}
FTPFileSystem a FileStorage implementation backed by ftp
func (*FTPFileSystem) Create ¶
func (fs *FTPFileSystem) Create(fileName, encoding string) (writer io.WriteCloser, err error)
func (*FTPFileSystem) Name ¶
func (fs *FTPFileSystem) Name() string
func (*FTPFileSystem) Open ¶
func (fs *FTPFileSystem) Open(fileName, encoding string) (io.ReadCloser, error)
type FileItemReader ¶
type FileItemReader interface {
//Open open file of the FileObjectModel, returns a handle for later read
Open(fd FileObjectModel) (handle interface{}, err error)
//Close open the file handle
Close(handle interface{}) error
//ReadItem read an item from the file handle
ReadItem(handle interface{}) (interface{}, error)
//SkipTo skip to the pos'th record in the file
SkipTo(handle interface{}, pos int64) error
//Count return number of items contained by the file
Count(fd FileObjectModel) (int64, error)
}
FileItemReader read object from file according a FileObjectModel
func GetFileItemReader ¶
func GetFileItemReader(ftype string) FileItemReader
GetFileItemReader get FileItemReader by type
type FileItemWriter ¶
type FileItemWriter interface {
//Open open file of the FileObjectModel, returns a handle for later write
Open(fd FileObjectModel) (handle interface{}, err error)
//Close open the file handle
Close(handle interface{}) error
//WriteItem write an item to the file
WriteItem(handle interface{}, data interface{}) error
}
FileItemWriter write object to file according to a FileObjectModel
func GetFileItemWriter ¶
func GetFileItemWriter(ftype string) FileItemWriter
GetFileItemWriter get FileItemWriter by type
type FileMerger ¶
type FileMerger interface {
//Merge merge multiple files into a simple one
Merge(src []FileObjectModel, dest FileObjectModel) (err error)
}
FileMerger merge multiple files into a simple one
type FileMove ¶
type FileMove struct {
FromFileName string
FromFileStore FileStorage
ToFileName string
ToFileStore FileStorage
}
FileMove represents source and destination of a file movement
type FileObjectModel ¶
type FileObjectModel struct {
FileStore FileStorage
FileName string
Type string
Encoding string
Header bool
FieldSeparator string
Checksum string
ItemPrototype interface{}
}
FileObjectModel represents mapping a file to/from a struct
func (FileObjectModel) ItemType ¶
func (fd FileObjectModel) ItemType() (reflect.Type, error)
ItemType returns the reflect.Type of the item within a FileObjectModel
func (FileObjectModel) String ¶
func (fd FileObjectModel) String() string
type FileSplitStrategy ¶
type FileSplitStrategy interface {
//DecideDestIndex decide the destination of every record, return the index of destination file in the dest array
DecideDestIndex(line string, dest []FileObjectModel) int
}
FileSplitStrategy decide the destination of every record
type FileSplitter ¶
type FileSplitter interface {
//Split split a src file into multiple destinations, decide the destination of every record according to a FileSplitStrategy
Split(src FileObjectModel, dest []FileObjectModel, strategy FileSplitStrategy) (err error)
}
FileSplitter split a src file into multiple dest ones
type FileStorage ¶
type FileStorage interface {
//Name return name of the file store
Name() string
//Exists check if a file with the specified fileName exists
Exists(fileName string) (ok bool, err error)
//Open return a reader of the specified fileName, transform encoding if necessary
Open(fileName, encoding string) (reader io.ReadCloser, err error)
//Create return a writer of the specified fileName, transform encoding if necessary
Create(fileName, encoding string) (writer io.WriteCloser, err error)
}
FileStorage represents a file store media, from which we can read or write files
type LocalFileSystem ¶
type LocalFileSystem struct {
}
LocalFileSystem a FileStorage implementation backed by local file system
func (*LocalFileSystem) Create ¶
func (fs *LocalFileSystem) Create(fileName, encoding string) (io.WriteCloser, error)
func (*LocalFileSystem) Name ¶
func (fs *LocalFileSystem) Name() string
func (*LocalFileSystem) Open ¶
func (fs *LocalFileSystem) Open(fileName, encoding string) (io.ReadCloser, error)
type MD5Checksumer ¶
type MD5Checksumer struct {
}
MD5Checksumer generate and verify a check file containing the md5 digest of the data file
func (*MD5Checksumer) Checksum ¶
func (ch *MD5Checksumer) Checksum(fd FileObjectModel) error
func (*MD5Checksumer) Verify ¶
func (ch *MD5Checksumer) Verify(fd FileObjectModel) (bool, error)
type MergeSplitter ¶
type MergeSplitter interface {
FileMerger
FileSplitter
}
MergeSplitter the combination of FileMerger and FileSplitter
func GetFileMergeSplitter ¶
func GetFileMergeSplitter(ftype string) MergeSplitter
GetFileMergeSplitter get MergeSplitter by type
type OKFlagChecksumer ¶
type OKFlagChecksumer struct {
}
OKFlagChecksumer generate and verify an empty file with '.ok' suffix indicating the data file completed
func (*OKFlagChecksumer) Checksum ¶
func (ch *OKFlagChecksumer) Checksum(fd FileObjectModel) error
func (*OKFlagChecksumer) Verify ¶
func (ch *OKFlagChecksumer) Verify(fd FileObjectModel) (bool, error)
type SHA1Checksumer ¶
type SHA1Checksumer struct {
}
SHA1Checksumer generate and verify a check file containing the sha-1 digest of the data file
func (*SHA1Checksumer) Checksum ¶
func (ch *SHA1Checksumer) Checksum(fd FileObjectModel) error
func (*SHA1Checksumer) Verify ¶
func (ch *SHA1Checksumer) Verify(fd FileObjectModel) (bool, error)
type SHA256Checksumer ¶
type SHA256Checksumer struct {
}
SHA256Checksumer generate and verify a check file containing the sha-256 digest of the data file
func (*SHA256Checksumer) Checksum ¶
func (ch *SHA256Checksumer) Checksum(fd FileObjectModel) error
func (*SHA256Checksumer) Verify ¶
func (ch *SHA256Checksumer) Verify(fd FileObjectModel) (bool, error)
type SHA512Checksumer ¶
type SHA512Checksumer struct {
}
SHA512Checksumer generate and verify a check file containing the sha-512 digest of the data file
func (*SHA512Checksumer) Checksum ¶
func (ch *SHA512Checksumer) Checksum(fd FileObjectModel) error
func (*SHA512Checksumer) Verify ¶
func (ch *SHA512Checksumer) Verify(fd FileObjectModel) (bool, error)