Documentation
¶
Index ¶
- Constants
- func DataKey(dataid string) string
- func DatasetFileKey(dataset types.DatasetFile) string
- func InducedTransformKey(itransformid string) string
- func StateKey(stateid string) string
- func TransformKey(transformid string) string
- type DataFile
- type LocalStorage
- func (store *LocalStorage) AddDataFile(dataFile types.DatasetFile) (dataID []string, err error)
- func (store *LocalStorage) AddInducedTransform(itransform types.InducedTransform) (itransformID string, err error)
- func (store *LocalStorage) AddTransformFile(transformFile string) (transform types.Transform, transformID string, err error)
- func (store *LocalStorage) Close() (err error)
- func (store *LocalStorage) Execute() (err error)
- func (store *LocalStorage) GetGraph() (graph types.ProtoMLGraph, err error)
- func (store *LocalStorage) GetTransformLogFile(transformId string) (paths string, err error)
- func (store *LocalStorage) Init(config persist.Config) (err error)
- func (store *LocalStorage) IsDone(itransformId string) (bool, error)
- func (store *LocalStorage) Run(itransformId string) (err error)
- func (store *LocalStorage) StartLuigi() (err error)
- func (store *LocalStorage) UpdateInducedTransform(itransformId string, itransform types.InducedTransform) (err error)
- type TaskInsert
- type TaskStatus
- type TaskStatusMsg
Constants ¶
View Source
const ( LOGTAG = "Persist-Local" BASE_STATE_DIRECTORY = ".ProtoML" ELASTIC_DIRECTORY = "elasticsearch" PROTOML_TRANSFORMS_DIRECTORY = "ProtoML-transforms/transforms" DIRECTORY_DEPTH = 4 HEX_CHARS_PER_DIRECTORY_LEVEL = 4 LUIGI_TASK = "ProtoML-persist/local/fiber/TransformTask.py" TASK_PARARMS_FILE = "params" TASK_LOG_FILE = "log" )
View Source
const DATAGROUPPARTS_TYPE = "dataparts"
Variables ¶
This section is empty.
Functions ¶
func InducedTransformKey ¶
func TransformKey ¶
Types ¶
type LocalStorage ¶
type LocalStorage struct {
Config persist.Config
ElasticProcess *exec.Cmd
LuigiProcess *exec.Cmd
LuigiTaskInsert chan TaskInsert
LuigiTaskStatus chan TaskStatus
FormatCollection *formatadaptor.FileFormatCollection
}
func (*LocalStorage) AddDataFile ¶
func (store *LocalStorage) AddDataFile(dataFile types.DatasetFile) (dataID []string, err error)
insert data file into persist
func (*LocalStorage) AddInducedTransform ¶
func (store *LocalStorage) AddInducedTransform(itransform types.InducedTransform) (itransformID string, err error)
add induced transform
func (*LocalStorage) AddTransformFile ¶
func (store *LocalStorage) AddTransformFile(transformFile string) (transform types.Transform, transformID string, err error)
load a transform from a file
func (*LocalStorage) Close ¶
func (store *LocalStorage) Close() (err error)
func (*LocalStorage) Execute ¶
func (store *LocalStorage) Execute() (err error)
execute entire pipeline
func (*LocalStorage) GetGraph ¶
func (store *LocalStorage) GetGraph() (graph types.ProtoMLGraph, err error)
get graph id vertices and id edges
func (*LocalStorage) GetTransformLogFile ¶
func (store *LocalStorage) GetTransformLogFile(transformId string) (paths string, err error)
get log file for transform
func (*LocalStorage) Run ¶
func (store *LocalStorage) Run(itransformId string) (err error)
func (*LocalStorage) StartLuigi ¶
func (store *LocalStorage) StartLuigi() (err error)
func (*LocalStorage) UpdateInducedTransform ¶
func (store *LocalStorage) UpdateInducedTransform(itransformId string, itransform types.InducedTransform) (err error)
update induced transform
type TaskStatus ¶
type TaskStatus struct {
TaskId string
TaskName string
MsgChan chan TaskStatusMsg
}
Source Files
Directories
Click to show internal directories.
Click to hide internal directories.