Documentation
¶
Overview ¶
Copyright 2018 StreamSets Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
Copyright 2018 StreamSets Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
Copyright 2018 StreamSets Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
Copyright 2018 StreamSets Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
Index ¶
Constants ¶
const ( ExecutionMode = "executionMode" DeliveryGuarantee = "deliveryGuarantee" ShouldRetry = "shouldRetry" RetryAttempts = "retryAttempts" MemoryLimit = "memoryLimit" MemoryLmtExceeded = "memoryLimitExceeded" NotifyOnStates = "notifyOnStates" EmailIds = "emailIDs" Constants = "constants" BadRecordsHandling = "badRecordsHandling" TestOriginStage = "testOriginStage" RateLimit = "rateLimit" MaxRunners = "maxRunners" StatsAggregatorStage = "statsAggregatorStage" ErrorRecordPolicy = "errorRecordPolicy" ClusterSlaveMemory = "clusterSlaveMemory" ClusterSlaveJavaOpts = "clusterSlaveJavaOpts" ClusterLauncherEnv = "clusterLauncherEnv" MesosDispatchUrl = "mesosDispatcherURL" HdfsS3ConfigDir = "hdfsS3ConfDir" WebHookConfigs = "webhookConfigs" )
const ( STAGE_TYPE = "stageType" SOURCE = "SOURCE" PROCESSOR = "PROCESSOR" TARGET = "TARGET" )
Variables ¶
This section is empty.
Functions ¶
func GetDefaultStatsAggregatorStageInstance ¶
func GetDefaultStatsAggregatorStageInstance() *common.StageConfiguration
func GetTrashErrorStageInstance ¶
func GetTrashErrorStageInstance() *common.StageConfiguration
Types ¶
type PipelineBean ¶
type PipelineBean struct {
Config PipelineConfigBean
Stages []StageBean
ErrorStage StageBean
StatsAggregatorStage StageBean
ElContext context.Context
}
func NewPipelineBean ¶
func NewPipelineBean( pipelineConfig common.PipelineConfiguration, runtimeParameters map[string]interface{}, ) (PipelineBean, []validation.Issue)
type PipelineConfigBean ¶
type PipelineConfigBean struct {
Version string
ExecutionMode string
DeliveryGuarantee string
ShouldRetry bool
RetryAttempts float64
MemoryLimit string
MemoryLimitExceeded string
NotifyOnStates []interface{}
EmailIDs []interface{}
Constants map[string]interface{}
BadRecordsHandling string
TestOriginStage string
ErrorRecordPolicy string
StatsAggregatorStage string
RateLimit float64
MaxRunners float64
}
func NewPipelineConfigBean ¶
func NewPipelineConfigBean(pipelineConfig common.PipelineConfiguration) PipelineConfigBean
type ServiceBean ¶
type ServiceBean struct {
Config *common.ServiceConfiguration
Service api.Service
}
type StageBean ¶
type StageBean struct {
Config *common.StageConfiguration
Stage api.Stage
SystemConfigs StageConfigBean
Services []ServiceBean
}
func NewStageBean ¶
func (*StageBean) IsProcessor ¶
type StageConfigBean ¶
type StageConfigBean struct {
StageOnRecordError string
StageRequiredFields []interface{}
StageRecordPreconditions []interface{}
}
func NewStageConfigBean ¶
func NewStageConfigBean(pipelineConfig *common.StageConfiguration) StageConfigBean
Source Files