dataparser

package

v0.0.0-...-ec16f60 Latest Latest Go to latest Published: Apr 15, 2021 License: Apache-2.0 Imports: 10 Imported by: 26

Details

Valid go.mod file

The Go module system was introduced in Go 1.11 and is the official dependency management solution for Go.
Redistributable license

Redistributable licenses place minimal restrictions on how software can be used, modified, and redistributed.
Tagged version

Modules with tagged versions give importers more predictable builds.
Stable version

When a project reaches major version v1 it is considered stable.
Learn more about best practices

Repository

github.com/streamsets/datacollector-edge

Documentation ¶

Overview ¶

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Index ¶

Constants
type DataParserFormatConfig
- func (d *DataParserFormatConfig) Init(dataFormat string, stageContext api.StageContext, issues []validation.Issue) []validation.Issue
type RegExConfig

Constants ¶

View Source

const (
	CompressedNone = "NONE"
	CompressedFile = "COMPRESSED_FILE"
)

Variables ¶

This section is empty.

Functions ¶

This section is empty.

Types ¶

type DataParserFormatConfig ¶

type DataParserFormatConfig struct {
	Compression         string `ConfigDef:"type=STRING,required=true"`
	FlePatternInArchive string `ConfigDef:"type=STRING,required=true"`

	/* Charset Related -- Shown last */
	Charset         string `ConfigDef:"type=STRING,required=true"`
	RemoveCtrlChars bool   `ConfigDef:"type=BOOLEAN,required=true"`

	/** For TEXT Content **/
	TextMaxLineLen                  float64 `ConfigDef:"type=NUMBER,required=true"`
	UseCustomDelimiter              bool    `ConfigDef:"type=BOOLEAN,required=true"`
	CustomDelimiter                 string  `ConfigDef:"type=STRING,required=true"`
	IncludeCustomDelimiterInTheText bool    `ConfigDef:"type=BOOLEAN,required=true"`

	/** For JSON **/
	JsonContent string `ConfigDef:"type=STRING,required=true"`

	/** For DELIMITED Content **/
	CsvFileFormat        string  `ConfigDef:"type=STRING,required=true"`
	CsvHeader            string  `ConfigDef:"type=STRING,required=true"`
	CsvAllowExtraColumns bool    `ConfigDef:"type=BOOLEAN,required=true"`
	CsvExtraColumnPrefix string  `ConfigDef:"type=STRING,required=true"`
	CsvMaxObjectLen      float64 `ConfigDef:"type=NUMBER,required=true"`
	CsvCustomDelimiter   string  `ConfigDef:"type=STRING,required=true"`
	CsvCustomEscape      string  `ConfigDef:"type=STRING,required=true"`
	CsvEnableComments    bool    `ConfigDef:"type=BOOLEAN,required=true"`
	CsvCommentMarker     string  `ConfigDef:"type=STRING,required=true"`
	CsvIgnoreEmptyLines  bool    `ConfigDef:"type=BOOLEAN,required=true"`
	CsvRecordType        string  `ConfigDef:"type=STRING,required=true"`
	CsvSkipStartLines    float64 `ConfigDef:"type=NUMBER,required=true"`
	ParseNull            bool    `ConfigDef:"type=BOOLEAN,required=true"`
	NullConstant         string  `ConfigDef:"type=STRING,required=true"`

	/** For XML Content **/
	XmlRecordElement            string            `ConfigDef:"type=STRING,required=true"`
	IncludeFieldXpathAttributes bool              `ConfigDef:"type=BOOLEAN,required=true"`
	XPathNamespaceContext       map[string]string `ConfigDef:"type=MAP,required=true"`
	OutputFieldAttributes       bool              `ConfigDef:"type=BOOLEAN,required=true"`
	XmlMaxObjectLen             float64           `ConfigDef:"type=NUMBER,required=true"`

	// LOG Configuration
	LogMode               string        `ConfigDef:"type=STRING,required=true"`
	LogMaxObjectLen       float64       `ConfigDef:"type=NUMBER,required=true"`
	RetainOriginalLine    bool          `ConfigDef:"type=BOOLEAN,required=true"`
	CustomLogFormat       string        `ConfigDef:"type=STRING,required=true"`
	Regex                 string        `ConfigDef:"type=STRING,required=true"`
	FieldPathsToGroupName []RegExConfig `ConfigDef:"type=MODEL" ListBeanModel:"name=fieldPathsToGroupName"`
	// GROK
	GrokPatternDefinition      string  `ConfigDef:"type=STRING,required=true"`
	GrokPattern                string  `ConfigDef:"type=STRING,required=true"`
	OnParseError               string  `ConfigDef:"type=STRING,required=true"`
	MaxStackTraceLines         float64 `ConfigDef:"type=NUMBER,required=true"`
	EnableLog4jCustomLogFormat bool    `ConfigDef:"type=BOOLEAN,required=true"`
	Log4jCustomLogFormat       string  `ConfigDef:"type=STRING,required=true"`

	/** For AVRO Content **/
	AvroSchemaSource                  string   `ConfigDef:"type=STRING,required=true"`
	AvroSchema                        string   `ConfigDef:"type=STRING,required=true"`
	RegisterSchema                    bool     `ConfigDef:"type=BOOLEAN,required=true"`
	SchemaRegistryUrlsForRegistration []string `ConfigDef:"type=LIST,required=true"`
	SchemaRegistryUrls                []string `ConfigDef:"type=LIST,required=true"`
	SchemaLookupMode                  string   `ConfigDef:"type=STRING,required=true"`
	Subject                           string   `ConfigDef:"type=STRING,required=true"`
	SchemaId                          float64  `ConfigDef:"type=STRING,required=true"`

	/** For Protobuf Content **/
	ProtoDescriptorFile string `ConfigDef:"type=STRING,required=true"`
	MessageType         string `ConfigDef:"type=STRING,required=true"`
	IsDelimited         bool   `ConfigDef:"type=BOOLEAN,required=true"`

	/** For Binary Content **/
	BinaryMaxObjectLen float64 `ConfigDef:"type=NUMBER,required=true"`

	// DATAGRAM
	DatagramMode    string `ConfigDef:"type=STRING,required=true"`
	TypesDbPath     string `ConfigDef:"type=STRING,required=true"`
	ConvertTime     bool   `ConfigDef:"type=BOOLEAN,required=true"`
	ExcludeInterval bool   `ConfigDef:"type=BOOLEAN,required=true"`
	AuthFilePath    string `ConfigDef:"type=STRING,required=true"`

	// Netflow v9
	NetflowOutputValuesMode         string  `ConfigDef:"type=STRING,required=true"`
	MaxTemplateCacheSize            float64 `ConfigDef:"type=NUMBER,required=true"`
	TemplateCacheTimeoutMs          float64 `ConfigDef:"type=NUMBER,required=true"`
	NetflowOutputValuesModeDatagram string  `ConfigDef:"type=STRING,required=true"`
	MaxTemplateCacheSizeDatagram    float64 `ConfigDef:"type=NUMBER,required=true"`
	TemplateCacheTimeoutMsDatagram  float64 `ConfigDef:"type=NUMBER,required=true"`

	/** For Whole File Content **/
	WholeFileMaxObjectLen float64 `ConfigDef:"type=NUMBER,required=true"`
	RateLimit             string  `ConfigDef:"type=STRING,required=true"`
	VerifyChecksum        bool    `ConfigDef:"type=BOOLEAN,required=true"`

	// Used to parse records from input stream
	RecordReaderFactory recordio.RecordReaderFactory

	// Used to create record for origins generating single line of text - Fail Tail & Directory Spooler
	RecordCreator recordio.RecordCreator
}

func (*DataParserFormatConfig) Init ¶

func (d *DataParserFormatConfig) Init(
	dataFormat string,
	stageContext api.StageContext,
	issues []validation.Issue,
) []validation.Issue

type RegExConfig ¶

type RegExConfig struct {
	FieldPath string  `ConfigDef:"type=STRING,required=true"`
	Group     float64 `ConfigDef:"type=NUMBER,required=true"`
}

Source Files ¶

View all Source files

dataparser.go

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL