Documentation ¶
Index ¶
- Constants
- Variables
- func CheckFatal(e error) error
- func CheckNotFatal(e error) error
- func CleanupStringASCII(s string, isToLower bool) string
- func CompressBlob(in []byte) ([]byte, error)
- func ElasticSearchPut(timeout time.Duration, endpoint string, indexPrefix string, env string, ...) error
- func Explode(x string) (a, b string)
- func FromJson(buf []byte, o interface{}) error
- func FromJsonBytes(buf []byte, o interface{}) error
- func FromJsonString(buf string, o interface{}) error
- func GetMD5Hash(text string) string
- func HttpGetJson(url string, o interface{}, username string, password string, ...) error
- func HumanFloat(value float64, units string) string
- func Implode(a, b string) string
- func IsPrivateIPv4(ip net.IP) bool
- func IsValidURL(urlString string) error
- func Lower16BitPrivateIP() (uint16, error)
- func ParseDurationString(duration string) int
- func ParseFloatOrDefault(val string, bits int, defaultVal float64) float64
- func ParseS3URL(urlStr string) (string, string, error)
- func ParseStringOrDefault(s *string, defaultVal string) string
- func PercentBar(percent float64) string
- func Prefix(s string, l int) string
- func PrivateIPv4() (net.IP, error)
- func ReadJsonFile(filename string, o interface{}) error
- func S3Exists(region *string, bucketName string, remoteFilename string) (bool, *s3.HeadObjectOutput, error)
- func S3Get(region *string, bucketName string, key string) ([]byte, string, error)
- func S3GetSignedURL(region *string, bucketName string, key string) (string, error)
- func S3GetToLocalfile(region *string, bucketName string, key string, localFilename string) (string, error)
- func S3Head(region *string, bucketName string, remoteFilename string) (bool, error)
- func S3PutBlob(region *string, timeout time.Duration, bucketName string, blob []byte, ...) (*s3.PutObjectOutput, string, error)
- func S3PutLocalFile(region *string, timeout time.Duration, bucketName string, localFilename string, ...) (*s3.PutObjectOutput, string, error)
- func S3PutReader(region *string, bucketName string, blob io.ReadSeeker, remoteFilename string, ...) (*s3.PutObjectOutput, error)
- func ToHexString(buf []byte) string
- func ToJsonBytes(v interface{}) []byte
- func ToJsonBytesNoIndent(v interface{}) []byte
- func ToJsonString(v interface{}) string
- func ToJsonStringNoIndent(v interface{}) string
- type Config
- type ConfigStream
- type LinesBatch
- type Server
- func (t *Server) AppendDebugLineToLogFile(line string)
- func (s *Server) GenerateUniqueId(idType string) (string, string, time.Time)
- func (t *Server) Initialize(config *Config)
- func (t *Server) ProcessLine(streamAddress *string, stream *ConfigStream, line *string)
- func (t *Server) ProcessLineToLocalFile(line *string)
- func (t *Server) ReadStream(streamAddress *string, stream *ConfigStream)
- func (t *Server) RunForever()
- func (t *Server) UploadBatch(batch *LinesBatch)
- func (t *Server) Write(p []byte) (n int, err error)
- type Stats
Constants ¶
View Source
const ( CONTENT_TYPE = "Content-Type" CONTENT_TYPE_AAC = "audio/aac" CONTENT_TYPE_JPEG = "image/jpeg" CONTENT_TYPE_RAW = "application/octet-stream" CONTENT_TYPE_BIN = "application/octet-stream" CONTENT_TYPE_FLAC = "audio/flac" CONTENT_TYPE_XML = "text/xml" CONTENT_TYPE_CSV = "text/csv" CONTENT_TYPE_JSON = "application/json" CONTENT_TYPE_HTML = "text/html" CONTENT_TYPE_TXT = "text/plain" CONTENT_TYPE_MP4 = "video/mp4" CONTENT_TYPE_PNG = "image/png" CONTENT_TYPE_KML = "application/vnd.google-earth.kml+xml" CONTENT_TYPE_GZ = "application/x-gzip" )
View Source
const HTTP_OK_STATUS_MAX = 300
View Source
const HTTP_OK_STATUS_MIN = 200
View Source
const ID_LINESD = "LINESD"
View Source
const LOG_FILES_CURRENT_LOG = "current" + LOG_FILES_EXT
View Source
const LOG_FILES_EXT = ".log"
View Source
const LOG_FILES_MAX_NUM = 10
View Source
const SEP = ":::"
View Source
const SERVICE_NAME = "linesd"
View Source
const VERSION_NUMBER = "0.0.5"
Variables ¶
View Source
var DurationREMult = regexp.MustCompile(`\d+([sSmMhHdDwW])`)
View Source
var DurationREValue = regexp.MustCompile(`(\d+).*`)
View Source
var (
ERR_INVALID_S3_URL = errors.New("S3 URL must be of form 's3://{bucket}/{key}'")
)
View Source
var (
ERR_S3_FAILURE = errors.New("can't establish S3 connection")
)
View Source
var FilterASCIIRE = regexp.MustCompile(`[^a-zA-Z_0-9\-]`)
Functions ¶
func CheckFatal ¶
func CheckNotFatal ¶
func CleanupStringASCII ¶
func CompressBlob ¶
func ElasticSearchPut ¶
func FromJsonBytes ¶
func FromJsonString ¶
func GetMD5Hash ¶
func HttpGetJson ¶
func HumanFloat ¶
func IsPrivateIPv4 ¶
func IsValidURL ¶
func Lower16BitPrivateIP ¶
func ParseDurationString ¶
func ParseFloatOrDefault ¶
func ParseStringOrDefault ¶
func PercentBar ¶
func PrivateIPv4 ¶
func ReadJsonFile ¶
func S3GetSignedURL ¶
func S3GetToLocalfile ¶
func S3PutLocalFile ¶
func S3PutReader ¶
func S3PutReader( region *string, bucketName string, blob io.ReadSeeker, remoteFilename string, contentType string, ) (*s3.PutObjectOutput, error)
func ToHexString ¶
func ToJsonBytes ¶
func ToJsonBytes(v interface{}) []byte
func ToJsonBytesNoIndent ¶
func ToJsonBytesNoIndent(v interface{}) []byte
func ToJsonString ¶
func ToJsonString(v interface{}) string
func ToJsonStringNoIndent ¶
func ToJsonStringNoIndent(v interface{}) string
Types ¶
type Config ¶
type Config struct { AWSRegion string `json:"aws_region"` AWSBucket string `json:"aws_bucket"` AWSKeyPrefix string `json:"aws_key_prefix"` AWSElasticSearchURL string `json:"aws_elastic_search"` Env string `json:"env"` ConcLimit int `json:"conc_limit"` IsShowBatches bool `json:"is_show_batches"` IsShowLines bool `json:"is_show_lines"` BatchSizeInLines int `json:"batch_size_in_lines"` BatchSizeInSeconds int `json:"batch_size_in_seconds"` Progress int `json:"progress"` Address string `json:"address"` Streams map[string]*ConfigStream `json:"streams"` // HTTP timeout: TimeoutSeconds int `json:"timeout_seconds"` // log to local files as well: LogFilesFolder string `json:"log_files_folder"` // max local file size in bytes: LogFilesFileSizeBytes int `json:"log_files_size_bytes"` // S3 destination prefix: AWSKeyPrefixEnv string }
type ConfigStream ¶
type LinesBatch ¶
type Server ¶
func (*Server) AppendDebugLineToLogFile ¶
func (*Server) GenerateUniqueId ¶
func (*Server) Initialize ¶
func (*Server) ProcessLine ¶
func (t *Server) ProcessLine(streamAddress *string, stream *ConfigStream, line *string)
func (*Server) ProcessLineToLocalFile ¶
func (*Server) ReadStream ¶
func (t *Server) ReadStream(streamAddress *string, stream *ConfigStream)
func (*Server) RunForever ¶
func (t *Server) RunForever()
func (*Server) UploadBatch ¶
func (t *Server) UploadBatch(batch *LinesBatch)
type Stats ¶
type Stats struct { Counters *prometheus.CounterVec `json:"-"` Gauges *prometheus.GaugeVec `json:"-"` }
Click to show internal directories.
Click to hide internal directories.