common

package
v0.7.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Oct 16, 2023 License: MIT Imports: 29 Imported by: 0

Documentation

Overview

Package common contains common functions and variables used by various scripts and services

Index

Constants

View Source 
const (
	SourceTagLocal      = "local"
	SourceTagBloxroute  = "bloxroute"
	SourceTagChainbound = "chainbound"
	SourceTagEden       = "eden"
	SourceTagAlchemy    = "alchemy"
	SourceTagInfura     = "infura"

	// Trash tx reasons
	TrashTxAlreadyOnChain = "tx-already-onchain"
	TrashTxSignatureError = "signature-error"

	// GRPCWindowSize is recommended window size by bloxroute-labs:
	// https://docs.bloxroute.com/streams/working-with-streams/creating-a-subscription/grpc
	GRPCWindowSize = 128 * 1024

	// TxAlreadyIncludedThreshold sets the threshold for discarding transactions (if included that many ms before received)
	TxAlreadyIncludedThreshold = 12_000
)

Variables

View Source 
var (
	ErrUnsupportedFileFormat = errors.New("unsupported file format")

	Printer = message.NewPrinter(language.English)
	Caser   = cases.Title(language.English)
	Title   = Caser.String
)
View Source 
var DefaultSourceComparisons = []SourceComp{
	{SourceTagBloxroute, SourceTagLocal},
	{SourceTagChainbound, SourceTagLocal},
	{SourceTagBloxroute, SourceTagChainbound},
	{SourceTagBloxroute, SourceTagEden},
	{SourceTagChainbound, SourceTagEden},
}
View Source 
var TxSummaryEntryCSVHeader = []string{
	"timestamp_ms",
	"hash",
	"chain_id",
	"from",
	"to",
	"value",
	"nonce",
	"gas",
	"gas_price",
	"gas_tip_cap",
	"gas_fee_cap",
	"data_size",
	"data_4bytes",
	"sources",
	"included_at_block_height",
	"included_block_timestamp_ms",
	"inclusion_delay_ms",
}

Functions

func FmtDateDay added in v0.6.0 

func FmtDateDay(t time.Time) string

func FmtDateDayTime added in v0.6.0 

func FmtDateDayTime(t time.Time) string

func FmtDuration added in v0.6.0 

func FmtDuration(d time.Duration) string

func GetAuthTokenAndURL added in v0.6.0 

func GetAuthTokenAndURL(auth string) (string, string)

GetAuthTokenAndURL takes in auth strings like "token" or "token@url" and returns token and url separately

func GetCSV 

func GetCSV(filename string) (rows [][]string, err error)

GetCSV returns a CSV content from a file (.csv or .csv.zip)

func GetCSVFromFiles added in v0.5.2 

func GetCSVFromFiles(filenames []string) (rows [][]string, err error)

func GetEnv 

func GetEnv(key, defaultValue string) string

func GetEnvInt added in v0.6.0 

func GetEnvInt(key string, defaultValue int) int

func GetLogger 

func GetLogger(debug, prod bool) *zap.SugaredLogger

func GetMemUsageHuman added in v0.6.0 

func GetMemUsageHuman() string

func GetMemUsageMb 

func GetMemUsageMb() uint64

func HumanBytes 

func HumanBytes(n uint64) string

HumanBytes returns size in the same format as AWS S3

func Int64DiffPercentFmt 

func Int64DiffPercentFmt(a, b int64, decimals uint) string

func Int64DiffPercentFmtC added in v0.6.0 

func Int64DiffPercentFmtC(a, b int64, decimals uint, fmtSuffix string) string

func IntDiffPercentFmt 

func IntDiffPercentFmt(a, b int, decimals uint) string

func IntDiffPercentFmtC added in v0.6.0 

func IntDiffPercentFmtC(a, b int, decimals uint, fmtSuffix string) string

func IsWebsocketProtocol added in v0.5.2 

func IsWebsocketProtocol(url string) bool

func LoadSourcelogFiles added in v0.5.2 

func LoadSourcelogFiles(log *zap.SugaredLogger, files []string) (txs map[string]map[string]int64, cntProcessedRecords int64)

LoadSourcelogFiles loads sourcelog .csv (or .csv.zip) files (format: <timestamp_ms>,<tx_hash>,<source>) and returns a map[hash][source] = timestampMs

func LoadTransactionCSVFiles 

func LoadTransactionCSVFiles(log *zap.SugaredLogger, txInputFiles, txBlacklistFiles []string) (txs map[string]*TxSummaryEntry, err error)

LoadTransactionCSVFiles loads transaction CSV files into a map[txHash]*TxSummaryEntry All transactions occurring in []knownTxsFiles are skipped

func LoadTrashFiles added in v0.5.2 

func LoadTrashFiles(log *zap.SugaredLogger, files []string) (txs map[string]map[string]*TrashEntry, err error)

LoadTrashFiles loads sourcelog .csv (or .csv.zip) files (format: <timestamp_ms>,<tx_hash>,<source>) and returns a map[hash][source] = *TrashEntry

func LoadTxHashesFromMetadataCSVFiles 

func LoadTxHashesFromMetadataCSVFiles(log *zap.SugaredLogger, files []string) (txs map[string]bool, err error)

LoadTxHashesFromMetadataCSVFiles loads transaction hashes from metadata CSV (or .csv.zip) files into a map[txHash]bool

func MustBeCSVFile added in v0.6.0 

func MustBeCSVFile(log *zap.SugaredLogger, fn string)

func MustBeFile 

func MustBeFile(log *zap.SugaredLogger, fn string, extensions []string)

func MustBeParquetFile added in v0.6.0 

func MustBeParquetFile(log *zap.SugaredLogger, fn string)

func MustNotExist 

func MustNotExist(log *zap.SugaredLogger, fn string)

func PrettyInt added in v0.6.0 

func PrettyInt(i int) string

func PrettyInt64 added in v0.6.0 

func PrettyInt64(i int64) string

func PrintMemUsage 

func PrintMemUsage()

func RLPDecode 

func RLPDecode(rlpBytes []byte) (*types.Transaction, error)

func RLPStringToTx 

func RLPStringToTx(rlpHex string) (*types.Transaction, error)

func SetupMarkdownTableWriter added in v0.6.0 

func SetupMarkdownTableWriter(table *tablewriter.Table)

func SourceAliasesFromEnv 

func SourceAliasesFromEnv() map[string]string

func TitleStrings added in v0.6.0 

func TitleStrings(s []string) []string

func TxSourcName 

func TxSourcName(uri string) string

func TxToRLPString 

func TxToRLPString(tx *types.Transaction) (string, error)

Types

type Analyzer2 added in v0.6.0 

type Analyzer2 struct {
	Transactions map[string]*TxSummaryEntry
	Sourelog     map[string]map[string]int64
	SourceComps  []SourceComp
	// contains filtered or unexported fields
}

func NewAnalyzer2 added in v0.6.0 

func NewAnalyzer2(opts Analyzer2Opts) *Analyzer2

func (*Analyzer2) Print added in v0.6.0 

func (a *Analyzer2) Print()

func (*Analyzer2) Sprint added in v0.6.0 

func (a *Analyzer2) Sprint() string

func (*Analyzer2) WriteToFile added in v0.6.0 

func (a *Analyzer2) WriteToFile(filename string) error

type Analyzer2Opts added in v0.6.0 

type Analyzer2Opts struct {
	Transactions map[string]*TxSummaryEntry
	Sourelog     map[string]map[string]int64 // [hash][source] = timestampMs
	SourceComps  []SourceComp
}

type BlxRawTxMsg 

type BlxRawTxMsg struct {
	Params struct {
		Result struct {
			RawTx string
		}
	}
}

type EdenRawTxMsg 

type EdenRawTxMsg struct {
	Params struct {
		Result struct {
			RLP string
		}
	}
}

type SourceComp 

type SourceComp struct {
	Source    string
	Reference string
}

func NewSourceComps 

func NewSourceComps(args []string) (srcComp []SourceComp)

type TrashEntry added in v0.5.2 

type TrashEntry struct {
	Timestamp int64
	Hash      string
	Source    string
	Reason    string
	Notes     string
}

func NewTrashEntryFromCSVRow added in v0.5.2 

func NewTrashEntryFromCSVRow(row []string) *TrashEntry

func (*TrashEntry) TrashEntryToCSVRow added in v0.5.2 

func (entry *TrashEntry) TrashEntryToCSVRow() string

type TxSummaryEntry 

type TxSummaryEntry struct {
	// The fields are written to CSV, and the order shouldn't change (for backwards compatibility)
	Timestamp int64  `parquet:"name=timestamp, type=INT64, convertedtype=TIMESTAMP_MILLIS"`
	Hash      string `parquet:"name=hash, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN, omitstats=true"`

	ChainID string `parquet:"name=chainId, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN"`
	From    string `parquet:"name=from, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN, omitstats=true"`
	To      string `parquet:"name=to, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN_DICTIONARY"`
	Value   string `parquet:"name=value, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN, omitstats=true"`
	Nonce   string `parquet:"name=nonce, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN, omitstats=true"`

	Gas       string `parquet:"name=gas, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN, omitstats=true"`
	GasPrice  string `parquet:"name=gasPrice, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN, omitstats=true"`
	GasTipCap string `parquet:"name=gasTipCap, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN, omitstats=true"`
	GasFeeCap string `parquet:"name=gasFeeCap, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN, omitstats=true"`

	DataSize   int64  `parquet:"name=dataSize, type=INT64"`
	Data4Bytes string `parquet:"name=data4Bytes, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN_DICTIONARY"`

	Sources []string `parquet:"name=sources, type=MAP, convertedtype=LIST, valuetype=BYTE_ARRAY, valueconvertedtype=UTF8"`

	// Inclusion stats
	IncludedAtBlockHeight  int64 `parquet:"name=includedAtBlockHeight, type=INT64"`
	IncludedBlockTimestamp int64 `parquet:"name=includedBlockTimestamp, type=INT64, convertedtype=TIMESTAMP_MILLIS"`
	InclusionDelayMs       int64 `parquet:"name=inclusionDelayMs, type=INT64"`

	// Finally, the raw transaction (not written to CSV)
	RawTx string `parquet:"name=rawTx, type=BYTE_ARRAY, encoding=PLAIN, omitstats=true"`
}

TxSummaryEntry is a struct that represents a single transaction in the summary CSV and Parquet file see also https://github.com/xitongsys/parquet-go for more details on parquet tags

func ParseTx 

func ParseTx(timestampMs int64, rawTxHex string) (TxSummaryEntry, *types.Transaction, error)

func (*TxSummaryEntry) HasSource added in v0.6.0 

func (t *TxSummaryEntry) HasSource(src string) bool

func (*TxSummaryEntry) RawTxHex 

func (t *TxSummaryEntry) RawTxHex() string

func (*TxSummaryEntry) ToCSVRow 

func (t *TxSummaryEntry) ToCSVRow() []string

func (*TxSummaryEntry) UpdateInclusionStatus added in v0.6.0 

func (t *TxSummaryEntry) UpdateInclusionStatus(ethClient *ethclient.Client) (*types.Header, error)

func (*TxSummaryEntry) WasIncludedBeforeReceived added in v0.6.0 

func (t *TxSummaryEntry) WasIncludedBeforeReceived() bool

type TxSummaryEntryNoRaw added in v0.6.0 

type TxSummaryEntryNoRaw struct {
	// The fields are written to CSV, and the order shouldn't change (for backwards compatibility)
	Timestamp int64  `parquet:"name=timestamp, type=INT64, convertedtype=TIMESTAMP_MILLIS"`
	Hash      string `parquet:"name=hash, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN, omitstats=true"`

	ChainID string `parquet:"name=chainId, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN"`
	From    string `parquet:"name=from, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN, omitstats=true"`
	To      string `parquet:"name=to, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN_DICTIONARY"`
	Value   string `parquet:"name=value, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN, omitstats=true"`
	Nonce   string `parquet:"name=nonce, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN, omitstats=true"`

	Gas       string `parquet:"name=gas, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN, omitstats=true"`
	GasPrice  string `parquet:"name=gasPrice, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN, omitstats=true"`
	GasTipCap string `parquet:"name=gasTipCap, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN, omitstats=true"`
	GasFeeCap string `parquet:"name=gasFeeCap, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN, omitstats=true"`

	DataSize   int64  `parquet:"name=dataSize, type=INT64"`
	Data4Bytes string `parquet:"name=data4Bytes, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN_DICTIONARY"`

	Sources []string `parquet:"name=sources, type=MAP, convertedtype=LIST, valuetype=BYTE_ARRAY, valueconvertedtype=UTF8"`

	// Inclusion stats
	IncludedAtBlockHeight  int64 `parquet:"name=includedAtBlockHeight, type=INT64"`
	IncludedBlockTimestamp int64 `parquet:"name=includedBlockTimestamp, type=INT64, convertedtype=TIMESTAMP_MILLIS"`
	InclusionDelayMs       int64 `parquet:"name=inclusionDelayMs, type=INT64"`
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.