cmd

package
v0.5.14 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Apr 29, 2024 License: Apache-2.0, MIT Imports: 43 Imported by: 1

Documentation

Index

Constants

This section is empty.

Variables

View Source
var App = &cli.App{
	Name:  "singularity",
	Usage: "A tool for large-scale clients with PB-scale data onboarding to Filecoin network",
	Description: `Database Backend Support:
  Singularity supports multiple database backend: sqlite3, postgres, mysql5.7+
  Use '--database-connection-string' or $DATABASE_CONNECTION_STRING to specify the database connection string.
    Example for postgres  - postgres://user:pass@example.com:5432/dbname
    Example for mysql     - mysql://user:pass@tcp(localhost:3306)/dbname?parseTime=true
    Example for sqlite3   - sqlite:/absolute/path/to/database.db
                or        - sqlite:relative/path/to/database.db

Network Support:
  Default settings in Singularity are for Mainnet. You may set below environment values for other network:
    For Calibration network:
      * Set LOTUS_API to https://api.calibration.node.glif.io/rpc/v1
      * Set MARKET_DEAL_URL to https://marketdeals-calibration.s3.amazonaws.com/StateMarketDeals.json.zst
      * Set LOTUS_TEST to 1
    For all other networks:
      * Set LOTUS_API to your network's Lotus API endpoint
      * Set MARKET_DEAL_URL to empty string
      * Set LOTUS_TEST to 0 or 1 based on whether the network address starts with 'f' or 't'
    Switching between different networks with the same database instance is not recommended.

Logging:
  Singularity uses go-log for logging and can be controlled by below environment variables:
    * GOLOG_LOG_LEVEL  - example values: debug, info, warn, error, dpanic, panic, fatal
    * GOLOG_LOG_FMT    - example values: color, nocolor, json
    * More details can be found at https://github.com/ipfs/go-log

Upgrading:
  Within each minor version upgrade, use "singularity admin init" to upgrade the database schema.
  For major version upgrade, please refer to the release notes for upgrade instructions.
`,
	EnableBashCompletion: true,
	Flags: []cli.Flag{
		&cli.StringFlag{
			Name:        "database-connection-string",
			Usage:       "Connection string to the database",
			DefaultText: "sqlite:" + "./singularity.db",
			Value:       "sqlite:" + "./singularity.db",
			EnvVars:     []string{"DATABASE_CONNECTION_STRING"},
		},
		&cli.BoolFlag{
			Name:  "json",
			Usage: "Enable JSON output",
			Value: false,
		},
		&cli.BoolFlag{
			Name:  "verbose",
			Usage: "Enable verbose output. This will print more columns for the result as well as full error trace",
			Value: false,
		},
		&cli.StringFlag{
			Name:     "lotus-api",
			Category: "Lotus",
			Usage:    "Lotus RPC API endpoint",
			Value:    "https://api.node.glif.io/rpc/v1",
			EnvVars:  []string{"LOTUS_API"},
		},
		&cli.StringFlag{
			Name:     "lotus-token",
			Category: "Lotus",
			Usage:    "Lotus RPC API token",
			Value:    "",
			EnvVars:  []string{"LOTUS_TOKEN"},
		},
		&cli.BoolFlag{
			Name:     "lotus-test",
			Category: "Lotus",
			Usage:    "Whether the runtime environment is using Testnet.",
			EnvVars:  []string{"LOTUS_TEST"},
		},
	},
	Before: func(c *cli.Context) error {
		if c.Bool("lotus-test") {
			address.CurrentNetwork = address.Testnet
			logger.Infow("Current network is set to Testnet")
		} else {
			address.CurrentNetwork = address.Mainnet
		}
		return nil
	},
	Commands: []*cli.Command{
		ez.PrepCmd,
		VersionCmd,
		{
			Name:     "admin",
			Usage:    "Admin commands",
			Category: "Operations",
			Subcommands: []*cli.Command{
				admin.InitCmd,
				admin.ResetCmd,
				admin.MigrateDatasetCmd,
				admin.MigrateScheduleCmd,
			},
		},
		DownloadCmd,
		tool.ExtractCarCmd,
		{
			Name:     "deal",
			Usage:    "Replication / Deal making management",
			Category: "Operations",
			Subcommands: []*cli.Command{
				{
					Name:  "schedule",
					Usage: "Schedule deals",
					Subcommands: []*cli.Command{
						schedule.CreateCmd,
						schedule.ListCmd,
						schedule.UpdateCmd,
						schedule.PauseCmd,
						schedule.ResumeCmd,
						schedule.RemoveCmd,
					},
				},
				deal.SendManualCmd,
				deal.ListCmd,
			},
		},
		{
			Name:     "run",
			Category: "Daemons",
			Usage:    "run different singularity components",
			Subcommands: []*cli.Command{
				run.APICmd,
				run.DatasetWorkerCmd,
				run.ContentProviderCmd,
				run.DealTrackerCmd,
				run.DealPusherCmd,
				run.DownloadServerCmd,
			},
		},
		{
			Name:     "wallet",
			Category: "Operations",
			Usage:    "Wallet management",
			Subcommands: []*cli.Command{
				wallet.ImportCmd,
				wallet.ListCmd,
				wallet.RemoveCmd,
			},
		},
		{
			Name:     "storage",
			Category: "Operations",
			Usage:    "Create and manage storage system connections",
			Subcommands: []*cli.Command{
				storage.CreateCmd,
				storage.ExploreCmd,
				storage.ListCmd,
				storage.RemoveCmd,
				storage.UpdateCmd,
				storage.RenameCmd,
			},
		},
		{
			Name:     "prep",
			Category: "Operations",
			Usage:    "Create and manage dataset preparations",
			Subcommands: []*cli.Command{
				dataprep.CreateCmd,
				dataprep.ListCmd,
				dataprep.StatusCmd,
				dataprep.RenameCmd,
				dataprep.AttachSourceCmd,
				dataprep.AttachOutputCmd,
				dataprep.DetachOutputCmd,
				dataprep.StartScanCmd,
				dataprep.PauseScanCmd,
				dataprep.StartPackCmd,
				dataprep.PausePackCmd,
				dataprep.StartDagGenCmd,
				dataprep.PauseDagGenCmd,
				dataprep.ListPiecesCmd,
				dataprep.AddPieceCmd,
				dataprep.ExploreCmd,
				dataprep.AttachWalletCmd,
				dataprep.ListWalletsCmd,
				dataprep.DetachWalletCmd,
				dataprep.RemoveCmd,
			},
		},
	},
}
View Source
var DownloadCmd = &cli.Command{
	Name:      "download",
	Usage:     "Download a CAR file from the metadata API",
	Category:  "Utility",
	Before:    cliutil.CheckNArgs,
	ArgsUsage: "<piece_cid>",
	Flags: func() []cli.Flag {
		flags := []cli.Flag{
			&cli.StringFlag{
				Name:     "api",
				Usage:    "URL of the metadata API",
				Value:    "http://127.0.0.1:7777",
				Category: "General Config",
			},
			&cli.StringFlag{
				Name:     "out-dir",
				Usage:    "Directory to write CAR files to",
				Value:    ".",
				Category: "General Config",
			},
			&cli.IntFlag{
				Name:     "concurrency",
				Usage:    "Number of concurrent downloads",
				Value:    10,
				Category: "General Config",
			},
			&cli.BoolFlag{
				Name:     "quiet",
				Usage:    "Suppress all output",
				Category: "General Config",
				Value:    false,
			},
		}

		flags = append(flags, storage.HTTPClientConfigFlagsForUpdate...)
		flags = append(flags, storage.CommonConfigFlags...)

		keys := make(map[string]struct{})
		for _, backend := range storagesystem.Backends {
			for _, providerOptions := range backend.ProviderOptions {
				for _, option := range providerOptions.Options {
					if !model.IsSecretConfigName(option.Name) {
						continue
					}
					flag := option.ToCLIFlag(backend.Prefix+"-", false, backend.Description)
					if _, ok := keys[flag.Names()[0]]; ok {
						continue
					}
					keys[flag.Names()[0]] = struct{}{}
					flags = append(flags, flag)
				}
			}
		}
		return flags
	}(),
	Action: func(c *cli.Context) error {
		api := c.String("api")
		outDir := c.String("out-dir")
		concurrency := c.Int("concurrency")
		piece := c.Args().First()
		config := map[string]string{}
		for _, key := range c.LocalFlagNames() {
			if c.IsSet(key) {
				if slices.Contains([]string{"api", "out-dir", "concurrency", "quiet"}, key) {
					continue
				}
				value := c.String(key)
				config[key] = value
			}
		}
		clientConfig, err := storage.GetClientConfigForUpdate(c)
		if err != nil {
			return errors.WithStack(err)
		}
		err = handler.DownloadHandler(c, piece, api, config, *clientConfig, outDir, concurrency)
		if err == nil {
			log.Logger("Download").Info("Download complete")
			return nil
		}
		return errors.WithStack(err)
	},
}
View Source
var Version string
View Source
var VersionCmd = &cli.Command{
	Name:    "version",
	Usage:   "Print version information",
	Aliases: []string{"v"},
	Action: func(context *cli.Context) error {
		buildInfo, ok := debug.ReadBuildInfo()
		if !ok {
			fmt.Println("unknown version")
		}

		version := buildInfo.Main.Version
		if version == "(devel)" || version == "" {
			version = Version
		}
		var revision string
		var modified string
		for _, setting := range buildInfo.Settings {
			switch setting.Key {
			case "vcs.revision":
				revision = setting.Value[:7]
			case "vcs.modified":
				modified = setting.Value
			}
		}
		if revision == "" {
			revision = "-unknown"
		} else {
			revision = "-" + revision
		}
		switch modified {
		case "true":
			modified = "-dirty"
		case "false":
			modified = ""
		case "":
			modified = "-unknown"
		default:
			modified = "-" + modified
		}
		v := fmt.Sprintf("singularity %s%s%s\n", version, revision, modified)
		_, err := context.App.Writer.Write([]byte(v))

		return errors.WithStack(err)
	},
}

Functions

func CalculateCommp added in v0.4.0

func CalculateCommp(t *testing.T, content []byte, targetPieceSize uint64) string

func CompareDirectories added in v0.4.0

func CompareDirectories(t *testing.T, dir1, dir2 string)

func Download added in v0.4.0

func Download(ctx context.Context, url string, nThreads int) ([]byte, error)

func GetAllPieceCIDs added in v0.4.0

func GetAllPieceCIDs(content string) []string

func GetFirstCID added in v0.4.0

func GetFirstCID(content string) string

func SetVersionJSON added in v0.4.0

func SetVersionJSON(versionJSON []byte) error

func SetupErrorHandler added in v0.4.0

func SetupErrorHandler()

func SetupHelpPager added in v0.3.0

func SetupHelpPager()

func WaitForServerReady added in v0.4.0

func WaitForServerReady(ctx context.Context, url string) error

Types

type Runner added in v0.4.0

type Runner struct {
	// contains filtered or unexported fields
}

func NewRunner added in v0.4.0

func NewRunner() *Runner

NewRunner creates a new Runner to capture CLI args

Note: tests invoking this function should stay in cmd.Test package because this function relies on environment variables to set database connection string so it won't work with parallel execution of different test packages.

func (*Runner) Run added in v0.4.0

func (r *Runner) Run(ctx context.Context, args string) (string, string, error)

func (*Runner) Save added in v0.4.0

func (r *Runner) Save(t *testing.T, tempDirs ...string)

func (*Runner) WithMode added in v0.4.0

func (r *Runner) WithMode(mode RunnerMode) *Runner

type RunnerMode added in v0.4.0

type RunnerMode string
const (
	Normal  RunnerMode = "normal"
	Verbose RunnerMode = "verbose"
	JSON    RunnerMode = "json"
)

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL