DATA-3918 Data pipelines CLI commands

cli/app.go

@@ -53,6 +53,7 @@ const (
 	generalFlagPart              = "part"
 	generalFlagPartName          = "part-name"
 	generalFlagPartID            = "part-id"
+	generalFlagID                = "id"
 	generalFlagName              = "name"
 	generalFlagMethod            = "method"
 	generalFlagDestination       = "destination"
@@ -74,7 +75,6 @@ const (
 	moduleFlagLocal           = "local"
 	moduleFlagHomeDir         = "home"
 	moduleCreateLocalOnly     = "local-only"
-	moduleFlagID              = "id"
 	moduleFlagIsPublic        = "public"
 	moduleFlagResourceType    = "resource-type"
 	moduleFlagModelName       = "model-name"
@@ -117,6 +117,10 @@ const (
 	dataFlagFilterTags                     = "filter-tags"
 	dataFlagTimeout                        = "timeout"
 
+	datapipelineFlagSchedule = "schedule"
+	datapipelineFlagMQL      = "mql"
+	datapipelineFlagMQLFile  = "mql-path"
+
 	packageFlagFramework = "model-framework"
 
 	oauthAppFlagClientID             = "client-id"
@@ -1508,6 +1512,150 @@ var app = &cli.App{
 				},
 			},
 		},
+		{
+			Name:            "datapipelines",
+			Usage:           "manage and track data pipelines",
+			UsageText:       createUsageText("datapipelines", nil, false, true),
+			HideHelpCommand: true,
+			Subcommands: []*cli.Command{
+				{
+					Name:  "list",
+					Usage: "list data pipelines for an organization ID",
+					UsageText: createUsageText("datapipelines list",
+						[]string{generalFlagOrgID}, true, false),
+					Flags: []cli.Flag{
+						&cli.StringFlag{
+							Name:     generalFlagOrgID,
+							Usage:    "organization ID for which data pipelines will be listed",
+							Required: true,
+						},
+					},
+					Action: createCommandWithT[datapipelineListArgs](DatapipelineListAction),
+				},
+				{
+					Name:      "describe",
+					Usage:     "describe a data pipeline and its status",
+					UsageText: createUsageText("datapipelines describe", []string{generalFlagID}, true, false),
+					Flags: []cli.Flag{
+						&cli.StringFlag{
+							Name:     generalFlagID,
+							Usage:    "ID of the data pipeline to describe",
+							Required: true,
+						},
+					},
+					Action: createCommandWithT[datapipelineDescribeArgs](DatapipelineDescribeAction),
+				},
+				{
+					Name:  "create",
+					Usage: "create a new data pipeline",
+					UsageText: createUsageText("datapipelines create",
+						[]string{generalFlagOrgID, generalFlagName, datapipelineFlagSchedule}, false, false,
+						fmt.Sprintf("[--%s=<%s> | --%s=<%s>]",
+							datapipelineFlagMQL, datapipelineFlagMQL,
+							datapipelineFlagMQLFile, datapipelineFlagMQLFile),
+					),
+					Flags: []cli.Flag{
+						&cli.StringFlag{
+							Name:     generalFlagOrgID,
+							Usage:    "organization ID for which data pipeline will be created",
+							Required: true,
+						},
+						&cli.StringFlag{
+							Name:     generalFlagName,
+							Usage:    "name of the new data pipeline",
+							Required: true,
+						},
+						&cli.StringFlag{
+							Name:     datapipelineFlagSchedule,
+							Usage:    "schedule of the new data pipeline (cron expression)",
+							Required: true,
+						},
+						&cli.StringFlag{
+							Name:  datapipelineFlagMQL,
+							Usage: "MQL query for the new data pipeline",
+						},
+						&cli.StringFlag{
+							Name:  datapipelineFlagMQLFile,
+							Usage: "path to JSON file containing MQL query for the new data pipeline",
+						},
+					},
+					Action: createCommandWithT[datapipelineCreateArgs](DatapipelineCreateAction),
+				},
+				{
+					Name:  "update",
+					Usage: "update a data pipeline",
+					UsageText: createUsageText("datapipelines update",
+						[]string{generalFlagID, generalFlagName, datapipelineFlagSchedule}, false, false,
+						fmt.Sprintf("[--%s=<%s> | --%s=<%s>]",
+							datapipelineFlagMQL, datapipelineFlagMQL,
+							datapipelineFlagMQLFile, datapipelineFlagMQLFile),
+					),
+					Flags: []cli.Flag{
+						&cli.StringFlag{
+							Name:     generalFlagID,
+							Usage:    "ID of the data pipeline to update",
+							Required: true,
+						},
+						&cli.StringFlag{
+							Name:  generalFlagName,
+							Usage: "name of the data pipeline to update",
+						},
+						&cli.StringFlag{
+							Name:  datapipelineFlagSchedule,
+							Usage: "schedule of the data pipeline to update (cron expression)",
+						},
+						&cli.StringFlag{
+							Name:  datapipelineFlagMQL,
+							Usage: "MQL query for the data pipeline to update",
+						},
+						&cli.StringFlag{
+							Name:  datapipelineFlagMQLFile,
+							Usage: "path to JSON file containing MQL query for the data pipeline to update",
+						},
+					},
+					Action: createCommandWithT[datapipelineUpdateArgs](DatapipelineUpdateAction),
+				},
+				{
+					Name:      "delete",
+					Usage:     "delete a data pipeline",
+					UsageText: createUsageText("datapipelines delete", []string{generalFlagID}, true, false),
+					Flags: []cli.Flag{
+						&cli.StringFlag{
+							Name:     generalFlagID,
+							Usage:    "ID of the data pipeline to delete",
+							Required: true,
+						},
+					},
+					Action: createCommandWithT[datapipelineDeleteArgs](DatapipelineDeleteAction),
+				},
+				{
+					Name:      "enable",
+					Usage:     "enable a data pipeline",
+					UsageText: createUsageText("datapipelines enable", []string{generalFlagID}, true, false),
+					Flags: []cli.Flag{
+						&cli.StringFlag{
+							Name:     generalFlagID,
+							Usage:    "ID of the data pipeline to enable",
+							Required: true,
+						},
+					},
+					Action: createCommandWithT[datapipelineEnableArgs](DatapipelineEnableAction),
+				},
+				{
+					Name:      "disable",
+					Usage:     "disable a data pipeline",
+					UsageText: createUsageText("datapipelines disable", []string{generalFlagID}, true, false),
+					Flags: []cli.Flag{
+						&cli.StringFlag{
+							Name:     generalFlagID,
+							Usage:    "ID of the data pipeline to disable",
+							Required: true,
+						},
+					},
+					Action: createCommandWithT[datapipelineDisableArgs](DatapipelineDisableAction),
+				},
+			},
+		},
 		{
 			Name:            "train",
 			Usage:           "train on data",
@@ -2550,7 +2698,7 @@ Example:
 									DefaultText: "all",
 								},
 								&cli.StringFlag{
-									Name:  moduleFlagID,
+									Name:  generalFlagID,
 									Usage: "restrict output to just return builds that match this id",
 								},
 							},
@@ -2560,10 +2708,10 @@ Example:
 							Name:      "logs",
 							Aliases:   []string{"log"},
 							Usage:     "get the logs from one of your cloud builds",
-							UsageText: createUsageText("module build logs", []string{moduleFlagID}, true, false),
+							UsageText: createUsageText("module build logs", []string{generalFlagID}, true, false),
 							Flags: []cli.Flag{
 								&cli.StringFlag{
-									Name:     moduleFlagID,
+									Name:     generalFlagID,
 									Usage:    "build that you want to get the logs for",
 									Required: true,
 								},
@@ -2645,7 +2793,7 @@ This won't work unless you have an existing installation of our GitHub app on yo
 							Usage: "name of module to restart. pass at most one of --name, --id",
 						},
 						&cli.StringFlag{
-							Name:  moduleFlagID,
+							Name:  generalFlagID,
 							Usage: "ID of module to restart, for example viam:wifi-sensor. pass at most one of --name, --id",
 						},
 						&cli.BoolFlag{
@@ -2679,7 +2827,7 @@ This won't work unless you have an existing installation of our GitHub app on yo
 							Value: ".",
 						},
 						&cli.StringFlag{
-							Name:        moduleFlagID,
+							Name:        generalFlagID,
 							Usage:       "module ID as org-id:name or namespace:name",
 							DefaultText: "will try to read from meta.json",
 						},

cli/auth.go

@@ -20,6 +20,7 @@ import (
 	"go.uber.org/multierr"
 	buildpb "go.viam.com/api/app/build/v1"
 	datapb "go.viam.com/api/app/data/v1"
+	datapipelinespb "go.viam.com/api/app/datapipelines/v1"
 	datasetpb "go.viam.com/api/app/dataset/v1"
 	mlinferencepb "go.viam.com/api/app/mlinference/v1"
 	mltrainingpb "go.viam.com/api/app/mltraining/v1"
@@ -528,6 +529,7 @@ func (c *viamClient) ensureLoggedInInner() error {
 	c.dataClient = datapb.NewDataServiceClient(conn)
 	c.packageClient = packagepb.NewPackageServiceClient(conn)
 	c.datasetClient = datasetpb.NewDatasetServiceClient(conn)
+	c.datapipelinesClient = datapipelinespb.NewDataPipelinesServiceClient(conn)
 	c.mlTrainingClient = mltrainingpb.NewMLTrainingServiceClient(conn)
 	c.mlInferenceClient = mlinferencepb.NewMLInferenceServiceClient(conn)
 	c.buildClient = buildpb.NewBuildServiceClient(conn)

cli/client.go

@@ -31,6 +31,7 @@ import (
 	"go.uber.org/zap"
 	buildpb "go.viam.com/api/app/build/v1"
 	datapb "go.viam.com/api/app/data/v1"
+	datapipelinespb "go.viam.com/api/app/datapipelines/v1"
 	datasetpb "go.viam.com/api/app/dataset/v1"
 	mlinferencepb "go.viam.com/api/app/mlinference/v1"
 	mltrainingpb "go.viam.com/api/app/mltraining/v1"
@@ -79,17 +80,18 @@ var errNoShellService = errors.New("shell service is not enabled on this machine
 // viamClient wraps a cli.Context and provides all the CLI command functionality
 // needed to talk to the app and data services but not directly to robot parts.
 type viamClient struct {
-	c                 *cli.Context
-	conf              *Config
-	client            apppb.AppServiceClient
-	dataClient        datapb.DataServiceClient
-	packageClient     packagepb.PackageServiceClient
-	datasetClient     datasetpb.DatasetServiceClient
-	mlTrainingClient  mltrainingpb.MLTrainingServiceClient
-	mlInferenceClient mlinferencepb.MLInferenceServiceClient
-	buildClient       buildpb.BuildServiceClient
-	baseURL           *url.URL
-	authFlow          *authFlow
+	c                   *cli.Context
+	conf                *Config
+	client              apppb.AppServiceClient
+	dataClient          datapb.DataServiceClient
+	packageClient       packagepb.PackageServiceClient
+	datasetClient       datasetpb.DatasetServiceClient
+	datapipelinesClient datapipelinespb.DataPipelinesServiceClient
+	mlTrainingClient    mltrainingpb.MLTrainingServiceClient
+	mlInferenceClient   mlinferencepb.MLInferenceServiceClient
+	buildClient         buildpb.BuildServiceClient
+	baseURL             *url.URL
+	authFlow            *authFlow
 
 	selectedOrg *apppb.Organization
 	selectedLoc *apppb.Location