Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DATA-3966 Change recent data to use hot data store type + pipeline support #4895

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

DATA-3966 Change recent data to use hot data store type + pipeline support #4895

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 24 additions & 8 deletions app/data_client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -277,9 +277,19 @@ type DataByFilterOptions struct {
IncludeInternalData bool
}

type TabularDataSource int32

const (
TabularDataSourceUnspecified TabularDataSource = iota
TabularDataSourceStandard
TabularDataSourceHotStorage
TabularDataSourcePipelineSink
)

// TabularDataByMQLOptions contains optional parameters for TabularDataByMQL.
type TabularDataByMQLOptions struct {
UseRecentData bool
DataSource TabularDataSource
PipelineID string
}

// BinaryDataCaptureUploadOptions represents optional parameters for the BinaryDataCaptureUpload method.
Expand Down Expand Up @@ -455,16 +465,22 @@ func (d *DataClient) TabularDataByMQL(
mqlBinary = append(mqlBinary, binary)
}

useRecentData := false
req := &pb.TabularDataByMQLRequest{
OrganizationId: organizationID,
MqlBinary: mqlBinary,
}
if opts != nil {
useRecentData = opts.UseRecentData
req.DataSource = &pb.TabularDataSource{
Type: pb.TabularDataSourceType(opts.DataSource),
PipelineId: &opts.PipelineID,
}
} else {
req.DataSource = &pb.TabularDataSource{
Type: pb.TabularDataSourceType_TABULAR_DATA_SOURCE_TYPE_STANDARD,
}
}

resp, err := d.dataClient.TabularDataByMQL(ctx, &pb.TabularDataByMQLRequest{
OrganizationId: organizationID,
MqlBinary: mqlBinary,
UseRecentData: &useRecentData,
})
resp, err := d.dataClient.TabularDataByMQL(ctx, req)
if err != nil {
return nil, err
}
Expand Down
4 changes: 2 additions & 2 deletions app/data_client_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -362,10 +362,10 @@ func TestDataClient(t *testing.T) {
response, err := client.TabularDataByMQL(context.Background(), organizationID, mqlQueries, nil)
test.That(t, err, test.ShouldBeNil)
test.That(t, response, test.ShouldResemble, rawData)
response, err = client.TabularDataByMQL(context.Background(), organizationID, mqlQueries, &TabularDataByMQLOptions{UseRecentData: false})
response, err = client.TabularDataByMQL(context.Background(), organizationID, mqlQueries, &TabularDataByMQLOptions{DataSource: TabularDataSourceStandard})
test.That(t, err, test.ShouldBeNil)
test.That(t, response, test.ShouldResemble, rawData)
response, err = client.TabularDataByMQL(context.Background(), organizationID, mqlQueries, &TabularDataByMQLOptions{UseRecentData: true})
response, err = client.TabularDataByMQL(context.Background(), organizationID, mqlQueries, &TabularDataByMQLOptions{DataSource: TabularDataSourceHotStorage})
test.That(t, err, test.ShouldBeNil)
test.That(t, response, test.ShouldResemble, rawData)
})
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ require (
go.uber.org/atomic v1.11.0
go.uber.org/multierr v1.11.0
go.uber.org/zap v1.27.0
go.viam.com/api v0.1.418
go.viam.com/api v0.1.424
go.viam.com/test v1.2.4
go.viam.com/utils v0.1.137
goji.io v2.0.2+incompatible
Expand Down
2 changes: 2 additions & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1524,6 +1524,8 @@ go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
go.viam.com/api v0.1.418 h1:XKDdwmG/4oZWU2RLuHj+mjfbw90umMKYw80Chw5fliI=
go.viam.com/api v0.1.418/go.mod h1:drvlBWaiHFxPziz5jayHvibez1qG7lylcNCC1LF8onU=
go.viam.com/api v0.1.424 h1:8MOooLs3aow7qvSOui/Qfm8DHGQlsSA6rmYe6vuft/U=
go.viam.com/api v0.1.424/go.mod h1:drvlBWaiHFxPziz5jayHvibez1qG7lylcNCC1LF8onU=
go.viam.com/test v1.2.4 h1:JYgZhsuGAQ8sL9jWkziAXN9VJJiKbjoi9BsO33TW3ug=
go.viam.com/test v1.2.4/go.mod h1:zI2xzosHdqXAJ/kFqcN+OIF78kQuTV2nIhGZ8EzvaJI=
go.viam.com/utils v0.1.137 h1:h7W9NpUIdEkWlm3OBvPTRRI2Ghx3EZ6PzN2sjVItb3A=
Expand Down
Loading