Skip to content

Commit 0bee591

Browse files
SungJin1212SungJin1212
committed
Add rolling update e2e test
Signed-off-by: SungJin1212 <[email protected]>
1 parent e739b3d commit 0bee591

File tree

11 files changed

+130
-29
lines changed

11 files changed

+130
-29
lines changed

‎integration/remote_write_v2_test.go

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,112 @@ import (
2323
"github.com/cortexproject/cortex/pkg/storage/tsdb"
2424
)
2525

26+
func TestIngesterRollingUpdate(t *testing.T) {
27+
28+
const blockRangePeriod = 5 * time.Second
29+
nonPushV2SupportImage := "quay.io/cortexproject/cortex:v1.18.1"
30+
31+
s, err := e2e.NewScenario(networkName)
32+
require.NoError(t, err)
33+
defer s.Close()
34+
35+
// Start dependencies.
36+
consul := e2edb.NewConsulWithName("consul")
37+
require.NoError(t, s.StartAndWaitReady(consul))
38+
39+
flags := mergeFlags(
40+
AlertmanagerLocalFlags(),
41+
map[string]string{
42+
"-store.engine": blocksStorageEngine,
43+
"-blocks-storage.backend": "filesystem",
44+
"-blocks-storage.tsdb.head-compaction-interval": "4m",
45+
"-blocks-storage.bucket-store.sync-interval": "15m",
46+
"-blocks-storage.bucket-store.index-cache.backend": tsdb.IndexCacheBackendInMemory,
47+
"-blocks-storage.bucket-store.bucket-index.enabled": "true",
48+
"-querier.query-store-for-labels-enabled": "true",
49+
"-blocks-storage.tsdb.block-ranges-period": blockRangePeriod.String(),
50+
"-blocks-storage.tsdb.ship-interval": "1s",
51+
"-blocks-storage.tsdb.retention-period": ((blockRangePeriod * 2) - 1).String(),
52+
"-blocks-storage.tsdb.enable-native-histograms": "true",
53+
// Ingester.
54+
"-ring.store": "consul",
55+
"-consul.hostname": consul.NetworkHTTPEndpoint(),
56+
// Distributor.
57+
"-distributor.replication-factor": "1",
58+
// Store-gateway.
59+
"-store-gateway.sharding-enabled": "false",
60+
// alert manager
61+
"-alertmanager.web.external-url": "http://localhost/alertmanager",
62+
},
63+
)
64+
65+
// make alert manager config dir
66+
require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{}))
67+
68+
path := path.Join(s.SharedDir(), "cortex-1")
69+
70+
flags = mergeFlags(flags, map[string]string{"-blocks-storage.filesystem.dir": path})
71+
// Start Cortex replicas.
72+
// Start all other services.
73+
ingester := e2ecortex.NewIngester("ingester", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), flags, nonPushV2SupportImage)
74+
distributor := e2ecortex.NewDistributor("distributor", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), flags, "")
75+
storeGateway := e2ecortex.NewStoreGateway("store-gateway", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), flags, "")
76+
querier := e2ecortex.NewQuerier("querier", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), mergeFlags(flags, map[string]string{
77+
"-querier.store-gateway-addresses": storeGateway.NetworkGRPCEndpoint()}), "")
78+
79+
require.NoError(t, s.StartAndWaitReady(querier, ingester, distributor, storeGateway))
80+
81+
// Wait until Cortex replicas have updated the ring state.
82+
require.NoError(t, distributor.WaitSumMetrics(e2e.Equals(512), "cortex_ring_tokens_total"))
83+
require.NoError(t, querier.WaitSumMetrics(e2e.Equals(512), "cortex_ring_tokens_total"))
84+
85+
c, err := e2ecortex.NewClient(distributor.HTTPEndpoint(), querier.HTTPEndpoint(), "", "", "user-1")
86+
require.NoError(t, err)
87+
88+
now := time.Now()
89+
90+
// series push
91+
symbols1, series, expectedVector := e2e.GenerateSeriesV2("test_series", now, prompb.Label{Name: "job", Value: "test"}, prompb.Label{Name: "foo", Value: "bar"})
92+
res, err := c.PushV2(symbols1, series)
93+
require.NoError(t, err)
94+
require.Equal(t, 200, res.StatusCode)
95+
96+
// sample
97+
result, err := c.Query("test_series", now)
98+
require.NoError(t, err)
99+
assert.Equal(t, expectedVector, result.(model.Vector))
100+
101+
// metadata
102+
metadata, err := c.Metadata("test_series", "")
103+
require.NoError(t, err)
104+
require.Equal(t, 1, len(metadata["test_series"]))
105+
106+
// histogram
107+
histogramIdx := rand.Uint32()
108+
symbols2, histogramSeries := e2e.GenerateHistogramSeriesV2("test_histogram", now, histogramIdx, false, prompb.Label{Name: "job", Value: "test"}, prompb.Label{Name: "float", Value: "false"})
109+
res, err = c.PushV2(symbols2, histogramSeries)
110+
require.NoError(t, err)
111+
require.Equal(t, 200, res.StatusCode)
112+
113+
symbols3, histogramFloatSeries := e2e.GenerateHistogramSeriesV2("test_histogram", now, histogramIdx, false, prompb.Label{Name: "job", Value: "test"}, prompb.Label{Name: "float", Value: "true"})
114+
res, err = c.PushV2(symbols3, histogramFloatSeries)
115+
require.NoError(t, err)
116+
require.Equal(t, 200, res.StatusCode)
117+
118+
testHistogramTimestamp := now.Add(blockRangePeriod * 2)
119+
expectedHistogram := tsdbutil.GenerateTestHistogram(int(histogramIdx))
120+
result, err = c.Query(`test_histogram`, testHistogramTimestamp)
121+
require.NoError(t, err)
122+
require.Equal(t, model.ValVector, result.Type())
123+
v := result.(model.Vector)
124+
require.Equal(t, 2, v.Len())
125+
for _, s := range v {
126+
require.NotNil(t, s.Histogram)
127+
require.Equal(t, float64(expectedHistogram.Count), float64(s.Histogram.Count))
128+
require.Equal(t, float64(expectedHistogram.Sum), float64(s.Histogram.Sum))
129+
}
130+
}
131+
26132
func TestIngest(t *testing.T) {
27133
const blockRangePeriod = 5 * time.Second
28134

‎pkg/cortex/modules_test.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@ import (
1313
"github.com/weaveworks/common/server"
1414

1515
"github.com/cortexproject/cortex/pkg/cortexpb"
16-
"github.com/cortexproject/cortex/pkg/cortexpbv2"
1716
)
1817

1918
func changeTargetConfig(c *Config) {
@@ -161,7 +160,7 @@ func TestCortex_InitRulerStorage(t *testing.T) {
161160

162161
type myPusher struct{}
163162

164-
func (p *myPusher) PushV2(ctx context.Context, req *cortexpbv2.WriteRequest) (*cortexpbv2.WriteResponse, error) {
163+
func (p *myPusher) PushV2(ctx context.Context, req *cortexpb.WriteRequestV2) (*cortexpb.WriteResponseV2, error) {
165164
return nil, nil
166165
}
167166

‎pkg/cortexpb/cortex.proto

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ enum MetricType {
2828
// https://github.com/prometheus/prometheus/blob/main/prompb/io/prometheus/write/v2/types.proto
2929
message WriteRequestV2 {
3030
reserved 1 to 2;
31-
cortexpb.SourceEnum Source = 3;
31+
SourceEnum Source = 3;
3232
repeated string symbols = 4;
3333
repeated TimeSeriesV2 timeseries = 5 [(gogoproto.nullable) = false, (gogoproto.customtype) = "PreallocTimeseriesV2"];
3434

‎pkg/cortexpb/slicesPool.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ type byteSlicePools struct {
1313
pools []sync.Pool
1414
}
1515

16-
func NewSlicePool(pools int) *byteSlicePools {
16+
func newSlicePool(pools int) *byteSlicePools {
1717
sp := byteSlicePools{}
1818
sp.init(pools)
1919
return &sp
@@ -32,7 +32,7 @@ func (sp *byteSlicePools) init(pools int) {
3232
}
3333
}
3434

35-
func (sp *byteSlicePools) GetSlice(size int) *[]byte {
35+
func (sp *byteSlicePools) getSlice(size int) *[]byte {
3636
index := int(math.Ceil(math.Log2(float64(size)))) - minPoolSizePower
3737

3838
if index >= len(sp.pools) {
@@ -50,7 +50,7 @@ func (sp *byteSlicePools) GetSlice(size int) *[]byte {
5050
return s
5151
}
5252

53-
func (sp *byteSlicePools) ReuseSlice(s *[]byte) {
53+
func (sp *byteSlicePools) reuseSlice(s *[]byte) {
5454
index := int(math.Floor(math.Log2(float64(cap(*s))))) - minPoolSizePower
5555

5656
if index >= len(sp.pools) || index < 0 {

‎pkg/cortexpb/slicesPool_test.go

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,22 +9,22 @@ import (
99
)
1010

1111
func TestFuzzyByteSlicePools(t *testing.T) {
12-
sut := NewSlicePool(20)
12+
sut := newSlicePool(20)
1313
maxByteSize := int(math.Pow(2, 20+minPoolSizePower-1))
1414

1515
for i := 0; i < 1000; i++ {
1616
size := rand.Int() % maxByteSize
17-
s := sut.GetSlice(size)
17+
s := sut.getSlice(size)
1818
assert.Equal(t, len(*s), size)
19-
sut.ReuseSlice(s)
19+
sut.reuseSlice(s)
2020
}
2121
}
2222

2323
func TestReturnSliceSmallerThanMin(t *testing.T) {
24-
sut := NewSlicePool(20)
24+
sut := newSlicePool(20)
2525
size := 3
2626
buff := make([]byte, 0, size)
27-
sut.ReuseSlice(&buff)
28-
buff2 := sut.GetSlice(size * 2)
27+
sut.reuseSlice(&buff)
28+
buff2 := sut.getSlice(size * 2)
2929
assert.Equal(t, len(*buff2), size*2)
3030
}

‎pkg/cortexpb/timeseries.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ var (
4747
}
4848
},
4949
}
50-
bytePool = NewSlicePool(20)
50+
bytePool = newSlicePool(20)
5151
)
5252

5353
// PreallocConfig configures how structures will be preallocated to optimise
@@ -86,7 +86,7 @@ func (p *PreallocTimeseries) Unmarshal(dAtA []byte) error {
8686

8787
func (p *PreallocWriteRequest) Marshal() (dAtA []byte, err error) {
8888
size := p.Size()
89-
p.data = bytePool.GetSlice(size)
89+
p.data = bytePool.getSlice(size)
9090
dAtA = *p.data
9191
n, err := p.MarshalToSizedBuffer(dAtA[:size])
9292
if err != nil {
@@ -97,7 +97,7 @@ func (p *PreallocWriteRequest) Marshal() (dAtA []byte, err error) {
9797

9898
func ReuseWriteRequest(req *PreallocWriteRequest) {
9999
if req.data != nil {
100-
bytePool.ReuseSlice(req.data)
100+
bytePool.reuseSlice(req.data)
101101
req.data = nil
102102
}
103103
req.Source = 0

‎pkg/cortexpb/timeseriesv2.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ var (
3434
}
3535
},
3636
}
37-
bytePoolV2 = NewSlicePool(20)
37+
bytePoolV2 = newSlicePool(20)
3838
)
3939

4040
// PreallocWriteRequestV2 is a WriteRequest which preallocs slices on Unmarshal.
@@ -51,7 +51,7 @@ func (p *PreallocWriteRequestV2) Unmarshal(dAtA []byte) error {
5151

5252
func (p *PreallocWriteRequestV2) Marshal() (dAtA []byte, err error) {
5353
size := p.Size()
54-
p.data = bytePool.GetSlice(size)
54+
p.data = bytePool.getSlice(size)
5555
dAtA = *p.data
5656
n, err := p.MarshalToSizedBuffer(dAtA[:size])
5757
if err != nil {
@@ -73,7 +73,7 @@ func (p *PreallocTimeseriesV2) Unmarshal(dAtA []byte) error {
7373

7474
func ReuseWriteRequestV2(req *PreallocWriteRequestV2) {
7575
if req.data != nil {
76-
bytePoolV2.ReuseSlice(req.data)
76+
bytePoolV2.reuseSlice(req.data)
7777
req.data = nil
7878
}
7979
req.Source = 0

‎pkg/ingester/client/cortex_mock_test.go

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,16 +6,15 @@ import (
66
"github.com/stretchr/testify/mock"
77

88
"github.com/cortexproject/cortex/pkg/cortexpb"
9-
"github.com/cortexproject/cortex/pkg/cortexpbv2"
109
)
1110

1211
type IngesterServerMock struct {
1312
mock.Mock
1413
}
1514

16-
func (m *IngesterServerMock) PushV2(ctx context.Context, r *cortexpbv2.WriteRequest) (*cortexpbv2.WriteResponse, error) {
15+
func (m *IngesterServerMock) PushV2(ctx context.Context, r *cortexpb.WriteRequestV2) (*cortexpb.WriteResponseV2, error) {
1716
args := m.Called(ctx, r)
18-
return args.Get(0).(*cortexpbv2.WriteResponse), args.Error(1)
17+
return args.Get(0).(*cortexpb.WriteResponseV2), args.Error(1)
1918
}
2019

2120
func (m *IngesterServerMock) Push(ctx context.Context, r *cortexpb.WriteRequest) (*cortexpb.WriteResponse, error) {

‎pkg/querier/tripperware/query.pb.go

Lines changed: 0 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/ruler/compat_test.go

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,20 +23,19 @@ import (
2323
"github.com/weaveworks/common/httpgrpc"
2424

2525
"github.com/cortexproject/cortex/pkg/cortexpb"
26-
"github.com/cortexproject/cortex/pkg/cortexpbv2"
2726
"github.com/cortexproject/cortex/pkg/querier/stats"
2827
"github.com/cortexproject/cortex/pkg/util/validation"
2928
)
3029

3130
type fakePusher struct {
3231
request *cortexpb.WriteRequest
33-
requestV2 *cortexpbv2.WriteRequest
32+
requestV2 *cortexpb.WriteRequestV2
3433
response *cortexpb.WriteResponse
35-
responseV2 *cortexpbv2.WriteResponse
34+
responseV2 *cortexpb.WriteResponseV2
3635
err error
3736
}
3837

39-
func (p *fakePusher) PushV2(ctx context.Context, r *cortexpbv2.WriteRequest) (*cortexpbv2.WriteResponse, error) {
38+
func (p *fakePusher) PushV2(ctx context.Context, r *cortexpb.WriteRequestV2) (*cortexpb.WriteResponseV2, error) {
4039
p.requestV2 = r
4140
return p.responseV2, p.err
4241
}

‎pkg/ruler/pusher_mock_test.go

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ import (
66
"github.com/stretchr/testify/mock"
77

88
"github.com/cortexproject/cortex/pkg/cortexpb"
9-
"github.com/cortexproject/cortex/pkg/cortexpbv2"
109
)
1110

1211
type pusherMock struct {
@@ -17,9 +16,9 @@ func newPusherMock() *pusherMock {
1716
return &pusherMock{}
1817
}
1918

20-
func (m *pusherMock) PushV2(ctx context.Context, req *cortexpbv2.WriteRequest) (*cortexpbv2.WriteResponse, error) {
19+
func (m *pusherMock) PushV2(ctx context.Context, req *cortexpb.WriteRequestV2) (*cortexpb.WriteResponseV2, error) {
2120
args := m.Called(ctx, req)
22-
return args.Get(0).(*cortexpbv2.WriteResponse), args.Error(1)
21+
return args.Get(0).(*cortexpb.WriteResponseV2), args.Error(1)
2322
}
2423

2524
func (m *pusherMock) Push(ctx context.Context, req *cortexpb.WriteRequest) (*cortexpb.WriteResponse, error) {

0 commit comments

Comments
 (0)