Skip to content

Commit 222da17

Browse files
xumengpandaxumengpanda
committed
Merge branch 'release-6.2' into mengxu/ha-code-read
2 parents 046a6e8 + 07e354c commit 222da17

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+1013
-241
lines changed

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,3 +89,4 @@ flow/coveragetool/obj
8989
temp/
9090
/compile_commands.json
9191
/.ccls-cache
92+
.clangd/

‎build/Dockerfile

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,10 @@ RUN yum install -y yum-utils &&\
88
http://opensource.wandisco.com/centos/6/git/x86_64/wandisco-git-release-6-1.noarch.rpm &&\
99
yum -y install devtoolset-8-8.1-1.el6 java-1.8.0-openjdk-devel \
1010
devtoolset-8-gcc-8.3.1 devtoolset-8-gcc-c++-8.3.1 \
11-
devtoolset-8-libubsan-devel devtoolset-8-valgrind-devel \
11+
devtoolset-8-libubsan-devel devtoolset-8-libasan-devel devtoolset-8-valgrind-devel \
1212
rh-python36-python-devel rh-ruby24 golang python27 rpm-build \
1313
mono-core debbuild python-pip dos2unix valgrind-devel ccache \
14-
distcc wget git &&\
14+
distcc wget git lz4 lz4-devel lz4-static &&\
1515
pip install boto3==1.1.1
1616

1717
USER root
@@ -61,8 +61,8 @@ RUN cd /opt/ && curl -L https://github.com/facebook/rocksdb/archive/v6.10.1.tar.
6161
ARG TIMEZONEINFO=America/Los_Angeles
6262
RUN rm -f /etc/localtime && ln -s /usr/share/zoneinfo/${TIMEZONEINFO} /etc/localtime
6363

64-
LABEL version=0.1.17
65-
ENV DOCKER_IMAGEVER=0.1.17
64+
LABEL version=0.1.19
65+
ENV DOCKER_IMAGEVER=0.1.19
6666
ENV JAVA_HOME=/usr/lib/jvm/java-1.8.0
6767
ENV CC=/opt/rh/devtoolset-8/root/usr/bin/gcc
6868
ENV CXX=/opt/rh/devtoolset-8/root/usr/bin/g++

‎build/Dockerfile.devel

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM foundationdb/foundationdb-build:0.1.17
1+
FROM foundationdb/foundationdb-build:0.1.19
22

33
USER root
44

@@ -50,8 +50,8 @@ RUN cp -iv /usr/local/bin/clang++ /usr/local/bin/clang++.deref &&\
5050
ldconfig &&\
5151
rm -rf /mnt/artifacts
5252

53-
LABEL version=0.11.9
54-
ENV DOCKER_IMAGEVER=0.11.9
53+
LABEL version=0.11.10
54+
ENV DOCKER_IMAGEVER=0.11.10
5555

5656
ENV CLANGCC=/usr/local/bin/clang.de8a65ef
5757
ENV CLANGCXX=/usr/local/bin/clang++.de8a65ef

‎build/docker-compose.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ version: "3"
22

33
services:
44
common: &common
5-
image: foundationdb/foundationdb-build:0.1.17
5+
image: foundationdb/foundationdb-build:0.1.19
66

77
build-setup: &build-setup
88
<<: *common

‎cmake/ConfigureCompiler.cmake

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -233,7 +233,6 @@ else()
233233
-Wno-unused-function
234234
-Wno-unused-local-typedef
235235
-Wno-unused-parameter
236-
-Wno-unused-value
237236
)
238237
if (USE_CCACHE)
239238
add_compile_options(

‎documentation/sphinx/source/release-notes/release-notes-620.rst

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,12 @@
44
Release Notes
55
#############
66

7+
6.2.28
8+
======
9+
* Log detailed team collection information when median available space ratio of all teams is too low. `(PR #3912) <https://github.com/apple/foundationdb/pull/3912>`_
10+
* Bug fix, blob client did not support authentication key sizes over 64 bytes. `(PR #3964) <https://github.com/apple/foundationdb/pull/3964>`_
11+
12+
713
6.2.27
814
======
915
* For clusters with a large number of shards, avoid slow tasks in the data distributor by adding yields to the shard map destruction. `(PR #3834) <https://github.com/apple/foundationdb/pull/3834>`_

‎fdbclient/BlobStore.actor.cpp

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -873,7 +873,12 @@ Future<BlobStoreEndpoint::ListResult> BlobStoreEndpoint::listBucket(std::string
873873
std::string BlobStoreEndpoint::hmac_sha1(std::string const &msg) {
874874
std::string key = secret;
875875

876-
// First pad the key to 64 bytes.
876+
// Hash key to shorten it if it is longer than SHA1 block size
877+
if(key.size() > 64) {
878+
key = SHA1::from_string(key);
879+
}
880+
881+
// Pad key up to SHA1 block size if needed
877882
key.append(64 - key.size(), '\0');
878883

879884
std::string kipad = key;

‎fdbclient/FDBTypes.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,10 @@ static std::string describe( Reference<T> const& item ) {
143143
return item->toString();
144144
}
145145

146+
static std::string describe(UID const& item) {
147+
return item.shortString();
148+
}
149+
146150
template <class T>
147151
static std::string describe( T const& item ) {
148152
return item.toString();

‎fdbrpc/FlowTransport.actor.cpp

Lines changed: 31 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -220,18 +220,36 @@ ACTOR Future<Void> pingLatencyLogger(TransportData* self) {
220220
if(!peer) {
221221
TraceEvent(SevWarnAlways, "MissingNetworkAddress").suppressFor(10.0).detail("PeerAddr", lastAddress);
222222
}
223+
if (peer->lastLoggedTime <= 0.0) {
224+
peer->lastLoggedTime = peer->lastConnectTime;
225+
}
226+
223227
if(peer && peer->pingLatencies.getPopulationSize() >= 10) {
224228
TraceEvent("PingLatency")
225-
.detail("PeerAddr", lastAddress)
226-
.detail("MinLatency", peer->pingLatencies.min())
227-
.detail("MaxLatency", peer->pingLatencies.max())
228-
.detail("MeanLatency", peer->pingLatencies.mean())
229-
.detail("MedianLatency", peer->pingLatencies.median())
230-
.detail("P90Latency", peer->pingLatencies.percentile(0.90))
231-
.detail("Count", peer->pingLatencies.getPopulationSize())
232-
.detail("BytesReceived", peer->bytesReceived - peer->lastLoggedBytesReceived)
233-
.detail("BytesSent", peer->bytesSent - peer->lastLoggedBytesSent);
229+
.detail("Elapsed", now() - peer->lastLoggedTime)
230+
.detail("PeerAddr", lastAddress)
231+
.detail("MinLatency", peer->pingLatencies.min())
232+
.detail("MaxLatency", peer->pingLatencies.max())
233+
.detail("MeanLatency", peer->pingLatencies.mean())
234+
.detail("MedianLatency", peer->pingLatencies.median())
235+
.detail("P90Latency", peer->pingLatencies.percentile(0.90))
236+
.detail("Count", peer->pingLatencies.getPopulationSize())
237+
.detail("BytesReceived", peer->bytesReceived - peer->lastLoggedBytesReceived)
238+
.detail("BytesSent", peer->bytesSent - peer->lastLoggedBytesSent)
239+
.detail("ConnectOutgoingCount", peer->connectOutgoingCount)
240+
.detail("ConnectIncomingCount", peer->connectIncomingCount)
241+
.detail("ConnectFailedCount", peer->connectFailedCount)
242+
.detail("ConnectMinLatency", peer->connectLatencies.min())
243+
.detail("ConnectMaxLatency", peer->connectLatencies.max())
244+
.detail("ConnectMeanLatency", peer->connectLatencies.mean())
245+
.detail("ConnectMedianLatency", peer->connectLatencies.median())
246+
.detail("ConnectP90Latency", peer->connectLatencies.percentile(0.90));
247+
peer->lastLoggedTime = now();
248+
peer->connectOutgoingCount = 0;
249+
peer->connectIncomingCount = 0;
250+
peer->connectFailedCount = 0;
234251
peer->pingLatencies.clear();
252+
peer->connectLatencies.clear();
235253
peer->lastLoggedBytesReceived = peer->bytesReceived;
236254
peer->lastLoggedBytesSent = peer->bytesSent;
237255
wait(delay(FLOW_KNOBS->PING_LOGGING_INTERVAL));
@@ -476,14 +494,15 @@ ACTOR Future<Void> connectionKeeper( Reference<Peer> self,
476494
std::max(0.0, self->lastConnectTime + self->reconnectionDelay -
477495
now()))); // Don't connect() to the same peer more than once per 2 sec
478496
self->lastConnectTime = now();
479-
497+
++self->connectOutgoingCount;
480498
TraceEvent("ConnectingTo", conn ? conn->getDebugID() : UID()).suppressFor(1.0).detail("PeerAddr", self->destination);
481499

482500
try {
483501
choose {
484502
when( Reference<IConnection> _conn = wait( INetworkConnections::net()->connect(self->destination) ) ) {
485503
conn = _conn;
486504
wait(conn->connectHandshake());
505+
self->connectLatencies.addSample(now() - self->lastConnectTime);
487506
if (FlowTransport::isClient()) {
488507
IFailureMonitor::failureMonitor().setStatus(self->destination, FailureStatus(false));
489508
}
@@ -505,6 +524,7 @@ ACTOR Future<Void> connectionKeeper( Reference<Peer> self,
505524
}
506525
}
507526
} catch( Error &e ) {
527+
++self->connectFailedCount;
508528
if(e.code() != error_code_connection_failed) {
509529
throw;
510530
}
@@ -648,6 +668,7 @@ void Peer::discardUnreliablePackets() {
648668
void Peer::onIncomingConnection( Reference<Peer> self, Reference<IConnection> conn, Future<Void> reader ) {
649669
// In case two processes are trying to connect to each other simultaneously, the process with the larger canonical NetworkAddress
650670
// gets to keep its outgoing connection.
671+
++self->connectIncomingCount;
651672
if ( !destination.isPublic() && !outgoingConnectionIdle ) throw address_in_use();
652673
NetworkAddress compatibleAddr = transport->localAddresses.address;
653674
if(transport->localAddresses.secondaryAddress.present() && transport->localAddresses.secondaryAddress.get().isTLS() == destination.isTLS()) {

‎fdbrpc/FlowTransport.h

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -127,15 +127,23 @@ struct Peer : public ReferenceCounted<Peer> {
127127
double lastDataPacketSentTime;
128128
int outstandingReplies;
129129
ContinuousSample<double> pingLatencies;
130+
double lastLoggedTime;
130131
int64_t lastLoggedBytesReceived;
131132
int64_t lastLoggedBytesSent;
132133

134+
// Cleared every time stats are logged for this peer.
135+
int connectOutgoingCount;
136+
int connectIncomingCount;
137+
int connectFailedCount;
138+
ContinuousSample<double> connectLatencies;
139+
133140
explicit Peer(TransportData* transport, NetworkAddress const& destination)
134141
: transport(transport), destination(destination), outgoingConnectionIdle(true), lastConnectTime(0.0),
135142
reconnectionDelay(FLOW_KNOBS->INITIAL_RECONNECTION_TIME), compatible(true), outstandingReplies(0),
136143
incompatibleProtocolVersionNewer(false), peerReferences(-1), bytesReceived(0), lastDataPacketSentTime(now()),
137-
pingLatencies(destination.isPublic() ? FLOW_KNOBS->PING_SAMPLE_AMOUNT : 1), lastLoggedBytesReceived(0),
138-
bytesSent(0), lastLoggedBytesSent(0) {}
144+
pingLatencies(destination.isPublic() ? FLOW_KNOBS->PING_SAMPLE_AMOUNT : 1), lastLoggedBytesReceived(0),
145+
bytesSent(0), lastLoggedBytesSent(0), lastLoggedTime(0.0), connectOutgoingCount(0), connectIncomingCount(0),
146+
connectFailedCount(0), connectLatencies(destination.isPublic() ? FLOW_KNOBS->NETWORK_CONNECT_SAMPLE_AMOUNT : 1) {}
139147

140148
void send(PacketBuffer* pb, ReliablePacket* rp, bool firstUnsent);
141149

‎fdbrpc/Stats.actor.cpp

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,9 @@ void CounterCollection::logToTraceEvent(TraceEvent &te) const {
7676
}
7777
}
7878

79-
ACTOR Future<Void> traceCounters(std::string traceEventName, UID traceEventID, double interval, CounterCollection* counters, std::string trackLatestName) {
79+
ACTOR Future<Void> traceCounters(std::string traceEventName, UID traceEventID, double interval,
80+
CounterCollection* counters, std::string trackLatestName,
81+
std::function<void(TraceEvent&)> decorator) {
8082
wait(delay(0)); // Give an opportunity for all members used in special counters to be initialized
8183

8284
for (ICounter* c : counters->counters)
@@ -89,6 +91,7 @@ ACTOR Future<Void> traceCounters(std::string traceEventName, UID traceEventID, d
8991
te.detail("Elapsed", now() - last_interval);
9092

9193
counters->logToTraceEvent(te);
94+
decorator(te);
9295

9396
if (!trackLatestName.empty()) {
9497
te.trackLatest(trackLatestName);

‎fdbrpc/Stats.h

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,9 @@ struct SpecialCounter : ICounter, FastAllocated<SpecialCounter<F>>, NonCopyable
132132
template <class F>
133133
static void specialCounter(CounterCollection& collection, std::string const& name, F && f) { new SpecialCounter<F>(collection, name, std::move(f)); }
134134

135-
Future<Void> traceCounters(std::string const& traceEventName, UID const& traceEventID, double const& interval, CounterCollection* const& counters, std::string const& trackLatestName = std::string());
135+
Future<Void> traceCounters(std::string const& traceEventName, UID const& traceEventID, double const& interval,
136+
CounterCollection* const& counters, std::string const& trackLatestName = std::string(),
137+
std::function<void(TraceEvent&)> const& decorator = [](TraceEvent& te) {});
136138

137139
class LatencyBands {
138140
public:

‎fdbrpc/sim2.actor.cpp

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -102,8 +102,6 @@ bool onlyBeforeSimulatorInit() {
102102

103103
const UID TOKEN_ENDPOINT_NOT_FOUND(-1, -1);
104104

105-
ISimulator* g_pSimulator = 0;
106-
thread_local ISimulator::ProcessInfo* ISimulator::currentProcess = 0;
107105
int openCount = 0;
108106

109107
struct SimClogging {

‎fdbrpc/simulator.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
#pragma once
2424

2525
#include "flow/flow.h"
26+
#include "flow/Histogram.h"
2627
#include "fdbrpc/FailureMonitor.h"
2728
#include "fdbrpc/Locality.h"
2829
#include "fdbrpc/IAsyncFile.h"
@@ -54,6 +55,7 @@ class ISimulator : public INetwork {
5455
LocalityData locality;
5556
ProcessClass startingClass;
5657
TDMetricCollection tdmetrics;
58+
HistogramRegistry histograms;
5759
std::map<NetworkAddress, Reference<IListener>> listenerMap;
5860
bool failed;
5961
bool excluded;

‎fdbserver/CMakeLists.txt

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,18 @@ add_library(fdb_sqlite STATIC
187187
sqlite/sqliteLimit.h
188188
sqlite/sqlite3.amalgamation.c)
189189

190+
if (WITH_ROCKSDB_EXPERIMENTAL)
191+
add_definitions(-DSSD_ROCKSDB_EXPERIMENTAL)
192+
# Set this to 0 if you want to compile RocksDB with `-march=native`.
193+
set(PORTABLE_ROCKSDB 1)
194+
195+
include(CompileRocksDB)
196+
# CompileRocksDB sets `lz4_LIBRARIES` to be the shared lib, we want to link
197+
# statically, so find the static library here.
198+
find_library(lz4_STATIC_LIBRARIES
199+
NAMES liblz4.a REQUIRED)
200+
endif()
201+
190202
# Suppress warnings in sqlite since it's third party
191203
if(NOT WIN32)
192204
target_compile_definitions(fdb_sqlite PRIVATE $<$<CONFIG:Debug>:NDEBUG>)
@@ -201,7 +213,14 @@ target_include_directories(fdbserver PRIVATE
201213
${CMAKE_BINARY_DIR}/bindings/c
202214
${CMAKE_CURRENT_BINARY_DIR}/workloads
203215
${CMAKE_CURRENT_SOURCE_DIR}/workloads)
204-
target_link_libraries(fdbserver PRIVATE fdbclient fdb_sqlite)
216+
if (WITH_ROCKSDB_EXPERIMENTAL)
217+
add_dependencies(fdbserver rocksdb)
218+
target_include_directories(fdbserver PRIVATE ${ROCKSDB_INCLUDE_DIR})
219+
target_link_libraries(fdbserver PRIVATE fdbclient fdb_sqlite ${ROCKSDB_LIBRARIES} ${lz4_STATIC_LIBRARIES})
220+
else()
221+
target_link_libraries(fdbserver PRIVATE fdbclient fdb_sqlite)
222+
endif()
223+
205224
if (GPERFTOOLS_FOUND)
206225
add_compile_definitions(USE_GPERFTOOLS)
207226
target_link_libraries(fdbserver PRIVATE gperftools)

0 commit comments

Comments
 (0)