Lockless changes for CSFB procedure, MO/MT call and SMS

cwf/gateway/docker/docker-compose.integ-test.yml

@@ -100,3 +100,9 @@ services:
     command: >
       /bin/bash -c "/usr/bin/redis-server /var/opt/magma/redis.conf --daemonize no &&
              /usr/bin/redis-cli shutdown"
+
+  uesim:
+    <<: *service
+    container_name: uesim
+    image: ${DOCKER_REGISTRY}uesim:${IMAGE_VERSION}
+    command: envdir /var/opt/magma/envdir /var/opt/magma/bin/uesim -logtostderr=true -v=0

cwf/gateway/docker/docker-compose.override.yml

@@ -34,4 +34,9 @@ services:
   sessiond:
     build:
       context: ${BUILD_CONTEXT}
-      dockerfile: lte/gateway/docker/c/Dockerfile
+      dockerfile: lte/gateway/docker/c/Dockerfile
+
+  uesim:
+    build:
+      context: ${BUILD_CONTEXT}
+      dockerfile: cwf/gateway/docker/go/Dockerfile

cwf/gateway/docker/go/Dockerfile

@@ -0,0 +1,122 @@
+ARG baseImage="ubuntu:xenial"
+FROM ${baseImage} as base
+
+# Add the magma apt repo
+RUN apt-get update && \
+    apt-get install -y apt-utils software-properties-common apt-transport-https
+COPY orc8r/tools/ansible/roles/pkgrepo/files/jfrog.pub /tmp/jfrog.pub
+RUN apt-key add /tmp/jfrog.pub && \
+    apt-add-repository "deb https://magma.jfrog.io/magma/list/dev/ xenial main"
+
+# Install the runtime deps.
+RUN apt-get update && apt-get install -y \
+    bzr \
+    curl \
+    daemontools \
+    gcc \
+    git \
+    libc-ares-dev \
+    libev-dev \
+    libevent-dev \
+    libffi-dev \
+    libjansson-dev \
+    libjemalloc-dev \
+    libssl-dev \
+    libsystemd-dev \
+    magma-nghttpx=1.31.1-1 \
+    make \
+    net-tools \
+    pkg-config \
+    python-cffi \
+    python3-pip \
+    redis-server \
+    rsyslog \
+    sudo \
+    unzip \
+    vim \
+    virtualenv
+
+# Install Golang 1.12.
+WORKDIR /usr/local
+RUN curl https://dl.google.com/go/go1.12.4.linux-amd64.tar.gz -O && \
+    tar xvf go1.12.4.linux-amd64.tar.gz && \
+    cp -r go/bin/* /usr/local/bin/
+
+# Install protobuf compiler.
+RUN curl -Lfs https://github.com/google/protobuf/releases/download/v3.1.0/protoc-3.1.0-linux-x86_64.zip -o protoc3.zip && \
+    unzip protoc3.zip -d protoc3 && \
+    mv protoc3/bin/protoc /bin/protoc && \
+    chmod a+rx /bin/protoc && \
+    mv protoc3/include/google /usr/include/ && \
+    chmod -R a+Xr /usr/include/google && \
+    rm -rf protoc3.zip protoc3
+
+ENV GOBIN /var/opt/magma/bin
+ENV MAGMA_ROOT /magma
+ENV PIP_CACHE_HOME ~/.pipcache
+ENV PYTHON_BUILD /build/python
+ENV PATH ${PYTHON_BUILD}/bin:${PATH}:${GOBIN}
+ENV GO111MODULE on
+# Use public go modules proxy
+ENV GOPROXY https://proxy.golang.org
+
+RUN printenv > /etc/environment
+
+
+# Copy just the go.mod and go.sum files to download the golang deps.
+# This step allows us to cache the downloads, and prevents reaching out to
+# the internet unless any of the go.mod or go.sum files are changed.
+COPY cwf/cloud/go/go.* $MAGMA_ROOT/cwf/cloud/go/
+COPY lte/cloud/go/go.* $MAGMA_ROOT/lte/cloud/go/
+COPY feg/cloud/go/go.* $MAGMA_ROOT/feg/cloud/go/
+COPY feg/cloud/go/protos/go.* $MAGMA_ROOT/feg/cloud/go/protos/
+COPY feg/radius/lib/go/ $MAGMA_ROOT/feg/radius/lib/go/
+COPY feg/radius/src/go.* $MAGMA_ROOT/feg/radius/src/
+COPY feg/gateway/go.* $MAGMA_ROOT/feg/gateway/
+COPY orc8r/cloud/go/go.* $MAGMA_ROOT/orc8r/cloud/go/
+COPY orc8r/gateway/go/go.* $MAGMA_ROOT/orc8r/gateway/go/
+WORKDIR $MAGMA_ROOT/cwf/gateway
+RUN go mod download; exit 0
+# Install protoc-gen-go
+RUN go install github.com/golang/protobuf/protoc-gen-go; exit 0
+
+# Copy the configs.
+COPY orc8r/cloud/docker/proxy/magma_headers.rb /etc/nghttpx/magma_headers.rb
+
+# Symlink python scripts.
+RUN ln -s /build/python/bin/generate_service_config.py /usr/local/bin/generate_service_config.py
+RUN ln -s /build/python/bin/generate_nghttpx_config.py /usr/local/bin/generate_nghttpx_config.py
+
+# Build the code.
+COPY cwf $MAGMA_ROOT/cwf
+COPY feg $MAGMA_ROOT/feg
+COPY lte/cloud $MAGMA_ROOT/lte/cloud
+COPY orc8r/cloud $MAGMA_ROOT/orc8r/cloud
+COPY orc8r/gateway/go $MAGMA_ROOT/orc8r/gateway/go
+
+# -----------------------------------------------------------------------------
+# Builder image with binary
+# -----------------------------------------------------------------------------
+FROM base as builder
+# Enable make gen if proto gen is required
+# RUN make -C $MAGMA_ROOT/cwf/gateway gen
+RUN make -C $MAGMA_ROOT/cwf/gateway build
+
+# -----------------------------------------------------------------------------
+# Production image
+# -----------------------------------------------------------------------------
+FROM ubuntu:xenial AS uesim
+
+# Install envdir.
+RUN apt-get -y update && apt-get -y install daemontools
+
+# Copy the build artifacts.
+COPY --from=builder /var/opt/magma/bin /var/opt/magma/bin
+
+# Copy the configs.
+COPY cwf/gateway/configs /etc/magma
+
+# Create empty envdir directory
+RUN mkdir -p /var/opt/magma/envdir
+
+RUN mkdir -p /var/opt/magma/configs

cwf/gateway/go.sum

@@ -74,6 +74,7 @@ github.com/facebookincubator/ent v0.0.0-20191128071424-29c7b0a0d805 h1:Gqrw3IfjW
 github.com/facebookincubator/ent v0.0.0-20191128071424-29c7b0a0d805/go.mod h1:3CGvz1m+D78JUti7JFkY6AkKj88McWuT80UcamLqEJQ=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fiorix/go-diameter/v4 v4.0.1-0.20200102140015-a6c006d17e34/go.mod h1:Qx/+pf+c9sBUHWq1d7EH3bkdwN8U0mUpdy9BieDw6UQ=
+github.com/fiorix/go-diameter/v4 v4.0.1/go.mod h1:Qx/+pf+c9sBUHWq1d7EH3bkdwN8U0mUpdy9BieDw6UQ=
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/getsentry/raven-go v0.1.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=

devmand/gateway/CMakeLists.txt

@@ -213,6 +213,7 @@ add_executable(devmantest
   ${PROJECT_SOURCE_DIR}/src/devmand/test/ConfigGeneratorTest.cpp
   ${PROJECT_SOURCE_DIR}/src/devmand/test/cli/RealCliDeviceTest.cpp
   ${PROJECT_SOURCE_DIR}/src/devmand/test/cli/CliScaleTest.cpp
+  ${PROJECT_SOURCE_DIR}/src/devmand/test/cli/CliTest.cpp
   ${PROJECT_SOURCE_DIR}/src/devmand/test/cli/PromptAwareCliTest.cpp
   ${PROJECT_SOURCE_DIR}/src/devmand/test/cli/TimeoutTrackingCliTest.cpp
   ${PROJECT_SOURCE_DIR}/src/devmand/test/cli/SshSessionTest.cpp

devmand/gateway/src/devmand/channels/cli/QueuedCli.cpp

@@ -100,7 +100,7 @@ SemiFuture<string> QueuedCli::executeWrite(const WriteCommand cmd) {
   Command command = cmd;
   if (!command.isMultiCommand()) {
     // Single line config command, execute with read
-    return executeRead(ReadCommand::create(cmd));
+    return executeRead(ReadCommand::create(cmd.raw(), true)); //skip cache
   }
 
   const vector<Command>& commands = command.splitMultiCommand();
@@ -119,7 +119,7 @@ SemiFuture<string> QueuedCli::executeWrite(const WriteCommand cmd) {
   }
 
   commmandsFutures.emplace_back(
-      executeRead(ReadCommand::create(commands.back()))
+      executeRead(ReadCommand::create(commands.back().raw(), true)) //skip cache
           .via(queuedParameters->serialExecutorKeepAlive));
 
   return reduce(

devmand/gateway/src/devmand/test/cli/CliTest.cpp

@@ -0,0 +1,92 @@
+// Copyright (c) 2019-present, Facebook, Inc.
+// All rights reserved.
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+#define LOG_WITH_GLOG
+#include <magma_logging.h>
+
+#include <boost/algorithm/string/trim.hpp>
+#include <devmand/Application.h>
+#include <devmand/channels/cli/IoConfigurationBuilder.h>
+#include <devmand/devices/cli/PlaintextCliDevice.h>
+#include <devmand/test/TestUtils.h>
+#include <devmand/test/cli/utils/Log.h>
+#include <devmand/test/cli/utils/MockCli.h>
+#include <devmand/test/cli/utils/Ssh.h>
+#include <gtest/gtest.h>
+
+namespace devmand {
+namespace test {
+namespace cli {
+
+using namespace devmand::channels::cli;
+using namespace devmand::cartography;
+using namespace devmand::devices;
+using namespace devmand::devices::cli;
+using namespace devmand::test::utils::cli;
+using namespace std;
+using namespace folly;
+using namespace devmand::test::utils::ssh;
+using devmand::channels::cli::IoConfigurationBuilder;
+using devmand::channels::cli::ReadCachingCli;
+using namespace std::chrono;
+
+class CliTest : public ::testing::Test {
+ protected:
+  shared_ptr<server> ssh;
+  unique_ptr<channels::cli::Engine> cliEngine;
+
+  void SetUp() override {
+    devmand::test::utils::log::initLog(MDEBUG);
+    cliEngine = make_unique<channels::cli::Engine>();
+    ssh = startSshServer();
+  }
+
+  void TearDown() override {
+    ssh->close();
+  }
+};
+
+static std::function<bool()> ensureConnected(const shared_ptr<Cli>& cli) {
+  return [cli]() {
+    try {
+      cli->executeRead(ReadCommand::create("echo 123", true)).get();
+        return true;
+    } catch (const exception& e) {
+        return false;
+    }
+  };
+}
+
+TEST_F(CliTest, writeMultipleTimesAllExecute) {
+  cartography::DeviceConfig deviceConfig;
+  devmand::cartography::ChannelConfig chnlCfg;
+  std::map<std::string, std::string> kvPairs;
+  kvPairs.insert(std::make_pair("port", "9999"));
+  kvPairs.insert(std::make_pair("username", "root"));
+  kvPairs.insert(std::make_pair("password", "root"));
+  chnlCfg.kvPairs = kvPairs;
+  deviceConfig.channelConfigs.insert(std::make_pair("cli", chnlCfg));
+  deviceConfig.ip = "localhost";
+  deviceConfig.id = "localhost-test-device";
+
+  IoConfigurationBuilder ioConfigurationBuilder(deviceConfig, *cliEngine);
+  const shared_ptr<Cli>& cli =
+      ioConfigurationBuilder.createAll(ReadCachingCli::createCache());
+  const function<bool()> &connectionTest = ensureConnected(cli);
+
+  EXPECT_BECOMES_TRUE(connectionTest());
+
+  cli->executeWrite(WriteCommand::create("sleep 2\nsleep2")).get();
+  steady_clock::time_point begin = steady_clock::now();
+  cli->executeWrite(WriteCommand::create("sleep 2")).get();
+  steady_clock::time_point end = steady_clock::now();
+  EXPECT_GE(duration_cast<milliseconds>(end - begin).count(), 2000);
+}
+
+} // namespace cli
+} // namespace test
+} // namespace devmand

docs/readmes/orc8r/deploy_build.md

@@ -22,7 +22,7 @@ When this job finishes, upload these images to your image registry:
 
 ```bash
 export MAGMA_TAG=v1.0.0
-for image in proxy controller prometheus-cache config-manager grafana
+for image in proxy controller prometheus-cache alertmanager-configurer prometheus-configurer grafana
 do
     ../../../orc8r/tools/docker/publish.sh -r REGISTRY -i ${image} -v ${MAGMA_TAG}
 done

docs/readmes/orc8r/deploy_install.md

@@ -172,8 +172,8 @@ prometheusPushAddresses:
   - "http://orc8r-prometheus-cache:9091/metrics"
 
 alertmanagerApiURL: "http://orc8r-alertmanager:9093/api/v2/alerts"
-prometheusConfigServiceURL: "http://orc8r-config-manager:9100"
-alertmanagerConfigServiceURL: "http://orc8r-config-manager:9101"
+prometheusConfigServiceURL: "http://orc8r-prometheus-configurer:9100"
+alertmanagerConfigServiceURL: "http://orc8r-alertmanager-configurer:9101"
 ```
 
 ## Initial Helm Deploy
@@ -265,10 +265,18 @@ metrics:
     nodeSelector:
       worker-type: metrics
 
-  configmanager:
+  alertmanagerConfigurer:
     create: true
     image:
-      repository: YOUR-DOCKER-REGISTRY/config-manager
+      repository: YOUR-DOCKER-REGISTRY/alertmanager-configurer
+      tag: YOUR-CONTAINER-TAG
+    nodeSelector:
+      worker-type: metrics
+
+  prometheusConfigurer:
+    create: true
+    image:
+      repository: YOUR-DOCKER-REGISTRY/prometheus-configurer
       tag: YOUR-CONTAINER-TAG
     nodeSelector:
       worker-type: metrics
@@ -350,16 +358,17 @@ First, find a `orc8r-controller-` pod in k8s:
 ```bash
 $ kubectl -n magma get pods
 
-NAME                                      READY   STATUS    RESTARTS   AGE
-orc8r-configmanager-896d784bc-chqr7       1/1     Running   0          X
-orc8r-controller-7757567bf5-cm4wn         1/1     Running   0          X
-orc8r-controller-7757567bf5-jshpv         1/1     Running   0          X
-orc8r-alertmanager-c8dc7cdb5-crzpl        1/1     Running   0          X
-orc8r-grafana-6446b97885-ck6g8            1/1     Running   0          X
-orc8r-prometheus-6c67bcc9d8-6lx22         1/1     Running   0          X
-orc8r-prometheus-cache-6bf7648446-9t9hx   1/1     Running   0          X
-orc8r-proxy-57cf989fcc-cg54z              1/1     Running   0          X
-orc8r-proxy-57cf989fcc-xn2cw              1/1     Running   0          X
+NAME                                              READY   STATUS    RESTARTS   AGE
+orc8r-alertmanager-configurer-c8dc7cdb5-crzpl     1/1     Running   0          X
+orc8r-controller-7757567bf5-cm4wn                 1/1     Running   0          X
+orc8r-controller-7757567bf5-jshpv                 1/1     Running   0          X
+orc8r-alertmanager-c8dc7cdb5-crzpl                1/1     Running   0          X
+orc8r-grafana-6446b97885-ck6g8                    1/1     Running   0          X
+orc8r-prometheus-6c67bcc9d8-6lx22                 1/1     Running   0          X
+orc8r-prometheus-cache-6bf7648446-9t9hx           1/1     Running   0          X
+orc8r-prometheus-configurer-896d784bc-chqr7       1/1     Running   0          X
+orc8r-proxy-57cf989fcc-cg54z                      1/1     Running   0          X
+orc8r-proxy-57cf989fcc-xn2cw                      1/1     Running   0          X
 ```
 
 Then:

feg/gateway/diameter/definitions.go

@@ -10,9 +10,10 @@ package diameter
 
 const (
 	// SuccessCode is the result code returned from a successful diameter call
-	SuccessCode          = 2001
-	LimitedSuccessCode   = 2002
-	DiameterRatingFailed = 5031
+	SuccessCode                = 2001
+	LimitedSuccessCode         = 2002
+	DiameterRatingFailed       = 5031
+	DiameterCreditLimitReached = 4012
 )
 
 var diamCodeToNameMap = map[uint32]string{

feg/gateway/go.mod

@@ -7,6 +7,8 @@
 module magma/feg/gateway
 
 replace (
+	github.com/fiorix/go-diameter/v4 => github.com/emakeev/go-diameter/v4 v4.0.2
+
 	magma/feg/cloud/go => ../../feg/cloud/go
 	magma/feg/cloud/go/protos => ../../feg/cloud/go/protos
 
@@ -16,7 +18,7 @@ replace (
 )
 
 require (
-	github.com/fiorix/go-diameter/v4 v4.0.1-0.20200102140015-a6c006d17e34
+	github.com/fiorix/go-diameter/v4 v4.0.1
 	github.com/go-redis/redis v6.14.1+incompatible
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
 	github.com/golang/protobuf v1.3.2