Merge pull request gchq#403 from gchq/dev

Dev

Author: at055612

Diff for: build.gradle

@@ -8,12 +8,15 @@ buildscript {
     dependencies {
         classpath group: 'gchq', name: 'urlDependencyPlugin', version: 'v0.1'
         classpath group: 'ca.cutterslade.gradle', name: 'gradle-dependency-analyze', version: '1.1.0'
-        classpath 'com.benjaminsproule:swagger-gradle-plugin:+'
     }
 }
 
 plugins {
+    //plugin for downloading content from the 'net
     id "de.undercouch.download" version "3.2.0"
+
+    //plugin for producing a tree of task dependencies, run task 'taskTree'
+    id "com.dorongold.task-tree" version "1.3"
 }
 
 apply plugin: 'gchq.urldependencies'
@@ -126,7 +129,6 @@ ext.libs = [
     hamcrest_core: "org.hamcrest:hamcrest-core:1.3",
     hbase: "org.apache.hbase:hbase-common:1.2.1", //should be set to be consistent with CDH version
     hessian: "com.caucho:hessian:4.0.38",
-    hibernate_annotations: "org.hibernate:hibernate-annotations:3.5.6-Final" /* LATEST 5.2.5.Final */,
     hibernate_core: "org.hibernate:hibernate-core:$versions.hibernate" /* LATEST 5.2.5.Final */,
     hibernate_entitymanager: "org.hibernate:hibernate-entitymanager:$versions.hibernate" /* LATEST 5.2.5.Final */,
     hibernate_jpa_api: "org.hibernate.javax.persistence:hibernate-jpa-2.1-api:1.0.0.Final",
@@ -184,9 +186,7 @@ ext.libs = [
     spring_web: "org.springframework:spring-web:$versions.spring", //was 4.2.1.RELEASE now 4.3.4.RELEASE
     spring_webmvc: "org.springframework:spring-webmvc:$versions.spring",
     stax_api: "stax:stax-api:1.0.1",
-    //swagger_jersey2_jaxrs: "io.swagger:swagger-jersey2-jaxrs:$versions.swagger",
     swagger_annotations: "io.swagger:swagger-annotations:$versions.swagger",
-    //swagger_core: "io.swagger:swagger-core:$versions.swagger",
     validation_api: "javax.validation:validation-api:1.0.0.GA", //GWT 2.8.0 throws errors on gwtCompile if validation-api 1.1.0.Final is used
     vavr: "io.vavr:vavr:0.9.0",
     ws_rs_api: "javax.ws.rs:javax.ws.rs-api:2.0.1",
@@ -210,7 +210,7 @@ ext.contentPacks = [
 ]
 
 task wrapper(type: Wrapper) {
-    gradleVersion = '3.5'
+    gradleVersion = '4.1'
 }
 
 allprojects {
@@ -223,6 +223,10 @@ allprojects {
 
 subprojects {
     apply plugin: 'java'
+
+    //sourceSets {
+        //logger.info "main.output.classesDirs: " + main.output.classesDirs
+    //}
 
     //analyze missing dependencies for the code base
     apply plugin: 'ca.cutterslade.analyze'
@@ -284,6 +288,8 @@ subprojects {
         }
     }
 
+    //ensure we download our libs before compiling java
+    tasks.compileJava.dependsOn downloadUrlDependencies
 }
 
 
@@ -390,3 +396,4 @@ urlDependencies {
             "https://github.com/gchq/hadoop-common-shaded/releases/download/$versions.hadoopCommonShaded/${urlLibs.hadoopCommonShaded}.jar")
 }
 
+

Diff for: gradle/wrapper/gradle-wrapper.jar

@@ -1,6 +1,6 @@
-#Fri Aug 11 09:22:55 BST 2017
+#Fri Sep 08 13:37:38 BST 2017
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-3.5-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.1-bin.zip

Diff for: gradle/wrapper/gradle-wrapper.properties

@@ -35,7 +35,7 @@ dependencies {
 def gwtSourceDirs = []
 parent.subprojects.each { p ->
     if (p.name != 'stroom-dashboard-gwt') {
-        gwtSourceDirs.add(p.sourceSets.main.output.classesDir)
+        gwtSourceDirs.addAll(p.sourceSets.main.output.classesDirs)
         gwtSourceDirs.addAll(p.sourceSets.main.java.srcDirs)
         gwtSourceDirs.addAll(p.sourceSets.main.resources.srcDirs)
     }

Diff for: stroom-app-gwt/build.gradle

@@ -1,16 +1,17 @@
 plugins {
-  id 'com.github.johnrengelman.shadow' version '1.2.4'
+    id 'com.github.johnrengelman.shadow' version '1.2.4'
+    id "com.benjaminsproule.swagger" version "0.1.4"
 }
 
 def swaggerDir = "${projectDir}/src/main/resources/swagger"
 
 apply plugin: 'application'
-apply plugin: 'com.benjaminsproule.swagger'
+apply plugin: 'java'
 mainClassName = 'stroom.startup.App'
 
 jar {
     //we need to first generate the swagger spec files so they can be added in to the jar
-    dependsOn generateSwaggerDocumentation
+    mustRunAfter generateSwaggerDocumentation
 
     manifest {
         attributes(
@@ -147,7 +148,7 @@ sourceSets {
 
 // This task lets us run the actual integration tests.
 task integrationTest(type: Test) {
-    testClassesDir = sourceSets.integrationTest.output.classesDir
+    testClassesDirs = sourceSets.integrationTest.output.classesDirs
     classpath = sourceSets.integrationTest.runtimeClasspath
 }
 
@@ -172,8 +173,8 @@ runShadow {
 //The gwtCompile tasks create files in stroom-app/src/main/resources/ui
 //so those tasks MUST be run first else this jar and the fat jar will be
 //missing all the GWT generated files and the UI will fail to start
-processResources.dependsOn(':stroom-app-gwt:gwtCompile')
-processResources.dependsOn(':stroom-dashboard-gwt:gwtCompile')
+processResources.mustRunAfter(':stroom-app-gwt:gwtCompile')
+processResources.mustRunAfter(':stroom-dashboard-gwt:gwtCompile')
 
 swagger {
     apiSource {
@@ -206,3 +207,5 @@ swagger {
     }
 }
 
+tasks.build.dependsOn generateSwaggerDocumentation
+tasks.build.dependsOn shadowJar

Diff for: stroom-app/build.gradle

@@ -136,7 +136,7 @@ public HealthCheck.Result getHealth() {
                             .withMessage("%s service(s) have no registered instances");
                 } else {
                     builder.healthy()
-                            .withMessage("All external services available");
+                            .withMessage("All services (local and remote) available");
                 }
                 return builder.withDetail("discovered-service-instances", serviceInstanceMap)
                         .build();

Diff for: stroom-app/src/main/java/stroom/servicediscovery/ServiceDiscovererImpl.java

@@ -69,9 +69,6 @@ private String getHostOrIp(final StroomPropertyService stroomPropertyService) {
 
     private void curatorStartupListener(ServiceDiscovery<String> serviceDiscovery) {
         try {
-            StringBuilder stringBuilder = new StringBuilder();
-            stringBuilder.append("Successfully registered the following services: ");
-
             Map<String, String> services = new TreeMap<>();
             Arrays.stream(RegisteredService.values())
                     .forEach(registeredService -> {
@@ -83,7 +80,7 @@ private void curatorStartupListener(ServiceDiscovery<String> serviceDiscovery) {
 
             health = HealthCheck.Result.builder()
                     .healthy()
-                    .withMessage("Services registered")
+                    .withMessage("Local services registered")
                     .withDetail("registered-services", services)
                     .build();
 

Diff for: stroom-app/src/main/java/stroom/servicediscovery/ServiceDiscoveryRegistrar.java

@@ -935,7 +935,7 @@
         </bean>
         <bean class="stroom.node.shared.GlobalProperty">
             <property name="name" value="stroom.services.stroomStats.version"/>
-            <property name="value" value="1"/>
+            <property name="value" value="2"/>
             <property name="description"
                       value="The version of the stroom-stats service" />
             <property name="editable" value="true"/>
@@ -1039,7 +1039,7 @@
             <property name="value"
                       value="localhost:2181"/>
             <property name="description"
-                      value="The Zookeeper quorum connection string, required for service discovery" />
+                      value="The Zookeeper quorum connection string, required for service discovery, in the form 'host1:port1,host2:port2,host3:port3'. The root znode to use in Zookeeper is defined in the property stroom.serviceDiscovery.zookeeperBasePath" />
             <property name="editable" value="true"/>
             <property name="requireUiRestart" value="true"/>
         </bean>

Diff for: stroom-core-server/src/main/resources/META-INF/spring/stroomCoreServerPropertyContext.xml

@@ -32,7 +32,7 @@ dependencies {
 def gwtSourceDirs = []
 parent.subprojects.each { p ->
     if (p.name != 'stroom-app-gwt') {
-        gwtSourceDirs.add(p.sourceSets.main.output.classesDir)
+        gwtSourceDirs.addAll(p.sourceSets.main.output.classesDirs)
         gwtSourceDirs.addAll(p.sourceSets.main.java.srcDirs)
         gwtSourceDirs.addAll(p.sourceSets.main.resources.srcDirs)
     }

Diff for: stroom-dashboard-gwt/build.gradle

@@ -61,6 +61,7 @@ public class KafkaProducerFilter extends AbstractSamplingFilter {
 
     private String topic;
     private String recordKey;
+    private StroomKafkaProducer.FlushMode flushMode;
 
     @Inject
     public KafkaProducerFilter(final ErrorReceiverProxy errorReceiverProxy,
@@ -96,4 +97,9 @@ public void setRecordKey(final String recordKey) {
     public void setTopic(final String topic) {
         this.topic = pathCreator.replaceAll(topic);
     }
+
+    @PipelineProperty(description="Flush the producer each time a message is sent")
+    public void setFlushOnSend(final boolean flushMode) {
+        this.flushMode = flushMode ? StroomKafkaProducer.FlushMode.FLUSH_ON_SEND : StroomKafkaProducer.FlushMode.NO_FLUSH;
+    }
 }

Diff for: stroom-pipeline/src/main/java/stroom/kafka/filter/KafkaProducerFilter.java

@@ -16,6 +16,7 @@ public class RollingKafkaDestination extends RollingDestination {
 
     private final String topic;
     private final String recordKey;
+    private final StroomKafkaProducer.FlushMode flushMode;
 
     private StringBuilder data = new StringBuilder();
 
@@ -25,11 +26,13 @@ public RollingKafkaDestination(final String key,
                                    final long creationTime,
                                    final StroomKafkaProducer stroomKafkaProducer,
                                    final String recordKey,
-                                   final String topic) {
+                                   final String topic,
+                                   final StroomKafkaProducer.FlushMode flushMode) {
         super(key, frequency, maxSize, creationTime);
         this.stroomKafkaProducer = stroomKafkaProducer;
         this.recordKey = recordKey;
         this.topic = topic;
+        this.flushMode = flushMode;
 
         setOutputStream(new ByteCountOutputStream(new OutputStream() {
             @Override
@@ -49,7 +52,7 @@ void onHeaderWritten(final ByteCountOutputStream outputStream,
     void onFooterWritten(final Consumer<Throwable> exceptionConsumer) {
         final ProducerRecord<String, String> newRecord = new ProducerRecord<>(topic, recordKey, data.toString());
         try {
-            stroomKafkaProducer.send(newRecord, StroomKafkaProducer.FlushMode.FLUSH_ON_SEND, e -> {
+            stroomKafkaProducer.send(newRecord, flushMode, e -> {
                 LOGGER.error("Unable to send record to Kafka!", e);
             });
         } catch (RuntimeException e) {

Diff for: stroom-pipeline/src/main/java/stroom/pipeline/destination/RollingKafkaDestination.java

@@ -13,7 +13,6 @@
 import stroom.util.spring.StroomScope;
 
 import javax.annotation.Resource;
-import javax.inject.Inject;
 import java.io.IOException;
 
 @Component
@@ -31,6 +30,7 @@ public class RollingKafkaAppender extends AbstractRollingAppender {
 
     private String topic;
     private String recordKey;
+    private StroomKafkaProducer.FlushMode flushMode;
 
     private String key;
 
@@ -62,7 +62,8 @@ public RollingDestination createDestination() throws IOException {
                 System.currentTimeMillis(),
                 stroomKafkaProducer,
                 recordKey,
-                topic);
+                topic,
+                flushMode);
     }
 
     @PipelineProperty(description = "The record key to apply to records, used to select patition. Replacement variables can be used in path strings such as ${feed}.")
@@ -74,4 +75,9 @@ public void setRecordKey(final String recordKey) {
     public void setTopic(final String topic) {
         this.topic = pathCreator.replaceAll(topic);
     }
+
+    @PipelineProperty(description="Flush the producer each time a message is sent")
+    public void setFlushOnSend(final boolean flushOnSend) {
+        this.flushMode = flushOnSend ? StroomKafkaProducer.FlushMode.FLUSH_ON_SEND : StroomKafkaProducer.FlushMode.NO_FLUSH;
+    }
 }

Diff for: stroom-pipeline/src/main/java/stroom/pipeline/server/writer/RollingKafkaAppender.java

@@ -6,7 +6,7 @@ dependencies {
 
     compile libs.hibernate_jpa_api
     compile libs.validation_api
-    compile libs.hibernate_annotations
+    compile libs.hibernate_core
 
     testCompile libs.junit