Skip to content

Commit 43b0d9e

Browse files
glenrobsonglenrobson
committed
Adding connection to AWS Elastic search
1 parent 5c60963 commit 43b0d9e

File tree

6 files changed

+262
-6
lines changed

6 files changed

+262
-6
lines changed

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,4 @@ src/main/webapp/demo.html
66
cache
77
index-2.6.1.html
88
src/main/webapp/stats
9+
.aws-credentials

‎pom.xml

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -165,12 +165,17 @@
165165
<dependency>
166166
<groupId>org.elasticsearch</groupId>
167167
<artifactId>elasticsearch</artifactId>
168-
<version>7.8.1</version>
168+
<version>7.7.1</version>
169169
</dependency>
170170
<dependency>
171171
<groupId>org.elasticsearch.client</groupId>
172172
<artifactId>elasticsearch-rest-high-level-client</artifactId>
173-
<version>7.8.1</version>
173+
<version>7.7.1</version>
174+
</dependency>
175+
<dependency>
176+
<groupId>com.amazonaws</groupId>
177+
<artifactId>aws-java-sdk-core</artifactId>
178+
<version>1.11.839</version>
174179
</dependency>
175180
<dependency>
176181
<groupId>junit</groupId>

‎src/main/java/uk/org/llgc/annotation/store/StoreConfig.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@
3535
public class StoreConfig extends HttpServlet {
3636
protected static Logger _logger = LogManager.getLogger(StoreConfig.class.getName());
3737
protected Map<String,String> _props = null;
38-
public final String[] ALLOWED_PROPS = {"baseURI","encoder","store","data_dir","store","repo_url","solr_connection","solr_collection"};
38+
public final String[] ALLOWED_PROPS = {"baseURI","encoder","store","data_dir","store","repo_url","solr_connection","elastic_connection"};
3939
protected AnnotationUtils _annotationUtils = null;
4040

4141
public StoreConfig() {
@@ -173,8 +173,10 @@ public StoreAdapter getStore() {
173173
try {
174174
tAdapter = new ElasticStore(_props.get("elastic_connection"));
175175
} catch (URISyntaxException tExcpt) {
176+
tExcpt.printStackTrace();
176177
throw new IllegalArgumentException("Failed to create Elastic connection due a problem with the conection URL");
177178
} catch (IOException tExcpt) {
179+
tExcpt.printStackTrace();
178180
throw new IllegalArgumentException("Failed to create Elastic connection due a problem with the conection URL");
179181
}
180182
} else {

‎src/main/java/uk/org/llgc/annotation/store/adapters/elastic/AWSRequestSigningApacheInterceptor.java

Lines changed: 188 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
/*
2+
* Copyright 2012-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with
5+
* the License. A copy of the License is located at
6+
*
7+
* http://aws.amazon.com/apache2.0
8+
*
9+
* or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
10+
* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
11+
* and limitations under the License.
12+
*/
13+
package uk.org.llgc.annotation.store.adapters.elastic;
14+
15+
import com.amazonaws.DefaultRequest;
16+
import com.amazonaws.auth.AWSCredentialsProvider;
17+
import com.amazonaws.auth.Signer;
18+
import com.amazonaws.http.HttpMethodName;
19+
import org.apache.http.Header;
20+
import org.apache.http.HttpEntityEnclosingRequest;
21+
import org.apache.http.HttpException;
22+
import org.apache.http.HttpHost;
23+
import org.apache.http.HttpRequest;
24+
import org.apache.http.HttpRequestInterceptor;
25+
import org.apache.http.NameValuePair;
26+
import org.apache.http.client.utils.URIBuilder;
27+
import org.apache.http.entity.BasicHttpEntity;
28+
import org.apache.http.message.BasicHeader;
29+
import org.apache.http.protocol.HttpContext;
30+
31+
import java.io.ByteArrayInputStream;
32+
import java.io.IOException;
33+
import java.net.URI;
34+
import java.net.URISyntaxException;
35+
import java.util.ArrayList;
36+
import java.util.List;
37+
import java.util.Map;
38+
import java.util.TreeMap;
39+
40+
import static org.apache.http.protocol.HttpCoreContext.HTTP_TARGET_HOST;
41+
42+
/**
43+
* THIS IS A HORRIBLE WAY OF USING THIS CLASS BUT:
44+
* this is recomended by the AWS docs here: https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/es-request-signing.html#es-request-signing-java
45+
* but the code isn't in maven: https://github.com/awslabs/aws-request-signing-apache-interceptor/issues/2
46+
* so this is the only real option...
47+
* An {@link HttpRequestInterceptor} that signs requests using any AWS {@link Signer}
48+
* and {@link AWSCredentialsProvider}.
49+
*/
50+
public class AWSRequestSigningApacheInterceptor implements HttpRequestInterceptor {
51+
/**
52+
* The service that we're connecting to. Technically not necessary.
53+
* Could be used by a future Signer, though.
54+
*/
55+
private final String service;
56+
57+
/**
58+
* The particular signer implementation.
59+
*/
60+
private final Signer signer;
61+
62+
/**
63+
* The source of AWS credentials for signing.
64+
*/
65+
private final AWSCredentialsProvider awsCredentialsProvider;
66+
67+
/**
68+
*
69+
* @param service service that we're connecting to
70+
* @param signer particular signer implementation
71+
* @param awsCredentialsProvider source of AWS credentials for signing
72+
*/
73+
public AWSRequestSigningApacheInterceptor(final String service,
74+
final Signer signer,
75+
final AWSCredentialsProvider awsCredentialsProvider) {
76+
this.service = service;
77+
this.signer = signer;
78+
this.awsCredentialsProvider = awsCredentialsProvider;
79+
}
80+
81+
/**
82+
* {@inheritDoc}
83+
*/
84+
@Override
85+
public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException {
86+
URIBuilder uriBuilder;
87+
try {
88+
uriBuilder = new URIBuilder(request.getRequestLine().getUri());
89+
} catch (URISyntaxException e) {
90+
throw new IOException("Invalid URI" , e);
91+
}
92+
93+
// Copy Apache HttpRequest to AWS DefaultRequest
94+
DefaultRequest<?> signableRequest = new DefaultRequest<>(service);
95+
96+
HttpHost host = (HttpHost) context.getAttribute(HTTP_TARGET_HOST);
97+
if (host != null) {
98+
signableRequest.setEndpoint(URI.create(host.toURI()));
99+
}
100+
final HttpMethodName httpMethod = HttpMethodName.fromValue(request.getRequestLine().getMethod());
101+
signableRequest.setHttpMethod(httpMethod);
102+
try {
103+
signableRequest.setResourcePath(uriBuilder.build().getRawPath());
104+
} catch (URISyntaxException e) {
105+
throw new IOException("Invalid URI" , e);
106+
}
107+
108+
String tContent = "";
109+
if (request instanceof HttpEntityEnclosingRequest) {
110+
HttpEntityEnclosingRequest httpEntityEnclosingRequest =
111+
(HttpEntityEnclosingRequest) request;
112+
if (httpEntityEnclosingRequest.getEntity() == null) {
113+
signableRequest.setContent(new ByteArrayInputStream(new byte[0]));
114+
} else {
115+
signableRequest.setContent(httpEntityEnclosingRequest.getEntity().getContent());
116+
}
117+
}
118+
signableRequest.setParameters(nvpToMapParams(uriBuilder.getQueryParams()));
119+
signableRequest.setHeaders(headerArrayToMap(request.getAllHeaders()));
120+
121+
// Sign it
122+
signer.sign(signableRequest, awsCredentialsProvider.getCredentials());
123+
124+
// Now copy everything back
125+
request.setHeaders(mapToHeaderArray(signableRequest.getHeaders()));
126+
if (request instanceof HttpEntityEnclosingRequest) {
127+
HttpEntityEnclosingRequest httpEntityEnclosingRequest = (HttpEntityEnclosingRequest) request;
128+
if (httpEntityEnclosingRequest.getEntity() != null) {
129+
BasicHttpEntity basicHttpEntity = new BasicHttpEntity();
130+
basicHttpEntity.setContent(signableRequest.getContent());
131+
httpEntityEnclosingRequest.setEntity(basicHttpEntity);
132+
}
133+
}
134+
}
135+
136+
/**
137+
*
138+
* @param params list of HTTP query params as NameValuePairs
139+
* @return a multimap of HTTP query params
140+
*/
141+
private static Map<String, List<String>> nvpToMapParams(final List<NameValuePair> params) {
142+
Map<String, List<String>> parameterMap = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
143+
for (NameValuePair nvp : params) {
144+
List<String> argsList =
145+
parameterMap.computeIfAbsent(nvp.getName(), k -> new ArrayList<>());
146+
argsList.add(nvp.getValue());
147+
}
148+
return parameterMap;
149+
}
150+
151+
/**
152+
* @param headers modeled Header objects
153+
* @return a Map of header entries
154+
*/
155+
private static Map<String, String> headerArrayToMap(final Header[] headers) {
156+
Map<String, String> headersMap = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
157+
for (Header header : headers) {
158+
if (!skipHeader(header)) {
159+
headersMap.put(header.getName(), header.getValue());
160+
}
161+
}
162+
return headersMap;
163+
}
164+
165+
/**
166+
* @param header header line to check
167+
* @return true if the given header should be excluded when signing
168+
*/
169+
private static boolean skipHeader(final Header header) {
170+
return ("content-length".equalsIgnoreCase(header.getName())
171+
&& "0".equals(header.getValue())) // Strip Content-Length: 0
172+
|| "host".equalsIgnoreCase(header.getName()); // Host comes from endpoint
173+
}
174+
175+
/**
176+
* @param mapHeaders Map of header entries
177+
* @return modeled Header objects
178+
*/
179+
private static Header[] mapToHeaderArray(final Map<String, String> mapHeaders) {
180+
Header[] headers = new Header[mapHeaders.size()];
181+
int i = 0;
182+
for (Map.Entry<String, String> headerEntry : mapHeaders.entrySet()) {
183+
headers[i++] = new BasicHeader(headerEntry.getKey(), headerEntry.getValue());
184+
}
185+
return headers;
186+
}
187+
}
188+

‎src/main/java/uk/org/llgc/annotation/store/adapters/elastic/ElasticStore.java

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,11 @@
6161
import org.elasticsearch.search.SearchHit;
6262
import org.elasticsearch.search.SearchHits;
6363

64+
import com.amazonaws.auth.AWS4Signer;
65+
import com.amazonaws.auth.AWSCredentialsProvider;
66+
import com.amazonaws.auth.EnvironmentVariableCredentialsProvider;
67+
import org.apache.http.HttpRequestInterceptor;
68+
6469
import java.text.ParseException;
6570

6671
import java.util.Map;
@@ -106,10 +111,21 @@ public RefreshPolicy getRefreshPolicy() {
106111
// Used in unit test to manage test
107112
public static RestHighLevelClient buildClient(final URI pConnectionURL) {
108113
//final CredentialsProvider credentialsProvider =new BasicCredentialsProvider();credentialsProvider.setCredentials(AuthScope.ANY,new UsernamePasswordCredentials("username", "password"));
109-
RestClientBuilder builder = RestClient.builder(new HttpHost(pConnectionURL.getHost(), pConnectionURL.getPort(), pConnectionURL.getScheme()));
110-
RestHighLevelClient tClient = new RestHighLevelClient(builder);
114+
if (System.getenv("AWS_REGION") != null && System.getenv("AWS_ACCESS_KEY_ID") != null && System.getenv("AWS_SECRET_ACCESS_KEY") != null) {
115+
String tServiceName = "es";
116+
AWSCredentialsProvider credentialsProvider = new EnvironmentVariableCredentialsProvider();
117+
AWS4Signer signer = new AWS4Signer();
118+
signer.setServiceName(tServiceName);
119+
signer.setRegionName(System.getenv("AWS_REGION")); // ENV
120+
HttpRequestInterceptor interceptor = new AWSRequestSigningApacheInterceptor(tServiceName, signer, credentialsProvider);
121+
return new RestHighLevelClient(RestClient.builder(new HttpHost(pConnectionURL.getHost(), pConnectionURL.getPort(), pConnectionURL.getScheme())).setHttpClientConfigCallback(hacb -> hacb.addInterceptorLast(interceptor)));
122+
} else {
123+
RestClientBuilder builder = RestClient.builder(new HttpHost(pConnectionURL.getHost(), pConnectionURL.getPort(), pConnectionURL.getScheme()));
124+
125+
RestHighLevelClient tClient = new RestHighLevelClient(builder);
111126

112-
return tClient ;
127+
return tClient ;
128+
}
113129
}
114130

115131
protected void createIndex() throws IOException {

‎src/test/resources/aws-elastic.properties

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
# Generic properties
2+
# ==================
3+
4+
# Uncomment this if you are behind a proxy or want a public URI
5+
#baseURI=http://dev.llgc.org.uk/annotation/
6+
7+
# Uncomment this if you would like to use an encoder which will work on
8+
# the annotation before its stored in the triplestore
9+
# encoder=uk.org.llgc.annotation.store.encoders.BookOfPeaceEncode
10+
11+
# if you are using Mirador versions greater than 2.1.4 then you need to uncomment the following
12+
# as the annotation structure changed between versions
13+
#encoder=uk.org.llgc.annotation.store.encoders.Mirador214
14+
15+
# Store configuration
16+
# ==================
17+
18+
# Uncomment this if you would like to use Jena as a backend
19+
#store=jena
20+
#data_dir=/annotation-data
21+
22+
# Uncomment the following if you want to use Sesame
23+
# store=sesame
24+
# repo_url=http://localhost:8080/openrdf-sesame/repositories/test-anno
25+
26+
# Uncomment the following if you want to use SOLR cores
27+
#store=solr
28+
#solr_connection=http://localhost:8983/solr/testannotations
29+
30+
# Uncomment the following if you want to use SOLR collections (Cloud)
31+
#store=solr-cloud
32+
#solr_connection=http://solr:8983/solr,http://solr:7574/solr
33+
#solr_collection=annotations
34+
35+
# Uncoment the following if you want to use ElasticSearch as a backend:
36+
store=elastic
37+
elastic_connection=https://search-elastic-sas-nfskj6q5cw7664b3cszjo6gwmi.eu-west-2.es.amazonaws.com/testannotations
38+
39+
# Note the following enviroment variables also need to be present to test AWS elastic search:
40+
# AWS_REGION="eu-west-2"
41+
# AWS_ACCESS_KEY_ID
42+
# AWS_SECRET_ACCESS_KEY
43+
# SAS_store="elastic"
44+
# SAS_elastic_connection

0 commit comments

Comments
 (0)