Skip to content

Implemented instance type #712

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 2 commits into
base: dev
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 0 additions & 5 deletions server/configs/application-dev.yml

This file was deleted.

5 changes: 4 additions & 1 deletion server/configs/application-local-dev-android.yml
Original file line number Diff line number Diff line change
@@ -1 +1,4 @@
extender.gradle.enabled: true
extender:
instance-type: BUILDER_ONLY
gradle:
enabled: true
1 change: 1 addition & 0 deletions server/configs/application-local-dev-app.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
extender:
instance-type: FRONTEND_ONLY
remote-builder:
enabled: true
platforms:
Expand Down
1 change: 1 addition & 0 deletions server/configs/application-local-dev.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
extender:
instance-type: BUILDER_ONLY
# sdk.location: /usr/local/extender/sdk
cache:
enabled: true
Expand Down
1 change: 1 addition & 0 deletions server/configs/application-standalone-dev.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ server:
port: 9010

extender:
instance-type: MIXED
sdk:
location: /usr/local/extender/sdk
cache-clear-on-exit: false
Expand Down
1 change: 1 addition & 0 deletions server/configs/application-test-app.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
extender:
instance-type: FRONTEND_ONLY
remote-builder:
enabled: true
platforms:
Expand Down
45 changes: 35 additions & 10 deletions server/src/main/java/com/defold/extender/ExtenderController.java
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@

import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import jakarta.transaction.NotSupportedException;

import java.io.File;
import java.io.FileInputStream;
Expand All @@ -46,9 +47,14 @@
import java.util.regex.Matcher;
import java.util.regex.Pattern;


@RestController
public class ExtenderController {
public enum InstanceType {
MIXED,
FRONTEND_ONLY,
BUILDER_ONLY
};

private static final Logger LOGGER = LoggerFactory.getLogger(ExtenderController.class);

private static final String LATEST = "latest";
Expand All @@ -65,6 +71,8 @@ public class ExtenderController {

private final RemoteEngineBuilder remoteEngineBuilder;
private Map<String, RemoteInstanceConfig> remoteBuilderPlatformMappings;
@Value("${extender.instance-type:MIXED}")
private InstanceType instanceType;
private final boolean remoteBuilderEnabled;

private static long maxPackageSize = 1024*1024*1024;
Expand Down Expand Up @@ -198,17 +206,31 @@ public void buildEngineAsync(HttpServletRequest _request,
// Regardless of success/fail status, we want to cache the uploaded files
DataCacheService.DataCacheServiceInfo uploadResultInfo = dataCacheService.cacheFiles(uploadDirectory);
metricsWriter.measureCacheUpload(uploadResultInfo.cachedFileSize.longValue(), uploadResultInfo.cachedFileCount.intValue());

String[] buildEnvDescription = ExtenderUtil.getSdksForPlatform(platform, defoldSdkService.getPlatformSdkMappings(sdkVersion));
// Build engine locally or on remote builder
if (remoteBuilderEnabled && isRemotePlatform(buildEnvDescription[0], buildEnvDescription[1])) {
LOGGER.info("Building engine on remote builder");
RemoteInstanceConfig remoteInstanceConfig = getRemoteBuilderConfig(buildEnvDescription[0], buildEnvDescription[1]);
this.remoteEngineBuilder.buildAsync(remoteInstanceConfig, uploadDirectory, platform, sdkVersion, jobDirectory, buildDirectory, metricsWriter);
} else {

if (instanceType.equals(InstanceType.BUILDER_ONLY)) {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If instance marked as "BUILDER_ONLY" - skip request for sdk mappings and start building.

asyncBuilder.asyncBuildEngine(metricsWriter, platform, sdkVersion, jobDirectory, uploadDirectory, buildDirectory);
} else {
String[] buildEnvDescription = null;
try {
// sdk version was removed (dev or beta)
buildEnvDescription = ExtenderUtil.getSdksForPlatform(platform, defoldSdkService.getPlatformSdkMappings(sdkVersion));
} catch(ExtenderException exc) {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Exception happened if we can't find sdk mappings for given hash.

if (instanceType.equals(InstanceType.FRONTEND_ONLY)) {
throw new NotSupportedException("Engine version unsupported. Please, update engine to the newer version.");
}
}
// Build engine locally or on remote builder
if (remoteBuilderEnabled && buildEnvDescription != null && isRemotePlatform(buildEnvDescription[0], buildEnvDescription[1])) {
LOGGER.info("Building engine on remote builder");
RemoteInstanceConfig remoteInstanceConfig = getRemoteBuilderConfig(buildEnvDescription[0], buildEnvDescription[1]);
this.remoteEngineBuilder.buildAsync(remoteInstanceConfig, uploadDirectory, platform, sdkVersion, jobDirectory, buildDirectory, metricsWriter);
} else if (instanceType.equals(InstanceType.MIXED)) {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If sdk mappings exist but there are no required sdk version defined and instance marked as "MIXED" - try build on current instance.

asyncBuilder.asyncBuildEngine(metricsWriter, platform, sdkVersion, jobDirectory, uploadDirectory, buildDirectory);
} else {
// no remote buidler was found and current instance can't build
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

buidler -> builder

throw new NotSupportedException("Engine version unsupported. Please, update engine to the newer version.");
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

SDK mappings exists but no record for given sdk and version was found - throw error.

}
}

response.getWriter().write(jobDirectory.getName());
response.getWriter().flush();
response.getWriter().close();
Expand All @@ -218,6 +240,9 @@ public void buildEngineAsync(HttpServletRequest _request,
throw new ExtenderException(e, "Client closed connection prematurely, build aborted");
} catch(FileUploadException e) {
throw new ExtenderException(e, "Bad request: " + e.getMessage());
} catch(NotSupportedException e) {
LOGGER.error("Unsupported engine version {}", sdkVersionString);
throw new ExtenderException("Unsupported engine version. Please, update engine to the newer version");
} catch(Exception e) {
LOGGER.error(String.format("Exception while building or sending response - SDK: %s", sdkVersion));
throw e;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -356,7 +356,9 @@ private JSONObject getRemotePlatformSdkMappings(String hash) throws IOException,
try {
JSONObject result = operation.get();
if (result == null) {
throw new ExtenderException(String.format("Cannot find or parse platform sdks mappings for hash: %s", hash));
String msg = String.format("Cannot find or parse platform sdks mappings for hash: %s", hash);
LOGGER.error(msg);
throw new ExtenderException(msg);
}
return result;
} catch (InterruptedException|ExecutionException exc) {
Expand Down
3 changes: 3 additions & 0 deletions server/src/main/resources/application.yml
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,9 @@ extender:
location: /tmp/results
cleanup-period: 20000
lifetime: 1200000
# see ExtenderController.InstanceType enum
# FRONTEND_ONLY, BUILDER_ONLY, MIXED
instance-type: MIXED

spring:
application:
Expand Down
16 changes: 16 additions & 0 deletions server/src/test/java/com/defold/extender/IntegrationTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import org.jf.dexlib2.iface.DexFile;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
import org.slf4j.Logger;
Expand Down Expand Up @@ -568,4 +569,19 @@ public void buildEngineWithDynamicLibs(TestConfiguration configuration) throws I
assertNotNull(zipFile.getEntry(getDynamicLibName(configuration.platform, "dynamic_specific2")));
}
}

@Test
public void testUnsupportedVersion() throws IOException, ExtenderClientException {
TestConfiguration configuration = new TestConfiguration(new DefoldVersion("non-exist", new Version(2, 10, 1), new String[]{ "x86_64-linux" }) , "x86_64-linux");
List<ExtenderResource> sourceFiles = Lists.newArrayList(
new FileExtenderResource("test-data/AndroidManifest.xml", "AndroidManifest.xml"),
new FileExtenderResource("test-data/ext2/ext.manifest"),
new FileExtenderResource("test-data/ext2/src/test_ext.cpp"),
new FileExtenderResource(String.format("test-data/ext2/lib/%s/%s", configuration.platform, getLibName(configuration.platform, "alib"))),
new FileExtenderResource(String.format("test-data/ext2/lib/%s/%s", configuration.platform, getLibName(configuration.platform, "blib")))
);

ExtenderClientException exc = assertThrows(ExtenderClientException.class, () -> doBuild(sourceFiles, configuration));
assertTrue(exc.getMessage().contains("Unsupported engine version"));
}
}