Skip to content

Commit 82fc589

Browse files
stuartwdouglasstuartwdouglas
authored
Merge pull request redhat-appstudio#788 from rnc/STONEBLD1198
STONEBLD-1198 Check for preexisting builds
2 parents 5f997e8 + 34f8e64 commit 82fc589

File tree

28 files changed

+574
-307
lines changed

28 files changed

+574
-307
lines changed

‎java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/build/BuildInfo.java

+36
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,12 @@ public Map<String, VersionRange> getTools() {
5252
int additionalMemory;
5353
List<String> allowedDifferences = new ArrayList<>();
5454

55+
List<String> gavs = new ArrayList<>();
56+
57+
String digest;
58+
59+
String image;
60+
5561
public BuildInfo setTools(Map<String, VersionRange> tools) {
5662
this.tools = tools;
5763
return this;
@@ -156,6 +162,33 @@ public BuildInfo setAllowedDifferences(List<String> allowedDifferences) {
156162
return this;
157163
}
158164

165+
public List<String> getGavs() {
166+
return gavs;
167+
}
168+
169+
public BuildInfo setGavs(List<String> gavs) {
170+
this.gavs = gavs;
171+
return this;
172+
}
173+
174+
public String getDigest() {
175+
return digest;
176+
}
177+
178+
public BuildInfo setDigest(String digest) {
179+
this.digest = digest;
180+
return this;
181+
}
182+
183+
public String getImage() {
184+
return image;
185+
}
186+
187+
public BuildInfo setImage(String image) {
188+
this.image = image;
189+
return this;
190+
}
191+
159192
@Override
160193
public String toString() {
161194
return "BuildInfo{" +
@@ -171,6 +204,9 @@ public String toString() {
171204
", disableSubmodules=" + disableSubmodules +
172205
", additionalMemory=" + additionalMemory +
173206
", allowedDifferences=" + allowedDifferences +
207+
", image=" + image +
208+
", digest=" + digest +
209+
", gavs=" + gavs +
174210
'}';
175211
}
176212
}

‎java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/build/LookupBuildInfoCommand.java

+112-8
Original file line numberDiff line numberDiff line change
@@ -6,42 +6,64 @@
66
import static com.redhat.hacbs.container.analyser.build.BuildInfo.MAVEN;
77
import static com.redhat.hacbs.container.analyser.build.BuildInfo.SBT;
88
import static com.redhat.hacbs.container.analyser.build.gradle.GradleUtils.GOOGLE_JAVA_FORMAT_PLUGIN;
9+
import static org.apache.commons.lang3.StringUtils.isBlank;
10+
import static org.apache.commons.lang3.StringUtils.isNotBlank;
911
import static org.eclipse.jgit.api.ResetCommand.ResetType.HARD;
1012

1113
import java.io.BufferedReader;
14+
import java.io.ByteArrayOutputStream;
1215
import java.io.IOException;
1316
import java.net.URI;
17+
import java.nio.charset.Charset;
1418
import java.nio.file.Files;
1519
import java.nio.file.Path;
1620
import java.util.ArrayList;
21+
import java.util.Arrays;
1722
import java.util.Collection;
1823
import java.util.Collections;
1924
import java.util.HashSet;
2025
import java.util.List;
2126
import java.util.Map;
2227
import java.util.Objects;
28+
import java.util.Optional;
2329
import java.util.Set;
2430

2531
import jakarta.enterprise.inject.Instance;
2632
import jakarta.inject.Inject;
2733

2834
import org.apache.commons.cli.ParseException;
35+
import org.apache.commons.codec.digest.DigestUtils;
2936
import org.apache.maven.cli.CLIManager;
3037
import org.apache.maven.model.Model;
3138
import org.apache.maven.model.io.xpp3.MavenXpp3Reader;
3239
import org.eclipse.jgit.api.Git;
40+
import org.eclipse.microprofile.config.inject.ConfigProperty;
3341
import org.eclipse.microprofile.rest.client.RestClientBuilder;
3442

3543
import com.fasterxml.jackson.databind.ObjectMapper;
44+
import com.google.cloud.tools.jib.api.Credential;
45+
import com.google.cloud.tools.jib.api.ImageReference;
46+
import com.google.cloud.tools.jib.api.InvalidImageReferenceException;
47+
import com.google.cloud.tools.jib.api.RegistryException;
48+
import com.google.cloud.tools.jib.blob.Blob;
49+
import com.google.cloud.tools.jib.image.json.ManifestTemplate;
50+
import com.google.cloud.tools.jib.image.json.OciManifestTemplate;
51+
import com.google.cloud.tools.jib.registry.ManifestAndDigest;
52+
import com.google.cloud.tools.jib.registry.RegistryClient;
3653
import com.redhat.hacbs.container.analyser.build.ant.AntUtils;
3754
import com.redhat.hacbs.container.analyser.build.gradle.GradleUtils;
3855
import com.redhat.hacbs.container.analyser.build.maven.MavenDiscoveryTask;
56+
import com.redhat.hacbs.container.analyser.deploy.containerregistry.ContainerUtil;
3957
import com.redhat.hacbs.container.analyser.location.VersionRange;
4058
import com.redhat.hacbs.recipies.build.BuildRecipeInfo;
4159
import com.redhat.hacbs.recipies.scm.ScmInfo;
4260
import com.redhat.hacbs.recipies.util.GitCredentials;
61+
import com.redhat.hacbs.resources.model.v1alpha1.Util;
62+
import com.redhat.hacbs.resources.model.v1alpha1.jbsconfigstatus.ImageRegistry;
4363

64+
import io.fabric8.kubernetes.client.KubernetesClientException;
4465
import io.quarkus.logging.Log;
66+
import io.vertx.core.json.JsonObject;
4567
import picocli.CommandLine;
4668

4769
@CommandLine.Command(name = "lookup-build-info")
@@ -53,11 +75,14 @@ public class LookupBuildInfoCommand implements Runnable {
5375
@CommandLine.Option(names = "--scm-url", required = true)
5476
String scmUrl;
5577

78+
@CommandLine.Option(names = "--scm-commit", required = true)
79+
String commit;
80+
5681
@CommandLine.Option(names = "--scm-tag", required = true)
5782
String tag;
5883

59-
// This is a subdirectory to operate upon.
60-
@CommandLine.Option(names = "--context", required = true)
84+
// This is a subdirectory to operate upon (i.e. path)
85+
@CommandLine.Option(names = "--context")
6186
String context;
6287

6388
@CommandLine.Option(names = "--version", required = true)
@@ -68,6 +93,13 @@ public class LookupBuildInfoCommand implements Runnable {
6893

6994
@CommandLine.Option(names = "--private-repo")
7095
boolean privateRepo;
96+
97+
@CommandLine.Option(names = "--registries", description = "Denotes registries to search for preexisting builds")
98+
String registries;
99+
100+
@ConfigProperty(name = "registry.token")
101+
Optional<String> envToken;
102+
71103
/**
72104
* The build info, in JSON format as per BuildRecipe.
73105
* <p>
@@ -84,7 +116,7 @@ public class LookupBuildInfoCommand implements Runnable {
84116
public void run() {
85117
try {
86118
ScmInfo info = new ScmInfo("git", this.scmUrl);
87-
Log.infof("LookupBuildInfo::resolving " + info.getUri());
119+
Log.infof("LookupBuildInfo resolving %s for version %s ", info.getUri(), version);
88120

89121
CacheBuildInfoLocator buildInfoLocator = RestClientBuilder.newBuilder().baseUri(new URI(cacheUrl))
90122
.build(CacheBuildInfoLocator.class);
@@ -99,8 +131,8 @@ public void run() {
99131
}
100132
}
101133

102-
Log.infof("Checking out %s at tag %s", scmUrl, tag);
103-
doBuildAnalysis(info.getUriWithoutFragment(), tag, context, buildRecipeInfo, privateRepo, buildInfoLocator);
134+
Log.infof("Cloning commit %s (tag %s) for path %s", commit, tag, context);
135+
doBuildAnalysis(info.getUriWithoutFragment(), commit, context, buildRecipeInfo, privateRepo, buildInfoLocator);
104136

105137
if (message != null) {
106138
Files.createFile(message);
@@ -133,7 +165,7 @@ private void doBuildAnalysis(String scmUrl, String scmTag, String context, Build
133165
skipTests = false;
134166
}
135167
long time = clone.getRepository().parseCommit(clone.getRepository().resolve(scmTag)).getCommitTime() * 1000L;
136-
if (context != null) {
168+
if (isNotBlank(context)) {
137169
path = path.resolve(context);
138170
}
139171
BuildInfo info = new BuildInfo();
@@ -165,8 +197,7 @@ private void doBuildAnalysis(String scmUrl, String scmTag, String context, Build
165197
try (BufferedReader pomReader = Files.newBufferedReader(pomFile)) {
166198
MavenXpp3Reader reader = new MavenXpp3Reader();
167199
Model model = reader.read(pomReader);
168-
169-
//TODO: we should do discoery on the whole tree
200+
//TODO: we should do discovery on the whole tree
170201
List<DiscoveryResult> results = new ArrayList<>();
171202
if (model.getVersion() != null && model.getVersion().endsWith("-SNAPSHOT")) {
172203
//not tagged properly, deal with it automatically
@@ -310,6 +341,79 @@ private void doBuildAnalysis(String scmUrl, String scmTag, String context, Build
310341
info.setAllowedDifferences(buildRecipeInfo.getAllowedDifferences());
311342
Log.infof("Got build recipe info %s", buildRecipeInfo);
312343
}
344+
if (registries != null) {
345+
String[] splitRegistries = registries.split(";", -1);
346+
347+
for (String value : splitRegistries) {
348+
ImageRegistry registry = Util.parseRegistry(value);
349+
// Meant to match Go code that does
350+
// util.HashString(abr.Status.SCMInfo.SCMURL + abr.Status.SCMInfo.Tag + abr.Status.SCMInfo.Path)
351+
String contextPath = context == null ? "" : context;
352+
String prependTag = isBlank(registry.getPrependTag()) ? "" : registry.getPrependTag() + "_";
353+
String imageId = prependTag + DigestUtils.md5Hex(scmUrl + tag + contextPath);
354+
String port = isBlank(registry.getPort()) ? "443" : registry.getPort();
355+
String fullName = registry.getHost() + (port.equals("443") ? "" : ":" + port) + "/" + registry.getOwner()
356+
+ "/" + registry.getRepository() + ":" + imageId;
357+
Credential credential = ContainerUtil.processToken(fullName, envToken.orElse(""));
358+
359+
Log.infof("Examining registry %s for image %s", fullName, imageId);
360+
try {
361+
//TODO consider authentication whether via env token or dockerconfig. Would need to pass
362+
// token in env as per ContainerRegistryDeployer?
363+
ImageReference reference = ImageReference.parse(fullName);
364+
RegistryClient registryClient = ContainerUtil.getRegistryClient(reference, credential,
365+
registry.getInsecure());
366+
Optional<ManifestAndDigest<ManifestTemplate>> manifestAndDigest = registryClient
367+
.checkManifest(reference.getTag().get());
368+
369+
if (manifestAndDigest.isPresent()) {
370+
// Found a potential manifest match for the build - now obtain the container manifest JSON
371+
// so we can obtain the correct GAVs.
372+
Blob blob = registryClient.pullBlob(((OciManifestTemplate) manifestAndDigest.get()
373+
.getManifest()).getContainerConfiguration()
374+
.getDigest(),
375+
s -> {
376+
}, s -> {
377+
});
378+
379+
ByteArrayOutputStream bao = new ByteArrayOutputStream();
380+
blob.writeTo(bao);
381+
String fromManifest = bao.toString(Charset.defaultCharset());
382+
JsonObject tagListJson = new JsonObject(fromManifest);
383+
JsonObject configJson = tagListJson.getJsonObject("config");
384+
JsonObject labels = configJson.getJsonObject("Labels");
385+
// ContainerRegistryDeployer uses a comma separated list
386+
List<String> gavList;
387+
if (labels.getMap().containsKey("io.jvmbuildservice.gavs")) {
388+
gavList = Arrays.asList(
389+
labels.getString("io.jvmbuildservice.gavs").split(",", -1));
390+
} else {
391+
// Backwards compatibility check for builds with older label format.
392+
String g = labels.getString("groupId");
393+
String v = labels.getString("version");
394+
gavList = new ArrayList<>();
395+
Arrays.stream(labels.getString("artifactId").split(","))
396+
.forEach(a -> gavList.add(g + ":" + a + ":" + v));
397+
}
398+
info.setGavs(gavList);
399+
info.setImage(fullName);
400+
info.setDigest(manifestAndDigest.get().getDigest().toString());
401+
Log.infof(
402+
"Found GAVs " + gavList + " and digest " + manifestAndDigest.get()
403+
.getDigest());
404+
break;
405+
}
406+
} catch (InvalidImageReferenceException | IOException | RegistryException e) {
407+
// Manually deleting a tag on quay.io gives
408+
// Tried to pull image manifest for <...>:975ea3800099190263d38f051c1a188a but failed
409+
// because: unknown error code: TAG_EXPIRED
410+
if (!e.getMessage().contains("TAG_EXPIRED")) {
411+
throw new KubernetesClientException(e.toString());
412+
}
413+
Log.errorf("Registry tag expired - " + e.getMessage());
414+
}
415+
}
416+
}
313417
ObjectMapper mapper = new ObjectMapper();
314418
Log.infof("Writing %s to %s", info, buildInfo.toFile());
315419
mapper.writeValue(buildInfo.toFile(), info);

‎java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/dependencies/AnalyseImage.java

+3-14
Original file line numberDiff line numberDiff line change
@@ -17,14 +17,10 @@
1717
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
1818
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
1919

20-
import com.google.cloud.tools.jib.api.Credential;
2120
import com.google.cloud.tools.jib.api.DescriptorDigest;
2221
import com.google.cloud.tools.jib.api.ImageReference;
2322
import com.google.cloud.tools.jib.api.InvalidImageReferenceException;
2423
import com.google.cloud.tools.jib.api.RegistryException;
25-
import com.google.cloud.tools.jib.event.EventHandlers;
26-
import com.google.cloud.tools.jib.frontend.CredentialRetrieverFactory;
27-
import com.google.cloud.tools.jib.http.FailoverHttpClient;
2824
import com.google.cloud.tools.jib.image.json.BuildableManifestTemplate;
2925
import com.google.cloud.tools.jib.image.json.ManifestTemplate;
3026
import com.google.cloud.tools.jib.image.json.V21ManifestTemplate;
@@ -33,6 +29,7 @@
3329
import com.google.cloud.tools.jib.registry.credentials.CredentialRetrievalException;
3430
import com.redhat.hacbs.classfile.tracker.NoCloseInputStream;
3531
import com.redhat.hacbs.classfile.tracker.TrackingData;
32+
import com.redhat.hacbs.container.analyser.deploy.containerregistry.ContainerUtil;
3633

3734
import io.quarkus.logging.Log;
3835
import picocli.CommandLine;
@@ -82,17 +79,9 @@ void doAnalysis(Set<String> gavs, Set<TrackingData> trackingData) throws Excepti
8279

8380
RegistryClient extractLayers(String image, Consumer<DescriptorDigest> layerConsumer)
8481
throws InvalidImageReferenceException, IOException, RegistryException, CredentialRetrievalException {
85-
ImageReference imageReference = ImageReference.parse(image);
86-
CredentialRetrieverFactory credentialRetrieverFactory = CredentialRetrieverFactory.forImage(imageReference,
87-
(s) -> System.err.println(s.getMessage()));
88-
RegistryClient.Factory factory = RegistryClient.factory(new EventHandlers.Builder().build(),
89-
imageReference.getRegistry(),
90-
imageReference.getRepository(), new FailoverHttpClient(false, false, s -> System.out.println(s.getMessage())));
91-
92-
Optional<Credential> optionalCredential = credentialRetrieverFactory.dockerConfig().retrieve();
93-
optionalCredential.ifPresent(factory::setCredential);
94-
RegistryClient registryClient = factory.newRegistryClient();
9582

83+
ImageReference imageReference = ImageReference.parse(image);
84+
RegistryClient registryClient = ContainerUtil.getRegistryClient(imageReference, null, false);
9685
ManifestAndDigest<ManifestTemplate> result = registryClient.pullManifest(imageReference.getQualifier());
9786
if (result.getManifest() instanceof V21ManifestTemplate) {
9887
V21ManifestTemplate template = (V21ManifestTemplate) result.getManifest();

‎java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/DeployCommand.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -339,7 +339,7 @@ private Optional<Gav> getGav(String entryName, String prependTag) {
339339
List<String> groupIdList = pathParts.subList(0, numberOfParts - 3);
340340
String groupId = String.join(DOT, groupIdList);
341341

342-
return Optional.of(Gav.create(groupId, artifactId, version, prependTag));
342+
return Optional.of(Gav.create(groupId, artifactId, version));
343343
}
344344
return Optional.empty();
345345
}

‎java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/DeployData.java

+2-7
Original file line numberDiff line numberDiff line change
@@ -10,14 +10,9 @@ public final class DeployData {
1010
private final Path artifactsPath;
1111
private final Set<Gav> gavs;
1212

13-
public DeployData(Path artifactsPath, Set<Gav> gavs) {
13+
public DeployData(Path artifactsPath, Set<String> gavs) {
1414
this.artifactsPath = artifactsPath;
15-
this.gavs = gavs;
16-
}
17-
18-
public DeployData(Path artifactsPath, Set<String> gavs, String prependTag) {
19-
this.artifactsPath = artifactsPath;
20-
this.gavs = gavs.stream().map(s -> Gav.parse(s, prependTag)).collect(Collectors.toSet());
15+
this.gavs = gavs.stream().map(Gav::parse).collect(Collectors.toSet());
2116
}
2217

2318
public Path getArtifactsPath() {

‎java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/DeployerUtil.java

-39
This file was deleted.

0 commit comments

Comments
 (0)