Merge main into add-huggingface-text-to-image

Author: lobo

.devcontainer/scripts/onCreateCommand.sh

@@ -1,5 +1,21 @@
 #!/bin/bash
 
+#
+# Copyright 2023-2024 the original author or authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 set -x
 
 az extension add --name spring

.editorconfig

@@ -8,3 +8,5 @@ indent_style = tab
 indent_size = 4
 continuation_indent_size = 8
 end_of_line = lf
+
+insert_final_newline = true

.mvn/extensions.xml

@@ -1,4 +1,20 @@
 <?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Copyright 2023-2024 the original author or authors.
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~      https://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
 <extensions>
     <extension>
       <groupId>fr.jcgay.maven</groupId>

.mvn/wrapper/maven-wrapper.properties

@@ -1,18 +1,17 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
 #
-#   https://www.apache.org/licenses/LICENSE-2.0
+# Copyright 2023-2024 the original author or authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 #
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
 distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.8.6/apache-maven-3.8.6-bin.zip
 wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.1.1/maven-wrapper-3.1.1.jar

README.md

@@ -1,4 +1,20 @@
 <?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Copyright 2023-2024 the original author or authors.
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~      https://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
 <project xmlns="http://maven.apache.org/POM/4.0.0"
 		 xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 		 xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

document-readers/markdown-reader/pom.xml

@@ -1,18 +1,45 @@
+/*
+ * Copyright 2023-2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.springframework.ai.reader.markdown;
 
-import org.commonmark.node.*;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.commonmark.node.AbstractVisitor;
+import org.commonmark.node.BlockQuote;
+import org.commonmark.node.Code;
+import org.commonmark.node.FencedCodeBlock;
+import org.commonmark.node.HardLineBreak;
+import org.commonmark.node.Heading;
+import org.commonmark.node.ListItem;
+import org.commonmark.node.Node;
+import org.commonmark.node.SoftLineBreak;
+import org.commonmark.node.Text;
+import org.commonmark.node.ThematicBreak;
 import org.commonmark.parser.Parser;
+
 import org.springframework.ai.document.Document;
 import org.springframework.ai.document.DocumentReader;
 import org.springframework.ai.reader.markdown.config.MarkdownDocumentReaderConfig;
 import org.springframework.core.io.DefaultResourceLoader;
 import org.springframework.core.io.Resource;
 
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.List;
-
 /**
  * Reads the given Markdown resource and groups headers, paragraphs, or text divided by
  * horizontal lines (depending on the
@@ -58,10 +85,10 @@ public MarkdownDocumentReader(Resource markdownResource, MarkdownDocumentReaderC
 	 */
 	@Override
 	public List<Document> get() {
-		try (var input = markdownResource.getInputStream()) {
-			Node node = parser.parseReader(new InputStreamReader(input));
+		try (var input = this.markdownResource.getInputStream()) {
+			Node node = this.parser.parseReader(new InputStreamReader(input));
 
-			DocumentVisitor documentVisitor = new DocumentVisitor(config);
+			DocumentVisitor documentVisitor = new DocumentVisitor(this.config);
 			node.accept(documentVisitor);
 
 			return documentVisitor.getDocuments();
@@ -90,7 +117,7 @@ public DocumentVisitor(MarkdownDocumentReaderConfig config) {
 
 		@Override
 		public void visit(org.commonmark.node.Document document) {
-			currentDocumentBuilder = Document.builder();
+			this.currentDocumentBuilder = Document.builder();
 			super.visit(document);
 		}
 
@@ -102,7 +129,7 @@ public void visit(Heading heading) {
 
 		@Override
 		public void visit(ThematicBreak thematicBreak) {
-			if (config.horizontalRuleCreateDocument) {
+			if (this.config.horizontalRuleCreateDocument) {
 				buildAndFlush();
 			}
 			super.visit(thematicBreak);
@@ -128,32 +155,32 @@ public void visit(ListItem listItem) {
 
 		@Override
 		public void visit(BlockQuote blockQuote) {
-			if (!config.includeBlockquote) {
+			if (!this.config.includeBlockquote) {
 				buildAndFlush();
 			}
 
 			translateLineBreakToSpace();
-			currentDocumentBuilder.withMetadata("category", "blockquote");
+			this.currentDocumentBuilder.withMetadata("category", "blockquote");
 			super.visit(blockQuote);
 		}
 
 		@Override
 		public void visit(Code code) {
-			currentParagraphs.add(code.getLiteral());
-			currentDocumentBuilder.withMetadata("category", "code_inline");
+			this.currentParagraphs.add(code.getLiteral());
+			this.currentDocumentBuilder.withMetadata("category", "code_inline");
 			super.visit(code);
 		}
 
 		@Override
 		public void visit(FencedCodeBlock fencedCodeBlock) {
-			if (!config.includeCodeBlock) {
+			if (!this.config.includeCodeBlock) {
 				buildAndFlush();
 			}
 
 			translateLineBreakToSpace();
-			currentParagraphs.add(fencedCodeBlock.getLiteral());
-			currentDocumentBuilder.withMetadata("category", "code_block");
-			currentDocumentBuilder.withMetadata("lang", fencedCodeBlock.getInfo());
+			this.currentParagraphs.add(fencedCodeBlock.getLiteral());
+			this.currentDocumentBuilder.withMetadata("category", "code_block");
+			this.currentDocumentBuilder.withMetadata("lang", fencedCodeBlock.getInfo());
 
 			buildAndFlush();
 
@@ -163,11 +190,11 @@ public void visit(FencedCodeBlock fencedCodeBlock) {
 		@Override
 		public void visit(Text text) {
 			if (text.getParent() instanceof Heading heading) {
-				currentDocumentBuilder.withMetadata("category", "header_%d".formatted(heading.getLevel()))
+				this.currentDocumentBuilder.withMetadata("category", "header_%d".formatted(heading.getLevel()))
 					.withMetadata("title", text.getLiteral());
 			}
 			else {
-				currentParagraphs.add(text.getLiteral());
+				this.currentParagraphs.add(text.getLiteral());
 			}
 
 			super.visit(text);
@@ -176,29 +203,29 @@ public void visit(Text text) {
 		public List<Document> getDocuments() {
 			buildAndFlush();
 
-			return documents;
+			return this.documents;
 		}
 
 		private void buildAndFlush() {
-			if (!currentParagraphs.isEmpty()) {
-				String content = String.join("", currentParagraphs);
+			if (!this.currentParagraphs.isEmpty()) {
+				String content = String.join("", this.currentParagraphs);
 
-				Document.Builder builder = currentDocumentBuilder.withContent(content);
+				Document.Builder builder = this.currentDocumentBuilder.withContent(content);
 
-				config.additionalMetadata.forEach(builder::withMetadata);
+				this.config.additionalMetadata.forEach(builder::withMetadata);
 
 				Document document = builder.build();
 
-				documents.add(document);
+				this.documents.add(document);
 
-				currentParagraphs.clear();
+				this.currentParagraphs.clear();
 			}
-			currentDocumentBuilder = Document.builder();
+			this.currentDocumentBuilder = Document.builder();
 		}
 
 		private void translateLineBreakToSpace() {
-			if (!currentParagraphs.isEmpty()) {
-				currentParagraphs.add(" ");
+			if (!this.currentParagraphs.isEmpty()) {
+				this.currentParagraphs.add(" ");
 			}
 		}
 

document-readers/markdown-reader/src/main/java/org/springframework/ai/reader/markdown/MarkdownDocumentReader.java

@@ -1,12 +1,28 @@
+/*
+ * Copyright 2023-2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.springframework.ai.reader.markdown.config;
 
+import java.util.HashMap;
+import java.util.Map;
+
 import org.springframework.ai.document.Document;
 import org.springframework.ai.reader.markdown.MarkdownDocumentReader;
 import org.springframework.util.Assert;
 
-import java.util.HashMap;
-import java.util.Map;
-
 /**
  * Common configuration for the {@link MarkdownDocumentReader}.
  *
@@ -23,10 +39,10 @@ public class MarkdownDocumentReaderConfig {
 	public final Map<String, Object> additionalMetadata;
 
 	public MarkdownDocumentReaderConfig(Builder builder) {
-		horizontalRuleCreateDocument = builder.horizontalRuleCreateDocument;
-		includeCodeBlock = builder.includeCodeBlock;
-		includeBlockquote = builder.includeBlockquote;
-		additionalMetadata = builder.additionalMetadata;
+		this.horizontalRuleCreateDocument = builder.horizontalRuleCreateDocument;
+		this.includeCodeBlock = builder.includeCodeBlock;
+		this.includeBlockquote = builder.includeBlockquote;
+		this.additionalMetadata = builder.additionalMetadata;
 	}
 
 	/**

document-readers/markdown-reader/src/main/java/org/springframework/ai/reader/markdown/config/MarkdownDocumentReaderConfig.java

@@ -1,12 +1,29 @@
+/*
+ * Copyright 2023-2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.springframework.ai.reader.markdown;
 
+import java.util.List;
+import java.util.Map;
+
 import org.junit.jupiter.api.Test;
+
 import org.springframework.ai.document.Document;
 import org.springframework.ai.reader.markdown.config.MarkdownDocumentReaderConfig;
 
-import java.util.List;
-import java.util.Map;
-
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.groups.Tuple.tuple;
 

document-readers/markdown-reader/src/test/java/org/springframework/ai/reader/markdown/MarkdownDocumentReaderTest.java

@@ -1,4 +1,20 @@
 <?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Copyright 2023-2024 the original author or authors.
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~      https://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
 <project xmlns="http://maven.apache.org/POM/4.0.0"
 	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
 	<modelVersion>4.0.0</modelVersion>