Upgrade to Lucene 10.2.0

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/VectorScorerBenchmark.java

@@ -19,6 +19,7 @@
 import org.apache.lucene.store.MMapDirectory;
 import org.apache.lucene.util.hnsw.RandomVectorScorer;
 import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;
+import org.apache.lucene.util.hnsw.UpdateableRandomVectorScorer;
 import org.apache.lucene.util.quantization.QuantizedByteVectorValues;
 import org.apache.lucene.util.quantization.ScalarQuantizer;
 import org.elasticsearch.common.logging.LogConfigurator;
@@ -76,10 +77,10 @@ public class VectorScorerBenchmark {
     float vec2Offset;
     float scoreCorrectionConstant;
 
-    RandomVectorScorer luceneDotScorer;
-    RandomVectorScorer luceneSqrScorer;
-    RandomVectorScorer nativeDotScorer;
-    RandomVectorScorer nativeSqrScorer;
+    UpdateableRandomVectorScorer luceneDotScorer;
+    UpdateableRandomVectorScorer luceneSqrScorer;
+    UpdateableRandomVectorScorer nativeDotScorer;
+    UpdateableRandomVectorScorer nativeSqrScorer;
 
     RandomVectorScorer luceneDotScorerQuery;
     RandomVectorScorer nativeDotScorerQuery;
@@ -118,12 +119,16 @@ public void setup() throws IOException {
         in = dir.openInput("vector.data", IOContext.DEFAULT);
         var values = vectorValues(dims, 2, in, VectorSimilarityFunction.DOT_PRODUCT);
         scoreCorrectionConstant = values.getScalarQuantizer().getConstantMultiplier();
-        luceneDotScorer = luceneScoreSupplier(values, VectorSimilarityFunction.DOT_PRODUCT).scorer(0);
+        luceneDotScorer = luceneScoreSupplier(values, VectorSimilarityFunction.DOT_PRODUCT).scorer();
+        luceneDotScorer.setScoringOrdinal(0);
         values = vectorValues(dims, 2, in, VectorSimilarityFunction.EUCLIDEAN);
-        luceneSqrScorer = luceneScoreSupplier(values, VectorSimilarityFunction.EUCLIDEAN).scorer(0);
+        luceneSqrScorer = luceneScoreSupplier(values, VectorSimilarityFunction.EUCLIDEAN).scorer();
+        luceneSqrScorer.setScoringOrdinal(0);
 
-        nativeDotScorer = factory.getInt7SQVectorScorerSupplier(DOT_PRODUCT, in, values, scoreCorrectionConstant).get().scorer(0);
-        nativeSqrScorer = factory.getInt7SQVectorScorerSupplier(EUCLIDEAN, in, values, scoreCorrectionConstant).get().scorer(0);
+        nativeDotScorer = factory.getInt7SQVectorScorerSupplier(DOT_PRODUCT, in, values, scoreCorrectionConstant).get().scorer();
+        nativeDotScorer.setScoringOrdinal(0);
+        nativeSqrScorer = factory.getInt7SQVectorScorerSupplier(EUCLIDEAN, in, values, scoreCorrectionConstant).get().scorer();
+        nativeSqrScorer.setScoringOrdinal(0);
 
         // setup for getInt7SQVectorScorer / query vector scoring
         float[] queryVec = new float[dims];

build-tools-internal/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 9.1.0
-lucene            = 10.1.0
+lucene            = 10.2.0
 
 bundled_jdk_vendor = openjdk
 bundled_jdk = 24+36@1f9ff9062db4449d8ca828c504ffae90
@@ -8,7 +8,7 @@ spatial4j         = 0.7
 jts               = 1.15.0
 jackson           = 2.15.0
 snakeyaml         = 2.0
-icu4j             = 68.2
+icu4j             = 77.1
 supercsv          = 2.4.0
 log4j             = 2.19.0
 slf4j             = 2.0.6

docs/Versions.asciidoc

@@ -1,8 +1,8 @@
 
 include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[]
 
-:lucene_version:        10.1.0
-:lucene_version_path:   10_1_0
+:lucene_version:        10.2.0
+:lucene_version_path:   10_2_0
 :jdk:                   11.0.2
 :jdk_major:             11
 :build_type:            tar

docs/changelog/126594.yaml

@@ -0,0 +1,5 @@
+pr: 126594
+summary: Upgrade to Lucene 10.2.0
+area: Search
+type: upgrade
+issues: []

libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/Int7SQVectorScorerSupplier.java

@@ -10,8 +10,8 @@
 package org.elasticsearch.simdvec.internal;
 
 import org.apache.lucene.store.MemorySegmentAccessInput;
-import org.apache.lucene.util.hnsw.RandomVectorScorer;
 import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;
+import org.apache.lucene.util.hnsw.UpdateableRandomVectorScorer;
 import org.apache.lucene.util.quantization.QuantizedByteVectorValues;
 import org.apache.lucene.util.quantization.ScalarQuantizedVectorSimilarity;
 
@@ -55,9 +55,6 @@ protected final void checkOrdinal(int ord) {
     }
 
     final float scoreFromOrds(int firstOrd, int secondOrd) throws IOException {
-        checkOrdinal(firstOrd);
-        checkOrdinal(secondOrd);
-
         final int length = dims;
         long firstByteOffset = (long) firstOrd * (length + Float.BYTES);
         long secondByteOffset = (long) secondOrd * (length + Float.BYTES);
@@ -92,13 +89,21 @@ protected final float fallbackScore(long firstByteOffset, long secondByteOffset)
     }
 
     @Override
-    public RandomVectorScorer scorer(int ord) {
-        checkOrdinal(ord);
-        return new RandomVectorScorer.AbstractRandomVectorScorer(values) {
+    public UpdateableRandomVectorScorer scorer() {
+        return new UpdateableRandomVectorScorer.AbstractUpdateableRandomVectorScorer(values) {
+            private int ord = -1;
+
             @Override
             public float score(int node) throws IOException {
+                checkOrdinal(node);
                 return scoreFromOrds(ord, node);
             }
+
+            @Override
+            public void setScoringOrdinal(int node) throws IOException {
+                checkOrdinal(node);
+                this.ord = node;
+            }
         };
     }
 

libs/simdvec/src/test/java/org/elasticsearch/simdvec/VectorScorerFactoryTests.java

@@ -19,8 +19,8 @@
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.store.MMapDirectory;
-import org.apache.lucene.util.hnsw.RandomVectorScorer;
 import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;
+import org.apache.lucene.util.hnsw.UpdateableRandomVectorScorer;
 import org.apache.lucene.util.quantization.QuantizedByteVectorValues;
 import org.apache.lucene.util.quantization.ScalarQuantizer;
 
@@ -50,6 +50,8 @@
 // @com.carrotsearch.randomizedtesting.annotations.Repeat(iterations = 100)
 public class VectorScorerFactoryTests extends AbstractVectorTestCase {
 
+    private static final float DELTA = 1e-4f;
+
     // bounds of the range of values that can be seen by int7 scalar quantized vectors
     static final byte MIN_INT7_VALUE = 0;
     static final byte MAX_INT7_VALUE = 127;
@@ -99,10 +101,13 @@ void testSimpleImpl(long maxChunkSize) throws IOException {
                         float scc = values.getScalarQuantizer().getConstantMultiplier();
                         float expected = luceneScore(sim, vec1, vec2, scc, vec1Correction, vec2Correction);
 
-                        var luceneSupplier = luceneScoreSupplier(values, VectorSimilarityType.of(sim)).scorer(0);
+                        var luceneSupplier = luceneScoreSupplier(values, VectorSimilarityType.of(sim)).scorer();
+                        luceneSupplier.setScoringOrdinal(0);
                         assertThat(luceneSupplier.score(1), equalTo(expected));
                         var supplier = factory.getInt7SQVectorScorerSupplier(sim, in, values, scc).get();
-                        assertThat(supplier.scorer(0).score(1), equalTo(expected));
+                        var scorer = supplier.scorer();
+                        scorer.setScoringOrdinal(0);
+                        assertThat(scorer.score(1), equalTo(expected));
 
                         if (Runtime.version().feature() >= 22) {
                             var qScorer = factory.getInt7SQVectorScorer(VectorSimilarityType.of(sim), values, query1).get();
@@ -134,24 +139,32 @@ public void testNonNegativeDotProduct() throws IOException {
                 float expected = 0f;
                 assertThat(luceneScore(DOT_PRODUCT, vec1, vec2, 1, -5, -5), equalTo(expected));
                 var supplier = factory.getInt7SQVectorScorerSupplier(DOT_PRODUCT, in, values, 1).get();
-                assertThat(supplier.scorer(0).score(1), equalTo(expected));
-                assertThat(supplier.scorer(0).score(1), greaterThanOrEqualTo(0f));
+                var scorer = supplier.scorer();
+                scorer.setScoringOrdinal(0);
+                assertThat(scorer.score(1), equalTo(expected));
+                assertThat(scorer.score(1), greaterThanOrEqualTo(0f));
                 // max inner product
                 expected = luceneScore(MAXIMUM_INNER_PRODUCT, vec1, vec2, 1, -5, -5);
                 supplier = factory.getInt7SQVectorScorerSupplier(MAXIMUM_INNER_PRODUCT, in, values, 1).get();
-                assertThat(supplier.scorer(0).score(1), greaterThanOrEqualTo(0f));
-                assertThat(supplier.scorer(0).score(1), equalTo(expected));
+                scorer = supplier.scorer();
+                scorer.setScoringOrdinal(0);
+                assertThat(scorer.score(1), greaterThanOrEqualTo(0f));
+                assertThat(scorer.score(1), equalTo(expected));
                 // cosine
                 expected = 0f;
                 assertThat(luceneScore(COSINE, vec1, vec2, 1, -5, -5), equalTo(expected));
                 supplier = factory.getInt7SQVectorScorerSupplier(COSINE, in, values, 1).get();
-                assertThat(supplier.scorer(0).score(1), equalTo(expected));
-                assertThat(supplier.scorer(0).score(1), greaterThanOrEqualTo(0f));
+                scorer = supplier.scorer();
+                scorer.setScoringOrdinal(0);
+                assertThat(scorer.score(1), equalTo(expected));
+                assertThat(scorer.score(1), greaterThanOrEqualTo(0f));
                 // euclidean
                 expected = luceneScore(EUCLIDEAN, vec1, vec2, 1, -5, -5);
                 supplier = factory.getInt7SQVectorScorerSupplier(EUCLIDEAN, in, values, 1).get();
-                assertThat(supplier.scorer(0).score(1), equalTo(expected));
-                assertThat(supplier.scorer(0).score(1), greaterThanOrEqualTo(0f));
+                scorer = supplier.scorer();
+                scorer.setScoringOrdinal(0);
+                assertThat(scorer.score(1), equalTo(expected));
+                assertThat(scorer.score(1), greaterThanOrEqualTo(0f));
             }
         }
     }
@@ -208,7 +221,9 @@ void testRandomSupplier(long maxChunkSize, Function<Integer, byte[]> byteArraySu
                         var values = vectorValues(dims, size, in, VectorSimilarityType.of(sim));
                         float expected = luceneScore(sim, vectors[idx0], vectors[idx1], correction, offsets[idx0], offsets[idx1]);
                         var supplier = factory.getInt7SQVectorScorerSupplier(sim, in, values, correction).get();
-                        assertThat(supplier.scorer(idx0).score(idx1), equalTo(expected));
+                        var scorer = supplier.scorer();
+                        scorer.setScoringOrdinal(idx0);
+                        assertThat(scorer.score(idx1), equalTo(expected));
                     }
                 }
             }
@@ -265,7 +280,7 @@ void testRandomScorerImpl(long maxChunkSize, Function<Integer, float[]> floatArr
 
                         var expected = luceneScore(sim, qVectors[idx0], qVectors[idx1], correction, corrections[idx0], corrections[idx1]);
                         var scorer = factory.getInt7SQVectorScorer(VectorSimilarityType.of(sim), values, vectors[idx0]).get();
-                        assertThat(scorer.score(idx1), equalTo(expected));
+                        assertEquals(scorer.score(idx1), expected, DELTA);
                     }
                 }
             }
@@ -313,7 +328,9 @@ void testRandomSliceImpl(int dims, long maxChunkSize, int initialPadding, Functi
                             var values = vectorValues(dims, size, in, VectorSimilarityType.of(sim));
                             float expected = luceneScore(sim, vectors[idx0], vectors[idx1], correction, offsets[idx0], offsets[idx1]);
                             var supplier = factory.getInt7SQVectorScorerSupplier(sim, in, values, correction).get();
-                            assertThat(supplier.scorer(idx0).score(idx1), equalTo(expected));
+                            var scorer = supplier.scorer();
+                            scorer.setScoringOrdinal(idx0);
+                            assertThat(scorer.score(idx1), equalTo(expected));
                         }
                     }
                 }
@@ -352,7 +369,9 @@ public void testLarge() throws IOException {
                         var values = vectorValues(dims, size, in, VectorSimilarityType.of(sim));
                         float expected = luceneScore(sim, vector(idx0, dims), vector(idx1, dims), correction, off0, off1);
                         var supplier = factory.getInt7SQVectorScorerSupplier(sim, in, values, correction).get();
-                        assertThat(supplier.scorer(idx0).score(idx1), equalTo(expected));
+                        var scorer = supplier.scorer();
+                        scorer.setScoringOrdinal(idx0);
+                        assertThat(scorer.score(idx1), equalTo(expected));
                     }
                 }
             }
@@ -391,8 +410,8 @@ void testRaceImpl(VectorSimilarityType sim) throws Exception {
                 var values = vectorValues(dims, 4, in, VectorSimilarityType.of(sim));
                 var scoreSupplier = factory.getInt7SQVectorScorerSupplier(sim, in, values, 1f).get();
                 var tasks = List.<Callable<Optional<Throwable>>>of(
-                    new ScoreCallable(scoreSupplier.copy().scorer(0), 1, expectedScore1),
-                    new ScoreCallable(scoreSupplier.copy().scorer(2), 3, expectedScore2)
+                    new ScoreCallable(scoreSupplier.copy().scorer(), 0, 1, expectedScore1),
+                    new ScoreCallable(scoreSupplier.copy().scorer(), 2, 3, expectedScore2)
                 );
                 var executor = Executors.newFixedThreadPool(2);
                 var results = executor.invokeAll(tasks);
@@ -408,14 +427,19 @@ void testRaceImpl(VectorSimilarityType sim) throws Exception {
 
     static class ScoreCallable implements Callable<Optional<Throwable>> {
 
-        final RandomVectorScorer scorer;
+        final UpdateableRandomVectorScorer scorer;
         final int ord;
         final float expectedScore;
 
-        ScoreCallable(RandomVectorScorer scorer, int ord, float expectedScore) {
-            this.scorer = scorer;
-            this.ord = ord;
-            this.expectedScore = expectedScore;
+        ScoreCallable(UpdateableRandomVectorScorer scorer, int queryOrd, int ord, float expectedScore) {
+            try {
+                this.scorer = scorer;
+                this.scorer.setScoringOrdinal(queryOrd);
+                this.ord = ord;
+                this.expectedScore = expectedScore;
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
         }
 
         @Override

modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/AbstractCompoundWordTokenFilterFactory.java

@@ -28,6 +28,8 @@ public abstract class AbstractCompoundWordTokenFilterFactory extends AbstractTok
     protected final int maxSubwordSize;
     protected final boolean onlyLongestMatch;
     protected final CharArraySet wordList;
+    // TODO expose this parameter?
+    protected final boolean reuseChars;
 
     protected AbstractCompoundWordTokenFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(name);
@@ -36,6 +38,8 @@ protected AbstractCompoundWordTokenFilterFactory(IndexSettings indexSettings, En
         minSubwordSize = settings.getAsInt("min_subword_size", CompoundWordTokenFilterBase.DEFAULT_MIN_SUBWORD_SIZE);
         maxSubwordSize = settings.getAsInt("max_subword_size", CompoundWordTokenFilterBase.DEFAULT_MAX_SUBWORD_SIZE);
         onlyLongestMatch = settings.getAsBoolean("only_longest_match", false);
+        // TODO is the default of true correct? see: https://github.com/apache/lucene/pull/14278
+        reuseChars = true;
         wordList = Analysis.getWordSet(env, settings, "word_list");
         if (wordList == null) {
             throw new IllegalArgumentException("word_list must be provided for [" + name + "], either as a path to a file, or directly");

modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/DictionaryCompoundWordTokenFilterFactory.java

@@ -28,6 +28,14 @@ public class DictionaryCompoundWordTokenFilterFactory extends AbstractCompoundWo
 
     @Override
     public TokenStream create(TokenStream tokenStream) {
-        return new DictionaryCompoundWordTokenFilter(tokenStream, wordList, minWordSize, minSubwordSize, maxSubwordSize, onlyLongestMatch);
+        return new DictionaryCompoundWordTokenFilter(
+            tokenStream,
+            wordList,
+            minWordSize,
+            minSubwordSize,
+            maxSubwordSize,
+            onlyLongestMatch,
+            reuseChars
+        );
     }
 }

server/src/main/java/org/elasticsearch/index/IndexVersions.java

@@ -159,6 +159,7 @@ private static Version parseUnchecked(String version) {
     public static final IndexVersion SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_UNSIGNED_LONG = def(9_019_0_00, Version.LUCENE_10_1_0);
     public static final IndexVersion SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_SCALED_FLOAT = def(9_020_0_00, Version.LUCENE_10_1_0);
     public static final IndexVersion USE_LUCENE101_POSTINGS_FORMAT = def(9_021_0_00, Version.LUCENE_10_1_0);
+    public static final IndexVersion UPGRADE_TO_LUCENE_10_2_0 = def(9_022_00_0, Version.LUCENE_10_2_0);
     /*
      * STOP! READ THIS FIRST! No, really,
      *        ____ _____ ___  ____  _        ____  _____    _    ____    _____ _   _ ___ ____    _____ ___ ____  ____ _____ _

server/src/main/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorsFormat.java

@@ -22,6 +22,7 @@
 import org.apache.lucene.util.VectorUtil;
 import org.apache.lucene.util.hnsw.RandomVectorScorer;
 import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;
+import org.apache.lucene.util.hnsw.UpdateableRandomVectorScorer;
 import org.apache.lucene.util.quantization.QuantizedByteVectorValues;
 
 import java.io.IOException;
@@ -130,18 +131,33 @@ public float score(int i) throws IOException {
     }
 
     static class HammingScorerSupplier implements RandomVectorScorerSupplier {
-        private final ByteVectorValues byteValues, byteValues1, byteValues2;
+        private final ByteVectorValues byteValues, targetValues;
 
         HammingScorerSupplier(ByteVectorValues byteValues) throws IOException {
             this.byteValues = byteValues;
-            this.byteValues1 = byteValues.copy();
-            this.byteValues2 = byteValues.copy();
+            this.targetValues = byteValues.copy();
         }
 
         @Override
-        public RandomVectorScorer scorer(int i) throws IOException {
-            byte[] query = byteValues1.vectorValue(i);
-            return new HammingVectorScorer(byteValues2, query);
+        public UpdateableRandomVectorScorer scorer() throws IOException {
+            return new UpdateableRandomVectorScorer.AbstractUpdateableRandomVectorScorer(targetValues) {
+                private final byte[] query = new byte[targetValues.dimension()];
+                private int currentOrd = -1;
+
+                @Override
+                public void setScoringOrdinal(int i) throws IOException {
+                    if (currentOrd == i) {
+                        return;
+                    }
+                    System.arraycopy(targetValues.vectorValue(i), 0, query, 0, query.length);
+                    this.currentOrd = i;
+                }
+
+                @Override
+                public float score(int i) throws IOException {
+                    return hammingScore(targetValues.vectorValue(i), query);
+                }
+            };
         }
 
         @Override