Skip to content

Commit 94955a1

Browse files
nicktindallnicktindall
committed
Merge remote-tracking branch 'origin/main' into ES-12631_add_metrics_max_queue_latency
# Conflicts: # server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/WriteLoadConstraintDeciderIT.java
2 parents 0bb4e53 + 068ca76 commit 94955a1

File tree

154 files changed

+4266
-803
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

154 files changed

+4266
-803
lines changed

‎.github/CODEOWNERS‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,11 @@ x-pack/plugin/apm-data/src/yamlRestTest/resources @elastic/obs-ds-intake-service
3434
x-pack/plugin/otel-data/src/main/resources @elastic/obs-ds-intake-services
3535
x-pack/plugin/otel-data/src/yamlRestTest/resources @elastic/obs-ds-intake-services
3636

37+
# Storage Engine
38+
x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/otlp @elastic/es-storage-engine
39+
x-pack/plugin/otel-data/src/test/java/org/elasticsearch/xpack/oteldata/otlp @elastic/es-storage-engine
40+
x-pack/plugin/otel-data/src/javaRestTest/java/org/elasticsearch/xpack/oteldata/otlp @elastic/es-storage-engine
41+
3742
# Delivery
3843
gradle @elastic/es-delivery
3944
build-conventions @elastic/es-delivery

‎benchmarks/src/main/java/org/elasticsearch/benchmark/exponentialhistogram/ExponentialHistogramMergeBench.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ private ExponentialHistogram asCompressedHistogram(ExponentialHistogram histogra
130130
CompressedExponentialHistogram.writeHistogramBytes(histoBytes, histogram.scale(), negativeBuckets, positiveBuckets);
131131
CompressedExponentialHistogram result = new CompressedExponentialHistogram();
132132
BytesRef data = histoBytes.bytes().toBytesRef();
133-
result.reset(histogram.zeroBucket().zeroThreshold(), totalCount, histogram.sum(), histogram.min(), data);
133+
result.reset(histogram.zeroBucket().zeroThreshold(), totalCount, histogram.sum(), histogram.min(), histogram.max(), data);
134134
return result;
135135
} catch (IOException e) {
136136
throw new RuntimeException(e);

‎benchmarks/src/main/java/org/elasticsearch/benchmark/vector/OSQScorerBenchmark.java‎

Lines changed: 88 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,10 @@
1414
import org.apache.lucene.store.IndexInput;
1515
import org.apache.lucene.store.IndexOutput;
1616
import org.apache.lucene.store.MMapDirectory;
17-
import org.apache.lucene.util.VectorUtil;
17+
import org.apache.lucene.store.NIOFSDirectory;
1818
import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;
1919
import org.elasticsearch.common.logging.LogConfigurator;
20+
import org.elasticsearch.core.IOUtils;
2021
import org.elasticsearch.simdvec.ES91OSQVectorsScorer;
2122
import org.elasticsearch.simdvec.internal.vectorization.ESVectorizationProvider;
2223
import org.openjdk.jmh.annotations.Benchmark;
@@ -29,6 +30,7 @@
2930
import org.openjdk.jmh.annotations.Scope;
3031
import org.openjdk.jmh.annotations.Setup;
3132
import org.openjdk.jmh.annotations.State;
33+
import org.openjdk.jmh.annotations.TearDown;
3234
import org.openjdk.jmh.annotations.Warmup;
3335
import org.openjdk.jmh.infra.Blackhole;
3436

@@ -66,9 +68,14 @@ public class OSQScorerBenchmark {
6668
float centroidDp;
6769

6870
byte[] scratch;
69-
ES91OSQVectorsScorer scorer;
71+
ES91OSQVectorsScorer scorerMmap;
72+
ES91OSQVectorsScorer scorerNfios;
7073

71-
IndexInput in;
74+
Directory dirMmap;
75+
IndexInput inMmap;
76+
77+
Directory dirNiofs;
78+
IndexInput inNiofs;
7279

7380
float[] scratchScores;
7481
float[] corrections;
@@ -84,18 +91,24 @@ public void setup() throws IOException {
8491
random.nextBytes(binaryVector);
8592
}
8693

87-
Directory dir = new MMapDirectory(Files.createTempDirectory("vectorData"));
88-
IndexOutput out = dir.createOutput("vectors", IOContext.DEFAULT);
94+
dirMmap = new MMapDirectory(Files.createTempDirectory("vectorDataMmap"));
95+
dirNiofs = new NIOFSDirectory(Files.createTempDirectory("vectorDataNFIOS"));
96+
IndexOutput outMmap = dirMmap.createOutput("vectors", IOContext.DEFAULT);
97+
IndexOutput outNfios = dirNiofs.createOutput("vectors", IOContext.DEFAULT);
8998
byte[] correctionBytes = new byte[14 * ES91OSQVectorsScorer.BULK_SIZE];
9099
for (int i = 0; i < numVectors; i += ES91OSQVectorsScorer.BULK_SIZE) {
91100
for (int j = 0; j < ES91OSQVectorsScorer.BULK_SIZE; j++) {
92-
out.writeBytes(binaryVectors[i + j], 0, binaryVectors[i + j].length);
101+
outMmap.writeBytes(binaryVectors[i + j], 0, binaryVectors[i + j].length);
102+
outNfios.writeBytes(binaryVectors[i + j], 0, binaryVectors[i + j].length);
93103
}
94104
random.nextBytes(correctionBytes);
95-
out.writeBytes(correctionBytes, 0, correctionBytes.length);
105+
outMmap.writeBytes(correctionBytes, 0, correctionBytes.length);
106+
outNfios.writeBytes(correctionBytes, 0, correctionBytes.length);
96107
}
97-
out.close();
98-
in = dir.openInput("vectors", IOContext.DEFAULT);
108+
outMmap.close();
109+
outNfios.close();
110+
inMmap = dirMmap.openInput("vectors", IOContext.DEFAULT);
111+
inNiofs = dirNiofs.openInput("vectors", IOContext.DEFAULT);
99112

100113
binaryQueries = new byte[numVectors][4 * length];
101114
for (byte[] binaryVector : binaryVectors) {
@@ -110,42 +123,40 @@ public void setup() throws IOException {
110123
centroidDp = random.nextFloat();
111124

112125
scratch = new byte[length];
113-
scorer = ESVectorizationProvider.getInstance().newES91OSQVectorsScorer(in, dims);
126+
scorerMmap = ESVectorizationProvider.getInstance().newES91OSQVectorsScorer(inMmap, dims);
127+
scorerNfios = ESVectorizationProvider.getInstance().newES91OSQVectorsScorer(inNiofs, dims);
114128
scratchScores = new float[16];
115129
corrections = new float[3];
116130
}
117131

132+
@TearDown
133+
public void teardown() throws IOException {
134+
IOUtils.close(dirMmap, inMmap, dirNiofs, inNiofs);
135+
}
136+
137+
@Benchmark
138+
public void scoreFromMemorySegmentOnlyVectorMmapScalar(Blackhole bh) throws IOException {
139+
scoreFromMemorySegmentOnlyVector(bh, inMmap, scorerMmap);
140+
}
141+
118142
@Benchmark
119143
@Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
120-
public void scoreFromArray(Blackhole bh) throws IOException {
121-
for (int j = 0; j < numQueries; j++) {
122-
in.seek(0);
123-
for (int i = 0; i < numVectors; i++) {
124-
in.readBytes(scratch, 0, length);
125-
float qDist = VectorUtil.int4BitDotProduct(binaryQueries[j], scratch);
126-
in.readFloats(corrections, 0, corrections.length);
127-
int addition = Short.toUnsignedInt(in.readShort());
128-
float score = scorer.score(
129-
result.lowerInterval(),
130-
result.upperInterval(),
131-
result.quantizedComponentSum(),
132-
result.additionalCorrection(),
133-
VectorSimilarityFunction.EUCLIDEAN,
134-
centroidDp,
135-
corrections[0],
136-
corrections[1],
137-
addition,
138-
corrections[2],
139-
qDist
140-
);
141-
bh.consume(score);
142-
}
143-
}
144+
public void scoreFromMemorySegmentOnlyVectorMmapVect(Blackhole bh) throws IOException {
145+
scoreFromMemorySegmentOnlyVector(bh, inMmap, scorerMmap);
146+
}
147+
148+
@Benchmark
149+
public void scoreFromMemorySegmentOnlyVectorNiofsScalar(Blackhole bh) throws IOException {
150+
scoreFromMemorySegmentOnlyVector(bh, inNiofs, scorerNfios);
144151
}
145152

146153
@Benchmark
147154
@Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
148-
public void scoreFromMemorySegmentOnlyVector(Blackhole bh) throws IOException {
155+
public void scoreFromMemorySegmentOnlyVectorNiofsVect(Blackhole bh) throws IOException {
156+
scoreFromMemorySegmentOnlyVector(bh, inNiofs, scorerNfios);
157+
}
158+
159+
private void scoreFromMemorySegmentOnlyVector(Blackhole bh, IndexInput in, ES91OSQVectorsScorer scorer) throws IOException {
149160
for (int j = 0; j < numQueries; j++) {
150161
in.seek(0);
151162
for (int i = 0; i < numVectors; i++) {
@@ -170,9 +181,29 @@ public void scoreFromMemorySegmentOnlyVector(Blackhole bh) throws IOException {
170181
}
171182
}
172183

184+
@Benchmark
185+
public void scoreFromMemorySegmentOnlyVectorBulkMmapScalar(Blackhole bh) throws IOException {
186+
scoreFromMemorySegmentOnlyVectorBulk(bh, inMmap, scorerMmap);
187+
}
188+
189+
@Benchmark
190+
@Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
191+
public void scoreFromMemorySegmentOnlyVectorBulkMmapVect(Blackhole bh) throws IOException {
192+
scoreFromMemorySegmentOnlyVectorBulk(bh, inMmap, scorerMmap);
193+
}
194+
195+
@Benchmark
196+
public void scoreFromMemorySegmentOnlyVectorBulkNiofsScalar(Blackhole bh) throws IOException {
197+
scoreFromMemorySegmentOnlyVectorBulk(bh, inNiofs, scorerNfios);
198+
}
199+
173200
@Benchmark
174201
@Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
175-
public void scoreFromMemorySegmentOnlyVectorBulk(Blackhole bh) throws IOException {
202+
public void scoreFromMemorySegmentOnlyVectorBulkNiofsVect(Blackhole bh) throws IOException {
203+
scoreFromMemorySegmentOnlyVectorBulk(bh, inNiofs, scorerNfios);
204+
}
205+
206+
private void scoreFromMemorySegmentOnlyVectorBulk(Blackhole bh, IndexInput in, ES91OSQVectorsScorer scorer) throws IOException {
176207
for (int j = 0; j < numQueries; j++) {
177208
in.seek(0);
178209
for (int i = 0; i < numVectors; i += 16) {
@@ -199,9 +230,29 @@ public void scoreFromMemorySegmentOnlyVectorBulk(Blackhole bh) throws IOExceptio
199230
}
200231
}
201232

233+
@Benchmark
234+
public void scoreFromMemorySegmentAllBulkMmapScalar(Blackhole bh) throws IOException {
235+
scoreFromMemorySegmentAllBulk(bh, inMmap, scorerMmap);
236+
}
237+
238+
@Benchmark
239+
@Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
240+
public void scoreFromMemorySegmentAllBulkMmapVect(Blackhole bh) throws IOException {
241+
scoreFromMemorySegmentAllBulk(bh, inMmap, scorerMmap);
242+
}
243+
244+
@Benchmark
245+
public void scoreFromMemorySegmentAllBulkNiofsScalar(Blackhole bh) throws IOException {
246+
scoreFromMemorySegmentAllBulk(bh, inNiofs, scorerNfios);
247+
}
248+
202249
@Benchmark
203250
@Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
204-
public void scoreFromMemorySegmentAllBulk(Blackhole bh) throws IOException {
251+
public void scoreFromMemorySegmentAllBulkNiofsVect(Blackhole bh) throws IOException {
252+
scoreFromMemorySegmentAllBulk(bh, inNiofs, scorerNfios);
253+
}
254+
255+
private void scoreFromMemorySegmentAllBulk(Blackhole bh, IndexInput in, ES91OSQVectorsScorer scorer) throws IOException {
205256
for (int j = 0; j < numQueries; j++) {
206257
in.seek(0);
207258
for (int i = 0; i < numVectors; i += 16) {

‎branches.json‎

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,29 @@
11
{
2-
"notice": "This file is not maintained outside of the main branch and should only be used for tooling.",
3-
"branches": [
2+
"notice" : "This file is not maintained outside of the main branch and should only be used for tooling.",
3+
"branches" : [
44
{
5-
"branch": "main"
5+
"branch" : "main",
6+
"version" : "9.2.0"
67
},
78
{
8-
"branch": "9.1"
9+
"branch" : "9.1",
10+
"version" : "9.1.4"
911
},
1012
{
11-
"branch": "9.0"
13+
"branch" : "9.0",
14+
"version" : "9.0.7"
1215
},
1316
{
14-
"branch": "8.19"
17+
"branch" : "8.19",
18+
"version" : "8.19.4"
1519
},
1620
{
17-
"branch": "8.18"
21+
"branch" : "8.18",
22+
"version" : "8.18.7"
1823
},
1924
{
20-
"branch": "7.17"
25+
"branch" : "7.17",
26+
"version" : "7.17.30"
2127
}
2228
]
2329
}

‎build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/VersionPropertiesPlugin.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@
1818

1919
public class VersionPropertiesPlugin implements Plugin<Project> {
2020

21+
public static final String VERSIONS_EXT = "versions";
22+
2123
@Override
2224
public void apply(Project project) {
2325
File workspaceDir = Util.locateElasticsearchWorkspace(project.getGradle());
@@ -28,6 +30,6 @@ public void apply(Project project) {
2830
.registerIfAbsent("versions", VersionPropertiesBuildService.class, spec -> {
2931
spec.getParameters().getInfoPath().set(infoPath);
3032
});
31-
project.getExtensions().add("versions", serviceProvider.get().getProperties());
33+
project.getExtensions().add(VERSIONS_EXT, serviceProvider.get().getProperties());
3234
}
3335
}

‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGitAwareGradleFuncTest.groovy‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,17 @@ abstract class AbstractGitAwareGradleFuncTest extends AbstractGradleFuncTest {
2626
execute("git clone ${remoteGitRepo.absolutePath} cloned", testProjectDir.root)
2727
buildFile = new File(testProjectDir.root, 'cloned/build.gradle')
2828
settingsFile = new File(testProjectDir.root, 'cloned/settings.gradle')
29+
versionPropertiesFile = new File(testProjectDir.root, 'cloned/build-tools-internal/version.properties')
30+
versionPropertiesFile.text = """
31+
elasticsearch = 9.1.0
32+
lucene = 10.2.2
33+
34+
bundled_jdk_vendor = openjdk
35+
bundled_jdk = 24+36@1f9ff9062db4449d8ca828c504ffae90
36+
minimumJdkVersion = 21
37+
minimumRuntimeJava = 21
38+
minimumCompilerJava = 21
39+
"""
2940
}
3041

3142
File setupGitRemote() {

‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPluginFuncTest.groovy‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -51,23 +51,23 @@ class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleF
5151
assertOutputContains(result.output, "[$bwcDistVersion] > Task :distribution:archives:darwin-tar:${expectedAssembleTaskName}")
5252

5353
where:
54-
bwcDistVersion | bwcProject | expectedAssembleTaskName
55-
"8.4.0" | "minor" | "extractedAssemble"
56-
"8.3.0" | "staged" | "extractedAssemble"
57-
"8.2.1" | "bugfix" | "extractedAssemble"
58-
"8.1.3" | "bugfix2" | "extractedAssemble"
54+
bwcDistVersion | bwcProject | expectedAssembleTaskName
55+
"8.4.0" | "major1" | "extractedAssemble"
56+
"8.3.0" | "major2" | "extractedAssemble"
57+
"8.2.1" | "major3" | "extractedAssemble"
58+
"8.1.3" | "major4" | "extractedAssemble"
5959
}
6060

6161
@Unroll
6262
def "supports #platform aarch distributions"() {
6363
when:
64-
def result = gradleRunner(":distribution:bwc:minor:buildBwc${platform.capitalize()}Aarch64Tar",
64+
def result = gradleRunner(":distribution:bwc:major1:buildBwc${platform.capitalize()}Aarch64Tar",
6565
"-DtestRemoteRepo=" + remoteGitRepo,
6666
"-Dbwc.remote=origin",
6767
"-Dbwc.dist.version=${bwcDistVersion}-SNAPSHOT")
6868
.build()
6969
then:
70-
result.task(":distribution:bwc:minor:buildBwc${platform.capitalize()}Aarch64Tar").outcome == TaskOutcome.SUCCESS
70+
result.task(":distribution:bwc:major1:buildBwc${platform.capitalize()}Aarch64Tar").outcome == TaskOutcome.SUCCESS
7171

7272
and: "assemble tasks triggered"
7373
assertOutputContains(result.output, "[$bwcDistVersion] > Task :distribution:archives:${platform}-aarch64-tar:extractedAssemble")
@@ -87,7 +87,7 @@ class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleF
8787
}
8888
8989
dependencies {
90-
expandedDist project(path: ":distribution:bwc:minor", configuration:"expanded-darwin-tar")
90+
expandedDist project(path: ":distribution:bwc:major1", configuration:"expanded-darwin-tar")
9191
}
9292
9393
tasks.register("resolveExpandedDistribution") {
@@ -109,12 +109,12 @@ class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleF
109109
.build()
110110
then:
111111
result.task(":resolveExpandedDistribution").outcome == TaskOutcome.SUCCESS
112-
result.task(":distribution:bwc:minor:buildBwcDarwinTar").outcome == TaskOutcome.SUCCESS
112+
result.task(":distribution:bwc:major1:buildBwcDarwinTar").outcome == TaskOutcome.SUCCESS
113113
and: "assemble task triggered"
114114
result.output.contains("[8.4.0] > Task :distribution:archives:darwin-tar:extractedAssemble")
115-
result.output.contains("expandedRootPath /distribution/bwc/minor/build/bwc/checkout-8.x/" +
115+
result.output.contains("expandedRootPath /distribution/bwc/major1/build/bwc/checkout-8.x/" +
116116
"distribution/archives/darwin-tar/build/install")
117-
result.output.contains("nested folder /distribution/bwc/minor/build/bwc/checkout-8.x/" +
117+
result.output.contains("nested folder /distribution/bwc/major1/build/bwc/checkout-8.x/" +
118118
"distribution/archives/darwin-tar/build/install/elasticsearch-8.4.0-SNAPSHOT")
119119
}
120120
}

‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
5151
def "resolves expanded bwc versions from source"() {
5252
given:
5353
internalBuild()
54-
bwcMinorProjectSetup()
54+
bwcMajor1ProjectSetup()
5555
buildFile << """
5656
apply plugin: 'elasticsearch.internal-distribution-download'
5757
@@ -72,16 +72,16 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
7272

7373
def result = gradleRunner("setupDistro").build()
7474
then:
75-
result.task(":distribution:bwc:minor:buildBwcExpandedTask").outcome == TaskOutcome.SUCCESS
75+
result.task(":distribution:bwc:major1:buildBwcExpandedTask").outcome == TaskOutcome.SUCCESS
7676
result.task(":setupDistro").outcome == TaskOutcome.SUCCESS
77-
assertExtractedDistroIsCreated("distribution/bwc/minor/build/install/elastic-distro",
77+
assertExtractedDistroIsCreated("distribution/bwc/major1/build/install/elastic-distro",
7878
'bwc-marker.txt')
7979
}
8080

8181
def "fails on resolving bwc versions with no bundled jdk"() {
8282
given:
8383
internalBuild()
84-
bwcMinorProjectSetup()
84+
bwcMajor1ProjectSetup()
8585
buildFile << """
8686
apply plugin: 'elasticsearch.internal-distribution-download'
8787
@@ -105,12 +105,12 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
105105
"without a bundled JDK is not supported.")
106106
}
107107

108-
private void bwcMinorProjectSetup() {
108+
private void bwcMajor1ProjectSetup() {
109109
settingsFile << """
110-
include ':distribution:bwc:minor'
110+
include ':distribution:bwc:major1'
111111
"""
112-
def bwcSubProjectFolder = testProjectDir.newFolder("distribution", "bwc", "minor")
113-
new File(bwcSubProjectFolder, 'bwc-marker.txt') << "bwc=minor"
112+
def bwcSubProjectFolder = testProjectDir.newFolder("distribution", "bwc", "major1")
113+
new File(bwcSubProjectFolder, 'bwc-marker.txt') << "bwc=major1"
114114
new File(bwcSubProjectFolder, 'build.gradle') << """
115115
apply plugin:'base'
116116

0 commit comments

Comments
 (0)