From 1fb7939ac044ca6f9526ecbf196c0ee95a835792 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Fri, 29 Aug 2025 00:06:35 +0800 Subject: [PATCH 1/2] [SPARK-53420][BUILD] Upgrade Parquet to 1.16.0 RC0 --- dev/deps/spark-deps-hadoop-3-hive-2.3 | 12 ++++++------ pom.xml | 12 +++++++++++- project/SparkBuild.scala | 1 + 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3 index a2ec364961155..9fa383ccb6830 100644 --- a/dev/deps/spark-deps-hadoop-3-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3-hive-2.3 @@ -240,12 +240,12 @@ orc-shims/2.2.0//orc-shims-2.2.0.jar oro/2.0.8//oro-2.0.8.jar osgi-resource-locator/1.0.3//osgi-resource-locator-1.0.3.jar paranamer/2.8.3//paranamer-2.8.3.jar -parquet-column/1.15.2//parquet-column-1.15.2.jar -parquet-common/1.15.2//parquet-common-1.15.2.jar -parquet-encoding/1.15.2//parquet-encoding-1.15.2.jar -parquet-format-structures/1.15.2//parquet-format-structures-1.15.2.jar -parquet-hadoop/1.15.2//parquet-hadoop-1.15.2.jar -parquet-jackson/1.15.2//parquet-jackson-1.15.2.jar +parquet-column/1.16.0//parquet-column-1.16.0.jar +parquet-common/1.16.0//parquet-common-1.16.0.jar +parquet-encoding/1.16.0//parquet-encoding-1.16.0.jar +parquet-format-structures/1.16.0//parquet-format-structures-1.16.0.jar +parquet-hadoop/1.16.0//parquet-hadoop-1.16.0.jar +parquet-jackson/1.16.0//parquet-jackson-1.16.0.jar pickle/1.5//pickle-1.5.jar py4j/0.10.9.9//py4j-0.10.9.9.jar remotetea-oncrpc/1.1.2//remotetea-oncrpc-1.1.2.jar diff --git a/pom.xml b/pom.xml index e6cb98665d109..8abe1b0c57f64 100644 --- a/pom.xml +++ b/pom.xml @@ -140,7 +140,7 @@ 3.9.1 10.16.1.1 - 1.15.2 + 1.16.0 2.2.0 shaded-protobuf 11.0.25 @@ -380,6 +380,16 @@ false + + apache-staging + https://repository.apache.org/content/groups/staging + + true + + + false + + diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index b1bbdad17759b..ae15b7fddf228 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -303,6 +303,7 @@ object SparkBuild extends PomBuild { // Google Mirror of Maven Central, placed first so that it's used instead of flaky Maven Central. // See https://storage-download.googleapis.com/maven-central/index.html for more info. "gcs-maven-central-mirror" at "https://maven-central.storage-download.googleapis.com/maven2/", + "parquet-staging" at "https://repository.apache.org/content/groups/staging/", DefaultMavenRepository, Resolver.mavenLocal, Resolver.file("ivyLocal", file(Path.userHome.absolutePath + "/.ivy2/local"))(Resolver.ivyStylePatterns) From b54dbcc4f21e30ceccd5cd6d5d1fb4120cd4fa32 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Wed, 3 Sep 2025 11:06:32 +0800 Subject: [PATCH 2/2] remove staging repo --- pom.xml | 10 ---------- project/SparkBuild.scala | 1 - 2 files changed, 11 deletions(-) diff --git a/pom.xml b/pom.xml index 8abe1b0c57f64..ec78cbab5343e 100644 --- a/pom.xml +++ b/pom.xml @@ -380,16 +380,6 @@ false - - apache-staging - https://repository.apache.org/content/groups/staging - - true - - - false - - diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index ae15b7fddf228..b1bbdad17759b 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -303,7 +303,6 @@ object SparkBuild extends PomBuild { // Google Mirror of Maven Central, placed first so that it's used instead of flaky Maven Central. // See https://storage-download.googleapis.com/maven-central/index.html for more info. "gcs-maven-central-mirror" at "https://maven-central.storage-download.googleapis.com/maven2/", - "parquet-staging" at "https://repository.apache.org/content/groups/staging/", DefaultMavenRepository, Resolver.mavenLocal, Resolver.file("ivyLocal", file(Path.userHome.absolutePath + "/.ivy2/local"))(Resolver.ivyStylePatterns)