Merge branch 'apache:master' into master

Author: kotlovs

.asf.yaml

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# https://cwiki.apache.org/confluence/display/INFRA/git+-+.asf.yaml+features
+# https://github.com/apache/infrastructure-asfyaml/blob/main/README.md
 ---
 github:
   description: "Apache Spark - A unified analytics engine for large-scale data processing"
@@ -31,6 +31,8 @@ github:
     merge: false
     squash: true
     rebase: true
+  ghp_branch: master
+  ghp_path: /docs
 
 notifications:
   pullrequests: reviews@spark.apache.org

.github/PULL_REQUEST_TEMPLATE

@@ -9,7 +9,7 @@ Thanks for sending a pull request!  Here are some tips for you:
   7. If you want to add a new configuration, please read the guideline first for naming configurations in
      'core/src/main/scala/org/apache/spark/internal/config/ConfigEntry.scala'.
   8. If you want to add or modify an error type or message, please read the guideline first in
-     'core/src/main/resources/error/README.md'.
+     'common/utils/src/main/resources/error/README.md'.
 -->
 
 ### What changes were proposed in this pull request?
@@ -33,7 +33,8 @@ Please clarify why the changes are needed. For instance,
 
 ### Does this PR introduce _any_ user-facing change?
 <!--
-Note that it means *any* user-facing change including all aspects such as the documentation fix.
+Note that it means *any* user-facing change including all aspects such as new features, bug fixes, or other behavior changes. Documentation-only updates are not considered user-facing changes.
+
 If yes, please clarify the previous behavior and the change this PR proposes - provide the console output, description and/or an example to show the behavior difference if possible.
 If possible, please also clarify if this is a user-facing change compared to the released Spark versions or within the unreleased branches such as master.
 If no, write 'No'.

.github/labeler.yml

@@ -17,140 +17,218 @@
 # under the License.
 #
 
-#
-# Pull Request Labeler Github Action Configuration: https://github.com/marketplace/actions/labeler
-#
-# Note that we currently cannot use the negatioon operator  (i.e. `!`)  for miniglob matches as they
-# would match any file that doesn't touch them. What's needed is the concept of `any `, which takes a
-# list of constraints / globs and then matches all of the constraints for either `any` of the files or
-# `all` of the files in the change set.
-#
-# However, `any`/`all` are not supported in a released version and testing off of the `main` branch
-# resulted in some other errors when testing.
-#
-# An issue has been opened upstream requesting that a release be cut that has support for all/any:
-#   - https://github.com/actions/labeler/issues/111
-#
-# While we wait for this issue to be handled upstream, we can remove
-# the negated / `!` matches for now and at least have labels again.
-#
 INFRA:
-  - ".github/**/*"
-  - "appveyor.yml"
-  - "tools/**/*"
-  - "dev/create-release/**/*"
-  - ".asf.yaml"
-  - ".gitattributes"
-  - ".gitignore"
-  - "dev/merge_spark_pr.py"
-  - "dev/run-tests-jenkins*"
+  - changed-files:
+    - any-glob-to-any-file: [
+     '.github/**/*',
+     'tools/**/*',
+     'dev/create-release/**/*',
+     '.asf.yaml',
+     '.gitattributes',
+     '.gitignore',
+     'dev/merge_spark_pr.py'
+    ]
+
 BUILD:
- # Can be supported when a stable release with correct all/any is released
- #- any: ['dev/**/*', '!dev/merge_spark_pr.py', '!dev/.rat-excludes']
- - "dev/**/*"
- - "build/**/*"
- - "project/**/*"
- - "assembly/**/*"
- - "**/*pom.xml"
- - "bin/docker-image-tool.sh"
- - "bin/find-spark-home*"
- - "scalastyle-config.xml"
- # These can be added in the above `any` clause (and the /dev/**/* glob removed) when
- # `any`/`all` support is released
- # - "!dev/merge_spark_pr.py"
- # - "!dev/run-tests-jenkins*"
- # - "!dev/.rat-excludes"
+  - changed-files:
+    - all-globs-to-any-file: [
+     'dev/**/*',
+     '!dev/merge_spark_pr.py'
+    ]
+    - any-glob-to-any-file: [
+     'build/**/*',
+     'project/**/*',
+     'assembly/**/*',
+     '**/*pom.xml',
+     'bin/docker-image-tool.sh',
+     'bin/find-spark-home*',
+     'scalastyle-config.xml'
+    ]
+
 DOCS:
-  - "docs/**/*"
-  - "**/README.md"
-  - "**/CONTRIBUTING.md"
+  - changed-files:
+    - any-glob-to-any-file: [
+     'docs/**/*',
+     '**/README.md',
+     '**/CONTRIBUTING.md',
+     'python/docs/**/*'
+    ]
+
 EXAMPLES:
-  - "examples/**/*"
-  - "bin/run-example*"
-# CORE needs to be updated when all/any are released upstream.
+  - changed-files:
+    - any-glob-to-any-file: [
+     'examples/**/*',
+     'bin/run-example*'
+    ]
+
 CORE:
-  # - any: ["core/**/*", "!**/*UI.scala", "!**/ui/**/*"] # If any file matches all of the globs defined in the list started by `any`, label is applied.
-  - "core/**/*"
-  - "common/kvstore/**/*"
-  - "common/network-common/**/*"
-  - "common/network-shuffle/**/*"
-  - "python/pyspark/*.py"
-  - "python/pyspark/tests/**/*.py"
+  - changed-files:
+    - all-globs-to-any-file: [
+     'core/**/*',
+     '!**/*UI.scala',
+     '!**/ui/**/*'
+    ]
+    - any-glob-to-any-file: [
+     'common/kvstore/**/*',
+     'common/network-common/**/*',
+     'common/network-shuffle/**/*',
+     'python/pyspark/*.py',
+     'python/pyspark/tests/**/*.py'
+    ]
+
 SPARK SUBMIT:
-  - "bin/spark-submit*"
+  - changed-files:
+    - any-glob-to-any-file: [
+     'bin/spark-submit*'
+    ]
+
 SPARK SHELL:
-  - "repl/**/*"
-  - "bin/spark-shell*"
+  - changed-files:
+    - any-glob-to-any-file: [
+     'repl/**/*',
+     'bin/spark-shell*'
+    ]
+
 SQL:
-#- any: ["**/sql/**/*", "!python/pyspark/sql/avro/**/*", "!python/pyspark/sql/streaming/**/*", "!python/pyspark/sql/tests/streaming/test_streaming.py"]
-  - "**/sql/**/*"
-  - "common/unsafe/**/*"
-  #- "!python/pyspark/sql/avro/**/*"
-  #- "!python/pyspark/sql/streaming/**/*"
-  #- "!python/pyspark/sql/tests/streaming/test_streaming.py"
-  - "bin/spark-sql*"
-  - "bin/beeline*"
-  - "sbin/*thriftserver*.sh"
-  - "**/*SQL*.R"
-  - "**/DataFrame.R"
-  - "**/*WindowSpec.R"
-  - "**/*catalog.R"
-  - "**/*column.R"
-  - "**/*functions.R"
-  - "**/*group.R"
-  - "**/*schema.R"
-  - "**/*types.R"
+  - changed-files:
+    - all-globs-to-any-file: [
+     '**/sql/**/*',
+     '!python/**/avro/**/*',
+     '!python/**/protobuf/**/*',
+     '!python/**/streaming/**/*'
+    ]
+    - any-glob-to-any-file: [
+     'common/unsafe/**/*',
+     'common/sketch/**/*',
+     'common/variant/**/*',
+     'bin/spark-sql*',
+     'bin/beeline*',
+     'sbin/*thriftserver*.sh',
+     '**/*SQL*.R',
+     '**/DataFrame.R',
+     '**/*WindowSpec.R',
+     '**/*catalog.R',
+     '**/*column.R',
+     '**/*functions.R',
+     '**/*group.R',
+     '**/*schema.R',
+     '**/*types.R'
+    ]
+
 AVRO:
-  - "connector/avro/**/*"
-  - "python/pyspark/sql/avro/**/*"
+  - changed-files:
+    - any-glob-to-any-file: [
+     'connector/avro/**/*',
+     'python/**/avro/**/*'
+    ]
+
 DSTREAM:
-  - "streaming/**/*"
-  - "data/streaming/**/*"
-  - "connector/kinesis*"
-  - "connector/kafka*"
-  - "python/pyspark/streaming/**/*"
+  - changed-files:
+    - any-glob-to-any-file: [
+     'streaming/**/*',
+     'data/streaming/**/*',
+     'connector/kinesis-asl/**/*',
+     'connector/kinesis-asl-assembly/**/*',
+     'connector/kafka-0-10/**/*',
+     'connector/kafka-0-10-assembly/**/*',
+     'connector/kafka-0-10-token-provider/**/*',
+     'python/pyspark/streaming/**/*'
+    ]
+
 GRAPHX:
-  - "graphx/**/*"
-  - "data/graphx/**/*"
+  - changed-files:
+    - any-glob-to-any-file: [
+     'graphx/**/*',
+     'data/graphx/**/*'
+    ]
+
 ML:
-  - "**/ml/**/*"
-  - "**/*mllib_*.R"
+  - changed-files:
+    - any-glob-to-any-file: [
+     '**/ml/**/*',
+     '**/*mllib_*.R'
+    ]
+
 MLLIB:
-  - "**/spark/mllib/**/*"
-  - "mllib-local/**/*"
-  - "python/pyspark/mllib/**/*"
+  - changed-files:
+    - any-glob-to-any-file: [
+     '**/mllib/**/*',
+     'mllib-local/**/*'
+    ]
+
 STRUCTURED STREAMING:
-  - "**/sql/**/streaming/**/*"
-  - "connector/kafka-0-10-sql/**/*"
-  - "python/pyspark/sql/streaming/**/*"
-  - "python/pyspark/sql/tests/streaming/test_streaming.py"
-  - "**/*streaming.R"
+  - changed-files:
+    - any-glob-to-any-file: [
+     '**/sql/**/streaming/**/*',
+     'connector/kafka-0-10-sql/**/*',
+     'python/pyspark/sql/**/streaming/**/*',
+     '**/*streaming.R'
+    ]
+
 PYTHON:
-  - "bin/pyspark*"
-  - "**/python/**/*"
+  - changed-files:
+    - any-glob-to-any-file: [
+     'bin/pyspark*',
+     '**/python/**/*'
+    ]
+
 PANDAS API ON SPARK:
-  - "python/pyspark/pandas/**/*"
+  - changed-files:
+    - any-glob-to-any-file: [
+     'python/pyspark/pandas/**/*'
+    ]
+
 R:
-  - "**/r/**/*"
-  - "**/R/**/*"
-  - "bin/sparkR*"
+  - changed-files:
+    - any-glob-to-any-file: [
+     '**/r/**/*',
+     '**/R/**/*',
+     'bin/sparkR*'
+    ]
+
 YARN:
-  - "resource-managers/yarn/**/*"
+  - changed-files:
+    - any-glob-to-any-file: [
+     'resource-managers/yarn/**/*'
+    ]
+
 KUBERNETES:
-  - "resource-managers/kubernetes/**/*"
+  - changed-files:
+    - any-glob-to-any-file: [
+     'bin/docker-image-tool.sh',
+     'resource-managers/kubernetes/**/*'
+    ]
+
 WINDOWS:
-  - "**/*.cmd"
-  - "R/pkg/tests/fulltests/test_Windows.R"
+  - changed-files:
+    - any-glob-to-any-file: [
+     '**/*.cmd',
+     'R/pkg/tests/fulltests/test_Windows.R'
+    ]
+
 WEB UI:
-  - "**/ui/**/*"
-  - "**/*UI.scala"
+  - changed-files:
+    - any-glob-to-any-file: [
+     '**/ui/**/*',
+     '**/*UI.scala'
+    ]
+
 DEPLOY:
-  - "sbin/**/*"
+  - changed-files:
+    - any-glob-to-any-file: [
+     'sbin/**/*'
+    ]
+
 CONNECT:
-  - "connector/connect/**/*"
-  - "**/sql/sparkconnect/**/*"
-  - "python/pyspark/sql/**/connect/**/*"
-  - "python/pyspark/ml/**/connect/**/*"
+  - changed-files:
+    - any-glob-to-any-file: [
+     'sql/connect/**/*',
+     'python/**/connect/**/*'
+    ]
+
 PROTOBUF:
-  - "connector/protobuf/**/*"
-  - "python/pyspark/sql/protobuf/**/*"
+  - changed-files:
+    - any-glob-to-any-file: [
+     'connector/protobuf/**/*',
+     'python/**/protobuf/**/*'
+    ]