From e30ef21d16222e090374e2751d253804b520a0fb Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Tue, 25 Oct 2022 17:07:42 +0200 Subject: [PATCH 1/8] Attempt to add Python 3.11 support Python 3.11 has been released as scheduled on October 25, 2022 and this is the first attempt to see how far Airflow (mostly dependencies) are from being ready to officially support 3.11. --- CI.rst | 2 +- LOCAL_VIRTUALENV.rst | 12 +- README.md | 18 +- airflow/__init__.py | 1 + airflow/providers/amazon/aws/hooks/dms.py | 2 +- airflow/providers/apache/hive/provider.yaml | 5 + airflow/utils/log/file_task_handler.py | 2 +- dev/README_RELEASE_AIRFLOW.md | 2 +- .../src/airflow_breeze/global_constants.py | 4 +- dev/breeze/tests/test_cache.py | 4 +- dev/breeze/tests/test_selective_checks.py | 64 ++--- .../prepare_provider_packages.py | 2 +- dev/retag_docker_images.py | 2 +- .../howto/upgrading-from-1-10/index.rst | 2 +- .../installation/prerequisites.rst | 2 +- docs/docker-stack/build.rst | 16 +- generated/provider_dependencies.json | 240 ++++++++++++------ images/breeze/output_ci-image_build.svg | 4 +- images/breeze/output_ci-image_pull.svg | 60 +++-- images/breeze/output_ci-image_verify.svg | 32 ++- images/breeze/output_k8s_build-k8s-image.svg | 52 ++-- .../breeze/output_k8s_configure-cluster.svg | 60 +++-- images/breeze/output_k8s_create-cluster.svg | 4 +- images/breeze/output_k8s_delete-cluster.svg | 34 +-- images/breeze/output_k8s_deploy-airflow.svg | 70 ++--- images/breeze/output_k8s_k9s.svg | 32 ++- images/breeze/output_k8s_logs.svg | 34 +-- .../breeze/output_k8s_run-complete-tests.svg | 68 ++--- images/breeze/output_k8s_shell.svg | 40 +-- images/breeze/output_k8s_status.svg | 36 +-- images/breeze/output_k8s_tests.svg | 68 ++--- images/breeze/output_k8s_upload-k8s-image.svg | 58 +++-- images/breeze/output_prod-image_build.svg | 4 +- images/breeze/output_prod-image_pull.svg | 60 +++-- images/breeze/output_prod-image_verify.svg | 34 +-- ...elease-management_generate-constraints.svg | 4 +- ...release-management_release-prod-images.svg | 2 +- images/breeze/output_setup_config.svg | 2 +- images/breeze/output_shell.svg | 110 ++++---- images/breeze/output_start-airflow.svg | 2 +- .../output_testing_docker-compose-tests.svg | 2 +- .../output_testing_integration-tests.svg | 76 +++--- images/breeze/output_testing_tests.svg | 112 ++++---- pyproject.toml | 21 +- ...re_commit_update_providers_dependencies.py | 14 +- setup.cfg | 1 + setup.py | 18 +- tests/dag_processing/test_processor.py | 98 ++++--- tests/decorators/test_python.py | 5 +- tests/models/test_param.py | 7 +- .../providers/apache/hive/hooks/test_hive.py | 11 +- .../utils/test_mlengine_operator_utils.py | 16 +- 52 files changed, 951 insertions(+), 680 deletions(-) diff --git a/CI.rst b/CI.rst index b0b5240423243..39a5ea6ec5775 100644 --- a/CI.rst +++ b/CI.rst @@ -59,7 +59,7 @@ Container Registry used as cache We are using GitHub Container Registry to store the results of the ``Build Images`` workflow which is used in the ``Tests`` workflow. -Currently in main version of Airflow we run tests in 4 different versions of Python (3.7, 3.8, 3.9, 3.10) +Currently in main version of Airflow we run tests in those versions of Python (3.7, 3.8, 3.9, 3.10, 3.11) which means that we have to build 8 images (4 CI ones and 4 PROD ones). Yet we run around 12 jobs with each of the CI images. That is a lot of time to just build the environment to run. Therefore we are utilising ``pull_request_target`` feature of GitHub Actions. diff --git a/LOCAL_VIRTUALENV.rst b/LOCAL_VIRTUALENV.rst index e52c529094454..00205215b196c 100644 --- a/LOCAL_VIRTUALENV.rst +++ b/LOCAL_VIRTUALENV.rst @@ -51,7 +51,7 @@ Required Software Packages Use system-level package managers like yum, apt-get for Linux, or Homebrew for macOS to install required software packages: -* Python (One of: 3.7, 3.8, 3.9, 3.10) +* Python (One of: 3.7, 3.8, 3.9, 3.10, 3.11) * MySQL 5.7+ * libxml @@ -102,7 +102,7 @@ Creating a Local virtualenv To use your IDE for Airflow development and testing, you need to configure a virtual environment. Ideally you should set up virtualenv for all Python versions that Airflow -supports (3.7, 3.8, 3.9, 3.10). +supports (3.7, 3.8, 3.9, 3.10, 3.11). To create and initialize the local virtualenv: @@ -122,7 +122,7 @@ To create and initialize the local virtualenv: .. code-block:: bash - conda create -n airflow python=3.7 # or 3.8, 3.9, 3.10 + conda create -n airflow python=3.7 # or 3.8, 3.9, 3.10, 3.11 conda activate airflow 2. Install Python PIP requirements: @@ -150,7 +150,7 @@ for different python versions). For development on current main source: .. code-block:: bash - # use the same version of python as you are working with, 3.7, 3.8, 3.9, or 3.10 + # use the same version of python as you are working with, 3.7, 3.8, 3.9, 3.10 or 3.11 pip install -e ".[devel,]" \ --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-source-providers-3.7.txt" @@ -163,7 +163,7 @@ You can also install Airflow in non-editable mode: .. code-block:: bash - # use the same version of python as you are working with, 3.7, 3.8, 3.9, or 3.10 + # use the same version of python as you are working with, 3.7, 3.8, 3.9, 3.10 or 3.11 pip install ".[devel,]" \ --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-source-providers-3.7.txt" @@ -173,7 +173,7 @@ sources, unless you set ``INSTALL_PROVIDERS_FROM_SOURCES`` environment variable .. code-block:: bash - # use the same version of python as you are working with, 3.7, 3.8, 3.9, or 3.10 + # use the same version of python as you are working with, 3.7, 3.8, 3.9, 3.10 or 3.11 INSTALL_PROVIDERS_FROM_SOURCES="true" pip install ".[devel,]" \ --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-source-providers-3.7.txt" diff --git a/README.md b/README.md index a9bc5aed7ee9e..52c97fe1f97e4 100644 --- a/README.md +++ b/README.md @@ -86,15 +86,15 @@ Airflow is not a streaming solution, but it is often used to process real-time d Apache Airflow is tested with: -| | Main version (dev) | Stable version (2.6.1) | -|------------|------------------------------|------------------------| -| Python | 3.7, 3.8, 3.9, 3.10 | 3.7, 3.8, 3.9, 3.10 | -| Platform | AMD64/ARM64(\*) | AMD64/ARM64(\*) | -| Kubernetes | 1.23, 1.24, 1.25, 1.26, 1.27 | 1.23, 1.24, 1.25, 1.26 | -| PostgreSQL | 11, 12, 13, 14, 15 | 11, 12, 13, 14, 15 | -| MySQL | 5.7, 8 | 5.7, 8 | -| SQLite | 3.15.0+ | 3.15.0+ | -| MSSQL | 2017(\*), 2019(\*) | 2017(\*), 2019(\*) | +| | Main version (dev) | Stable version (2.6.1) | +|------------|------------------------------|---------------------------| +| Python | 3.7, 3.8, 3.9, 3.10, 3.11 | 3.7, 3.8, 3.9, 3.10, 3.11 | +| Platform | AMD64/ARM64(\*) | AMD64/ARM64(\*) | +| Kubernetes | 1.23, 1.24, 1.25, 1.26, 1.27 | 1.23, 1.24, 1.25, 1.26 | +| PostgreSQL | 11, 12, 13, 14, 15 | 11, 12, 13, 14, 15 | +| MySQL | 5.7, 8 | 5.7, 8 | +| SQLite | 3.15.0+ | 3.15.0+ | +| MSSQL | 2017(\*), 2019(\*) | 2017(\*), 2019(\*) | \* Experimental diff --git a/airflow/__init__.py b/airflow/__init__.py index cf5622af69b5a..f0174e5d59f49 100644 --- a/airflow/__init__.py +++ b/airflow/__init__.py @@ -72,6 +72,7 @@ PY38 = sys.version_info >= (3, 8) PY39 = sys.version_info >= (3, 9) PY310 = sys.version_info >= (3, 10) +PY311 = sys.version_info >= (3, 11) # Things to lazy import in form {local_name: ('target_module', 'target_name')} __lazy_imports: dict[str, tuple[str, str]] = { diff --git a/airflow/providers/amazon/aws/hooks/dms.py b/airflow/providers/amazon/aws/hooks/dms.py index 94cb5ac8ba2eb..a8ab6bf0a0dd6 100644 --- a/airflow/providers/amazon/aws/hooks/dms.py +++ b/airflow/providers/amazon/aws/hooks/dms.py @@ -207,7 +207,7 @@ def wait_for_task_status(self, replication_task_arn: str, status: DmsTaskWaiterS raise TypeError("Status must be an instance of DmsTaskWaiterStatus") dms_client = self.get_conn() - waiter = dms_client.get_waiter(f"replication_task_{status}") + waiter = dms_client.get_waiter(f"replication_task_{status.value}") waiter.wait( Filters=[ { diff --git a/airflow/providers/apache/hive/provider.yaml b/airflow/providers/apache/hive/provider.yaml index 0ec7e0959ba25..afeb2b2000dd8 100644 --- a/airflow/providers/apache/hive/provider.yaml +++ b/airflow/providers/apache/hive/provider.yaml @@ -63,6 +63,11 @@ dependencies: - sasl>=0.3.1; python_version>="3.9" - thrift>=0.9.2 +# Excluded because SASL is not yet compatible +# with 3.11. See https://github.com/cloudera/python-sasl/issues/30 +excluded-python-versions: + - "3.11" + integrations: - integration-name: Apache Hive external-doc-url: https://hive.apache.org/ diff --git a/airflow/utils/log/file_task_handler.py b/airflow/utils/log/file_task_handler.py index 28d8c628ccec9..194045c62635a 100644 --- a/airflow/utils/log/file_task_handler.py +++ b/airflow/utils/log/file_task_handler.py @@ -201,7 +201,7 @@ def add_triggerer_suffix(full_path, job_id=None): triggerer instances. """ full_path = Path(full_path).as_posix() - full_path += f".{LogType.TRIGGER}" + full_path += f".{LogType.TRIGGER.value}" if job_id: full_path += f".{job_id}.log" return full_path diff --git a/dev/README_RELEASE_AIRFLOW.md b/dev/README_RELEASE_AIRFLOW.md index 6d42b592083f7..7e10bcfe0ce58 100644 --- a/dev/README_RELEASE_AIRFLOW.md +++ b/dev/README_RELEASE_AIRFLOW.md @@ -670,7 +670,7 @@ the older branches, you should set the "skip" field to true. ## Verify production images ```shell script -for PYTHON in 3.7 3.8 3.9 3.10 +for PYTHON in 3.7 3.8 3.9 3.10 3.11 do docker pull apache/airflow:${VERSION}-python${PYTHON} breeze prod-image verify --image-name apache/airflow:${VERSION}-python${PYTHON} diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index eec14c2951ca0..a72f6b7513831 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -36,7 +36,7 @@ APACHE_AIRFLOW_GITHUB_REPOSITORY = "apache/airflow" # Checked before putting in build cache -ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS = ["3.7", "3.8", "3.9", "3.10"] +ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11"] DEFAULT_PYTHON_MAJOR_MINOR_VERSION = ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS[0] ALLOWED_ARCHITECTURES = [Architecture.X86_64, Architecture.ARM] ALLOWED_BACKENDS = ["sqlite", "mysql", "postgres", "mssql"] @@ -174,7 +174,7 @@ def get_default_platform_machine() -> str: PYTHONDONTWRITEBYTECODE = True PRODUCTION_IMAGE = False -ALL_PYTHON_MAJOR_MINOR_VERSIONS = ["3.7", "3.8", "3.9", "3.10"] +ALL_PYTHON_MAJOR_MINOR_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11"] CURRENT_PYTHON_MAJOR_MINOR_VERSIONS = ALL_PYTHON_MAJOR_MINOR_VERSIONS CURRENT_POSTGRES_VERSIONS = ["11", "12", "13", "14", "15"] DEFAULT_POSTGRES_VERSION = CURRENT_POSTGRES_VERSIONS[0] diff --git a/dev/breeze/tests/test_cache.py b/dev/breeze/tests/test_cache.py index a37a91165b4fc..1db27494abd07 100644 --- a/dev/breeze/tests/test_cache.py +++ b/dev/breeze/tests/test_cache.py @@ -36,8 +36,8 @@ [ ("backend", "mysql", (True, ["sqlite", "mysql", "postgres", "mssql"]), None), ("backend", "xxx", (False, ["sqlite", "mysql", "postgres", "mssql"]), None), - ("python_major_minor_version", "3.8", (True, ["3.7", "3.8", "3.9", "3.10"]), None), - ("python_major_minor_version", "3.5", (False, ["3.7", "3.8", "3.9", "3.10"]), None), + ("python_major_minor_version", "3.8", (True, ["3.7", "3.8", "3.9", "3.10", "3.11"]), None), + ("python_major_minor_version", "3.5", (False, ["3.7", "3.8", "3.9", "3.10", "3.11"]), None), ("missing", "value", None, AttributeError), ], ) diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 4ac7adb34d286..7a23c3c551e65 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -266,10 +266,10 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): ("setup.py",), { "affected-providers-list-as-string": ALL_PROVIDERS_AFFECTED, - "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", - "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", - "python-versions": "['3.7', '3.8', '3.9', '3.10']", - "python-versions-list-as-string": "3.7 3.8 3.9 3.10", + "all-python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']", + "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11", + "python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']", + "python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11", "image-build": "true", "needs-helm-tests": "true", "run-tests": "true", @@ -289,10 +289,10 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): ("generated/provider_dependencies.json",), { "affected-providers-list-as-string": ALL_PROVIDERS_AFFECTED, - "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", - "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", - "python-versions": "['3.7', '3.8', '3.9', '3.10']", - "python-versions-list-as-string": "3.7 3.8 3.9 3.10", + "all-python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']", + "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11", + "python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']", + "python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11", "image-build": "true", "needs-helm-tests": "true", "run-tests": "true", @@ -397,10 +397,10 @@ def test_expected_output_pull_request_main( "main", { "affected-providers-list-as-string": ALL_PROVIDERS_AFFECTED, - "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", - "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", - "python-versions": "['3.7', '3.8', '3.9', '3.10']", - "python-versions-list-as-string": "3.7 3.8 3.9 3.10", + "all-python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']", + "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11", + "python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']", + "python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11", "image-build": "true", "run-tests": "true", "docs-build": "true", @@ -424,10 +424,10 @@ def test_expected_output_pull_request_main( "main", { "affected-providers-list-as-string": ALL_PROVIDERS_AFFECTED, - "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", - "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", - "python-versions": "['3.7', '3.8', '3.9', '3.10']", - "python-versions-list-as-string": "3.7 3.8 3.9 3.10", + "all-python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']", + "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11", + "python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']", + "python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11", "image-build": "true", "run-tests": "true", "docs-build": "true", @@ -449,10 +449,10 @@ def test_expected_output_pull_request_main( "main", { "affected-providers-list-as-string": ALL_PROVIDERS_AFFECTED, - "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", - "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", - "python-versions": "['3.7', '3.8', '3.9', '3.10']", - "python-versions-list-as-string": "3.7 3.8 3.9 3.10", + "all-python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']", + "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11", + "python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']", + "python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11", "image-build": "true", "run-tests": "true", "docs-build": "true", @@ -474,10 +474,10 @@ def test_expected_output_pull_request_main( "v2-3-stable", { "affected-providers-list-as-string": ALL_PROVIDERS_AFFECTED, - "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", - "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", - "python-versions": "['3.7', '3.8', '3.9', '3.10']", - "python-versions-list-as-string": "3.7 3.8 3.9 3.10", + "all-python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']", + "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11", + "python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']", + "python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11", "image-build": "true", "run-tests": "true", "docs-build": "true", @@ -787,8 +787,8 @@ def test_expected_output_pull_request_target( "main", { "affected-providers-list-as-string": ALL_PROVIDERS_AFFECTED, - "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", - "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", + "all-python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']", + "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11", "image-build": "true", "needs-helm-tests": "true", "run-tests": "true", @@ -807,8 +807,8 @@ def test_expected_output_pull_request_target( "v2-3-stable", { "affected-providers-list-as-string": ALL_PROVIDERS_AFFECTED, - "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", - "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", + "all-python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']", + "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11", "image-build": "true", "needs-helm-tests": "false", "run-tests": "true", @@ -826,8 +826,8 @@ def test_expected_output_pull_request_target( "main", { "affected-providers-list-as-string": ALL_PROVIDERS_AFFECTED, - "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", - "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", + "all-python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']", + "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11", "image-build": "true", "needs-helm-tests": "true", "run-tests": "true", @@ -878,8 +878,8 @@ def test_no_commit_provided_trigger_full_build_for_any_event_type(github_event): ) assert_outputs_are_printed( { - "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", - "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", + "all-python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']", + "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11", "image-build": "true", "needs-helm-tests": "true", "run-tests": "true", diff --git a/dev/provider_packages/prepare_provider_packages.py b/dev/provider_packages/prepare_provider_packages.py index a2b7787f490ca..f78f53a122d14 100755 --- a/dev/provider_packages/prepare_provider_packages.py +++ b/dev/provider_packages/prepare_provider_packages.py @@ -52,7 +52,7 @@ from rich.syntax import Syntax from yaml import safe_load -ALL_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10"] +ALL_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11"] MIN_AIRFLOW_VERSION = "2.4.0" # In case you have some providers that you want to have different min-airflow version for, diff --git a/dev/retag_docker_images.py b/dev/retag_docker_images.py index 8c9f6eb78d386..ecc304eaa8b4c 100755 --- a/dev/retag_docker_images.py +++ b/dev/retag_docker_images.py @@ -31,7 +31,7 @@ import rich_click as click -PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10"] +PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11"] GHCR_IO_PREFIX = "ghcr.io" diff --git a/docs/apache-airflow/howto/upgrading-from-1-10/index.rst b/docs/apache-airflow/howto/upgrading-from-1-10/index.rst index c599f357f7c30..66a6cccb927b0 100644 --- a/docs/apache-airflow/howto/upgrading-from-1-10/index.rst +++ b/docs/apache-airflow/howto/upgrading-from-1-10/index.rst @@ -34,7 +34,7 @@ Airflow 1.10 was the last release series to support Python 2. Airflow 2.0.0 requires Python 3.6+ and has been tested with Python versions 3.6, 3.7 and 3.8. Python 3.9 support was added from Airflow 2.1.2. -Airflow 2.3.0 dropped support for Python 3.6. It's tested with Python 3.7, 3.8, 3.9 and 3.10. +Airflow 2.3.0 dropped support for Python 3.6. It's tested with Python 3.7, 3.8, 3.9, 3.10, 3.11. If you have a specific task that still requires Python 2 then you can use the ``@task.virtualenv``, ``@task.docker`` or ``@task.kubernetes`` decorators for this. diff --git a/docs/apache-airflow/installation/prerequisites.rst b/docs/apache-airflow/installation/prerequisites.rst index 82a0310721e3f..d10650bad0563 100644 --- a/docs/apache-airflow/installation/prerequisites.rst +++ b/docs/apache-airflow/installation/prerequisites.rst @@ -20,7 +20,7 @@ Prerequisites Starting with Airflow 2.3.0, Airflow is tested with:. -* Python: 3.7, 3.8, 3.9, 3.10 +* Python: 3.7, 3.8, 3.9, 3.10, 3.11 * Databases: diff --git a/docs/docker-stack/build.rst b/docs/docker-stack/build.rst index 7beed0afed6b0..56edb3837d6dd 100644 --- a/docs/docker-stack/build.rst +++ b/docs/docker-stack/build.rst @@ -288,14 +288,14 @@ There are two types of images you can extend your image from: Naming conventions for the images: -+----------------+------------------+---------------------------------+--------------------------------------+ -| Image | Python | Standard image | Slim image | -+================+==================+=================================+======================================+ -| Latest default | 3.7 | apache/airflow:latest | apache/airflow:slim-latest | -| Default | 3.7 | apache/airflow:X.Y.Z | apache/airflow:slim-X.Y.Z | -| Latest | 3.7,3.8,3.9,3.10 | apache/airflow:latest-pythonN.M | apache/airflow:slim-latest-pythonN.M | -| Specific | 3.7,3.8,3.9,3.10 | apache/airflow:X.Y.Z-pythonN.M | apache/airflow:slim-X.Y.Z-pythonN.M | -+----------------+------------------+---------------------------------+--------------------------------------+ ++----------------+-----------------------+---------------------------------+--------------------------------------+ +| Image | Python | Standard image | Slim image | ++================+=======================+=================================+======================================+ +| Latest default | 3.7 | apache/airflow:latest | apache/airflow:slim-latest | +| Default | 3.7 | apache/airflow:X.Y.Z | apache/airflow:slim-X.Y.Z | +| Latest | 3.7,3.8,3.9,3.10,3.11 | apache/airflow:latest-pythonN.M | apache/airflow:slim-latest-pythonN.M | +| Specific | 3.7,3.8,3.9,3.10,3.11 | apache/airflow:X.Y.Z-pythonN.M | apache/airflow:slim-X.Y.Z-pythonN.M | ++----------------+-----------------------+---------------------------------+--------------------------------------+ * The "latest" image is always the latest released stable version available. diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 0d836769f4d61..ae286f97e3382 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -6,14 +6,16 @@ ], "cross-providers-deps": [ "http" - ] + ], + "excluded-python-versions": [] }, "alibaba": { "deps": [ "apache-airflow>=2.4.0", "oss2>=2.14.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "amazon": { "deps": [ @@ -41,7 +43,8 @@ "mongo", "salesforce", "ssh" - ] + ], + "excluded-python-versions": [] }, "apache.beam": { "deps": [ @@ -50,14 +53,16 @@ ], "cross-providers-deps": [ "google" - ] + ], + "excluded-python-versions": [] }, "apache.cassandra": { "deps": [ "apache-airflow>=2.4.0", "cassandra-driver>=3.13.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "apache.drill": { "deps": [ @@ -67,7 +72,8 @@ ], "cross-providers-deps": [ "common.sql" - ] + ], + "excluded-python-versions": [] }, "apache.druid": { "deps": [ @@ -78,7 +84,8 @@ "cross-providers-deps": [ "apache.hive", "common.sql" - ] + ], + "excluded-python-versions": [] }, "apache.flink": { "deps": [ @@ -88,14 +95,16 @@ ], "cross-providers-deps": [ "cncf.kubernetes" - ] + ], + "excluded-python-versions": [] }, "apache.hdfs": { "deps": [ "apache-airflow>=2.4.0", "hdfs[avro,dataframe,kerberos]>=2.0.4" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "apache.hive": { "deps": [ @@ -115,6 +124,9 @@ "presto", "samba", "vertica" + ], + "excluded-python-versions": [ + "3.11" ] }, "apache.impala": { @@ -124,7 +136,8 @@ ], "cross-providers-deps": [ "common.sql" - ] + ], + "excluded-python-versions": [] }, "apache.kafka": { "deps": [ @@ -132,14 +145,16 @@ "asgiref", "confluent-kafka>=1.8.2" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "apache.kylin": { "deps": [ "apache-airflow>=2.4.0", "kylinpy>=2.6" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "apache.livy": { "deps": [ @@ -150,13 +165,15 @@ ], "cross-providers-deps": [ "http" - ] + ], + "excluded-python-versions": [] }, "apache.pig": { "deps": [ "apache-airflow>=2.4.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "apache.pinot": { "deps": [ @@ -166,41 +183,47 @@ ], "cross-providers-deps": [ "common.sql" - ] + ], + "excluded-python-versions": [] }, "apache.spark": { "deps": [ "apache-airflow>=2.4.0", "pyspark" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "apache.sqoop": { "deps": [ "apache-airflow>=2.4.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "arangodb": { "deps": [ "apache-airflow>=2.4.0", "python-arango>=7.3.2" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "asana": { "deps": [ "apache-airflow>=2.4.0", "asana>=0.10" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "atlassian.jira": { "deps": [ "apache-airflow>=2.4.0", "atlassian-python-api>=1.14.2" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "celery": { "deps": [ @@ -208,14 +231,16 @@ "celery>=5.2.3,<6", "flower>=1.0.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "cloudant": { "deps": [ "apache-airflow>=2.4.0", "cloudant>=2.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "cncf.kubernetes": { "deps": [ @@ -225,13 +250,15 @@ "kubernetes>=21.7.0,<24", "kubernetes_asyncio>=18.20.1,<25" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "common.sql": { "deps": [ "sqlparse>=0.4.2" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "databricks": { "deps": [ @@ -243,14 +270,16 @@ ], "cross-providers-deps": [ "common.sql" - ] + ], + "excluded-python-versions": [] }, "datadog": { "deps": [ "apache-airflow>=2.4.0", "datadog>=0.14.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "dbt.cloud": { "deps": [ @@ -261,7 +290,8 @@ ], "cross-providers-deps": [ "http" - ] + ], + "excluded-python-versions": [] }, "dingding": { "deps": [ @@ -270,7 +300,8 @@ ], "cross-providers-deps": [ "http" - ] + ], + "excluded-python-versions": [] }, "discord": { "deps": [ @@ -279,7 +310,8 @@ ], "cross-providers-deps": [ "http" - ] + ], + "excluded-python-versions": [] }, "docker": { "deps": [ @@ -287,7 +319,8 @@ "docker>=5.0.3", "python-dotenv>=0.21.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "elasticsearch": { "deps": [ @@ -299,7 +332,8 @@ ], "cross-providers-deps": [ "common.sql" - ] + ], + "excluded-python-versions": [] }, "exasol": { "deps": [ @@ -310,25 +344,29 @@ ], "cross-providers-deps": [ "common.sql" - ] + ], + "excluded-python-versions": [] }, "facebook": { "deps": [ "apache-airflow>=2.4.0", "facebook-business>=6.0.2" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "ftp": { "deps": [], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "github": { "deps": [ "PyGithub!=1.58", "apache-airflow>=2.4.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "google": { "deps": [ @@ -403,7 +441,8 @@ "sftp", "ssh", "trino" - ] + ], + "excluded-python-versions": [] }, "grpc": { "deps": [ @@ -412,7 +451,8 @@ "google-auth>=1.0.0, <3.0.0", "grpcio>=1.15.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "hashicorp": { "deps": [ @@ -421,7 +461,8 @@ ], "cross-providers-deps": [ "google" - ] + ], + "excluded-python-versions": [] }, "http": { "deps": [ @@ -430,11 +471,13 @@ "requests>=2.26.0", "requests_toolbelt" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "imap": { "deps": [], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "influxdb": { "deps": [ @@ -442,7 +485,8 @@ "influxdb-client>=1.19.0", "requests>=2.26.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "jdbc": { "deps": [ @@ -452,14 +496,16 @@ ], "cross-providers-deps": [ "common.sql" - ] + ], + "excluded-python-versions": [] }, "jenkins": { "deps": [ "apache-airflow>=2.4.0", "python-jenkins>=1.0.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "microsoft.azure": { "deps": [ @@ -486,7 +532,8 @@ "google", "oracle", "sftp" - ] + ], + "excluded-python-versions": [] }, "microsoft.mssql": { "deps": [ @@ -496,21 +543,24 @@ ], "cross-providers-deps": [ "common.sql" - ] + ], + "excluded-python-versions": [] }, "microsoft.psrp": { "deps": [ "apache-airflow>=2.4.0", "pypsrp>=0.8.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "microsoft.winrm": { "deps": [ "apache-airflow>=2.4.0", "pywinrm>=0.4" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "mongo": { "deps": [ @@ -518,7 +568,8 @@ "dnspython>=1.13.0", "pymongo>=3.6.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "mysql": { "deps": [ @@ -532,14 +583,16 @@ "presto", "trino", "vertica" - ] + ], + "excluded-python-versions": [] }, "neo4j": { "deps": [ "apache-airflow>=2.4.0", "neo4j>=4.2.1" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "odbc": { "deps": [ @@ -549,13 +602,15 @@ ], "cross-providers-deps": [ "common.sql" - ] + ], + "excluded-python-versions": [] }, "openfaas": { "deps": [ "apache-airflow>=2.4.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "openlineage": { "deps": [ @@ -565,14 +620,16 @@ "openlineage-integration-common>=0.22.0", "openlineage-python>=0.22.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "opsgenie": { "deps": [ "apache-airflow>=2.4.0", "opsgenie-sdk>=2.1.5" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "oracle": { "deps": [ @@ -582,14 +639,16 @@ ], "cross-providers-deps": [ "common.sql" - ] + ], + "excluded-python-versions": [] }, "pagerduty": { "deps": [ "apache-airflow>=2.4.0", "pdpyras>=4.1.2" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "papermill": { "deps": [ @@ -597,14 +656,16 @@ "papermill[all]>=1.2.1", "scrapbook[all]" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "plexus": { "deps": [ "apache-airflow>=2.4.0", "arrow>=0.16.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "postgres": { "deps": [ @@ -615,7 +676,8 @@ "cross-providers-deps": [ "amazon", "common.sql" - ] + ], + "excluded-python-versions": [] }, "presto": { "deps": [ @@ -627,7 +689,8 @@ "cross-providers-deps": [ "common.sql", "google" - ] + ], + "excluded-python-versions": [] }, "qubole": { "deps": [ @@ -637,14 +700,16 @@ ], "cross-providers-deps": [ "common.sql" - ] + ], + "excluded-python-versions": [] }, "redis": { "deps": [ "apache-airflow>=2.4.0", "redis>=3.2.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "salesforce": { "deps": [ @@ -652,28 +717,32 @@ "pandas>=0.17.1", "simple-salesforce>=1.0.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "samba": { "deps": [ "apache-airflow>=2.4.0", "smbprotocol>=1.5.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "segment": { "deps": [ "analytics-python>=1.2.9", "apache-airflow>=2.4.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "sendgrid": { "deps": [ "apache-airflow>=2.4.0", "sendgrid>=6.0.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "sftp": { "deps": [ @@ -682,14 +751,16 @@ ], "cross-providers-deps": [ "ssh" - ] + ], + "excluded-python-versions": [] }, "singularity": { "deps": [ "apache-airflow>=2.4.0", "spython>=0.0.56" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "slack": { "deps": [ @@ -699,13 +770,15 @@ ], "cross-providers-deps": [ "common.sql" - ] + ], + "excluded-python-versions": [] }, "smtp": { "deps": [ "apache-airflow>=2.4.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "snowflake": { "deps": [ @@ -717,7 +790,8 @@ "cross-providers-deps": [ "common.sql", "slack" - ] + ], + "excluded-python-versions": [] }, "sqlite": { "deps": [ @@ -725,7 +799,8 @@ ], "cross-providers-deps": [ "common.sql" - ] + ], + "excluded-python-versions": [] }, "ssh": { "deps": [ @@ -733,27 +808,31 @@ "paramiko>=2.6.0", "sshtunnel>=0.3.2" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "tableau": { "deps": [ "apache-airflow>=2.4.0", "tableauserverclient" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "tabular": { "deps": [ "apache-airflow>=2.4.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "telegram": { "deps": [ "apache-airflow>=2.4.0", "python-telegram-bot>=20.0.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "trino": { "deps": [ @@ -765,7 +844,8 @@ "cross-providers-deps": [ "common.sql", "google" - ] + ], + "excluded-python-versions": [] }, "vertica": { "deps": [ @@ -775,13 +855,15 @@ ], "cross-providers-deps": [ "common.sql" - ] + ], + "excluded-python-versions": [] }, "zendesk": { "deps": [ "apache-airflow>=2.4.0", "zenpy>=2.0.24" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] } } diff --git a/images/breeze/output_ci-image_build.svg b/images/breeze/output_ci-image_build.svg index 8319e04d12496..05742c982012d 100644 --- a/images/breeze/output_ci-image_build.svg +++ b/images/breeze/output_ci-image_build.svg @@ -280,7 +280,7 @@ ╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ │--python-pPython major/minor version used in Airflow image for images.│ -│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ +│(>3.7< | 3.8 | 3.9 | 3.10 | 3.11)                           │ │[default: 3.7]                                              │ │--upgrade-to-newer-dependencies-uWhen set, upgrade all PIP packages to latest.│ │--upgrade-on-failureWhen set, attempt to run upgrade to newer dependencies when regular build      │ @@ -300,7 +300,7 @@ │(INTEGER RANGE)                                                            │ │[default: 4; 1<=x<=8]                                                      │ │--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.7 3.8 3.9 3.10]                                                   │ +│[default: 3.7 3.8 3.9 3.10 3.11]                                              │ │--skip-cleanupSkip cleanup of temporary files created during parallel run.│ │--debug-resourcesWhether to show resource information while running in parallel.│ │--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ diff --git a/images/breeze/output_ci-image_pull.svg b/images/breeze/output_ci-image_pull.svg index d0e5c8d141651..3c54da94e3bb4 100644 --- a/images/breeze/output_ci-image_pull.svg +++ b/images/breeze/output_ci-image_pull.svg @@ -1,4 +1,4 @@ - + ╭─ Pull image flags ───────────────────────────────────────────────────────────────────────────────────────────────────╮ │--image-tag-tTag of the image which is used to pull the image.(TEXT)[default: latest]│ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ -│--verifyVerify image.│ -│--wait-for-imageWait until image is available.│ -│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful when you   │ -│build or pull image with --image-tag.                                                      │ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Parallel running ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ -│--parallelismMaximum number of processes to use while running the operation in parallel.│ -│(INTEGER RANGE)                                                            │ -│[default: 4; 1<=x<=8]                                                      │ -│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.7 3.8 3.9 3.10]                                                   │ -│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ -│--debug-resourcesWhether to show resource information while running in parallel.│ -│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10 | 3.11)                           │ +│[default: 3.7]                                              │ +│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ +│--verifyVerify image.│ +│--wait-for-imageWait until image is available.│ +│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful when you   │ +│build or pull image with --image-tag.                                                      │ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Parallel running ───────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ +│--parallelismMaximum number of processes to use while running the operation in parallel.│ +│(INTEGER RANGE)                                                            │ +│[default: 4; 1<=x<=8]                                                      │ +│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ +│[default: 3.7 3.8 3.9 3.10 3.11]                                              │ +│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ +│--debug-resourcesWhether to show resource information while running in parallel.│ +│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_ci-image_verify.svg b/images/breeze/output_ci-image_verify.svg index 6c18349eed7f2..d32439aa4db70 100644 --- a/images/breeze/output_ci-image_verify.svg +++ b/images/breeze/output_ci-image_verify.svg @@ -1,4 +1,4 @@ - +