Merge branch 'main' into jhcipar/AE-1371/worker-concurrency-new-jobs

Author: deanq

.github/workflows/CI-e2e.yml

@@ -1,93 +1,38 @@
-# Performs a full test of the package within production environment.
-
-name: CI | End-to-End Runpod Python Tests
-
+name: CI-e2e
 on:
   push:
-    branches:
-      - main
-
+    branches: [main]
   pull_request:
-    branches:
-      - main
-
+    branches: [main]
   workflow_dispatch:
 
 jobs:
-  e2e-build:
-    name: Build and push mock-worker Docker image
+  e2e:
     if: github.repository == 'runpod/runpod-python'
     runs-on: ubuntu-latest
-    outputs:
-      docker_tag: ${{ steps.output_docker_tag.outputs.docker_tag }}
-
+    timeout-minutes: 15
     steps:
-      - name: Checkout Repo
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 2
-
-      - name: Clone and patch mock-worker
-        run: |
-          git clone https://github.com/runpod-workers/mock-worker
-          GIT_SHA=${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
-          echo "git+https://github.com/runpod/runpod-python.git@$GIT_SHA" > mock-worker/builder/requirements.txt
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+      - uses: actions/checkout@v4
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+      - uses: astral-sh/setup-uv@v6
 
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
+      - uses: actions/setup-python@v5
         with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
+          python-version: "3.12"
 
-      - name: Define Docker Tag
-        id: docker_tag
+      - name: Install dependencies
         run: |
-          DOCKER_TAG=${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
-          echo "DOCKER_TAG=$(echo $DOCKER_TAG | cut -c 1-7)" >> $GITHUB_ENV
-
-      - name: Set Docker Tag as Output
-        id: output_docker_tag
-        run: echo "docker_tag=${{ env.DOCKER_TAG }}" >> $GITHUB_OUTPUT
-
-      - name: Build and push Docker image
-        uses: docker/build-push-action@v6
-        with:
-          context: ./mock-worker
-          file: ./mock-worker/Dockerfile
-          push: true
-          tags: ${{ vars.DOCKERHUB_REPO }}/${{ vars.DOCKERHUB_IMG }}:${{ env.DOCKER_TAG }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-
-  test:
-    name: Run End-to-End Tests
-    runs-on: ubuntu-latest
-    needs: [e2e-build]
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Run Tests
-        id: run-tests
-        uses: runpod/runpod-test-runner@v2.1.0
-        with:
-          image-tag: ${{ vars.DOCKERHUB_REPO }}/${{ vars.DOCKERHUB_IMG }}:${{ needs.e2e-build.outputs.docker_tag }}
-          runpod-api-key: ${{ secrets.RUNPOD_API_KEY }}
-          request-timeout: 1200
-
-      - name: Verify Tests
-        env:
-          TOTAL_TESTS: ${{ steps.run-tests.outputs.total-tests }}
-          SUCCESSFUL_TESTS: ${{ steps.run-tests.outputs.succeeded }}
+          uv venv
+          source .venv/bin/activate
+          uv pip install -e ".[test]" --quiet || uv pip install -e .
+          uv pip install runpod-flash pytest pytest-asyncio pytest-timeout pytest-rerunfailures httpx
+          uv pip install -e . --reinstall --no-deps
+          python -c "import runpod; print(f'runpod: {runpod.__version__} from {runpod.__file__}')"
+
+      - name: Run e2e tests
         run: |
-          echo "Total tests: $TOTAL_TESTS"
-          echo "Successful tests: $SUCCESSFUL_TESTS"
-          if [ "$TOTAL_TESTS" != "$SUCCESSFUL_TESTS" ]; then
-              exit 1
-          fi
+          source .venv/bin/activate
+          pytest tests/e2e/ -v -p no:xdist --timeout=600 --reruns 1 --reruns-delay 5 --log-cli-level=INFO -o "addopts="
+        env:
+          RUNPOD_API_KEY: ${{ secrets.RUNPOD_API_KEY }}
+          RUNPOD_SDK_GIT_REF: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}

CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## [1.8.2](https://github.com/runpod/runpod-python/compare/v1.8.1...v1.8.2) (2026-03-24)
+
+
+### Bug Fixes
+
+* **config:** handle corrupted config.toml in credential functions ([#481](https://github.com/runpod/runpod-python/issues/481)) ([9894894](https://github.com/runpod/runpod-python/commit/9894894ee2022e7db0777c9dd24c23208e52f90c))
+* use flashBootType instead of appending -fb ([#484](https://github.com/runpod/runpod-python/issues/484)) ([7938936](https://github.com/runpod/runpod-python/commit/7938936158c351d6e00caebbf4242e085f7565ae))
+
 ## [1.8.1](https://github.com/runpod/runpod-python/compare/v1.8.0...v1.8.1) (2025-11-19)
 
 

pytest.ini

@@ -1,5 +1,9 @@
 [pytest]
 addopts = --durations=10 --cov-config=.coveragerc --timeout=120 --timeout_method=thread --cov=runpod --cov-report=xml --cov-report=term-missing --cov-fail-under=90 -W error -p no:cacheprovider -p no:unraisableexception
 python_files = tests.py test_*.py *_test.py
-norecursedirs = venv *.egg-info .git build
+norecursedirs = venv *.egg-info .git build tests/e2e
 asyncio_mode = auto
+markers =
+    qb: Queue-based tests (local execution, fast)
+    lb: Load-balanced tests (remote provisioning, slow)
+    cold_start: Cold start benchmark (starts own server)

runpod/api/mutations/endpoints.py

@@ -1,4 +1,4 @@
-""" Runpod | API Wrapper | Mutations | Endpoints """
+f"""Runpod | API Wrapper | Mutations | Endpoints"""
 
 # pylint: disable=too-many-arguments
 
@@ -23,7 +23,7 @@ def generate_endpoint_mutation(
 
     # ------------------------------ Required Fields ----------------------------- #
     if flashboot:
-        name = name + "-fb"
+        input_fields.append('flashBootType: "FLASHBOOT"')
 
     input_fields.append(f'name: "{name}"')
     input_fields.append(f'templateId: "{template_id}"')
@@ -75,12 +75,12 @@ def generate_endpoint_mutation(
             workersMax
             allowedCudaVersions
             gpuCount
+            flashBootType
         }}
     }}
     """
 
 
-
 def update_endpoint_template_mutation(endpoint_id: str, template_id: str):
     """Generate a string for a GraphQL mutation to update an existing endpoint's template."""
     input_fields = []

runpod/cli/groups/config/functions.py

@@ -31,11 +31,15 @@ def set_credentials(api_key: str, profile: str = "default", overwrite=False) ->
     Path(CREDENTIAL_FILE).touch(exist_ok=True)
 
     if not overwrite:
-        with open(CREDENTIAL_FILE, "rb") as cred_file:
-            if profile in toml.load(cred_file):
-                raise ValueError(
-                    "Profile already exists. Use `update_credentials` instead."
-                )
+        try:
+            with open(CREDENTIAL_FILE, "rb") as cred_file:
+                existing = toml.load(cred_file)
+        except (TypeError, ValueError):
+            existing = {}
+        if profile in existing:
+            raise ValueError(
+                "Profile already exists. Use `update_credentials` instead."
+            )
 
     with open(CREDENTIAL_FILE, "w", encoding="UTF-8") as cred_file:
         cred_file.write("[" + profile + "]\n")
@@ -72,12 +76,18 @@ def check_credentials(profile: str = "default"):
 def get_credentials(profile="default"):
     """
     Returns the credentials for the specified profile from ~/.runpod/config.toml
+
+    Returns None if the file does not exist, is not valid TOML, or does not
+    contain the requested profile.
     """
     if not os.path.exists(CREDENTIAL_FILE):
         return None
 
-    with open(CREDENTIAL_FILE, "rb") as cred_file:
-        credentials = toml.load(cred_file)
+    try:
+        with open(CREDENTIAL_FILE, "rb") as cred_file:
+            credentials = toml.load(cred_file)
+    except (TypeError, ValueError):
+        return None
 
     if profile not in credentials:
         return None

tests/e2e/__init__.py

@@ -0,0 +1,73 @@
+"""E2E test fixtures: provision real endpoints, configure SDK, clean up."""
+
+import logging
+import os
+import subprocess
+from pathlib import Path
+
+import pytest
+import runpod
+
+from tests.e2e.e2e_provisioner import load_test_cases, provision_endpoints
+
+log = logging.getLogger(__name__)
+REQUEST_TIMEOUT = 300  # seconds per job request
+
+# Repo root: tests/e2e/conftest.py -> ../../
+_REPO_ROOT = Path(__file__).resolve().parents[2]
+
+
+@pytest.fixture(scope="session", autouse=True)
+def verify_local_runpod():
+    """Fail fast if the local runpod-python is not installed."""
+    log.info("runpod version=%s path=%s", runpod.__version__, runpod.__file__)
+    runpod_path = Path(runpod.__file__).resolve()
+    if not runpod_path.is_relative_to(_REPO_ROOT):
+        pytest.fail(
+            f"Expected runpod installed from {_REPO_ROOT} but got {runpod_path}. "
+            "Run: pip install -e . --force-reinstall --no-deps"
+        )
+
+
+@pytest.fixture(scope="session")
+def require_api_key():
+    """Skip entire session if RUNPOD_API_KEY is not set."""
+    key = os.environ.get("RUNPOD_API_KEY")
+    if not key:
+        pytest.skip("RUNPOD_API_KEY not set")
+    log.info("RUNPOD_API_KEY is set (length=%d)", len(key))
+
+
+@pytest.fixture(scope="session")
+def test_cases():
+    """Load test cases from tests.json."""
+    cases = load_test_cases()
+    log.info("Loaded %d test cases: %s", len(cases), [c.get("id") for c in cases])
+    return cases
+
+
+@pytest.fixture(scope="session")
+def endpoints(require_api_key, test_cases):
+    """Provision one endpoint per unique hardwareConfig.
+
+    Endpoints deploy lazily on first .run()/.runsync() call.
+    """
+    eps = provision_endpoints(test_cases)
+    for key, ep in eps.items():
+        log.info("Endpoint ready: name=%s image=%s template.dockerArgs=%s", ep.name, ep.image, ep.template.dockerArgs if ep.template else "N/A")
+    yield eps
+
+    log.info("Cleaning up all provisioned endpoints")
+    try:
+        result = subprocess.run(
+            ["flash", "undeploy", "--all", "--force"],
+            capture_output=True,
+            text=True,
+            timeout=120,
+        )
+        if result.returncode == 0:
+            log.info("Undeployed all endpoints")
+        else:
+            log.warning("flash undeploy --all --force failed (rc=%d): %s", result.returncode, result.stderr)
+    except Exception:
+        log.exception("Failed to undeploy endpoints")