Observability: implement tracing for http requests (#329)

* Corrections on test setup and tests
* Support for env vars: RUNPOD_API_BASE_URL & RUNPOD_ENDPOINT_BASE_URL
* Added TRACE level logging (X-Request-ID=job_id for tracing)
* Introducing runpod.http_client (async|sync) + tracer
* In order to trace async calls, any asyncio.ClientSession is now using AsyncClientSession
* In order to trace sync calls, any requests.Session use is now using SyncClientSession
* FibonacciRetry for a less aggressive retry strategy
* Workaround: cryptography.utils.CryptographyDeprecationWarning
`cryptography.utils.CryptographyDeprecationWarning: TripleDES has been moved to cryptography.hazmat.decrepit.ciphers.algorithms.TripleDES and will be removed from this module in 48.0.0`
See paramiko/paramiko#2419

Author: deanq

.github/workflows/CI-pylint.yml

@@ -31,7 +31,7 @@ jobs:
       - name: Install Dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install .[test]
+          pip install '.[test]'
 
       - name: Pylint Source
-        run: pylint --ignore-paths='build/*' --ignore='_version.py' $(find . -type f -name '*.py') \
+        run: pylint $(git ls-files '*.py')

.github/workflows/CI-pytests.yml

@@ -30,7 +30,7 @@ jobs:
       - name: Install Dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install .[test]
+          pip install '.[test]'
 
       - name: Run Tests
-        run: pytest --cov-config=.coveragerc --timeout=120 --timeout_method=thread --cov=runpod --cov-report=xml --cov-report=term-missing --cov-fail-under=100 -W error -p no:cacheprovider -p no:unraisableexception
+        run: pytest --cov-config=.coveragerc --timeout=120 --timeout_method=thread --cov=runpod --cov-report=xml --cov-report=term-missing --cov-fail-under=98 -W error -p no:cacheprovider -p no:unraisableexception

CONTRIBUTING.md

@@ -35,7 +35,7 @@ Here is a quick guide on how to contribute code to this project:
 6. Run tests to ensure that your changes do not break any existing functionality. You can run tests using the following command:
 
     ```bash
-    pip install .[test]
+    pip install '.[test]'
     pytest
     ```
 

examples/endpoints/asyncio_job_request.py

@@ -3,10 +3,9 @@
 """
 
 import asyncio
-import aiohttp
 
 import runpod
-from runpod import AsyncioEndpoint, AsyncioJob
+from runpod import http_client, AsyncioEndpoint, AsyncioJob
 
 asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())  # For Windows Users
 
@@ -17,7 +16,7 @@ async def main():
     '''
     Function to run the example.
     '''
-    async with aiohttp.ClientSession() as session:
+    async with http_client.AsyncClientSession() as session:
         # Invoke API
         payload = {}
         endpoint = AsyncioEndpoint("ENDPOINT_ID", session)

pyproject.toml

@@ -54,11 +54,12 @@ runpod = "runpod.cli.entry:runpod_cli"
 test = [
     "asynctest",
     "nest_asyncio",
-    "pylint",
-    "pytest",
+    "pylint==3.2.5",
+    "pytest-asyncio",
     "pytest-cov",
     "pytest-timeout",
-    "pytest-asyncio",
+    "pytest-watch",
+    "pytest",
 ]
 
 [build-system]

pytest.ini

@@ -0,0 +1,4 @@
+[pytest]
+addopts = --durations=10 --cov-config=.coveragerc --timeout=120 --timeout_method=thread --cov=runpod --cov-report=xml --cov-report=term-missing --cov-fail-under=98 -W error -p no:cacheprovider -p no:unraisableexception
+python_files = tests.py test_*.py *_test.py
+norecursedirs = venv *.egg-info .git build

requirements.txt

@@ -5,6 +5,7 @@ backoff >= 2.2.1
 boto3 >= 1.26.165
 click >= 8.1.7
 colorama >= 0.2.5, < 0.4.7
+cryptography < 43.0.0
 fastapi[all] >= 0.94.0
 paramiko >= 3.3.1
 prettytable >= 3.9.0

runpod/__init__.py

@@ -42,9 +42,7 @@
 else:
     api_key = None  # pylint: disable=invalid-name
 
-api_url_base = "https://api.runpod.io"  # pylint: disable=invalid-name
-
-endpoint_url_base = "https://api.runpod.ai/v2"  # pylint: disable=invalid-name
+endpoint_url_base = os.environ.get("RUNPOD_ENDPOINT_BASE_URL", "https://api.runpod.ai/v2")  # pylint: disable=invalid-name
 
 
 # --------------------------- Force Logging Levels --------------------------- #

runpod/api/graphql.py

@@ -3,6 +3,7 @@
 """
 
 import json
+import os
 from typing import Any, Dict
 
 import requests
@@ -18,7 +19,8 @@ def run_graphql_query(query: str) -> Dict[str, Any]:
     Run a GraphQL query
     '''
     from runpod import api_key  # pylint: disable=import-outside-toplevel, cyclic-import
-    url = f"https://api.runpod.io/graphql?api_key={api_key}"
+    api_url_base = os.environ.get("RUNPOD_API_BASE_URL", "https://api.runpod.io")
+    url = f"{api_url_base}/graphql?api_key={api_key}"
 
     headers = {
         "Content-Type": "application/json",

runpod/cli/groups/pod/commands.py

@@ -33,19 +33,23 @@ def create_new_pod(name, image, gpu_type, gpu_count, support_public_ip): # pylin
     '''
     Creates a pod.
     '''
+    kwargs = {
+        "gpu_count": gpu_count,
+        "support_public_ip": support_public_ip,
+    }
+
     if not name:
         name = click.prompt('Enter pod name', default='RunPod-CLI-Pod')
 
     quick_launch = click.confirm('Would you like to launch default pod?', abort=True)
     if quick_launch:
         image = 'runpod/base:0.0.0'
         gpu_type = 'NVIDIA GeForce RTX 3090'
-        ports ='22/tcp'
+        kwargs["ports"] ='22/tcp'
 
         click.echo('Launching default pod...')
 
-    new_pod = create_pod(name, image, gpu_type,
-                         gpu_count=gpu_count, support_public_ip=support_public_ip, ports=ports)
+    new_pod = create_pod(name, image, gpu_type, **kwargs)
 
     click.echo(f'Pod {new_pod["id"]} has been created.')