docs: add documentation versioning support (#1815)

## Summary

- Add Docusaurus versioning support with version dropdown in the navbar.
- Add manual doc snapshots for versions 1.6 and 0.6 (with API
reference).
- Automate doc version creation on stable releases in the release
workflow (minor/major create new versions, patches override the latest).

## Issue

- Closes: #1302

---

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: vdusek

.github/workflows/manual_release_stable.yaml

@@ -104,15 +104,83 @@ jobs:
       # TODO: add job for publish package to Conda
       # https://github.com/apify/crawlee-python/issues/104
 
+  version_docs:
+    name: Version docs
+    needs: [release_prepare, changelog_update, pypi_publish]
+    runs-on: ubuntu-latest
+    outputs:
+      version_docs_commitish: ${{ steps.commit_versioned_docs.outputs.commit_long_sha }}
+    permissions:
+      contents: write
+    env:
+      NODE_VERSION: 22
+      PYTHON_VERSION: 3.14
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+        with:
+          token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
+          ref: ${{ needs.changelog_update.outputs.changelog_commitish }}
+
+      - name: Set up Node
+        uses: actions/setup-node@v6
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Set up uv package manager
+        uses: astral-sh/setup-uv@v7
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Install Python dependencies
+        run: uv run poe install-dev
+
+      - name: Install website dependencies
+        run: |
+          cd website
+          corepack enable
+          yarn install
+
+      - name: Snapshot the current version
+        run: |
+          cd website
+          VERSION="$(python -c "import tomllib, pathlib; print(tomllib.loads(pathlib.Path('../pyproject.toml').read_text())['project']['version'])")"
+          MAJOR_MINOR="$(echo "$VERSION" | cut -d. -f1-2)"
+          export MAJOR_MINOR
+          # Remove existing version if present (patch releases override)
+          rm -rf "versioned_docs/version-${MAJOR_MINOR}"
+          rm -rf "versioned_sidebars/version-${MAJOR_MINOR}-sidebars.json"
+          jq 'map(select(. != env.MAJOR_MINOR))' versions.json > tmp.json && mv tmp.json versions.json
+          # Copy changelog
+          cp ../CHANGELOG.md ../docs/changelog.md
+          # Build API reference and create version snapshots
+          bash build_api_reference.sh
+          npx docusaurus docs:version "$MAJOR_MINOR"
+          npx docusaurus api:version "$MAJOR_MINOR"
+
+      - name: Commit and push versioned docs
+        id: commit_versioned_docs
+        uses: EndBug/add-and-commit@v10
+        with:
+          add: "website/versioned_docs website/versioned_sidebars website/versions.json"
+          message: "docs: version ${{ needs.release_prepare.outputs.version_number }} docs [skip ci]"
+          default_author: github_actions
+
   doc_release:
     name: Doc release
-    needs: [changelog_update, pypi_publish]
+    needs: [changelog_update, pypi_publish, version_docs]
     permissions:
       contents: write
       pages: write
       id-token: write
     uses: ./.github/workflows/_release_docs.yaml
     with:
-      # Use the ref from the changelog update to include the updated changelog.
-      ref: ${{ needs.changelog_update.outputs.changelog_commitish }}
+      # Use the version_docs commit to include both changelog and versioned docs.
+      ref: ${{ needs.version_docs.outputs.version_docs_commitish }}
     secrets: inherit

docs/guides/code_examples/request_loaders/rl_tandem_example_explicit.py

@@ -12,7 +12,7 @@ async def main() -> None:
     # Open the default request queue.
     request_queue = await RequestQueue.open()
 
-    # And combine them together to a sinhle request manager.
+    # And combine them together to a single request manager.
     request_manager = RequestManagerTandem(request_list, request_queue)
 
     # Create a crawler and pass the request manager to it.

pyproject.toml

@@ -140,6 +140,9 @@ packages = ["src/crawlee"]
 [tool.ruff]
 line-length = 120
 include = ["src/**/*.py", "tests/**/*.py", "docs/**/*.py", "website/**/*.py"]
+exclude = [
+    "website/versioned_docs/**",
+]
 extend-exclude = ["src/crawlee/project_template"]
 
 [tool.ruff.lint]
@@ -251,6 +254,7 @@ include = ["src", "tests", "scripts", "docs", "website"]
 exclude = [
     "src/crawlee/project_template",
     "docs/guides/code_examples/storage_clients/custom_storage_client_example.py",
+    "website/versioned_docs",
 ]
 
 [[tool.ty.overrides]]

typos.toml

@@ -13,7 +13,8 @@ extend-exclude = [
     "*.lock",
     "*.min.js",
     "*.min.css",
-    "CHANGELOG.md",
+    "**/CHANGELOG.md",
+    "**/changelog.md",
 ]
 
 [default.extend-identifiers]
@@ -25,3 +26,4 @@ asend = "asend" # Python async generator method
 # Add project-specific words that should not be treated as typos
 mke = "mke" # Sennheiser MKE product name
 consts = "consts"  # Common abbreviation for "constants"
+certifi = "certifi" # Python package name

website/docusaurus.config.js

@@ -94,7 +94,6 @@ module.exports = {
                     path: '../docs',
                     sidebarPath: './sidebars.js',
                     rehypePlugins: [externalLinkProcessor],
-                    // disableVersioning: true,
                     editUrl: (doc) => {
                         return `https://github.com/apify/crawlee-python/edit/master/website/${doc.versionDocsDirPath}/${doc.docPath}`;
                     },
@@ -118,6 +117,7 @@ module.exports = {
                 },
                 sortSidebar: groupSort,
                 routeBasePath: 'api',
+                python: true,
                 pythonOptions: {
                     pythonModulePath: path.join(__dirname, '../src/crawlee'),
                     moduleShortcutsPath: path.join(__dirname, 'module_shortcuts.json'),
@@ -172,6 +172,12 @@ module.exports = {
                     includeVersionedDocs: false,
                     enableLlmsFullTxt: true,
                     relativePaths: false,
+                    excludeRoutes: [
+                        '/python/api/[0-9]*/**',
+                        '/python/api/[0-9]*',
+                        '/python/api/next/**',
+                        '/python/api/next',
+                    ],
                 },
             },
         ],
@@ -281,6 +287,12 @@ module.exports = {
                     label: 'Blog',
                     position: 'left',
                 },
+                {
+                    type: 'docsVersionDropdown',
+                    position: 'right',
+                    dropdownItemsBefore: [],
+                    dropdownItemsAfter: [],
+                },
             ],
         },
         colorMode: {

website/src/theme/Navbar/Content/index.js

@@ -1,5 +1,4 @@
 import Link from '@docusaurus/Link';
-import { useLocation } from '@docusaurus/router';
 import { useThemeConfig } from '@docusaurus/theme-common';
 import {
     splitNavbarItems,
@@ -38,32 +37,10 @@ function NavbarContentLayout({ left, right }) {
     );
 }
 
-const VERSIONS_ITEM = {
-    type: 'docsVersionDropdown',
-    position: 'left',
-    label: 'Versions',
-    dropdownItemsAfter: [
-        {
-            href: 'https://sdk.apify.com/docs/guides/getting-started',
-            label: '2.2',
-        },
-        {
-            href: 'https://sdk.apify.com/docs/1.3.1/guides/getting-started',
-            label: '1.3',
-        },
-    ],
-    dropdownItemsBefore: [],
-};
-
 export default function NavbarContent() {
-    const location = useLocation();
     const mobileSidebar = useNavbarMobileSidebar();
     const items = useNavbarItems();
-    const effectiveItems = location.pathname?.endsWith('/python/')
-        || location.pathname?.endsWith('/python')
-        ? items
-        : [...items, VERSIONS_ITEM];
-    const [leftItems, rightItems] = splitNavbarItems(effectiveItems);
+    const [leftItems, rightItems] = splitNavbarItems(items);
     const searchBarItem = items.find((item) => item.type === 'search');
     return (
         <NavbarContentLayout

website/versioned_docs/version-0.6/api-packages.json

@@ -0,0 +1 @@
+[{"entryPoints":{"index":{"label":"Index","path":"src/index.ts"}},"packageRoot":".","packagePath":".","packageSlug":".","packageName":"crawlee"}]