Implement intelligent chunking for large repository wiki generation and add XML structure prompt

.github/workflows/docker-build-push.yml

@@ -11,13 +11,13 @@ on:
 env:
   REGISTRY: ghcr.io
   IMAGE_NAME: ${{ github.repository }}
-  
+
 concurrency:
   # This concurrency group ensures that only one job in the group runs at a time.
   # If a new job is triggered, the previous one will be canceled.
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
-  
+
 jobs:
   build-and-push:
     strategy:
@@ -39,7 +39,7 @@ jobs:
           platform=${{ matrix.platform }}
           echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
           echo "GHCR_IMAGE=ghcr.io/${GITHUB_REPOSITORY@L}" >> $GITHUB_ENV
-      
+
       - name: Checkout repository
         uses: actions/checkout@v4
 
@@ -75,21 +75,21 @@ jobs:
           outputs: type=image,name=${{ env.GHCR_IMAGE }},push-by-digest=true,name-canonical=true,push=${{ github.event_name != 'pull_request' }},oci-mediatypes=true
           cache-from: type=gha,scope=${{ github.repository }}-${{ github.ref_name }}-${{ matrix.platform }}
           cache-to: type=gha,mode=max,scope=${{ github.repository }}-${{ github.ref_name }}-${{ matrix.platform }}
-          
+
       - name: Export digest
         run: |
           mkdir -p /tmp/digests
           digest="${{ steps.build.outputs.digest }}"
           touch "/tmp/digests/${digest#sha256:}"
 
-      - name: Upload digest 
+      - name: Upload digest
         uses: actions/upload-artifact@v4
         with:
           name: digests-${{ env.PLATFORM_PAIR }}
           path: /tmp/digests/*
           if-no-files-found: error
           retention-days: 1
-  
+
   merge:
     name: merge Docker manifests
     runs-on: ubuntu-latest
@@ -105,7 +105,7 @@ jobs:
         id: prepare
         run: |
           echo "GHCR_IMAGE=ghcr.io/${GITHUB_REPOSITORY@L}" >> $GITHUB_ENV
-      
+
       - name: Download digests
         uses: actions/download-artifact@v4
         with:

.pre-commit-config.yaml

@@ -0,0 +1,71 @@
+# Pre-commit hooks for deepwiki-open (Next.js frontend + FastAPI backend)
+#
+# Setup:
+#   pipx install pre-commit        # or: pip install pre-commit
+#   pre-commit install             # install the git hook
+#   pre-commit run --all-files     # run once across the whole repo
+#
+# Update pinned versions later with:  pre-commit autoupdate
+
+default_language_version:
+  python: python3.12
+
+# Never touch generated / vendored / log / backup files.
+exclude: |
+  (?x)^(
+    .*\.bak|
+    api/logs/.*|
+    .*/__pycache__/.*|
+    public/.*\.svg|
+    yarn\.lock|
+    package-lock\.json|
+    uv\.lock|
+    api/poetry\.lock
+  )$
+
+repos:
+  # ---------------------------------------------------------------------------
+  # General hygiene
+  # ---------------------------------------------------------------------------
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: trailing-whitespace
+        args: [--markdown-linebreak-ext=md]
+      - id: end-of-file-fixer
+      - id: mixed-line-ending
+        args: [--fix=lf]
+      - id: check-merge-conflict
+      - id: check-case-conflict
+      - id: check-added-large-files
+        args: [--maxkb=1024]
+      - id: check-yaml
+      - id: check-toml
+      - id: check-json
+        # .vscode/*.json and tsconfig.json are JSONC (allow comments) -> skip
+        exclude: ^(\.vscode/|tsconfig\.json$)
+      - id: detect-private-key
+
+  # ---------------------------------------------------------------------------
+  # Secret scanning
+  # Blocks commits that contain tokens, keys, or credentials.
+  # ---------------------------------------------------------------------------
+  - repo: https://github.com/gitleaks/gitleaks
+    rev: v8.21.2
+    hooks:
+      - id: gitleaks
+
+  # ---------------------------------------------------------------------------
+  # Python — ruff replaces flake8 + isort + black (lint + format in one tool).
+  # Scoped to the backend and test dirs only.
+  # ---------------------------------------------------------------------------
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.8.6
+    hooks:
+      - id: ruff
+        name: ruff (lint + autofix)
+        args: [--fix]
+        files: ^(api|test)/.*\.py$  # disable tests folders for now
+      - id: ruff-format
+        name: ruff (format)
+        files: ^(api|test)/.*\.py$

Dockerfile

@@ -2,7 +2,6 @@
 
 # Build argument for custom certificates directory
 ARG CUSTOM_CERT_DIR="certs"
-
 FROM node:20-alpine3.22 AS node_base
 
 FROM node_base AS node_deps
@@ -35,6 +34,8 @@ RUN python -m pip install poetry==2.0.1 --no-cache-dir && \
 
 # Use Python 3.11 as final image
 FROM python:3.11-slim
+ARG AST_CHUNKING=false
+ENV DEEPWIKI_AST_CHUNKING=${AST_CHUNKING}
 
 # Set working directory
 WORKDIR /app
@@ -66,9 +67,12 @@ RUN if [ -n "${CUSTOM_CERT_DIR}" ]; then \
     fi
 
 ENV PATH="/opt/venv/bin:$PATH"
+ENV TIKTOKEN_CACHE_DIR=/opt/tiktoken_cache/
 
 # Copy Python dependencies
 COPY --from=py_deps /api/.venv /opt/venv
+RUN mkdir -p "$TIKTOKEN_CACHE_DIR" && \
+    python -c "import tiktoken; tiktoken.get_encoding('cl100k_base')"
 COPY api/ ./api/
 
 # Copy Node app

Dockerfile-litellm

@@ -0,0 +1,125 @@
+# syntax=docker/dockerfile:1-labs
+
+FROM node:20-alpine AS node_base
+
+FROM node_base AS node_deps
+WORKDIR /app
+COPY package.json package-lock.json ./
+RUN npm ci --legacy-peer-deps
+
+FROM node_base AS node_builder
+WORKDIR /app
+COPY --from=node_deps /app/node_modules ./node_modules
+COPY --exclude=./api . .
+RUN NODE_ENV=production npm run build
+
+FROM python:3.11-slim AS py_deps
+WORKDIR /api
+COPY api/pyproject.toml .
+COPY api/poetry.lock .
+RUN python -m pip install poetry==2.0.1 --no-cache-dir && \
+    poetry config virtualenvs.create true --local && \
+    poetry config virtualenvs.in-project true --local && \
+    poetry config virtualenvs.options.always-copy --local true && \
+    POETRY_MAX_WORKERS=10 poetry install --no-interaction --no-ansi --only main && \
+    poetry cache clear --all .
+
+FROM python:3.11-slim AS ollama_base
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    curl zstd && rm -rf /var/lib/apt/lists/*
+
+# Detect architecture and download appropriate Ollama version
+# ARG TARGETARCH can be set at build time with --build-arg TARGETARCH=arm64 or TARGETARCH=amd64
+ARG TARGETARCH
+RUN OLLAMA_ARCH="" && \
+    if [ "$TARGETARCH" = "arm64" ]; then \
+        echo "Building for ARM64 architecture." && \
+        OLLAMA_ARCH="arm64"; \
+    elif [ "$TARGETARCH" = "amd64" ]; then \
+        echo "Building for AMD64 architecture." && \
+        OLLAMA_ARCH="amd64"; \
+    else \
+        echo "Error: Unsupported architecture '$TARGETARCH'. Supported architectures are 'arm64' and 'amd64'." >&2 && \
+        exit 1; \
+    fi && \
+    (set -o pipefail; \
+     curl -fL "https://ollama.com/download/ollama-linux-${OLLAMA_ARCH}.tar.zst" \
+      | zstd -d | tar -x -C /usr)
+
+RUN ollama serve > /dev/null 2>&1 & \
+    sleep 20 && \
+    ollama pull nomic-embed-text && \
+    ollama pull qwen3:1.7b
+
+# Use Python 3.11 as final image
+FROM python:3.11-slim
+
+# Set working directory
+WORKDIR /app
+
+# Install Node.js and npm
+RUN apt-get update && apt-get install -y \
+    curl \
+    gnupg \
+    git \
+    ca-certificates \
+    && mkdir -p /etc/apt/keyrings \
+    && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
+    && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list \
+    && apt-get update \
+    && apt-get install -y nodejs \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+ENV PATH="/opt/venv/bin:$PATH"
+ENV TIKTOKEN_CACHE_DIR=/opt/tiktoken_cache/
+
+# Copy Python dependencies
+COPY --from=py_deps /api/.venv /opt/venv
+RUN mkdir -p "$TIKTOKEN_CACHE_DIR" && \
+    python -c "import tiktoken; tiktoken.get_encoding('cl100k_base')"
+COPY api/ ./api/
+
+# Copy Node app
+COPY --from=node_builder /app/public ./public
+COPY --from=node_builder /app/.next/standalone ./
+COPY --from=node_builder /app/.next/static ./.next/static
+COPY --from=ollama_base /usr/bin/ollama /usr/local/bin/
+COPY --from=ollama_base /root/.ollama /root/.ollama
+
+# Expose the port the app runs on
+EXPOSE ${PORT:-8001} 3000
+
+# Create a script to run both backend and frontend
+RUN echo '#!/bin/bash\n\
+# Start ollama serve in background\n\
+ollama serve > /dev/null 2>&1 &\n\
+\n\
+# Load environment variables from .env file if it exists\n\
+if [ -f .env ]; then\n\
+  export $(grep -v "^#" .env | xargs -r)\n\
+fi\n\
+\n\
+# Check for required environment variables\n\
+if [ -z "$OPENAI_API_KEY" ] || [ -z "$GOOGLE_API_KEY" ]; then\n\
+  echo "Warning: OPENAI_API_KEY and/or GOOGLE_API_KEY environment variables are not set."\n\
+  echo "These are required for DeepWiki to function properly."\n\
+  echo "You can provide them via a mounted .env file or as environment variables when running the container."\n\
+fi\n\
+\n\
+# Start the API server in the background with the configured port\n\
+python -m api.main --port ${PORT:-8001} &\n\
+PORT=3000 HOSTNAME=0.0.0.0 node server.js &\n\
+wait -n\n\
+exit $?' > /app/start.sh && chmod +x /app/start.sh
+
+# Set environment variables
+ENV PORT=8001
+ENV NODE_ENV=production
+ENV SERVER_BASE_URL=http://localhost:${PORT:-8001}
+
+# Create empty .env file (will be overridden if one exists at runtime)
+RUN touch .env
+
+# Command to run the application
+CMD ["/app/start.sh"]

Dockerfile-ollama-local

@@ -25,8 +25,9 @@ RUN python -m pip install poetry==2.0.1 --no-cache-dir && \
     poetry cache clear --all .
 
 FROM python:3.11-slim AS ollama_base
-RUN apt-get update && apt-get install -y \
-    curl
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    curl zstd && rm -rf /var/lib/apt/lists/*
+
 # Detect architecture and download appropriate Ollama version
 # ARG TARGETARCH can be set at build time with --build-arg TARGETARCH=arm64 or TARGETARCH=amd64
 ARG TARGETARCH=arm64
@@ -41,9 +42,9 @@ RUN OLLAMA_ARCH="" && \
         echo "Error: Unsupported architecture '$TARGETARCH'. Supported architectures are 'arm64' and 'amd64'." >&2 && \
         exit 1; \
     fi && \
-    curl -L "https://ollama.com/download/ollama-linux-${OLLAMA_ARCH}.tgz" -o ollama.tgz && \
-    tar -C /usr -xzf ollama.tgz && \
-    rm ollama.tgz
+    (set -o pipefail; \
+     curl -fL "https://ollama.com/download/ollama-linux-${OLLAMA_ARCH}.tar.zst" \
+      | zstd -d | tar -x -C /usr)
 
 RUN ollama serve > /dev/null 2>&1 & \
     sleep 20 && \
@@ -70,9 +71,12 @@ RUN apt-get update && apt-get install -y \
     && rm -rf /var/lib/apt/lists/*
 
 ENV PATH="/opt/venv/bin:$PATH"
+ENV TIKTOKEN_CACHE_DIR=/opt/tiktoken_cache/
 
 # Copy Python dependencies
 COPY --from=py_deps /api/.venv /opt/venv
+RUN mkdir -p "$TIKTOKEN_CACHE_DIR" && \
+    python -c "import tiktoken; tiktoken.get_encoding('cl100k_base')"
 COPY api/ ./api/
 
 # Copy Node app

Ollama-instruction.md

@@ -80,15 +80,21 @@ npm run dev
 
 ## Alternative using Dockerfile
 
-1. Build the docker image `docker build -f Dockerfile-ollama-local -t deepwiki:ollama-local .`
-2. Run the container:
+1. Configure the Local Embedder for Ollama:
+```
+cp api/config/embedder.ollama.json.bak api/config/embedder.json
+# overwrite api/config/embedder.json? (y/n [n]) y
+```
+
+2. Build the docker image `docker build -f Dockerfile-ollama-local -t deepwiki:ollama-local .`
+3. Run the container:
    ```bash
    # For regular use
    docker run -p 3000:3000 -p 8001:8001 --name deepwiki \
      -v ~/.adalflow:/root/.adalflow \
      -e OLLAMA_HOST=your_ollama_host \
      deepwiki:ollama-local
-   
+
    # For local repository analysis
    docker run -p 3000:3000 -p 8001:8001 --name deepwiki \
      -v ~/.adalflow:/root/.adalflow \
@@ -97,9 +103,9 @@ npm run dev
      deepwiki:ollama-local
    ```
 
-3. When using local repositories in the interface: use `/app/local-repos/repo-name` as the local repository path.
+4. When using local repositories in the interface: use `/app/local-repos/repo-name` as the local repository path.
 
-4. Open http://localhost:3000 in your browser
+5. Open http://localhost:3000 in your browser
 
 Note: For Apple Silicon Macs, the Dockerfile automatically uses ARM64 binaries for better performance.
 

README.es.md

@@ -8,6 +8,7 @@
 2. Generará documentación completa
 3. Creará diagramas visuales para explicar cómo funciona todo
 4. Organizará todo en una wiki fácil de navegar
+5. Generará un mapa de código (codemap) para recorridos guiados centrados en el código
 
 [!["Buy Me A Coffee"](https://www.buymeacoffee.com/assets/img/custom_images/orange_img.png)](https://buymeacoffee.com/sheing)
 
@@ -289,6 +290,9 @@ Para usar Investigación Profunda, simplemente activa el interruptor "Investigac
 ![Función de Investigación Profunda](screenshots/DeepResearch.png)
 *Investigación Profunda realiza investigaciones de múltiples turnos para temas complejos*
 
+![Codemap](screenshots/Codemap.png)
+*Codemap*
+
 ### Video de Demostración
 
 [![Video de Demostración de DeepWiki](https://img.youtube.com/vi/zGANs8US8B4/0.jpg)](https://youtu.be/zGANs8US8B4)
@@ -331,4 +335,3 @@ Este proyecto está licenciado bajo la Licencia MIT - consulta el archivo [LICEN
 ## ⭐ Historial de Estrellas
 
 [![Gráfico de Historial de Estrellas](https://api.star-history.com/svg?repos=AsyncFuncAI/deepwiki-open&type=Date)](https://star-history.com/#AsyncFuncAI/deepwiki-open&Date)
-