Add devcontainers supporting all the brokers (airtai#1839)

* WIP: Add devcontainer with multiple brokers * Update devcontainer * Use python 3.10 for all pre-commit hooks * Fix failing kafka container
IvanKirpichnikov · Oct 8, 2024 · a817094 · a817094
1 parent d60f3bc
commit a817094
Show file tree

Hide file tree

Showing 5 changed files with 121 additions and 39 deletions.
diff --git a/.devcontainer/devcontainer.env b/.devcontainer/devcontainer.env
@@ -0,0 +1 @@
+CONTAINER_PREFIX=${USER}
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -1,26 +1,51 @@
-// For format details, see https://aka.ms/devcontainer.json. For config options, see the
-// README at: https://github.com/devcontainers/templates/tree/main/src/python
 {
-	"name": "Python 3",
-	"service": "python-3.12-faststream-studio",
-	"dockerComposeFile": [
+    "name": "python-3.12",
+    "dockerComposeFile": [
         "./docker-compose.yaml"
     ],
-	"shutdownAction": "stopCompose",
-	"forwardPorts": [
-        "kafka-faststream:9092"
-    ],
-	"updateContentCommand": "bash .devcontainer/setup.sh",
-	"workspaceFolder": "/workspaces/faststream"
-	// Features to add to the dev container. More info: https://containers.dev/features.
-	// "features": {},
-
-	// Use 'postCreateCommand' to run commands after the container is created.
-	// "postCreateCommand": "pip3 install --user -r requirements.txt",
-
-	// Configure tool-specific properties.
-	// "customizations": {},
-
-	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
-	// "remoteUser": "root"
+    "service": "python-3.12-faststream",
+    "shutdownAction": "stopCompose",
+    "workspaceFolder": "/workspaces/faststream",
+    // "runArgs": [],
+    "remoteEnv": {},
+    "features": {
+        "ghcr.io/devcontainers/features/common-utils:2": {
+            "installZsh": true,
+            "installOhMyZsh": true,
+            "configureZshAsDefaultShell": true,
+            "username": "vscode",
+            "userUid": "1000",
+            "userGid": "1000"
+            // "upgradePackages": "true"
+        },
+        "ghcr.io/devcontainers/features/git:1": {},
+        "ghcr.io/devcontainers/features/git-lfs:1": {},
+        "ghcr.io/devcontainers/features/python:1": {
+            "version": "3.12" // python 3.12 is installed here
+        }
+    },
+    "updateContentCommand": "bash .devcontainer/setup.sh",
+    "postCreateCommand": [],
+    "customizations": {
+        "vscode": {
+            "settings": {
+                "python.linting.enabled": true,
+                "python.testing.pytestEnabled": true,
+                "editor.formatOnSave": true,
+                "editor.codeActionsOnSave": {
+                    "source.organizeImports": "always"
+                },
+                "[python]": {
+                    "editor.defaultFormatter": "ms-python.vscode-pylance"
+                },
+                "editor.rulers": [
+                    80
+                ]
+            },
+            "extensions": [
+                "ms-python.python",
+                "ms-python.vscode-pylance"
+            ]
+        }
+    }
 }
diff --git a/.devcontainer/docker-compose.yaml b/.devcontainer/docker-compose.yaml
@@ -1,33 +1,88 @@
 version: '3'
 
 services:
-  python-3.12-faststream-studio:  # nosemgrep
-    image: mcr.microsoft.com/devcontainers/python:3.12
-    container_name: python-3.12-faststream-studio
+  python-3.12-faststream:  # nosemgrep
+    # Installing python 3.10 here which is needed for pre-commit
+    # Actual python 3.12 is installed in devcontainer.json
+    image: mcr.microsoft.com/devcontainers/python:3.10
+    container_name: python-3.12-faststream
     volumes:
       - ../:/workspaces/faststream:cached
     command: sleep infinity
-    network_mode: "host"
+    env_file:
+      - ./devcontainer.env
+    networks:
+      - faststream-network
+    extra_hosts:
+      - "localhost:host-gateway"
 
-  kafka-faststream:  # nosemgrep
+  # nosemgrep: yaml.docker-compose.security.writable-filesystem-service.writable-filesystem-service
+  rabbitmq-faststream:
+    image: rabbitmq:alpine
+    container_name: faststream-${USER}-rabbitmq-py312
+    env_file:
+      - ./devcontainer.env
+    ports:
+      - "5672:5672"
+    # https://semgrep.dev/r?q=yaml.docker-compose.security.no-new-privileges.no-new-privileges
+    security_opt:
+      - no-new-privileges:true
+    networks:
+      - faststream-network
+  # nosemgrep: yaml.docker-compose.security.writable-filesystem-service.writable-filesystem-service
+  kafka-faststream:
     image: bitnami/kafka:3.5.0
-    container_name: kafka-faststream
+    container_name: faststream-${USER}-kafka-py312
+    env_file:
+      - ./devcontainer.env
     ports:
-      - 9092:9092
+      - "9092:9092"
+      - "9093:9093"
     environment:
       KAFKA_ENABLE_KRAFT: "true"
       KAFKA_CFG_NODE_ID: "1"
       KAFKA_CFG_PROCESS_ROLES: "broker,controller"
       KAFKA_CFG_CONTROLLER_LISTENER_NAMES: "CONTROLLER"
       KAFKA_CFG_LISTENERS: "PLAINTEXT://:9092,CONTROLLER://:9093"
       KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: "CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT"
-      KAFKA_CFG_ADVERTISED_LISTENERS: "PLAINTEXT://127.0.0.1:9092"
+      KAFKA_CFG_ADVERTISED_LISTENERS: "PLAINTEXT://localhost:9092"
       KAFKA_BROKER_ID: "1"
-      KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: "1@kafka-faststream:9093"
+      KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: "1@localhost:9093"
       ALLOW_PLAINTEXT_LISTENER: "true"
+    # https://semgrep.dev/r?q=yaml.docker-compose.security.no-new-privileges.no-new-privileges
+    security_opt:
+      - no-new-privileges:true
+    networks:
+      - faststream-network
+  # nosemgrep: yaml.docker-compose.security.writable-filesystem-service.writable-filesystem-service
+  nats-faststream:
+    image: nats
+    container_name: faststream-${USER}-nats-py312
+    command: -js
+    env_file:
+      - ./devcontainer.env
+    ports:
+      - 4222:4222
+      - 8222:8222  # management
+    # https://semgrep.dev/r?q=yaml.docker-compose.security.no-new-privileges.no-new-privileges
+    security_opt:
+      - no-new-privileges:true
+    networks:
+      - faststream-network
+  # nosemgrep: yaml.docker-compose.security.writable-filesystem-service.writable-filesystem-service
+  redis-faststream:
+    image: redis:alpine
+    container_name: faststream-${USER}-redis-py312
+    env_file:
+      - ./devcontainer.env
+    ports:
+      - 6379:6379
+    # https://semgrep.dev/r?q=yaml.docker-compose.security.no-new-privileges.no-new-privileges
+    security_opt:
+      - no-new-privileges:true
     networks:
       - faststream-network
 
 networks:
   faststream-network:
-    name: "${USER}-faststream-network"
+    name: "faststream-${USER}-network"
diff --git a/.devcontainer/setup.sh b/.devcontainer/setup.sh
@@ -1,7 +1,8 @@
-pip install uv
+# Update pip
+pip install --upgrade pip
 
-uv venv .venv
+# Install dev packages
+pip install -e ".[dev]"
 
-uv pip install -e ".[dev]"
-
-source .venv/bin/activate
+# Install pre-commit hooks if not installed already
+pre-commit install
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -38,7 +38,7 @@ repos:
         stages: [pre-commit, pre-merge-commit, manual]
         entry: "scripts/lint-pre-commit.sh"
         language: python
-        language_version: python3.8
+        language_version: python3.10
         types: [python]
         require_serial: true
         verbose: true
@@ -49,7 +49,7 @@ repos:
         name: Static analysis
         entry: "scripts/static-pre-commit.sh"
         language: python
-        language_version: python3.8
+        language_version: python3.10
         types: [python]
         require_serial: true
         verbose: true