diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 00000000..7a52405a --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,347 @@ +# ASU Microservices Architecture + +## Overview + +ASU uses a true microservices architecture where services are **completely independent** with **NO shared code**. Services communicate only via HTTP. + +## Architecture + +``` +┌──────────────────────────────────────────────────────────┐ +│ Load Balancer │ +│ /api/v1/build/prepare → asu-prepare:8001 │ +│ /api/v1/build → asu:8000 │ +└──────────────────────────────────────────────────────────┘ + │ │ + ▼ ▼ + ┌────────────────┐ ┌────────────────┐ + │ asu-prepare/ │◄──HTTP────│ asu/ │ + │ (Separate repo)│ │ (Main repo) │ + ├────────────────┤ ├────────────────┤ + │ • FastAPI │ │ • FastAPI │ + │ • Pydantic │ │ • Redis/RQ │ + │ • Package │ │ • Podman │ + │ resolution │ │ • ImageBuilder │ + │ • 512MB RAM │ │ • 4GB+ RAM │ + │ • 0.5 CPU │ │ • 4+ CPU │ + └────────────────┘ └────────────────┘ + │ + ├─ Redis + └─ Workers (scalable) +``` + +## Services + +### 1. Prepare Service (`asu-prepare/`) + +**Purpose:** Lightweight package resolution and validation. + +**Location:** `asu-prepare/` directory (separate codebase) + +**Files:** +- `main.py` - FastAPI app +- `build_request.py` - Data models +- `package_changes.py` - Migration logic +- `package_resolution.py` - Resolution algorithm +- `config.py` - Minimal config +- `Containerfile` - Container definition + +**Dependencies:** +- FastAPI, Uvicorn, Pydantic +- NO Redis, NO Podman, NO build tools + +**API:** +- `POST /api/v1/prepare` - Resolve packages + +**Communication:** +- **Receives:** HTTP requests from clients or build service +- **Returns:** JSON with resolved packages + +**Resources:** +- CPU: 0.5 cores +- RAM: 512MB +- Response time: <1s + +### 2. Build Service (`asu/`) + +**Purpose:** Heavy firmware building service. + +**Location:** `asu/` directory (main codebase) + +**Dependencies:** +- Everything from original ASU +- **PLUS** httpx for calling prepare service + +**Communication:** +- **Calls:** Prepare service via HTTP for package resolution +- **Uses:** Redis for queue/cache +- **Uses:** Podman for builds + +**Modified Files:** +- `asu/routers/api.py` - Proxies prepare requests via HTTP +- `asu/config.py` - Added `prepare_service_url` setting + +## Communication Flow + +### Prepare Request + +``` +Client + ↓ POST /api/v1/build/prepare +Build Service (asu/) + ↓ HTTP POST /api/v1/prepare +Prepare Service (asu-prepare/) + ↓ Package resolution +Build Service + ↓ Add cache info +Client +``` + +### Build Request (Direct) + +``` +Client + ↓ POST /api/v1/build +Build Service + ↓ Apply package changes locally + ↓ Queue build job +Workers + ↓ Build firmware +Client +``` + +### Build Request (Prepared) + +``` +Client + ↓ POST /api/v1/build/prepare +Prepare Service + ↓ Return resolved packages +Client (user approval) + ↓ POST /api/v1/build?skip_package_resolution=true +Build Service + ↓ Queue build (no package changes) +Workers + ↓ Build firmware +Client +``` + +## Key Design Principles + +### 1. No Shared Code + +- Each service has its OWN copy of necessary files +- NO `from asu.X import Y` between services +- Communication ONLY via HTTP + +**Benefits:** +- Can deploy/version independently +- No dependency conflicts +- Clear service boundaries +- Could rewrite in different language + +### 2. HTTP-Only Communication + +Services communicate via HTTP, not Python imports: + +```python +# Build service calling prepare service +async with httpx.AsyncClient() as client: + response = await client.post( + "http://asu-prepare:8001/api/v1/prepare", + json=build_request.model_dump() + ) +``` + +### 3. Independent Deployment + +Each service can be deployed separately: + +```bash +# Deploy only prepare service +cd asu-prepare +podman build -t asu-prepare . +podman run -p 8001:8001 asu-prepare + +# Deploy only build service +cd .. +podman build -t asu-build . +podman run -p 8000:8000 asu-build +``` + +## Deployment + +### Microservices (Recommended) + +```bash +podman-compose -f podman-compose.microservices.yml up -d +``` + +**Services Started:** +- `asu-prepare` - Prepare service (port 8001) +- `asu-build` - Build service (port 8000) +- `asu-worker` - Build workers (scalable) +- `redis` - Queue and cache +- `nginx` - Load balancer (optional) + +### Monolithic (Backward Compatible) + +```bash +podman-compose up -d +``` + +All functionality in one container (original behavior). + +## Scaling + +### Horizontal Scaling + +```bash +# Scale prepare service (cheap, lightweight) +podman-compose -f podman-compose.microservices.yml up -d --scale prepare=5 + +# Scale build workers (expensive, heavy) +podman-compose -f podman-compose.microservices.yml up -d --scale worker=10 +``` + +### Resource Allocation + +| Service | Instances | CPU/each | RAM/each | Total | +|---------|-----------|----------|----------|-------| +| Prepare | 5 | 0.5 | 512MB | 2.5 CPU, 2.5GB | +| Build | 1 | 4 | 4GB | 4 CPU, 4GB | +| Workers | 4 | 2 | 2GB | 8 CPU, 8GB | +| Redis | 1 | 1 | 1GB | 1 CPU, 1GB | +| **Total** | **11** | - | - | **15.5 CPU, 15.5GB** | + +## Configuration + +### Environment Variables + +**Prepare Service:** +- `UPSTREAM_URL` - OpenWrt downloads URL + +**Build Service:** +- `PREPARE_SERVICE_URL` - Prepare service URL (default: `http://asu-prepare:8001`) +- `REDIS_URL` - Redis connection string +- All existing ASU variables + +### Service Discovery + +Build service finds prepare service via: +1. Environment variable `PREPARE_SERVICE_URL` +2. Default: `http://asu-prepare:8001` (container name) + +## Migration Path + +### From Monolithic + +1. Deploy microservices alongside monolithic +2. Route prepare requests to new service +3. Monitor and test +4. Gradually shift build requests +5. Decommission monolithic + +### Code Duplication + +Yes, there is code duplication (BuildRequest model, package logic). This is **intentional**: + +- **Pros:** Complete independence, separate versioning, no coupling +- **Cons:** Updates must be made to both services + +**Philosophy:** Prefer duplication over coupling for microservices. + +## Benefits + +✅ **True Independence:** Services can be developed separately +✅ **Technology Freedom:** Could rewrite prepare in Go, Rust, etc. +✅ **Easy Scaling:** Scale services independently based on load +✅ **Clear Boundaries:** HTTP API is the contract +✅ **Fault Isolation:** Prepare failure doesn't affect builds +✅ **Resource Efficiency:** Run many cheap prepare instances + +## Drawbacks + +❌ **Code Duplication:** Models and logic duplicated +❌ **Network Overhead:** HTTP calls vs. function calls +❌ **Complexity:** More moving parts +❌ **Consistency:** Updates must be synchronized + +## When to Use + +**Use Microservices When:** +- High traffic (>1000 prepares/day) +- Need independent scaling +- Different teams own services +- Want deployment flexibility + +**Use Monolithic When:** +- Small deployment (<100 builds/day) +- Single admin +- Simplicity preferred +- Limited resources + +## Testing + +### Prepare Service + +```bash +cd asu-prepare +poetry run pytest +``` + +### Build Service + +```bash +cd .. +poetry run pytest +``` + +### Integration Testing + +```bash +# Start both services +podman-compose -f podman-compose.microservices.yml up -d + +# Test prepare +curl -X POST http://localhost:8001/api/v1/prepare \ + -H "Content-Type: application/json" \ + -d '{"version":"24.10.0","target":"ath79/generic","profile":"test","packages":["luci","auc"]}' + +# Test build calling prepare +curl -X POST http://localhost:8000/api/v1/build/prepare \ + -H "Content-Type: application/json" \ + -d '{"version":"24.10.0","target":"ath79/generic","profile":"test","packages":["luci"]}' +``` + +## Monitoring + +### Health Checks + +```bash +# Prepare service +curl http://asu-prepare:8001/health + +# Build service +curl http://asu-build:8000/health +``` + +### Metrics + +Each service exposes metrics independently: +- Request count +- Response time +- Error rate +- Resource usage + +## Future Enhancements + +1. **Service Mesh:** Istio/Linkerd for advanced routing +2. **gRPC:** Replace HTTP JSON with gRPC for performance +3. **Event Bus:** Redis Streams or RabbitMQ for async communication +4. **API Gateway:** Kong or similar for centralized routing +5. **Separate Languages:** Rewrite prepare in Go for performance + +## Summary + +ASU uses true microservices with complete code separation. Services communicate only via HTTP, enabling independent development, deployment, and scaling. diff --git a/README.md b/README.md index b5f991d2..b3617d5d 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,43 @@ For security reasons each build happens inside a container so that one build can't affect another build. For this to work a Podman container runs an API service so workers can themselfs execute builds inside containers. +### Microservices Architecture + +ASU can be deployed as independent microservices that run in separate containers: + +1. **Prepare Service** (Lightweight) + - Handles `/api/v1/build/prepare` endpoint + - Package resolution and validation only + - No heavy dependencies (Redis, Podman, ImageBuilder) + - Fast, stateless, can scale horizontally + - Minimal resource requirements (512MB RAM, 0.5 CPU) + +2. **Build Service** (Heavy) + - Handles `/api/v1/build` endpoint + - Actual firmware image building + - Requires Redis, RQ, Podman, ImageBuilder + - Resource-intensive (4GB+ RAM, 4+ CPU cores) + - Can scale workers independently + +**Benefits:** +- **Cost Efficiency**: Run many prepare instances cheaply, fewer build instances +- **Better Performance**: Prepare requests don't wait for build resources +- **Independent Scaling**: Scale each service based on demand +- **Resource Isolation**: Build failures don't affect prepare service +- **Deployment Flexibility**: Deploy services on different infrastructure + +**Deployment Options:** + +```bash +# Option 1: Monolithic (current, all-in-one) +podman-compose up -d + +# Option 2: Microservices (separate containers) +podman-compose -f podman-compose.microservices.yml up -d +``` + +See `podman-compose.microservices.yml` and `Containerfile.prepare` / `Containerfile.build` for microservices configuration. + ### Installation The server uses `podman-compose` to manage the containers. On a Debian based @@ -204,3 +241,88 @@ server: - [https://sysupgrade.openwrt.org/docs/](https://sysupgrade.openwrt.org/docs/) - [https://sysupgrade.openwrt.org/redoc](https://sysupgrade.openwrt.org/redoc/) + +#### Two-Step Build Process (Optional) + +For better user experience, clients can use a two-step build process to show +users what package changes will be made before building: + +##### Step 1: Prepare (Optional) + +```bash +POST /api/v1/build/prepare +``` + +This endpoint validates the request, applies package changes/migrations, and +returns what packages will be installed. This allows users to review and approve +changes before the actual build starts. + +**Example Request:** + +```json +{ + "version": "24.10.0", + "target": "ath79/generic", + "profile": "tplink_archer-c7-v5", + "packages": ["luci", "auc"], + "from_version": "23.05.0" +} +``` + +**Example Response:** + +```json +{ + "status": "prepared", + "original_packages": ["luci", "auc"], + "resolved_packages": ["luci", "owut"], + "changes": [ + { + "type": "migration", + "action": "replace", + "from_package": "auc", + "to_package": "owut", + "reason": "Package renamed in 24.10", + "automatic": true + } + ], + "prepared_request": { + "version": "24.10.0", + "target": "ath79/generic", + "profile": "tplink_archer-c7-v5", + "packages": ["luci", "owut"], + "diff_packages": false, + ... + }, + "request_hash": "abc123...", + "cache_available": false +} +``` + +##### Step 2: Build + +```bash +POST /api/v1/build?skip_package_resolution=true +``` + +When called with a prepared request and `skip_package_resolution=true`, the +endpoint builds exactly what was prepared without further package modifications. + +**Example Request:** + +```json +{ + "version": "24.10.0", + "target": "ath79/generic", + "profile": "tplink_archer-c7-v5", + "packages": ["luci", "owut"], + "diff_packages": false +} +``` + +#### Backward Compatibility + +Existing clients continue to work unchanged. The `/api/v1/build` endpoint still +applies package changes automatically when called directly without the prepare +step. The two-step process is entirely optional and provides enhanced user +control over package migrations. diff --git a/ansible/README.md b/ansible/README.md new file mode 100644 index 00000000..2beefd1f --- /dev/null +++ b/ansible/README.md @@ -0,0 +1,168 @@ +# ASU Ansible Deployment + +Ansible role to deploy the OpenWrt ASU (Attendant Sysupgrade Server) using Podman. + +## Features + +- Installs Podman and podman-compose +- Creates a non-root `asu` user for running containers +- Deploys ASU microservices architecture from the Git repository +- Configures Caddy reverse proxy with automatic HTTPS +- Configures environment variables +- Sets up systemd service for automatic startup +- Idempotent: running twice automatically rebuilds containers + +## Requirements + +- Target system: Linux with systemd +- Ansible 2.9+ +- Target OS: RHEL/Fedora/CentOS or Debian/Ubuntu (with appropriate package manager) + +## Role Variables + +Available variables are listed below, along with default values (see `defaults/main.yml`): + +```yaml +# User configuration +asu_user: asu +asu_group: asu +asu_home: /home/asu + +# Application paths +asu_app_dir: /home/asu/asu +public_path: /var/lib/asu/public + +# Domain configuration for Caddy reverse proxy +caddy_domain: "" # Set to domain for automatic HTTPS (e.g., "sysupgrade.staging.openwrt.org") + +# Force rebuild on every run +force_rebuild: true + +# Environment variables +allow_defaults: 0 +squid_cache: 0 +log_level: INFO +upstream_url: https://downloads.openwrt.org +``` + +## Directory Structure + +``` +ansible/ +├── roles/ +│ └── asu-deploy/ +│ ├── defaults/ +│ │ └── main.yml # Default variables +│ ├── tasks/ +│ │ └── main.yml # Main tasks +│ ├── handlers/ +│ │ └── main.yml # Handlers for restarts +│ └── templates/ +│ ├── env.j2 # .env template +│ └── asu-podman.service.j2 # systemd service +├── playbook.yml # Example playbook +└── inventory.ini # Inventory file +``` + +## Usage + +1. Edit the inventory file `inventory.ini`: + +```ini +[asu_servers] +your-server.example.com ansible_user=root +``` + +2. Customize variables in `playbook.yml` if needed: + +```yaml +- name: Deploy ASU with Podman + hosts: asu_servers + become: true + + roles: + - role: asu-deploy + vars: + caddy_domain: "sysupgrade.staging.openwrt.org" + force_rebuild: true + public_path: /var/lib/asu/public +``` + +3. Run the playbook: + +```bash +cd ansible +ansible-playbook -i inventory.ini playbook.yml +``` + +## Idempotency and Automatic Rebuilds + +The role is designed to automatically rebuild containers when: + +- `force_rebuild: true` is set (default) +- The Git repository has updates +- The `.env` file changes + +Running the playbook twice will: +1. First run: Install Podman, create user, clone repo, build and start containers +2. Second run: Check for updates, rebuild if needed (due to `force_rebuild: true`) + +To disable automatic rebuilds on every run, set `force_rebuild: false` in your playbook. + +## Managing the Service + +After deployment, the containers are managed by systemd: + +```bash +# On the target server +sudo systemctl status asu-podman +sudo systemctl restart asu-podman +sudo systemctl stop asu-podman +sudo systemctl start asu-podman + +# Or using podman-compose directly as the asu user +sudo -u asu podman-compose -f /home/asu/asu/podman-compose.yml ps +sudo -u asu podman-compose -f /home/asu/asu/podman-compose.yml logs +``` + +## Configuring Domain and HTTPS + +The role deploys Caddy as a reverse proxy. Configure the domain: + +```yaml +roles: + - role: asu-deploy + vars: + caddy_domain: "sysupgrade.staging.openwrt.org" # Automatic HTTPS + # caddy_domain: "" # HTTP only on port 80 +``` + +When a domain is set, Caddy will automatically obtain and manage Let's Encrypt certificates. + +## Security Notes + +- The `asu` user is created without root privileges +- Containers run in rootless mode under the `asu` user +- Podman socket is user-specific (`/run/user//podman/podman.sock`) +- User lingering is enabled to allow services to run without login + +## Troubleshooting + +Check container logs: +```bash +sudo -u asu podman-compose -f /home/asu/asu/podman-compose.yml logs -f +``` + +Check systemd service: +```bash +sudo systemctl status asu-podman +sudo journalctl -u asu-podman -f +``` + +Manual rebuild: +```bash +sudo -u asu bash +cd /home/asu/asu +podman-compose down +podman-compose up -d --build +``` diff --git a/ansible/inventory.ini b/ansible/inventory.ini new file mode 100644 index 00000000..9b23cf52 --- /dev/null +++ b/ansible/inventory.ini @@ -0,0 +1,8 @@ +[asu_servers] +46.224.121.175 ansible_user=root +# Add your server(s) here +# example.com ansible_user=root +# 192.168.1.100 ansible_user=admin ansible_become=true + +# Example with SSH key +# asu-prod.example.com ansible_user=deploy ansible_ssh_private_key_file=~/.ssh/id_rsa diff --git a/ansible/playbook.yml b/ansible/playbook.yml new file mode 100644 index 00000000..2619d2b5 --- /dev/null +++ b/ansible/playbook.yml @@ -0,0 +1,30 @@ +--- +- name: Deploy ASU with Podman + hosts: asu_servers + become: true + + roles: + - role: asu-deploy + vars: + # User configuration + asu_user: asu + asu_group: asu + asu_repository: https://github.com/aparcar/asu.git + asu_branch: claude/analyze-code-PCj61 + + # Paths + public_path: /var/lib/asu/public + asu_app_dir: /home/asu/asu + + # Domain configuration for Caddy + # Set to empty string for no domain (HTTP only on port 80) + # Set to a domain for automatic HTTPS with Let's Encrypt + caddy_domain: "sysupgrade.staging.openwrt.org" + + # Force rebuild on every run (set to true for automatic rebuilds) + force_rebuild: true + + # Environment variables + allow_defaults: 0 + log_level: INFO + upstream_url: https://downloads.openwrt.org diff --git a/ansible/roles/asu-deploy/defaults/main.yml b/ansible/roles/asu-deploy/defaults/main.yml new file mode 100644 index 00000000..e475fdfb --- /dev/null +++ b/ansible/roles/asu-deploy/defaults/main.yml @@ -0,0 +1,30 @@ +--- +# Default variables for asu-deploy role + +# User configuration +asu_user: asu +asu_group: asu +asu_home: /home/asu +asu_repository: https://github.com/openwrt/asu.git +asu_branch: main + +# Application paths +asu_app_dir: /home/asu/asu +public_path: /var/lib/asu/public +container_socket_path: /run/user/{{ asu_uid }}/podman/podman.sock + +# Environment variables +allow_defaults: 0 +squid_cache: 0 +redis_host: redis +redis_port: 6379 +log_level: INFO +upstream_url: https://downloads.openwrt.org + +# Domain configuration for Caddy +# Set to empty string for no domain (uses :80) +caddy_domain: "" +# Example: "sysupgrade.staging.openwrt.org" + +# Force rebuild on every run +force_rebuild: true diff --git a/ansible/roles/asu-deploy/handlers/main.yml b/ansible/roles/asu-deploy/handlers/main.yml new file mode 100644 index 00000000..6364c6c0 --- /dev/null +++ b/ansible/roles/asu-deploy/handlers/main.yml @@ -0,0 +1,21 @@ +--- +- name: Rebuild and restart containers + ansible.builtin.command: + cmd: podman-compose up -d --build + chdir: "{{ asu_app_dir }}" + become: true + become_user: "{{ asu_user }}" + environment: + PUBLIC_PATH: "{{ public_path }}" + CONTAINER_SOCKET_PATH: "{{ container_socket_path }}" + +- name: Reload systemd + ansible.builtin.systemd: + daemon_reload: true + become: true + +- name: Reload Caddy + ansible.builtin.systemd: + name: caddy + state: reloaded + become: true diff --git a/ansible/roles/asu-deploy/tasks/main.yml b/ansible/roles/asu-deploy/tasks/main.yml new file mode 100644 index 00000000..e887bf96 --- /dev/null +++ b/ansible/roles/asu-deploy/tasks/main.yml @@ -0,0 +1,167 @@ +--- +- name: Install Podman and dependencies + ansible.builtin.package: + name: + - podman + - podman-compose + - git + - caddy + state: present + become: true + +- name: Enable and start Podman socket + ansible.builtin.systemd: + name: podman.socket + enabled: true + state: started + become: true + +- name: Create asu group + ansible.builtin.group: + name: "{{ asu_group }}" + state: present + become: true + +- name: Create asu user + ansible.builtin.user: + name: "{{ asu_user }}" + group: "{{ asu_group }}" + home: "{{ asu_home }}" + shell: /bin/bash + create_home: true + system: false + become: true + register: asu_user_info + +- name: Set asu_uid fact + ansible.builtin.set_fact: + asu_uid: "{{ asu_user_info.uid }}" + +- name: Enable lingering for asu user (allows user services to run without login) + ansible.builtin.command: + cmd: loginctl enable-linger {{ asu_user }} + become: true + changed_when: false + +- name: Create public data directory + ansible.builtin.file: + path: "{{ public_path }}" + state: directory + owner: "{{ asu_user }}" + group: "{{ asu_group }}" + mode: '0755' + become: true + +- name: Create application directory + ansible.builtin.file: + path: "{{ asu_app_dir }}" + state: directory + owner: "{{ asu_user }}" + group: "{{ asu_group }}" + mode: '0755' + become: true + +- name: Check if repository exists + ansible.builtin.stat: + path: "{{ asu_app_dir }}/.git" + register: git_repo + +- name: Clone ASU repository + ansible.builtin.git: + repo: "{{ asu_repository }}" + version: "{{ asu_branch }}" + dest: "{{ asu_app_dir }}" + force: true + become: true + become_user: "{{ asu_user }}" + when: not git_repo.stat.exists + +- name: Update ASU repository + ansible.builtin.git: + repo: "{{ asu_repository }}" + version: "{{ asu_branch }}" + dest: "{{ asu_app_dir }}" + force: true + become: true + become_user: "{{ asu_user }}" + when: git_repo.stat.exists + register: git_update + +- name: Deploy .env file + ansible.builtin.template: + src: env.j2 + dest: "{{ asu_app_dir }}/.env" + owner: "{{ asu_user }}" + group: "{{ asu_group }}" + mode: '0644' + become: true + register: env_file + notify: Rebuild and restart containers + +- name: Deploy podman-compose.yml + ansible.builtin.copy: + src: "{{ playbook_dir }}/../podman-compose.yml" + dest: "{{ asu_app_dir }}/podman-compose.yml" + owner: "{{ asu_user }}" + group: "{{ asu_group }}" + mode: '0644' + become: true + register: compose_file_copy + notify: Rebuild and restart containers + +- name: Deploy Caddyfile to /etc/caddy + ansible.builtin.template: + src: Caddyfile.j2 + dest: /etc/caddy/Caddyfile + owner: root + group: root + mode: '0644' + become: true + register: caddyfile + notify: Reload Caddy + +- name: Enable and start Caddy service + ansible.builtin.systemd: + name: caddy + enabled: true + state: started + become: true + +- name: Stop existing containers + ansible.builtin.command: + cmd: podman-compose down + chdir: "{{ asu_app_dir }}" + become: true + become_user: "{{ asu_user }}" + failed_when: false + changed_when: false + when: force_rebuild or (git_update is defined and git_update.changed) or env_file.changed or compose_file_copy.changed + +- name: Build and start containers with podman-compose + ansible.builtin.command: + cmd: podman-compose up -d --build + chdir: "{{ asu_app_dir }}" + become: true + become_user: "{{ asu_user }}" + environment: + PUBLIC_PATH: "{{ public_path }}" + CONTAINER_SOCKET_PATH: "{{ container_socket_path }}" + when: force_rebuild or (git_update is defined and git_update.changed) or env_file.changed or compose_file_copy.changed + register: compose_up + +- name: Create systemd service for podman-compose + ansible.builtin.template: + src: asu-podman.service.j2 + dest: "/etc/systemd/system/asu-podman.service" + owner: root + group: root + mode: '0644' + become: true + notify: Reload systemd + +- name: Enable asu-podman service + ansible.builtin.systemd: + name: asu-podman + enabled: true + daemon_reload: true + become: true diff --git a/ansible/roles/asu-deploy/templates/Caddyfile.j2 b/ansible/roles/asu-deploy/templates/Caddyfile.j2 new file mode 100644 index 00000000..3d24fd6d --- /dev/null +++ b/ansible/roles/asu-deploy/templates/Caddyfile.j2 @@ -0,0 +1,41 @@ +{ + # Global options +{% if caddy_domain %} + email admin@{{ caddy_domain }} +{% else %} + auto_https off +{% endif %} +} + +{% if caddy_domain %} +{{ caddy_domain }} { +{% else %} +:80 { +{% endif %} + # Logging + log { + output file /var/log/caddy/access.log + format json + } + + # Enable compression + encode gzip + + # Route /api/v1/build/prepare to prepare service + handle /api/v1/build/prepare* { + reverse_proxy localhost:8001 { + health_uri / + health_interval 10s + health_timeout 5s + } + } + + # Route everything else to build service + handle { + reverse_proxy localhost:8000 { + health_uri /json/v1/overview.json + health_interval 10s + health_timeout 5s + } + } +} diff --git a/ansible/roles/asu-deploy/templates/asu-podman.service.j2 b/ansible/roles/asu-deploy/templates/asu-podman.service.j2 new file mode 100644 index 00000000..ec9b4c84 --- /dev/null +++ b/ansible/roles/asu-deploy/templates/asu-podman.service.j2 @@ -0,0 +1,20 @@ +[Unit] +Description=ASU Podman Compose Application +Requires=podman.service +After=network-online.target podman.service +Wants=network-online.target + +[Service] +Type=oneshot +RemainAfterExit=yes +User={{ asu_user }} +Group={{ asu_group }} +WorkingDirectory={{ asu_app_dir }} +Environment="PUBLIC_PATH={{ public_path }}" +Environment="CONTAINER_SOCKET_PATH={{ container_socket_path }}" + +ExecStart=/usr/bin/podman-compose up -d +ExecStop=/usr/bin/podman-compose down + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/asu-deploy/templates/env.j2 b/ansible/roles/asu-deploy/templates/env.j2 new file mode 100644 index 00000000..4e2777a4 --- /dev/null +++ b/ansible/roles/asu-deploy/templates/env.j2 @@ -0,0 +1,21 @@ +# Path to public files directory +PUBLIC_PATH={{ public_path }} + +# Path to Podman socket +CONTAINER_SOCKET_PATH={{ container_socket_path }} + +# Set to 1 to allow custom scripts running on first boot +ALLOW_DEFAULTS={{ allow_defaults }} + +# Set to 1 to enable Squid cache for ImageBuilder downloads +SQUID_CACHE={{ squid_cache }} + +# Redis server configuration +REDIS_HOST={{ redis_host }} +REDIS_PORT={{ redis_port }} + +# Log level (DEBUG, INFO, WARNING, ERROR, CRITICAL) +LOG_LEVEL={{ log_level }} + +# Upstream URL +UPSTREAM_URL={{ upstream_url }} diff --git a/asu-prepare/Containerfile b/asu-prepare/Containerfile new file mode 100644 index 00000000..39450593 --- /dev/null +++ b/asu-prepare/Containerfile @@ -0,0 +1,27 @@ +# Containerfile for ASU Prepare Service +# Minimal, lightweight service with NO build dependencies + +FROM python:3.11-slim + +WORKDIR /app + +# Install minimal dependencies only +COPY pyproject.toml ./ +RUN pip install poetry && \ + poetry config virtualenvs.create false && \ + poetry install --only main --no-root --no-interaction --no-ansi && \ + rm -rf ~/.cache/pypoetry + +# Copy prepare service code ONLY +COPY *.py ./ +COPY README.md ./ + +# Expose port +EXPOSE 8001 + +# Health check +HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ + CMD python -c "import httpx; httpx.get('http://localhost:8001/health')" || exit 1 + +# Run prepare service +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8001"] diff --git a/asu-prepare/README.md b/asu-prepare/README.md new file mode 100644 index 00000000..9b479a63 --- /dev/null +++ b/asu-prepare/README.md @@ -0,0 +1,142 @@ +# ASU Prepare Service + +Lightweight microservice for OpenWrt firmware build request preparation. + +## Purpose + +This service handles **package resolution and validation** for OpenWrt firmware build requests. It runs completely independently from the build service and has NO dependencies on: + +- ❌ Redis/RQ (no queue management) +- ❌ Podman (no container operations) +- ❌ ImageBuilder (no firmware building) +- ❌ Build infrastructure + +## What It Does + +✅ Validates build requests +✅ Applies package changes/migrations (e.g., `auc` → `owut`) +✅ Resolves hardware-specific package requirements +✅ Returns prepared package lists for user approval +✅ Tracks all changes made + +## API + +### POST /api/v1/prepare + +Prepare a build request without executing it. + +**Request:** +```json +{ + "version": "24.10.0", + "target": "ath79/generic", + "profile": "tplink_archer-c7-v5", + "packages": ["luci", "auc"] +} +``` + +**Response:** +```json +{ + "status": "prepared", + "original_packages": ["luci", "auc"], + "resolved_packages": ["luci", "owut"], + "changes": [ + { + "type": "migration", + "action": "replace", + "from_package": "auc", + "to_package": "owut", + "reason": "Package renamed in 24.10", + "automatic": true + } + ], + "prepared_request": { ... }, + "request_hash": "abc123..." +} +``` + +### GET /health + +Health check endpoint for load balancers. + +## Running + +### Development + +```bash +cd asu-prepare +poetry install +poetry run uvicorn main:app --reload --port 8001 +``` + +### Production (Docker/Podman) + +```bash +podman build -t asu-prepare -f Containerfile . +podman run -p 8001:8001 asu-prepare +``` + +## Dependencies + +Minimal dependencies (no heavy build tools): + +- FastAPI - Web framework +- Uvicorn - ASGI server +- Pydantic - Data validation + +## Resource Requirements + +- **CPU**: 0.5 cores +- **RAM**: 512MB +- **Response Time**: <1 second +- **Scalability**: Horizontal (stateless) + +## Architecture + +This service is completely stateless and can be scaled horizontally: + +``` +Load Balancer + ↓ + ┌───┴───┬───────┬───────┐ + ↓ ↓ ↓ ↓ +Prepare Prepare Prepare Prepare + (1) (2) (3) (4) +``` + +Each instance can handle requests independently. + +## Communication with Build Service + +The build service makes HTTP requests to this service: + +```python +# In build service +import httpx + +async with httpx.AsyncClient() as client: + response = await client.post( + "http://prepare-service:8001/api/v1/prepare", + json=build_request.model_dump() + ) + prepared = response.json() +``` + +## Testing + +```bash +poetry run pytest +``` + +## Configuration + +Environment variables: + +- `UPSTREAM_URL` - OpenWrt downloads URL (default: https://downloads.openwrt.org) +- `MAX_DEFAULTS_LENGTH` - Max first-boot script size (default: 20480) +- `MAX_CUSTOM_ROOTFS_SIZE_MB` - Max custom rootfs size (default: 1024) + +## License + +Same as ASU main project diff --git a/asu-prepare/__init__.py b/asu-prepare/__init__.py new file mode 100644 index 00000000..c64cdcea --- /dev/null +++ b/asu-prepare/__init__.py @@ -0,0 +1,19 @@ +""" +ASU Prepare Service - Independent Microservice + +This is a standalone microservice that handles package resolution and +validation for OpenWrt firmware build requests. + +It does NOT: +- Build firmware (no Podman, no ImageBuilder) +- Queue jobs (no Redis, no RQ) +- Store state (completely stateless) + +It DOES: +- Validate build requests +- Apply package changes/migrations +- Return resolved package lists +- Provide detailed change tracking +""" + +__version__ = "1.0.0" diff --git a/asu-prepare/config.py b/asu-prepare/config.py new file mode 100644 index 00000000..8a9b3525 --- /dev/null +++ b/asu-prepare/config.py @@ -0,0 +1,36 @@ +""" +Minimal configuration for prepare service + +No Redis, No Podman, No build infrastructure. +Only validation and package resolution settings. +""" + +from pydantic_settings import BaseSettings + + +class PrepareSettings(BaseSettings): + """Settings for the prepare service""" + + # Defaults validation + max_defaults_length: int = 20480 + max_custom_rootfs_size_mb: int = 1024 + + # Upstream for validation data + upstream_url: str = "https://downloads.openwrt.org" + + # Service identification + service_name: str = "asu-prepare" + service_version: str = "1.0.0" + + # Supported branches (simplified, could be loaded dynamically) + branches: dict[str, dict[str, str]] = { + "SNAPSHOT": {"name": "SNAPSHOT", "path": "snapshots"}, + "24.10": {"name": "24.10", "path": "releases/{version}"}, + "23.05": {"name": "23.05", "path": "releases/{version}"}, + "22.03": {"name": "22.03", "path": "releases/{version}"}, + "21.02": {"name": "21.02", "path": "releases/{version}"}, + } + + +# Global settings instance +settings = PrepareSettings() diff --git a/asu-prepare/main.py b/asu-prepare/main.py new file mode 100644 index 00000000..147548b2 --- /dev/null +++ b/asu-prepare/main.py @@ -0,0 +1,182 @@ +""" +ASU Prepare Service - Standalone FastAPI Application + +This is a completely independent microservice that runs separately from +the build service. It handles only package resolution and validation. + +No dependencies on: +- Redis/RQ +- Podman +- Build infrastructure +- ASU build service code + +Communication: +- Accepts HTTP requests +- Returns JSON responses +- Build service calls this via HTTP +""" + +import logging +from typing import Optional +from copy import deepcopy + +from fastapi import FastAPI, Response, status +from fastapi.responses import JSONResponse +from fastapi.middleware.cors import CORSMiddleware + +from prepare_request import PrepareRequest +from package_resolution import PackageResolver +from config import settings + +# Configure logging +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", +) +log = logging.getLogger("asu-prepare") + +# Create FastAPI app +app = FastAPI( + title="ASU Prepare Service", + description="Package resolution and validation service for OpenWrt firmware builds", + version=settings.service_version, +) + +# Add CORS middleware to allow cross-origin requests from build service +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], # In production, restrict to build service URLs + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + + +def calculate_request_hash(prepare_request: PrepareRequest) -> str: + """ + Calculate a reproducible hash for a build request. + + This is a simplified version that doesn't require the full util.py module. + """ + import hashlib + import json + + # Serialize request to consistent JSON + data = prepare_request.model_dump() + # Sort packages for consistency + if "packages" in data and data["packages"]: + data["packages"] = sorted(data["packages"]) + + # Create hash + serialized = json.dumps(data, sort_keys=True) + return hashlib.sha256(serialized.encode()).hexdigest() + + +@app.get("/") +async def root(): + """Root endpoint - service information""" + return { + "service": settings.service_name, + "version": settings.service_version, + "status": "running", + "endpoints": { + "prepare": "POST /api/v1/prepare", + "health": "GET /health", + }, + } + + +@app.get("/health") +async def health_check(): + """Health check endpoint for load balancers""" + return {"status": "healthy", "service": settings.service_name} + + +@app.post("/api/v1/prepare") +async def prepare(prepare_request: PrepareRequest, response: Response): + """ + Prepare a build request without executing it. + + This endpoint: + 1. Validates basic request structure (Pydantic does this) + 2. Sanitizes the profile name + 3. Applies package changes based on version/target/profile + 4. Returns the final package list and changes for user approval + 5. Does NOT queue a build job (no Redis access) + 6. Does NOT check cache (no Redis access) + + The build service will call this endpoint, then handle queueing and caching. + """ + try: + # Sanitize the profile + prepare_request.profile = prepare_request.profile.replace(",", "_") + + # Create a copy to preserve the original + request_copy = deepcopy(prepare_request) + + # Resolve packages and track changes + resolver = PackageResolver() + final_packages, changes = resolver.resolve(request_copy) + + # Create prepared request (with resolved packages) + # This is what the build service will receive + prepared_request = PrepareRequest( + distro=prepare_request.distro, + version=prepare_request.version, + from_version=prepare_request.from_version, + target=prepare_request.target, + profile=request_copy.profile, # Use sanitized profile + packages=final_packages, + ) + + # Calculate hash of the prepared request + request_hash = calculate_request_hash(prepared_request) + + log.info( + f"Prepared request for {prepare_request.version}/{prepare_request.target}/" + f"{prepare_request.profile} with {len(changes)} changes" + ) + + return { + "status": "prepared", + # Original request info + "original_packages": prepare_request.packages, + # Resolved packages + "resolved_packages": final_packages, + # What changed + "changes": [c.to_dict() for c in changes], + # Prepared request to send to /build + "prepared_request": prepared_request.model_dump(), + "request_hash": request_hash, + } + + except Exception as e: + log.error(f"Error preparing request: {e}", exc_info=True) + response.status_code = status.HTTP_500_INTERNAL_SERVER_ERROR + return { + "status": "error", + "detail": f"Failed to prepare request: {str(e)}", + } + + +@app.get("/api/v1/status") +async def service_status(): + """Service status endpoint""" + return { + "service": settings.service_name, + "version": settings.service_version, + "status": "operational", + "capabilities": { + "package_resolution": True, + "package_migration": True, + "request_validation": True, + "build_execution": False, # This service does NOT build + "caching": False, # This service does NOT cache + }, + } + + +if __name__ == "__main__": + import uvicorn + + uvicorn.run(app, host="0.0.0.0", port=8001, log_level="info") diff --git a/asu-prepare/package_changes.py b/asu-prepare/package_changes.py new file mode 100644 index 00000000..96bd12f1 --- /dev/null +++ b/asu-prepare/package_changes.py @@ -0,0 +1,196 @@ +import logging + +from prepare_request import PrepareRequest + +log = logging.getLogger("asu-prepare") + + +# Language pack replacements are done generically on a per-version basis. +# Note that the version comparison below applies to all versions the same +# or newer, so for example "24.10" applies to snapshots, too. +language_packs = { + "24.10": { + "luci-i18n-opkg-": "luci-i18n-package-manager-", + }, +} + + +def apply_package_changes(prepare_request: PrepareRequest): + """ + Apply package changes to the request + + Args: + prepare_request (PrepareRequest): The prepare request + """ + + def _add_if_missing(package): + if package not in prepare_request.packages: + prepare_request.packages.append(package) + log.debug(f"Added {package} to packages") + + # 23.05 specific changes + if prepare_request.version.startswith("23.05"): + # mediatek/mt7622 specific changes + if prepare_request.target == "mediatek/mt7622": + _add_if_missing("kmod-mt7622-firmware") + + # ath79/generic specific changes + elif prepare_request.target == "ath79/generic": + if prepare_request.profile in { + "buffalo_wzr-hp-g300nh-s", + "dlink_dir-825-b1", + "netgear_wndr3700", + "netgear_wndr3700-v2", + "netgear_wndr3800", + "netgear_wndr3800ch", + "netgear_wndrmac-v1", + "netgear_wndrmac-v2", + "trendnet_tew-673gru", + }: + _add_if_missing("kmod-switch-rtl8366s") + + elif prepare_request.profile == "buffalo_wzr-hp-g300nh-rb": + _add_if_missing("kmod-switch-rtl8366rb") + + if prepare_request.version.startswith("24.10"): + # `auc` no longer exists here + if "auc" in prepare_request.packages: + prepare_request.packages.remove("auc") + _add_if_missing("owut") + + # 25.12 specific changes + if prepare_request.version.startswith("25.12"): + # Changes for https://github.com/openwrt/openwrt/commit/8a7239009c5f4b28b696042b70ed1f8f89902915 + if prepare_request.target == "kirkwood/generic": + if prepare_request.profile in { + "checkpoint_l-50", + "endian_4i-edge-200", + "linksys_e4200-v2", + "linksys_ea3500", + "linksys_ea4500", + }: + _add_if_missing("kmod-dsa-mv88e6xxx") + # Changes for https://github.com/openwrt/openwrt/commit/eaa82118eadfd495f8512d55c01c1935b8b42c51 + elif prepare_request.target == "mvebu/cortexa9": + if prepare_request.profile in { + "cznic_turris-omnia", + "fortinet_fg-30e", + "fortinet_fwf-30e", + "fortinet_fg-50e", + "fortinet_fg-51e", + "fortinet_fg-52e", + "fortinet_fwf-50e-2r", + "fortinet_fwf-51e", + "iij_sa-w2", + "linksys_wrt1200ac", + "linksys_wrt1900acs", + "linksys_wrt1900ac-v1", + "linksys_wrt1900ac-v2", + "linksys_wrt3200acm", + "linksys_wrt32x", + "marvell_a370-rd", + }: + _add_if_missing("kmod-dsa-mv88e6xxx") + # Changes for https://github.com/openwrt/openwrt/commit/eaa82118eadfd495f8512d55c01c1935b8b42c51 + elif prepare_request.target == "mvebu/cortexa53": + if prepare_request.profile in { + "glinet_gl-mv1000", + "globalscale_espressobin", + "globalscale_espressobin-emmc", + "globalscale_espressobin-ultra", + "globalscale_espressobin-v7", + "globalscale_espressobin-v7-emmc", + "methode_udpu", + }: + _add_if_missing("kmod-dsa-mv88e6xxx") + # Changes for https://github.com/openwrt/openwrt/commit/eaa82118eadfd495f8512d55c01c1935b8b42c51 + elif prepare_request.target == "mvebu/cortexa72": + if prepare_request.profile in { + "checkpoint_v-80", + "checkpoint_v-81", + "globalscale_mochabin", + "mikrotik_rb5009", + "solidrun_clearfog-pro", + }: + _add_if_missing("kmod-dsa-mv88e6xxx") + # Changes for https://github.com/openwrt/openwrt/commit/a18d95f35bd54ade908e8ec3158435859402552d + elif prepare_request.target == "lantiq/xrx200": + if prepare_request.profile in { + "arcadyan_arv7519rw22", + "arcadyan_vgv7510kw22-brn", + "arcadyan_vgv7510kw22-nor", + "avm_fritz7412", + "avm_fritz7430", + "buffalo_wbmr-300hpd", + }: + _add_if_missing("xrx200-rev1.1-phy22f-firmware") + _add_if_missing("xrx200-rev1.2-phy22f-firmware") + elif prepare_request.profile in { + "tplink_vr200", + "tplink_vr200v", + "arcadyan_vgv7519-brn", + "arcadyan_vgv7519-nor", + "arcadyan_vrv9510kwac23", + "avm_fritz3370-rev2-hynix", + "avm_fritz3370-rev2-micron", + "avm_fritz3390", + "avm_fritz3490", + "avm_fritz3490-micron", + "avm_fritz5490", + "avm_fritz5490-micron", + "avm_fritz7360sl", + "avm_fritz7360-v2", + "avm_fritz7362sl", + "avm_fritz7490", + "avm_fritz7490-micron", + "bt_homehub-v5a", + "lantiq_easy80920-nand", + "lantiq_easy80920-nor", + "zyxel_p-2812hnu-f1", + "zyxel_p-2812hnu-f3", + }: + _add_if_missing("xrx200-rev1.1-phy11g-firmware") + _add_if_missing("xrx200-rev1.2-phy11g-firmware") + # Changes for https://github.com/openwrt/openwrt/commit/a18d95f35bd54ade908e8ec3158435859402552d + elif prepare_request.target == "lantiq/xrx200_legacy": + if prepare_request.profile in { + "alphanetworks_asl56026", + "netgear_dm200", + }: + _add_if_missing("xrx200-rev1.1-phy22f-firmware") + _add_if_missing("xrx200-rev1.2-phy22f-firmware") + elif prepare_request.profile in { + "tplink_tdw8970", + "tplink_tdw8980", + "arcadyan_vg3503j", + }: + _add_if_missing("xrx200-rev1.1-phy11g-firmware") + _add_if_missing("xrx200-rev1.2-phy11g-firmware") + # Changes for https://github.com/openwrt/openwrt/commit/3b7a92754e81432024b232c7cd7fe32593891ee0 + elif prepare_request.target == "bcm53xx/generic": + if prepare_request.profile in { + "meraki_mr32", + }: + _add_if_missing("kmod-hci-uart") + elif prepare_request.target == "ipq40xx/generic": + if prepare_request.profile in { + "linksys_whw03", + "linksys_whw03v2", + }: + _add_if_missing("kmod-hci-uart") + elif prepare_request.target == "qualcommax/ipq807x": + if prepare_request.profile in { + "linksys_mx4200v1", + "linksys_mx8500", + "zyxel_nbg7815", + }: + _add_if_missing("kmod-hci-uart") + + # TODO: if we ever fully implement 'packages_versions', this needs rework + for version, packages in language_packs.items(): + if prepare_request.version >= version: # Includes snapshots + for i, package in enumerate(prepare_request.packages): + for old, new in packages.items(): + if package.startswith(old): + lang = package.replace(old, "") + prepare_request.packages[i] = f"{new}{lang}" diff --git a/asu-prepare/package_resolution.py b/asu-prepare/package_resolution.py new file mode 100644 index 00000000..86de55dc --- /dev/null +++ b/asu-prepare/package_resolution.py @@ -0,0 +1,187 @@ +""" +Package resolution logic + +This module handles: +- Applying package changes based on version/target/profile +- Tracking what changes were made +- Calculating final package lists for prepare endpoint +""" + +import logging +from typing import Optional + +from prepare_request import PrepareRequest +from package_changes import apply_package_changes + +log = logging.getLogger("asu-prepare") + + +class PackageChange: + """Represents a single package change""" + + def __init__( + self, + change_type: str, # migration, addition, removal + action: str, # replace, add, remove + package: Optional[str] = None, + from_package: Optional[str] = None, + to_package: Optional[str] = None, + reason: str = "", + automatic: bool = True, + ): + self.type = change_type + self.action = action + self.package = package + self.from_package = from_package + self.to_package = to_package + self.reason = reason + self.automatic = automatic + + def to_dict(self): + """Convert to dictionary for JSON serialization""" + result = { + "type": self.type, + "action": self.action, + "reason": self.reason, + "automatic": self.automatic, + } + + if self.package: + result["package"] = self.package + if self.from_package: + result["from_package"] = self.from_package + if self.to_package: + result["to_package"] = self.to_package + + return result + + +class PackageResolver: + """Resolves packages for a build request""" + + def __init__(self): + self.changes: list[PackageChange] = [] + + def resolve( + self, prepare_request: PrepareRequest + ) -> tuple[list[str], list[PackageChange]]: + """ + Resolve packages for a build request. + + This method applies package changes based on version/target/profile + and tracks what was changed. + + Args: + prepare_request: The prepare request to resolve packages for + + Returns: + Tuple of (final_packages, changes_applied) + """ + self.changes = [] + + # Make a deep copy to track changes + original_packages = prepare_request.packages.copy() + + # Apply package changes (existing logic from package_changes.py) + apply_package_changes(prepare_request) + + # Track what changed + self._track_changes(original_packages, prepare_request.packages, prepare_request) + + return prepare_request.packages, self.changes + + def _track_changes( + self, + original: list[str], + modified: list[str], + prepare_request: PrepareRequest, + ): + """Track what changes were made""" + original_set = set(original) + modified_set = set(modified) + + added = modified_set - original_set + removed = original_set - modified_set + + # Detect migrations (package renames) + # Check for known migrations from package_changes.py + for removed_pkg in list(removed): + # Check if this is a known migration + migration = self._find_migration( + removed_pkg, prepare_request.version, prepare_request + ) + if migration and migration in added: + self.changes.append( + PackageChange( + change_type="migration", + action="replace", + from_package=removed_pkg, + to_package=migration, + reason=f"Package renamed in {prepare_request.version}", + automatic=True, + ) + ) + removed.remove(removed_pkg) + added.remove(migration) + + # Remaining removals + for pkg in removed: + self.changes.append( + PackageChange( + change_type="removal", + action="remove", + package=pkg, + reason="Package no longer available or needed", + automatic=True, + ) + ) + + # Remaining additions + for pkg in added: + reason = self._get_addition_reason(pkg, prepare_request) + self.changes.append( + PackageChange( + change_type="addition", + action="add", + package=pkg, + reason=reason, + automatic=True, + ) + ) + + def _find_migration( + self, package: str, version: str, prepare_request: PrepareRequest + ) -> Optional[str]: + """Find if package was migrated to another name""" + # Check for known migrations + if version.startswith("24.10"): + if package == "auc": + return "owut" + + # Check for language pack renames + from package_changes import language_packs + + for lang_version, packages in language_packs.items(): + if version >= lang_version: + for old, new in packages.items(): + if package.startswith(old): + lang = package.replace(old, "") + return f"{new}{lang}" + + return None + + def _get_addition_reason(self, package: str, prepare_request: PrepareRequest) -> str: + """Determine why package was added""" + if package.startswith("kmod-"): + # Check if it's a hardware-specific module + if prepare_request.target: + return f"Required kernel module for {prepare_request.target}" + return "Required kernel module" + + if package.startswith("luci-i18n-"): + return "Language pack" + + if package.startswith("xrx200-"): + return "Required PHY firmware" + + return "Required for this version/target/profile" diff --git a/asu-prepare/prepare_request.py b/asu-prepare/prepare_request.py new file mode 100644 index 00000000..5bcaeaae --- /dev/null +++ b/asu-prepare/prepare_request.py @@ -0,0 +1,102 @@ +""" +PrepareRequest model for the prepare service. + +This is a minimal model that only includes fields needed for package +resolution and migration. Fields like defaults, rootfs_size_mb, +repositories, etc. are NOT needed for the prepare step - those are +only relevant for the actual build step. + +The prepare service only needs to know: +- What version/target/profile you're building for +- What packages you want +- What version you're upgrading from (for migrations) +""" + +from typing import Annotated + +from pydantic import BaseModel, Field + +STRING_PATTERN = r"^[\w.,-]*$" +TARGET_PATTERN = r"^[\w]*/[\w]*$" + + +class PrepareRequest(BaseModel): + """ + Minimal request for the prepare endpoint. + + This only includes fields needed to resolve packages and apply migrations. + All build-specific fields (defaults, rootfs_size_mb, repositories, etc.) + are handled by the build service, not the prepare service. + """ + + version: Annotated[ + str, + Field( + examples=["23.05.2", "24.10.0", "SNAPSHOT"], + description=""" + The OpenWrt version to build for. This determines which + package migrations and changes should be applied. + """.strip(), + pattern=STRING_PATTERN, + ), + ] + + target: Annotated[ + str, + Field( + examples=["ath79/generic", "x86/64", "mediatek/mt7622"], + description=""" + The target platform. This determines which hardware-specific + packages need to be added (e.g., kernel modules, firmware). + """.strip(), + pattern=TARGET_PATTERN, + ), + ] + + profile: Annotated[ + str, + Field( + examples=["tplink_tl-wdr4300-v1", "generic", "linksys_e4200-v2"], + description=""" + The device profile. Some profiles require specific packages + (e.g., switch drivers, PHY firmware). + """.strip(), + pattern=STRING_PATTERN, + ), + ] + + packages: Annotated[ + list[Annotated[str, Field(pattern=STRING_PATTERN)]], + Field( + examples=[["luci", "vim", "tmux"], ["auc", "luci-i18n-opkg-en"]], + description=""" + List of packages to include in the build. The prepare service + will apply migrations (e.g., auc → owut) and add required + hardware-specific packages. + """.strip(), + ), + ] = [] + + from_version: Annotated[ + str | None, + Field( + examples=["23.05.0", "24.10.0"], + description=""" + Optional: The version the device is currently running. + This can be used for future migration logic that depends + on the upgrade path, but is not currently used. + """.strip(), + pattern=STRING_PATTERN, + ), + ] = None + + distro: Annotated[ + str, + Field( + description=""" + Distribution name. Currently only 'openwrt' is supported. + Included for consistency with build API. + """.strip(), + pattern=STRING_PATTERN, + ), + ] = "openwrt" diff --git a/asu-prepare/pyproject.toml b/asu-prepare/pyproject.toml new file mode 100644 index 00000000..4a5a7a4d --- /dev/null +++ b/asu-prepare/pyproject.toml @@ -0,0 +1,27 @@ +[tool.poetry] +name = "asu-prepare" +version = "1.0.0" +description = "ASU Prepare Service - Lightweight package resolution microservice" +authors = ["OpenWrt Community"] +readme = "README.md" +packages = [{include = "*"}] + +[tool.poetry.dependencies] +python = "^3.11" +fastapi = "^0.119.0" +uvicorn = "^0.37.0" +pydantic = "^2.12.0" +pydantic-settings = "^2.7.0" + +[tool.poetry.group.dev.dependencies] +pytest = "^8.4.2" +httpx = "^0.28.1" +ruff = "^0.14.9" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" + +[tool.ruff] +line-length = 88 +target-version = "py311" diff --git a/asu-prepare/pytest.ini b/asu-prepare/pytest.ini new file mode 100644 index 00000000..693e1280 --- /dev/null +++ b/asu-prepare/pytest.ini @@ -0,0 +1,13 @@ +[pytest] +testpaths = tests +python_files = test_*.py +python_classes = Test* +python_functions = test_* +addopts = + -v + --strict-markers + --tb=short + --color=yes +markers = + slow: marks tests as slow (deselect with '-m "not slow"') + integration: marks tests as integration tests diff --git a/asu-prepare/tests/README.md b/asu-prepare/tests/README.md new file mode 100644 index 00000000..d5b49a0f --- /dev/null +++ b/asu-prepare/tests/README.md @@ -0,0 +1,157 @@ +# ASU Prepare Service Tests + +Comprehensive test suite for the asu-prepare microservice. + +## Test Structure + +``` +tests/ +├── __init__.py # Test package initialization +├── conftest.py # Pytest fixtures and configuration +├── test_api.py # FastAPI endpoint tests +├── test_build_request.py # BuildRequest model validation tests +├── test_package_resolution.py # Package resolution logic tests +├── test_package_changes.py # Package changes logic tests +└── test_integration.py # Integration and workflow tests +``` + +## Running Tests + +### Run all tests +```bash +cd asu-prepare +pytest +``` + +### Run specific test file +```bash +pytest tests/test_api.py +``` + +### Run specific test class +```bash +pytest tests/test_api.py::TestPrepareEndpoint +``` + +### Run specific test +```bash +pytest tests/test_api.py::TestPrepareEndpoint::test_prepare_basic_request +``` + +### Run with verbose output +```bash +pytest -v +``` + +### Run with coverage +```bash +pytest --cov=. --cov-report=html +``` + +## Test Categories + +### API Tests (`test_api.py`) +- Root endpoint functionality +- Health check endpoint +- Prepare endpoint validation and responses +- Status endpoint capabilities +- Request/response formats +- Error handling + +### Build Request Tests (`test_build_request.py`) +- Pydantic model validation +- Required field validation +- Pattern matching (version, target, profile, packages) +- Default values +- Serialization/deserialization + +### Package Resolution Tests (`test_package_resolution.py`) +- Basic package resolution +- Package migrations (e.g., auc → owut) +- Language pack migrations +- Change tracking +- PackageChange model + +### Package Changes Tests (`test_package_changes.py`) +- Version-specific changes (23.05, 24.10, 25.12) +- Target-specific additions +- Profile-specific kernel modules +- Hardware-specific firmware +- Language pack replacements + +### Integration Tests (`test_integration.py`) +- Complete prepare workflow +- Multi-step changes +- Idempotent operations +- Service independence verification +- Stateless operation verification + +## Test Fixtures + +### `client` +FastAPI TestClient for making HTTP requests to the service. + +### `sample_build_request` +Basic valid build request for testing standard flows. + +### `migration_build_request` +Request that triggers auc → owut migration (24.10). + +### `language_pack_migration_request` +Request that triggers language pack migration (24.10). + +### `hardware_specific_request` +Request for hardware needing additional modules (23.05). + +### `dsa_mv88e6xxx_request` +Request needing kmod-dsa-mv88e6xxx (25.12). + +### `xrx200_phy_firmware_request` +Request needing XRX200 PHY firmware (25.12). + +## Key Testing Principles + +1. **Service Independence**: Tests verify the prepare service has no dependencies on Redis, Podman, or build infrastructure. + +2. **Stateless Operation**: Tests verify the service is stateless and produces consistent results. + +3. **Validation**: Tests ensure Pydantic models properly validate all input fields. + +4. **Package Changes**: Tests verify all version/target/profile-specific package changes work correctly. + +5. **Migration Tracking**: Tests verify all package changes are properly tracked and reported. + +## Adding New Tests + +When adding new package changes logic: + +1. Add test fixtures in `conftest.py` if needed +2. Add specific test cases in `test_package_changes.py` +3. Add resolution tests in `test_package_resolution.py` +4. Add integration workflow test in `test_integration.py` + +Example: +```python +def test_new_version_package_change(self): + """26.01 should add new-package for specific target""" + request = BuildRequest( + version="26.01.0", + target="new/target", + profile="test", + packages=["luci"], + ) + + apply_package_changes(request) + + assert "new-package" in request.packages +``` + +## Continuous Integration + +These tests are designed to run quickly without external dependencies: +- No Redis required +- No Podman required +- No network requests (all mocked) +- Fast execution (< 1 second total) + +Perfect for CI/CD pipelines. diff --git a/asu-prepare/tests/__init__.py b/asu-prepare/tests/__init__.py new file mode 100644 index 00000000..bb36c089 --- /dev/null +++ b/asu-prepare/tests/__init__.py @@ -0,0 +1 @@ +"""Tests for asu-prepare service""" diff --git a/asu-prepare/tests/conftest.py b/asu-prepare/tests/conftest.py new file mode 100644 index 00000000..1b150bdc --- /dev/null +++ b/asu-prepare/tests/conftest.py @@ -0,0 +1,78 @@ +"""Test configuration and fixtures for asu-prepare service""" + +import pytest +from fastapi.testclient import TestClient + + +@pytest.fixture +def client(): + """FastAPI test client""" + from main import app + + return TestClient(app) + + +@pytest.fixture +def sample_build_request(): + """Sample valid build request""" + return { + "version": "23.05.5", + "target": "ath79/generic", + "profile": "tplink_tl-wdr4300-v1", + "packages": ["luci", "vim"], + } + + +@pytest.fixture +def migration_build_request(): + """Build request that triggers package migrations""" + return { + "version": "24.10.0", + "target": "ath79/generic", + "profile": "tplink_tl-wdr4300-v1", + "packages": ["luci", "auc"], # auc should migrate to owut in 24.10 + } + + +@pytest.fixture +def language_pack_migration_request(): + """Build request that triggers language pack migration""" + return { + "version": "24.10.0", + "target": "ath79/generic", + "profile": "tplink_tl-wdr4300-v1", + "packages": ["luci", "luci-i18n-opkg-en"], # Should migrate to luci-i18n-package-manager-en + } + + +@pytest.fixture +def hardware_specific_request(): + """Build request for hardware that needs additional modules (23.05)""" + return { + "version": "23.05.0", + "target": "mediatek/mt7622", + "profile": "test-device", + "packages": ["luci"], + } + + +@pytest.fixture +def dsa_mv88e6xxx_request(): + """Build request that needs kmod-dsa-mv88e6xxx (25.12)""" + return { + "version": "25.12.0", + "target": "kirkwood/generic", + "profile": "checkpoint_l-50", + "packages": ["luci"], + } + + +@pytest.fixture +def xrx200_phy_firmware_request(): + """Build request that needs XRX200 PHY firmware (25.12)""" + return { + "version": "25.12.0", + "target": "lantiq/xrx200", + "profile": "arcadyan_arv7519rw22", + "packages": ["luci"], + } diff --git a/asu-prepare/tests/test_api.py b/asu-prepare/tests/test_api.py new file mode 100644 index 00000000..6ffbc5f7 --- /dev/null +++ b/asu-prepare/tests/test_api.py @@ -0,0 +1,156 @@ +"""Tests for the FastAPI endpoints""" + +import pytest + + +class TestRootEndpoint: + """Tests for the root endpoint""" + + def test_root_returns_service_info(self, client): + """Root endpoint should return service information""" + response = client.get("/") + assert response.status_code == 200 + data = response.json() + assert data["service"] == "asu-prepare" + assert "version" in data + assert data["status"] == "running" + assert "endpoints" in data + + def test_root_lists_endpoints(self, client): + """Root endpoint should list available endpoints""" + response = client.get("/") + data = response.json() + assert "prepare" in data["endpoints"] + assert "health" in data["endpoints"] + + +class TestHealthCheck: + """Tests for the health check endpoint""" + + def test_health_check_returns_healthy(self, client): + """Health check should return healthy status""" + response = client.get("/health") + assert response.status_code == 200 + data = response.json() + assert data["status"] == "healthy" + assert data["service"] == "asu-prepare" + + +class TestPrepareEndpoint: + """Tests for the /api/v1/prepare endpoint""" + + def test_prepare_basic_request(self, client, sample_build_request): + """Prepare endpoint should accept valid build request""" + response = client.post("/api/v1/prepare", json=sample_build_request) + assert response.status_code == 200 + data = response.json() + assert data["status"] == "prepared" + assert "resolved_packages" in data + assert "changes" in data + assert "prepared_request" in data + assert "request_hash" in data + + def test_prepare_preserves_original_packages(self, client, sample_build_request): + """Prepare endpoint should preserve original package list""" + response = client.post("/api/v1/prepare", json=sample_build_request) + data = response.json() + assert data["original_packages"] == sample_build_request["packages"] + + def test_prepare_sanitizes_profile(self, client): + """Prepare endpoint should sanitize profile names""" + request = { + "version": "23.05.5", + "target": "ath79/generic", + "profile": "test,profile,with,commas", + "packages": ["luci"], + } + response = client.post("/api/v1/prepare", json=request) + assert response.status_code == 200 + data = response.json() + # Profile should have commas replaced with underscores + assert "," not in data["prepared_request"]["profile"] + assert data["prepared_request"]["profile"] == "test_profile_with_commas" + + def test_prepare_returns_only_minimal_fields(self, client): + """Prepared request should only contain minimal fields needed for migration""" + request = { + "version": "23.05.5", + "target": "ath79/generic", + "profile": "test", + "packages": ["luci"], + } + response = client.post("/api/v1/prepare", json=request) + data = response.json() + prepared = data["prepared_request"] + + # Should have minimal fields + assert "version" in prepared + assert "target" in prepared + assert "profile" in prepared + assert "packages" in prepared + + # Should NOT have build-specific fields + assert "diff_packages" not in prepared + assert "rootfs_size_mb" not in prepared + assert "repositories" not in prepared + assert "repository_keys" not in prepared + + def test_prepare_returns_hash(self, client, sample_build_request): + """Prepare endpoint should return request hash""" + response = client.post("/api/v1/prepare", json=sample_build_request) + data = response.json() + assert "request_hash" in data + assert len(data["request_hash"]) == 64 # SHA256 hash + + def test_prepare_same_request_same_hash(self, client, sample_build_request): + """Same request should produce same hash""" + response1 = client.post("/api/v1/prepare", json=sample_build_request) + response2 = client.post("/api/v1/prepare", json=sample_build_request) + hash1 = response1.json()["request_hash"] + hash2 = response2.json()["request_hash"] + assert hash1 == hash2 + + def test_prepare_invalid_request(self, client): + """Invalid request should return validation error""" + invalid_request = { + "version": "23.05.5", + # Missing required fields + } + response = client.post("/api/v1/prepare", json=invalid_request) + assert response.status_code == 422 # Validation error + + def test_prepare_invalid_pattern(self, client): + """Request with invalid pattern should fail validation""" + invalid_request = { + "version": "23.05.5", + "target": "invalid target", # Space not allowed + "profile": "test", + "packages": ["luci"], + } + response = client.post("/api/v1/prepare", json=invalid_request) + assert response.status_code == 422 + + +class TestStatusEndpoint: + """Tests for the /api/v1/status endpoint""" + + def test_status_returns_service_info(self, client): + """Status endpoint should return service information""" + response = client.get("/api/v1/status") + assert response.status_code == 200 + data = response.json() + assert data["service"] == "asu-prepare" + assert data["status"] == "operational" + + def test_status_lists_capabilities(self, client): + """Status endpoint should list service capabilities""" + response = client.get("/api/v1/status") + data = response.json() + capabilities = data["capabilities"] + # Prepare service can do these + assert capabilities["package_resolution"] is True + assert capabilities["package_migration"] is True + assert capabilities["request_validation"] is True + # But NOT these (build service responsibilities) + assert capabilities["build_execution"] is False + assert capabilities["caching"] is False diff --git a/asu-prepare/tests/test_build_request.py b/asu-prepare/tests/test_build_request.py new file mode 100644 index 00000000..d0435aa3 --- /dev/null +++ b/asu-prepare/tests/test_build_request.py @@ -0,0 +1,149 @@ +"""Tests for PrepareRequest model validation""" + +import pytest +from pydantic import ValidationError +from prepare_request import PrepareRequest + + +class TestPrepareRequestValidation: + """Tests for PrepareRequest Pydantic validation""" + + def test_minimal_valid_request(self): + """Minimal valid request should work""" + request = PrepareRequest( + version="23.05.5", + target="ath79/generic", + profile="test", + ) + + assert request.version == "23.05.5" + assert request.target == "ath79/generic" + assert request.profile == "test" + assert request.packages == [] + + def test_full_valid_request(self): + """Full valid request with all fields should work""" + request = PrepareRequest( + distro="openwrt", + version="23.05.5", + from_version="23.05.0", + target="ath79/generic", + profile="tplink_tl-wdr4300-v1", + packages=["luci", "vim", "tmux"], + ) + + assert request.distro == "openwrt" + assert request.from_version == "23.05.0" + assert len(request.packages) == 3 + + def test_default_values(self): + """Default values should be set correctly""" + request = PrepareRequest( + version="23.05.5", + target="ath79/generic", + profile="test", + ) + + assert request.distro == "openwrt" + assert request.from_version is None + assert request.packages == [] + + def test_missing_required_field(self): + """Missing required field should raise ValidationError""" + with pytest.raises(ValidationError): + PrepareRequest( + version="23.05.5", + # Missing target and profile + ) + + def test_invalid_version_pattern(self): + """Invalid version pattern should raise ValidationError""" + with pytest.raises(ValidationError): + PrepareRequest( + version="23.05.5 invalid", # Space not allowed + target="ath79/generic", + profile="test", + ) + + def test_invalid_target_pattern(self): + """Invalid target pattern should raise ValidationError""" + with pytest.raises(ValidationError): + PrepareRequest( + version="23.05.5", + target="invalid", # Must be format: arch/subarch + profile="test", + ) + + def test_invalid_profile_pattern(self): + """Invalid profile pattern should raise ValidationError""" + with pytest.raises(ValidationError): + PrepareRequest( + version="23.05.5", + target="ath79/generic", + profile="test profile", # Space not allowed + ) + + def test_invalid_package_pattern(self): + """Invalid package name pattern should raise ValidationError""" + with pytest.raises(ValidationError): + PrepareRequest( + version="23.05.5", + target="ath79/generic", + profile="test", + packages=["vim", "invalid package"], # Space not allowed + ) + + def test_valid_package_patterns(self): + """Valid package name patterns should work""" + request = PrepareRequest( + version="23.05.5", + target="ath79/generic", + profile="test", + packages=["vim", "luci-i18n-base-en", "kmod-usb-core", "lib.so.1"], + ) + + assert len(request.packages) == 4 + + def test_snapshot_version(self): + """SNAPSHOT version should be valid""" + request = PrepareRequest( + version="SNAPSHOT", + target="ath79/generic", + profile="test", + ) + + assert request.version == "SNAPSHOT" + + +class TestPrepareRequestSerialization: + """Tests for PrepareRequest serialization""" + + def test_model_dump(self): + """model_dump should produce dictionary""" + request = PrepareRequest( + version="23.05.5", + target="ath79/generic", + profile="test", + packages=["luci", "vim"], + ) + + data = request.model_dump() + + assert isinstance(data, dict) + assert data["version"] == "23.05.5" + assert data["target"] == "ath79/generic" + assert data["packages"] == ["luci", "vim"] + + def test_model_dump_json(self): + """model_dump_json should produce JSON string""" + request = PrepareRequest( + version="23.05.5", + target="ath79/generic", + profile="test", + ) + + json_str = request.model_dump_json() + + assert isinstance(json_str, str) + assert "23.05.5" in json_str + assert "ath79/generic" in json_str diff --git a/asu-prepare/tests/test_integration.py b/asu-prepare/tests/test_integration.py new file mode 100644 index 00000000..3b8c01a9 --- /dev/null +++ b/asu-prepare/tests/test_integration.py @@ -0,0 +1,228 @@ +"""Integration tests for the complete prepare workflow""" + +import pytest + + +class TestPrepareIntegration: + """Integration tests for the complete prepare workflow""" + + def test_complete_prepare_workflow(self, client): + """Test complete prepare workflow from request to response""" + request = { + "version": "24.10.0", + "target": "ath79/generic", + "profile": "tplink_tl-wdr4300-v1", + "packages": ["luci", "auc", "luci-i18n-opkg-en"], + } + + response = client.post("/api/v1/prepare", json=request) + + assert response.status_code == 200 + data = response.json() + + # Should have migrated packages + assert "auc" not in data["resolved_packages"] + assert "owut" in data["resolved_packages"] + assert "luci-i18n-opkg-en" not in data["resolved_packages"] + assert "luci-i18n-package-manager-en" in data["resolved_packages"] + + # Should track changes + assert len(data["changes"]) >= 2 + + # Should have prepared request ready to send to build service + assert data["prepared_request"]["version"] == "24.10.0" + assert "owut" in data["prepared_request"]["packages"] + assert "auc" not in data["prepared_request"]["packages"] + + def test_hardware_specific_workflow(self, client): + """Test workflow with hardware-specific additions""" + request = { + "version": "25.12.0", + "target": "kirkwood/generic", + "profile": "checkpoint_l-50", + "packages": ["luci"], + } + + response = client.post("/api/v1/prepare", json=request) + + assert response.status_code == 200 + data = response.json() + + # Should have added hardware-specific module + assert "kmod-dsa-mv88e6xxx" in data["resolved_packages"] + + # Should track the addition + additions = [c for c in data["changes"] if c["type"] == "addition"] + assert len(additions) >= 1 + + def test_multiple_changes_workflow(self, client): + """Test workflow with multiple types of changes""" + request = { + "version": "25.12.0", + "target": "lantiq/xrx200", + "profile": "arcadyan_arv7519rw22", + "packages": ["luci"], + } + + response = client.post("/api/v1/prepare", json=request) + + assert response.status_code == 200 + data = response.json() + + # Should have added PHY firmware packages + assert "xrx200-rev1.1-phy22f-firmware" in data["resolved_packages"] + assert "xrx200-rev1.2-phy22f-firmware" in data["resolved_packages"] + + # Should track multiple additions + additions = [c for c in data["changes"] if c["type"] == "addition"] + assert len(additions) >= 2 + + def test_prepare_then_build_workflow_simulation(self, client): + """Simulate prepare -> build workflow""" + # Step 1: Prepare the request + original_request = { + "version": "24.10.0", + "target": "ath79/generic", + "profile": "test,device", # Will be sanitized + "packages": ["luci", "auc"], + "diff_packages": True, + } + + prepare_response = client.post("/api/v1/prepare", json=original_request) + assert prepare_response.status_code == 200 + prepare_data = prepare_response.json() + + # Step 2: Get the prepared request + prepared_request = prepare_data["prepared_request"] + + # Verify prepared request is ready for build + assert prepared_request["profile"] == "test_device" # Sanitized + assert "owut" in prepared_request["packages"] # Migrated + assert "auc" not in prepared_request["packages"] # Removed + + # Step 3: Verify request hash for caching + request_hash = prepare_data["request_hash"] + assert len(request_hash) == 64 + + def test_idempotent_prepare(self, client): + """Preparing the same request twice should give same results""" + request = { + "version": "24.10.0", + "target": "ath79/generic", + "profile": "test", + "packages": ["luci", "auc"], + } + + response1 = client.post("/api/v1/prepare", json=request) + response2 = client.post("/api/v1/prepare", json=request) + + data1 = response1.json() + data2 = response2.json() + + assert data1["resolved_packages"] == data2["resolved_packages"] + assert data1["request_hash"] == data2["request_hash"] + assert len(data1["changes"]) == len(data2["changes"]) + + def test_from_version_preserved(self, client): + """from_version should be preserved in prepared request""" + request = { + "version": "24.10.0", + "from_version": "23.05.0", + "target": "ath79/generic", + "profile": "test", + "packages": ["luci"], + } + + response = client.post("/api/v1/prepare", json=request) + data = response.json() + + assert data["prepared_request"]["from_version"] == "23.05.0" + + def test_build_specific_fields_not_in_prepare(self, client): + """Build-specific fields should NOT be in prepared request""" + request = { + "version": "23.05.5", + "target": "ath79/generic", + "profile": "test", + "packages": ["luci"], + } + + response = client.post("/api/v1/prepare", json=request) + data = response.json() + + prepared = data["prepared_request"] + + # Prepare service should NOT include build-specific fields + assert "rootfs_size_mb" not in prepared + assert "repositories" not in prepared + assert "repository_keys" not in prepared + assert "client" not in prepared + assert "diff_packages" not in prepared + + # Should only have minimal fields needed for package resolution + assert "version" in prepared + assert "target" in prepared + assert "profile" in prepared + assert "packages" in prepared + + +class TestServiceIndependence: + """Tests to verify service independence""" + + def test_no_redis_dependency(self, client): + """Prepare service should work without Redis""" + # This test verifies that no Redis connection is attempted + request = { + "version": "23.05.5", + "target": "ath79/generic", + "profile": "test", + "packages": ["luci"], + } + + # Should succeed even though no Redis is available + response = client.post("/api/v1/prepare", json=request) + assert response.status_code == 200 + + def test_no_build_execution(self, client): + """Prepare service should not execute builds""" + # Verify capabilities show build_execution: False + response = client.get("/api/v1/status") + data = response.json() + + assert data["capabilities"]["build_execution"] is False + + def test_no_caching(self, client): + """Prepare service should not handle caching""" + # Verify capabilities show caching: False + response = client.get("/api/v1/status") + data = response.json() + + assert data["capabilities"]["caching"] is False + + def test_stateless_operation(self, client): + """Prepare service should be stateless""" + # Same request should produce same response regardless of order + request1 = { + "version": "24.10.0", + "target": "ath79/generic", + "profile": "test1", + "packages": ["luci"], + } + request2 = { + "version": "23.05.5", + "target": "x86/64", + "profile": "test2", + "packages": ["vim"], + } + + # Process in one order + r1a = client.post("/api/v1/prepare", json=request1) + r2a = client.post("/api/v1/prepare", json=request2) + + # Process in reverse order + r2b = client.post("/api/v1/prepare", json=request2) + r1b = client.post("/api/v1/prepare", json=request1) + + # Results should be identical + assert r1a.json()["request_hash"] == r1b.json()["request_hash"] + assert r2a.json()["request_hash"] == r2b.json()["request_hash"] diff --git a/asu-prepare/tests/test_package_changes.py b/asu-prepare/tests/test_package_changes.py new file mode 100644 index 00000000..af57099a --- /dev/null +++ b/asu-prepare/tests/test_package_changes.py @@ -0,0 +1,268 @@ +"""Tests for package changes logic""" + +import pytest +from prepare_request import PrepareRequest +from package_changes import apply_package_changes + + +class TestPackageChanges2305: + """Tests for 23.05 specific package changes""" + + def test_mediatek_mt7622_firmware(self): + """23.05 mediatek/mt7622 should add kmod-mt7622-firmware""" + request = PrepareRequest( + version="23.05.0", + target="mediatek/mt7622", + profile="test-device", + packages=["luci"], + ) + + apply_package_changes(request) + + assert "kmod-mt7622-firmware" in request.packages + + def test_ath79_rtl8366s_switch(self): + """23.05 ath79 specific profiles should add rtl8366s switch""" + profiles_needing_switch = [ + "buffalo_wzr-hp-g300nh-s", + "dlink_dir-825-b1", + "netgear_wndr3700", + "netgear_wndr3700-v2", + "netgear_wndr3800", + ] + + for profile in profiles_needing_switch: + request = PrepareRequest( + version="23.05.0", + target="ath79/generic", + profile=profile, + packages=["luci"], + ) + + apply_package_changes(request) + + assert "kmod-switch-rtl8366s" in request.packages + + def test_ath79_rtl8366rb_switch(self): + """23.05 buffalo_wzr-hp-g300nh-rb should add rtl8366rb switch""" + request = PrepareRequest( + version="23.05.0", + target="ath79/generic", + profile="buffalo_wzr-hp-g300nh-rb", + packages=["luci"], + ) + + apply_package_changes(request) + + assert "kmod-switch-rtl8366rb" in request.packages + + +class TestPackageChanges2410: + """Tests for 24.10 specific package changes""" + + def test_auc_to_owut_migration(self): + """24.10 should migrate auc to owut""" + request = PrepareRequest( + version="24.10.0", + target="ath79/generic", + profile="test", + packages=["luci", "auc", "vim"], + ) + + apply_package_changes(request) + + assert "auc" not in request.packages + assert "owut" in request.packages + assert "luci" in request.packages + assert "vim" in request.packages + + def test_auc_migration_snapshot(self): + """SNAPSHOT should also migrate auc to owut""" + request = PrepareRequest( + version="24.10-SNAPSHOT", + target="ath79/generic", + profile="test", + packages=["auc"], + ) + + apply_package_changes(request) + + assert "auc" not in request.packages + assert "owut" in request.packages + + def test_language_pack_migration(self): + """24.10 should migrate luci-i18n-opkg-* to luci-i18n-package-manager-*""" + request = PrepareRequest( + version="24.10.0", + target="ath79/generic", + profile="test", + packages=["luci-i18n-opkg-en", "luci-i18n-opkg-de"], + ) + + apply_package_changes(request) + + assert "luci-i18n-opkg-en" not in request.packages + assert "luci-i18n-opkg-de" not in request.packages + assert "luci-i18n-package-manager-en" in request.packages + assert "luci-i18n-package-manager-de" in request.packages + + +class TestPackageChanges2512: + """Tests for 25.12 specific package changes""" + + def test_kirkwood_dsa_mv88e6xxx(self): + """25.12 kirkwood specific profiles should add kmod-dsa-mv88e6xxx""" + profiles = [ + "checkpoint_l-50", + "endian_4i-edge-200", + "linksys_e4200-v2", + "linksys_ea3500", + "linksys_ea4500", + ] + + for profile in profiles: + request = PrepareRequest( + version="25.12.0", + target="kirkwood/generic", + profile=profile, + packages=["luci"], + ) + + apply_package_changes(request) + + assert "kmod-dsa-mv88e6xxx" in request.packages + + def test_mvebu_cortexa9_dsa_mv88e6xxx(self): + """25.12 mvebu/cortexa9 specific profiles should add kmod-dsa-mv88e6xxx""" + profiles = [ + "cznic_turris-omnia", + "linksys_wrt1200ac", + "linksys_wrt3200acm", + ] + + for profile in profiles: + request = PrepareRequest( + version="25.12.0", + target="mvebu/cortexa9", + profile=profile, + packages=["luci"], + ) + + apply_package_changes(request) + + assert "kmod-dsa-mv88e6xxx" in request.packages + + def test_lantiq_xrx200_phy22f_firmware(self): + """25.12 lantiq/xrx200 specific profiles should add phy22f firmware""" + profiles = [ + "arcadyan_arv7519rw22", + "arcadyan_vgv7510kw22-brn", + "avm_fritz7412", + ] + + for profile in profiles: + request = PrepareRequest( + version="25.12.0", + target="lantiq/xrx200", + profile=profile, + packages=["luci"], + ) + + apply_package_changes(request) + + assert "xrx200-rev1.1-phy22f-firmware" in request.packages + assert "xrx200-rev1.2-phy22f-firmware" in request.packages + + def test_lantiq_xrx200_phy11g_firmware(self): + """25.12 lantiq/xrx200 specific profiles should add phy11g firmware""" + profiles = [ + "tplink_vr200", + "avm_fritz7490", + "bt_homehub-v5a", + ] + + for profile in profiles: + request = PrepareRequest( + version="25.12.0", + target="lantiq/xrx200", + profile=profile, + packages=["luci"], + ) + + apply_package_changes(request) + + assert "xrx200-rev1.1-phy11g-firmware" in request.packages + assert "xrx200-rev1.2-phy11g-firmware" in request.packages + + def test_bcm53xx_hci_uart(self): + """25.12 bcm53xx/generic meraki_mr32 should add kmod-hci-uart""" + request = PrepareRequest( + version="25.12.0", + target="bcm53xx/generic", + profile="meraki_mr32", + packages=["luci"], + ) + + apply_package_changes(request) + + assert "kmod-hci-uart" in request.packages + + def test_ipq40xx_hci_uart(self): + """25.12 ipq40xx/generic linksys_whw03 should add kmod-hci-uart""" + request = PrepareRequest( + version="25.12.0", + target="ipq40xx/generic", + profile="linksys_whw03", + packages=["luci"], + ) + + apply_package_changes(request) + + assert "kmod-hci-uart" in request.packages + + +class TestPackageChangesGeneric: + """Tests for generic package changes logic""" + + def test_add_if_missing_does_not_duplicate(self): + """Package should not be added if already present""" + request = PrepareRequest( + version="23.05.0", + target="mediatek/mt7622", + profile="test", + packages=["luci", "kmod-mt7622-firmware"], # Already present + ) + + original_count = len(request.packages) + apply_package_changes(request) + + # Should not add duplicate + assert request.packages.count("kmod-mt7622-firmware") == 1 + assert len(request.packages) == original_count + + def test_version_prefix_matching(self): + """Version matching should work with prefixes""" + # 23.05.5 should match 23.05 + request = PrepareRequest( + version="23.05.5", + target="mediatek/mt7622", + profile="test", + packages=["luci"], + ) + + apply_package_changes(request) + assert "kmod-mt7622-firmware" in request.packages + + def test_no_changes_for_other_targets(self): + """No changes should be applied for unrelated targets""" + request = PrepareRequest( + version="23.05.0", + target="x86/64", # No special handling + profile="generic", + packages=["luci"], + ) + + original_packages = request.packages.copy() + apply_package_changes(request) + + assert request.packages == original_packages diff --git a/asu-prepare/tests/test_package_resolution.py b/asu-prepare/tests/test_package_resolution.py new file mode 100644 index 00000000..58d91b91 --- /dev/null +++ b/asu-prepare/tests/test_package_resolution.py @@ -0,0 +1,142 @@ +"""Tests for package resolution logic""" + +import pytest +from prepare_request import PrepareRequest +from package_resolution import PackageResolver, PackageChange + + +class TestPackageResolver: + """Tests for the PackageResolver class""" + + def test_resolver_basic_resolution(self): + """Resolver should handle basic package resolution""" + resolver = PackageResolver() + request = PrepareRequest( + version="23.05.5", + target="ath79/generic", + profile="test", + packages=["luci", "vim"], + ) + + final_packages, changes = resolver.resolve(request) + + assert isinstance(final_packages, list) + assert isinstance(changes, list) + assert "luci" in final_packages + assert "vim" in final_packages + + def test_resolver_no_changes(self): + """Resolver should return empty changes when nothing changes""" + resolver = PackageResolver() + request = PrepareRequest( + version="23.05.5", + target="ath79/generic", + profile="test", + packages=["luci"], + ) + + final_packages, changes = resolver.resolve(request) + + # For basic packages with no special handling, no changes expected + assert len(changes) == 0 + assert final_packages == ["luci"] + + def test_resolver_auc_migration(self): + """Resolver should migrate auc to owut in 24.10""" + resolver = PackageResolver() + request = PrepareRequest( + version="24.10.0", + target="ath79/generic", + profile="test", + packages=["luci", "auc"], + ) + + final_packages, changes = resolver.resolve(request) + + # auc should be removed + assert "auc" not in final_packages + # owut should be added + assert "owut" in final_packages + # Should track the migration + assert len(changes) >= 1 + migration = next((c for c in changes if c.type == "migration"), None) + assert migration is not None + assert migration.from_package == "auc" + assert migration.to_package == "owut" + + def test_resolver_language_pack_migration(self): + """Resolver should migrate language packs in 24.10""" + resolver = PackageResolver() + request = PrepareRequest( + version="24.10.0", + target="ath79/generic", + profile="test", + packages=["luci", "luci-i18n-opkg-en"], + ) + + final_packages, changes = resolver.resolve(request) + + # Old language pack should be replaced + assert "luci-i18n-opkg-en" not in final_packages + # New language pack should be present + assert "luci-i18n-package-manager-en" in final_packages + + def test_resolver_multiple_language_packs(self): + """Resolver should migrate multiple language packs""" + resolver = PackageResolver() + request = PrepareRequest( + version="24.10.0", + target="ath79/generic", + profile="test", + packages=["luci-i18n-opkg-en", "luci-i18n-opkg-de", "luci-i18n-opkg-fr"], + ) + + final_packages, changes = resolver.resolve(request) + + # All old language packs should be replaced + assert "luci-i18n-opkg-en" not in final_packages + assert "luci-i18n-opkg-de" not in final_packages + assert "luci-i18n-opkg-fr" not in final_packages + # New ones should be present + assert "luci-i18n-package-manager-en" in final_packages + assert "luci-i18n-package-manager-de" in final_packages + assert "luci-i18n-package-manager-fr" in final_packages + + +class TestPackageChange: + """Tests for the PackageChange class""" + + def test_package_change_to_dict(self): + """PackageChange should convert to dictionary""" + change = PackageChange( + change_type="migration", + action="replace", + from_package="auc", + to_package="owut", + reason="Package renamed", + automatic=True, + ) + + result = change.to_dict() + + assert result["type"] == "migration" + assert result["action"] == "replace" + assert result["from_package"] == "auc" + assert result["to_package"] == "owut" + assert result["reason"] == "Package renamed" + assert result["automatic"] is True + + def test_package_change_optional_fields(self): + """PackageChange should handle optional fields""" + change = PackageChange( + change_type="addition", + action="add", + package="vim", + reason="User requested", + ) + + result = change.to_dict() + + assert "package" in result + assert "from_package" not in result + assert "to_package" not in result diff --git a/asu/build.py b/asu/build.py index 7cb55b38..fb74845b 100644 --- a/asu/build.py +++ b/asu/build.py @@ -34,13 +34,16 @@ log = logging.getLogger("rq.worker") -def _build(build_request: BuildRequest, job=None): +def _build(build_request: BuildRequest, job=None, skip_package_resolution=False): """Build image request and setup ImageBuilders automatically The `request` dict contains properties of the requested image. Args: - request (dict): Contains all properties of requested image + build_request (BuildRequest): Contains all properties of requested image + job: RQ job instance + skip_package_resolution (bool): If True, skip package resolution. + Used when building from a prepared request. """ build_start: float = perf_counter() @@ -221,7 +224,9 @@ def _build(build_request: BuildRequest, job=None): .split() ) - apply_package_changes(build_request) + # Only apply package changes if not already prepared + if not skip_package_resolution: + apply_package_changes(build_request) build_cmd_packages = build_request.packages @@ -433,9 +438,17 @@ def _build(build_request: BuildRequest, job=None): return json_content -def build(build_request: BuildRequest, job=None): +def build(build_request: BuildRequest, job=None, skip_package_resolution=False): + """ + Build a firmware image. + + Args: + build_request: The build request parameters + job: RQ job instance + skip_package_resolution: If True, skip package resolution + """ try: - result = _build(build_request, job) + result = _build(build_request, job, skip_package_resolution) except Exception: # Log all build errors, including internal server errors. add_build_event("failures") diff --git a/asu/build_request.py b/asu/build_request.py index 1fc18437..01d7f837 100644 --- a/asu/build_request.py +++ b/asu/build_request.py @@ -31,6 +31,19 @@ class BuildRequest(BaseModel): pattern=STRING_PATTERN, ), ] + from_version: Annotated[ + str | None, + Field( + examples=["23.05.0"], + description=""" + The version the device is currently running. This allows the + server to apply appropriate package migrations when upgrading + from an older version. If not provided, package changes are + applied based on the target version only. + """.strip(), + pattern=STRING_PATTERN, + ), + ] = None version_code: Annotated[ str, Field( diff --git a/asu/config.py b/asu/config.py index 92c57041..a0141b9c 100644 --- a/asu/config.py +++ b/asu/config.py @@ -66,6 +66,7 @@ class Settings(BaseSettings): public_path: Path = Path.cwd() / "public" redis_url: str = "redis://localhost:6379" upstream_url: str = "https://downloads.openwrt.org" + prepare_service_url: str = "http://asu-prepare:8001" allow_defaults: bool = False async_queue: bool = True branches_file: Union[str, Path, None] = None diff --git a/asu/package_resolution.py b/asu/package_resolution.py new file mode 100644 index 00000000..c156043c --- /dev/null +++ b/asu/package_resolution.py @@ -0,0 +1,188 @@ +""" +Package resolution logic + +This module handles: +- Applying package changes based on version/target/profile +- Tracking what changes were made +- Calculating final package lists for prepare endpoint +""" + +import logging +from typing import Optional +from copy import deepcopy + +from asu.build_request import BuildRequest +from asu.package_changes import apply_package_changes + +log = logging.getLogger("rq.worker") + + +class PackageChange: + """Represents a single package change""" + + def __init__( + self, + change_type: str, # migration, addition, removal + action: str, # replace, add, remove + package: Optional[str] = None, + from_package: Optional[str] = None, + to_package: Optional[str] = None, + reason: str = "", + automatic: bool = True, + ): + self.type = change_type + self.action = action + self.package = package + self.from_package = from_package + self.to_package = to_package + self.reason = reason + self.automatic = automatic + + def to_dict(self): + """Convert to dictionary for JSON serialization""" + result = { + "type": self.type, + "action": self.action, + "reason": self.reason, + "automatic": self.automatic, + } + + if self.package: + result["package"] = self.package + if self.from_package: + result["from_package"] = self.from_package + if self.to_package: + result["to_package"] = self.to_package + + return result + + +class PackageResolver: + """Resolves packages for a build request""" + + def __init__(self): + self.changes: list[PackageChange] = [] + + def resolve( + self, build_request: BuildRequest + ) -> tuple[list[str], list[PackageChange]]: + """ + Resolve packages for a build request. + + This method applies package changes based on version/target/profile + and tracks what was changed. + + Args: + build_request: The build request to resolve packages for + + Returns: + Tuple of (final_packages, changes_applied) + """ + self.changes = [] + + # Make a deep copy to track changes + original_packages = build_request.packages.copy() + + # Apply package changes (existing logic from package_changes.py) + apply_package_changes(build_request) + + # Track what changed + self._track_changes(original_packages, build_request.packages, build_request) + + return build_request.packages, self.changes + + def _track_changes( + self, + original: list[str], + modified: list[str], + build_request: BuildRequest, + ): + """Track what changes were made""" + original_set = set(original) + modified_set = set(modified) + + added = modified_set - original_set + removed = original_set - modified_set + + # Detect migrations (package renames) + # Check for known migrations from package_changes.py + for removed_pkg in list(removed): + # Check if this is a known migration + migration = self._find_migration( + removed_pkg, build_request.version, build_request + ) + if migration and migration in added: + self.changes.append( + PackageChange( + change_type="migration", + action="replace", + from_package=removed_pkg, + to_package=migration, + reason=f"Package renamed in {build_request.version}", + automatic=True, + ) + ) + removed.remove(removed_pkg) + added.remove(migration) + + # Remaining removals + for pkg in removed: + self.changes.append( + PackageChange( + change_type="removal", + action="remove", + package=pkg, + reason="Package no longer available or needed", + automatic=True, + ) + ) + + # Remaining additions + for pkg in added: + reason = self._get_addition_reason(pkg, build_request) + self.changes.append( + PackageChange( + change_type="addition", + action="add", + package=pkg, + reason=reason, + automatic=True, + ) + ) + + def _find_migration( + self, package: str, version: str, build_request: BuildRequest + ) -> Optional[str]: + """Find if package was migrated to another name""" + # Check for known migrations + if version.startswith("24.10"): + if package == "auc": + return "owut" + + # Check for language pack renames + from asu.package_changes import language_packs + + for lang_version, packages in language_packs.items(): + if version >= lang_version: + for old, new in packages.items(): + if package.startswith(old): + lang = package.replace(old, "") + return f"{new}{lang}" + + return None + + def _get_addition_reason(self, package: str, build_request: BuildRequest) -> str: + """Determine why package was added""" + if package.startswith("kmod-"): + # Check if it's a hardware-specific module + if build_request.target: + return f"Required kernel module for {build_request.target}" + return "Required kernel module" + + if package.startswith("luci-i18n-"): + return "Language pack" + + if package.startswith("xrx200-"): + return "Required PHY firmware" + + return "Required for this version/target/profile" diff --git a/asu/routers/api.py b/asu/routers/api.py index b26c85de..534e848c 100644 --- a/asu/routers/api.py +++ b/asu/routers/api.py @@ -1,6 +1,7 @@ import logging from typing import Union +import httpx from fastapi import APIRouter, Header, Request from fastapi.responses import RedirectResponse, Response from rq.job import Job @@ -208,7 +209,20 @@ def api_v1_build_post( response: Response, request: Request, user_agent: str = Header(None), + skip_package_resolution: bool = False, ): + """ + Build a firmware image. + + Args: + build_request: The build request parameters + skip_package_resolution: If True, skip package resolution (used when + building from a prepared request). Default: False + + If skip_package_resolution=True, assumes packages are already resolved + and skips package changes/migrations. This should be used when calling + /build after /build/prepare. + """ # Sanitize the profile in case the client did not (bug in older LuCI app). build_request.profile = build_request.profile.replace(",", "_") @@ -224,7 +238,7 @@ def api_v1_build_post( if build_request.client: client = build_request.client - elif user_agent.startswith("auc"): + elif user_agent and user_agent.startswith("auc"): client = user_agent.replace(" (", "/").replace(")", "") else: client = "unknown/0" @@ -237,10 +251,13 @@ def api_v1_build_post( if job is None: add_build_event("cache-misses") - content, status = validate_request(request.app, build_request) - if content: - response.status_code = status - return content + # Only validate if not already prepared + # Prepared requests have already been validated + if not skip_package_resolution: + content, status = validate_request(request.app, build_request) + if content: + response.status_code = status + return content job_queue_length = len(get_queue()) if job_queue_length > settings.max_pending_jobs: @@ -254,6 +271,7 @@ def api_v1_build_post( job = get_queue().enqueue( build, build_request, + skip_package_resolution=skip_package_resolution, job_id=request_hash, result_ttl=result_ttl, failure_ttl=failure_ttl, diff --git a/misc/Caddyfile b/misc/Caddyfile index b74cb365..bfe64cb6 100644 --- a/misc/Caddyfile +++ b/misc/Caddyfile @@ -1,27 +1,70 @@ -#{ -# auto_https disable_redirects -# preferred_chains { -# root_common_name "ISRG Root X1" -# } -#} -# -#sysupgrade.openwrt.org sysupgrade.openwrt.org:80 { -# root * /path/to/asu/ -# file_server /json/ -# file_server /store/ -# header Access-Control-Allow-Methods "POST, GET, OPTIONS" -# header Access-Control-Allow-Headers "*" -# header Access-Control-Allow-Origin "*" -# reverse_proxy * localhost:8000 -#} +# Caddy configuration for ASU Microservices +# Routes requests to appropriate service + +{ + # Global options + auto_https off +} :80 { - root * /site/ - file_server /json/ - file_server /store/ - header Access-Control-Allow-Methods "POST, GET, OPTIONS" - header Access-Control-Allow-Headers "*" - header Access-Control-Allow-Origin "*" - reverse_proxy * server:8000 - reverse_proxy /stats grafana:3000 + # Prepare endpoint → prepare service + handle /api/v1/prepare* { + reverse_proxy prepare:8001 { + # Timeouts for prepare service (should be fast) + transport http { + dial_timeout 10s + response_header_timeout 30s + } + } + } + + # Build endpoint → build service + handle /api/v1/build* { + reverse_proxy build:8000 { + # Longer timeouts for build service (can take time) + transport http { + dial_timeout 30s + response_header_timeout 600s + } + } + } + + # Stats and other endpoints → build service + handle /api/v1/* { + reverse_proxy build:8000 + } + + # JSON endpoints → build service (for now) + # Could be moved to prepare service if needed + handle /json/* { + reverse_proxy build:8000 + } + + # Static files → build service + handle /static/* { + reverse_proxy build:8000 + } + + # Store (firmware downloads) → build service + handle /store/* { + reverse_proxy build:8000 + } + + # Health checks + handle /health/prepare { + reverse_proxy prepare:8001 { + rewrite /health + } + } + + handle /health/build { + reverse_proxy build:8000 { + rewrite /health + } + } + + # Root and other paths → build service (catch-all) + handle { + reverse_proxy build:8000 + } } diff --git a/podman-compose.yml b/podman-compose.yml index 3c467125..107f9c3c 100644 --- a/podman-compose.yml +++ b/podman-compose.yml @@ -1,81 +1,110 @@ +# Podman Compose for Microservices Architecture +# Completely independent prepare and build services with NO shared code + +version: '3' + services: - server: - image: "docker.io/openwrt/asu:latest" + # Lightweight prepare service (SEPARATE CODEBASE in asu-prepare/) + # Only handles package resolution and validation + # No Redis, No Podman, No shared code with build service + prepare: build: - context: . + context: ./asu-prepare dockerfile: Containerfile - restart: unless-stopped - command: uvicorn --host 0.0.0.0 asu.main:app - env_file: .env + container_name: asu-prepare + ports: + - "127.0.0.1:8001:8001" environment: - REDIS_URL: "redis://redis:6379/0" - volumes: - - $PUBLIC_PATH/store:$PUBLIC_PATH/store:ro + - UPSTREAM_URL=${UPSTREAM_URL:-https://downloads.openwrt.org} + restart: unless-stopped + networks: + - asu-network + # Minimal resources for lightweight service + deploy: + resources: + limits: + cpus: '1' + memory: 512M + + # Heavy build service (SEPARATE CODEBASE in asu/) + # Handles actual firmware building + # Calls prepare service via HTTP + # Requires Redis, RQ, Podman access + build: + build: + context: . + dockerfile: Containerfile + container_name: asu-build ports: - "127.0.0.1:8000:8000" + environment: + - PUBLIC_PATH=/var/lib/asu/public + - REDIS_URL=redis://redis:6379 + - UPSTREAM_URL=${UPSTREAM_URL:-https://downloads.openwrt.org} + - PREPARE_SERVICE_URL=http://asu-prepare:8001 + - ALLOW_DEFAULTS=${ALLOW_DEFAULTS:-0} + volumes: + - ${PUBLIC_PATH:-./public}:/var/lib/asu/public + restart: unless-stopped depends_on: - redis + - prepare + networks: + - asu-network + # More resources for heavy service + deploy: + resources: + limits: + cpus: '4' + memory: 4G + + # Redis for build service only + # Prepare service doesn't need it + redis: + image: redis/redis-stack-server:latest + container_name: asu-redis + ports: + - "6379:6379" + # volumes: + # - redis-data:/data + restart: unless-stopped + networks: + - asu-network + # Worker for build service + # Can scale independently worker: - image: "docker.io/openwrt/asu:latest" build: context: . dockerfile: Containerfile - restart: unless-stopped - command: rqworker --logging_level INFO - env_file: .env + container_name: asu-worker + command: ["rq", "worker"] environment: - REDIS_URL: "redis://redis:6379/0" + - PUBLIC_PATH=/var/lib/asu/public + - REDIS_URL=redis://redis:6379 + - UPSTREAM_URL=${UPSTREAM_URL:-https://downloads.openwrt.org} + - PREPARE_SERVICE_URL=http://asu-prepare:8001 volumes: - - $PUBLIC_PATH:$PUBLIC_PATH:rw + - ${PUBLIC_PATH:-./public}:/var/lib/asu/public - $CONTAINER_SOCKET_PATH:$CONTAINER_SOCKET_PATH:rw + restart: unless-stopped depends_on: - redis + - build + - prepare + networks: + - asu-network + deploy: + mode: replicated + replicas: 1 # Can scale workers independently + resources: + limits: + cpus: '2' + memory: 2G - redis: - image: "docker.io/redis/redis-stack-server" - restart: unless-stopped - volumes: - - ./redis-data:/data/:rw - ports: - - "127.0.0.1:6379:6379" - - # Optionally add more workers - # worker2: - # image: "docker.io/openwrt/asu:latest" - # restart: unless-stopped - # command: rqworker --logging_level INFO - # env_file: .env - # environment: - # REDIS_URL: "redis://redis:6379/0" - # volumes: - # - $PUBLIC_PATH:$PUBLIC_PATH:rw - # - $CONTAINER_SOCKET_PATH:$CONTAINER_SOCKET_PATH:rw - # depends_on: - # - redis - # - # Optionally add a Squid cache container when using `SQUID_CACHE` - # squid: - # image: "docker.io/ubuntu/squid:latest" - # restart: unless-stopped - # ports: - # - "127.0.0.1:3128:3128" - # volumes: - # - ".squid.conf:/etc/squid/conf.d/snippet.conf:ro" - # - "./squid-data/:/var/spool/squid/:rw" +networks: + asu-network: + driver: bridge - # Optionally add a Grafana container when using `SERVER_STATS` - # grafana: - # image: docker.io/grafana/grafana-oss - # container_name: grafana - # restart: unless-stopped - # ports: - # - "127.0.0.1:3000:3000" - # depends_on: - # - redis - # environment: - # GF_SERVER_DOMAIN: sysupgrade.openwrt.org - # GF_SERVER_ROOT_URL: https://sysupgrade.openwrt.org/stats/ - # GF_SERVER_SERVE_FROM_SUB_PATH: "true" - # volumes: - # - ./grafana-data:/var/lib/grafana +volumes: + redis-data: diff --git a/tests/README_INTEGRATION.md b/tests/README_INTEGRATION.md new file mode 100644 index 00000000..5a92c290 --- /dev/null +++ b/tests/README_INTEGRATION.md @@ -0,0 +1,182 @@ +# Integration Tests for ASU Prepare + Build Workflow + +This directory contains integration tests and tools for testing the two-step prepare/build workflow. + +## Prerequisites + +1. **Running Services**: + - Prepare service at `http://localhost:8001` + - Build service at `http://localhost:8000` + +2. **Python Dependencies**: + ```bash + pip install requests pytest + ``` + +## Test Files + +### Integration Test Suite + +**`test_integration_prepare_build.py`** - Comprehensive integration tests + +Run all integration tests: +```bash +pytest tests/test_integration_prepare_build.py -v +``` + +Run only if services are available: +```bash +pytest tests/test_integration_prepare_build.py -v -m integration +``` + +### Interactive Client + +**`uclient.py`** - Interactive command-line client for testing + +Usage: +```bash +# Two-step workflow (default - shows changes before building) +python tests/uclient.py tests/configs/basic_build.json + +# Prepare-only mode (see changes without building) +python tests/uclient.py tests/configs/migration_auc_to_owut.json --prepare-only + +# Legacy single-step workflow (bypasses prepare) +python tests/uclient.py tests/configs/basic_build.json --legacy +``` + +## Test Configurations + +The `configs/` directory contains test scenarios: + +### Basic Tests + +- **`basic_build.json`** - Simple build with no migrations +- **`migration_auc_to_owut.json`** - Tests auc → owut migration (24.10) +- **`language_pack_migration.json`** - Tests language pack migration (24.10) +- **`hardware_specific.json`** - Tests hardware-specific package addition (25.12) + +### Running Examples + +1. **See package changes without building**: + ```bash + python tests/uclient.py tests/configs/migration_auc_to_owut.json --prepare-only + ``` + + Output: + ``` + 📝 Preparing build request... + ✅ Preparation complete! + 📦 Package changes (1): + 🔄 Migration: auc → owut + Reason: Package renamed in 24.10.0 + ``` + +2. **Full build with migration**: + ```bash + python tests/uclient.py tests/configs/migration_auc_to_owut.json + ``` + + You'll see the changes and be asked to confirm before building. + +3. **Hardware-specific packages**: + ```bash + python tests/uclient.py tests/configs/hardware_specific.json --prepare-only + ``` + + Shows that `kmod-dsa-mv88e6xxx` will be added automatically. + +## Integration Test Scenarios + +The test suite covers: + +### Prepare Endpoint Tests +- ✅ Basic prepare requests +- ✅ Package migrations (auc → owut) +- ✅ Language pack migrations +- ✅ Profile name sanitization +- ✅ Hardware-specific package additions + +### Complete Workflow Tests +- ✅ Prepare → Build workflow +- ✅ Migration + Build +- ✅ Changes shown before building +- ✅ Request hash consistency + +### Service Independence Tests +- ✅ Prepare works without build service +- ✅ Services report different capabilities +- ✅ Prepare has no build endpoint + +## Environment Variables + +Override default URLs: +```bash +export PREPARE_SERVICE_URL=http://prepare-service:8001 +export BUILD_SERVICE_URL=http://build-service:8000 + +pytest tests/test_integration_prepare_build.py -v +``` + +## Continuous Integration + +For CI/CD pipelines, skip integration tests if services aren't running: + +```bash +# Only run integration tests if services are available +pytest tests/test_integration_prepare_build.py -v -m integration +``` + +Tests will automatically skip if services aren't reachable. + +## Creating New Test Configs + +Create a JSON file in `configs/`: + +```json +{ + "url": "http://localhost:8000", + "prepare_url": "http://localhost:8001", + "version": "24.10.0", + "target": "ath79/generic", + "profile": "my-device", + "packages": ["luci", "vim"], + "diff_packages": false, + "defaults": "...", + "rootfs_size_mb": 512 +} +``` + +Required fields: +- `url` - Build service URL +- `version` - OpenWrt version +- `target` - Target platform +- `profile` - Device profile + +Optional fields: +- `prepare_url` - Prepare service URL (default: http://localhost:8001) +- `packages` - Package list +- `from_version` - Version upgrading from (for migrations) +- `diff_packages` - Absolute vs additional packages +- `defaults` - Default configuration +- `rootfs_size_mb` - Custom rootfs size +- `repositories` - Custom package repositories +- `repository_keys` - Repository signing keys + +## Debugging + +Enable verbose output: +```bash +python tests/uclient.py tests/configs/basic_build.json --prepare-only 2>&1 | tee debug.log +``` + +Check service health: +```bash +curl http://localhost:8001/health +curl http://localhost:8000/health +``` + +View service status: +```bash +curl http://localhost:8001/api/v1/status | jq +``` diff --git a/tests/configs/basic_build.json b/tests/configs/basic_build.json new file mode 100644 index 00000000..c34f0800 --- /dev/null +++ b/tests/configs/basic_build.json @@ -0,0 +1,8 @@ +{ + "url": "http://localhost:8000", + "prepare_url": "http://localhost:8001", + "version": "23.05.5", + "target": "x86/64", + "profile": "generic", + "packages": ["luci"] +} diff --git a/tests/configs/hardware_specific.json b/tests/configs/hardware_specific.json new file mode 100644 index 00000000..75028256 --- /dev/null +++ b/tests/configs/hardware_specific.json @@ -0,0 +1,9 @@ +{ + "url": "http://localhost:8000", + "prepare_url": "http://localhost:8001", + "version": "25.12.0", + "target": "kirkwood/generic", + "profile": "checkpoint_l-50", + "packages": ["luci"], + "comment": "This tests hardware-specific package addition: kmod-dsa-mv88e6xxx should be added" +} diff --git a/tests/configs/language_pack_migration.json b/tests/configs/language_pack_migration.json new file mode 100644 index 00000000..d2b5e738 --- /dev/null +++ b/tests/configs/language_pack_migration.json @@ -0,0 +1,9 @@ +{ + "url": "http://localhost:8000", + "prepare_url": "http://localhost:8001", + "version": "24.10.0", + "target": "ath79/generic", + "profile": "tplink_tl-wdr4300-v1", + "packages": ["luci", "luci-i18n-opkg-en", "luci-i18n-opkg-de"], + "comment": "This tests language pack migration in 24.10: luci-i18n-opkg-* -> luci-i18n-package-manager-*" +} diff --git a/tests/configs/migration_auc_to_owut.json b/tests/configs/migration_auc_to_owut.json new file mode 100644 index 00000000..8489863d --- /dev/null +++ b/tests/configs/migration_auc_to_owut.json @@ -0,0 +1,9 @@ +{ + "url": "http://localhost:8000", + "prepare_url": "http://localhost:8001", + "version": "24.10.0", + "target": "x86/64", + "profile": "generic", + "packages": ["luci", "auc"], + "comment": "This tests auc -> owut migration in 24.10" +} diff --git a/tests/regressions/asu.js b/tests/regressions/asu.js new file mode 100644 index 00000000..c7d1981b --- /dev/null +++ b/tests/regressions/asu.js @@ -0,0 +1,44 @@ +const fs = require("fs"); + +let rawdata = fs.readFileSync("tests/regressions/regression-gh1176.json"); +let request_json = JSON.parse(rawdata); +let url = "https://sysupgrade.openwrt.org"; + +async function do_request(data) { + request = { + packages: data.packages, + target: data.target, + version: data.version, + profile: data.profile, + }; + + return fetch(url + "/api/v1/build", { + method: "POST", + mode: "cors", + cache: "no-cache", + credentials: "same-origin", + headers: { + "Content-Type": "application/json", + }, + redirect: "follow", + referrerPolicy: "no-referrer", + body: JSON.stringify(request), + }).then((response) => response.json()); +} + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function run() { + while (true) { + const foo = await do_request(request_json); + console.log(foo); + if (foo.status != 202) { + return; + } + await sleep(1000); + } +} + +run(); diff --git a/tests/regressions/regression-gh1003.json b/tests/regressions/regression-gh1003.json new file mode 100644 index 00000000..2a3c23e6 --- /dev/null +++ b/tests/regressions/regression-gh1003.json @@ -0,0 +1,175 @@ +{ + "url": "https://sysupgrade.openwrt.org", + "revision": "r24012-d8dd03c46f", + "advanced_mode": "0", + "sha256_unsigned": "", + "branch": "23.05", + "efi": null, + "profile": "avm,fritzbox-7530", + "target": "ipq40xx/generic", + "version": "23.05.5", + "packages": [ + "ath10k-board-qca4019", + "ath10k-firmware-qca4019-ct", + "base-files", + "busybox", + "ca-bundle", + "cgi-io", + "curl", + "ddns-scripts-noip", + "dnsmasq", + "dropbear", + "firewall4", + "fritz-caldata", + "fritz-tffs-nand", + "fstools", + "fwtool", + "getrandom", + "hostapd-common", + "iw", + "iwinfo", + "jansson", + "jshn", + "jsonfilter", + "kernel", + "kmod-ath", + "kmod-ath10k-ct", + "kmod-cfg80211", + "kmod-crypto-acompress", + "kmod-crypto-aead", + "kmod-crypto-ccm", + "kmod-crypto-cmac", + "kmod-crypto-crc32c", + "kmod-crypto-ctr", + "kmod-crypto-gcm", + "kmod-crypto-gf128", + "kmod-crypto-ghash", + "kmod-crypto-hash", + "kmod-crypto-hmac", + "kmod-crypto-manager", + "kmod-crypto-null", + "kmod-crypto-rng", + "kmod-crypto-seqiv", + "kmod-crypto-sha512", + "kmod-gpio-button-hotplug", + "kmod-hwmon-core", + "kmod-leds-gpio", + "kmod-lib-crc-ccitt", + "kmod-lib-crc32c", + "kmod-lib-lzo", + "kmod-mac80211", + "kmod-nf-conntrack", + "kmod-nf-conntrack6", + "kmod-nf-flow", + "kmod-nf-log", + "kmod-nf-log6", + "kmod-nf-nat", + "kmod-nf-reject", + "kmod-nf-reject6", + "kmod-nfnetlink", + "kmod-nft-core", + "kmod-nft-fib", + "kmod-nft-nat", + "kmod-nft-offload", + "kmod-nls-base", + "kmod-ppp", + "kmod-pppoe", + "kmod-pppox", + "kmod-slhc", + "kmod-usb-core", + "kmod-usb-dwc3", + "kmod-usb-dwc3-qcom", + "kmod-usb-net-rndis", + "kmod-usb-xhci-hcd", + "kmod-usb3", + "libblobmsg-json", + "libc", + "libcurl", + "libiwinfo", + "libiwinfo-data", + "libjson-c", + "libjson-script", + "liblua", + "liblucihttp", + "liblucihttp-ucode", + "libmbedtls", + "libmnl", + "libnftnl", + "libnl-tiny", + "libubox", + "libubus", + "libuci", + "libuclient", + "libucode", + "libustream-mbedtls", + "logd", + "ltq-vdsl-vr11-app", + "lua", + "luci", + "luci-app-attendedsysupgrade", + "luci-app-ddns", + "luci-app-firewall", + "luci-app-openvpn", + "luci-app-opkg", + "luci-app-wol", + "luci-base", + "luci-light", + "luci-mod-admin-full", + "luci-mod-network", + "luci-mod-status", + "luci-mod-system", + "luci-proto-ipv6", + "luci-proto-ppp", + "luci-proto-wireguard", + "luci-ssl", + "luci-theme-bootstrap", + "mtd", + "netifd", + "nftables-json", + "odhcp6c", + "odhcpd-ipv6only", + "openvpn-openssl", + "openwrt-keyring", + "opkg", + "ppp", + "ppp-mod-pppoe", + "procd", + "procd-seccomp", + "procd-ujail", + "px5g-mbedtls", + "qrencode", + "rpcd", + "rpcd-mod-file", + "rpcd-mod-iwinfo", + "rpcd-mod-luci", + "rpcd-mod-rrdns", + "rpcd-mod-ucode", + "ubi-utils", + "uboot-envtools", + "ubox", + "ubus", + "ubusd", + "uci", + "uclient-fetch", + "ucode", + "ucode-mod-fs", + "ucode-mod-html", + "ucode-mod-math", + "ucode-mod-nl80211", + "ucode-mod-rtnl", + "ucode-mod-ubus", + "ucode-mod-uci", + "ucode-mod-uloop", + "uhttpd", + "uhttpd-mod-ubus", + "urandom-seed", + "urngd", + "usign", + "wireguard-tools", + "wireless-regdb", + "wpad-basic-mbedtls" + ], + "diff_packages": true, + "filesystem": "squashfs", + "client": "luci/git-23.339.51123-138595a" +} diff --git a/tests/regressions/regression-gh1179.json b/tests/regressions/regression-gh1179.json new file mode 100644 index 00000000..b78db00c --- /dev/null +++ b/tests/regressions/regression-gh1179.json @@ -0,0 +1,103 @@ +{ + "url": "https://sysupgrade.openwrt.org", + "branch": "24.10", + "revision": "r28304-6dacba30a7", + "efi": null, + "advanced_mode": "1", + "request_hash": "", + "sha256_unsigned": "", + "client": "luci/undefined", + "packages_versions": { + "kmod-usb-storage": "6.6.69-r1", + "libopenssl": "3.0.15-r1", + "libpam": "1.5.2-r1", + "libc": "1.2.5-r4", + "avahi-dbus-daemon": "0.8-r9", + "opkg": "2024.10.1638eccbb1-r1", + "wpad-basic-mbedtls": "2024.09.155ace39b0-r2", + "luci-i18n-base-ja": "25.021.3071836b6107", + "libpthread": "1.2.5-r4", + "libubus-lua": "2025.01.02afa57cce-r1", + "nano": "8.3-r1", + "kmod-tun": "6.6.69-r1", + "luci-i18n-attendedsysupgrade-ja": "25.021.3071836b6107", + "libtasn1": "4.19.0-r2", + "swconfig": "12", + "kmod-crypto-md5": "6.6.69-r1", + "kmod-lib-crc-ccitt": "6.6.69-r1", + "getrandom": "2024.04.2685f10530-r1", + "kmod-asn1-decoder": "6.6.69-r1", + "kmod-pppoe": "6.6.69-r1", + "libuuid": "2.40.2-r1", + "libcap": "2.69-r1", + "libavahi-client": "0.8-r9", + "procd-ujail": "2024.12.2242d39376-r1", + "base-files": "16496dacba30a7", + "libustream-mbedtls": "2024.07.2899bd3d2b-r1", + "ddns-scripts": "2.8.2-r52", + "firewall4": "2024.12.1818fc0ead-r1", + "uboot-envtools": "2024.07-r1", + "kmod-usb-ohci": "6.6.69-r1", + "dnsmasq": "2.90-r3", + "attr": "2.5.2-r3", + "kmod-crypto-sha256": "6.6.69-r1", + "kmod-crypto-acompress": "6.6.69-r1", + "ddns-scripts-noip": "2.8.2-r52", + "luci-app-ddns": "25.024.211134d6c9a4", + "kmod-usb2": "6.6.69-r1", + "libmbedtls": "3.6.2-r1", + "libgnutls": "3.8.5-r1", + "odhcp6c": "2024.09.25b6ae9ffa-r1", + "kmod-ath9k": "6.6.69.6.12.6-r1", + "uci": "2025.01.2016ff0bad-r1", + "lua": "5.1.5-r11", + "kmod-fs-ext4": "6.6.69-r1", + "luci-ssl": "25.021.3071836b6107", + "dropbear": "2024.86-r1", + "luci-i18n-package-manager-ja": "25.021.3071836b6107", + "kmod-nls-utf8": "6.6.69-r1", + "libgmp": "6.3.0-r1", + "mtd": "26", + "odhcpd-ipv6only": "2024.05.08a2988231-r1", + "kmod-owl-loader": "6.6.69.6.12.6-r1", + "samba4-server": "4.18.8-r1", + "urandom-seed": "3", + "ppp": "2.5.1-r1", + "kmod-gpio-button-hotplug": "6.6.69-r5", + "logd": "2024.04.2685f10530-r1", + "libreadline": "8.2-r2", + "luci-i18n-ddns-ja": "25.024.211134d6c9a4", + "kmod-dnsresolver": "6.6.69-r1", + "kmod-crypto-ecb": "6.6.69-r1", + "kmod-crypto-des": "6.6.69-r1", + "attendedsysupgrade-common": "9", + "kmod-ppp": "6.6.69-r1", + "luci-i18n-firewall-ja": "25.021.3071836b6107", + "luci-app-samba4": "25.024.211134d6c9a4", + "luci-proto-relay": "25.021.3071836b6107", + "liblua": "5.1.5-r11", + "ca-bundle": "20240203-r1", + "libuclient": "2024.10.2288ae8f20-r1", + "liblucihttp-lua": "2023.03.159b5b683f-r1", + "luci": "25.021.3071836b6107", + "owut": "2025.01.06e623a900-r1", + "kmod-usb-ledtrig-usbport": "6.6.69-r1", + "luci-i18n-openvpn-ja": "25.024.211134d6c9a4", + "kmod-lib-lzo": "6.6.69-r1", + "kmod-ath": "6.6.69.6.12.6-r1", + "kernel": "6.6.698a59efb5b60b2afd0fd0c33f40800cbf-r1", + "kmod-fs-cifs": "6.6.69-r1", + "libatomic": "13.3.0-r4", + "libtirpc": "1.3.4-r1", + "openvpn-mbedtls": "2.6.12-r1", + "kmod-scsi-core": "6.6.69-r1", + "urngd": "2023.11.0144365eb1-r1", + "ppp-mod-pppoe": "2.5.1-r1", + "luci-app-openvpn": "25.024.21113~4d6c9a4" + }, + "profile": "buffalo,wzr-hp-ag300h", + "target": "ath79/generic", + "version": "24.10.0-rc7", + "diff_packages": true, + "filesystem": "squashfs" +} diff --git a/tests/regressions/regression-gh1189.json b/tests/regressions/regression-gh1189.json new file mode 100644 index 00000000..ae39db5f --- /dev/null +++ b/tests/regressions/regression-gh1189.json @@ -0,0 +1,48 @@ +{ + "url": "https://sysupgrade.openwrt.org", + "branch": "24.10", + "revision": "r28417-daef29c75d", + "efi": null, + "advanced_mode": "0", + "request_hash": "", + "sha256_unsigned": "", + "client": "luci/25.027.83426170375e", + "packages_versions": { + "luci-proto-wireguard": "25.027.83426170375e", + "libc": "1.2.5-r4", + "luci-app-travelmate": "25.027.83426170375e", + "opkg": "2024.10.1638eccbb1-r1", + "wpad-basic-mbedtls": "2024.09.155ace39b0-r2", + "procd-ujail": "2024.12.2242d39376-r1", + "base-files": "1653daef29c75d", + "libustream-mbedtls": "2024.07.2899bd3d2b-r1", + "firewall4": "2024.12.1818fc0ead-r1", + "dnsmasq": "2.90-r4", + "odhcp6c": "2024.09.25b6ae9ffa-r1", + "uci": "2025.01.2016ff0bad-r1", + "dropbear": "2024.86-r1", + "mtd": "26", + "odhcpd-ipv6only": "2024.05.08a2988231-r1", + "travelmate": "2.1.3-r3", + "urandom-seed": "3", + "luci-mod-dashboard": "25.027.83426170375e", + "ppp": "2.5.1-r1", + "kmod-leds-gpio": "6.6.73-r1", + "kmod-gpio-button-hotplug": "6.6.73-r5", + "logd": "2024.04.2685f10530-r1", + "luci-app-attendedsysupgrade": "25.027.83426170375e", + "kmod-crypto-hw-eip93": "6.6.73-r1", + "luci-app-wifischedule": "25.027.83426170375e", + "kmod-mt7915-firmware": "6.6.73.2025.01.148e4f72b6-r1", + "ca-bundle": "20240203-r1", + "luci": "25.027.83426170375e", + "kernel": "6.6.733abe85def815b59c6c75ac1f92135cb6-r1", + "urngd": "2023.11.0144365eb1-r1", + "ppp-mod-pppoe": "2.5.1-r1" + }, + "profile": "totolink,x5000r", + "target": "ramips/mt7621", + "version": "24.10.0-rc7", + "diff_packages": true, + "filesystem": "squashfs" +} diff --git a/tests/regressions/regression-gh1200.json b/tests/regressions/regression-gh1200.json new file mode 100644 index 00000000..cee4a348 --- /dev/null +++ b/tests/regressions/regression-gh1200.json @@ -0,0 +1,153 @@ +{ + "url": "http://localhost:8000", + "revision": "r24106-10cc5fcd00", + "advanced_mode": "1", + "sha256_unsigned": "", + "branch": "23.05", + "efi": null, + "profile": "netgear,wndr3800", + "target": "ath79/generic", + "version": "24.10.0", + "defaults": "a", + "packages": [ + "auc", + "base-files", + "busybox", + "ca-bundle", + "cgi-io", + "collectd-mod-ethstat", + "collectd-mod-ipstatistics", + "collectd-mod-irq", + "collectd-mod-load", + "collectd-mod-ping", + "collectd-mod-powerdns", + "collectd-mod-sqm", + "collectd-mod-thermal", + "collectd-mod-wireless", + "curl", + "dnsmasq", + "dropbear", + "firewall", + "fstools", + "fwtool", + "getrandom", + "hostapd-common", + "ip6tables-nft", + "iptables-nft", + "iw", + "iwinfo", + "jshn", + "jsonfilter", + "kernel", + "kmod-ath", + "kmod-ath9k", + "kmod-ath9k-common", + "kmod-cfg80211", + "kmod-gpio-button-hotplug", + "kmod-ip6tables", + "kmod-ipt-conntrack", + "kmod-ipt-core", + "kmod-ipt-nat", + "kmod-ipt-offload", + "kmod-leds-reset", + "kmod-lib-crc-ccitt", + "kmod-mac80211", + "kmod-nf-conntrack", + "kmod-nf-conntrack6", + "kmod-nf-flow", + "kmod-nf-ipt", + "kmod-nf-ipt6", + "kmod-nf-nat", + "kmod-nf-reject", + "kmod-nf-reject6", + "kmod-owl-loader", + "kmod-ppp", + "kmod-pppoe", + "kmod-pppox", + "kmod-slhc", + "kmod-switch-rtl8366s", + "kmod-usb-ledtrig-usbport", + "kmod-usb-ohci", + "kmod-usb2", + "libblobmsg-json", + "libc", + "libip4tc", + "libip6tc", + "libiwinfo", + "libiwinfo-lua", + "libjson-c", + "libjson-script", + "liblua", + "liblucihttp", + "liblucihttp-lua", + "libnl-tiny", + "libubox", + "libubus", + "libubus-lua", + "libuci", + "libuclient", + "libustream-wolfssl", + "libwolfssl", + "libxtables", + "logd", + "lua", + "lua-argparse", + "lua-bit32", + "lualanes", + "luaposix", + "luarocks", + "luci", + "luci-app-attendedsysupgrade", + "luci-app-firewall", + "luci-app-opkg", + "luci-app-snmpd", + "luci-app-sqm", + "luci-app-statistics", + "luci-base", + "luci-lib-base", + "luci-lib-ip", + "luci-lib-jsonc", + "luci-lib-nixio", + "luci-mod-admin-full", + "luci-mod-network", + "luci-mod-status", + "luci-mod-system", + "luci-proto-ipv6", + "luci-proto-ppp", + "luci-theme-bootstrap", + "mtd", + "netifd", + "netperf", + "odhcp6c", + "odhcpd-ipv6only", + "openwrt-keyring", + "opkg", + "ppp", + "ppp-mod-pppoe", + "procd", + "px5g-wolfssl", + "rpcd", + "rpcd-mod-file", + "rpcd-mod-iwinfo", + "rpcd-mod-luci", + "rpcd-mod-rrdns", + "swconfig", + "uboot-envtools", + "ubox", + "ubus", + "ubusd", + "uci", + "uclient-fetch", + "uhttpd", + "uhttpd-mod-ubus", + "umdns", + "urandom-seed", + "urngd", + "usign", + "wireless-regdb", + "wpad-basic-wolfssl" + ], + "diff_packages": true, + "filesystem": "squashfs", + "client": "luci/git-23.339.51123-138595a" +} diff --git a/tests/regressions/regression-gh458.json b/tests/regressions/regression-gh458.json new file mode 100644 index 00000000..d92c15b8 --- /dev/null +++ b/tests/regressions/regression-gh458.json @@ -0,0 +1,231 @@ +{ + "revision": "r21968-acd8e94d20", + "advanced_mode": "1", + "branch": "SNAPSHOT", + "efi": null, + "profile": "raspberrypi,4-model-b", + "target": "bcm27xx/bcm2711", + "version": "SNAPSHOT", + "packages": [ + "adblock", + "attendedsysupgrade-common", + "base-files", + "bcm27xx-gpu-fw", + "bcm27xx-userland", + "bmon", + "brcmfmac-nvram-43455-sdio", + "busybox", + "ca-bundle", + "collectd", + "collectd-mod-cpu", + "collectd-mod-interface", + "collectd-mod-iwinfo", + "collectd-mod-load", + "collectd-mod-memory", + "collectd-mod-network", + "collectd-mod-rrdtool", + "collectd-mod-sensors", + "collectd-mod-thermal", + "confuse", + "coreutils", + "coreutils-sort", + "cypress-firmware-43455-sdio", + "dnsmasq", + "dropbear", + "e2fsprogs", + "firewall4", + "fstools", + "fwtool", + "getrandom", + "hostapd-common", + "htop", + "iftop", + "iperf3", + "iptraf-ng", + "iw-full", + "iwinfo", + "jansson", + "jshn", + "jsonfilter", + "kernel", + "kmod-brcmfmac", + "kmod-brcmutil", + "kmod-cfg80211", + "kmod-crypto-acompress", + "kmod-crypto-crc32c", + "kmod-crypto-hash", + "kmod-crypto-kpp", + "kmod-crypto-lib-chacha20", + "kmod-crypto-lib-chacha20poly1305", + "kmod-crypto-lib-curve25519", + "kmod-crypto-lib-poly1305", + "kmod-fixed-phy", + "kmod-fs-vfat", + "kmod-hid", + "kmod-hid-generic", + "kmod-input-core", + "kmod-input-evdev", + "kmod-lib-crc-ccitt", + "kmod-lib-crc32c", + "kmod-lib-lzo", + "kmod-libphy", + "kmod-mdio-devres", + "kmod-mii", + "kmod-mmc", + "kmod-nf-conntrack", + "kmod-nf-conntrack-netlink", + "kmod-nf-conntrack6", + "kmod-nf-flow", + "kmod-nf-log", + "kmod-nf-log6", + "kmod-nf-nat", + "kmod-nf-reject", + "kmod-nf-reject6", + "kmod-nfnetlink", + "kmod-nft-core", + "kmod-nft-fib", + "kmod-nft-nat", + "kmod-nft-offload", + "kmod-nls-base", + "kmod-nls-cp437", + "kmod-nls-iso8859-1", + "kmod-nls-utf8", + "kmod-phy-microchip", + "kmod-phy-realtek", + "kmod-ppp", + "kmod-pppoe", + "kmod-pppox", + "kmod-r8169", + "kmod-slhc", + "kmod-sound-arm-bcm2835", + "kmod-sound-core", + "kmod-udptunnel4", + "kmod-udptunnel6", + "kmod-usb-core", + "kmod-usb-hid", + "kmod-usb-net", + "kmod-usb-net-lan78xx", + "kmod-wireguard", + "libblkid", + "libblobmsg-json", + "libc", + "libcomerr", + "libext2fs", + "libf2fs", + "libiperf3", + "libiwinfo", + "libiwinfo-data", + "libjson-c", + "libjson-script", + "libltdl", + "liblua", + "liblucihttp", + "liblucihttp-lua", + "liblucihttp-ucode", + "libmnl", + "libncurses", + "libnftnl", + "libnl-core", + "libnl-route", + "libnl-tiny", + "libpcap", + "librrd1", + "libsensors", + "libsmartcols", + "libss", + "libsysfs", + "libubox", + "libubus", + "libubus-lua", + "libuclient", + "libucode", + "libustream-wolfssl", + "libuuid", + "libuv", + "libwolfssl", + "lm-sensors", + "logd", + "lua", + "luci", + "luci-app-adblock", + "luci-app-attendedsysupgrade", + "luci-app-diag-core", + "luci-app-firewall", + "luci-app-nlbwmon", + "luci-app-opkg", + "luci-app-statistics", + "luci-base", + "luci-compat", + "luci-lib-base", + "luci-lib-ip", + "luci-lib-jsonc", + "luci-lib-nixio", + "luci-lua-runtime", + "luci-mod-admin-full", + "luci-mod-dashboard", + "luci-mod-network", + "luci-mod-status", + "luci-mod-system", + "luci-proto-ipv6", + "luci-proto-ppp", + "luci-proto-wireguard", + "luci-ssl", + "luci-theme-bootstrap", + "luci-theme-material", + "luci-theme-openwrt", + "luci-theme-openwrt-2020", + "mkf2fs", + "mtd", + "nano-full", + "netdata", + "netifd", + "nftables-json", + "nlbwmon", + "odhcp6c", + "odhcpd-ipv6only", + "openwrt-keyring", + "opkg", + "partx-utils", + "ppp", + "ppp-mod-pppoe", + "procd", + "procd-seccomp", + "procd-ujail", + "px5g-wolfssl", + "r8169-firmware", + "rpcd", + "rpcd-mod-file", + "rpcd-mod-iwinfo", + "rpcd-mod-luci", + "rpcd-mod-rpcsys", + "rpcd-mod-rrdns", + "rpcd-mod-ucode", + "rrdtool1", + "sysfsutils", + "tcpdump", + "terminfo", + "ubox", + "ubus", + "ubusd", + "uci", + "uclient-fetch", + "ucode", + "ucode-mod-fs", + "ucode-mod-html", + "ucode-mod-lua", + "ucode-mod-math", + "ucode-mod-ubus", + "ucode-mod-uci", + "uhttpd", + "uhttpd-mod-ubus", + "urandom-seed", + "usign", + "wireguard-tools", + "wireless-regdb", + "wpad-basic-wolfssl", + "zlib" + ], + "diff_packages": true, + "filesystem": "squashfs", + "client": "luci/git-22.285.67526-18bfcca" +} diff --git a/tests/regressions/regression-gh782.json b/tests/regressions/regression-gh782.json new file mode 100644 index 00000000..68c1e1f3 --- /dev/null +++ b/tests/regressions/regression-gh782.json @@ -0,0 +1,151 @@ +{ + "profile": "tplink,archer-c7-v5", + "target": "ath79/generic", + "version": "23.05.3", + "packages": [ + "ath10k-board-qca988x", + "ath10k-firmware-qca988x-ct", + "base-files", + "busybox", + "ca-bundle", + "cgi-io", + "dnsmasq", + "dropbear", + "firewall4", + "fstools", + "fwtool", + "getrandom", + "hostapd-common", + "iw", + "iwinfo", + "jansson", + "jshn", + "jsonfilter", + "kernel", + "kmod-ath", + "kmod-ath10k-ct", + "kmod-ath9k", + "kmod-ath9k-common", + "kmod-cfg80211", + "kmod-crypto-aead", + "kmod-crypto-ccm", + "kmod-crypto-cmac", + "kmod-crypto-crc32c", + "kmod-crypto-ctr", + "kmod-crypto-gcm", + "kmod-crypto-gf128", + "kmod-crypto-ghash", + "kmod-crypto-hash", + "kmod-crypto-hmac", + "kmod-crypto-manager", + "kmod-crypto-null", + "kmod-crypto-rng", + "kmod-crypto-seqiv", + "kmod-crypto-sha256", + "kmod-gpio-button-hotplug", + "kmod-lib-crc-ccitt", + "kmod-lib-crc32c", + "kmod-mac80211", + "kmod-nf-conntrack", + "kmod-nf-conntrack6", + "kmod-nf-flow", + "kmod-nf-log", + "kmod-nf-log6", + "kmod-nf-nat", + "kmod-nf-reject", + "kmod-nf-reject6", + "kmod-nfnetlink", + "kmod-nft-core", + "kmod-nft-fib", + "kmod-nft-nat", + "kmod-nft-offload", + "kmod-ppp", + "kmod-pppoe", + "kmod-pppox", + "kmod-slhc", + "kmod-usb-ledtrig-usbport", + "kmod-usb2", + "libblobmsg-json", + "libiwinfo", + "libiwinfo-data", + "libiwinfo-lua", + "libjson-c", + "libjson-script", + "liblua", + "liblucihttp", + "liblucihttp-lua", + "libmnl", + "libnftnl", + "libnl-tiny", + "libubox", + "libubus", + "libubus-lua", + "libuci", + "libuclient", + "libucode", + "libustream-wolfssl", + "libwolfssl", + "logd", + "lua", + "luci", + "luci-app-attendedsysupgrade", + "luci-app-firewall", + "luci-app-opkg", + "luci-app-wireguard", + "luci-base", + "luci-i18n-attendedsysupgrade-en", + "luci-i18n-wireguard-en", + "luci-lib-base", + "luci-lib-ip", + "luci-lib-jsonc", + "luci-lib-nixio", + "luci-mod-admin-full", + "luci-mod-network", + "luci-mod-status", + "luci-mod-system", + "luci-proto-ipv6", + "luci-proto-ppp", + "luci-proto-wireguard", + "luci-ssl", + "luci-theme-bootstrap", + "mtd", + "netifd", + "nftables-json", + "odhcp6c", + "odhcpd-ipv6only", + "openwrt-keyring", + "opkg", + "ppp", + "ppp-mod-pppoe", + "procd", + "procd-seccomp", + "procd-ujail", + "px5g-wolfssl", + "rpcd", + "rpcd-mod-file", + "rpcd-mod-iwinfo", + "rpcd-mod-luci", + "rpcd-mod-rrdns", + "swconfig", + "uboot-envtools", + "ubox", + "ubus", + "ubusd", + "uci", + "uclient-fetch", + "ucode", + "ucode-mod-fs", + "ucode-mod-ubus", + "ucode-mod-uci", + "uhttpd", + "uhttpd-mod-ubus", + "urandom-seed", + "urngd", + "usign", + "wireless-regdb", + "wpad-basic-wolfssl" + ], + "diff_packages": true, + "filesystem": "squashfs", + "client": "luci/git-23.093.42303-58b861d" +} diff --git a/tests/regressions/regression-gh783.json b/tests/regressions/regression-gh783.json new file mode 100644 index 00000000..a12c260e --- /dev/null +++ b/tests/regressions/regression-gh783.json @@ -0,0 +1,164 @@ +{ + "url": "https://sysupgrade.openwrt.org", + "revision": "r23630-842932a63d", + "advanced_mode": "0", + "sha256_unsigned": "", + "branch": "23.05", + "efi": null, + "profile": "linksys,e8450-ubi", + "target": "mediatek/mt7622", + "version": "23.05.3", + "packages": [ + "auc", + "base-files", + "busybox", + "ca-bundle", + "ca-certificates", + "cgi-io", + "diffutils", + "dropbear", + "firewall4", + "fstools", + "fwtool", + "getrandom", + "hostapd-common", + "irqbalance", + "iw", + "iwinfo", + "jansson", + "jshn", + "jsonfilter", + "kernel", + "kmod-cfg80211", + "kmod-crypto-aead", + "kmod-crypto-ccm", + "kmod-crypto-cmac", + "kmod-crypto-crc32c", + "kmod-crypto-ctr", + "kmod-crypto-gcm", + "kmod-crypto-gf128", + "kmod-crypto-ghash", + "kmod-crypto-hash", + "kmod-crypto-hmac", + "kmod-crypto-manager", + "kmod-crypto-null", + "kmod-crypto-rng", + "kmod-crypto-seqiv", + "kmod-crypto-sha256", + "kmod-gpio-button-hotplug", + "kmod-hwmon-core", + "kmod-leds-gpio", + "kmod-lib-crc-ccitt", + "kmod-lib-crc32c", + "kmod-mac80211", + "kmod-mt76-connac", + "kmod-mt76-core", + "kmod-mt7615-common", + "kmod-mt7615-firmware", + "kmod-mt7615e", + "kmod-mt7622-firmware", + "kmod-mt7915-firmware", + "kmod-mt7915e", + "kmod-nf-conntrack", + "kmod-nf-conntrack6", + "kmod-nf-flow", + "kmod-nf-log", + "kmod-nf-log6", + "kmod-nf-nat", + "kmod-nf-reject", + "kmod-nf-reject6", + "kmod-nfnetlink", + "kmod-nft-core", + "kmod-nft-fib", + "kmod-nft-nat", + "kmod-nft-offload", + "kmod-ppp", + "kmod-pppoe", + "kmod-pppox", + "kmod-slhc", + "kmod-usb3", + "less", + "libblobmsg-json", + "libc", + "libiwinfo", + "libiwinfo-data", + "libiwinfo-lua", + "libjson-c", + "libjson-script", + "liblua", + "liblucihttp", + "liblucihttp-lua", + "libmnl", + "libnftnl", + "libnl-tiny", + "libubox", + "libubus", + "libubus-lua", + "libuci", + "libuclient", + "libucode", + "libustream-openssl", + "logd", + "lua", + "luci", + "luci-app-attendedsysupgrade", + "luci-app-dawn", + "luci-app-firewall", + "luci-app-opkg", + "luci-app-uhttpd", + "luci-base", + "luci-lib-base", + "luci-lib-ip", + "luci-lib-jsonc", + "luci-lib-nixio", + "luci-lib-px5g", + "luci-mod-admin-full", + "luci-mod-network", + "luci-mod-status", + "luci-mod-system", + "luci-proto-ipv6", + "luci-proto-ppp", + "luci-theme-bootstrap", + "mtd", + "netifd", + "nftables-json", + "ntpclient", + "odhcp6c", + "odhcpd-ipv6only", + "openssl-util", + "openwrt-keyring", + "opkg", + "ppp", + "ppp-mod-pppoe", + "procd", + "procd-seccomp", + "procd-ujail", + "rpcd", + "rpcd-mod-file", + "rpcd-mod-iwinfo", + "rpcd-mod-luci", + "rpcd-mod-rrdns", + "ubi-utils", + "uboot-envtools", + "ubox", + "ubus", + "ubusd", + "uci", + "uclient-fetch", + "ucode", + "ucode-mod-fs", + "ucode-mod-ubus", + "ucode-mod-uci", + "uhttpd", + "uhttpd-mod-ubus", + "urandom-seed", + "urngd", + "usign", + "vim-full", + "wireless-regdb", + "wpad-openssl" + ], + "diff_packages": true, + "filesystem": "squashfs", + "client": "luci/git-23.306.39494-6f7b129" +} diff --git a/tests/regressions/regression-gh784.json b/tests/regressions/regression-gh784.json new file mode 100644 index 00000000..6611ebdc --- /dev/null +++ b/tests/regressions/regression-gh784.json @@ -0,0 +1,383 @@ +{ + "url": "http://localhost:5001", + "profile": "rpi-4", + "target": "bcm27xx/bcm2711", + "packages": [ + "alpine-repositories", + "apk", + "ar", + "attendedsysupgrade-common", + "base-files", + "bash", + "bcm27xx-gpu-fw", + "bcm27xx-userland", + "bind-dig", + "bind-libs", + "binutils", + "block-mount", + "brcmfmac-firmware-usb", + "brcmfmac-nvram-43455-sdio", + "busybox", + "ca-bundle", + "cfdisk", + "cgi-io", + "collectd", + "collectd-mod-cpu", + "collectd-mod-cpufreq", + "collectd-mod-ethstat", + "collectd-mod-exec", + "collectd-mod-interface", + "collectd-mod-iwinfo", + "collectd-mod-load", + "collectd-mod-memory", + "collectd-mod-network", + "collectd-mod-rrdtool", + "collectd-mod-sensors", + "curl", + "cypress-firmware-43455-sdio", + "darkstat", + "dawn", + "diffutils", + "dropbear", + "dtc", + "e2fsprogs", + "f2fs-tools", + "f2fsck", + "fail2ban", + "firewall4", + "fstools", + "fwtool", + "gcc", + "gdisk", + "getrandom", + "git", + "git-http", + "hostapd-common", + "hostapd-utils", + "htop", + "httping", + "https-dns-proxy", + "iftop", + "ip-tiny", + "ip6tables-zz-legacy", + "iperf3", + "ipset", + "iptables-mod-conntrack-extra", + "iptables-mod-ipopt", + "iptables-zz-legacy", + "irqbalance", + "iw", + "iwinfo", + "jansson", + "jq", + "jshn", + "jsonfilter", + "kernel", + "kmod-brcmfmac", + "kmod-brcmutil", + "kmod-cfg80211", + "kmod-crypto-acompress", + "kmod-crypto-aead", + "kmod-crypto-ccm", + "kmod-crypto-cmac", + "kmod-crypto-crc32", + "kmod-crypto-crc32c", + "kmod-crypto-ctr", + "kmod-crypto-gcm", + "kmod-crypto-gf128", + "kmod-crypto-ghash", + "kmod-crypto-hash", + "kmod-crypto-hmac", + "kmod-crypto-kpp", + "kmod-crypto-lib-chacha20", + "kmod-crypto-lib-chacha20poly1305", + "kmod-crypto-lib-curve25519", + "kmod-crypto-lib-poly1305", + "kmod-crypto-manager", + "kmod-crypto-null", + "kmod-crypto-rng", + "kmod-crypto-seqiv", + "kmod-crypto-sha256", + "kmod-crypto-sha512", + "kmod-fixed-phy", + "kmod-fs-ext4", + "kmod-fs-f2fs", + "kmod-fs-vfat", + "kmod-hid", + "kmod-hid-generic", + "kmod-hwmon-core", + "kmod-hwmon-raspberrypi", + "kmod-i2c-algo-bit", + "kmod-i2c-bcm2835", + "kmod-i2c-core", + "kmod-i2c-gpio", + "kmod-input-core", + "kmod-input-evdev", + "kmod-ip6tables", + "kmod-ipt-conntrack", + "kmod-ipt-conntrack-extra", + "kmod-ipt-core", + "kmod-ipt-ipopt", + "kmod-ipt-ipset", + "kmod-ipt-raw", + "kmod-lib-crc-ccitt", + "kmod-lib-crc16", + "kmod-lib-crc32c", + "kmod-lib-lzo", + "kmod-libphy", + "kmod-mac80211", + "kmod-mdio-devres", + "kmod-mii", + "kmod-mmc", + "kmod-nf-conncount", + "kmod-nf-conntrack", + "kmod-nf-conntrack-netlink", + "kmod-nf-conntrack6", + "kmod-nf-flow", + "kmod-nf-ipt", + "kmod-nf-ipt6", + "kmod-nf-log", + "kmod-nf-log6", + "kmod-nf-nat", + "kmod-nf-reject", + "kmod-nf-reject6", + "kmod-nfnetlink", + "kmod-nft-core", + "kmod-nft-fib", + "kmod-nft-nat", + "kmod-nft-offload", + "kmod-nls-base", + "kmod-nls-cp437", + "kmod-nls-iso8859-1", + "kmod-nls-utf8", + "kmod-of-mdio", + "kmod-phy-microchip", + "kmod-phy-realtek", + "kmod-ppp", + "kmod-pppoe", + "kmod-pppox", + "kmod-r8169", + "kmod-scsi-core", + "kmod-slhc", + "kmod-sound-arm-bcm2835", + "kmod-sound-core", + "kmod-tun", + "kmod-udptunnel4", + "kmod-udptunnel6", + "kmod-usb-core", + "kmod-usb-ehci", + "kmod-usb-hid", + "kmod-usb-net", + "kmod-usb-net-cdc-ether", + "kmod-usb-net-cdc-ncm", + "kmod-usb-net-lan78xx", + "kmod-usb-net-rtl8152", + "kmod-usb-storage", + "kmod-usb-storage-uas", + "kmod-usb-xhci-hcd", + "kmod-usb2", + "kmod-usb3", + "kmod-wireguard", + "libatomic", + "libbfd", + "libblkid", + "libblobmsg-json", + "libbz2", + "libc", + "libcap", + "libcares", + "libcomerr", + "libctf", + "libcurl", + "libev", + "libevdev", + "libevent2", + "libevent2-core", + "libext2fs", + "libf2fs", + "libfdisk", + "libffi", + "libgcc", + "libgcrypt", + "libgmp", + "libgpg-error", + "libip4tc", + "libip6tc", + "libiperf3", + "libipset", + "libiptext", + "libiptext6", + "libiwinfo-data", + "libiwinfo-lua", + "libiwinfo", + "libjpeg-turbo", + "libjson-c", + "libjson-script", + "libltdl", + "liblua", + "liblucihttp-lua", + "liblucihttp-ucode", + "liblucihttp", + "libmbedtls", + "libmnl", + "libmount", + "libncurses", + "libnetfilter-conntrack", + "libnettle", + "libnfnetlink", + "libnftnl", + "libnghttp2", + "libnl-tiny", + "libopcodes", + "libopenssl-conf", + "libopenssl", + "libpcap", + "libpcre", + "libpcre2", + "libpng", + "libpopt", + "libpthread", + "libpython3", + "libqrencode", + "libreadline", + "librrd1", + "libsensors", + "libsmartcols", + "libsqlite3", + "libss", + "libstdcpp", + "libsysfs", + "libtirpc", + "libubox", + "libubus-lua", + "libubus", + "libuci-lua", + "libuci", + "libuclient", + "libucode", + "libudev-zero", + "libusb-1.0", + "libustream-mbedtls", + "libuuid", + "libuv", + "libwebp", + "libxtables", + "libzstd", + "lm-sensors", + "logd", + "lsof", + "lua", + "luci-app-advanced-reboot", + "luci-app-attendedsysupgrade", + "luci-app-dawn", + "luci-app-firewall", + "luci-app-https-dns-proxy", + "luci-app-mwan3", + "luci-app-nlbwmon", + "luci-app-opkg", + "luci-app-statistics", + "luci-base", + "luci-lib-base", + "luci-lib-ip", + "luci-lib-json", + "luci-lib-jsonc", + "luci-lib-nixio", + "luci-light", + "luci-lua-runtime", + "luci-mod-admin-full", + "luci-mod-network", + "luci-mod-status", + "luci-mod-system", + "luci-proto-ipv6", + "luci-proto-ppp", + "luci-proto-wireguard", + "luci-ssl", + "luci-theme-bootstrap", + "make", + "mesh11sd", + "mkf2fs", + "mtd", + "mtr-json", + "mwan3", + "netifd", + "nftables-json", + "nlbwmon", + "objdump", + "odhcp6c", + "odhcpd-ipv6only", + "openssl-util", + "openwrt-keyring", + "opkg", + "partx-utils", + "pingcheck", + "ppp", + "ppp-mod-pppoe", + "procd", + "procd-seccomp", + "procd-ujail", + "px5g-mbedtls", + "python3-base", + "python3-ctypes", + "python3-distutils", + "python3-email", + "python3-light", + "python3-logging", + "python3-pkg-resources", + "python3-sqlite3", + "python3-urllib", + "qrencode", + "r8152-firmware", + "r8169-firmware", + "resize2fs", + "resolveip", + "rpcd", + "rpcd-mod-file", + "rpcd-mod-iwinfo", + "rpcd-mod-luci", + "rpcd-mod-rpcsys", + "rpcd-mod-rrdns", + "rpcd-mod-ucode", + "rrdtool1", + "rsync", + "shadow-common", + "shadow-groupadd", + "shadow-useradd", + "shadow-usermod", + "sudo", + "sysfsutils", + "terminfo", + "tmux", + "tune2fs", + "ubox", + "ubus", + "ubusd", + "uci", + "uclient-fetch", + "ucode", + "ucode-mod-fs", + "ucode-mod-html", + "ucode-mod-lua", + "ucode-mod-math", + "ucode-mod-nl80211", + "ucode-mod-rtnl", + "ucode-mod-ubus", + "ucode-mod-uci", + "ucode-mod-uloop", + "uhttpd", + "uhttpd-mod-ubus", + "umdns", + "urandom-seed", + "usbids", + "usbutils", + "usign", + "wireguard-tools", + "wireless-regdb", + "xtables-legacy", + "zlib", + "zsh" + ], + "defaults": "", + "version": "23.05.3", + "diff_packages": true, + "client": "ofs/%GIT_VERSION%" +} diff --git a/tests/test_api.py b/tests/test_api.py index fb62343c..d3fe1628 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -758,3 +758,147 @@ def test_api_stats(client): assert response.status_code == 200 data = response.json() assert data["queue_length"] == 0 + + +def test_api_build_prepare(client): + """Test the prepare endpoint""" + response = client.post( + "/api/v1/build/prepare", + json=dict( + version="1.2.3", + target="testtarget/testsubtarget", + profile="testprofile", + packages=["test1", "test2"], + ), + ) + assert response.status_code == 200 + data = response.json() + + # Check response structure + assert data["status"] == "prepared" + assert "original_packages" in data + assert "resolved_packages" in data + assert "changes" in data + assert "prepared_request" in data + assert "request_hash" in data + assert "cache_available" in data + + # Check original packages + assert data["original_packages"] == ["test1", "test2"] + + # Check prepared request structure + prepared = data["prepared_request"] + assert prepared["version"] == "1.2.3" + assert prepared["target"] == "testtarget/testsubtarget" + assert prepared["profile"] == "testprofile" + assert prepared["diff_packages"] is False # Should be False after preparation + + +def test_api_build_prepare_with_auc_migration(client): + """Test prepare endpoint with auc → owut migration""" + response = client.post( + "/api/v1/build/prepare", + json=dict( + version="24.10.0", + target="testtarget/testsubtarget", + profile="testprofile", + packages=["luci", "auc"], + ), + ) + assert response.status_code == 200 + data = response.json() + + # Check that auc was replaced with owut + assert "owut" in data["resolved_packages"] + assert "auc" not in data["resolved_packages"] + + # Check changes + assert len(data["changes"]) >= 1 + migration = next( + (c for c in data["changes"] if c.get("from_package") == "auc"), None + ) + assert migration is not None + assert migration["type"] == "migration" + assert migration["to_package"] == "owut" + + +def test_api_build_prepare_validation_error(client): + """Test prepare endpoint with invalid version""" + response = client.post( + "/api/v1/build/prepare", + json=dict( + version="99.99.99", # Invalid version + target="testtarget/testsubtarget", + profile="testprofile", + packages=["test1"], + ), + ) + assert response.status_code == 400 + data = response.json() + assert "detail" in data + + +def test_api_build_from_prepared_request(client): + """Test building from a prepared request""" + # First, prepare the request + prep_response = client.post( + "/api/v1/build/prepare", + json=dict( + version="1.2.3", + target="testtarget/testsubtarget", + profile="testprofile", + packages=["test1", "test2"], + ), + ) + assert prep_response.status_code == 200 + prep_data = prep_response.json() + + # Then build using the prepared request + build_response = client.post( + "/api/v1/build?skip_package_resolution=true", + json=prep_data["prepared_request"], + ) + assert build_response.status_code == 200 + build_data = build_response.json() + + # Verify the build used the prepared packages + assert build_data["request"]["packages"] == prep_data["resolved_packages"] + + +def test_api_build_with_skip_package_resolution_flag(client): + """Test that skip_package_resolution flag works correctly""" + # Build with auc but skip package resolution + response = client.post( + "/api/v1/build?skip_package_resolution=true", + json=dict( + version="24.10.0", + target="testtarget/testsubtarget", + profile="testprofile", + packages=["owut"], # Already migrated + diff_packages=False, + ), + ) + assert response.status_code == 200 + data = response.json() + + # Should keep owut, not try to migrate + assert "owut" in data["request"]["packages"] + + +def test_api_build_prepare_with_from_version(client): + """Test prepare endpoint with from_version parameter""" + response = client.post( + "/api/v1/build/prepare", + json=dict( + version="24.10.0", + from_version="23.05.0", + target="testtarget/testsubtarget", + profile="testprofile", + packages=["luci"], + ), + ) + assert response.status_code == 200 + data = response.json() + + # Check that from_version is preserved in prepared request + assert data["prepared_request"]["from_version"] == "23.05.0" diff --git a/tests/test_integration_prepare_build.py b/tests/test_integration_prepare_build.py new file mode 100644 index 00000000..d39ea8e6 --- /dev/null +++ b/tests/test_integration_prepare_build.py @@ -0,0 +1,314 @@ +""" +Integration tests for the prepare + build workflow. + +These tests require both services to be running: +- asu-prepare service on http://localhost:8001 +- asu-build service on http://localhost:8000 + +Run with: + pytest tests/test_integration_prepare_build.py -v + +Or skip if services not running: + pytest tests/test_integration_prepare_build.py -v -m "not integration" +""" + +import pytest +import requests +import time +import os + +# Configuration +PREPARE_URL = os.getenv("PREPARE_SERVICE_URL", "http://localhost:8001") +BUILD_URL = os.getenv("BUILD_SERVICE_URL", "http://localhost:8000") +MAX_POLL_ATTEMPTS = 60 # 60 seconds max +POLL_INTERVAL = 1 # 1 second between polls + + +def check_service_available(url): + """Check if a service is available""" + try: + response = requests.get(f"{url}/health", timeout=2) + return response.status_code == 200 + except requests.exceptions.RequestException: + return False + + +@pytest.fixture(scope="module") +def services_available(): + """Check if both services are available""" + prepare_available = check_service_available(PREPARE_URL) + build_available = check_service_available(BUILD_URL) + + if not prepare_available: + pytest.skip(f"Prepare service not available at {PREPARE_URL}") + if not build_available: + pytest.skip(f"Build service not available at {BUILD_URL}") + + return True + + +@pytest.mark.integration +class TestPrepareEndpointIntegration: + """Integration tests for the prepare endpoint""" + + def test_prepare_basic_request(self, services_available): + """Test basic prepare request""" + request = { + "version": "23.05.5", + "target": "ath79/generic", + "profile": "tplink_tl-wdr4300-v1", + "packages": ["luci"], + } + + response = requests.post(f"{PREPARE_URL}/api/v1/prepare", json=request) + + assert response.status_code == 200 + data = response.json() + + assert data["status"] == "prepared" + assert "resolved_packages" in data + assert "changes" in data + assert "prepared_request" in data + assert "request_hash" in data + + def test_prepare_with_migration(self, services_available): + """Test prepare with package migration (auc -> owut)""" + request = { + "version": "24.10.0", + "target": "ath79/generic", + "profile": "tplink_tl-wdr4300-v1", + "packages": ["luci", "auc"], + } + + response = requests.post(f"{PREPARE_URL}/api/v1/prepare", json=request) + + assert response.status_code == 200 + data = response.json() + + # Should have migrated auc to owut + assert "auc" not in data["resolved_packages"] + assert "owut" in data["resolved_packages"] + + # Should have tracked the migration + migrations = [c for c in data["changes"] if c["type"] == "migration"] + assert len(migrations) >= 1 + assert any(c["from_package"] == "auc" and c["to_package"] == "owut" for c in migrations) + + def test_prepare_with_language_pack_migration(self, services_available): + """Test prepare with language pack migration""" + request = { + "version": "24.10.0", + "target": "ath79/generic", + "profile": "tplink_tl-wdr4300-v1", + "packages": ["luci", "luci-i18n-opkg-en"], + } + + response = requests.post(f"{PREPARE_URL}/api/v1/prepare", json=request) + + assert response.status_code == 200 + data = response.json() + + # Should have migrated language pack + assert "luci-i18n-opkg-en" not in data["resolved_packages"] + assert "luci-i18n-package-manager-en" in data["resolved_packages"] + + def test_prepare_profile_sanitization(self, services_available): + """Test that prepare sanitizes profile names""" + request = { + "version": "23.05.5", + "target": "ath79/generic", + "profile": "test,profile,with,commas", + "packages": ["luci"], + } + + response = requests.post(f"{PREPARE_URL}/api/v1/prepare", json=request) + + assert response.status_code == 200 + data = response.json() + + # Profile should be sanitized + assert "," not in data["prepared_request"]["profile"] + assert data["prepared_request"]["profile"] == "test_profile_with_commas" + + +@pytest.mark.integration +class TestPrepareBuildWorkflow: + """Integration tests for the complete prepare -> build workflow""" + + def poll_build(self, build_url, timeout=MAX_POLL_ATTEMPTS): + """Poll build endpoint until completion or timeout""" + attempts = 0 + while attempts < timeout: + response = requests.get(build_url) + + if response.status_code == 200: + return response.json() + elif response.status_code == 202: + data = response.json() + print(f"Building... {data.get('imagebuilder_status', 'unknown')}") + time.sleep(POLL_INTERVAL) + attempts += 1 + else: + # Error occurred + return response.json() + + raise TimeoutError(f"Build did not complete within {timeout} seconds") + + def test_prepare_then_build_basic(self, services_available): + """Test complete workflow: prepare -> build""" + # Step 1: Prepare + prepare_request = { + "version": "23.05.5", + "target": "x86/64", + "profile": "generic", + "packages": ["luci"], + } + + prepare_response = requests.post( + f"{PREPARE_URL}/api/v1/prepare", + json=prepare_request + ) + + assert prepare_response.status_code == 200 + prepare_data = prepare_response.json() + + # Step 2: Build with prepared request + # Note: Build service needs additional fields + build_request = { + **prepare_data["prepared_request"], + "diff_packages": False, # Build service needs this + } + + build_response = requests.post( + f"{BUILD_URL}/api/v1/build", + json=build_request + ) + + # Build should be accepted (202) or completed (200) + assert build_response.status_code in [200, 202] + + if build_response.status_code == 202: + # Poll until completion + build_data = build_response.json() + request_hash = build_data["request_hash"] + + # Poll using request_hash + final_data = self.poll_build(f"{BUILD_URL}/api/v1/build/{request_hash}") + + # Should have completed successfully + assert "images" in final_data or "detail" in final_data + + def test_prepare_with_migration_then_build(self, services_available): + """Test workflow with package migration""" + # Step 1: Prepare with migration + prepare_request = { + "version": "24.10.0", + "target": "x86/64", + "profile": "generic", + "packages": ["luci", "auc"], # auc will migrate to owut + } + + prepare_response = requests.post( + f"{PREPARE_URL}/api/v1/prepare", + json=prepare_request + ) + + assert prepare_response.status_code == 200 + prepare_data = prepare_response.json() + + # Verify migration happened + assert "owut" in prepare_data["resolved_packages"] + assert "auc" not in prepare_data["resolved_packages"] + + # Step 2: Build with migrated packages + build_request = { + **prepare_data["prepared_request"], + "diff_packages": False, + } + + build_response = requests.post( + f"{BUILD_URL}/api/v1/build", + json=build_request + ) + + assert build_response.status_code in [200, 202] + + def test_prepare_shows_changes_before_build(self, services_available): + """Test that prepare shows changes before committing to build""" + # Use a profile that requires additional packages + prepare_request = { + "version": "25.12.0", + "target": "kirkwood/generic", + "profile": "checkpoint_l-50", + "packages": ["luci"], + } + + prepare_response = requests.post( + f"{PREPARE_URL}/api/v1/prepare", + json=prepare_request + ) + + assert prepare_response.status_code == 200 + prepare_data = prepare_response.json() + + # Should show that kmod-dsa-mv88e6xxx will be added + assert "kmod-dsa-mv88e6xxx" in prepare_data["resolved_packages"] + + # Should have changes tracked + additions = [c for c in prepare_data["changes"] if c["type"] == "addition"] + assert len(additions) >= 1 + assert any("kmod-dsa-mv88e6xxx" in c.get("package", "") for c in additions) + + # User can see changes BEFORE building + print("Changes that will be applied:") + for change in prepare_data["changes"]: + if change["type"] == "migration": + print(f" 🔄 {change['from_package']} → {change['to_package']}") + elif change["type"] == "addition": + print(f" ➕ {change['package']}") + elif change["type"] == "removal": + print(f" ➖ {change['package']}") + + +@pytest.mark.integration +class TestServiceIndependence: + """Test that services are truly independent""" + + def test_prepare_works_without_build_service(self): + """Prepare service should work even if build service is down""" + if not check_service_available(PREPARE_URL): + pytest.skip("Prepare service not available") + + request = { + "version": "23.05.5", + "target": "ath79/generic", + "profile": "test", + "packages": ["luci"], + } + + response = requests.post(f"{PREPARE_URL}/api/v1/prepare", json=request) + + # Should work regardless of build service status + assert response.status_code == 200 + + def test_prepare_has_no_build_capability(self, services_available): + """Prepare service should not have build endpoint""" + response = requests.post( + f"{PREPARE_URL}/api/v1/build", + json={"version": "23.05.5", "target": "x86/64", "profile": "generic"} + ) + + # Should return 404 or 405 (not found / method not allowed) + assert response.status_code in [404, 405] + + def test_services_report_different_capabilities(self, services_available): + """Services should report different capabilities""" + prepare_status = requests.get(f"{PREPARE_URL}/api/v1/status").json() + build_status = requests.get(f"{BUILD_URL}/api/v1/status").json() + + # Prepare service capabilities + assert prepare_status["capabilities"]["package_resolution"] is True + assert prepare_status["capabilities"]["build_execution"] is False + + # Build service capabilities (when /status endpoint exists) + # Note: Build service may not have status endpoint yet diff --git a/tests/test_package_resolution.py b/tests/test_package_resolution.py new file mode 100644 index 00000000..e520ef63 --- /dev/null +++ b/tests/test_package_resolution.py @@ -0,0 +1,224 @@ +"""Tests for package resolution functionality""" + +import pytest + +from asu.build_request import BuildRequest +from asu.package_resolution import PackageResolver, PackageChange + + +def test_package_change_to_dict(): + """Test PackageChange serialization""" + change = PackageChange( + change_type="migration", + action="replace", + from_package="auc", + to_package="owut", + reason="Package renamed in 24.10", + automatic=True, + ) + + result = change.to_dict() + assert result["type"] == "migration" + assert result["action"] == "replace" + assert result["from_package"] == "auc" + assert result["to_package"] == "owut" + assert result["reason"] == "Package renamed in 24.10" + assert result["automatic"] is True + + +def test_package_resolver_no_changes(): + """Test resolver with no package changes needed""" + resolver = PackageResolver() + + request = BuildRequest( + version="23.05.5", + target="ath79/generic", + profile="tplink_archer-c7-v5", + packages=["luci", "htop"], + ) + + final_packages, changes = resolver.resolve(request) + + assert "luci" in final_packages + assert "htop" in final_packages + # No changes should be made for basic packages on 23.05 + assert len(changes) == 0 + + +def test_package_resolver_auc_migration(): + """Test auc → owut migration in 24.10""" + resolver = PackageResolver() + + request = BuildRequest( + version="24.10.0", + target="ath79/generic", + profile="tplink_archer-c7-v5", + packages=["luci", "auc"], + ) + + final_packages, changes = resolver.resolve(request) + + # auc should be replaced with owut + assert "owut" in final_packages + assert "auc" not in final_packages + assert "luci" in final_packages + + # Should have one migration change + assert len(changes) == 1 + migration = changes[0] + assert migration.type == "migration" + assert migration.action == "replace" + assert migration.from_package == "auc" + assert migration.to_package == "owut" + + +def test_package_resolver_hardware_dependencies(): + """Test hardware-specific package addition""" + resolver = PackageResolver() + + request = BuildRequest( + version="23.05.5", + target="mediatek/mt7622", + profile="linksys_e8450", + packages=["luci"], + ) + + final_packages, changes = resolver.resolve(request) + + # Should add kmod-mt7622-firmware + assert "kmod-mt7622-firmware" in final_packages + assert "luci" in final_packages + + # Should have one addition + assert len(changes) == 1 + addition = changes[0] + assert addition.type == "addition" + assert addition.action == "add" + assert addition.package == "kmod-mt7622-firmware" + + +def test_package_resolver_language_pack_rename(): + """Test language pack renaming in 24.10""" + resolver = PackageResolver() + + request = BuildRequest( + version="24.10.0", + target="ath79/generic", + profile="tplink_archer-c7-v5", + packages=["luci", "luci-i18n-opkg-en"], + ) + + final_packages, changes = resolver.resolve(request) + + # Language pack should be renamed + assert "luci-i18n-package-manager-en" in final_packages + assert "luci-i18n-opkg-en" not in final_packages + assert "luci" in final_packages + + # Should have one migration + assert len(changes) == 1 + migration = changes[0] + assert migration.type == "migration" + assert migration.from_package == "luci-i18n-opkg-en" + assert migration.to_package == "luci-i18n-package-manager-en" + + +def test_package_resolver_multiple_changes(): + """Test multiple package changes in one request""" + resolver = PackageResolver() + + request = BuildRequest( + version="25.12.0", + target="kirkwood/generic", + profile="checkpoint_l-50", + packages=["luci"], + ) + + final_packages, changes = resolver.resolve(request) + + # Should add kmod-dsa-mv88e6xxx + assert "kmod-dsa-mv88e6xxx" in final_packages + assert "luci" in final_packages + + # Should have at least one addition + assert len(changes) >= 1 + addition = next((c for c in changes if c.package == "kmod-dsa-mv88e6xxx"), None) + assert addition is not None + assert addition.type == "addition" + + +def test_package_resolver_lantiq_firmware(): + """Test lantiq PHY firmware additions in 25.12""" + resolver = PackageResolver() + + request = BuildRequest( + version="25.12.0", + target="lantiq/xrx200", + profile="arcadyan_arv7519rw22", + packages=["luci"], + ) + + final_packages, changes = resolver.resolve(request) + + # Should add PHY firmware packages + assert "xrx200-rev1.1-phy22f-firmware" in final_packages + assert "xrx200-rev1.2-phy22f-firmware" in final_packages + assert "luci" in final_packages + + # Should have two additions + firmware_additions = [ + c + for c in changes + if c.type == "addition" and "firmware" in c.package + ] + assert len(firmware_additions) == 2 + + +def test_package_resolver_get_addition_reason(): + """Test addition reason detection""" + resolver = PackageResolver() + + request = BuildRequest( + version="23.05.5", + target="ath79/generic", + profile="test", + packages=[], + ) + + # Test kernel module + reason = resolver._get_addition_reason("kmod-usb-core", request) + assert "kernel module" in reason.lower() + + # Test language pack + reason = resolver._get_addition_reason("luci-i18n-base-en", request) + assert "language pack" in reason.lower() + + # Test PHY firmware + reason = resolver._get_addition_reason("xrx200-rev1.1-phy11g-firmware", request) + assert "firmware" in reason.lower() + + # Test generic package + reason = resolver._get_addition_reason("htop", request) + assert "version/target/profile" in reason.lower() + + +def test_package_resolver_preserves_original_request(): + """Test that resolver doesn't modify the original request object""" + resolver = PackageResolver() + + original_packages = ["luci", "auc"] + request = BuildRequest( + version="24.10.0", + target="ath79/generic", + profile="test", + packages=original_packages.copy(), + ) + + # Packages will be modified during resolve + final_packages, changes = resolver.resolve(request) + + # The request.packages will be modified (this is expected) + # But we track changes correctly + assert len(changes) == 1 + assert changes[0].from_package == "auc" + assert changes[0].to_package == "owut" diff --git a/tests/test_services.py b/tests/test_services.py new file mode 100644 index 00000000..5a32ea8e --- /dev/null +++ b/tests/test_services.py @@ -0,0 +1,158 @@ +"""Tests for independent microservices""" + +import pytest + +from asu.build_request import BuildRequest +from asu.services.prepare_service import PrepareService, get_prepare_service +from asu.services.build_service import BuildService, get_build_service + + +class TestPrepareService: + """Tests for the independent prepare service""" + + def test_prepare_service_standalone(self): + """Test prepare service can run without app""" + service = PrepareService(app=None) + + request = BuildRequest( + version="24.10.0", + target="ath79/generic", + profile="test", + packages=["luci", "auc"], + ) + + # Should work without validation (no app provided) + result = service.prepare(request) + + assert result["status"] == "prepared" + assert "owut" in result["resolved_packages"] + assert "auc" not in result["resolved_packages"] + assert len(result["changes"]) >= 1 + + def test_prepare_service_singleton(self): + """Test prepare service singleton""" + service1 = get_prepare_service() + service2 = get_prepare_service() + + assert service1 is service2 + + def test_prepare_service_no_changes(self): + """Test prepare service with no package changes""" + service = PrepareService(app=None) + + request = BuildRequest( + version="23.05.5", + target="ath79/generic", + profile="test", + packages=["luci"], + ) + + result = service.prepare(request) + + assert result["status"] == "prepared" + assert result["resolved_packages"] == ["luci"] + assert len(result["changes"]) == 0 + + def test_prepare_service_sanitizes_profile(self): + """Test prepare service sanitizes profile names""" + service = PrepareService(app=None) + + request = BuildRequest( + version="23.05.5", + target="ath79/generic", + profile="test,profile", # Invalid comma + packages=["luci"], + ) + + result = service.prepare(request) + + # Profile should be sanitized + assert "," not in result["prepared_request"]["profile"] + + def test_prepare_service_preserves_from_version(self): + """Test prepare service preserves from_version""" + service = PrepareService(app=None) + + request = BuildRequest( + version="24.10.0", + from_version="23.05.0", + target="ath79/generic", + profile="test", + packages=["luci"], + ) + + result = service.prepare(request) + + assert result["prepared_request"]["from_version"] == "23.05.0" + + +class TestBuildService: + """Tests for the independent build service""" + + def test_build_service_singleton(self): + """Test build service singleton""" + service1 = get_build_service() + service2 = get_build_service() + + assert service1 is service2 + + +class TestServiceIndependence: + """Tests to verify services are truly independent""" + + def test_prepare_has_no_redis_dependency(self): + """Verify prepare service doesn't import Redis at module level""" + import asu.services.prepare_service as prepare_module + + # Check module doesn't have redis in its globals + module_names = [name for name in dir(prepare_module)] + + # Should not have Redis or RQ imports at module level + assert "redis" not in [n.lower() for n in module_names] + assert "Queue" not in module_names + assert "get_queue" not in module_names + + def test_prepare_has_no_podman_dependency(self): + """Verify prepare service doesn't import Podman""" + import asu.services.prepare_service as prepare_module + + module_names = [name for name in dir(prepare_module)] + + # Should not have Podman imports + assert "podman" not in [n.lower() for n in module_names] + assert "get_podman" not in module_names + + def test_prepare_has_no_build_dependency(self): + """Verify prepare service doesn't import build logic""" + import asu.services.prepare_service as prepare_module + + module_names = [name for name in dir(prepare_module)] + + # Should not import build function + assert "build" not in [n for n in module_names if not n.startswith("_")] + + def test_services_share_common_models(self): + """Verify both services use the same models""" + from asu.services.prepare_service import PrepareService + from asu.services.build_service import BuildService + from asu.build_request import BuildRequest + + # Both should use BuildRequest + prepare = PrepareService() + build = BuildService() + + # Both should accept BuildRequest (no TypeError) + request = BuildRequest( + version="23.05.5", + target="test/test", + profile="test", + ) + + # Prepare should work + result = prepare.prepare(request) + assert result["status"] == "prepared" + + # Build should work (will fail due to missing Redis in tests, + # but should not fail on BuildRequest type) + # We just verify it accepts the same type + assert isinstance(request, BuildRequest) diff --git a/tests/uclient.py b/tests/uclient.py new file mode 100644 index 00000000..1fe0b0c8 --- /dev/null +++ b/tests/uclient.py @@ -0,0 +1,252 @@ +#!/usr/bin/env python3 +""" +ASU Client - Two-step workflow (prepare + build) + +Usage: + # Use two-step workflow (default): + python uclient.py config.json + + # Use legacy single-step workflow: + python uclient.py config.json --legacy + + # Use prepare-only mode to see changes without building: + python uclient.py config.json --prepare-only + +Config JSON format: +{ + "url": "http://localhost:8000", + "prepare_url": "http://localhost:8001", // Optional, for two-step mode + "version": "23.05.5", + "target": "ath79/generic", + "profile": "tplink_tl-wdr4300-v1", + "packages": ["luci", "vim"], + "defaults": "...", // Optional + "diff_packages": true // Optional +} +""" + +import json +import sys +from pathlib import Path +from time import sleep +from typing import Optional + +import requests + + +def do_prepare(prepare_url: str, data: dict) -> dict: + """ + Step 1: Call prepare endpoint to resolve packages and get changes. + + Returns the prepare response with resolved packages and changes. + """ + request = { + "version": data["version"], + "target": data["target"], + "profile": data["profile"], + } + + if "packages" in data: + request["packages"] = data["packages"] + + if "packages_versions" in data: + request["packages"] = list(data["packages_versions"].keys()) + + if "from_version" in data: + request["from_version"] = data["from_version"] + + print(f"📝 Preparing build request...") + response = requests.post(f"{prepare_url}/api/v1/prepare", json=request) + + if response.status_code != 200: + print(f"❌ Prepare failed: {response.status_code}") + print(response.text) + sys.exit(1) + + return response.json() + + +def show_prepare_results(prepare_data: dict): + """Display prepare results to user""" + print("\n✅ Preparation complete!") + print(f"Request hash: {prepare_data['request_hash']}") + + if prepare_data["changes"]: + print(f"\n📦 Package changes ({len(prepare_data['changes'])}):") + for change in prepare_data["changes"]: + if change["type"] == "migration": + print(f" 🔄 Migration: {change['from_package']} → {change['to_package']}") + print(f" Reason: {change['reason']}") + elif change["type"] == "addition": + print(f" ➕ Addition: {change['package']}") + print(f" Reason: {change['reason']}") + elif change["type"] == "removal": + print(f" ➖ Removal: {change['package']}") + print(f" Reason: {change['reason']}") + else: + print("\n✨ No package changes needed") + + print(f"\nFinal packages ({len(prepare_data['resolved_packages'])}):") + for pkg in sorted(prepare_data['resolved_packages']): + print(f" - {pkg}") + + +def do_build(build_url: str, prepare_data: dict, original_data: dict) -> dict: + """ + Step 2: Call build endpoint with prepared request. + + Polls until build completes or fails. + """ + # Build request combines prepared request with build-specific fields + request = { + **prepare_data["prepared_request"], + "diff_packages": original_data.get("diff_packages", False), + } + + # Add build-specific fields if present + if "defaults" in original_data: + request["defaults"] = original_data["defaults"] + + if "rootfs_size_mb" in original_data: + request["rootfs_size_mb"] = original_data["rootfs_size_mb"] + + if "repositories" in original_data: + request["repositories"] = original_data["repositories"] + + if "repository_keys" in original_data: + request["repository_keys"] = original_data["repository_keys"] + + print(f"\n🔨 Starting build...") + response = requests.post(f"{build_url}/api/v1/build", json=request) + + # Poll until completion + while response.status_code == 202: + response_json = response.json() + status = response_json.get("imagebuilder_status", "unknown") + + if "queue_position" in response_json: + print(f"⏳ Queued at position {response_json['queue_position']}") + else: + print(f"⚙️ Building: {status}") + + sleep(1) + + # Poll using request hash + request_hash = response_json["request_hash"] + response = requests.get(f"{build_url}/api/v1/build/{request_hash}") + + return response.json() + + +def do_legacy_build(build_url: str, data: dict) -> dict: + """ + Legacy single-step workflow (direct to /build without prepare). + + This is the old behavior for backwards compatibility. + """ + request = { + "target": data["target"], + "version": data["version"], + "profile": data["profile"], + } + + if "packages" in data: + request["packages"] = data["packages"] + + if "packages_versions" in data: + request["packages"] = list(data["packages_versions"].keys()) + + if "defaults" in data: + request["defaults"] = data["defaults"] + + if "diff_packages" in data: + request["diff_packages"] = data["diff_packages"] + + print(f"🔨 Building (legacy mode)...") + response = requests.post(f"{build_url}/api/v1/build", json=request) + + # Poll until completion + while response.status_code == 202: + response_json = response.json() + print(f"⚙️ Building: {response_json.get('imagebuilder_status', 'unknown')}") + sleep(1) + request_hash = response_json["request_hash"] + response = requests.get(f"{build_url}/api/v1/build/{request_hash}") + + return response.json() + + +def show_build_results(result: dict): + """Display build results""" + print("\n" + "="*60) + + if "detail" in result: + # Error occurred + print(f"❌ Build failed: {result['detail']}") + if "stdout" in result: + print("\nSTDOUT:") + print(result["stdout"]) + if "stderr" in result: + print("\nSTDERR:") + print(result["stderr"]) + sys.exit(1) + else: + # Success + print("✅ Build completed successfully!") + print(f"\nVersion: {result.get('version_number', 'unknown')}") + print(f"Request hash: {result['request_hash']}") + + if "images" in result: + print(f"\n📦 Images ({len(result['images'])}):") + for image in result["images"]: + print(f" - {image['name']}") + print(f" Type: {image.get('type', 'unknown')}") + print(f" SHA256: {image.get('sha256', 'unknown')}") + + +def main(): + if len(sys.argv) < 2: + print("Usage: python uclient.py [--legacy] [--prepare-only]") + sys.exit(1) + + config_path = Path(sys.argv[1]) + legacy_mode = "--legacy" in sys.argv + prepare_only = "--prepare-only" in sys.argv + + data = json.loads(config_path.read_text()) + + build_url = data["url"] + prepare_url = data.get("prepare_url", "http://localhost:8001") + + if legacy_mode: + # Old single-step workflow + print("🔧 Using legacy single-step workflow") + result = do_legacy_build(build_url, data) + show_build_results(result) + else: + # New two-step workflow + print("🚀 Using two-step workflow (prepare + build)") + + # Step 1: Prepare + prepare_data = do_prepare(prepare_url, data) + show_prepare_results(prepare_data) + + if prepare_only: + print("\n✋ Stopping after prepare (--prepare-only mode)") + print("\nTo build with these changes, remove --prepare-only flag") + sys.exit(0) + + # Ask user to confirm if there are changes + if prepare_data["changes"]: + response = input("\n❓ Proceed with build? [Y/n] ") + if response.lower() in ["n", "no"]: + print("❌ Build cancelled by user") + sys.exit(0) + + # Step 2: Build + result = do_build(build_url, prepare_data, data) + show_build_results(result) + + +if __name__ == "__main__": + main()