GQL SQL Support

.coveragerc

@@ -1,12 +1,15 @@
 # .coveragerc to control coverage.py
 [run]
 branch = True
-source = datajunction
-# omit = bad_file.py
+source = dj
+omit =
+    */dj/sql/parsing/backends/grammar/generated/*
+    */dj/sql/parsing/backends/antlr4.py
+    */dj/sql/parsing/ast.py
 
 [paths]
 source =
-    src/
+    dj/
     */site-packages/
 
 [report]
@@ -26,3 +29,5 @@ exclude_lines =
     # Don't complain if non-runnable code isn't run:
     if 0:
     if __name__ == .__main__.:
+
+    if TYPE_CHECKING:

.github/pull_request_template.md

@@ -0,0 +1,15 @@
+### Summary
+
+<!-- What's this change about? -->
+
+### Test Plan
+
+<!-- How did you test your change? -->
+
+- [ ] PR has an associated issue: #
+- [ ] `make check` passes
+- [ ] `make test` shows 100% unit test coverage
+
+### Deployment Plan
+
+<!-- Any special instructions around deployment? -->

.github/release.yml

@@ -0,0 +1,36 @@
+# .github/release.yml
+# Configuration for auto-generated release notes
+
+changelog:
+  exclude:
+    labels:
+      - ignore-for-release
+      - skip-changelog
+      - duplicate
+  categories:
+    - title: "⚠️ Breaking Changes"
+      labels:
+        - Semver-Major
+        - breaking-change
+    - title: "🚀 Features"
+      labels:
+        - Semver-Minor
+        - enhancement
+        - feature
+    - title: "🐛 Bug Fixes"
+      labels:
+        - bug
+        - bugfix
+        - fix
+    - title: "📚 Documentation"
+      labels:
+        - documentation
+        - docs
+    - title: "🧹 Maintenance"
+      labels:
+        - chore
+        - maintenance
+        - dependencies
+    - title: "Other Changes"
+      labels:
+        - "*"

.github/workflows/client-integration-tests.yml

@@ -0,0 +1,114 @@
+name: "Manual : Run client integration tests"
+on:
+  schedule:
+    - cron:  '0 12 * * *'
+  workflow_dispatch:
+jobs:
+  python-client-integration:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.10', '3.11']
+    steps:
+      - uses: actions/checkout@v2
+      - name: Build and launch DJ demo environment
+        uses: hoverkraft-tech/compose-action@v2.0.1
+        with:
+          compose-file: "./docker-compose.yml"
+          services: |
+            dj
+            postgres_metadata
+            db-migration
+            db-seed
+            djqs
+            djqs-db-migration
+            djrs-redis
+            djrs-worker
+            djrs-beat
+            dj-trino
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - uses: pdm-project/setup-pdm@v3
+        name: Setup PDM
+        with:
+          python-version: ${{ matrix.python-version }}
+          architecture: x64
+          prerelease: true
+          enable-pep582: true
+      - name: Install dependencies
+        run: |
+          pdm sync -d
+          cd ./datajunction-clients/python; pdm install -d -G pandas
+      - name: Python client integration tests
+        run: cd datajunction-clients/python && make test PYTEST_ARGS="--integration -k test_integration"
+
+  javascript-client-integration:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node-version: [18.x]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Build and launch DJ demo environment
+        uses: hoverkraft-tech/compose-action@v2.0.1
+        with:
+          compose-file: "./docker-compose.yml"
+          services: |
+            dj
+            postgres_metadata
+            db-migration
+            db-seed
+            djqs
+            djqs-db-migration
+            djrs-redis
+            djrs-worker
+            djrs-beat
+            dj-trino
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+      - name: Install Dev Dependencies
+        run: npm install --only=dev
+        working-directory: ./datajunction-clients/javascript
+      - name: Javascript client integration tests
+        run: npm test
+        working-directory: ./datajunction-clients/javascript
+
+  java-client-integration:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        java-version: [ 17 ]
+        distribution: [ 'temurin' ]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Build and launch DJ demo environment
+        uses: hoverkraft-tech/compose-action@v2.0.1
+        with:
+          compose-file: "./docker-compose.yml"
+          services: |
+            dj
+            postgres_metadata
+            db-migration
+            db-seed
+            djqs
+            djqs-db-migration
+            djrs-redis
+            djrs-worker
+            djrs-beat
+            dj-trino
+      - name: Set up JDK ${{ matrix.java-version }}
+        uses: actions/setup-java@v3
+        with:
+          java-version: ${{ matrix.java-version }}
+          distribution: ${{ matrix.distribution }}
+      - name: Build with Gradle
+        uses: gradle/gradle-build-action@v3
+        with:
+          arguments: cleanTest test
+          build-root-directory: ./datajunction-clients/java
+

.github/workflows/create-release.yml

@@ -0,0 +1,121 @@
+name: "Create Version Bump PR"
+
+on:
+  schedule:
+    - cron: '0 3 * * MON'  # Each Monday at 3am UTC
+  workflow_dispatch:
+    inputs:
+      bump:
+        type: choice
+        description: "Version bump type"
+        required: true
+        default: patch
+        options:
+          - patch
+          - minor
+          - major
+
+jobs:
+  create-pr:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install Hatch
+        run: |
+          pip install hatch 'virtualenv!=21.0.0'
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+
+      - name: Calculate next version
+        id: version
+        working-directory: ./datajunction-server
+        run: |
+          CURRENT_VERSION=$(hatch version)
+          echo "Current version: $CURRENT_VERSION"
+
+          BUMP_TYPE="${{ github.event.inputs.bump || 'patch' }}"
+          echo "Bump type: $BUMP_TYPE"
+
+          # Parse current version (handles X.Y.Z and X.Y.ZaN formats)
+          IFS='.' read -r MAJOR MINOR PATCH <<< "${CURRENT_VERSION%%a*}"
+
+          # Calculate new version
+          case $BUMP_TYPE in
+            major) NEW_VERSION="$((MAJOR + 1)).0.0" ;;
+            minor) NEW_VERSION="${MAJOR}.$((MINOR + 1)).0" ;;
+            patch) NEW_VERSION="${MAJOR}.${MINOR}.$((PATCH + 1))" ;;
+          esac
+
+          echo "New version: $NEW_VERSION"
+          echo "NEW_VERSION=$NEW_VERSION" >> $GITHUB_OUTPUT
+
+      - name: Configure Git
+        run: |
+          git config user.name "GitHub Actions Bot"
+          git config user.email "actions@github.com"
+
+      - name: Create release branch and update versions
+        run: |
+          VERSION="${{ steps.version.outputs.NEW_VERSION }}"
+          BRANCH="release/v$VERSION"
+
+          git checkout -b $BRANCH
+
+          # Update Python packages
+          cd datajunction-server && hatch version $VERSION && cd ..
+          cd datajunction-query && hatch version $VERSION && cd ..
+          cd datajunction-reflection && hatch version $VERSION && cd ..
+          cd datajunction-clients/python && hatch version $VERSION && cd ../..
+
+          # Update JavaScript packages
+          cd datajunction-ui && npm version $VERSION --no-git-tag-version && cd ..
+          cd datajunction-clients/javascript && npm version $VERSION --no-git-tag-version && cd ../..
+
+          git add -A
+          git commit -m "Bump version to v$VERSION"
+          git push origin $BRANCH
+
+      - name: Create Pull Request
+        run: |
+          VERSION="${{ steps.version.outputs.NEW_VERSION }}"
+          gh pr create \
+            --title "Release v$VERSION" \
+            --body "## Release v$VERSION
+
+          This PR bumps all package versions to v$VERSION.
+
+          **When merged**, packages will be automatically published to:
+          - PyPI: datajunction-server, datajunction-query, datajunction-reflection, datajunction
+          - npm: datajunction, datajunction-ui
+
+          A GitHub Release will also be created with auto-generated release notes.
+
+          ---
+          *Auto-generated by Create Version Bump workflow*" \
+            --base main \
+            --head "release/v$VERSION"
+        env:
+          GITHUB_TOKEN: ${{ secrets.REPO_SCOPED_TOKEN }}
+
+      - name: Summary
+        run: |
+          VERSION="${{ steps.version.outputs.NEW_VERSION }}"
+          echo "## Version Bump PR Created" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "**Version:** v$VERSION" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "Review the PR and merge when ready to publish." >> $GITHUB_STEP_SUMMARY

.github/workflows/generate-openapi-client.yml

@@ -0,0 +1,65 @@
+name: "Manual : Generate OpenAPI client"
+on:
+  workflow_dispatch:
+jobs:
+  generate-python-client:
+    env:
+      PDM_DEPS: 'urllib3<2'
+    runs-on: ubuntu-latest
+    name: Generate Python Client
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Install DJ
+        run: |
+          python -m pip install --upgrade pip
+          pip install .
+
+      - name: Generate OpenAPI Spec
+        run: ./scripts/generate-openapi.py -o openapi.json
+
+      - name: Configure Git
+        run: |
+          git config user.name "GitHub Actions Bot"
+          git config user.email "<>"
+
+      - name: Commit OpenAPI Spec
+        run: |
+          git add openapi.json
+          git commit -m "Updating OpenAPI Spec"
+
+      - name: Generate Python client
+        uses: openapi-generators/openapitools-generator-action@v1.4.0
+        with:
+          generator: python
+          openapi-file: openapi.json
+          config-file: ./.github/files/python-client-gen.yml
+          command-args: --skip-validate-spec
+
+      - name: Move client to right directory
+        run: |
+          mkdir -p ./openapi/python
+          cp -r python-client/* ./openapi/python/
+          rm -rf python-client
+
+      - name: Set short sha
+        id: sha
+        run: echo "short_sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+
+      - name: Commit generated files
+        run: |
+          git add openapi/python/
+          git commit -m "Update DJ Python client"
+          git checkout -b ci-pr/python-client-${{ steps.sha.outputs.short_sha }}
+          git push --set-upstream origin ci-pr/python-client-${{ steps.sha.outputs.short_sha }}
+
+      - name: Open a PR
+        run: gh pr create -B main -H ci-pr/python-client-${{ steps.sha.outputs.short_sha }} --title 'Update Python Client - ${{ steps.sha.outputs.short_sha }}' --body '(This PR was generated by a GitHub action)'
+        env:
+          GITHUB_TOKEN: ${{ secrets.REPO_SCOPED_TOKEN }}