Attach availability to materialization

.coveragerc

@@ -1,12 +1,15 @@
 # .coveragerc to control coverage.py
 [run]
 branch = True
-source = datajunction
-# omit = bad_file.py
+source = dj
+omit =
+    */dj/sql/parsing/backends/grammar/generated/*
+    */dj/sql/parsing/backends/antlr4.py
+    */dj/sql/parsing/ast.py
 
 [paths]
 source =
-    src/
+    dj/
     */site-packages/
 
 [report]
@@ -26,3 +29,5 @@ exclude_lines =
     # Don't complain if non-runnable code isn't run:
     if 0:
     if __name__ == .__main__.:
+
+    if TYPE_CHECKING:

.github/pull_request_template.md

@@ -0,0 +1,15 @@
+### Summary
+
+<!-- What's this change about? -->
+
+### Test Plan
+
+<!-- How did you test your change? -->
+
+- [ ] PR has an associated issue: #
+- [ ] `make check` passes
+- [ ] `make test` shows 100% unit test coverage
+
+### Deployment Plan
+
+<!-- Any special instructions around deployment? -->

.github/workflows/client-integration-tests.yml

@@ -0,0 +1,114 @@
+name: "Manual : Run client integration tests"
+on:
+  schedule:
+    - cron:  '0 12 * * *'
+  workflow_dispatch:
+jobs:
+  python-client-integration:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.10', '3.11']
+    steps:
+      - uses: actions/checkout@v2
+      - name: Build and launch DJ demo environment
+        uses: hoverkraft-tech/compose-action@v2.0.1
+        with:
+          compose-file: "./docker-compose.yml"
+          services: |
+            dj
+            postgres_metadata
+            db-migration
+            db-seed
+            djqs
+            djqs-db-migration
+            djrs-redis
+            djrs-worker
+            djrs-beat
+            dj-trino
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - uses: pdm-project/setup-pdm@v3
+        name: Setup PDM
+        with:
+          python-version: ${{ matrix.python-version }}
+          architecture: x64
+          prerelease: true
+          enable-pep582: true
+      - name: Install dependencies
+        run: |
+          pdm sync -d
+          cd ./datajunction-clients/python; pdm install -d -G pandas
+      - name: Python client integration tests
+        run: cd datajunction-clients/python && make test PYTEST_ARGS="--integration -k test_integration"
+
+  javascript-client-integration:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node-version: [18.x]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Build and launch DJ demo environment
+        uses: hoverkraft-tech/compose-action@v2.0.1
+        with:
+          compose-file: "./docker-compose.yml"
+          services: |
+            dj
+            postgres_metadata
+            db-migration
+            db-seed
+            djqs
+            djqs-db-migration
+            djrs-redis
+            djrs-worker
+            djrs-beat
+            dj-trino
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+      - name: Install Dev Dependencies
+        run: npm install --only=dev
+        working-directory: ./datajunction-clients/javascript
+      - name: Javascript client integration tests
+        run: npm test
+        working-directory: ./datajunction-clients/javascript
+
+  java-client-integration:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        java-version: [ 17 ]
+        distribution: [ 'temurin' ]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Build and launch DJ demo environment
+        uses: hoverkraft-tech/compose-action@v2.0.1
+        with:
+          compose-file: "./docker-compose.yml"
+          services: |
+            dj
+            postgres_metadata
+            db-migration
+            db-seed
+            djqs
+            djqs-db-migration
+            djrs-redis
+            djrs-worker
+            djrs-beat
+            dj-trino
+      - name: Set up JDK ${{ matrix.java-version }}
+        uses: actions/setup-java@v3
+        with:
+          java-version: ${{ matrix.java-version }}
+          distribution: ${{ matrix.distribution }}
+      - name: Build with Gradle
+        uses: gradle/gradle-build-action@v3
+        with:
+          arguments: cleanTest test
+          build-root-directory: ./datajunction-clients/java
+

.github/workflows/generate-openapi-client.yml

@@ -0,0 +1,65 @@
+name: "Manual : Generate OpenAPI client"
+on:
+  workflow_dispatch:
+jobs:
+  generate-python-client:
+    env:
+      PDM_DEPS: 'urllib3<2'
+    runs-on: ubuntu-latest
+    name: Generate Python Client
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Install DJ
+        run: |
+          python -m pip install --upgrade pip
+          pip install .
+
+      - name: Generate OpenAPI Spec
+        run: ./scripts/generate-openapi.py -o openapi.json
+
+      - name: Configure Git
+        run: |
+          git config user.name "GitHub Actions Bot"
+          git config user.email "<>"
+
+      - name: Commit OpenAPI Spec
+        run: |
+          git add openapi.json
+          git commit -m "Updating OpenAPI Spec"
+
+      - name: Generate Python client
+        uses: openapi-generators/openapitools-generator-action@v1.4.0
+        with:
+          generator: python
+          openapi-file: openapi.json
+          config-file: ./.github/files/python-client-gen.yml
+          command-args: --skip-validate-spec
+
+      - name: Move client to right directory
+        run: |
+          mkdir -p ./openapi/python
+          cp -r python-client/* ./openapi/python/
+          rm -rf python-client
+
+      - name: Set short sha
+        id: sha
+        run: echo "short_sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+
+      - name: Commit generated files
+        run: |
+          git add openapi/python/
+          git commit -m "Update DJ Python client"
+          git checkout -b ci-pr/python-client-${{ steps.sha.outputs.short_sha }}
+          git push --set-upstream origin ci-pr/python-client-${{ steps.sha.outputs.short_sha }}
+
+      - name: Open a PR
+        run: gh pr create -B main -H ci-pr/python-client-${{ steps.sha.outputs.short_sha }} --title 'Update Python Client - ${{ steps.sha.outputs.short_sha }}' --body '(This PR was generated by a GitHub action)'
+        env:
+          GITHUB_TOKEN: ${{ secrets.REPO_SCOPED_TOKEN }}

.github/workflows/generate-openapi-spec.yml

@@ -0,0 +1,61 @@
+name: "Manual : Generate OpenAPI spec"
+on:
+  workflow_dispatch:
+jobs:
+  generate-openapi-spec:
+    env:
+      PDM_DEPS: 'urllib3<2'
+    defaults:
+      run:
+        working-directory: ./datajunction-server
+    runs-on: ubuntu-latest
+    name: OpenAPI Spec
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Set up Node
+        uses: actions/setup-node@v3
+        with:
+          node-version: 19
+
+      - name: Install DJ
+        run: |
+          python -m pip install --upgrade pip
+          pip install .
+
+      - name: Generate OpenAPI Spec
+        run: |
+          ./scripts/generate-openapi.py -o ../openapi.json
+
+      - name: Generate Markdown Docs from Spec
+        run: |
+          npm install -g widdershins
+          widdershins ../openapi.json -o ../docs/content/0.1.0/docs/developers/the-datajunction-api-specification.md --code=true --omitBody=true --summary=true
+
+      - name: Configure Git
+        run: |
+          git config user.name "GitHub Actions Bot"
+          git config user.email "<>"
+
+      - name: Set short sha
+        id: sha
+        run: echo "short_sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+
+      - name: Commit OpenAPI Spec
+        run: |
+          git add ../openapi.json
+          git add ../docs/content/0.1.0/docs/developers/the-datajunction-api-specification.md
+          git commit -m "Updating OpenAPI Spec"
+          git checkout -b ci-pr/python-client-${{ steps.sha.outputs.short_sha }}
+          git push --set-upstream origin ci-pr/python-client-${{ steps.sha.outputs.short_sha }}
+
+      - name: Open a PR
+        run: gh pr create -B main -H ci-pr/python-client-${{ steps.sha.outputs.short_sha }} --title 'Update OpenAPI Spec - ${{ steps.sha.outputs.short_sha }}' --body '(This PR was generated by a GitHub action)'
+        env:
+          GITHUB_TOKEN: ${{ secrets.REPO_SCOPED_TOKEN }}

.github/workflows/python-package.yml → .github/workflows/python-package-daily.yml

@@ -1,33 +1,32 @@
 # This workflow will install Python dependencies, run tests and lint with a variety of Python versions
 # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
 
-name: Python package
+name: "@Daily : Run tests with latest dependencies"
 
 on:
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
+  schedule:
+    - cron:  '0 6 * * *'
 
 jobs:
-  build:
+  daily:
 
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.8, 3.9, '3.10']
+        python-version: ['3.10', '3.11.0rc1']
 
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
+    - name: Install latest dependencies
       run: |
         python -m pip install --upgrade pip
         pip install -e '.[testing]'
     - name: Test with pytest
       run: |
-        pytest --cov=src/datajunction -vv tests/ --doctest-modules src/datajunction
+        pre-commit run --all-files
+        pytest --cov-fail-under=100 --cov=dj -vv tests/ --doctest-modules dj --without-integration --without-slow-integration