Aggify 1.0: async-first MongoDB ODM rewrite (no MongoEngine)

.github/workflows/docs.yml

@@ -0,0 +1,18 @@
+name: Docs
+
+on:
+  push:
+    branches: ["main"]
+
+permissions:
+  contents: write
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with: { python-version: "3.12" }
+      - run: pip install -e . mkdocs mkdocs-material "mkdocstrings[python]"
+      - run: mkdocs gh-deploy --force

.github/workflows/python-app.yml

@@ -1,53 +1,70 @@
-# This workflow will install Python dependencies, run tests and lint with a single version of Python
-# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
-
-name: Python application
+name: CI
 
 on:
   push:
-    branches: [ "main" ]
+    branches: ["main"]
   pull_request:
-    branches: [ "main" ]
+    branches: ["main"]
 
 permissions:
   contents: read
 
 jobs:
-  test:
+  lint:
     runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with: { python-version: "3.12", cache: pip }
+      - run: pip install -e . ruff mypy "pydantic>=2" fastapi "django>=4.2" grpcio
+      - run: ruff check aggify tests
+      - run: mypy aggify
 
+  test:
+    runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
-
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+        mongo: ["6.0", "7.0", "8.0"]
+        exclude:
+          - { python-version: "3.10", mongo: "6.0" }
+          - { python-version: "3.11", mongo: "6.0" }
+          - { python-version: "3.9", mongo: "8.0" }
+          - { python-version: "3.10", mongo: "8.0" }
     steps:
-      - name: Checkout Repo
-        uses: actions/checkout@v3
-
-      - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install flake8 pytest pytest-cov
-          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-
-      - name: Lint with flake8
-        run: |
-          # stop the build if there are Python syntax errors or undefined names
-          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-      - name: Test with pytest
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with: { python-version: "${{ matrix.python-version }}", cache: pip }
+      - name: Start MongoDB ${{ matrix.mongo }} (single-node replica set)
         run: |
-          pytest --cov=./ --cov-report=xml
-
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v2
+          docker run -d --name mongo -p 27017:27017 mongo:${{ matrix.mongo }} --replSet rs0 --bind_ip_all
+          for i in $(seq 1 30); do
+            docker exec mongo mongosh --quiet --eval 'db.runCommand({ping:1}).ok' >/dev/null 2>&1 && break
+            sleep 2
+          done
+          docker exec mongo mongosh --quiet --eval 'rs.initiate({_id:"rs0",members:[{_id:0,host:"localhost:27017"}]})'
+          for i in $(seq 1 30); do
+            docker exec mongo mongosh --quiet --eval 'db.hello().isWritablePrimary' | grep -q true && break
+            sleep 2
+          done
+      - name: Install
+        run: pip install -e . pytest pytest-asyncio pytest-cov hypothesis httpx "pydantic>=2" fastapi grpcio protobuf "django>=4.2"
+      - name: Test
+        env:
+          AGGIFY_TEST_MONGO_URI: mongodb://localhost:27017/?directConnection=true
+        run: pytest --cov=aggify --cov-report=xml
+      - uses: codecov/codecov-action@v4
+        if: matrix.python-version == '3.12' && matrix.mongo == '7.0'
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          fail_ci_if_error: true
+          files: coverage.xml
+
+  docs:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with: { python-version: "3.12", cache: pip }
+      - run: pip install -e . mkdocs mkdocs-material "mkdocstrings[python]"
+      - run: mkdocs build --strict

.gitignore

@@ -304,4 +304,11 @@ pyrightconfig.json
 .history
 .ionide
 test.py
-models.py
+models.py
+# local virtualenvs / tooling
+.venv*/
+.hypothesis/
+
+# test models are source code
+!tests/models.py
+!docs/examples/shop_models.py

CHANGELOG.md

@@ -0,0 +1,36 @@
+# Changelog
+
+## 1.0.0 - 2026-10-17 - complete rewrite
+
+Aggify is now a standalone, async-first MongoDB ODM with a Django-style API. **MongoEngine is no longer used or required.**
+
+### Added
+- `Document` / `EmbeddedDocument` with typed fields (strict validation, `Decimal128`, tz-aware UTC datetimes, enums,
+  references, GeoJSON points, ...), `Meta` options, abstract bases, managers, signals, indexes.
+- `QuerySet` / `Manager`: lazy, immutable, Django-style (`filter/exclude/order_by/values/annotate/aggregate/select_related/
+  prefetch_related/only/defer/...`), automatic joins across references, `Q` and `F`, 25 lookups, update operators and
+  expression updates, bulk operations. Every I/O method has an async twin (`aget`, `alist`, `asave`, ...).
+- Native async via pymongo's `AsyncMongoClient` (one client per event loop), sync API on `MongoClient`.
+- Change tracking: `save()` sends only changed fields; optional optimistic locking.
+- Transactions: `transaction()` / `atransaction()` (ContextVar based, nestable).
+- Expression library (arithmetic, strings, dates, arrays, `Case/When`, `Filter/Map/Reduce`, aggregates, windows).
+- Rebuilt `Aggify` pipeline builder: immutable, schema-aware, all common stages (`$facet`, `$bucket`,
+  `$setWindowFields`, `$graphLookup`, `$unionWith`, `$merge`, ...), sync + async execution.
+- Integrations: Pydantic schema generation, FastAPI helpers, gRPC (`grpc.aio`) helpers, Django app (settings,
+  shortcuts, forms, middleware, `aggify_indexes`, `ObjectIdField` for Django ORM models).
+- Testing helpers: `capture_commands`, `assert_num_queries`, `assert_pipeline`, `temporary_database`.
+- Documentation site (mkdocs) with a cookbook of complex queries; Persian overview.
+
+### Security
+- Operator injection prevented by field typing and recursive value scanning; `Raw()` is the only escape hatch.
+- Regex lookups escape input (no regex injection / ReDoS); explicit `regex` is length-capped.
+- Server-side JavaScript (`$where`, `$function`, `$accumulator`) blocked unless enabled.
+- Validated field names/dictionary keys; strict constructors (mass assignment); resource limits
+  (`max_limit`, `max_in_list`, `default_max_time_ms`); credentials masked in `repr`.
+
+### Fixed (compared with 0.x)
+- `$subtract` / `$divide` with 3 operands, mutation of shared `F` / `Q` objects, `$unwind` using the wrong path,
+  silently dropped `$match` conditions, magic `let` substitution, `[n]` meaning `limit n`.
+
+### Breaking
+See the [upgrade guide](docs/migration/aggify-0.md).

CONTRIBUTING.md

@@ -32,6 +32,18 @@ git push origin feature-or-bugfix-name
 
 10 - **Merge:** Once your pull request is approved, it will be merged into the main repository.
 
+## Development setup
+```shell
+make install     # editable install + test/doc dependencies
+make mongo       # single-node replica set in Docker on :27018 (transactions need a replica set)
+make test        # pytest (unit + integration + framework tests)
+make lint typecheck docs
+```
+Tests marked `mongo` skip themselves when no server is reachable; set `AGGIFY_TEST_MONGO_URI` to use your own.
+A new lookup, field or stage needs: a unit test of the compiled output, an integration test against MongoDB and
+(if user-facing) a docs entry. Anything touching query construction must keep `tests/integration/test_differential.py`
+and `tests/test_security.py` green.
+
 ## Guidelines
 * Follow Python PEP 8 style guidelines for code formatting.
 * Write clear and concise code comments and docstrings.

Makefile

@@ -1,20 +1,53 @@
-# Makefile for aggify
+PYTHON ?= python
+MONGO_PORT ?= 27018
 
-# Environment variables
-PYTHON = python
+.PHONY: help install mongo mongo-stop test test-all lint typecheck docs docs-serve bench build clean publish
 
-.PHONY: init build clean
+help:
+	@grep -E '^[a-z-]+:' Makefile | cut -d: -f1 | tr '\n' ' '; echo
 
-init:
-	$(PYTHON) -m pip install build twine
+install:
+	$(PYTHON) -m pip install -e . pytest pytest-asyncio pytest-cov hypothesis httpx ruff mypy "pydantic>=2" fastapi grpcio protobuf "django>=4.2" mkdocs mkdocs-material "mkdocstrings[python]"
+
+mongo:        ## throw-away single-node replica set on :$(MONGO_PORT) (needed for the transaction tests)
+	-docker rm -f aggify-test-mongo >/dev/null 2>&1
+	docker run -d --name aggify-test-mongo -p $(MONGO_PORT):27017 mongo:7 --replSet rs0 --bind_ip_all
+	@sleep 4
+	docker exec aggify-test-mongo mongosh --quiet --eval 'rs.initiate({_id:"rs0",members:[{_id:0,host:"localhost:27017"}]})'
+
+mongo-stop:
+	-docker rm -f aggify-test-mongo
+
+test:
+	AGGIFY_TEST_MONGO_URI="mongodb://localhost:$(MONGO_PORT)/?directConnection=true" $(PYTHON) -m pytest
+
+cov:
+	AGGIFY_TEST_MONGO_URI="mongodb://localhost:$(MONGO_PORT)/?directConnection=true" $(PYTHON) -m pytest --cov=aggify --cov-report=term-missing
+
+test-all:
+	tox
+
+lint:
+	ruff check aggify tests
+
+typecheck:
+	mypy aggify
+
+docs:
+	mkdocs build --strict
+
+docs-serve:
+	mkdocs serve
+
+bench:
+	$(PYTHON) benchmarks/bench.py "mongodb://localhost:$(MONGO_PORT)/?directConnection=true"
 
 build:
 	$(PYTHON) -m build
 
 clean:
-	rm -rf dist/
+	rm -rf dist build site .pytest_cache .mypy_cache .ruff_cache
 
-# Publish to PyPI using twine
-publish:
+publish: clean build
 	twine check dist/*
 	twine upload dist/*