Draft: Visibility sync since v1.3.0 (existing PRs + Enveritas fixes/features)

.github/workflows/ci.yml

@@ -15,7 +15,7 @@ jobs:
         go-version: '1.23.5'
     - uses: sqlc-dev/setup-sqlc@v4
       with:
-        sqlc-version: '1.28.0'
+        sqlc-version: '1.30.0'
     - run: make
     - run: make test
     - run: sqlc diff

README.md

@@ -20,6 +20,29 @@ sql:
           emit_async_querier: true
 ```
 
+### Configuration Options
+
+These are the supported `options` for the `py` plugin. Add them under the `codegen[].options` section of your `sqlc.yaml`.
+
+- package: Module path used for imports in generated query files (e.g., `from <package> import models`).
+- emit_sync_querier: Emit a synchronous `Querier` class using `sqlalchemy.engine.Connection`.
+- emit_async_querier: Emit an asynchronous `AsyncQuerier` class using `sqlalchemy.ext.asyncio.AsyncConnection`.
+- emit_pydantic_models: Emit Pydantic models instead of `dataclasses` for models.py. See the section below.
+- emit_str_enum: Emit enums as `enum.StrEnum` (Python >=3.11). When false, emit `(str, enum.Enum)`.  See the section below.
+- emit_schema_name_prefix: When true, prefix non-default schema to generated types to avoid name collisions. Examples:
+  - false (default): `Book`, `BookStatus`
+  - true: `MySchemaBook`, `MySchemaBookStatus` when the objects live in schema `my_schema`.
+- emit_exact_table_names: When true, do not singularize table names for model class names.
+- query_parameter_limit: Integer controlling when query params are grouped into a single struct argument.
+  - If the number of parameters exceeds this value, a single `Params` struct is emitted.
+  - Set to 0 to always emit a struct; omit or set to a large value to keep separate parameters.
+- inflection_exclude_table_names: A list of table names to exclude from singularization when `emit_exact_table_names` is false.
+- overrides: Column type overrides; see the section below.
+
+Notes
+- out: Controlled by `codegen[].out` at the sqlc level. The plugin’s `out` option is not used; prefer the top-level `out` value.
+
+
 ### Emit Pydantic Models instead of `dataclasses`
 
 Option: `emit_pydantic_models`
@@ -76,3 +99,51 @@ class Status(str, enum.Enum):
     OPEN = "op!en"
     CLOSED = "clo@sed"
 ```
+
+### Override Column Types
+
+Option: `overrides`
+
+You can override the SQL to Python type mapping for specific columns or database types using the `overrides` option. This is useful for columns with JSON data or other custom types.
+
+Example configuration:
+
+```yaml
+options:
+  package: authors
+  emit_pydantic_models: true
+  overrides:
+    - column: "some_table.payload"
+      py_import: "my_lib.models"
+      py_type: "Payload"
+    - db_type: "jsonb"
+      py_import: "my_lib.models"
+      py_type: "Payload"
+```
+
+This will:
+1. Override the column `payload` in `some_table` to use the type `Payload`
+2. Override any column with the database type `jsonb` to use the type `Payload`
+3. Add an import for `my_lib.models` to the models file
+
+Example output:
+
+```python
+# Code generated by sqlc. DO NOT EDIT.
+# versions:
+#   sqlc v1.30.0
+
+import datetime
+import pydantic
+from typing import Any
+
+import my_lib.models
+
+
+class SomeTable(pydantic.BaseModel):
+    id: int
+    created_at: datetime.datetime
+    payload: my_lib.models.Payload
+```
+
+This is similar to the [overrides functionality in the Go version of sqlc](https://docs.sqlc.dev/en/stable/howto/overrides.html#overriding-types).

examples/src/authors/models.py

@@ -1,6 +1,6 @@
 # Code generated by sqlc. DO NOT EDIT.
 # versions:
-#   sqlc v1.28.0
+#   sqlc v1.30.0
 import dataclasses
 from typing import Optional
 

examples/src/authors/query.py

@@ -1,6 +1,6 @@
 # Code generated by sqlc. DO NOT EDIT.
 # versions:
-#   sqlc v1.28.0
+#   sqlc v1.30.0
 # source: query.sql
 from typing import AsyncIterator, Iterator, Optional
 

examples/src/booktest/models.py

@@ -1,6 +1,6 @@
 # Code generated by sqlc. DO NOT EDIT.
 # versions:
-#   sqlc v1.28.0
+#   sqlc v1.30.0
 import dataclasses
 import datetime
 import enum

examples/src/booktest/query.py

@@ -1,6 +1,6 @@
 # Code generated by sqlc. DO NOT EDIT.
 # versions:
-#   sqlc v1.28.0
+#   sqlc v1.30.0
 # source: query.sql
 import dataclasses
 import datetime

examples/src/jets/models.py

@@ -1,6 +1,6 @@
 # Code generated by sqlc. DO NOT EDIT.
 # versions:
-#   sqlc v1.28.0
+#   sqlc v1.30.0
 import dataclasses
 
 

examples/src/jets/query-building.py

@@ -1,6 +1,6 @@
 # Code generated by sqlc. DO NOT EDIT.
 # versions:
-#   sqlc v1.28.0
+#   sqlc v1.30.0
 # source: query-building.sql
 from typing import AsyncIterator, Optional
 

examples/src/ondeck/city.py

@@ -1,6 +1,6 @@
 # Code generated by sqlc. DO NOT EDIT.
 # versions:
-#   sqlc v1.28.0
+#   sqlc v1.30.0
 # source: city.sql
 from typing import AsyncIterator, Optional
 

examples/src/ondeck/models.py

@@ -1,6 +1,6 @@
 # Code generated by sqlc. DO NOT EDIT.
 # versions:
-#   sqlc v1.28.0
+#   sqlc v1.30.0
 import dataclasses
 import datetime
 import enum

examples/src/ondeck/venue.py

@@ -1,6 +1,6 @@
 # Code generated by sqlc. DO NOT EDIT.
 # versions:
-#   sqlc v1.28.0
+#   sqlc v1.30.0
 # source: venue.sql
 import dataclasses
 from typing import AsyncIterator, List, Optional

internal/config.go

@@ -1,13 +1,45 @@
 package python
 
+import (
+	"bytes"
+	json "encoding/json"
+	"fmt"
+	"strings"
+)
+
+type OverrideColumn struct {
+	Column   string `json:"column"`
+	DbType   string `json:"db_type"`
+	PyType   string `json:"py_type"`
+	PyImport string `json:"py_import"`
+}
+
 type Config struct {
-	EmitExactTableNames         bool     `json:"emit_exact_table_names"`
-	EmitSyncQuerier             bool     `json:"emit_sync_querier"`
-	EmitAsyncQuerier            bool     `json:"emit_async_querier"`
-	Package                     string   `json:"package"`
-	Out                         string   `json:"out"`
-	EmitPydanticModels          bool     `json:"emit_pydantic_models"`
-	EmitStrEnum                 bool     `json:"emit_str_enum"`
-	QueryParameterLimit         *int32   `json:"query_parameter_limit"`
-	InflectionExcludeTableNames []string `json:"inflection_exclude_table_names"`
+	EmitExactTableNames         bool             `json:"emit_exact_table_names"`
+	EmitSyncQuerier             bool             `json:"emit_sync_querier"`
+	EmitAsyncQuerier            bool             `json:"emit_async_querier"`
+	Package                     string           `json:"package"`
+	Out                         string           `json:"out"`
+	EmitPydanticModels          bool             `json:"emit_pydantic_models"`
+	EmitStrEnum                 bool             `json:"emit_str_enum"`
+	EmitSchemaNamePrefix        bool             `json:"emit_schema_name_prefix"`
+	QueryParameterLimit         *int32           `json:"query_parameter_limit"`
+	InflectionExcludeTableNames []string         `json:"inflection_exclude_table_names"`
+	Overrides                   []OverrideColumn `json:"overrides"`
+}
+
+func parseConfig(raw []byte) (Config, error) {
+	var conf Config
+	if len(raw) == 0 {
+		return conf, nil
+	}
+
+	dec := json.NewDecoder(bytes.NewReader(raw))
+	dec.DisallowUnknownFields()
+	if err := dec.Decode(&conf); err != nil {
+		msg := strings.TrimPrefix(err.Error(), "json: ")
+		return Config{}, fmt.Errorf("invalid plugin options: %s", msg)
+	}
+
+	return conf, nil
 }

internal/config_test.go

@@ -0,0 +1,32 @@
+package python
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestParseConfigDisallowUnknownFields(t *testing.T) {
+	_, err := parseConfig([]byte(`{"emit_sync_querier":true,"db_typ":"jsonb"}`))
+	if err == nil {
+		t.Fatal("expected unknown field error, got nil")
+	}
+	if !strings.Contains(err.Error(), "invalid plugin options") {
+		t.Fatalf("expected error to reference plugin options, got: %v", err)
+	}
+	if !strings.Contains(err.Error(), `unknown field "db_typ"`) {
+		t.Fatalf("expected unknown field in error, got: %v", err)
+	}
+}
+
+func TestParseConfigValid(t *testing.T) {
+	conf, err := parseConfig([]byte(`{"emit_sync_querier":true,"overrides":[{"db_type":"jsonb","py_type":"str"}]}`))
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if !conf.EmitSyncQuerier {
+		t.Fatal("expected emit_sync_querier to be true")
+	}
+	if len(conf.Overrides) != 1 || conf.Overrides[0].DbType != "jsonb" || conf.Overrides[0].PyType != "str" {
+		t.Fatal("unexpected parsed overrides")
+	}
+}

internal/endtoend/endtoend_test.go

@@ -101,7 +101,7 @@ func TestGenerate(t *testing.T) {
 			cmd.Dir = dir
 			got, err := cmd.CombinedOutput()
 			if diff := cmp.Diff(string(want), string(got)); diff != "" {
-				t.Errorf("sqlc diff mismatch (-want +got):\n%s", diff)
+				t.Errorf("sqlc diff mismatch (-want +got):\n%s", got)
 			}
 			if len(want) == 0 && err != nil {
 				t.Error(err)

internal/endtoend/testdata/dataclasses_with_reserved_keywords/db/models.py

@@ -0,0 +1,10 @@
+# Code generated by sqlc. DO NOT EDIT.
+# versions:
+#   sqlc v1.30.0
+import dataclasses
+
+
+@dataclasses.dataclass()
+class Author:
+    id: int
+    class_: str

internal/endtoend/testdata/dataclasses_with_reserved_keywords/db/query.py

@@ -0,0 +1,44 @@
+# Code generated by sqlc. DO NOT EDIT.
+# versions:
+#   sqlc v1.30.0
+# source: query.sql
+from typing import Optional
+
+import sqlalchemy
+import sqlalchemy.ext.asyncio
+
+from db import models
+
+
+GET_AUTHOR = """-- name: get_author \\:one
+SELECT id, class FROM authors
+WHERE id = :p1 LIMIT 1
+"""
+
+
+class Querier:
+    def __init__(self, conn: sqlalchemy.engine.Connection):
+        self._conn = conn
+
+    def get_author(self, *, id: int) -> Optional[models.Author]:
+        row = self._conn.execute(sqlalchemy.text(GET_AUTHOR), {"p1": id}).first()
+        if row is None:
+            return None
+        return models.Author(
+            id=row[0],
+            class_=row[1],
+        )
+
+
+class AsyncQuerier:
+    def __init__(self, conn: sqlalchemy.ext.asyncio.AsyncConnection):
+        self._conn = conn
+
+    async def get_author(self, *, id: int) -> Optional[models.Author]:
+        row = (await self._conn.execute(sqlalchemy.text(GET_AUTHOR), {"p1": id})).first()
+        if row is None:
+            return None
+        return models.Author(
+            id=row[0],
+            class_=row[1],
+        )

internal/endtoend/testdata/dataclasses_with_reserved_keywords/query.sql

@@ -0,0 +1,3 @@
+-- name: GetAuthor :one
+SELECT * FROM authors
+WHERE id = $1 LIMIT 1;

internal/endtoend/testdata/dataclasses_with_reserved_keywords/schema.sql

@@ -0,0 +1,4 @@
+CREATE TABLE authors (
+          id    BIGSERIAL PRIMARY KEY,
+          class text      NOT NULL
+);

internal/endtoend/testdata/dataclasses_with_reserved_keywords/sqlc.yaml

@@ -0,0 +1,18 @@
+version: "2"
+plugins:
+  - name: py
+    wasm:
+      url: file://../../../../bin/sqlc-gen-python.wasm
+      sha256: "a5d1ad0ead1ecadb0608684d7c7a04327762299b769e6cca9c8f838c5df89788"
+sql:
+  - schema: schema.sql
+    queries: query.sql
+    engine: postgresql
+    codegen:
+      - plugin: py
+        out: db
+        options:
+          package: db
+          emit_sync_querier: true
+          emit_async_querier: true
+          emit_pydantic_models: false

internal/endtoend/testdata/emit_pydantic_models/db/models.py

@@ -1,11 +1,15 @@
 # Code generated by sqlc. DO NOT EDIT.
 # versions:
-#   sqlc v1.28.0
+#   sqlc v1.30.0
 import pydantic
 from typing import Optional
 
 
 class Author(pydantic.BaseModel):
+    model_config = pydantic.ConfigDict(
+        validate_by_alias=True,
+        validate_by_name=True,
+    )
     id: int
     name: str
     bio: Optional[str]

internal/endtoend/testdata/emit_pydantic_models/db/query.py

@@ -1,6 +1,6 @@
 # Code generated by sqlc. DO NOT EDIT.
 # versions:
-#   sqlc v1.28.0
+#   sqlc v1.30.0
 # source: query.sql
 from typing import AsyncIterator, Iterator, Optional
 

internal/endtoend/testdata/emit_pydantic_models/sqlc.yaml

@@ -3,7 +3,7 @@ plugins:
 - name: py
   wasm:
     url: file://../../../../bin/sqlc-gen-python.wasm
-    sha256: "d6846ffad948181e611e883cedd2d2be66e091edc1273a0abc6c9da18399e0ca"
+    sha256: "a5d1ad0ead1ecadb0608684d7c7a04327762299b769e6cca9c8f838c5df89788"
 sql:
 - schema: schema.sql
   queries: query.sql

internal/endtoend/testdata/emit_pydantic_models_with_reserved_keywords/db/models.py

@@ -0,0 +1,15 @@
+# Code generated by sqlc. DO NOT EDIT.
+# versions:
+#   sqlc v1.30.0
+import pydantic
+
+
+class Author(pydantic.BaseModel):
+    model_config = pydantic.ConfigDict(
+        validate_by_alias=True,
+        validate_by_name=True,
+    )
+    id: int
+    class_: str = pydantic.Field(
+        alias="class",
+    )