feat: Add SensitiveColumns to tables schema

examples/simple_plugin/go.mod

@@ -32,7 +32,7 @@ require (
 	github.com/bahlo/generic-list-go v0.2.0 // indirect
 	github.com/buger/jsonparser v1.1.1 // indirect
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
-	github.com/cloudquery/cloudquery-api-go v1.13.8 // indirect
+	github.com/cloudquery/cloudquery-api-go v1.13.9 // indirect
 	github.com/cloudquery/codegen v0.3.26 // indirect
 	github.com/cloudquery/plugin-pb-go v1.26.10 // indirect
 	github.com/cloudquery/plugin-sdk/v2 v2.7.0 // indirect

examples/simple_plugin/go.sum

@@ -50,8 +50,8 @@ github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMU
 github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
 github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
 github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
-github.com/cloudquery/cloudquery-api-go v1.13.8 h1:8n5D0G2wynbUdexr1GS8ND8i0uOwm0gXKNipJsijPe0=
-github.com/cloudquery/cloudquery-api-go v1.13.8/go.mod h1:ZhEjPkDGDL2KZKlQLUnsgQ0mPz3qC7qftr37q3q+IcA=
+github.com/cloudquery/cloudquery-api-go v1.13.9 h1:XudJusEJ0+kPa2/GXoZPuphR8gTRvHZ49dq53sTrWME=
+github.com/cloudquery/cloudquery-api-go v1.13.9/go.mod h1:ZhEjPkDGDL2KZKlQLUnsgQ0mPz3qC7qftr37q3q+IcA=
 github.com/cloudquery/codegen v0.3.26 h1:cWORVpObYW5/0LnjC0KO/Ocg1+vbZivJfFd+sMpb5ZY=
 github.com/cloudquery/codegen v0.3.26/go.mod h1:bg/M1JxFvNVABMLMFb/uAQmTGAyI2L/E4zL4kho9RFs=
 github.com/cloudquery/plugin-pb-go v1.26.10 h1:VNRk3JMLR7+pCXGCk4729I8r3vTrn64qonCs+4KY7+M=

go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/licensemanager v1.30.3
 	github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.29.0
 	github.com/bradleyjkemp/cupaloy/v2 v2.8.0
-	github.com/cloudquery/cloudquery-api-go v1.13.8
+	github.com/cloudquery/cloudquery-api-go v1.13.9
 	github.com/cloudquery/codegen v0.3.26
 	github.com/cloudquery/plugin-pb-go v1.26.10
 	github.com/cloudquery/plugin-sdk/v2 v2.7.0

go.sum

@@ -50,8 +50,8 @@ github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMU
 github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
 github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
 github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
-github.com/cloudquery/cloudquery-api-go v1.13.8 h1:8n5D0G2wynbUdexr1GS8ND8i0uOwm0gXKNipJsijPe0=
-github.com/cloudquery/cloudquery-api-go v1.13.8/go.mod h1:ZhEjPkDGDL2KZKlQLUnsgQ0mPz3qC7qftr37q3q+IcA=
+github.com/cloudquery/cloudquery-api-go v1.13.9 h1:XudJusEJ0+kPa2/GXoZPuphR8gTRvHZ49dq53sTrWME=
+github.com/cloudquery/cloudquery-api-go v1.13.9/go.mod h1:ZhEjPkDGDL2KZKlQLUnsgQ0mPz3qC7qftr37q3q+IcA=
 github.com/cloudquery/codegen v0.3.26 h1:cWORVpObYW5/0LnjC0KO/Ocg1+vbZivJfFd+sMpb5ZY=
 github.com/cloudquery/codegen v0.3.26/go.mod h1:bg/M1JxFvNVABMLMFb/uAQmTGAyI2L/E4zL4kho9RFs=
 github.com/cloudquery/jsonschema v0.0.0-20240220124159-92878faa2a66 h1:OZLPSIBYEfvkAUeOeM8CwTgVQy5zhayI99ishCrsFV0=

internal/memdb/memdb.go

@@ -59,6 +59,7 @@ func GetNewClient(options ...Option) plugin.NewClientFunc {
 					},
 				},
 				PermissionsNeeded: []string{"permission1"},
+				SensitiveColumns:  []string{"col1"},
 				Relations: schema.Tables{
 					{
 						Name: "table2",

plugin/testing_validation.go

@@ -14,5 +14,12 @@ func ValidateNoEmptyColumns(t *testing.T, tables schema.Tables, messages message
 		if len(emptyColumns) > 0 {
 			t.Fatalf("found empty column(s): %v in %s", emptyColumns, table.Name)
 		}
+		nonMatchingColumns, nonMatchingJSONColumns := schema.FindNotMatchingSensitiveColumns(table)
+		if len(nonMatchingColumns) > 0 {
+			t.Fatalf("found non-matching sensitive column(s): %v in %s", nonMatchingColumns, table.Name)
+		}
+		if len(nonMatchingJSONColumns) > 0 {
+			t.Fatalf("found non-matching sensitive JSON column(s): %v in %s", nonMatchingJSONColumns, table.Name)
+		}
 	}
 }

schema/arrow.go

@@ -27,6 +27,7 @@ const (
 	MetadataTableDependsOn         = "cq:table_depends_on"
 	MetadataTableIsPaid            = "cq:table_paid"
 	MetadataTablePermissionsNeeded = "cq:table_permissions_needed"
+	MetadataTableSensitiveColumns  = "cq:table_sensitive_columns"
 )
 
 type Schemas []*arrow.Schema

schema/table.go

@@ -66,6 +66,8 @@ type Table struct {
 	Description string `json:"description"`
 	// List of permissions needed to access this table, if any. For example ["Microsoft.Network/dnsZones/read"] or ["storage.buckets.list"]
 	PermissionsNeeded []string `json:"permissions_needed"`
+	// List of columns that may contain sensitive or secret data
+	SensitiveColumns []string `json:"sensitive_columns"`
 	// Columns are the set of fields that are part of this table
 	Columns ColumnList `json:"columns"`
 	// Relations are a set of related tables defines
@@ -188,6 +190,7 @@ func NewTableFromArrowSchema(sc *arrow.Schema) (*Table, error) {
 	title, _ := tableMD.GetValue(MetadataTableTitle)
 	dependsOn, _ := tableMD.GetValue(MetadataTableDependsOn)
 	permissionsNeeded, _ := tableMD.GetValue(MetadataTablePermissionsNeeded)
+	sensitiveColumns, _ := tableMD.GetValue(MetadataTableSensitiveColumns)
 	var parent *Table
 	if dependsOn != "" {
 		parent = &Table{Name: dependsOn}
@@ -200,6 +203,8 @@ func NewTableFromArrowSchema(sc *arrow.Schema) (*Table, error) {
 
 	var permissionsNeededArr []string
 	_ = json.Unmarshal([]byte(permissionsNeeded), &permissionsNeededArr)
+	var sensitiveColumnsArr []string
+	_ = json.Unmarshal([]byte(sensitiveColumns), &sensitiveColumnsArr)
 	table := &Table{
 		Name:              name,
 		Description:       description,
@@ -208,6 +213,7 @@ func NewTableFromArrowSchema(sc *arrow.Schema) (*Table, error) {
 		Title:             title,
 		Parent:            parent,
 		PermissionsNeeded: permissionsNeededArr,
+		SensitiveColumns:  sensitiveColumnsArr,
 	}
 	if isIncremental, found := tableMD.GetValue(MetadataIncremental); found {
 		table.IsIncremental = isIncremental == MetadataTrue
@@ -493,6 +499,8 @@ func (t *Table) ToArrowSchema() *arrow.Schema {
 	}
 	asJSON, _ := json.Marshal(t.PermissionsNeeded)
 	md[MetadataTablePermissionsNeeded] = string(asJSON)
+	asJSON, _ = json.Marshal(t.SensitiveColumns)
+	md[MetadataTableSensitiveColumns] = string(asJSON)
 
 	schemaMd := arrow.MetadataFrom(md)
 	for i, c := range t.Columns {

schema/table_test.go

@@ -731,6 +731,7 @@ func TestTablesToAndFromArrow(t *testing.T) {
 				{Name: "multiple_attributes", Type: arrow.BinaryTypes.String, PrimaryKey: true, IncrementalKey: true, NotNull: true, Unique: true},
 			},
 			PermissionsNeeded: []string{"storage.buckets.list", "compute.acceleratorTypes.list", "test,test"},
+			SensitiveColumns:  []string{"string", "json"},
 		},
 	}
 

schema/validators.go

@@ -2,6 +2,8 @@ package schema
 
 import (
 	"encoding/json"
+	"slices"
+	"strings"
 
 	"github.com/apache/arrow-go/v18/arrow"
 	"github.com/cloudquery/plugin-sdk/v4/types"
@@ -40,6 +42,36 @@ func FindEmptyColumns(table *Table, records []arrow.Record) []string {
 	return emptyColumns
 }
 
+func FindNotMatchingSensitiveColumns(table *Table) (nonMatchingColumns []string, nonMatchingJSONColumns []string) {
+	if len(table.SensitiveColumns) == 0 {
+		return []string{}, []string{}
+	}
+
+	nonMatchingColumns = make([]string, 0)
+	nonMatchingJSONColumns = make([]string, 0)
+	tableColumns := table.Columns.Names()
+	for _, c := range table.SensitiveColumns {
+		isJSONPath := false
+		if strings.Contains(c, ".") {
+			c = strings.Split(c, ".")[0]
+			isJSONPath = true
+		}
+		if !slices.Contains(tableColumns, c) {
+			nonMatchingColumns = append(nonMatchingColumns, c)
+			continue
+		}
+		if !isJSONPath {
+			continue
+		}
+		col := table.Columns.Get(c)
+		if !arrow.TypeEqual(col.Type, types.ExtensionTypes.JSON) {
+			nonMatchingJSONColumns = append(nonMatchingJSONColumns, c)
+			continue
+		}
+	}
+	return nonMatchingColumns, nonMatchingJSONColumns
+}
+
 func isEmptyJSON(msg json.RawMessage) bool {
 	if len(msg) == 0 {
 		return true

serve/package.go

@@ -103,6 +103,7 @@ func (s *PluginServe) writeTablesJSON(ctx context.Context, dir string) error {
 			Title:             &table.Title,
 			Columns:           &columns,
 			PermissionsNeeded: &table.PermissionsNeeded,
+			SensitiveColumns:  &table.SensitiveColumns,
 		})
 	}
 	buffer := &bytes.Buffer{}

serve/testdata/memdbtables.json

@@ -17,7 +17,8 @@
     "relations": ["table2"],
     "title": "",
     "is_paid": false,
-    "permissions_needed": ["permission1"]
+    "permissions_needed": ["permission1"],
+    "sensitive_columns": ["col1"]
   },
   {
     "columns": [
@@ -38,7 +39,8 @@
     "name": "table2",
     "title": "",
     "is_paid": false,
-    "permissions_needed": null
+    "permissions_needed": null,
+    "sensitive_columns": null
   },
   {
     "columns": [
@@ -59,6 +61,7 @@
     "name": "table3",
     "title": "",
     "is_paid": true,
-    "permissions_needed": null
+    "permissions_needed": null,
+    "sensitive_columns": null
   }
 ]