feat: implement attachment filtering functionality
This commit is contained in:
parent
955ff0cad6
commit
78aa41336a
12 changed files with 210 additions and 15 deletions
|
|
@ -99,6 +99,9 @@ var (
|
||||||
defaultOnce sync.Once
|
defaultOnce sync.Once
|
||||||
defaultInst *Engine
|
defaultInst *Engine
|
||||||
defaultErr error
|
defaultErr error
|
||||||
|
defaultAttachmentOnce sync.Once
|
||||||
|
defaultAttachmentInst *Engine
|
||||||
|
defaultAttachmentErr error
|
||||||
)
|
)
|
||||||
|
|
||||||
// DefaultEngine returns the process-wide memo filter engine.
|
// DefaultEngine returns the process-wide memo filter engine.
|
||||||
|
|
@ -109,6 +112,14 @@ func DefaultEngine() (*Engine, error) {
|
||||||
return defaultInst, defaultErr
|
return defaultInst, defaultErr
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DefaultAttachmentEngine returns the process-wide attachment filter engine.
|
||||||
|
func DefaultAttachmentEngine() (*Engine, error) {
|
||||||
|
defaultAttachmentOnce.Do(func() {
|
||||||
|
defaultAttachmentInst, defaultAttachmentErr = NewEngine(NewAttachmentSchema())
|
||||||
|
})
|
||||||
|
return defaultAttachmentInst, defaultAttachmentErr
|
||||||
|
}
|
||||||
|
|
||||||
func normalizeLegacyFilter(expr string) string {
|
func normalizeLegacyFilter(expr string) string {
|
||||||
expr = rewriteNumericLogicalOperand(expr, "&&")
|
expr = rewriteNumericLogicalOperand(expr, "&&")
|
||||||
expr = rewriteNumericLogicalOperand(expr, "||")
|
expr = rewriteNumericLogicalOperand(expr, "||")
|
||||||
|
|
|
||||||
|
|
@ -243,6 +243,62 @@ func NewSchema() Schema {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NewAttachmentSchema constructs the attachment filter schema and CEL environment.
|
||||||
|
func NewAttachmentSchema() Schema {
|
||||||
|
fields := map[string]Field{
|
||||||
|
"filename": {
|
||||||
|
Name: "filename",
|
||||||
|
Kind: FieldKindScalar,
|
||||||
|
Type: FieldTypeString,
|
||||||
|
Column: Column{Table: "resource", Name: "filename"},
|
||||||
|
SupportsContains: true,
|
||||||
|
Expressions: map[DialectName]string{},
|
||||||
|
},
|
||||||
|
"mime_type": {
|
||||||
|
Name: "mime_type",
|
||||||
|
Kind: FieldKindScalar,
|
||||||
|
Type: FieldTypeString,
|
||||||
|
Column: Column{Table: "resource", Name: "type"},
|
||||||
|
Expressions: map[DialectName]string{},
|
||||||
|
},
|
||||||
|
"create_time": {
|
||||||
|
Name: "create_time",
|
||||||
|
Kind: FieldKindScalar,
|
||||||
|
Type: FieldTypeTimestamp,
|
||||||
|
Column: Column{Table: "resource", Name: "created_ts"},
|
||||||
|
Expressions: map[DialectName]string{
|
||||||
|
DialectMySQL: "UNIX_TIMESTAMP(%s)",
|
||||||
|
DialectPostgres: "EXTRACT(EPOCH FROM TO_TIMESTAMP(%s))",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"memo": {
|
||||||
|
Name: "memo",
|
||||||
|
Kind: FieldKindScalar,
|
||||||
|
Type: FieldTypeString,
|
||||||
|
Column: Column{Table: "resource", Name: "memo_uid"},
|
||||||
|
Expressions: map[DialectName]string{},
|
||||||
|
AllowedComparisonOps: map[ComparisonOperator]bool{
|
||||||
|
CompareEq: true,
|
||||||
|
CompareNeq: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
envOptions := []cel.EnvOption{
|
||||||
|
cel.Variable("filename", cel.StringType),
|
||||||
|
cel.Variable("mime_type", cel.StringType),
|
||||||
|
cel.Variable("create_time", cel.IntType),
|
||||||
|
cel.Variable("memo", cel.StringType),
|
||||||
|
nowFunction,
|
||||||
|
}
|
||||||
|
|
||||||
|
return Schema{
|
||||||
|
Name: "attachment",
|
||||||
|
Fields: fields,
|
||||||
|
EnvOptions: envOptions,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// columnExpr returns the field expression for the given dialect, applying
|
// columnExpr returns the field expression for the given dialect, applying
|
||||||
// any schema-specific overrides (e.g. UNIX timestamp conversions).
|
// any schema-specific overrides (e.g. UNIX timestamp conversions).
|
||||||
func (f Field) columnExpr(d DialectName) string {
|
func (f Field) columnExpr(d DialectName) string {
|
||||||
|
|
|
||||||
|
|
@ -101,9 +101,9 @@ message ListAttachmentsRequest {
|
||||||
string page_token = 2 [(google.api.field_behavior) = OPTIONAL];
|
string page_token = 2 [(google.api.field_behavior) = OPTIONAL];
|
||||||
|
|
||||||
// Optional. Filter to apply to the list results.
|
// Optional. Filter to apply to the list results.
|
||||||
// Example: "type=image/png" or "filename:*.jpg"
|
// Example: "mime_type==\"image/png\"" or "filename.contains(\"test\")"
|
||||||
// Supported operators: =, !=, <, <=, >, >=, :
|
// Supported operators: =, !=, <, <=, >, >=, : (contains), in
|
||||||
// Supported fields: filename, type, size, create_time, memo
|
// Supported fields: filename, mime_type, create_time, memo
|
||||||
string filter = 3 [(google.api.field_behavior) = OPTIONAL];
|
string filter = 3 [(google.api.field_behavior) = OPTIONAL];
|
||||||
|
|
||||||
// Optional. The order to sort results by.
|
// Optional. The order to sort results by.
|
||||||
|
|
|
||||||
|
|
@ -201,9 +201,9 @@ type ListAttachmentsRequest struct {
|
||||||
// Provide this to retrieve the subsequent page.
|
// Provide this to retrieve the subsequent page.
|
||||||
PageToken string `protobuf:"bytes,2,opt,name=page_token,json=pageToken,proto3" json:"page_token,omitempty"`
|
PageToken string `protobuf:"bytes,2,opt,name=page_token,json=pageToken,proto3" json:"page_token,omitempty"`
|
||||||
// Optional. Filter to apply to the list results.
|
// Optional. Filter to apply to the list results.
|
||||||
// Example: "type=image/png" or "filename:*.jpg"
|
// Example: "mime_type==\"image/png\"" or "filename.contains(\"test\")"
|
||||||
// Supported operators: =, !=, <, <=, >, >=, :
|
// Supported operators: =, !=, <, <=, >, >=, : (contains), in
|
||||||
// Supported fields: filename, type, size, create_time, memo
|
// Supported fields: filename, mime_type, create_time, memo
|
||||||
Filter string `protobuf:"bytes,3,opt,name=filter,proto3" json:"filter,omitempty"`
|
Filter string `protobuf:"bytes,3,opt,name=filter,proto3" json:"filter,omitempty"`
|
||||||
// Optional. The order to sort results by.
|
// Optional. The order to sort results by.
|
||||||
// Example: "create_time desc" or "filename asc"
|
// Example: "create_time desc" or "filename asc"
|
||||||
|
|
|
||||||
|
|
@ -97,9 +97,9 @@ paths:
|
||||||
in: query
|
in: query
|
||||||
description: |-
|
description: |-
|
||||||
Optional. Filter to apply to the list results.
|
Optional. Filter to apply to the list results.
|
||||||
Example: "type=image/png" or "filename:*.jpg"
|
Example: "mime_type==\"image/png\"" or "filename.contains(\"test\")"
|
||||||
Supported operators: =, !=, <, <=, >, >=, :
|
Supported operators: =, !=, <, <=, >, >=, : (contains), in
|
||||||
Supported fields: filename, type, size, create_time, memo
|
Supported fields: filename, mime_type, create_time, memo
|
||||||
schema:
|
schema:
|
||||||
type: string
|
type: string
|
||||||
- name: orderBy
|
- name: orderBy
|
||||||
|
|
|
||||||
|
|
@ -21,6 +21,7 @@ import (
|
||||||
|
|
||||||
"github.com/usememos/memos/internal/profile"
|
"github.com/usememos/memos/internal/profile"
|
||||||
"github.com/usememos/memos/internal/util"
|
"github.com/usememos/memos/internal/util"
|
||||||
|
"github.com/usememos/memos/plugin/filter"
|
||||||
"github.com/usememos/memos/plugin/storage/s3"
|
"github.com/usememos/memos/plugin/storage/s3"
|
||||||
v1pb "github.com/usememos/memos/proto/gen/api/v1"
|
v1pb "github.com/usememos/memos/proto/gen/api/v1"
|
||||||
storepb "github.com/usememos/memos/proto/gen/store"
|
storepb "github.com/usememos/memos/proto/gen/store"
|
||||||
|
|
@ -156,6 +157,14 @@ func (s *APIV1Service) ListAttachments(ctx context.Context, request *v1pb.ListAt
|
||||||
Offset: &offset,
|
Offset: &offset,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Parse filter if provided
|
||||||
|
if request.Filter != "" {
|
||||||
|
if err := s.validateAttachmentFilter(ctx, request.Filter); err != nil {
|
||||||
|
return nil, status.Errorf(codes.InvalidArgument, "invalid filter: %v", err)
|
||||||
|
}
|
||||||
|
findAttachment.Filters = append(findAttachment.Filters, request.Filter)
|
||||||
|
}
|
||||||
|
|
||||||
attachments, err := s.Store.ListAttachments(ctx, findAttachment)
|
attachments, err := s.Store.ListAttachments(ctx, findAttachment)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, status.Errorf(codes.Internal, "failed to list attachments: %v", err)
|
return nil, status.Errorf(codes.Internal, "failed to list attachments: %v", err)
|
||||||
|
|
@ -472,3 +481,29 @@ func isValidMimeType(mimeType string) bool {
|
||||||
matched, _ := regexp.MatchString(`^[a-zA-Z0-9][a-zA-Z0-9!#$&^_.+-]{0,126}/[a-zA-Z0-9][a-zA-Z0-9!#$&^_.+-]{0,126}$`, mimeType)
|
matched, _ := regexp.MatchString(`^[a-zA-Z0-9][a-zA-Z0-9!#$&^_.+-]{0,126}/[a-zA-Z0-9][a-zA-Z0-9!#$&^_.+-]{0,126}$`, mimeType)
|
||||||
return matched
|
return matched
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *APIV1Service) validateAttachmentFilter(ctx context.Context, filterStr string) error {
|
||||||
|
if filterStr == "" {
|
||||||
|
return errors.New("filter cannot be empty")
|
||||||
|
}
|
||||||
|
|
||||||
|
engine, err := filter.DefaultAttachmentEngine()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
var dialect filter.DialectName
|
||||||
|
switch s.Profile.Driver {
|
||||||
|
case "mysql":
|
||||||
|
dialect = filter.DialectMySQL
|
||||||
|
case "postgres":
|
||||||
|
dialect = filter.DialectPostgres
|
||||||
|
default:
|
||||||
|
dialect = filter.DialectSQLite
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := engine.CompileToStatement(ctx, filterStr, filter.RenderOptions{Dialect: dialect}); err != nil {
|
||||||
|
return errors.Wrap(err, "failed to compile filter")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -51,6 +51,7 @@ type FindAttachment struct {
|
||||||
MemoIDList []int32
|
MemoIDList []int32
|
||||||
HasRelatedMemo bool
|
HasRelatedMemo bool
|
||||||
StorageType *storepb.AttachmentStorageType
|
StorageType *storepb.AttachmentStorageType
|
||||||
|
Filters []string
|
||||||
Limit *int
|
Limit *int
|
||||||
Offset *int
|
Offset *int
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -9,6 +9,7 @@ import (
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
"google.golang.org/protobuf/encoding/protojson"
|
"google.golang.org/protobuf/encoding/protojson"
|
||||||
|
|
||||||
|
"github.com/usememos/memos/plugin/filter"
|
||||||
storepb "github.com/usememos/memos/proto/gen/store"
|
storepb "github.com/usememos/memos/proto/gen/store"
|
||||||
"github.com/usememos/memos/store"
|
"github.com/usememos/memos/store"
|
||||||
)
|
)
|
||||||
|
|
@ -83,6 +84,16 @@ func (d *DB) ListAttachments(ctx context.Context, find *store.FindAttachment) ([
|
||||||
where, args = append(where, "`resource`.`storage_type` = ?"), append(args, find.StorageType.String())
|
where, args = append(where, "`resource`.`storage_type` = ?"), append(args, find.StorageType.String())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if len(find.Filters) > 0 {
|
||||||
|
engine, err := filter.DefaultAttachmentEngine()
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Wrap(err, "failed to get filter engine")
|
||||||
|
}
|
||||||
|
if err := filter.AppendConditions(ctx, engine, find.Filters, filter.DialectMySQL, &where, &args); err != nil {
|
||||||
|
return nil, errors.Wrap(err, "failed to append filter conditions")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fields := []string{
|
fields := []string{
|
||||||
"`resource`.`id` AS `id`",
|
"`resource`.`id` AS `id`",
|
||||||
"`resource`.`uid` AS `uid`",
|
"`resource`.`uid` AS `uid`",
|
||||||
|
|
|
||||||
|
|
@ -9,6 +9,7 @@ import (
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
"google.golang.org/protobuf/encoding/protojson"
|
"google.golang.org/protobuf/encoding/protojson"
|
||||||
|
|
||||||
|
"github.com/usememos/memos/plugin/filter"
|
||||||
storepb "github.com/usememos/memos/proto/gen/store"
|
storepb "github.com/usememos/memos/proto/gen/store"
|
||||||
"github.com/usememos/memos/store"
|
"github.com/usememos/memos/store"
|
||||||
)
|
)
|
||||||
|
|
@ -72,6 +73,16 @@ func (d *DB) ListAttachments(ctx context.Context, find *store.FindAttachment) ([
|
||||||
where, args = append(where, "resource.storage_type = "+placeholder(len(args)+1)), append(args, v.String())
|
where, args = append(where, "resource.storage_type = "+placeholder(len(args)+1)), append(args, v.String())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if len(find.Filters) > 0 {
|
||||||
|
engine, err := filter.DefaultAttachmentEngine()
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Wrap(err, "failed to get filter engine")
|
||||||
|
}
|
||||||
|
if err := filter.AppendConditions(ctx, engine, find.Filters, filter.DialectPostgres, &where, &args); err != nil {
|
||||||
|
return nil, errors.Wrap(err, "failed to append filter conditions")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fields := []string{
|
fields := []string{
|
||||||
"resource.id AS id",
|
"resource.id AS id",
|
||||||
"resource.uid AS uid",
|
"resource.uid AS uid",
|
||||||
|
|
|
||||||
|
|
@ -9,6 +9,7 @@ import (
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
"google.golang.org/protobuf/encoding/protojson"
|
"google.golang.org/protobuf/encoding/protojson"
|
||||||
|
|
||||||
|
"github.com/usememos/memos/plugin/filter"
|
||||||
storepb "github.com/usememos/memos/proto/gen/store"
|
storepb "github.com/usememos/memos/proto/gen/store"
|
||||||
"github.com/usememos/memos/store"
|
"github.com/usememos/memos/store"
|
||||||
)
|
)
|
||||||
|
|
@ -76,6 +77,16 @@ func (d *DB) ListAttachments(ctx context.Context, find *store.FindAttachment) ([
|
||||||
where, args = append(where, "`resource`.`storage_type` = ?"), append(args, find.StorageType.String())
|
where, args = append(where, "`resource`.`storage_type` = ?"), append(args, find.StorageType.String())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if len(find.Filters) > 0 {
|
||||||
|
engine, err := filter.DefaultAttachmentEngine()
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Wrap(err, "failed to get filter engine")
|
||||||
|
}
|
||||||
|
if err := filter.AppendConditions(ctx, engine, find.Filters, filter.DialectSQLite, &where, &args); err != nil {
|
||||||
|
return nil, errors.Wrap(err, "failed to append filter conditions")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fields := []string{
|
fields := []string{
|
||||||
"`resource`.`id` AS `id`",
|
"`resource`.`id` AS `id`",
|
||||||
"`resource`.`uid` AS `uid`",
|
"`resource`.`uid` AS `uid`",
|
||||||
|
|
|
||||||
|
|
@ -61,3 +61,62 @@ func TestAttachmentStore(t *testing.T) {
|
||||||
require.ErrorContains(t, err, "attachment not found")
|
require.ErrorContains(t, err, "attachment not found")
|
||||||
ts.Close()
|
ts.Close()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestAttachmentStoreWithFilter(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
ts := NewTestingStore(ctx, t)
|
||||||
|
|
||||||
|
_, err := ts.CreateAttachment(ctx, &store.Attachment{
|
||||||
|
UID: shortuuid.New(),
|
||||||
|
CreatorID: 101,
|
||||||
|
Filename: "test.png",
|
||||||
|
Blob: []byte("test"),
|
||||||
|
Type: "image/png",
|
||||||
|
Size: 1000,
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
_, err = ts.CreateAttachment(ctx, &store.Attachment{
|
||||||
|
UID: shortuuid.New(),
|
||||||
|
CreatorID: 101,
|
||||||
|
Filename: "test.jpg",
|
||||||
|
Blob: []byte("test"),
|
||||||
|
Type: "image/jpeg",
|
||||||
|
Size: 2000,
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
_, err = ts.CreateAttachment(ctx, &store.Attachment{
|
||||||
|
UID: shortuuid.New(),
|
||||||
|
CreatorID: 101,
|
||||||
|
Filename: "test.pdf",
|
||||||
|
Blob: []byte("test"),
|
||||||
|
Type: "application/pdf",
|
||||||
|
Size: 3000,
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
attachments, err := ts.ListAttachments(ctx, &store.FindAttachment{
|
||||||
|
CreatorID: &[]int32{101}[0],
|
||||||
|
Filters: []string{`mime_type == "image/png"`},
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Len(t, attachments, 1)
|
||||||
|
require.Equal(t, "image/png", attachments[0].Type)
|
||||||
|
|
||||||
|
attachments, err = ts.ListAttachments(ctx, &store.FindAttachment{
|
||||||
|
CreatorID: &[]int32{101}[0],
|
||||||
|
Filters: []string{`mime_type in ["image/png", "image/jpeg"]`},
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Len(t, attachments, 2)
|
||||||
|
|
||||||
|
attachments, err = ts.ListAttachments(ctx, &store.FindAttachment{
|
||||||
|
CreatorID: &[]int32{101}[0],
|
||||||
|
Filters: []string{`filename.contains("test")`},
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Len(t, attachments, 3)
|
||||||
|
|
||||||
|
ts.Close()
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -139,9 +139,9 @@ export type ListAttachmentsRequest = Message<"memos.api.v1.ListAttachmentsReques
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Optional. Filter to apply to the list results.
|
* Optional. Filter to apply to the list results.
|
||||||
* Example: "type=image/png" or "filename:*.jpg"
|
* Example: "mime_type==\"image/png\"" or "filename.contains(\"test\")"
|
||||||
* Supported operators: =, !=, <, <=, >, >=, :
|
* Supported operators: =, !=, <, <=, >, >=, : (contains), in
|
||||||
* Supported fields: filename, type, size, create_time, memo
|
* Supported fields: filename, mime_type, create_time, memo
|
||||||
*
|
*
|
||||||
* @generated from field: string filter = 3;
|
* @generated from field: string filter = 3;
|
||||||
*/
|
*/
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue