package store import ( "context" "log/slog" "os" "path/filepath" "github.com/pkg/errors" "github.com/usememos/memos/internal/base" "github.com/usememos/memos/internal/storage" storepb "github.com/usememos/memos/proto/gen/store" ) type Attachment struct { // ID is the system generated unique identifier for the attachment. ID int32 // UID is the user defined unique identifier for the attachment. UID string // Standard fields CreatorID int32 CreatedTs int64 UpdatedTs int64 // Domain specific fields Filename string Blob []byte Type string Size int64 StorageType storepb.AttachmentStorageType Reference string Payload *storepb.AttachmentPayload // The related memo ID. MemoID *int32 // Policy is present for transport-facing direct-link mutations. Drivers // revalidate it in the same transaction as the attachment insert. Policy *MemoWritePolicy // Composed field MemoUID *string } type FindAttachment struct { GetBlob bool ID *int32 UID *string CreatorID *int32 Filename *string FilenameSearch *string MemoID *int32 MemoIDList []int32 HasRelatedMemo bool Filters []string // Access applies memo-local authorization to linked attachments // before pagination. Unlinked attachments are visible only to the active // matching creator. Nil is reserved for trusted internal callers. Access *MemoAccessScope Limit *int Offset *int SkipDefaultLimit bool } type UpdateAttachment struct { ID int32 UID *string UpdatedTs *int64 Filename *string MemoID *int32 Payload *storepb.AttachmentPayload // Policy is present for transport-facing updates. Drivers discover the // current attachment binding and authorize any linked memo in the same // transaction as the update. Policy *MemoWritePolicy } type DeleteAttachment struct { ID int32 MemoID *int32 } const ( thumbnailCacheFolder = ".thumbnail_cache" motionCacheFolder = ".motion_cache" ) type deleteAttachmentStorageFailpointKey struct{} type createAttachmentPolicyFailpointKey struct{} type createAttachmentPostCommitFailpointKey struct{} // ErrDeleteAttachmentStorageFailpoint is returned by the test-only attachment storage failpoint. var ErrDeleteAttachmentStorageFailpoint = errors.New("delete attachment storage failpoint") // ErrCreateAttachmentPostCommitFailpoint is returned after the test-only attachment create failpoint persists a row. var ErrCreateAttachmentPostCommitFailpoint = errors.New("create attachment post-commit failpoint") // WithDeleteAttachmentStorageFailpoint forces DeleteAttachmentStorage to return a failpoint error. func WithDeleteAttachmentStorageFailpoint(ctx context.Context) context.Context { return context.WithValue(ctx, deleteAttachmentStorageFailpointKey{}, true) } // WithCreateAttachmentPolicyFailpoint forces a policy-bearing attachment create to fail before its database insert. func WithCreateAttachmentPolicyFailpoint(ctx context.Context) context.Context { return context.WithValue(ctx, createAttachmentPolicyFailpointKey{}, true) } // WithCreateAttachmentPostCommitFailpoint forces CreateAttachment to return an error after its database insert succeeds. func WithCreateAttachmentPostCommitFailpoint(ctx context.Context) context.Context { return context.WithValue(ctx, createAttachmentPostCommitFailpointKey{}, true) } func (s *Store) CreateAttachment(ctx context.Context, create *Attachment) (*Attachment, error) { if !base.UIDMatcher.MatchString(create.UID) { return nil, errors.New("invalid uid") } if err := validateMemoWritePolicy(create.Policy); err != nil { return nil, err } var ( attachment *Attachment err error ) if create.Policy != nil { if create.MemoID == nil { return nil, errors.New("attachment write policy requires a memo") } if create.CreatorID != create.Policy.ActorUserID { return nil, ErrMemoPermissionDenied } if shouldFailCreateAttachmentPolicy(ctx) { return nil, ErrMemoSpaceMembershipRequired } attachment, err = s.driver.CreateAttachment(ctx, create) } else { attachment, err = s.driver.CreateAttachment(ctx, create) } if err != nil { return nil, err } if shouldFailCreateAttachmentPostCommit(ctx) { return nil, ErrCreateAttachmentPostCommitFailpoint } return attachment, nil } func (s *Store) ListAttachments(ctx context.Context, find *FindAttachment) ([]*Attachment, error) { // Set default limits to prevent loading too many attachments at once shouldApplyDefaultLimit := find.Limit == nil && find.MemoID == nil && len(find.MemoIDList) == 0 && !find.SkipDefaultLimit if shouldApplyDefaultLimit && find.GetBlob { // When fetching blobs, we should be especially careful with limits defaultLimit := 10 find.Limit = &defaultLimit } else if shouldApplyDefaultLimit { // Even without blobs, let's default to a reasonable limit defaultLimit := 100 find.Limit = &defaultLimit } return s.driver.ListAttachments(ctx, find) } func (s *Store) GetAttachment(ctx context.Context, find *FindAttachment) (*Attachment, error) { attachments, err := s.ListAttachments(ctx, find) if err != nil { return nil, err } if len(attachments) == 0 { return nil, nil } return attachments[0], nil } func (s *Store) UpdateAttachment(ctx context.Context, update *UpdateAttachment) error { if update.UID != nil && !base.UIDMatcher.MatchString(*update.UID) { return errors.New("invalid uid") } if err := validateMemoWritePolicy(update.Policy); err != nil { return err } if update.Policy != nil { if update.MemoID != nil { return errors.New("policy-bearing attachment updates cannot change memo binding") } return s.driver.UpdateAttachment(ctx, update) } return s.driver.UpdateAttachment(ctx, update) } func (s *Store) DeleteAttachment(ctx context.Context, delete *DeleteAttachment) error { attachment, err := s.GetAttachment(ctx, &FindAttachment{ID: &delete.ID}) if err != nil { return errors.Wrap(err, "failed to get attachment") } if attachment == nil { return errors.New("attachment not found") } if err := s.DeleteAttachmentStorage(ctx, attachment); err != nil { if attachment.StorageType == storepb.AttachmentStorageType_LOCAL { return errors.Wrap(err, "failed to delete local file") } slog.Warn("Failed to delete attachment storage", slog.Any("err", err)) } return s.driver.DeleteAttachment(ctx, delete) } func (s *Store) DeleteAttachments(ctx context.Context, attachments []*Attachment) error { if len(attachments) == 0 { return nil } deletes := make([]*DeleteAttachment, 0, len(attachments)) for _, attachment := range attachments { if attachment == nil { continue } deletes = append(deletes, &DeleteAttachment{ID: attachment.ID, MemoID: attachment.MemoID}) } if len(deletes) == 0 { return nil } if err := s.driver.DeleteAttachments(ctx, deletes); err != nil { return err } return s.deleteAttachmentStorageSnapshots(ctx, attachments) } func (s *Store) deleteAttachmentStorageSnapshots(ctx context.Context, attachments []*Attachment) error { instanceStorageSetting, instanceStorageSettingErr := s.getAttachmentStorageCleanupInstanceSetting(ctx, attachments) for _, attachment := range attachments { if attachment == nil { continue } var err error if instanceStorageSettingErr != nil && AttachmentNeedsInstanceStorageSetting(attachment) { err = instanceStorageSettingErr } else { err = s.deleteAttachmentStorageImpl(ctx, attachment, instanceStorageSetting) } if err != nil { if attachment.StorageType == storepb.AttachmentStorageType_LOCAL { return errors.Wrap(err, "failed to delete local file") } slog.Warn("Failed to delete attachment storage", slog.Any("err", err)) } } return nil } func (s *Store) DeleteAttachmentStorage(ctx context.Context, attachment *Attachment) error { return s.deleteAttachmentStorageImpl(ctx, attachment, nil) } // DeleteAttachmentStorageWithInstanceSetting deletes attachment storage using a preloaded instance storage setting. func (s *Store) DeleteAttachmentStorageWithInstanceSetting(ctx context.Context, attachment *Attachment, instanceStorageSetting *storepb.InstanceStorageSetting) error { return s.deleteAttachmentStorageImpl(ctx, attachment, instanceStorageSetting) } func (s *Store) deleteAttachmentStorageImpl(ctx context.Context, attachment *Attachment, instanceStorageSetting *storepb.InstanceStorageSetting) error { if attachment == nil { return nil } if shouldFailDeleteAttachmentStorage(ctx) { return ErrDeleteAttachmentStorageFailpoint } if attachment.StorageType == storepb.AttachmentStorageType_LOCAL { if err := func() error { p := filepath.FromSlash(attachment.Reference) if !filepath.IsAbs(p) { p = filepath.Join(s.profile.Data, p) } err := os.Remove(p) if err != nil && !os.IsNotExist(err) { return errors.Wrap(err, "failed to delete local file") } return nil }(); err != nil { return err } } else if attachment.StorageType == storepb.AttachmentStorageType_S3 { if err := func() error { s3ObjectPayload := attachment.Payload.GetS3Object() if s3ObjectPayload == nil { return errors.Errorf("No s3 object found") } // Deletes resolve with the registry like reads do, so a legacy // attachment picks up rotated credentials for its namespace instead // of deleting with the stale embedded config. if instanceStorageSetting == nil { var err error instanceStorageSetting, err = s.GetInstanceStorageSetting(ctx) if err != nil { if AttachmentNeedsInstanceStorageSetting(attachment) { return errors.Wrap(err, "failed to get instance storage setting") } // The embedded config is sufficient on its own; keep the // delete best-effort when the settings read fails. instanceStorageSetting = nil } } resolvedStorage, err := ResolveStorage(instanceStorageSetting, s3ObjectPayload.StorageId, s3ObjectPayload.S3Config) if err != nil { return errors.Wrap(err, "failed to resolve storage") } driver, err := s.StorageDriver(ctx, resolvedStorage) if err != nil { return errors.Wrap(err, "failed to create storage driver") } if err := driver.DeleteObject(ctx, s3ObjectPayload.Key); err != nil { return errors.Wrap(err, "Failed to delete s3 object") } return nil }(); err != nil { return err } } s.deleteAttachmentDerivedCaches(attachment) return nil } func (s *Store) getAttachmentStorageCleanupInstanceSetting(ctx context.Context, attachments []*Attachment) (*storepb.InstanceStorageSetting, error) { for _, attachment := range attachments { if AttachmentNeedsInstanceStorageSetting(attachment) { instanceStorageSetting, err := s.GetInstanceStorageSetting(ctx) if err != nil { return nil, errors.Wrap(err, "failed to get instance storage setting") } return instanceStorageSetting, nil } } return nil, nil } // ResolveAttachmentS3Driver resolves the storage driver referenced by an S3 // attachment payload, validating the payload and supplying the instance setting. func (s *Store) ResolveAttachmentS3Driver(ctx context.Context, attachment *Attachment) (storage.Driver, *storepb.AttachmentPayload_S3Object, error) { if attachment == nil { return nil, nil, errors.New("attachment is missing") } if attachment.Payload == nil { return nil, nil, errors.New("attachment payload is missing") } s3Object := attachment.Payload.GetS3Object() if s3Object == nil { return nil, nil, errors.New("S3 object payload is missing") } if s3Object.Key == "" { return nil, nil, errors.New("S3 object key is missing") } instanceStorageSetting, err := s.GetInstanceStorageSetting(ctx) if err != nil { return nil, nil, errors.Wrap(err, "failed to get instance storage setting") } resolvedStorage, err := ResolveStorage(instanceStorageSetting, s3Object.StorageId, s3Object.S3Config) if err != nil { return nil, nil, errors.Wrap(err, "failed to resolve storage") } driver, err := s.StorageDriver(ctx, resolvedStorage) if err != nil { return nil, nil, errors.Wrap(err, "failed to create storage driver") } return driver, s3Object, nil } // AttachmentNeedsInstanceStorageSetting reports whether cleanup should load // the configured storage registry for an S3 attachment. func AttachmentNeedsInstanceStorageSetting(attachment *Attachment) bool { if attachment == nil || attachment.StorageType != storepb.AttachmentStorageType_S3 { return false } return attachment.Payload.GetS3Object() != nil } func (s *Store) deleteAttachmentDerivedCaches(attachment *Attachment) { for _, cachePath := range []string{ filepath.Join(s.profile.Data, thumbnailCacheFolder, attachment.UID+".jpeg"), filepath.Join(s.profile.Data, motionCacheFolder, attachment.UID+".mp4"), } { if err := os.Remove(cachePath); err != nil && !os.IsNotExist(err) { slog.Warn("Failed to delete derived attachment cache", slog.String("path", cachePath), slog.Any("err", err)) } } } func shouldFailDeleteAttachmentStorage(ctx context.Context) bool { failpoint, ok := ctx.Value(deleteAttachmentStorageFailpointKey{}).(bool) return ok && failpoint } func shouldFailCreateAttachmentPolicy(ctx context.Context) bool { failpoint, ok := ctx.Value(createAttachmentPolicyFailpointKey{}).(bool) return ok && failpoint } func shouldFailCreateAttachmentPostCommit(ctx context.Context) bool { failpoint, ok := ctx.Value(createAttachmentPostCommitFailpointKey{}).(bool) return ok && failpoint }